forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
29.3 KiB
1//! End-to-end: CI against the real `forge` binary, driven by the real `forge-runner` library.
2//!
3//! The runner side uses `HostExecutor` (behind the `test-host-executor` feature), so jobs run as
4//! plain `sh` on the test host. Everything else is the production path: runner registration over
5//! the HTTP API, claims, the job's git fetch over smart HTTP with its job token, log shipping with
6//! masking, artifact upload, and finish.
7//!
8//! Scenarios:
9//! - a push run: secrets are masked in the log, artifacts download, a failure skips dependants;
10//! - a fork pull request: it waits for approval, then runs without secrets;
11//! - a live log: `log.txt` streams while a job runs and ends when the job finishes.
12
13mod common;
14use common::{BIN, Server, launch, pick_ports, write_config};
15use forge_ci::proto::{JobAssignment, Outcome, RegisterRequest};
16use forge_runner::client::ForgeClient;
17use forge_runner::config::{Limits, RunnerConfig};
18use forge_runner::executor::HostExecutor;
19use forge_runner::runner::{RunOptions, Runner};
20use std::collections::BTreeMap;
21use std::fs;
22use std::path::{Path, PathBuf};
23use std::process::{Command, Output};
24use std::sync::Arc;
25use std::time::{Duration, Instant};
26use tokio_util::sync::CancellationToken;
27
28use reqwest::header::{LOCATION, SET_COOKIE};
29
30const ALICE_PW: &str = "alice-password-123";
31const BOB_PW: &str = "bob-password-4567";
32const BOB_EMAIL: &str = "bob@users.noreply.127.0.0.1";
33const REPO_PATH: &str = "/alice/r";
34const SECRET_VALUE: &str = "supersecretvalue";
35const LABELS: &[&str] = &["linux"];
36
37/// The push-triggered workflow. Image strings are ignored by `HostExecutor`.
38const CI_TOML: &str = r#"[on]
39push = ["main"]
40pull_request = true
41
42[jobs.build]
43image = "docker.io/library/alpine:3"
44run = "echo hello; echo $MY_SECRET; mkdir -p out; echo data > out/a.txt"
45secrets = ["MY_SECRET"]
46artifacts = ["out/*.txt"]
47
48[jobs.after]
49image = "docker.io/library/alpine:3"
50run = "echo after"
51needs = ["build"]
52
53[jobs.broken]
54image = "docker.io/library/alpine:3"
55run = "exit 3"
56
57[jobs.skipme]
58image = "docker.io/library/alpine:3"
59run = "echo never"
60needs = ["broken"]
61"#;
62
63/// The same workflow with the build line changed to print the secret in a form that stays
64/// readable when the variable is unset. Bob's pull request uses it, so `set -u` does not abort.
65const CI_TOML_FORK: &str = r#"[on]
66push = ["main"]
67pull_request = true
68
69[jobs.build]
70image = "docker.io/library/alpine:3"
71run = "echo hello; echo \"value=[${MY_SECRET-}]\"; mkdir -p out; echo data > out/a.txt"
72secrets = ["MY_SECRET"]
73artifacts = ["out/*.txt"]
74
75[jobs.after]
76image = "docker.io/library/alpine:3"
77run = "echo after"
78needs = ["build"]
79
80[jobs.broken]
81image = "docker.io/library/alpine:3"
82run = "exit 3"
83
84[jobs.skipme]
85image = "docker.io/library/alpine:3"
86run = "echo never"
87needs = ["broken"]
88"#;
89
90/// A parsed HTTP response. Only the parts the test needs are kept.
91struct Resp {
92 status: u16,
93 location: Option<String>,
94 set_cookies: Vec<String>,
95 body: String,
96}
97
98struct Http {
99 client: reqwest::Client,
100 base: String,
101}
102
103impl Http {
104 fn new(port: u16) -> Self {
105 let client = reqwest::Client::builder()
106 .redirect(reqwest::redirect::Policy::none())
107 .timeout(Duration::from_secs(60))
108 .build()
109 .expect("http client");
110 Self {
111 client,
112 base: format!("http://127.0.0.1:{port}"),
113 }
114 }
115
116 async fn get(&self, path: &str, cookie: Option<&str>) -> Resp {
117 self.send(self.client.get(format!("{}{path}", self.base)), cookie)
118 .await
119 }
120
121 async fn get_bearer(&self, path: &str, token: &str) -> Resp {
122 let req = self
123 .client
124 .get(format!("{}{path}", self.base))
125 .bearer_auth(token);
126 self.send(req, None).await
127 }
128
129 async fn post_form(&self, path: &str, form: &[(&str, &str)], cookie: Option<&str>) -> Resp {
130 self.send(
131 self.client.post(format!("{}{path}", self.base)).form(form),
132 cookie,
133 )
134 .await
135 }
136
137 async fn send(&self, req: reqwest::RequestBuilder, cookie: Option<&str>) -> Resp {
138 let req = match cookie {
139 Some(cookie) => req.header(reqwest::header::COOKIE, cookie),
140 None => req,
141 };
142 let res = req.send().await.expect("http request");
143 let status = res.status().as_u16();
144 let location = res
145 .headers()
146 .get(LOCATION)
147 .and_then(|v| v.to_str().ok())
148 .map(str::to_owned);
149 let set_cookies = res
150 .headers()
151 .get_all(SET_COOKIE)
152 .iter()
153 .filter_map(|v| v.to_str().ok().map(str::to_owned))
154 .collect();
155 let body = res.text().await.unwrap_or_default();
156 Resp {
157 status,
158 location,
159 set_cookies,
160 body,
161 }
162 }
163}
164
165/// Returns `name=value` for the cookie whose name ends with `suffix`. Cleared cookies are ignored.
166fn cookie_named(resp: &Resp, suffix: &str) -> Option<String> {
167 resp.set_cookies.iter().find_map(|set| {
168 let pair = set.split(';').next()?;
169 let (name, value) = pair.split_once('=')?;
170 (name.ends_with(suffix) && !value.is_empty()).then(|| format!("{name}={value}"))
171 })
172}
173
174/// The value of the first `<input name="...">` on a page.
175fn input_value(body: &str, name: &str) -> String {
176 let doc = scraper::Html::parse_document(body);
177 let selector = scraper::Selector::parse(&format!("input[name=\"{name}\"]")).expect("selector");
178 doc.select(&selector)
179 .next()
180 .and_then(|el| el.value().attr("value"))
181 .unwrap_or_else(|| panic!("no input named {name} on page"))
182 .to_owned()
183}
184
185/// The text of the first element matching `selector`.
186fn first_text(body: &str, selector: &str) -> String {
187 let doc = scraper::Html::parse_document(body);
188 let selector = scraper::Selector::parse(selector).expect("selector");
189 doc.select(&selector)
190 .next()
191 .unwrap_or_else(|| panic!("no element matches {selector:?}"))
192 .text()
193 .collect::<String>()
194 .trim()
195 .to_owned()
196}
197
198fn run_admin(config: &Path, username: &str, extra: &[&str], password: &str) {
199 let out = Command::new(BIN)
200 .arg("--config")
201 .arg(config)
202 .args(["admin", "create-user", username])
203 .args(extra)
204 .env("FORGE_PASSWORD", password)
205 .output()
206 .expect("run admin create-user");
207 ok(&out, &format!("create-user {username}"));
208}
209
210fn ok(out: &Output, what: &str) {
211 assert!(
212 out.status.success(),
213 "{what} failed:\n{}",
214 String::from_utf8_lossy(&out.stderr)
215 );
216}
217
218/// Runs git in `cwd` with `env` applied on top of the inherited environment. Returns stdout.
219fn git_env(cwd: &Path, args: &[&str], env: &[(&str, &str)]) -> String {
220 let mut cmd = Command::new("git");
221 cmd.current_dir(cwd)
222 .args(["-c", "credential.helper="])
223 .args(args)
224 .env("GIT_TERMINAL_PROMPT", "0");
225 for (key, value) in env {
226 cmd.env(key, value);
227 }
228 let out = cmd.output().expect("run git");
229 assert!(
230 out.status.success(),
231 "git {args:?} failed:\n{}",
232 String::from_utf8_lossy(&out.stderr)
233 );
234 String::from_utf8_lossy(&out.stdout).trim().to_owned()
235}
236
237fn git(cwd: &Path, args: &[&str]) -> String {
238 git_env(cwd, args, &[])
239}
240
241/// Writes `file` (creating its directory), commits it with `who` as author and committer, and
242/// returns the new HEAD.
243fn commit(
244 repo: &Path,
245 file: &str,
246 contents: &str,
247 message: &str,
248 who: &str,
249 email: &str,
250) -> String {
251 let path = repo.join(file);
252 if let Some(parent) = path.parent() {
253 fs::create_dir_all(parent).expect("create dir");
254 }
255 fs::write(&path, contents).expect("write file");
256 git(repo, &["add", file]);
257 git_env(
258 repo,
259 &["-c", "commit.gpgsign=false", "commit", "-q", "-m", message],
260 &[
261 ("GIT_AUTHOR_NAME", who),
262 ("GIT_AUTHOR_EMAIL", email),
263 ("GIT_COMMITTER_NAME", who),
264 ("GIT_COMMITTER_EMAIL", email),
265 ],
266 );
267 git(repo, &["rev-parse", "HEAD"])
268}
269
270/// Clones `url` into `into` (a path that does not exist yet) and returns it.
271fn clone(url: &str, into: &Path) -> PathBuf {
272 let parent = into.parent().expect("clone target has a parent");
273 git(
274 parent,
275 &["clone", "-q", url, into.to_str().expect("utf-8 path")],
276 );
277 into.to_path_buf()
278}
279
280/// The HTTPS remote for `owner/repo`, authenticated with a token.
281fn remote(port: u16, user: &str, token: &str, owner: &str, repo: &str) -> String {
282 format!("http://{user}:{token}@127.0.0.1:{port}/{owner}/{repo}.git")
283}
284
285/// The CSRF token shown on any settings page of the signed-in user.
286async fn csrf_from(http: &Http, path: &str, session: &str) -> String {
287 let page = http.get(path, Some(session)).await;
288 assert_eq!(page.status, 200, "GET {path}");
289 input_value(&page.body, "csrf")
290}
291
292/// Logs in through the real `/login` form and returns the `__Host-forge_session` cookie header.
293async fn login(http: &Http, username: &str, password: &str) -> String {
294 let page = http.get("/login", None).await;
295 assert_eq!(page.status, 200, "GET /login");
296 let precsrf_cookie = cookie_named(&page, "forge_precsrf").expect("precsrf cookie");
297 let precsrf = input_value(&page.body, "precsrf");
298 let res = http
299 .post_form(
300 "/login",
301 &[
302 ("username", username),
303 ("password", password),
304 ("next", "/"),
305 ("precsrf", &precsrf),
306 ],
307 Some(&precsrf_cookie),
308 )
309 .await;
310 assert_eq!(res.status, 303, "login as {username}: {}", res.body);
311 cookie_named(&res, "forge_session").expect("session cookie after login")
312}
313
314/// Creates an access token through `/settings/tokens` and returns the token shown once.
315async fn create_token(http: &Http, session: &str, name: &str) -> String {
316 let csrf = csrf_from(http, "/settings/tokens", session).await;
317 let res = http
318 .post_form(
319 "/settings/tokens",
320 &[("csrf", &csrf), ("name", name), ("expires", "30")],
321 Some(session),
322 )
323 .await;
324 assert_eq!(res.status, 200, "create token: {}", res.body);
325 let token = first_text(&res.body, "pre.code code");
326 assert!(token.starts_with("forge_"), "unexpected token shape");
327 token
328}
329
330/// Creates a public repo `name` owned by alice through `/new`.
331async fn create_repo(http: &Http, session: &str, name: &str) {
332 let page = http.get("/new", Some(session)).await;
333 assert_eq!(page.status, 200, "GET /new");
334 let csrf = input_value(&page.body, "csrf");
335 let res = http
336 .post_form(
337 "/new",
338 &[
339 ("csrf", &csrf),
340 ("name", name),
341 ("description", ""),
342 ("visibility", "public"),
343 ("default_branch", "main"),
344 ],
345 Some(session),
346 )
347 .await;
348 assert_eq!(res.status, 303, "create repo {name}: {}", res.body);
349 assert_eq!(
350 res.location.as_deref(),
351 Some(format!("/alice/{name}").as_str())
352 );
353}
354
355/// Adds or replaces a repo secret through the settings form.
356async fn set_secret(http: &Http, session: &str, name: &str, value: &str) {
357 let csrf = csrf_from(http, &format!("{REPO_PATH}/settings/secrets"), session).await;
358 let res = http
359 .post_form(
360 &format!("{REPO_PATH}/settings/secrets"),
361 &[("csrf", &csrf), ("name", name), ("value", value)],
362 Some(session),
363 )
364 .await;
365 assert_eq!(res.status, 303, "set secret {name}: {}", res.body);
366}
367
368/// A running server with its own temp data dir. Field order matters: the server is dropped (and
369/// killed) before the temp dir is removed.
370struct Booted {
371 _server: Server,
372 _root: tempfile::TempDir,
373 dir: PathBuf,
374 http: Http,
375 base_url: String,
376 port: u16,
377}
378
379/// The server's log, for assertion messages.
380fn server_log(b: &Booted) -> String {
381 fs::read_to_string(b.dir.join("serve.log")).unwrap_or_default()
382}
383
384/// Creates alice (admin) and bob, then boots the server on fresh ports.
385async fn boot() -> Booted {
386 // Keep the path short: the internal unix socket path is limited to 108 bytes.
387 let root = tempfile::Builder::new()
388 .prefix("ci2e")
389 .tempdir_in("/tmp")
390 .expect("temp dir");
391 let dir = root.path().to_path_buf();
392 let ports = pick_ports();
393 let config = write_config(&dir, ports);
394
395 run_admin(&config, "alice", &["--admin"], ALICE_PW);
396 run_admin(&config, "bob", &[], BOB_PW);
397 let (server, ports) = launch(&dir, ports).await;
398 Booted {
399 _server: server,
400 _root: root,
401 dir,
402 http: Http::new(ports.http),
403 base_url: format!("http://127.0.0.1:{}", ports.http),
404 port: ports.http,
405 }
406}
407
408/// A registered runner and the secret it claims with.
409struct Worker {
410 runner: Runner,
411 secret: String,
412 labels: Vec<String>,
413}
414
415impl Worker {
416 /// One claim. `None` means the forge had nothing for this runner (after its long poll).
417 async fn claim(&self) -> Option<JobAssignment> {
418 self.runner
419 .client()
420 .claim(&self.secret, &self.labels)
421 .await
422 .expect("claim")
423 }
424
425 /// Claims and runs jobs until the forge has nothing left. Returns how many ran.
426 async fn drain(&self) -> usize {
427 let mut ran = 0;
428 while let Some(job) = self.claim().await {
429 self.runner.process(job, CancellationToken::new()).await;
430 ran += 1;
431 }
432 ran
433 }
434}
435
436/// Registers a runner through the admin page and the runner API. The admin is `admin_session`.
437async fn register_runner(b: &Booted, admin_session: &str, name: &str) -> Worker {
438 let page = b.http.get("/admin/runners", Some(admin_session)).await;
439 assert_eq!(page.status, 200, "GET /admin/runners");
440 let csrf = input_value(&page.body, "csrf");
441 let res = b
442 .http
443 .post_form(
444 "/admin/runners/tokens",
445 &[("csrf", &csrf)],
446 Some(admin_session),
447 )
448 .await;
449 assert_eq!(res.status, 200, "create registration token: {}", res.body);
450 let token = first_text(&res.body, "pre code");
451
452 let client = ForgeClient::new(&b.base_url).expect("runner client");
453 let registered = client
454 .register(&RegisterRequest {
455 token,
456 name: name.to_owned(),
457 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
458 })
459 .await
460 .expect("register runner");
461 let config = RunnerConfig {
462 url: b.base_url.clone(),
463 runner_id: registered.runner_id,
464 runner_secret: registered.runner_secret.clone(),
465 name: name.to_owned(),
466 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
467 work_dir: b.dir.join("runner-work"),
468 concurrency: 1,
469 limits: Limits::default(),
470 executor: forge_runner::config::ExecutorKind::default(),
471 images: forge_runner::config::ImagesConfig::default(),
472 sandbox: forge_runner::config::SandboxConfig::default(),
473 registries: Vec::new(),
474 };
475 let runner =
476 Runner::new(config, Arc::new(HostExecutor), RunOptions::default()).expect("runner");
477 Worker {
478 runner,
479 secret: registered.runner_secret,
480 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
481 }
482}
483
484/// One commit status as the status API reports it.
485#[derive(Debug, Clone)]
486struct Status {
487 state: String,
488 description: String,
489 target_url: String,
490}
491
492/// The statuses on `sha` of alice/r, keyed by context.
493async fn statuses(http: &Http, token: &str, sha: &str) -> BTreeMap<String, Status> {
494 let res = http
495 .get_bearer(
496 &format!("/api/v1/repos/alice/r/commits/{sha}/statuses"),
497 token,
498 )
499 .await;
500 assert_eq!(res.status, 200, "statuses for {sha}: {}", res.body);
501 let json: serde_json::Value = serde_json::from_str(&res.body).expect("statuses json");
502 json["statuses"]
503 .as_array()
504 .expect("statuses array")
505 .iter()
506 .map(|s| {
507 let field = |k: &str| s[k].as_str().unwrap_or_default().to_owned();
508 (
509 field("context"),
510 Status {
511 state: field("state"),
512 description: field("description"),
513 target_url: field("target_url"),
514 },
515 )
516 })
517 .collect()
518}
519
520/// Polls the statuses until every context in `contexts` is present, then returns them.
521async fn wait_statuses(
522 http: &Http,
523 token: &str,
524 sha: &str,
525 contexts: &[&str],
526) -> BTreeMap<String, Status> {
527 let deadline = Instant::now() + Duration::from_secs(20);
528 loop {
529 let found = statuses(http, token, sha).await;
530 if contexts.iter().all(|c| found.contains_key(*c)) {
531 return found;
532 }
533 assert!(
534 Instant::now() < deadline,
535 "statuses never showed {contexts:?}; got {found:?}"
536 );
537 tokio::time::sleep(Duration::from_millis(100)).await;
538 }
539}
540
541/// The job id at the end of a status `target_url` (`.../ci/jobs/<id>`).
542fn job_id_of(status: &Status) -> String {
543 status
544 .target_url
545 .rsplit('/')
546 .next()
547 .filter(|id| !id.is_empty())
548 .unwrap_or_else(|| panic!("no job id in {:?}", status.target_url))
549 .to_owned()
550}
551
552/// Pushes `.forge/ci.toml` (with `contents`) to `main` as alice. Returns the pushed commit.
553fn push_ci_config(work: &Path, contents: &str) -> String {
554 let sha = commit(
555 work,
556 ".forge/ci.toml",
557 contents,
558 "add ci",
559 "alice",
560 "alice@example.test",
561 );
562 git(work, &["push", "-q", "origin", "HEAD:refs/heads/main"]);
563 sha
564}
565
566#[tokio::test]
567async fn push_run_masks_secrets_keeps_artifacts_and_skips_dependants() {
568 let b = boot().await;
569 let alice = login(&b.http, "alice", ALICE_PW).await;
570 let token = create_token(&b.http, &alice, "ci-token").await;
571 create_repo(&b.http, &alice, "r").await;
572 set_secret(&b.http, &alice, "MY_SECRET", SECRET_VALUE).await;
573 let worker = register_runner(&b, &alice, "runner-a").await;
574
575 let work = clone(
576 &remote(b.port, "alice", &token, "alice", "r"),
577 &b.dir.join("alice-work"),
578 );
579 let sha = push_ci_config(&work, CI_TOML);
580
581 let found = wait_statuses(
582 &b.http,
583 &token,
584 &sha,
585 &["ci/build", "ci/after", "ci/broken", "ci/skipme"],
586 )
587 .await;
588 assert_eq!(found["ci/build"].state, "pending", "build is queued");
589
590 let ran = worker.drain().await;
591 assert_eq!(ran, 3, "build, after and broken run; skipme never does");
592
593 let found = statuses(&b.http, &token, &sha).await;
594 assert_eq!(found["ci/build"].state, "success");
595 assert_eq!(found["ci/after"].state, "success");
596 assert_eq!(found["ci/broken"].state, "failure");
597 assert_eq!(found["ci/skipme"].state, "error");
598 assert_eq!(found["ci/skipme"].description, "skipped");
599
600 // The build log: the secret is masked, never printed.
601 let build_id = job_id_of(&found["ci/build"]);
602 let log = b
603 .http
604 .get(&format!("{REPO_PATH}/ci/jobs/{build_id}/log.txt"), None)
605 .await;
606 assert_eq!(log.status, 200, "GET log.txt");
607 assert!(log.body.contains("hello"), "log has the echo: {}", log.body);
608 assert!(log.body.contains("***"), "secret is masked: {}", log.body);
609 assert!(
610 !log.body.contains(SECRET_VALUE),
611 "secret leaked into the log: {}",
612 log.body
613 );
614
615 // The artifact comes back with the right bytes. `out/a.txt` is stored as `out_a.txt`.
616 let artifact = b
617 .http
618 .get(
619 &format!("{REPO_PATH}/ci/jobs/{build_id}/artifacts/out_a.txt"),
620 Some(&alice),
621 )
622 .await;
623 assert_eq!(artifact.status, 200, "GET artifact: {}", server_log(&b));
624 assert_eq!(artifact.body, "data\n", "artifact bytes");
625
626 // The run page lists every job.
627 let run = b
628 .http
629 .get(&format!("{REPO_PATH}/ci/runs/1"), Some(&alice))
630 .await;
631 assert_eq!(run.status, 200, "GET run page");
632 for job in ["build", "after", "broken", "skipme"] {
633 assert!(run.body.contains(job), "run page lists {job}");
634 }
635}
636
637#[tokio::test]
638async fn fork_pull_waits_for_approval_and_runs_without_secrets() {
639 let b = boot().await;
640 let alice = login(&b.http, "alice", ALICE_PW).await;
641 let token = create_token(&b.http, &alice, "ci-token").await;
642 create_repo(&b.http, &alice, "r").await;
643 set_secret(&b.http, &alice, "MY_SECRET", SECRET_VALUE).await;
644 let worker = register_runner(&b, &alice, "runner-a").await;
645
646 // Base branch with the workflow, run to completion (run 1).
647 let work = clone(
648 &remote(b.port, "alice", &token, "alice", "r"),
649 &b.dir.join("alice-work"),
650 );
651 let base_sha = push_ci_config(&work, CI_TOML);
652 wait_statuses(&b.http, &token, &base_sha, &["ci/build"]).await;
653 worker.drain().await;
654
655 // Bob forks and changes the workflow on a branch.
656 let bob = login(&b.http, "bob", BOB_PW).await;
657 let bob_token = create_token(&b.http, &bob, "bob-token").await;
658 let csrf = csrf_from(&b.http, "/settings/tokens", &bob).await;
659 let fork = b
660 .http
661 .post_form(&format!("{REPO_PATH}/fork"), &[("csrf", &csrf)], Some(&bob))
662 .await;
663 assert_eq!(fork.status, 303, "fork: {}\n{}", fork.body, server_log(&b));
664
665 let bob_work = clone(
666 &remote(b.port, "bob", &bob_token, "bob", "r"),
667 &b.dir.join("bob-work"),
668 );
669 git(&bob_work, &["checkout", "-q", "-b", "bob-feature"]);
670 let head = commit(
671 &bob_work,
672 ".forge/ci.toml",
673 CI_TOML_FORK,
674 "print the secret",
675 "bob",
676 BOB_EMAIL,
677 );
678 git(
679 &bob_work,
680 &["push", "-q", "origin", "HEAD:refs/heads/bob-feature"],
681 );
682
683 // Bob opens the pull request. The run waits for the owner.
684 let compare = b
685 .http
686 .get(
687 &format!("{REPO_PATH}/compare/main...bob:bob-feature"),
688 Some(&bob),
689 )
690 .await;
691 assert_eq!(compare.status, 200, "GET compare");
692 let csrf = csrf_from(&b.http, "/settings/tokens", &bob).await;
693 let open = b
694 .http
695 .post_form(
696 &format!("{REPO_PATH}/pulls"),
697 &[
698 ("csrf", &csrf),
699 ("title", "Bob change"),
700 ("body", ""),
701 ("base", "main"),
702 ("head", "bob:bob-feature"),
703 ],
704 Some(&bob),
705 )
706 .await;
707 assert_eq!(open.status, 303, "open pull: {}", open.body);
708
709 let found = wait_statuses(&b.http, &token, &head, &["ci/build"]).await;
710 assert_eq!(found["ci/build"].state, "pending");
711 assert_eq!(found["ci/build"].description, "Waiting for approval");
712
713 let run_page = b
714 .http
715 .get(&format!("{REPO_PATH}/ci/runs/2"), Some(&alice))
716 .await;
717 assert_eq!(run_page.status, 200, "GET run 2");
718 assert!(
719 run_page.body.contains("Approve and run"),
720 "run 2 waits for approval: {}",
721 run_page.body
722 );
723
724 // The owner approves, and the job runs with no secrets.
725 let approve = b
726 .http
727 .post_form(
728 &format!("{REPO_PATH}/ci/runs/2/approve"),
729 &[("csrf", &input_value(&run_page.body, "csrf"))],
730 Some(&alice),
731 )
732 .await;
733 assert_eq!(approve.status, 303, "approve run 2: {}", approve.body);
734 worker.drain().await;
735
736 let found = statuses(&b.http, &token, &head).await;
737 assert_eq!(found["ci/build"].state, "success", "fork build runs");
738 assert_eq!(found["ci/skipme"].state, "error");
739
740 let build_id = job_id_of(&found["ci/build"]);
741 let log = b
742 .http
743 .get(&format!("{REPO_PATH}/ci/jobs/{build_id}/log.txt"), None)
744 .await;
745 assert_eq!(log.status, 200, "GET log.txt");
746 assert!(
747 log.body.contains("value=[]"),
748 "the secret is empty for a fork run: {}",
749 log.body
750 );
751 assert!(!log.body.contains("***"), "no value to mask: {}", log.body);
752 assert!(!log.body.contains(SECRET_VALUE), "secret leaked");
753}
754
755#[tokio::test]
756async fn live_log_streams_until_the_job_finishes() {
757 let b = boot().await;
758 let alice = login(&b.http, "alice", ALICE_PW).await;
759 let token = create_token(&b.http, &alice, "ci-token").await;
760 create_repo(&b.http, &alice, "r").await;
761 let worker = register_runner(&b, &alice, "runner-a").await;
762
763 let slow = r#"[on]
764push = ["main"]
765
766[jobs.slow]
767image = "docker.io/library/alpine:3"
768run = "echo started; sleep 5; echo done"
769"#;
770 let work = clone(
771 &remote(b.port, "alice", &token, "alice", "r"),
772 &b.dir.join("alice-work"),
773 );
774 let sha = push_ci_config(&work, slow);
775 let found = wait_statuses(&b.http, &token, &sha, &["ci/slow"]).await;
776 let job_id = job_id_of(&found["ci/slow"]);
777
778 let job = worker.claim().await.expect("the slow job is claimed");
779 assert_eq!(job.job_id, job_id, "claimed the job the status points at");
780 let runner = worker.runner.clone();
781 let handle = tokio::spawn(async move { runner.process(job, CancellationToken::new()).await });
782
783 // Read the stream chunk by chunk. The first line appears while the job is still sleeping.
784 let url = format!("{}{REPO_PATH}/ci/jobs/{job_id}/log.txt", b.http.base);
785 let mut res = b.http.client.get(url).send().await.expect("GET log.txt");
786 assert_eq!(res.status().as_u16(), 200, "GET log.txt");
787 let mut body = String::new();
788 let mut seen_started = false;
789 let read = async {
790 while let Some(chunk) = res.chunk().await.expect("log chunk") {
791 body.push_str(&String::from_utf8_lossy(&chunk));
792 if !seen_started && body.contains("started") {
793 seen_started = true;
794 assert!(
795 !handle.is_finished(),
796 "the first line arrived only after the job had finished"
797 );
798 }
799 }
800 };
801 tokio::time::timeout(Duration::from_secs(90), read)
802 .await
803 .expect("the log stream ends after the job finishes");
804
805 assert!(seen_started, "the live log showed the first line: {body}");
806 assert!(
807 body.contains("done"),
808 "the stream ends with the last line: {body}"
809 );
810 let outcome = handle.await.expect("job task");
811 assert_eq!(outcome, Outcome::Success);
812 let found = statuses(&b.http, &token, &sha).await;
813 assert_eq!(found["ci/slow"].state, "success");
814}
815
816#[tokio::test]
817async fn ci_job_token_clones_the_repos_wiki_read_only() {
818 let b = boot().await;
819 let alice = login(&b.http, "alice", ALICE_PW).await;
820 let token = create_token(&b.http, &alice, "ci-token").await;
821 create_repo(&b.http, &alice, "r").await;
822 let worker = register_runner(&b, &alice, "runner-a").await;
823
824 // The wiki gets its first page through the web form.
825 let csrf = input_value(
826 &b.http
827 .get(&format!("{REPO_PATH}/wiki/-/new"), Some(&alice))
828 .await
829 .body,
830 "csrf",
831 );
832 let saved = b
833 .http
834 .post_form(
835 &format!("{REPO_PATH}/wiki/-/new"),
836 &[
837 ("csrf", &csrf),
838 ("name", "Home"),
839 ("base", ""),
840 ("content", "# Welcome from the wiki\n"),
841 ("message", "first page"),
842 ],
843 Some(&alice),
844 )
845 .await;
846 assert_eq!(saved.status, 303, "create Home: {}", saved.body);
847
848 // A slow job keeps running, so its token is valid while the test clones.
849 let slow = r#"[on]
850push = ["main"]
851
852[jobs.slow]
853image = "docker.io/library/alpine:3"
854run = "echo started; sleep 5; echo done"
855"#;
856 let work = clone(
857 &remote(b.port, "alice", &token, "alice", "r"),
858 &b.dir.join("alice-work"),
859 );
860 let sha = push_ci_config(&work, slow);
861 let found = wait_statuses(&b.http, &token, &sha, &["ci/slow"]).await;
862 let job = worker.claim().await.expect("the slow job is claimed");
863 assert_eq!(job.job_id, job_id_of(&found["ci/slow"]));
864
865 // The job reads the wiki with its own credential: `ci-job` and `<job id>:<job token>`.
866 let wiki_url = format!(
867 "http://ci-job:{}:{}@127.0.0.1:{}/alice/r.wiki.git",
868 job.job_id, job.job_token, b.port
869 );
870 let job_wiki = clone(&wiki_url, &b.dir.join("job-wiki"));
871 let home = fs::read_to_string(job_wiki.join("Home.md")).expect("Home.md in the job's clone");
872 assert!(home.contains("Welcome from the wiki"), "{home}");
873
874 // Pushing to the wiki is refused for a job token.
875 fs::write(job_wiki.join("Home.md"), "# changed by a job\n").expect("write");
876 git(&job_wiki, &["add", "Home.md"]);
877 git_env(
878 &job_wiki,
879 &[
880 "-c",
881 "commit.gpgsign=false",
882 "commit",
883 "-q",
884 "-m",
885 "job edit",
886 ],
887 &[
888 ("GIT_AUTHOR_NAME", "job"),
889 ("GIT_AUTHOR_EMAIL", "job@example.invalid"),
890 ("GIT_COMMITTER_NAME", "job"),
891 ("GIT_COMMITTER_EMAIL", "job@example.invalid"),
892 ],
893 );
894 let push = Command::new("git")
895 .current_dir(&job_wiki)
896 .args([
897 "-c",
898 "credential.helper=",
899 "push",
900 "-q",
901 "origin",
902 "HEAD:main",
903 ])
904 .env("GIT_TERMINAL_PROMPT", "0")
905 .output()
906 .expect("run git");
907 assert!(
908 !push.status.success(),
909 "a CI job token must not push to the wiki"
910 );
911}