forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
29.4 KiB
1//! End-to-end: CI against the real `forge` binary, driven by the real `forge-runner` library.
2//!
3//! The runner side uses `HostExecutor` (behind the `test-host-executor` feature), so jobs run as
4//! plain `sh` on the test host. Everything else is the production path: runner registration over
5//! the HTTP API, claims, the job's git fetch over smart HTTP with its job token, log shipping with
6//! masking, artifact upload, and finish.
7//!
8//! Scenarios:
9//! - a push run: secrets are masked in the log, artifacts download, a failure skips dependants;
10//! - a fork pull request: it waits for approval, then runs without secrets;
11//! - a live log: `log.txt` streams while a job runs and ends when the job finishes.
12
13mod common;
14use common::{BIN, Server, launch, pick_ports, write_config};
15use forge_ci::proto::{JobAssignment, Outcome, RegisterRequest};
16use forge_runner::client::ForgeClient;
17use forge_runner::config::{Limits, RunnerConfig};
18use forge_runner::executor::HostExecutor;
19use forge_runner::runner::{RunOptions, Runner};
20use std::collections::BTreeMap;
21use std::fs;
22use std::path::{Path, PathBuf};
23use std::process::{Command, Output};
24use std::sync::Arc;
25use std::time::{Duration, Instant};
26use tokio_util::sync::CancellationToken;
27
28use reqwest::header::{LOCATION, SET_COOKIE};
29
30const ALICE_PW: &str = "alice-password-123";
31const BOB_PW: &str = "bob-password-4567";
32const BOB_EMAIL: &str = "bob@users.noreply.127.0.0.1";
33const REPO_PATH: &str = "/alice/r";
34const SECRET_VALUE: &str = "supersecretvalue";
35const LABELS: &[&str] = &["linux"];
36
37/// The push-triggered workflow. Image strings are ignored by `HostExecutor`.
38const CI_TOML: &str = r#"[on]
39push = ["main"]
40pull_request = true
41
42[jobs.build]
43image = "docker.io/library/alpine:3"
44run = "echo hello; echo $MY_SECRET; mkdir -p out; echo data > out/a.txt"
45secrets = ["MY_SECRET"]
46artifacts = ["out/*.txt"]
47
48[jobs.after]
49image = "docker.io/library/alpine:3"
50run = "echo after"
51needs = ["build"]
52
53[jobs.broken]
54image = "docker.io/library/alpine:3"
55run = "exit 3"
56
57[jobs.skipme]
58image = "docker.io/library/alpine:3"
59run = "echo never"
60needs = ["broken"]
61"#;
62
63/// The same workflow with the build line changed to print the secret in a form that stays
64/// readable when the variable is unset. Bob's pull request uses it, so `set -u` does not abort.
65const CI_TOML_FORK: &str = r#"[on]
66push = ["main"]
67pull_request = true
68
69[jobs.build]
70image = "docker.io/library/alpine:3"
71run = "echo hello; echo \"value=[${MY_SECRET-}]\"; mkdir -p out; echo data > out/a.txt"
72secrets = ["MY_SECRET"]
73artifacts = ["out/*.txt"]
74
75[jobs.after]
76image = "docker.io/library/alpine:3"
77run = "echo after"
78needs = ["build"]
79
80[jobs.broken]
81image = "docker.io/library/alpine:3"
82run = "exit 3"
83
84[jobs.skipme]
85image = "docker.io/library/alpine:3"
86run = "echo never"
87needs = ["broken"]
88"#;
89
90/// A parsed HTTP response. Only the parts the test needs are kept.
91struct Resp {
92 status: u16,
93 location: Option<String>,
94 set_cookies: Vec<String>,
95 body: String,
96}
97
98struct Http {
99 client: reqwest::Client,
100 base: String,
101}
102
103impl Http {
104 fn new(port: u16) -> Self {
105 let client = reqwest::Client::builder()
106 .redirect(reqwest::redirect::Policy::none())
107 .timeout(Duration::from_secs(60))
108 .build()
109 .expect("http client");
110 Self {
111 client,
112 base: format!("http://127.0.0.1:{port}"),
113 }
114 }
115
116 async fn get(&self, path: &str, cookie: Option<&str>) -> Resp {
117 self.send(self.client.get(format!("{}{path}", self.base)), cookie)
118 .await
119 }
120
121 async fn get_bearer(&self, path: &str, token: &str) -> Resp {
122 let req = self
123 .client
124 .get(format!("{}{path}", self.base))
125 .bearer_auth(token);
126 self.send(req, None).await
127 }
128
129 async fn post_form(&self, path: &str, form: &[(&str, &str)], cookie: Option<&str>) -> Resp {
130 self.send(
131 self.client.post(format!("{}{path}", self.base)).form(form),
132 cookie,
133 )
134 .await
135 }
136
137 async fn send(&self, req: reqwest::RequestBuilder, cookie: Option<&str>) -> Resp {
138 let req = match cookie {
139 Some(cookie) => req.header(reqwest::header::COOKIE, cookie),
140 None => req,
141 };
142 let res = req.send().await.expect("http request");
143 let status = res.status().as_u16();
144 let location = res
145 .headers()
146 .get(LOCATION)
147 .and_then(|v| v.to_str().ok())
148 .map(str::to_owned);
149 let set_cookies = res
150 .headers()
151 .get_all(SET_COOKIE)
152 .iter()
153 .filter_map(|v| v.to_str().ok().map(str::to_owned))
154 .collect();
155 let body = res.text().await.unwrap_or_default();
156 Resp {
157 status,
158 location,
159 set_cookies,
160 body,
161 }
162 }
163}
164
165/// Returns `name=value` for the cookie whose name ends with `suffix`. Cleared cookies are ignored.
166fn cookie_named(resp: &Resp, suffix: &str) -> Option<String> {
167 resp.set_cookies.iter().find_map(|set| {
168 let pair = set.split(';').next()?;
169 let (name, value) = pair.split_once('=')?;
170 (name.ends_with(suffix) && !value.is_empty()).then(|| format!("{name}={value}"))
171 })
172}
173
174/// The value of the first `<input name="...">` on a page.
175fn input_value(body: &str, name: &str) -> String {
176 let doc = scraper::Html::parse_document(body);
177 let selector = scraper::Selector::parse(&format!("input[name=\"{name}\"]")).expect("selector");
178 doc.select(&selector)
179 .next()
180 .and_then(|el| el.value().attr("value"))
181 .unwrap_or_else(|| panic!("no input named {name} on page"))
182 .to_owned()
183}
184
185/// The text of the first element matching `selector`.
186fn first_text(body: &str, selector: &str) -> String {
187 let doc = scraper::Html::parse_document(body);
188 let selector = scraper::Selector::parse(selector).expect("selector");
189 doc.select(&selector)
190 .next()
191 .unwrap_or_else(|| panic!("no element matches {selector:?}"))
192 .text()
193 .collect::<String>()
194 .trim()
195 .to_owned()
196}
197
198fn run_admin(config: &Path, username: &str, extra: &[&str], password: &str) {
199 let out = Command::new(BIN)
200 .arg("--config")
201 .arg(config)
202 .args(["admin", "create-user", username])
203 .args(extra)
204 .env("FORGE_PASSWORD", password)
205 .output()
206 .expect("run admin create-user");
207 ok(&out, &format!("create-user {username}"));
208}
209
210fn ok(out: &Output, what: &str) {
211 assert!(
212 out.status.success(),
213 "{what} failed:\n{}",
214 String::from_utf8_lossy(&out.stderr)
215 );
216}
217
218/// Runs git in `cwd` with `env` applied on top of the inherited environment. Returns stdout.
219fn git_env(cwd: &Path, args: &[&str], env: &[(&str, &str)]) -> String {
220 let mut cmd = Command::new("git");
221 cmd.current_dir(cwd)
222 .args(["-c", "credential.helper="])
223 .args(args)
224 .env("GIT_TERMINAL_PROMPT", "0");
225 for (key, value) in env {
226 cmd.env(key, value);
227 }
228 let out = cmd.output().expect("run git");
229 assert!(
230 out.status.success(),
231 "git {args:?} failed:\n{}",
232 String::from_utf8_lossy(&out.stderr)
233 );
234 String::from_utf8_lossy(&out.stdout).trim().to_owned()
235}
236
237fn git(cwd: &Path, args: &[&str]) -> String {
238 git_env(cwd, args, &[])
239}
240
241/// Writes `file` (creating its directory), commits it with `who` as author and committer, and
242/// returns the new HEAD.
243fn commit(
244 repo: &Path,
245 file: &str,
246 contents: &str,
247 message: &str,
248 who: &str,
249 email: &str,
250) -> String {
251 let path = repo.join(file);
252 if let Some(parent) = path.parent() {
253 fs::create_dir_all(parent).expect("create dir");
254 }
255 fs::write(&path, contents).expect("write file");
256 git(repo, &["add", file]);
257 git_env(
258 repo,
259 &["-c", "commit.gpgsign=false", "commit", "-q", "-m", message],
260 &[
261 ("GIT_AUTHOR_NAME", who),
262 ("GIT_AUTHOR_EMAIL", email),
263 ("GIT_COMMITTER_NAME", who),
264 ("GIT_COMMITTER_EMAIL", email),
265 ],
266 );
267 git(repo, &["rev-parse", "HEAD"])
268}
269
270/// Clones `url` into `into` (a path that does not exist yet) and returns it.
271fn clone(url: &str, into: &Path) -> PathBuf {
272 let parent = into.parent().expect("clone target has a parent");
273 git(
274 parent,
275 &["clone", "-q", url, into.to_str().expect("utf-8 path")],
276 );
277 into.to_path_buf()
278}
279
280/// The HTTPS remote for `owner/repo`, authenticated with a token.
281fn remote(port: u16, user: &str, token: &str, owner: &str, repo: &str) -> String {
282 format!("http://{user}:{token}@127.0.0.1:{port}/{owner}/{repo}.git")
283}
284
285/// The CSRF token shown on any settings page of the signed-in user.
286async fn csrf_from(http: &Http, path: &str, session: &str) -> String {
287 let page = http.get(path, Some(session)).await;
288 assert_eq!(page.status, 200, "GET {path}");
289 input_value(&page.body, "csrf")
290}
291
292/// Logs in through the real `/login` form and returns the `__Host-forge_session` cookie header.
293async fn login(http: &Http, username: &str, password: &str) -> String {
294 let page = http.get("/login", None).await;
295 assert_eq!(page.status, 200, "GET /login");
296 let precsrf_cookie = cookie_named(&page, "forge_precsrf").expect("precsrf cookie");
297 let precsrf = input_value(&page.body, "precsrf");
298 let res = http
299 .post_form(
300 "/login",
301 &[
302 ("username", username),
303 ("password", password),
304 ("next", "/"),
305 ("precsrf", &precsrf),
306 ],
307 Some(&precsrf_cookie),
308 )
309 .await;
310 assert_eq!(res.status, 303, "login as {username}: {}", res.body);
311 cookie_named(&res, "forge_session").expect("session cookie after login")
312}
313
314/// Creates an access token through `/settings/tokens` and returns the token shown once.
315async fn create_token(http: &Http, session: &str, name: &str) -> String {
316 let csrf = csrf_from(http, "/settings/tokens", session).await;
317 let res = http
318 .post_form(
319 "/settings/tokens",
320 &[("csrf", &csrf), ("name", name), ("expires", "30")],
321 Some(session),
322 )
323 .await;
324 assert_eq!(res.status, 200, "create token: {}", res.body);
325 let token = first_text(&res.body, "pre.code code");
326 assert!(token.starts_with("forge_"), "unexpected token shape");
327 token
328}
329
330/// Creates a public repo `name` owned by alice through `/new`.
331async fn create_repo(http: &Http, session: &str, name: &str) {
332 let page = http.get("/new", Some(session)).await;
333 assert_eq!(page.status, 200, "GET /new");
334 let csrf = input_value(&page.body, "csrf");
335 let res = http
336 .post_form(
337 "/new",
338 &[
339 ("csrf", &csrf),
340 ("name", name),
341 ("description", ""),
342 ("visibility", "public"),
343 ("default_branch", "main"),
344 ],
345 Some(session),
346 )
347 .await;
348 assert_eq!(res.status, 303, "create repo {name}: {}", res.body);
349 assert_eq!(
350 res.location.as_deref(),
351 Some(format!("/alice/{name}").as_str())
352 );
353}
354
355/// Adds or replaces a repo secret through the settings form.
356async fn set_secret(http: &Http, session: &str, name: &str, value: &str) {
357 let csrf = csrf_from(http, &format!("{REPO_PATH}/settings/secrets"), session).await;
358 let res = http
359 .post_form(
360 &format!("{REPO_PATH}/settings/secrets"),
361 &[("csrf", &csrf), ("name", name), ("value", value)],
362 Some(session),
363 )
364 .await;
365 assert_eq!(res.status, 303, "set secret {name}: {}", res.body);
366}
367
368/// A running server with its own temp data dir. Field order matters: the server is dropped (and
369/// killed) before the temp dir is removed.
370struct Booted {
371 _server: Server,
372 _root: tempfile::TempDir,
373 dir: PathBuf,
374 http: Http,
375 base_url: String,
376 port: u16,
377}
378
379/// The server's log, for assertion messages.
380fn server_log(b: &Booted) -> String {
381 fs::read_to_string(b.dir.join("serve.log")).unwrap_or_default()
382}
383
384/// Creates alice (admin) and bob, then boots the server on fresh ports.
385async fn boot() -> Booted {
386 // Keep the path short: the internal unix socket path is limited to 108 bytes.
387 let root = tempfile::Builder::new()
388 .prefix("ci2e")
389 .tempdir_in("/tmp")
390 .expect("temp dir");
391 let dir = root.path().to_path_buf();
392 let ports = pick_ports();
393 let config = write_config(&dir, ports);
394
395 run_admin(&config, "alice", &["--admin"], ALICE_PW);
396 run_admin(&config, "bob", &[], BOB_PW);
397 let (server, ports) = launch(&dir, ports).await;
398 Booted {
399 _server: server,
400 _root: root,
401 dir,
402 http: Http::new(ports.http),
403 base_url: format!("http://127.0.0.1:{}", ports.http),
404 port: ports.http,
405 }
406}
407
408/// A registered runner and the secret it claims with.
409struct Worker {
410 runner: Runner,
411 secret: String,
412 labels: Vec<String>,
413}
414
415impl Worker {
416 /// One claim. `None` means the forge had nothing for this runner (after its long poll).
417 async fn claim(&self) -> Option<JobAssignment> {
418 self.runner
419 .client()
420 .claim(&self.secret, &self.labels)
421 .await
422 .expect("claim")
423 .job
424 }
425
426 /// Claims and runs jobs until the forge has nothing left. Returns how many ran.
427 async fn drain(&self) -> usize {
428 let mut ran = 0;
429 while let Some(job) = self.claim().await {
430 self.runner.process(job, CancellationToken::new()).await;
431 ran += 1;
432 }
433 ran
434 }
435}
436
437/// Registers a runner through the admin page and the runner API. The admin is `admin_session`.
438async fn register_runner(b: &Booted, admin_session: &str, name: &str) -> Worker {
439 let page = b.http.get("/admin/runners", Some(admin_session)).await;
440 assert_eq!(page.status, 200, "GET /admin/runners");
441 let csrf = input_value(&page.body, "csrf");
442 let res = b
443 .http
444 .post_form(
445 "/admin/runners/tokens",
446 &[("csrf", &csrf)],
447 Some(admin_session),
448 )
449 .await;
450 assert_eq!(res.status, 200, "create registration token: {}", res.body);
451 let token = first_text(&res.body, "pre code");
452
453 let client = ForgeClient::new(&b.base_url).expect("runner client");
454 let registered = client
455 .register(&RegisterRequest {
456 token,
457 name: name.to_owned(),
458 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
459 })
460 .await
461 .expect("register runner");
462 let config = RunnerConfig {
463 url: b.base_url.clone(),
464 runner_id: registered.runner_id,
465 runner_secret: registered.runner_secret.clone(),
466 name: name.to_owned(),
467 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
468 work_dir: b.dir.join("runner-work"),
469 concurrency: 1,
470 limits: Limits::default(),
471 executor: forge_runner::config::ExecutorKind::default(),
472 images: forge_runner::config::ImagesConfig::default(),
473 sandbox: forge_runner::config::SandboxConfig::default(),
474 registries: Vec::new(),
475 // The e2e runner must not replace the test binary under itself.
476 auto_update: false,
477 };
478 let runner =
479 Runner::new(config, Arc::new(HostExecutor), RunOptions::default()).expect("runner");
480 Worker {
481 runner,
482 secret: registered.runner_secret,
483 labels: LABELS.iter().map(|l| (*l).to_owned()).collect(),
484 }
485}
486
487/// One commit status as the status API reports it.
488#[derive(Debug, Clone)]
489struct Status {
490 state: String,
491 description: String,
492 target_url: String,
493}
494
495/// The statuses on `sha` of alice/r, keyed by context.
496async fn statuses(http: &Http, token: &str, sha: &str) -> BTreeMap<String, Status> {
497 let res = http
498 .get_bearer(
499 &format!("/api/v1/repos/alice/r/commits/{sha}/statuses"),
500 token,
501 )
502 .await;
503 assert_eq!(res.status, 200, "statuses for {sha}: {}", res.body);
504 let json: serde_json::Value = serde_json::from_str(&res.body).expect("statuses json");
505 json["statuses"]
506 .as_array()
507 .expect("statuses array")
508 .iter()
509 .map(|s| {
510 let field = |k: &str| s[k].as_str().unwrap_or_default().to_owned();
511 (
512 field("context"),
513 Status {
514 state: field("state"),
515 description: field("description"),
516 target_url: field("target_url"),
517 },
518 )
519 })
520 .collect()
521}
522
523/// Polls the statuses until every context in `contexts` is present, then returns them.
524async fn wait_statuses(
525 http: &Http,
526 token: &str,
527 sha: &str,
528 contexts: &[&str],
529) -> BTreeMap<String, Status> {
530 let deadline = Instant::now() + Duration::from_secs(20);
531 loop {
532 let found = statuses(http, token, sha).await;
533 if contexts.iter().all(|c| found.contains_key(*c)) {
534 return found;
535 }
536 assert!(
537 Instant::now() < deadline,
538 "statuses never showed {contexts:?}; got {found:?}"
539 );
540 tokio::time::sleep(Duration::from_millis(100)).await;
541 }
542}
543
544/// The job id at the end of a status `target_url` (`.../ci/jobs/<id>`).
545fn job_id_of(status: &Status) -> String {
546 status
547 .target_url
548 .rsplit('/')
549 .next()
550 .filter(|id| !id.is_empty())
551 .unwrap_or_else(|| panic!("no job id in {:?}", status.target_url))
552 .to_owned()
553}
554
555/// Pushes `.forge/ci.toml` (with `contents`) to `main` as alice. Returns the pushed commit.
556fn push_ci_config(work: &Path, contents: &str) -> String {
557 let sha = commit(
558 work,
559 ".forge/ci.toml",
560 contents,
561 "add ci",
562 "alice",
563 "alice@example.test",
564 );
565 git(work, &["push", "-q", "origin", "HEAD:refs/heads/main"]);
566 sha
567}
568
569#[tokio::test]
570async fn push_run_masks_secrets_keeps_artifacts_and_skips_dependants() {
571 let b = boot().await;
572 let alice = login(&b.http, "alice", ALICE_PW).await;
573 let token = create_token(&b.http, &alice, "ci-token").await;
574 create_repo(&b.http, &alice, "r").await;
575 set_secret(&b.http, &alice, "MY_SECRET", SECRET_VALUE).await;
576 let worker = register_runner(&b, &alice, "runner-a").await;
577
578 let work = clone(
579 &remote(b.port, "alice", &token, "alice", "r"),
580 &b.dir.join("alice-work"),
581 );
582 let sha = push_ci_config(&work, CI_TOML);
583
584 let found = wait_statuses(
585 &b.http,
586 &token,
587 &sha,
588 &["ci/build", "ci/after", "ci/broken", "ci/skipme"],
589 )
590 .await;
591 assert_eq!(found["ci/build"].state, "pending", "build is queued");
592
593 let ran = worker.drain().await;
594 assert_eq!(ran, 3, "build, after and broken run; skipme never does");
595
596 let found = statuses(&b.http, &token, &sha).await;
597 assert_eq!(found["ci/build"].state, "success");
598 assert_eq!(found["ci/after"].state, "success");
599 assert_eq!(found["ci/broken"].state, "failure");
600 assert_eq!(found["ci/skipme"].state, "error");
601 assert_eq!(found["ci/skipme"].description, "skipped");
602
603 // The build log: the secret is masked, never printed.
604 let build_id = job_id_of(&found["ci/build"]);
605 let log = b
606 .http
607 .get(&format!("{REPO_PATH}/ci/jobs/{build_id}/log.txt"), None)
608 .await;
609 assert_eq!(log.status, 200, "GET log.txt");
610 assert!(log.body.contains("hello"), "log has the echo: {}", log.body);
611 assert!(log.body.contains("***"), "secret is masked: {}", log.body);
612 assert!(
613 !log.body.contains(SECRET_VALUE),
614 "secret leaked into the log: {}",
615 log.body
616 );
617
618 // The artifact comes back with the right bytes. `out/a.txt` is stored as `out_a.txt`.
619 let artifact = b
620 .http
621 .get(
622 &format!("{REPO_PATH}/ci/jobs/{build_id}/artifacts/out_a.txt"),
623 Some(&alice),
624 )
625 .await;
626 assert_eq!(artifact.status, 200, "GET artifact: {}", server_log(&b));
627 assert_eq!(artifact.body, "data\n", "artifact bytes");
628
629 // The run page lists every job.
630 let run = b
631 .http
632 .get(&format!("{REPO_PATH}/ci/runs/1"), Some(&alice))
633 .await;
634 assert_eq!(run.status, 200, "GET run page");
635 for job in ["build", "after", "broken", "skipme"] {
636 assert!(run.body.contains(job), "run page lists {job}");
637 }
638}
639
640#[tokio::test]
641async fn fork_pull_waits_for_approval_and_runs_without_secrets() {
642 let b = boot().await;
643 let alice = login(&b.http, "alice", ALICE_PW).await;
644 let token = create_token(&b.http, &alice, "ci-token").await;
645 create_repo(&b.http, &alice, "r").await;
646 set_secret(&b.http, &alice, "MY_SECRET", SECRET_VALUE).await;
647 let worker = register_runner(&b, &alice, "runner-a").await;
648
649 // Base branch with the workflow, run to completion (run 1).
650 let work = clone(
651 &remote(b.port, "alice", &token, "alice", "r"),
652 &b.dir.join("alice-work"),
653 );
654 let base_sha = push_ci_config(&work, CI_TOML);
655 wait_statuses(&b.http, &token, &base_sha, &["ci/build"]).await;
656 worker.drain().await;
657
658 // Bob forks and changes the workflow on a branch.
659 let bob = login(&b.http, "bob", BOB_PW).await;
660 let bob_token = create_token(&b.http, &bob, "bob-token").await;
661 let csrf = csrf_from(&b.http, "/settings/tokens", &bob).await;
662 let fork = b
663 .http
664 .post_form(&format!("{REPO_PATH}/fork"), &[("csrf", &csrf)], Some(&bob))
665 .await;
666 assert_eq!(fork.status, 303, "fork: {}\n{}", fork.body, server_log(&b));
667
668 let bob_work = clone(
669 &remote(b.port, "bob", &bob_token, "bob", "r"),
670 &b.dir.join("bob-work"),
671 );
672 git(&bob_work, &["checkout", "-q", "-b", "bob-feature"]);
673 let head = commit(
674 &bob_work,
675 ".forge/ci.toml",
676 CI_TOML_FORK,
677 "print the secret",
678 "bob",
679 BOB_EMAIL,
680 );
681 git(
682 &bob_work,
683 &["push", "-q", "origin", "HEAD:refs/heads/bob-feature"],
684 );
685
686 // Bob opens the pull request. The run waits for the owner.
687 let compare = b
688 .http
689 .get(
690 &format!("{REPO_PATH}/compare/main...bob:bob-feature"),
691 Some(&bob),
692 )
693 .await;
694 assert_eq!(compare.status, 200, "GET compare");
695 let csrf = csrf_from(&b.http, "/settings/tokens", &bob).await;
696 let open = b
697 .http
698 .post_form(
699 &format!("{REPO_PATH}/pulls"),
700 &[
701 ("csrf", &csrf),
702 ("title", "Bob change"),
703 ("body", ""),
704 ("base", "main"),
705 ("head", "bob:bob-feature"),
706 ],
707 Some(&bob),
708 )
709 .await;
710 assert_eq!(open.status, 303, "open pull: {}", open.body);
711
712 let found = wait_statuses(&b.http, &token, &head, &["ci/build"]).await;
713 assert_eq!(found["ci/build"].state, "pending");
714 assert_eq!(found["ci/build"].description, "Waiting for approval");
715
716 let run_page = b
717 .http
718 .get(&format!("{REPO_PATH}/ci/runs/2"), Some(&alice))
719 .await;
720 assert_eq!(run_page.status, 200, "GET run 2");
721 assert!(
722 run_page.body.contains("Approve and run"),
723 "run 2 waits for approval: {}",
724 run_page.body
725 );
726
727 // The owner approves, and the job runs with no secrets.
728 let approve = b
729 .http
730 .post_form(
731 &format!("{REPO_PATH}/ci/runs/2/approve"),
732 &[("csrf", &input_value(&run_page.body, "csrf"))],
733 Some(&alice),
734 )
735 .await;
736 assert_eq!(approve.status, 303, "approve run 2: {}", approve.body);
737 worker.drain().await;
738
739 let found = statuses(&b.http, &token, &head).await;
740 assert_eq!(found["ci/build"].state, "success", "fork build runs");
741 assert_eq!(found["ci/skipme"].state, "error");
742
743 let build_id = job_id_of(&found["ci/build"]);
744 let log = b
745 .http
746 .get(&format!("{REPO_PATH}/ci/jobs/{build_id}/log.txt"), None)
747 .await;
748 assert_eq!(log.status, 200, "GET log.txt");
749 assert!(
750 log.body.contains("value=[]"),
751 "the secret is empty for a fork run: {}",
752 log.body
753 );
754 assert!(!log.body.contains("***"), "no value to mask: {}", log.body);
755 assert!(!log.body.contains(SECRET_VALUE), "secret leaked");
756}
757
758#[tokio::test]
759async fn live_log_streams_until_the_job_finishes() {
760 let b = boot().await;
761 let alice = login(&b.http, "alice", ALICE_PW).await;
762 let token = create_token(&b.http, &alice, "ci-token").await;
763 create_repo(&b.http, &alice, "r").await;
764 let worker = register_runner(&b, &alice, "runner-a").await;
765
766 let slow = r#"[on]
767push = ["main"]
768
769[jobs.slow]
770image = "docker.io/library/alpine:3"
771run = "echo started; sleep 5; echo done"
772"#;
773 let work = clone(
774 &remote(b.port, "alice", &token, "alice", "r"),
775 &b.dir.join("alice-work"),
776 );
777 let sha = push_ci_config(&work, slow);
778 let found = wait_statuses(&b.http, &token, &sha, &["ci/slow"]).await;
779 let job_id = job_id_of(&found["ci/slow"]);
780
781 let job = worker.claim().await.expect("the slow job is claimed");
782 assert_eq!(job.job_id, job_id, "claimed the job the status points at");
783 let runner = worker.runner.clone();
784 let handle = tokio::spawn(async move { runner.process(job, CancellationToken::new()).await });
785
786 // Read the stream chunk by chunk. The first line appears while the job is still sleeping.
787 let url = format!("{}{REPO_PATH}/ci/jobs/{job_id}/log.txt", b.http.base);
788 let mut res = b.http.client.get(url).send().await.expect("GET log.txt");
789 assert_eq!(res.status().as_u16(), 200, "GET log.txt");
790 let mut body = String::new();
791 let mut seen_started = false;
792 let read = async {
793 while let Some(chunk) = res.chunk().await.expect("log chunk") {
794 body.push_str(&String::from_utf8_lossy(&chunk));
795 if !seen_started && body.contains("started") {
796 seen_started = true;
797 assert!(
798 !handle.is_finished(),
799 "the first line arrived only after the job had finished"
800 );
801 }
802 }
803 };
804 tokio::time::timeout(Duration::from_secs(90), read)
805 .await
806 .expect("the log stream ends after the job finishes");
807
808 assert!(seen_started, "the live log showed the first line: {body}");
809 assert!(
810 body.contains("done"),
811 "the stream ends with the last line: {body}"
812 );
813 let outcome = handle.await.expect("job task");
814 assert_eq!(outcome, Outcome::Success);
815 let found = statuses(&b.http, &token, &sha).await;
816 assert_eq!(found["ci/slow"].state, "success");
817}
818
819#[tokio::test]
820async fn ci_job_token_clones_the_repos_wiki_read_only() {
821 let b = boot().await;
822 let alice = login(&b.http, "alice", ALICE_PW).await;
823 let token = create_token(&b.http, &alice, "ci-token").await;
824 create_repo(&b.http, &alice, "r").await;
825 let worker = register_runner(&b, &alice, "runner-a").await;
826
827 // The wiki gets its first page through the web form.
828 let csrf = input_value(
829 &b.http
830 .get(&format!("{REPO_PATH}/wiki/-/new"), Some(&alice))
831 .await
832 .body,
833 "csrf",
834 );
835 let saved = b
836 .http
837 .post_form(
838 &format!("{REPO_PATH}/wiki/-/new"),
839 &[
840 ("csrf", &csrf),
841 ("name", "Home"),
842 ("base", ""),
843 ("content", "# Welcome from the wiki\n"),
844 ("message", "first page"),
845 ],
846 Some(&alice),
847 )
848 .await;
849 assert_eq!(saved.status, 303, "create Home: {}", saved.body);
850
851 // A slow job keeps running, so its token is valid while the test clones.
852 let slow = r#"[on]
853push = ["main"]
854
855[jobs.slow]
856image = "docker.io/library/alpine:3"
857run = "echo started; sleep 5; echo done"
858"#;
859 let work = clone(
860 &remote(b.port, "alice", &token, "alice", "r"),
861 &b.dir.join("alice-work"),
862 );
863 let sha = push_ci_config(&work, slow);
864 let found = wait_statuses(&b.http, &token, &sha, &["ci/slow"]).await;
865 let job = worker.claim().await.expect("the slow job is claimed");
866 assert_eq!(job.job_id, job_id_of(&found["ci/slow"]));
867
868 // The job reads the wiki with its own credential: `ci-job` and `<job id>:<job token>`.
869 let wiki_url = format!(
870 "http://ci-job:{}:{}@127.0.0.1:{}/alice/r.wiki.git",
871 job.job_id, job.job_token, b.port
872 );
873 let job_wiki = clone(&wiki_url, &b.dir.join("job-wiki"));
874 let home = fs::read_to_string(job_wiki.join("Home.md")).expect("Home.md in the job's clone");
875 assert!(home.contains("Welcome from the wiki"), "{home}");
876
877 // Pushing to the wiki is refused for a job token.
878 fs::write(job_wiki.join("Home.md"), "# changed by a job\n").expect("write");
879 git(&job_wiki, &["add", "Home.md"]);
880 git_env(
881 &job_wiki,
882 &[
883 "-c",
884 "commit.gpgsign=false",
885 "commit",
886 "-q",
887 "-m",
888 "job edit",
889 ],
890 &[
891 ("GIT_AUTHOR_NAME", "job"),
892 ("GIT_AUTHOR_EMAIL", "job@example.invalid"),
893 ("GIT_COMMITTER_NAME", "job"),
894 ("GIT_COMMITTER_EMAIL", "job@example.invalid"),
895 ],
896 );
897 let push = Command::new("git")
898 .current_dir(&job_wiki)
899 .args([
900 "-c",
901 "credential.helper=",
902 "push",
903 "-q",
904 "origin",
905 "HEAD:main",
906 ])
907 .env("GIT_TERMINAL_PROMPT", "0")
908 .output()
909 .expect("run git");
910 assert!(
911 !push.status.success(),
912 "a CI job token must not push to the wiki"
913 );
914}