forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
16.6 KiB
1//! Real sandbox tests. They run `forge-runner __sandbox` with a rootfs built here from the host's
2//! shell and a few utilities plus their shared libraries. Nothing is pulled from a registry, and
3//! there is no network access. The tests skip, with a message, when unprivileged user
4//! namespaces are unavailable.
5
6use std::fs;
7use std::io::Write;
8use std::path::{Path, PathBuf};
9use std::process::{Command, Output, Stdio};
10use std::time::{Duration, Instant};
11
12use std::os::unix::fs::MetadataExt;
13
14use forge_runner::config::Limits;
15use forge_runner::sandbox::SandboxRequest;
16use forge_runner::sandbox::idmap::{IdMap, UsernsMode};
17
18/// Utilities copied into the test rootfs. `sh` is the job shell, and the rest are used by the
19/// test scripts.
20const BINARIES: &[&str] = &[
21 "sh", "id", "cat", "grep", "touch", "sleep", "unshare", "chown", "stat", "mkdir",
22];
23
24fn userns_available() -> bool {
25 let max = fs::read_to_string("/proc/sys/user/max_user_namespaces")
26 .ok()
27 .and_then(|s| s.trim().parse::<u64>().ok())
28 .unwrap_or(0);
29 if max == 0 {
30 return false;
31 }
32 Command::new("unshare")
33 .args(["--user", "--map-root-user", "true"])
34 .status()
35 .is_ok_and(|s| s.success())
36}
37
38/// Skips the test when user namespaces are unavailable. Returns `true` when the test should run.
39fn can_run() -> bool {
40 if userns_available() {
41 true
42 } else {
43 eprintln!("skipping: unprivileged user namespaces are not available on this host");
44 false
45 }
46}
47
48/// The shared libraries `ldd` reports for `binary`, as absolute host paths.
49fn shared_libs(binary: &Path) -> Vec<PathBuf> {
50 let Ok(out) = Command::new("ldd").arg(binary).output() else {
51 return Vec::new();
52 };
53 String::from_utf8_lossy(&out.stdout)
54 .lines()
55 .filter_map(|line| {
56 let line = line.trim();
57 let path = match line.split_once(" => ") {
58 Some((_, rest)) => rest.split(" (").next()?,
59 None => line.split(" (").next()?,
60 };
61 path.starts_with('/').then(|| PathBuf::from(path))
62 })
63 .collect()
64}
65
66fn copy_into(root: &Path, host: &Path) {
67 let real = fs::canonicalize(host).expect("canonicalize host file");
68 let dest = root.join(host.strip_prefix("/").unwrap_or(host));
69 if let Some(parent) = dest.parent() {
70 fs::create_dir_all(parent).expect("create dest dir");
71 }
72 fs::copy(&real, &dest).expect("copy file into rootfs");
73}
74
75/// Builds a tiny rootfs in `root`: the binaries in `/bin`, their shared libraries at the same
76/// absolute paths, and the empty mount points the spec needs.
77fn build_rootfs(root: &Path) {
78 for dir in [
79 "bin",
80 "proc",
81 "dev",
82 "sys",
83 "tmp",
84 "workspace",
85 "etc",
86 "root",
87 ] {
88 fs::create_dir_all(root.join(dir)).expect("create rootfs dir");
89 }
90 for name in ["null", "zero", "full", "random", "urandom"] {
91 fs::write(root.join("dev").join(name), b"").expect("placeholder device");
92 }
93 fs::write(root.join("etc/hosts"), b"").expect("placeholder hosts");
94 fs::write(root.join("etc/hostname"), b"").expect("placeholder hostname");
95 fs::write(root.join("etc/resolv.conf"), b"").expect("placeholder resolv.conf");
96
97 let mut libs: Vec<PathBuf> = Vec::new();
98 for name in BINARIES {
99 let host = ["/usr/bin", "/bin"]
100 .iter()
101 .map(|dir| Path::new(dir).join(name))
102 .find(|p| p.exists())
103 .unwrap_or_else(|| panic!("{name} not found on this host"));
104 let real = fs::canonicalize(&host).expect("canonicalize binary");
105 fs::copy(&real, root.join("bin").join(name)).expect("copy binary");
106 libs.extend(shared_libs(&real));
107 }
108 libs.sort();
109 libs.dedup();
110 for lib in &libs {
111 copy_into(root, lib);
112 }
113}
114
115fn request(dir: &Path, rootfs: &Path, run: &str, network: bool) -> SandboxRequest {
116 let workspace = dir.join("workspace");
117 fs::create_dir_all(&workspace).expect("create workspace");
118 SandboxRequest {
119 job_id: "job_01".to_owned(),
120 rootfs: rootfs.to_owned(),
121 job_dir: dir.join("jobs/job_01"),
122 workspace,
123 image_env: Vec::new(),
124 env: vec![("FORGE_TEST".to_owned(), "yes".to_owned())],
125 run: run.to_owned(),
126 network,
127 limits: Limits::default(),
128 idmap: IdMap::single(),
129 }
130}
131
132/// The subuid map, or `None` with a message when the host can't provide one.
133fn subuid_map() -> Option<IdMap> {
134 match IdMap::resolve(UsernsMode::Subuid) {
135 Ok(map) if map.is_subuid() => Some(map),
136 Ok(_) => None,
137 Err(e) => {
138 eprintln!("skipping: subordinate ids are not available: {e}");
139 None
140 }
141 }
142}
143
144/// Spawns the helper and writes the request to its stdin. Returns the child, not yet waited on.
145fn spawn_helper(req: &SandboxRequest) -> std::process::Child {
146 let mut child = Command::new(env!("CARGO_BIN_EXE_forge-runner"))
147 .arg("__sandbox")
148 .stdin(Stdio::piped())
149 .stdout(Stdio::piped())
150 .stderr(Stdio::piped())
151 .spawn()
152 .expect("spawn helper");
153 let json = serde_json::to_vec(req).expect("serialize request");
154 let mut stdin = child.stdin.take().expect("helper stdin");
155 stdin.write_all(&json).expect("write request");
156 drop(stdin);
157 child
158}
159
160fn run_helper(req: &SandboxRequest) -> Output {
161 spawn_helper(req).wait_with_output().expect("helper output")
162}
163
164fn stdout_of(out: &Output) -> String {
165 String::from_utf8_lossy(&out.stdout).into_owned()
166}
167
168fn stderr_of(out: &Output) -> String {
169 String::from_utf8_lossy(&out.stderr).into_owned()
170}
171
172#[test]
173fn job_runs_as_root_with_no_capabilities() {
174 if !can_run() {
175 return;
176 }
177 let dir = tempfile::tempdir().expect("tempdir");
178 let rootfs = dir.path().join("rootfs");
179 build_rootfs(&rootfs);
180 let run = "echo hi; id -u; grep -E '^(CapEff|NoNewPrivs|Seccomp):' /proc/self/status; \
181 touch /workspace/f; echo \"$CI $FORGE_CI $HOME $FORGE_TEST\"";
182 let req = request(dir.path(), &rootfs, run, false);
183 let out = run_helper(&req);
184
185 let stdout = stdout_of(&out);
186 let stderr = stderr_of(&out);
187 assert_eq!(out.status.code(), Some(0), "stderr: {stderr}");
188 assert!(stdout.lines().any(|l| l == "hi"), "stdout: {stdout}");
189 assert!(stdout.lines().any(|l| l == "0"), "uid is not 0: {stdout}");
190 assert!(
191 stdout.contains("CapEff:\t0000000000000000"),
192 "capabilities not empty: {stdout}"
193 );
194 assert!(
195 stdout.contains("NoNewPrivs:\t1"),
196 "no_new_privs off: {stdout}"
197 );
198 // Seccomp mode 2 is a filter. The profile is installed by `sandbox::exec::FilteredExecutor`.
199 assert!(
200 stdout.contains("Seccomp:\t2"),
201 "seccomp filter not active: {stdout}"
202 );
203 assert!(
204 stdout.lines().any(|l| l == "true true /root yes"),
205 "env wrong: {stdout}"
206 );
207 assert!(
208 req.workspace.join("f").exists(),
209 "workspace file missing on host"
210 );
211 assert!(!req.job_dir.join("bundle/config.json").exists());
212 assert!(!req.job_dir.exists(), "job dir was not removed");
213}
214
215#[test]
216fn exit_code_is_passed_through() {
217 if !can_run() {
218 return;
219 }
220 let dir = tempfile::tempdir().expect("tempdir");
221 let rootfs = dir.path().join("rootfs");
222 build_rootfs(&rootfs);
223 let out = run_helper(&request(dir.path(), &rootfs, "exit 7", false));
224 assert_eq!(out.status.code(), Some(7), "stderr: {}", stderr_of(&out));
225}
226
227#[test]
228fn unshare_is_denied_inside_the_job() {
229 if !can_run() {
230 return;
231 }
232 let dir = tempfile::tempdir().expect("tempdir");
233 let rootfs = dir.path().join("rootfs");
234 build_rootfs(&rootfs);
235 // Creating a user namespace works on the host. The seccomp filter makes unshare(2) fail with
236 // EPERM inside the job, so this must print "denied".
237 let run = "if unshare --user true 2>/dev/null; then echo allowed; else echo denied; fi";
238 let out = run_helper(&request(dir.path(), &rootfs, run, false));
239 let stdout = stdout_of(&out);
240 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
241 assert!(stdout.lines().any(|l| l == "denied"), "stdout: {stdout}");
242 assert!(!stdout.lines().any(|l| l == "allowed"), "stdout: {stdout}");
243}
244
245#[test]
246fn network_false_has_only_loopback() {
247 if !can_run() {
248 return;
249 }
250 let dir = tempfile::tempdir().expect("tempdir");
251 let rootfs = dir.path().join("rootfs");
252 build_rootfs(&rootfs);
253 let out = run_helper(&request(dir.path(), &rootfs, "cat /proc/net/dev", false));
254 let stdout = stdout_of(&out);
255 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
256 let interfaces: Vec<&str> = stdout
257 .lines()
258 .filter_map(|l| l.split_once(':').map(|(name, _)| name.trim()))
259 .collect();
260 assert_eq!(interfaces, ["lo"], "unexpected interfaces: {stdout}");
261}
262
263#[test]
264fn network_true_shares_the_host_network() {
265 if !can_run() {
266 return;
267 }
268 let dir = tempfile::tempdir().expect("tempdir");
269 let rootfs = dir.path().join("rootfs");
270 build_rootfs(&rootfs);
271 let out = run_helper(&request(dir.path(), &rootfs, "cat /proc/net/dev", true));
272 let stdout = stdout_of(&out);
273 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
274 let names = |text: &str| -> Vec<String> {
275 let mut names: Vec<String> = text
276 .lines()
277 .filter_map(|l| l.split_once(':').map(|(name, _)| name.trim().to_owned()))
278 .collect();
279 names.sort();
280 names
281 };
282 let host = fs::read_to_string("/proc/net/dev").expect("host /proc/net/dev");
283 assert_eq!(
284 names(&stdout),
285 names(&host),
286 "job does not see the host network"
287 );
288}
289
290#[test]
291fn host_sysctl_is_read_only() {
292 if !can_run() {
293 return;
294 }
295 let dir = tempfile::tempdir().expect("tempdir");
296 let rootfs = dir.path().join("rootfs");
297 build_rootfs(&rootfs);
298 let run = "if echo x > /proc/sys/kernel/hostname; then echo WROTE; else echo DENIED; fi";
299 let out = run_helper(&request(dir.path(), &rootfs, run, false));
300 let stdout = stdout_of(&out);
301 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
302 assert!(stdout.contains("DENIED"), "write was not denied: {stdout}");
303 assert!(!stdout.contains("WROTE"), "write was allowed: {stdout}");
304}
305
306#[test]
307fn writes_land_in_the_overlay_not_the_image_rootfs() {
308 if !can_run() {
309 return;
310 }
311 let dir = tempfile::tempdir().expect("tempdir");
312 let rootfs = dir.path().join("rootfs");
313 build_rootfs(&rootfs);
314 let out = run_helper(&request(
315 dir.path(),
316 &rootfs,
317 "touch /etc/forge-marker",
318 false,
319 ));
320 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
321 assert!(
322 !rootfs.join("etc/forge-marker").exists(),
323 "the job wrote into the image rootfs"
324 );
325}
326
327#[test]
328fn sigterm_kills_the_container_and_the_helper_exits() {
329 if !can_run() {
330 return;
331 }
332 let dir = tempfile::tempdir().expect("tempdir");
333 let rootfs = dir.path().join("rootfs");
334 build_rootfs(&rootfs);
335 let req = request(dir.path(), &rootfs, "sleep 60", false);
336 let child = spawn_helper(&req);
337 // Give the container time to start, so that SIGTERM hits a running job.
338 std::thread::sleep(Duration::from_secs(2));
339 let pid = nix::unistd::Pid::from_raw(i32::try_from(child.id()).expect("pid fits i32"));
340 nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGTERM).expect("send SIGTERM");
341
342 let started = Instant::now();
343 let out = child.wait_with_output().expect("helper output");
344 assert!(
345 started.elapsed() < Duration::from_secs(20),
346 "helper did not exit after SIGTERM"
347 );
348 assert_eq!(out.status.code(), Some(137), "stderr: {}", stderr_of(&out));
349}
350
351#[test]
352fn setup_failure_exits_125_with_a_reason() {
353 let dir = tempfile::tempdir().expect("tempdir");
354 let missing = dir.path().join("no-such-rootfs");
355 let req = request(dir.path(), &missing, "echo never", false);
356 let out = run_helper(&req);
357 assert_eq!(out.status.code(), Some(125));
358 assert!(
359 stderr_of(&out).contains("forge-sandbox: "),
360 "stderr: {}",
361 stderr_of(&out)
362 );
363}
364
365#[test]
366fn rootfs_is_copied_when_the_overlay_cannot_be_mounted() {
367 if !can_run() {
368 return;
369 }
370 let dir = tempfile::tempdir().expect("tempdir");
371 // A comma in a lower dir can't be written in overlay options, so the overlay mount fails
372 // and the helper falls back to copying the rootfs.
373 let rootfs = dir.path().join("rootfs,copy");
374 build_rootfs(&rootfs);
375 let out = run_helper(&request(
376 dir.path(),
377 &rootfs,
378 "echo copied; touch /etc/forge-marker",
379 false,
380 ));
381 let stdout = stdout_of(&out);
382 let stderr = stderr_of(&out);
383 assert_eq!(out.status.code(), Some(0), "stderr: {stderr}");
384 assert!(stdout.lines().any(|l| l == "copied"), "stdout: {stdout}");
385 assert!(
386 stderr.contains("overlay mount failed"),
387 "no fallback warning: {stderr}"
388 );
389 assert!(!rootfs.join("etc/forge-marker").exists());
390}
391
392/// A request that runs under `map` (subuid mode when `map` has a range).
393fn subuid_request(dir: &Path, rootfs: &Path, run: &str, map: &IdMap) -> SandboxRequest {
394 let mut req = request(dir, rootfs, run, false);
395 req.idmap = map.clone();
396 req
397}
398
399#[test]
400fn subuid_job_can_chown_to_a_subordinate_id() {
401 let Some(map) = subuid_map() else { return };
402 if !can_run() {
403 return;
404 }
405 let dir = tempfile::tempdir().expect("tempdir");
406 let rootfs = dir.path().join("rootfs");
407 build_rootfs(&rootfs);
408 let out = run_helper(&subuid_request(
409 dir.path(),
410 &rootfs,
411 "touch /tmp/f && chown 1000:1000 /tmp/f && stat -c '%u %g' /tmp/f",
412 &map,
413 ));
414 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
415 assert_eq!(stdout_of(&out).trim(), "1000 1000");
416}
417
418#[test]
419fn subuid_uid_and_gid_maps_have_one_plus_the_range() {
420 let Some(map) = subuid_map() else { return };
421 if !can_run() {
422 return;
423 }
424 let dir = tempfile::tempdir().expect("tempdir");
425 let rootfs = dir.path().join("rootfs");
426 build_rootfs(&rootfs);
427 let out = run_helper(&subuid_request(
428 dir.path(),
429 &rootfs,
430 "cat /proc/self/uid_map; cat /proc/self/gid_map",
431 &map,
432 ));
433 let stdout = stdout_of(&out);
434 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
435 // Each line is `inside outside count`. The job's root plus the subordinate range make the
436 // map's size, which is `1 + n`, in one or two lines. Sum the counts for each map.
437 let sizes: Vec<u32> = stdout
438 .lines()
439 .filter_map(|l| l.split_whitespace().nth(2)?.parse().ok())
440 .collect();
441 assert_eq!(sizes.len(), 4, "stdout: {stdout}");
442 let (uid_sizes, gid_sizes) = sizes.split_at(2);
443 assert_eq!(
444 uid_sizes.iter().sum::<u32>(),
445 map.size(),
446 "stdout: {stdout}"
447 );
448 assert_eq!(
449 gid_sizes.iter().sum::<u32>(),
450 map.size(),
451 "stdout: {stdout}"
452 );
453 assert!(map.size() > 1, "the range is empty");
454}
455
456#[test]
457fn subuid_job_files_are_owned_by_the_runner_afterwards() {
458 let Some(map) = subuid_map() else { return };
459 if !can_run() {
460 return;
461 }
462 let dir = tempfile::tempdir().expect("tempdir");
463 let rootfs = dir.path().join("rootfs");
464 build_rootfs(&rootfs);
465 let req = subuid_request(
466 dir.path(),
467 &rootfs,
468 "touch /workspace/a && chown 1000:1000 /workspace/a \
469 && mkdir /workspace/d && chown 1000:1000 /workspace/d \
470 && touch /workspace/d/x && chown 1000:1000 /workspace/d/x",
471 &map,
472 );
473 let out = run_helper(&req);
474 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
475 for path in ["a", "d", "d/x"] {
476 let path = req.workspace.join(path);
477 let meta = fs::symlink_metadata(&path).expect("stat workspace entry");
478 assert_eq!(
479 (meta.uid(), meta.gid()),
480 (map.uid, map.gid),
481 "{} is not owned by the runner",
482 path.display()
483 );
484 }
485}
486
487#[test]
488fn subuid_job_dir_is_removed_afterwards() {
489 let Some(map) = subuid_map() else { return };
490 if !can_run() {
491 return;
492 }
493 let dir = tempfile::tempdir().expect("tempdir");
494 let rootfs = dir.path().join("rootfs");
495 build_rootfs(&rootfs);
496 let req = subuid_request(
497 dir.path(),
498 &rootfs,
499 "touch /workspace/a && chown 1000:1000 /workspace/a",
500 &map,
501 );
502 let out = run_helper(&req);
503 assert_eq!(out.status.code(), Some(0), "stderr: {}", stderr_of(&out));
504 assert!(!req.job_dir.exists(), "the job dir was left behind");
505}