forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
16.6 KiB
1//! Host checks for the builtin executor, behind `forge-runner check`.
2//!
3//! The check runs one trivial job through the real sandbox, with a rootfs of host binaries, and
4//! reports each requirement as ok, failed, or skipped. It also holds [`build_rootfs`], which
5//! builds such a rootfs for the tests.
6
7use std::fs;
8use std::io;
9use std::path::{Path, PathBuf};
10use std::process::{ExitStatus, Stdio};
11
12use tokio::io::AsyncWriteExt;
13use tokio::process::Command;
14
15use crate::config::{Limits, UsernsMode};
16use crate::sandbox::SandboxRequest;
17use crate::sandbox::idmap::IdMap;
18
19/// The probe job. It prints the seccomp mode of the job, then succeeds.
20const PROBE_SCRIPT: &str = "grep '^Seccomp:' /proc/self/status; true";
21/// The probe job in subuid mode. It also chowns a file in `/tmp` to 1000:1000 and prints its
22/// owner, which needs the subordinate range.
23const PROBE_SUBUID_SCRIPT: &str = "grep '^Seccomp:' /proc/self/status; \
24 touch /tmp/forge-probe && chown 1000:1000 /tmp/forge-probe \
25 && stat -c 'owner %u %g' /tmp/forge-probe; true";
26/// The utilities the probe needs in its rootfs.
27const PROBE_BINARIES: &[&str] = &["sh", "grep", "touch", "chown", "stat"];
28/// The controllers the job cgroup needs for `[limits]`.
29const LIMIT_CONTROLLERS: [&str; 3] = ["cpu", "memory", "pids"];
30
31/// Copies `binaries` (looked up in `/usr/bin` and `/bin`) and their shared libraries into `root`,
32/// at the same absolute paths. It also creates the empty mount points the sandbox needs.
33pub fn build_rootfs(root: &Path, binaries: &[&str]) -> io::Result<()> {
34 for dir in [
35 "bin",
36 "proc",
37 "dev",
38 "sys",
39 "tmp",
40 "workspace",
41 "etc",
42 "root",
43 ] {
44 fs::create_dir_all(root.join(dir))?;
45 }
46 for name in ["null", "zero", "full", "random", "urandom"] {
47 fs::write(root.join("dev").join(name), b"")?;
48 }
49 for name in ["etc/hosts", "etc/hostname", "etc/resolv.conf"] {
50 fs::write(root.join(name), b"")?;
51 }
52
53 let mut libs: Vec<PathBuf> = Vec::new();
54 for name in binaries {
55 let host = ["/usr/bin", "/bin"]
56 .iter()
57 .map(|dir| Path::new(dir).join(name))
58 .find(|p| p.exists())
59 .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("{name} not found")))?;
60 let real = fs::canonicalize(&host)?;
61 fs::copy(&real, root.join("bin").join(name))?;
62 libs.extend(shared_libs(&real));
63 }
64 libs.sort();
65 libs.dedup();
66 for lib in &libs {
67 copy_into(root, lib)?;
68 }
69 Ok(())
70}
71
72/// The shared libraries `ldd` reports for `binary`, as absolute host paths.
73fn shared_libs(binary: &Path) -> Vec<PathBuf> {
74 let Ok(out) = std::process::Command::new("ldd").arg(binary).output() else {
75 return Vec::new();
76 };
77 String::from_utf8_lossy(&out.stdout)
78 .lines()
79 .filter_map(|line| {
80 let line = line.trim();
81 let path = match line.split_once(" => ") {
82 Some((_, rest)) => rest.split(" (").next()?,
83 None => line.split(" (").next()?,
84 };
85 path.starts_with('/').then(|| PathBuf::from(path))
86 })
87 .collect()
88}
89
90fn copy_into(root: &Path, host: &Path) -> io::Result<()> {
91 let real = fs::canonicalize(host)?;
92 let dest = root.join(host.strip_prefix("/").unwrap_or(host));
93 if let Some(parent) = dest.parent() {
94 fs::create_dir_all(parent)?;
95 }
96 fs::copy(&real, &dest)?;
97 Ok(())
98}
99
100/// How one requirement came out.
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102pub enum Status {
103 Ok,
104 Fail,
105 Skip,
106}
107
108/// One line of the report.
109#[derive(Debug, Clone, PartialEq, Eq)]
110pub struct Line {
111 pub status: Status,
112 pub name: &'static str,
113 pub detail: String,
114}
115
116impl Line {
117 fn ok(name: &'static str, detail: impl Into<String>) -> Self {
118 Self {
119 status: Status::Ok,
120 name,
121 detail: detail.into(),
122 }
123 }
124
125 fn fail(name: &'static str, detail: impl Into<String>) -> Self {
126 Self {
127 status: Status::Fail,
128 name,
129 detail: detail.into(),
130 }
131 }
132
133 fn skip(name: &'static str, detail: impl Into<String>) -> Self {
134 Self {
135 status: Status::Skip,
136 name,
137 detail: detail.into(),
138 }
139 }
140}
141
142/// The helper's output and exit status for the probe job.
143struct Probe {
144 status: ExitStatus,
145 stdout: String,
146 stderr: String,
147}
148
149/// Runs the probe job through the sandbox helper at `helper`, under `map`. `work_dir` must be
150/// absolute.
151async fn run_probe(helper: &Path, work_dir: &Path, map: &IdMap) -> Result<Probe, String> {
152 let root = work_dir.join(format!("check-{}", std::process::id()));
153 let rootfs = root.join("rootfs");
154 let build_into = rootfs.clone();
155 let built = tokio::task::spawn_blocking(move || build_rootfs(&build_into, PROBE_BINARIES))
156 .await
157 .map_err(|e| format!("cannot build the test rootfs: {e}"))?;
158 built.map_err(|e| format!("cannot build the test rootfs: {e}"))?;
159 let workspace = root.join("workspace");
160 fs::create_dir_all(&workspace)
161 .map_err(|e| format!("cannot create {}: {e}", workspace.display()))?;
162
163 let script = if map.is_subuid() {
164 PROBE_SUBUID_SCRIPT
165 } else {
166 PROBE_SCRIPT
167 };
168 let request = SandboxRequest {
169 job_id: "forge-check".to_owned(),
170 rootfs,
171 job_dir: root.join("job"),
172 workspace,
173 image_env: Vec::new(),
174 env: Vec::new(),
175 run: script.to_owned(),
176 network: false,
177 limits: Limits::default(),
178 idmap: map.clone(),
179 };
180 let json = serde_json::to_vec(&request).map_err(|e| e.to_string())?;
181 let output = async {
182 let mut child = Command::new(helper)
183 .arg("__sandbox")
184 .stdin(Stdio::piped())
185 .stdout(Stdio::piped())
186 .stderr(Stdio::piped())
187 .kill_on_drop(true)
188 .spawn()?;
189 if let Some(mut stdin) = child.stdin.take() {
190 stdin.write_all(&json).await?;
191 }
192 child.wait_with_output().await
193 }
194 .await
195 .map_err(|e| format!("cannot start the sandbox helper: {e}"));
196 let _ = fs::remove_dir_all(&root);
197 let output = output?;
198 Ok(Probe {
199 status: output.status,
200 stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
201 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
202 })
203}
204
205/// The helper's `forge-sandbox: ` lines, without the prefix.
206fn sandbox_lines(stderr: &str) -> Vec<&str> {
207 stderr
208 .lines()
209 .filter_map(|line| line.strip_prefix("forge-sandbox: "))
210 .collect()
211}
212
213/// The cgroup v2 controllers enabled for the job cgroup, which is a child of the runner's parent
214/// slice (see the helper). Returns the enabled list, or why it could not be read.
215fn delegated_controllers() -> Result<Vec<String>, String> {
216 let text = fs::read_to_string("/proc/self/cgroup")
217 .map_err(|e| format!("cannot read /proc/self/cgroup: {e}"))?;
218 let own = text
219 .lines()
220 .find_map(|line| line.strip_prefix("0::"))
221 .ok_or_else(|| "cgroup v2 is not in use by this process".to_owned())?;
222 let own = PathBuf::from(own.trim());
223 let base = own.parent().unwrap_or(&own);
224 let file = Path::new("/sys/fs/cgroup")
225 .join(base.strip_prefix("/").unwrap_or(base))
226 .join("cgroup.subtree_control");
227 let enabled =
228 fs::read_to_string(&file).map_err(|e| format!("cannot read {}: {e}", file.display()))?;
229 Ok(enabled.split_whitespace().map(str::to_owned).collect())
230}
231
232/// Builds the report from the probe result. Pure, so the rules can be tested.
233fn report(probe: Result<&Probe, &str>, controllers: Result<&[String], &str>) -> Vec<Line> {
234 let mut lines = Vec::new();
235
236 match controllers {
237 Ok(enabled) => {
238 let missing: Vec<&str> = LIMIT_CONTROLLERS
239 .iter()
240 .copied()
241 .filter(|c| !enabled.iter().any(|e| e == c))
242 .collect();
243 if missing.is_empty() {
244 lines.push(Line::ok(
245 "cgroup v2 delegation",
246 format!("controllers enabled: {}", enabled.join(" ")),
247 ));
248 } else {
249 lines.push(Line::fail(
250 "cgroup v2 delegation",
251 format!(
252 "missing {}; [limits] cannot be applied (enabled: {})",
253 missing.join(" "),
254 enabled.join(" ")
255 ),
256 ));
257 }
258 }
259 Err(reason) => lines.push(Line::fail("cgroup v2 delegation", reason)),
260 }
261
262 let probe = match probe {
263 Ok(probe) => probe,
264 Err(reason) => {
265 lines.push(Line::fail("sandbox run", reason));
266 for name in [
267 "user namespaces",
268 "overlayfs in a user namespace",
269 "resource limits",
270 "seccomp filter",
271 ] {
272 lines.push(Line::skip(name, "not tested: the probe job did not run"));
273 }
274 return lines;
275 }
276 };
277
278 let setup_errors = sandbox_lines(&probe.stderr);
279 let userns_error = setup_errors
280 .iter()
281 .find(|l| l.contains("user namespaces unavailable"));
282 let started = probe.stdout.lines().any(|l| l.starts_with("Seccomp:"));
283
284 if let Some(line) = userns_error {
285 lines.push(Line::fail("user namespaces", *line));
286 } else if started {
287 lines.push(Line::ok(
288 "user namespaces",
289 "unprivileged user namespaces work",
290 ));
291 } else {
292 lines.push(Line::skip(
293 "user namespaces",
294 "not tested: the probe job did not start",
295 ));
296 }
297
298 if !started {
299 let reason = setup_errors
300 .first()
301 .map(|l| (*l).to_owned())
302 .unwrap_or_else(|| format!("the probe exited with {}", probe.status));
303 lines.push(Line::fail("sandbox run", reason));
304 lines.push(Line::skip(
305 "overlayfs in a user namespace",
306 "not tested: the probe job did not start",
307 ));
308 lines.push(Line::skip(
309 "resource limits",
310 "not tested: the probe job did not start",
311 ));
312 lines.push(Line::skip(
313 "seccomp filter",
314 "not tested: the probe job did not start",
315 ));
316 return lines;
317 }
318
319 lines.push(Line::ok(
320 "sandbox run",
321 format!("probe job exited with {}", probe.status),
322 ));
323
324 match setup_errors
325 .iter()
326 .find(|l| l.starts_with("warning: overlay mount failed"))
327 {
328 Some(line) => lines.push(Line::fail(
329 "overlayfs in a user namespace",
330 format!("{line}; jobs copy the rootfs instead (slower, uses disk)"),
331 )),
332 None => lines.push(Line::ok(
333 "overlayfs in a user namespace",
334 "overlay mounts work",
335 )),
336 }
337
338 match setup_errors
339 .iter()
340 .find(|l| l.starts_with("warning: resource limits unavailable"))
341 {
342 Some(line) => lines.push(Line::fail("resource limits", *line)),
343 None => lines.push(Line::ok(
344 "resource limits",
345 "cpu, memory and pids limits apply",
346 )),
347 }
348
349 let mode = probe
350 .stdout
351 .lines()
352 .find_map(|l| l.strip_prefix("Seccomp:"))
353 .map(str::trim)
354 .unwrap_or_default();
355 if mode == "2" {
356 lines.push(Line::ok(
357 "seccomp filter",
358 "the job runs with a seccomp filter",
359 ));
360 } else {
361 lines.push(Line::fail(
362 "seccomp filter",
363 format!("the job's seccomp mode is {mode:?}, expected 2"),
364 ));
365 }
366 lines
367}
368
369/// The subordinate id check, in subuid mode. The probe must have started, and must have chowned
370/// its file in `/tmp` to 1000:1000, which is container id 1000 and needs the subordinate range.
371fn ownership_line(probe: &Probe) -> Line {
372 const NAME: &str = "subordinate ids";
373 if !probe.stdout.lines().any(|l| l.starts_with("Seccomp:")) {
374 return Line::skip(NAME, "not tested: the probe job did not start");
375 }
376 if probe.stdout.lines().any(|l| l.trim() == "owner 1000 1000") {
377 return Line::ok(NAME, "a job can chown files to 1000:1000");
378 }
379 let reason = probe
380 .stderr
381 .lines()
382 .find(|l| !l.starts_with("forge-sandbox: ") && !l.trim().is_empty())
383 .unwrap_or("no output");
384 Line::fail(
385 NAME,
386 format!("a job cannot chown 1000:1000 in /tmp: {reason}"),
387 )
388}
389
390/// Runs every check and prints one line per requirement. Returns `true` when all pass.
391///
392/// `helper` is the `forge-runner` binary, and `work_dir` is where the probe runs. The probe needs
393/// the same filesystem that jobs will use. `mode` is the `[sandbox] userns` setting.
394pub async fn run(helper: &Path, work_dir: &Path, mode: UsernsMode) -> bool {
395 let work_dir = match std::path::absolute(work_dir) {
396 Ok(path) => path,
397 Err(e) => {
398 println!("FAIL work dir: {}: {e}", work_dir.display());
399 return false;
400 }
401 };
402 let map = match IdMap::resolve(mode) {
403 Ok(map) => map,
404 Err(e) => {
405 println!("FAIL user ids: {e}");
406 return false;
407 }
408 };
409 let probe = run_probe(helper, &work_dir, &map).await;
410 let controllers = delegated_controllers();
411 let mut lines = vec![Line::ok("user ids", map.describe())];
412 lines.extend(report(
413 probe.as_ref().map_err(String::as_str),
414 controllers.as_deref().map_err(String::as_str),
415 ));
416 if map.is_subuid() {
417 lines.push(match &probe {
418 Ok(probe) => ownership_line(probe),
419 Err(_) => Line::skip("subordinate ids", "not tested: the probe job did not run"),
420 });
421 }
422 let mut all_ok = true;
423 for line in &lines {
424 let tag = match line.status {
425 Status::Ok => "ok ",
426 Status::Fail => {
427 all_ok = false;
428 "FAIL"
429 }
430 Status::Skip => {
431 all_ok = false;
432 "skip"
433 }
434 };
435 println!("{tag} {}: {}", line.name, line.detail);
436 }
437 all_ok
438}
439
440#[cfg(test)]
441mod tests {
442 use super::*;
443 use std::os::unix::process::ExitStatusExt;
444
445 fn probe(stdout: &str, stderr: &str) -> Probe {
446 Probe {
447 status: ExitStatus::from_raw(0),
448 stdout: stdout.to_owned(),
449 stderr: stderr.to_owned(),
450 }
451 }
452
453 fn all() -> Vec<String> {
454 LIMIT_CONTROLLERS.iter().map(|c| (*c).to_owned()).collect()
455 }
456
457 #[test]
458 fn a_healthy_host_passes_every_check() {
459 let p = probe("Seccomp:\t2\n", "");
460 let lines = report(Ok(&p), Ok(&all()));
461 assert!(lines.iter().all(|l| l.status == Status::Ok), "{lines:?}");
462 }
463
464 #[test]
465 fn missing_userns_is_reported_with_its_reason() {
466 let p = probe("", "forge-sandbox: user namespaces unavailable: EPERM\n");
467 let lines = report(Ok(&p), Ok(&all()));
468 let userns = lines
469 .iter()
470 .find(|l| l.name == "user namespaces")
471 .expect("line");
472 assert_eq!(userns.status, Status::Fail);
473 assert!(userns.detail.contains("EPERM"));
474 assert!(
475 lines
476 .iter()
477 .any(|l| l.name == "sandbox run" && l.status == Status::Fail)
478 );
479 }
480
481 #[test]
482 fn overlay_and_limit_warnings_are_failures() {
483 let p = probe(
484 "Seccomp:\t2\n",
485 "forge-sandbox: warning: overlay mount failed (EPERM), copying the rootfs instead\n\
486 forge-sandbox: warning: resource limits unavailable: cgroup\n",
487 );
488 let lines = report(Ok(&p), Ok(&all()));
489 let overlay = lines
490 .iter()
491 .find(|l| l.name.starts_with("overlayfs"))
492 .expect("line");
493 assert_eq!(overlay.status, Status::Fail);
494 let limits = lines
495 .iter()
496 .find(|l| l.name == "resource limits")
497 .expect("line");
498 assert_eq!(limits.status, Status::Fail);
499 }
500
501 #[test]
502 fn missing_controllers_fail_the_delegation_check() {
503 let p = probe("Seccomp:\t2\n", "");
504 let enabled = vec!["memory".to_owned()];
505 let lines = report(Ok(&p), Ok(&enabled));
506 let line = &lines[0];
507 assert_eq!(line.status, Status::Fail);
508 assert!(line.detail.contains("cpu pids"));
509 }
510
511 #[test]
512 fn a_probe_that_never_ran_skips_the_rest() {
513 let lines = report(Err("cannot build the test rootfs"), Ok(&all()));
514 assert_eq!(lines[1].status, Status::Fail);
515 assert!(lines.iter().skip(2).all(|l| l.status == Status::Skip));
516 }
517}