forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
20.5 KiB
1//! The sandbox helper: the body of `forge-runner __sandbox`.
2//!
3//! The process is single-threaded and has no tokio runtime, because libcontainer forks and
4//! clones. It reads one [`SandboxRequest`] from stdin, prepares the rootfs, creates and starts a
5//! container, and waits for it. Its stdout and stderr are the container's, so the runner just
6//! forwards them.
7//!
8//! Setup happens in this order:
9//! 1. Enter a user namespace with the request's [`IdMap`](super::idmap::IdMap): root is the
10//! runner's uid, and in subuid mode container ids `1..=n` are the subordinate range. The
11//! overlay mount needs CAP_SYS_ADMIN in a user namespace that owns the mount namespace.
12//! 2. An overlay mount of the image rootfs. If that fails, the rootfs is copied instead.
13//! 3. The OCI spec is written to `bundle/config.json` (0600) and libcontainer builds the
14//! container from it. The file is deleted right after `build()`, since libcontainer has
15//! read it by then.
16//! 4. The container starts and the helper waits, polling so that SIGTERM is never missed. On
17//! SIGTERM the container is killed and deleted. The exit code is the job's.
18//!
19//! Exit codes: the job's code, 128 + signal if the job was signalled, or 125 for a sandbox setup
20//! failure. A setup failure prints `forge-sandbox: <reason>` on stderr.
21
22use std::fs::{self, OpenOptions};
23use std::io::{self, Write};
24use std::os::fd::AsFd;
25use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt, lchown};
26use std::path::{Path, PathBuf};
27use std::process::Command;
28use std::sync::atomic::{AtomicBool, Ordering};
29use std::time::Duration;
30
31use libcontainer::container::Container;
32use libcontainer::container::builder::ContainerBuilder;
33use libcontainer::syscall::syscall::SyscallType;
34use nix::errno::Errno;
35use nix::mount::{MntFlags, MsFlags, mount, umount2};
36use nix::sys::signal::{SaFlags, SigAction, SigHandler, SigSet, Signal, sigaction};
37use nix::sys::wait::{WaitPidFlag, WaitStatus, waitpid};
38use nix::unistd::Pid;
39use oci_spec::runtime::Spec;
40
41use super::exec::FilteredExecutor;
42use super::idmap::enter_mapped_userns;
43use super::seccomp::build_filter;
44use super::spec::{HOSTNAME, build_spec};
45use super::{SandboxError, SandboxRequest};
46use crate::workspace::validate_job_id;
47
48/// Exit code for a sandbox setup failure. The runner maps it to a sandbox error.
49pub const EXIT_SETUP_FAILED: i32 = 125;
50/// How often the wait loop checks for SIGTERM and polls the child.
51const POLL: Duration = Duration::from_millis(50);
52const HOSTS: &str = "127.0.0.1 localhost\n::1 localhost ip6-localhost ip6-loopback\n";
53
54/// Set by the SIGTERM and SIGINT handler. The wait loop reads it.
55static TERMINATE: AtomicBool = AtomicBool::new(false);
56
57extern "C" fn on_terminate(_signal: i32) {
58 // Only an atomic store here, which is async-signal-safe.
59 TERMINATE.store(true, Ordering::SeqCst);
60}
61
62/// Entry point for `forge-runner __sandbox`. Returns the process exit code.
63pub fn helper_main() -> i32 {
64 // libcontainer logs through tracing. Its logs appear only when the operator sets RUST_LOG.
65 if std::env::var_os("RUST_LOG").is_some() {
66 let _ = tracing_subscriber::fmt()
67 .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
68 .with_writer(io::stderr)
69 .try_init();
70 }
71 match run() {
72 Ok(code) => code,
73 Err(e) => {
74 eprintln!("forge-sandbox: {e}");
75 EXIT_SETUP_FAILED
76 }
77 }
78}
79
80/// The directories under `job_dir` that the helper uses.
81struct Layout {
82 job_dir: PathBuf,
83 bundle: PathBuf,
84 state: PathBuf,
85 etc: PathBuf,
86 upper: PathBuf,
87 work: PathBuf,
88 merged: PathBuf,
89}
90
91impl Layout {
92 fn new(job_dir: &Path) -> Self {
93 let overlay = job_dir.join("overlay");
94 Self {
95 job_dir: job_dir.to_owned(),
96 bundle: job_dir.join("bundle"),
97 state: job_dir.join("state"),
98 etc: job_dir.join("etc"),
99 upper: overlay.join("upper"),
100 work: overlay.join("work"),
101 merged: overlay.join("merged"),
102 }
103 }
104}
105
106fn run() -> Result<i32, SandboxError> {
107 let req: SandboxRequest = serde_json::from_reader(io::stdin().lock())
108 .map_err(|e| SandboxError::Invalid(format!("cannot read request: {e}")))?;
109 validate_request(&req)?;
110 install_signal_handlers()?;
111
112 let layout = Layout::new(&req.job_dir);
113 prepare_dirs(&layout)?;
114 write_etc_files(&layout)?;
115
116 enter_mapped_userns(&req.idmap).map_err(|e| SandboxError::Setup(e.to_string()))?;
117 let rootfs = mount_rootfs(&req, &layout)?;
118
119 // Clean up on failure too: the state dir holds the job env.
120 let result = run_job(&req, &layout, &rootfs);
121 // The container is gone, so nothing writes to the workspace now. In subuid mode the job may
122 // have left files owned by subordinate ids, which the runner can't remove, so they go back
123 // to the runner.
124 if req.idmap.is_subuid() {
125 reown_workspace(&req.workspace);
126 }
127 cleanup(&layout);
128 result
129}
130
131/// Sets every entry under `workspace` to uid and gid 0 of this namespace, which is the runner's
132/// own id on the host. Symlinks are not followed. Entries already owned by 0:0 are left alone.
133/// Failures are warnings only, because the job has already run.
134fn reown_workspace(workspace: &Path) {
135 let mut stack = vec![workspace.to_owned()];
136 while let Some(path) = stack.pop() {
137 let meta = match fs::symlink_metadata(&path) {
138 Ok(meta) => meta,
139 Err(e) => {
140 warn(format!("cannot stat {}: {e}", path.display()));
141 continue;
142 }
143 };
144 if (meta.uid() != 0 || meta.gid() != 0)
145 && let Err(e) = lchown(&path, Some(0), Some(0))
146 {
147 warn(format!("cannot chown {}: {e}", path.display()));
148 }
149 if !meta.is_dir() {
150 continue;
151 }
152 match fs::read_dir(&path) {
153 Ok(entries) => {
154 for entry in entries {
155 match entry {
156 Ok(entry) => stack.push(entry.path()),
157 Err(e) => warn(format!("cannot read {}: {e}", path.display())),
158 }
159 }
160 }
161 Err(e) => warn(format!("cannot read {}: {e}", path.display())),
162 }
163 }
164}
165
166fn validate_request(req: &SandboxRequest) -> Result<(), SandboxError> {
167 validate_job_id(&req.job_id).map_err(|e| SandboxError::Invalid(e.to_string()))?;
168 for (what, path) in [
169 ("rootfs", &req.rootfs),
170 ("job_dir", &req.job_dir),
171 ("workspace", &req.workspace),
172 ] {
173 if !path.is_absolute() {
174 return Err(SandboxError::Invalid(format!("{what} must be absolute")));
175 }
176 }
177 if !req.workspace.is_dir() {
178 return Err(SandboxError::Invalid("workspace is not a directory".into()));
179 }
180 if !req.rootfs.is_dir() {
181 return Err(SandboxError::Invalid("rootfs is not a directory".into()));
182 }
183 if req.run.contains('\0') {
184 return Err(SandboxError::Invalid("run script contains NUL".into()));
185 }
186 Ok(())
187}
188
189/// Installs the SIGTERM and SIGINT handler. It does not set SA_RESTART. The wait loop polls,
190/// so the handler only sets a flag.
191fn install_signal_handlers() -> Result<(), SandboxError> {
192 let action = SigAction::new(
193 SigHandler::Handler(on_terminate),
194 SaFlags::empty(),
195 SigSet::empty(),
196 );
197 for sig in [Signal::SIGTERM, Signal::SIGINT] {
198 // SAFETY: the handler only performs an atomic store, which is async-signal-safe.
199 unsafe { sigaction(sig, &action) }
200 .map_err(|e| SandboxError::Setup(format!("cannot install signal handler: {e}")))?;
201 }
202 Ok(())
203}
204
205fn prepare_dirs(layout: &Layout) -> Result<(), SandboxError> {
206 for dir in [
207 &layout.job_dir,
208 &layout.bundle,
209 &layout.state,
210 &layout.etc,
211 &layout.upper,
212 &layout.work,
213 &layout.merged,
214 ] {
215 fs::create_dir_all(dir)
216 .map_err(SandboxError::io(format!("cannot create {}", dir.display())))?;
217 }
218 for dir in [&layout.job_dir, &layout.bundle, &layout.state] {
219 fs::set_permissions(dir, fs::Permissions::from_mode(0o700))
220 .map_err(SandboxError::io(format!("cannot chmod {}", dir.display())))?;
221 }
222 Ok(())
223}
224
225/// The files bind mounted at `/etc/hosts` and `/etc/hostname`.
226fn write_etc_files(layout: &Layout) -> Result<(), SandboxError> {
227 let hosts = layout.etc.join("hosts");
228 fs::write(&hosts, HOSTS).map_err(SandboxError::io("cannot write hosts"))?;
229 let hostname = layout.etc.join("hostname");
230 fs::write(&hostname, format!("{HOSTNAME}\n"))
231 .map_err(SandboxError::io("cannot write hostname"))?;
232 for file in [&hosts, &hostname] {
233 fs::set_permissions(file, fs::Permissions::from_mode(0o644))
234 .map_err(SandboxError::io(format!("cannot chmod {}", file.display())))?;
235 }
236 Ok(())
237}
238
239/// Mounts the overlay at `merged` and returns it as the rootfs. If the overlay fails, the image
240/// rootfs is copied instead, and a warning is printed.
241fn mount_rootfs(req: &SandboxRequest, layout: &Layout) -> Result<PathBuf, SandboxError> {
242 match mount_overlay(&req.rootfs, layout) {
243 Ok(()) => Ok(layout.merged.clone()),
244 Err(e) => {
245 warn(format!(
246 "overlay mount failed ({e}), copying the rootfs instead"
247 ));
248 copy_rootfs(&req.rootfs, &layout.merged)?;
249 Ok(layout.merged.clone())
250 }
251 }
252}
253
254fn mount_overlay(lower: &Path, layout: &Layout) -> Result<(), SandboxError> {
255 // Overlay options are comma separated, and the colon separates lower dirs, so paths
256 // containing either can't be expressed.
257 for p in [lower, &layout.upper, &layout.work] {
258 let s = p.to_string_lossy();
259 if s.contains(',') || s.contains(':') {
260 return Err(SandboxError::Setup(format!(
261 "overlay path has ',' or ':': {s}"
262 )));
263 }
264 }
265 let options = format!(
266 "lowerdir={},upperdir={},workdir={}",
267 lower.display(),
268 layout.upper.display(),
269 layout.work.display()
270 );
271 mount(
272 Some("overlay"),
273 layout.merged.as_path(),
274 Some("overlay"),
275 MsFlags::empty(),
276 Some(options.as_str()),
277 )
278 .map_err(|e| SandboxError::Setup(e.to_string()))
279}
280
281/// A full copy of the rootfs, reflinked where the filesystem supports it.
282fn copy_rootfs(src: &Path, dst: &Path) -> Result<(), SandboxError> {
283 let cp = ["/usr/bin/cp", "/bin/cp"]
284 .into_iter()
285 .find(|p| Path::new(p).exists())
286 .unwrap_or("cp");
287 let status = Command::new(cp)
288 .arg("-a")
289 .arg("--reflink=auto")
290 .arg(format!("{}/.", src.display()))
291 .arg(format!("{}/", dst.display()))
292 .status()
293 .map_err(SandboxError::io("cannot run cp for the rootfs copy"))?;
294 if status.success() {
295 Ok(())
296 } else {
297 Err(SandboxError::Setup(format!("rootfs copy failed: {status}")))
298 }
299}
300
301/// Runs the job with `[limits]` applied through its cgroup. If the container can't be built that
302/// way, it is rebuilt in the same cgroup without the limit values, and a warning goes to the job
303/// log.
304///
305/// libcontainer 0.7 cannot run a container in a user namespace without a cgroup manager, and it
306/// only avoids systemd when the cgroup path is absolute. So "run without cgroups" is not
307/// available. If the job cgroup can't be created at all, the job fails with exit 125.
308fn run_job(req: &SandboxRequest, layout: &Layout, rootfs: &Path) -> Result<i32, SandboxError> {
309 let cgroup = job_cgroup_path(&req.job_id, std::process::id())?;
310 let mut spec = build_spec(req, rootfs)?;
311 set_cgroup_path(&mut spec, &cgroup);
312
313 match launch_and_wait(layout, &req.job_id, &spec) {
314 Ok(code) => Ok(code),
315 Err(SandboxError::Setup(reason)) if reason.starts_with("container: ") => {
316 warn(format!("resource limits unavailable: {reason}"));
317 strip_limits(&mut spec);
318 let retry_id = format!("{}-nolimits", req.job_id);
319 launch_and_wait(layout, &retry_id, &spec)
320 }
321 Err(e) => Err(e),
322 }
323}
324
325/// The job's cgroup path, absolute and relative to the cgroup2 mount.
326///
327/// It must not be the runner's own cgroup. libcontainer kills every process in the job cgroup on
328/// delete, so that would kill the runner. A cgroup that holds processes also can't enable
329/// controllers for its children. The job cgroup goes in the parent slice instead, when the parent
330/// is a plain domain cgroup. The runner's user owns both, because systemd delegates the user
331/// slice.
332///
333/// The name is `forge-job-<job id>-<helper pid>`, so the runner can compute it after it has
334/// spawned the helper. It uses the runner's own cgroup, which is also the helper's, to find the
335/// parent slice. The path is relative to the cgroup2 mount, so the directory is
336/// `/sys/fs/cgroup` joined with it.
337pub fn job_cgroup_path(job_id: &str, helper_pid: u32) -> Result<PathBuf, SandboxError> {
338 let text = fs::read_to_string("/proc/self/cgroup")
339 .map_err(SandboxError::io("cannot read /proc/self/cgroup"))?;
340 let own = text
341 .lines()
342 .find_map(|line| line.strip_prefix("0::"))
343 .ok_or_else(|| SandboxError::Setup("cgroup v2 is not available".into()))?;
344 let own = PathBuf::from(own.trim());
345 let base = match own.parent() {
346 Some(parent) if is_plain_domain(parent) => parent.to_path_buf(),
347 _ => own,
348 };
349 // The helper's pid keeps the name unique even if two helpers share a job id.
350 Ok(base.join(format!("forge-job-{job_id}-{helper_pid}")))
351}
352
353/// Whether `cgroup` (absolute, inside the cgroup2 mount) is a normal domain cgroup. A threaded
354/// one can't hold domain children.
355fn is_plain_domain(cgroup: &Path) -> bool {
356 let file = Path::new("/sys/fs/cgroup")
357 .join(cgroup.strip_prefix("/").unwrap_or(cgroup))
358 .join("cgroup.type");
359 fs::read_to_string(file).is_ok_and(|t| t.trim() == "domain")
360}
361
362fn set_cgroup_path(spec: &mut Spec, cgroup: &Path) {
363 if let Some(linux) = spec.linux_mut() {
364 linux.set_cgroups_path(Some(cgroup.to_owned()));
365 }
366}
367
368/// Removes the resource values. The cgroup path stays, so libcontainer still uses the fs driver.
369fn strip_limits(spec: &mut Spec) {
370 if let Some(linux) = spec.linux_mut() {
371 linux.set_resources(None);
372 }
373}
374
375/// Writes the bundle, builds, starts, and waits. Build and start errors are returned as
376/// `Setup("container: …")` so the caller can tell them apart from setup errors.
377fn launch_and_wait(layout: &Layout, id: &str, spec: &Spec) -> Result<i32, SandboxError> {
378 // Start each attempt with an empty state dir, so a failed attempt leaves no state behind.
379 // The dir may not exist yet on the first attempt, so a failure here is not an error.
380 let _ = fs::remove_dir_all(&layout.state);
381 fs::create_dir_all(&layout.state).map_err(SandboxError::io("cannot create state dir"))?;
382 fs::set_permissions(&layout.state, fs::Permissions::from_mode(0o700))
383 .map_err(SandboxError::io("cannot chmod state dir"))?;
384
385 write_bundle(layout, spec)?;
386 let built = build_container(layout, id);
387 // libcontainer has read the bundle config during build, and the env it holds is now in
388 // state. Delete the bundle copy now.
389 let _ = fs::remove_file(layout.bundle.join("config.json"));
390 let mut container = match built {
391 Ok(c) => c,
392 Err(e) => {
393 return Err(SandboxError::Setup(format!(
394 "container: cannot create: {e}"
395 )));
396 }
397 };
398 let pid = match start_container(&mut container) {
399 Ok(pid) => pid,
400 Err(e) => {
401 let _ = container.delete(true);
402 return Err(e);
403 }
404 };
405 wait_for_init(&mut container, pid)
406}
407
408/// Writes `bundle/config.json` (0600). It holds the job env and secrets, so it is removed right
409/// after `build()`.
410fn write_bundle(layout: &Layout, spec: &Spec) -> Result<(), SandboxError> {
411 let json = serde_json::to_vec(spec)
412 .map_err(|e| SandboxError::Setup(format!("cannot serialize spec: {e}")))?;
413 let path = layout.bundle.join("config.json");
414 let mut file = OpenOptions::new()
415 .write(true)
416 .create(true)
417 .truncate(true)
418 .mode(0o600)
419 .open(&path)
420 .map_err(SandboxError::io(format!(
421 "cannot create {}",
422 path.display()
423 )))?;
424 file.set_permissions(fs::Permissions::from_mode(0o600))
425 .map_err(SandboxError::io("cannot chmod bundle config"))?;
426 file.write_all(&json)
427 .map_err(SandboxError::io("cannot write bundle config"))
428}
429
430/// Creates the container. Its stdin is `/dev/null`. Its stdout and stderr are dups of the
431/// helper's, so the runner receives them.
432fn build_container(layout: &Layout, id: &str) -> Result<Container, String> {
433 let io_err = |e: io::Error| e.to_string();
434 let devnull = fs::File::open("/dev/null").map_err(io_err)?;
435 let stdout = io::stdout().as_fd().try_clone_to_owned().map_err(io_err)?;
436 let stderr = io::stderr().as_fd().try_clone_to_owned().map_err(io_err)?;
437 // The seccomp BPF is compiled here, before libcontainer forks. The executor only installs it.
438 let filter = build_filter().map_err(|e| e.to_string())?;
439 let builder = ContainerBuilder::new(id.to_owned(), SyscallType::default())
440 .validate_id()
441 .and_then(|b| b.with_root_path(&layout.state))
442 .map_err(|e| e.to_string())?;
443 builder
444 .with_stdin(devnull)
445 .with_stdout(stdout)
446 .with_stderr(stderr)
447 .with_executor(FilteredExecutor::new(filter))
448 .as_init(&layout.bundle)
449 .with_systemd(false)
450 .build()
451 .map_err(|e| e.to_string())
452}
453
454/// Starts the container and returns the init pid. The init is a child of this process.
455fn start_container(container: &mut Container) -> Result<Pid, SandboxError> {
456 container
457 .start()
458 .map_err(|e| SandboxError::Setup(format!("container: cannot start: {e}")))?;
459 // libcontainer has its own nix, so its Pid is a different type. The raw value is the same.
460 container
461 .pid()
462 .map(|pid| Pid::from_raw(pid.as_raw()))
463 .ok_or_else(|| SandboxError::Setup("container: no init pid".into()))
464}
465
466/// SIGKILL, as libcontainer's own signal type. Its `Signal` is not nix's, so it is built from
467/// the number.
468const SIGKILL_NUMBER: i32 = 9;
469
470fn kill_container(container: &mut Container) -> Result<(), String> {
471 let sigkill = libcontainer::signal::Signal::try_from(SIGKILL_NUMBER)
472 .map_err(|_| "cannot build SIGKILL".to_owned())?;
473 container.kill(sigkill, true).map_err(|e| e.to_string())
474}
475
476/// Waits for the init process. On SIGTERM the container is killed. The child is then reaped,
477/// and `delete` removes the container's state. Returns the job's exit code.
478fn wait_for_init(container: &mut Container, pid: Pid) -> Result<i32, SandboxError> {
479 let mut killed = false;
480 loop {
481 if !killed && TERMINATE.load(Ordering::SeqCst) {
482 killed = true;
483 if let Err(e) = kill_container(container) {
484 warn(format!("cannot kill container: {e}"));
485 }
486 }
487 match waitpid(pid, Some(WaitPidFlag::WNOHANG)) {
488 Ok(WaitStatus::StillAlive) => std::thread::sleep(POLL),
489 Ok(WaitStatus::Exited(_, code)) => {
490 delete_container(container);
491 return Ok(code);
492 }
493 Ok(WaitStatus::Signaled(_, sig, _)) => {
494 delete_container(container);
495 return Ok(128 + sig as i32);
496 }
497 Ok(_) => {}
498 Err(Errno::EINTR) => {}
499 Err(e) => {
500 let _ = kill_container(container);
501 delete_container(container);
502 return Err(SandboxError::Setup(format!("waiting for the job: {e}")));
503 }
504 }
505 }
506}
507
508fn delete_container(container: &mut Container) {
509 if let Err(e) = container.delete(true) {
510 warn(format!("cannot delete container: {e}"));
511 }
512}
513
514/// Removes the whole job dir: state and bundle (which hold the env), and the overlay upper dir
515/// (which may hold files the job wrote). The container is gone by now. The overlay is still
516/// mounted in this process's mount namespace, so detach it first. The helper is still root in its
517/// user namespace, so it can remove the overlay work files that the runner's uid could not.
518fn cleanup(layout: &Layout) {
519 let _ = umount2(layout.merged.as_path(), MntFlags::MNT_DETACH);
520 if let Err(e) = fs::remove_dir_all(&layout.job_dir) {
521 warn(format!("cannot remove {}: {e}", layout.job_dir.display()));
522 }
523}
524
525fn warn(message: impl std::fmt::Display) {
526 eprintln!("forge-sandbox: warning: {message}");
527}