forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
11.7 KiB
1//! The `forge-runner` command line: `register`, `run` and `check`.
2//!
3//! The standalone `forge-runner` binary and `forge runner` both call [`main_with_args`], so
4//! the two behave the same way.
5
6use std::ffi::OsString;
7use std::path::PathBuf;
8use std::process::ExitCode;
9use std::sync::Arc;
10use std::time::Duration;
11
12use clap::{CommandFactory, Parser, Subcommand};
13use forge_ci::proto::RegisterRequest;
14use tokio_util::sync::CancellationToken;
15use tracing_subscriber::EnvFilter;
16
17use crate::client::ForgeClient;
18use crate::config::{self, ExecutorKind, ImagesConfig, Limits, RunnerConfig, UsernsMode};
19use crate::executor::{BuiltinExecutor, Executor, PodmanExecutor};
20use crate::oci::ImageStore;
21use crate::runner::{RunOptions, Runner};
22use crate::sandbox::idmap::IdMap;
23
24/// How often unused images are garbage collected, after the one at startup.
25const GC_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
26
27#[derive(Parser)]
28#[command(
29 name = "forge-runner",
30 version,
31 about = "Runs forge CI jobs in a rootless sandbox (builtin) or rootless podman"
32)]
33struct Cli {
34 #[command(subcommand)]
35 command: Command,
36}
37
38#[derive(Subcommand)]
39enum Command {
40 /// Register this machine with a forge and write the runner config (mode 0600).
41 Register {
42 /// Forge base URL, for example https://forge.example
43 #[arg(long)]
44 url: String,
45 /// Registration token shown by the forge admin page.
46 #[arg(long)]
47 token: String,
48 /// Runner name shown in the forge.
49 #[arg(long)]
50 name: String,
51 /// Comma-separated runner labels, for example linux,x86_64.
52 #[arg(long, value_delimiter = ',')]
53 labels: Vec<String>,
54 /// Where to write the config. Default: ~/.config/forge-runner/runner.toml
55 #[arg(long)]
56 config: Option<PathBuf>,
57 /// Where job workspaces go. Default: ~/.cache/forge-runner/work
58 #[arg(long)]
59 work_dir: Option<PathBuf>,
60 },
61 /// Claim and run jobs until interrupted.
62 Run {
63 /// The config file. Default: ~/.config/forge-runner/runner.toml
64 #[arg(long)]
65 config: Option<PathBuf>,
66 },
67 /// Check that this host can run builtin jobs: user namespaces, overlayfs, cgroup limits,
68 /// seccomp and, in subuid mode, subordinate ids. Runs one trivial job through the real sandbox.
69 Check {
70 /// Where the probe runs. It should be on the filesystem that jobs use.
71 /// Default: ~/.cache/forge-runner/work
72 #[arg(long)]
73 work_dir: Option<PathBuf>,
74 /// The runner config, for its `[sandbox] userns` setting. Default: the default config
75 /// path, if that file exists, otherwise `auto`.
76 #[arg(long)]
77 config: Option<PathBuf>,
78 },
79}
80
81/// Runs the `forge-runner` command line. `args` are the arguments after the program name, so
82/// `register --url ...` and not `forge-runner register --url ...`. `bin_name` is shown in help
83/// and error messages, for example `forge runner`.
84pub fn main_with_args<I, T>(bin_name: &str, args: I) -> ExitCode
85where
86 I: IntoIterator<Item = T>,
87 T: Into<OsString>,
88{
89 let args: Vec<OsString> = args.into_iter().map(Into::into).collect();
90 // The sandbox helper is single-threaded and must not have a tokio runtime, because
91 // libcontainer forks and clones. Dispatch it before anything else starts.
92 if args.first().is_some_and(is_sandbox_arg) {
93 std::process::exit(hidden_dispatch(&args));
94 }
95 let runtime = match tokio::runtime::Builder::new_multi_thread()
96 .enable_all()
97 .build()
98 {
99 Ok(runtime) => runtime,
100 Err(e) => {
101 eprintln!("{bin_name}: cannot start the runtime: {e}");
102 return ExitCode::FAILURE;
103 }
104 };
105 // Another subscriber may already be installed, for example by forge-web. Keep that one.
106 let _ = tracing_subscriber::fmt()
107 .with_env_filter(
108 EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
109 )
110 .with_writer(std::io::stderr)
111 .try_init();
112 let argv = std::iter::once(OsString::from(bin_name)).chain(args);
113 let cli = match Cli::command()
114 .bin_name(bin_name)
115 .try_get_matches_from(argv)
116 .and_then(|matches| <Cli as clap::FromArgMatches>::from_arg_matches(&matches))
117 {
118 Ok(cli) => cli,
119 Err(e) => {
120 let code = e.exit_code();
121 let _ = e.print();
122 return ExitCode::from(code as u8);
123 }
124 };
125 runtime.block_on(async {
126 match run(cli).await {
127 Ok(()) => ExitCode::SUCCESS,
128 Err(e) => {
129 eprintln!("{bin_name}: {e}");
130 ExitCode::FAILURE
131 }
132 }
133 })
134}
135
136/// Whether this process was started as a hidden sandbox helper: argv[1] is `__sandbox`,
137/// `__sandbox-rm` or `__sandbox-unpack`. Callers that have their own runtime or subscriber check
138/// this first, and then call [`hidden_main`].
139pub fn is_sandbox_invocation() -> bool {
140 std::env::args_os()
141 .nth(1)
142 .is_some_and(|arg| is_sandbox_arg(&arg))
143}
144
145/// Runs the hidden helper named by argv[1]. The helpers are single-threaded and must not start
146/// a tokio runtime. Returns the process exit code.
147pub fn hidden_main() -> i32 {
148 let args: Vec<OsString> = std::env::args_os().skip(1).collect();
149 hidden_dispatch(&args)
150}
151
152fn is_sandbox_arg(arg: &OsString) -> bool {
153 arg.as_encoded_bytes().starts_with(b"__sandbox")
154}
155
156/// Routes a hidden helper command. `args` start with the command name.
157fn hidden_dispatch(args: &[OsString]) -> i32 {
158 match args.first().map(|arg| arg.as_encoded_bytes()) {
159 Some(b"__sandbox") => crate::sandbox::helper_main(),
160 Some(b"__sandbox-rm") => crate::sandbox::idmap::rm_helper_main(&args[1..]),
161 Some(b"__sandbox-unpack") => crate::oci::unpack_helper_main(),
162 _ => {
163 eprintln!("forge-sandbox: unknown helper command");
164 125
165 }
166 }
167}
168
169async fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
170 match cli.command {
171 Command::Register {
172 url,
173 token,
174 name,
175 labels,
176 config: path,
177 work_dir,
178 } => {
179 let client = ForgeClient::new(&url)?;
180 let resp = client
181 .register(&RegisterRequest {
182 token,
183 name: name.clone(),
184 labels: labels.clone(),
185 })
186 .await?;
187 let path = match path {
188 Some(p) => p,
189 None => config::default_config_path()?,
190 };
191 let work_dir = match work_dir {
192 Some(w) => w,
193 None => config::default_work_dir()?,
194 };
195 let cfg = RunnerConfig {
196 url,
197 runner_id: resp.runner_id.clone(),
198 runner_secret: resp.runner_secret,
199 name,
200 labels,
201 work_dir,
202 concurrency: 1,
203 limits: Limits::default(),
204 executor: ExecutorKind::default(),
205 images: ImagesConfig::default(),
206 sandbox: config::SandboxConfig::default(),
207 registries: Vec::new(),
208 };
209 config::save(&path, &cfg)?;
210 println!(
211 "registered runner {} and wrote {}",
212 resp.runner_id,
213 path.display()
214 );
215 Ok(())
216 }
217 Command::Run { config: path } => {
218 let path = match path {
219 Some(p) => p,
220 None => config::default_config_path()?,
221 };
222 let cfg = config::load(&path)?;
223 let shutdown = CancellationToken::new();
224 let executor: Arc<dyn Executor> = match cfg.executor {
225 ExecutorKind::Podman => Arc::new(PodmanExecutor::new(cfg.limits.clone())),
226 ExecutorKind::Builtin => {
227 // Resolved once. In subuid mode a failure here is a startup error.
228 let map = IdMap::resolve(cfg.sandbox.userns)?;
229 tracing::info!("sandbox ids: {}", map.describe());
230 let helper = std::env::current_exe()?;
231 let root = cfg.work_dir.join("oci");
232 spawn_image_gc(
233 root.clone(),
234 cfg.images.keep_days,
235 map.clone(),
236 helper.clone(),
237 shutdown.clone(),
238 );
239 let images = ImageStore::new(root)
240 .with_max_unpacked_bytes(gib(cfg.limits.image_max_gb))
241 .with_idmap(map.clone(), helper.clone());
242 let jobs_dir = cfg.work_dir.join("jobs");
243 Arc::new(
244 BuiltinExecutor::new(images, cfg.limits.clone(), helper, jobs_dir)
245 .with_idmap(map)
246 .with_registries(&cfg.registries),
247 )
248 }
249 };
250 let runner = Runner::new(cfg, executor, RunOptions::default())?;
251 let stop = shutdown.clone();
252 tokio::spawn(async move {
253 if tokio::signal::ctrl_c().await.is_ok() {
254 tracing::info!("interrupt received, cancelling running jobs");
255 stop.cancel();
256 }
257 });
258 runner.run(shutdown).await;
259 Ok(())
260 }
261 Command::Check {
262 work_dir,
263 config: config_path,
264 } => {
265 let work_dir = match work_dir {
266 Some(w) => w,
267 None => config::default_work_dir()?,
268 };
269 let mode = check_userns_mode(config_path)?;
270 let helper = std::env::current_exe()?;
271 if crate::check::run(&helper, &work_dir, mode).await {
272 Ok(())
273 } else {
274 Err("some host checks failed, see the report above".into())
275 }
276 }
277 }
278}
279
280/// The `[sandbox] userns` setting for `check`. An explicit config must exist. Without one, the
281/// default config is used if it exists, and otherwise the mode is `auto`.
282fn check_userns_mode(config_path: Option<PathBuf>) -> Result<UsernsMode, config::ConfigError> {
283 if let Some(path) = config_path {
284 return Ok(config::load(&path)?.sandbox.userns);
285 }
286 match config::default_config_path() {
287 Ok(path) if path.exists() => Ok(config::load(&path)?.sandbox.userns),
288 _ => Ok(UsernsMode::Auto),
289 }
290}
291
292/// Garbage collects unused images now and then once a day, until `shutdown` is cancelled. Images
293/// are removed under `map`, the same as when they were unpacked.
294fn spawn_image_gc(
295 root: PathBuf,
296 keep_days: u32,
297 map: IdMap,
298 helper: PathBuf,
299 shutdown: CancellationToken,
300) {
301 tokio::spawn(async move {
302 loop {
303 let (root, map, helper) = (root.clone(), map.clone(), helper.clone());
304 let result = tokio::task::spawn_blocking(move || {
305 ImageStore::new(root).with_idmap(map, helper).gc(keep_days)
306 })
307 .await;
308 match result {
309 Ok(Ok(stats)) => tracing::info!(?stats, "image garbage collection finished"),
310 Ok(Err(e)) => tracing::warn!("image garbage collection failed: {e}"),
311 Err(e) => tracing::warn!("image garbage collection task failed: {e}"),
312 }
313 tokio::select! {
314 _ = tokio::time::sleep(GC_INTERVAL) => {}
315 _ = shutdown.cancelled() => break,
316 }
317 }
318 });
319}
320
321/// `image_max_gb` as bytes.
322fn gib(gb: u64) -> u64 {
323 gb.saturating_mul(1024 * 1024 * 1024)
324}