forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
12.0 KiB
1//! The `forge-runner` command line: `register`, `run` and `check`.
2//!
3//! The standalone `forge-runner` binary and `forge runner` both call [`main_with_args`], so
4//! the two behave the same way.
5
6use std::ffi::OsString;
7use std::path::PathBuf;
8use std::process::ExitCode;
9use std::sync::Arc;
10use std::time::Duration;
11
12use clap::{CommandFactory, Parser, Subcommand};
13use forge_ci::proto::RegisterRequest;
14use tokio_util::sync::CancellationToken;
15use tracing_subscriber::EnvFilter;
16
17use crate::client::ForgeClient;
18use crate::config::{self, ExecutorKind, ImagesConfig, Limits, RunnerConfig, UsernsMode};
19use crate::executor::{BuiltinExecutor, Executor, PodmanExecutor};
20use crate::oci::ImageStore;
21use crate::runner::{RunEnd, RunOptions, Runner};
22use crate::sandbox::idmap::IdMap;
23use crate::update;
24
25/// How often unused images are garbage collected, after the one at startup.
26const GC_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
27
28#[derive(Parser)]
29#[command(
30 name = "forge-runner",
31 version,
32 about = "Runs forge CI jobs in a rootless sandbox (builtin) or rootless podman"
33)]
34struct Cli {
35 #[command(subcommand)]
36 command: Command,
37}
38
39#[derive(Subcommand)]
40enum Command {
41 /// Register this machine with a forge and write the runner config (mode 0600).
42 Register {
43 /// Forge base URL, for example https://forge.example
44 #[arg(long)]
45 url: String,
46 /// Registration token shown by the forge admin page.
47 #[arg(long)]
48 token: String,
49 /// Runner name shown in the forge.
50 #[arg(long)]
51 name: String,
52 /// Comma-separated runner labels, for example linux,x86_64.
53 #[arg(long, value_delimiter = ',')]
54 labels: Vec<String>,
55 /// Where to write the config. Default: ~/.config/forge-runner/runner.toml
56 #[arg(long)]
57 config: Option<PathBuf>,
58 /// Where job workspaces go. Default: ~/.cache/forge-runner/work
59 #[arg(long)]
60 work_dir: Option<PathBuf>,
61 },
62 /// Claim and run jobs until interrupted.
63 Run {
64 /// The config file. Default: ~/.config/forge-runner/runner.toml
65 #[arg(long)]
66 config: Option<PathBuf>,
67 },
68 /// Check that this host can run builtin jobs: user namespaces, overlayfs, cgroup limits,
69 /// seccomp and, in subuid mode, subordinate ids. Runs one trivial job through the real sandbox.
70 Check {
71 /// Where the probe runs. It should be on the filesystem that jobs use.
72 /// Default: ~/.cache/forge-runner/work
73 #[arg(long)]
74 work_dir: Option<PathBuf>,
75 /// The runner config, for its `[sandbox] userns` setting. Default: the default config
76 /// path, if that file exists, otherwise `auto`.
77 #[arg(long)]
78 config: Option<PathBuf>,
79 },
80}
81
82/// Runs the `forge-runner` command line. `args` are the arguments after the program name, so
83/// `register --url ...` and not `forge-runner register --url ...`. `bin_name` is shown in help
84/// and error messages, for example `forge runner`.
85pub fn main_with_args<I, T>(bin_name: &str, args: I) -> ExitCode
86where
87 I: IntoIterator<Item = T>,
88 T: Into<OsString>,
89{
90 let args: Vec<OsString> = args.into_iter().map(Into::into).collect();
91 // The sandbox helper is single-threaded and must not have a tokio runtime, because
92 // libcontainer forks and clones. Dispatch it before anything else starts.
93 if args.first().is_some_and(is_sandbox_arg) {
94 std::process::exit(hidden_dispatch(&args));
95 }
96 let runtime = match tokio::runtime::Builder::new_multi_thread()
97 .enable_all()
98 .build()
99 {
100 Ok(runtime) => runtime,
101 Err(e) => {
102 eprintln!("{bin_name}: cannot start the runtime: {e}");
103 return ExitCode::FAILURE;
104 }
105 };
106 // Another subscriber may already be installed, for example by forge-web. Keep that one.
107 let _ = tracing_subscriber::fmt()
108 .with_env_filter(
109 EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
110 )
111 .with_writer(std::io::stderr)
112 .try_init();
113 let argv = std::iter::once(OsString::from(bin_name)).chain(args);
114 let cli = match Cli::command()
115 .bin_name(bin_name)
116 .try_get_matches_from(argv)
117 .and_then(|matches| <Cli as clap::FromArgMatches>::from_arg_matches(&matches))
118 {
119 Ok(cli) => cli,
120 Err(e) => {
121 let code = e.exit_code();
122 let _ = e.print();
123 return ExitCode::from(code as u8);
124 }
125 };
126 runtime.block_on(async {
127 match run(cli).await {
128 Ok(code) => code,
129 Err(e) => {
130 eprintln!("{bin_name}: {e}");
131 ExitCode::FAILURE
132 }
133 }
134 })
135}
136
137/// Whether this process was started as a hidden sandbox helper: argv[1] is `__sandbox`,
138/// `__sandbox-rm` or `__sandbox-unpack`. Callers that have their own runtime or subscriber check
139/// this first, and then call [`hidden_main`].
140pub fn is_sandbox_invocation() -> bool {
141 std::env::args_os()
142 .nth(1)
143 .is_some_and(|arg| is_sandbox_arg(&arg))
144}
145
146/// Runs the hidden helper named by argv[1]. The helpers are single-threaded and must not start
147/// a tokio runtime. Returns the process exit code.
148pub fn hidden_main() -> i32 {
149 let args: Vec<OsString> = std::env::args_os().skip(1).collect();
150 hidden_dispatch(&args)
151}
152
153fn is_sandbox_arg(arg: &OsString) -> bool {
154 arg.as_encoded_bytes().starts_with(b"__sandbox")
155}
156
157/// Routes a hidden helper command. `args` start with the command name.
158fn hidden_dispatch(args: &[OsString]) -> i32 {
159 match args.first().map(|arg| arg.as_encoded_bytes()) {
160 Some(b"__sandbox") => crate::sandbox::helper_main(),
161 Some(b"__sandbox-rm") => crate::sandbox::idmap::rm_helper_main(&args[1..]),
162 Some(b"__sandbox-unpack") => crate::oci::unpack_helper_main(),
163 _ => {
164 eprintln!("forge-sandbox: unknown helper command");
165 125
166 }
167 }
168}
169
170async fn run(cli: Cli) -> Result<ExitCode, Box<dyn std::error::Error>> {
171 match cli.command {
172 Command::Register {
173 url,
174 token,
175 name,
176 labels,
177 config: path,
178 work_dir,
179 } => {
180 let client = ForgeClient::new(&url)?;
181 let resp = client
182 .register(&RegisterRequest {
183 token,
184 name: name.clone(),
185 labels: labels.clone(),
186 })
187 .await?;
188 let path = match path {
189 Some(p) => p,
190 None => config::default_config_path()?,
191 };
192 let work_dir = match work_dir {
193 Some(w) => w,
194 None => config::default_work_dir()?,
195 };
196 let cfg = RunnerConfig {
197 url,
198 runner_id: resp.runner_id.clone(),
199 runner_secret: resp.runner_secret,
200 name,
201 labels,
202 work_dir,
203 concurrency: 1,
204 limits: Limits::default(),
205 executor: ExecutorKind::default(),
206 images: ImagesConfig::default(),
207 sandbox: config::SandboxConfig::default(),
208 registries: Vec::new(),
209 auto_update: true,
210 };
211 config::save(&path, &cfg)?;
212 println!(
213 "registered runner {} and wrote {}",
214 resp.runner_id,
215 path.display()
216 );
217 Ok(ExitCode::SUCCESS)
218 }
219 Command::Run { config: path } => {
220 let path = match path {
221 Some(p) => p,
222 None => config::default_config_path()?,
223 };
224 let cfg = config::load(&path)?;
225 let shutdown = CancellationToken::new();
226 let executor: Arc<dyn Executor> = match cfg.executor {
227 ExecutorKind::Podman => Arc::new(PodmanExecutor::new(cfg.limits.clone())),
228 ExecutorKind::Builtin => {
229 // Resolved once. In subuid mode a failure here is a startup error.
230 let map = IdMap::resolve(cfg.sandbox.userns)?;
231 tracing::info!("sandbox ids: {}", map.describe());
232 let helper = std::env::current_exe()?;
233 let root = cfg.work_dir.join("oci");
234 spawn_image_gc(
235 root.clone(),
236 cfg.images.keep_days,
237 map.clone(),
238 helper.clone(),
239 shutdown.clone(),
240 );
241 let images = ImageStore::new(root)
242 .with_max_unpacked_bytes(gib(cfg.limits.image_max_gb))
243 .with_idmap(map.clone(), helper.clone());
244 let jobs_dir = cfg.work_dir.join("jobs");
245 Arc::new(
246 BuiltinExecutor::new(images, cfg.limits.clone(), helper, jobs_dir)
247 .with_idmap(map)
248 .with_registries(&cfg.registries),
249 )
250 }
251 };
252 let runner = Runner::new(cfg, executor, RunOptions::default())?;
253 let stop = shutdown.clone();
254 tokio::spawn(async move {
255 if tokio::signal::ctrl_c().await.is_ok() {
256 tracing::info!("interrupt received, cancelling running jobs");
257 stop.cancel();
258 }
259 });
260 match runner.run(shutdown).await {
261 RunEnd::Stopped => Ok(ExitCode::SUCCESS),
262 // systemd restarts the runner on this code, and it starts the new binary.
263 RunEnd::Updated => Ok(ExitCode::from(update::RESTART_EXIT_CODE)),
264 }
265 }
266 Command::Check {
267 work_dir,
268 config: config_path,
269 } => {
270 let work_dir = match work_dir {
271 Some(w) => w,
272 None => config::default_work_dir()?,
273 };
274 let mode = check_userns_mode(config_path)?;
275 let helper = std::env::current_exe()?;
276 if crate::check::run(&helper, &work_dir, mode).await {
277 Ok(ExitCode::SUCCESS)
278 } else {
279 Err("some host checks failed, see the report above".into())
280 }
281 }
282 }
283}
284
285/// The `[sandbox] userns` setting for `check`. An explicit config must exist. Without one, the
286/// default config is used if it exists, and otherwise the mode is `auto`.
287fn check_userns_mode(config_path: Option<PathBuf>) -> Result<UsernsMode, config::ConfigError> {
288 if let Some(path) = config_path {
289 return Ok(config::load(&path)?.sandbox.userns);
290 }
291 match config::default_config_path() {
292 Ok(path) if path.exists() => Ok(config::load(&path)?.sandbox.userns),
293 _ => Ok(UsernsMode::Auto),
294 }
295}
296
297/// Garbage collects unused images now and then once a day, until `shutdown` is cancelled. Images
298/// are removed under `map`, the same as when they were unpacked.
299fn spawn_image_gc(
300 root: PathBuf,
301 keep_days: u32,
302 map: IdMap,
303 helper: PathBuf,
304 shutdown: CancellationToken,
305) {
306 tokio::spawn(async move {
307 loop {
308 let (root, map, helper) = (root.clone(), map.clone(), helper.clone());
309 let result = tokio::task::spawn_blocking(move || {
310 ImageStore::new(root).with_idmap(map, helper).gc(keep_days)
311 })
312 .await;
313 match result {
314 Ok(Ok(stats)) => tracing::info!(?stats, "image garbage collection finished"),
315 Ok(Err(e)) => tracing::warn!("image garbage collection failed: {e}"),
316 Err(e) => tracing::warn!("image garbage collection task failed: {e}"),
317 }
318 tokio::select! {
319 _ = tokio::time::sleep(GC_INTERVAL) => {}
320 _ = shutdown.cancelled() => break,
321 }
322 }
323 });
324}
325
326/// `image_max_gb` as bytes.
327fn gib(gb: u64) -> u64 {
328 gb.saturating_mul(1024 * 1024 * 1024)
329}