forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
9.1 KiB
1//! Artifact collection: expand a job's globs inside its workspace and flatten the matches into
2//! upload names.
3//!
4//! The walk never follows symlinks, only regular files count, and every match is canonicalized
5//! and checked to stay inside the canonical workspace.
6
7use std::collections::HashSet;
8use std::fs::FileType;
9use std::path::{Path, PathBuf};
10
11use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
12
13/// The server accepts names matching `[A-Za-z0-9._+-]{1,128}` and not starting with `.`.
14pub const MAX_NAME_LEN: usize = 128;
15/// The server caps artifacts per job at 50 by default.
16pub const MAX_FILES: usize = 50;
17
18/// Errors from collecting artifacts. Any of them fails the job.
19#[derive(Debug, thiserror::Error)]
20pub enum ArtifactError {
21 #[error("invalid artifact pattern {0:?}: must be relative, without ..")]
22 Pattern(String),
23 #[error("invalid artifact glob: {0}")]
24 Glob(#[from] globset::Error),
25 #[error("cannot flatten artifact path {0:?} into a valid name")]
26 Name(String),
27 #[error("two artifacts flatten to the same name {0:?}")]
28 Duplicate(String),
29 #[error("artifact {name:?} is {size} bytes, over the {limit} byte limit")]
30 TooLarge { name: String, size: u64, limit: u64 },
31 #[error("more than {MAX_FILES} artifacts matched")]
32 TooMany,
33 #[error("artifact io: {0}")]
34 Io(#[from] std::io::Error),
35}
36
37/// One file to upload.
38#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct Artifact {
40 /// The flattened upload name.
41 pub name: String,
42 /// The file on disk, inside the workspace.
43 pub path: PathBuf,
44 pub size: u64,
45}
46
47/// Turns a workspace-relative path into one upload name: `/` becomes `_`, characters outside
48/// the server alphabet become `_`, and a leading `.` is replaced so the name is never hidden.
49pub fn flatten_name(rel: &str) -> Result<String, ArtifactError> {
50 let joined = rel
51 .split('/')
52 .filter(|c| !c.is_empty())
53 .collect::<Vec<_>>()
54 .join("_");
55 let mut name: String = joined
56 .chars()
57 .map(|c| {
58 if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '+' | '-') {
59 c
60 } else {
61 '_'
62 }
63 })
64 .collect();
65 while name.starts_with('.') {
66 name.replace_range(0..1, "_");
67 }
68 if name.is_empty() || name.len() > MAX_NAME_LEN {
69 return Err(ArtifactError::Name(rel.to_owned()));
70 }
71 Ok(name)
72}
73
74/// Rejects patterns that are absolute or climb out of the workspace.
75pub fn check_pattern(pattern: &str) -> Result<(), ArtifactError> {
76 let bad =
77 pattern.is_empty() || pattern.starts_with('/') || pattern.split('/').any(|c| c == "..");
78 if bad {
79 Err(ArtifactError::Pattern(pattern.to_owned()))
80 } else {
81 Ok(())
82 }
83}
84
85fn build_globs(patterns: &[String]) -> Result<GlobSet, ArtifactError> {
86 let mut builder = GlobSetBuilder::new();
87 for p in patterns {
88 check_pattern(p)?;
89 builder.add(GlobBuilder::new(p).literal_separator(true).build()?);
90 }
91 Ok(builder.build()?)
92}
93
94/// Expands `patterns` inside `workspace`. Returns the matches sorted by name.
95pub fn collect(
96 workspace: &Path,
97 patterns: &[String],
98 max_bytes_per_file: u64,
99) -> Result<Vec<Artifact>, ArtifactError> {
100 if patterns.is_empty() {
101 return Ok(Vec::new());
102 }
103 let globs = build_globs(patterns)?;
104 let root = workspace.canonicalize()?;
105
106 let mut found: Vec<(String, PathBuf)> = Vec::new();
107 let mut stack: Vec<(PathBuf, String)> = vec![(root.clone(), String::new())];
108 while let Some((dir, prefix)) = stack.pop() {
109 let mut entries: Vec<_> = std::fs::read_dir(&dir)?.collect::<Result<_, _>>()?;
110 entries.sort_by_key(|e| e.file_name());
111 for entry in entries {
112 let file_name = entry.file_name();
113 let Some(file_name) = file_name.to_str() else {
114 continue;
115 };
116 let rel = if prefix.is_empty() {
117 file_name.to_owned()
118 } else {
119 format!("{prefix}/{file_name}")
120 };
121 // DirEntry::file_type does not follow symlinks.
122 let file_type: FileType = entry.file_type()?;
123 if file_type.is_symlink() {
124 continue;
125 }
126 if file_type.is_dir() {
127 if file_name != ".git" {
128 stack.push((entry.path(), rel));
129 }
130 } else if file_type.is_file() && globs.is_match(&rel) {
131 found.push((rel, entry.path()));
132 }
133 }
134 }
135
136 found.sort();
137 if found.len() > MAX_FILES {
138 return Err(ArtifactError::TooMany);
139 }
140
141 let mut seen = HashSet::new();
142 let mut out = Vec::with_capacity(found.len());
143 for (rel, path) in found {
144 // Defense in depth: the walk never enters symlinks, but check the real location too.
145 let real = path.canonicalize()?;
146 if !real.starts_with(&root) {
147 continue;
148 }
149 let meta = std::fs::symlink_metadata(&real)?;
150 if !meta.is_file() {
151 continue;
152 }
153 let name = flatten_name(&rel)?;
154 if !seen.insert(name.clone()) {
155 return Err(ArtifactError::Duplicate(name));
156 }
157 if meta.len() > max_bytes_per_file {
158 return Err(ArtifactError::TooLarge {
159 name,
160 size: meta.len(),
161 limit: max_bytes_per_file,
162 });
163 }
164 out.push(Artifact {
165 name,
166 path: real,
167 size: meta.len(),
168 });
169 }
170 Ok(out)
171}
172
173#[cfg(test)]
174mod tests {
175 use super::*;
176 use std::os::unix::fs::symlink;
177
178 fn ws_with(files: &[(&str, &str)]) -> (tempfile::TempDir, PathBuf) {
179 let dir = tempfile::tempdir().expect("tempdir");
180 let ws = dir.path().join("ws");
181 std::fs::create_dir_all(&ws).expect("mkdir");
182 for (rel, body) in files {
183 let p = ws.join(rel);
184 std::fs::create_dir_all(p.parent().expect("parent")).expect("mkdir");
185 std::fs::write(&p, body).expect("write");
186 }
187 (dir, ws)
188 }
189
190 fn names(found: &[Artifact]) -> Vec<&str> {
191 found.iter().map(|a| a.name.as_str()).collect()
192 }
193
194 #[test]
195 fn flattening_maps_separators_and_hidden_names() {
196 assert_eq!(
197 flatten_name("dist/app/v1.tar.gz").expect("ok"),
198 "dist_app_v1.tar.gz"
199 );
200 assert_eq!(flatten_name(".hidden/x").expect("ok"), "_hidden_x");
201 assert_eq!(flatten_name("a b/c~d").expect("ok"), "a_b_c_d");
202 assert!(flatten_name("").is_err());
203 assert!(flatten_name(&"x".repeat(MAX_NAME_LEN + 1)).is_err());
204 }
205
206 #[test]
207 fn globs_match_inside_the_workspace() {
208 let (_dir, ws) = ws_with(&[
209 ("dist/app.bin", "a"),
210 ("dist/sub/b.txt", "b"),
211 ("other.txt", "c"),
212 ]);
213 let found = collect(&ws, &["dist/**".to_owned()], 1024).expect("collect");
214 assert_eq!(names(&found), vec!["dist_app.bin", "dist_sub_b.txt"]);
215 let found = collect(&ws, &["dist/*".to_owned()], 1024).expect("collect");
216 assert_eq!(names(&found), vec!["dist_app.bin"]);
217 }
218
219 #[test]
220 fn parent_directory_patterns_are_rejected() {
221 let (_dir, ws) = ws_with(&[("x.txt", "x")]);
222 assert!(matches!(
223 collect(&ws, &["../*".to_owned()], 1024),
224 Err(ArtifactError::Pattern(_))
225 ));
226 assert!(matches!(
227 collect(&ws, &["/etc/passwd".to_owned()], 1024),
228 Err(ArtifactError::Pattern(_))
229 ));
230 }
231
232 #[test]
233 fn a_file_outside_the_workspace_is_never_reached() {
234 let (dir, ws) = ws_with(&[("x.txt", "x")]);
235 std::fs::write(dir.path().join("secret.txt"), "outside").expect("write");
236 symlink(dir.path().join("secret.txt"), ws.join("link.txt")).expect("symlink");
237 let found = collect(&ws, &["**".to_owned()], 1024).expect("collect");
238 assert_eq!(names(&found), vec!["x.txt"]);
239 }
240
241 #[test]
242 fn symlink_to_etc_is_skipped() {
243 let (_dir, ws) = ws_with(&[("dist/real.txt", "r")]);
244 symlink("/etc/passwd", ws.join("dist/passwd")).expect("symlink");
245 symlink("/etc", ws.join("etc-dir")).expect("symlink");
246 let found = collect(&ws, &["**".to_owned()], 1024).expect("collect");
247 assert_eq!(names(&found), vec!["dist_real.txt"]);
248 }
249
250 #[test]
251 fn oversized_files_are_refused() {
252 let (_dir, ws) = ws_with(&[("big.bin", "0123456789")]);
253 assert!(matches!(
254 collect(&ws, &["*.bin".to_owned()], 4),
255 Err(ArtifactError::TooLarge { .. })
256 ));
257 }
258
259 #[test]
260 fn colliding_flattened_names_are_refused() {
261 let (_dir, ws) = ws_with(&[("a/b.txt", "1"), ("a_b.txt", "2")]);
262 assert!(matches!(
263 collect(&ws, &["**".to_owned()], 1024),
264 Err(ArtifactError::Duplicate(_))
265 ));
266 }
267
268 #[test]
269 fn git_dir_is_not_collected() {
270 let (_dir, ws) = ws_with(&[(".git/config", "c"), ("keep.txt", "k")]);
271 let found = collect(&ws, &["**".to_owned()], 1024).expect("collect");
272 assert_eq!(names(&found), vec!["keep.txt"]);
273 }
274}