forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
8.4 KiB
1//! Garbage collection of LFS objects (`admin lfs-gc`). An object file no repo links is deleted once
2//! it is older than the minimum age. Upload temp files of the same age are abandoned uploads and
3//! are deleted too. The age gate keeps a fresh upload from being collected before its link row is
4//! written.
5
6use std::path::{Path, PathBuf};
7use std::time::{Duration, SystemTime};
8
9use forge_core::Store;
10use serde::{Deserialize, Serialize};
11
12use super::{is_valid_oid, lfs_root, tmp_dir};
13
14/// The age an unlinked file must reach before it is deleted.
15pub const GC_MIN_AGE: Duration = Duration::from_secs(24 * 60 * 60);
16
17/// What one run did.
18#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
19pub struct GcSummary {
20 /// Object files looked at.
21 pub checked: u64,
22 /// Object files deleted.
23 pub removed: u64,
24 /// Bytes in the deleted object files.
25 pub bytes_freed: u64,
26 /// Abandoned upload temp files deleted.
27 pub temp_removed: u64,
28}
29
30/// An object file found on disk.
31struct Candidate {
32 oid: String,
33 path: PathBuf,
34 size: u64,
35 modified: SystemTime,
36}
37
38/// Deletes unlinked objects and abandoned temp files older than `min_age`.
39pub async fn gc(
40 store: &Store,
41 data_dir: &Path,
42 min_age: Duration,
43) -> forge_core::Result<GcSummary> {
44 let now = SystemTime::now();
45 let mut summary = GcSummary::default();
46
47 let temp = tmp_dir(data_dir);
48 let temp_files = tokio::task::spawn_blocking(move || list_files(&temp))
49 .await
50 .map_err(|e| forge_core::Error::Io(std::io::Error::other(e)))??;
51 for (path, modified) in temp_files {
52 if is_older_than(modified, now, min_age) && remove_if_present(&path).await? {
53 summary.temp_removed += 1;
54 }
55 }
56
57 let root = lfs_root(data_dir);
58 let objects = tokio::task::spawn_blocking(move || collect_objects(&root))
59 .await
60 .map_err(|e| forge_core::Error::Io(std::io::Error::other(e)))??;
61 for candidate in objects {
62 summary.checked += 1;
63 if !is_older_than(candidate.modified, now, min_age) {
64 continue;
65 }
66 if store.lfs_oid_is_linked(&candidate.oid).await? {
67 continue;
68 }
69 // Look again just before deleting: an upload may have renamed the file into place and
70 // linked it since the listing was taken.
71 let fresh = tokio::fs::metadata(&candidate.path).await?;
72 if !is_older_than(fresh.modified()?, now, min_age)
73 || store.lfs_oid_is_linked(&candidate.oid).await?
74 {
75 continue;
76 }
77 if remove_if_present(&candidate.path).await? {
78 summary.removed += 1;
79 summary.bytes_freed += candidate.size;
80 }
81 }
82 Ok(summary)
83}
84
85/// True when `modified` is at least `min_age` before `now`. A time in the future is never old.
86fn is_older_than(modified: SystemTime, now: SystemTime, min_age: Duration) -> bool {
87 now.duration_since(modified).is_ok_and(|age| age >= min_age)
88}
89
90/// Deletes `path`. `Ok(false)` when it was already gone.
91async fn remove_if_present(path: &Path) -> std::io::Result<bool> {
92 match tokio::fs::remove_file(path).await {
93 Ok(()) => Ok(true),
94 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
95 Err(e) => Err(e),
96 }
97}
98
99/// The regular files directly inside `dir`, with their modification times. A missing directory is
100/// empty.
101fn list_files(dir: &Path) -> forge_core::Result<Vec<(PathBuf, SystemTime)>> {
102 let mut found = Vec::new();
103 let Ok(entries) = std::fs::read_dir(dir) else {
104 return Ok(found);
105 };
106 for entry in entries {
107 let entry = entry?;
108 let meta = entry.metadata()?;
109 if meta.is_file() {
110 found.push((entry.path(), meta.modified()?));
111 }
112 }
113 Ok(found)
114}
115
116/// Every object file under `root`, found through the `aa/bb/<oid>` layout. Anything that does not
117/// match that layout (the temp directory, stray files) is skipped.
118fn collect_objects(root: &Path) -> forge_core::Result<Vec<Candidate>> {
119 let mut found = Vec::new();
120 let Ok(first) = std::fs::read_dir(root) else {
121 return Ok(found);
122 };
123 for a in first {
124 let a = a?;
125 let Some(a_name) = fan_out_name(&a)? else {
126 continue;
127 };
128 for b in std::fs::read_dir(a.path())? {
129 let b = b?;
130 let Some(b_name) = fan_out_name(&b)? else {
131 continue;
132 };
133 for file in std::fs::read_dir(b.path())? {
134 let file = file?;
135 let meta = file.metadata()?;
136 let Some(oid) = file.file_name().to_str().map(str::to_owned) else {
137 continue;
138 };
139 if !meta.is_file()
140 || !is_valid_oid(&oid)
141 || !oid.starts_with(&format!("{a_name}{b_name}"))
142 {
143 continue;
144 }
145 found.push(Candidate {
146 oid,
147 path: file.path(),
148 size: meta.len(),
149 modified: meta.modified()?,
150 });
151 }
152 }
153 }
154 Ok(found)
155}
156
157/// The name of a two-character fan-out directory, or `None` for anything else.
158fn fan_out_name(entry: &std::fs::DirEntry) -> forge_core::Result<Option<String>> {
159 if !entry.file_type()?.is_dir() {
160 return Ok(None);
161 }
162 Ok(entry
163 .file_name()
164 .to_str()
165 .filter(|name| name.len() == 2)
166 .map(str::to_owned))
167}
168
169#[cfg(test)]
170mod tests {
171 use super::*;
172 use forge_core::Visibility;
173
174 const LINKED: &str = "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393";
175 const ORPHAN: &str = "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3";
176
177 fn write_object(data: &Path, oid: &str, bytes: &[u8]) -> PathBuf {
178 let path = super::super::object_path(data, oid);
179 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
180 std::fs::write(&path, bytes).unwrap();
181 path
182 }
183
184 #[tokio::test]
185 async fn removes_unlinked_objects_and_keeps_linked_ones() {
186 let dir = tempfile::tempdir().unwrap();
187 let data = dir.path();
188 let store = Store::open(&data.join("forge.db")).await.unwrap();
189 let user = store.create_user("alice", None, "h", false).await.unwrap();
190 let repo = store
191 .create_repo(&user.id, "demo", "", Visibility::Public, "main")
192 .await
193 .unwrap();
194 let linked = write_object(data, LINKED, b"kept");
195 let orphan = write_object(data, ORPHAN, b"12345");
196 store.insert_lfs_object(&repo.id, LINKED, 4).await.unwrap();
197
198 let summary = gc(&store, data, Duration::ZERO).await.unwrap();
199
200 assert_eq!(summary.checked, 2);
201 assert_eq!(summary.removed, 1);
202 assert_eq!(summary.bytes_freed, 5);
203 assert!(linked.exists());
204 assert!(!orphan.exists());
205 }
206
207 #[tokio::test]
208 async fn young_files_and_temp_files_survive_the_age_gate() {
209 let dir = tempfile::tempdir().unwrap();
210 let data = dir.path();
211 let store = Store::open(&data.join("forge.db")).await.unwrap();
212 let orphan = write_object(data, ORPHAN, b"fresh");
213 std::fs::create_dir_all(tmp_dir(data)).unwrap();
214 let temp = tmp_dir(data).join("upload.part");
215 std::fs::write(&temp, b"partial").unwrap();
216
217 let summary = gc(&store, data, GC_MIN_AGE).await.unwrap();
218
219 assert_eq!(summary.removed, 0);
220 assert_eq!(summary.temp_removed, 0);
221 assert!(orphan.exists());
222 assert!(temp.exists());
223 }
224
225 #[tokio::test]
226 async fn abandoned_temp_files_are_removed_once_old() {
227 let dir = tempfile::tempdir().unwrap();
228 let data = dir.path();
229 let store = Store::open(&data.join("forge.db")).await.unwrap();
230 std::fs::create_dir_all(tmp_dir(data)).unwrap();
231 let temp = tmp_dir(data).join("upload.part");
232 std::fs::write(&temp, b"partial").unwrap();
233
234 let summary = gc(&store, data, Duration::ZERO).await.unwrap();
235
236 assert_eq!(summary.temp_removed, 1);
237 assert!(!temp.exists());
238 }
239
240 #[tokio::test]
241 async fn missing_store_is_a_no_op() {
242 let dir = tempfile::tempdir().unwrap();
243 let store = Store::open(&dir.path().join("forge.db")).await.unwrap();
244 let summary = gc(&store, dir.path(), GC_MIN_AGE).await.unwrap();
245 assert_eq!(summary, GcSummary::default());
246 }
247}