forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
11.1 KiB
1//! Checks a backup set end to end. Every problem found is recorded in the report, so one run shows
2//! all of them. Only a set that cannot be read at all, such as a missing or unparseable manifest,
3//! is an error.
4
5use std::collections::BTreeSet;
6use std::path::{Path, PathBuf};
7
8use forge_core::store::snapshot::verify_snapshot;
9use forge_core::{Error, Result};
10use forge_git::GitEnv;
11
12use super::{
13 DB_FILE, FORMAT_VERSION, FileEntry, KEY_FILE, MANIFEST_FILE, Manifest, Scope, VerifyReport,
14 create_private_dir, empty_sha256, hash_file, io_err, is_known_path,
15};
16
17/// Directories a set may contain. Anything else is reported as unexpected.
18const DIRS: [&str; 3] = ["db", "repos", "keys"];
19
20/// Verifies the set at `path`. `git` runs the bundle checks. Scratch repos go in a private
21/// directory next to `path`, never in `/tmp`, and are removed before this returns.
22pub async fn verify_backup(path: &Path, git: &GitEnv) -> Result<VerifyReport> {
23 let path = std::path::absolute(path).map_err(io_err)?;
24 let bytes = std::fs::read(path.join(MANIFEST_FILE)).map_err(io_err)?;
25 let manifest: Manifest = serde_json::from_slice(&bytes)
26 .map_err(|e| Error::Invalid(format!("manifest.json does not parse: {e}")))?;
27 if manifest.format != FORMAT_VERSION {
28 return Err(Error::Invalid(format!(
29 "unsupported manifest format {} (this binary reads {FORMAT_VERSION})",
30 manifest.format
31 )));
32 }
33
34 let mut problems = Vec::new();
35 check_layout(&path, &manifest, &mut problems)?;
36 let total_bytes = check_files(&path, &manifest, &mut problems).await?;
37 let schema_version = check_db(&path, &manifest, &mut problems).await;
38 let bundles_verified = check_bundles(&path, &manifest, git, &mut problems).await?;
39
40 Ok(VerifyReport {
41 path,
42 scope: manifest.scope,
43 files: manifest.files.len(),
44 bytes: total_bytes,
45 repos: manifest.repos.len(),
46 bundles_verified,
47 empty_repos: manifest.repos.iter().filter(|r| r.bundle.is_none()).count(),
48 schema_version,
49 problems,
50 })
51}
52
53/// Every file on disk must be listed, and the scope, key and repo entries must agree with each
54/// other.
55fn check_layout(root: &Path, manifest: &Manifest, problems: &mut Vec<String>) -> Result<()> {
56 let mut on_disk = Vec::new();
57 walk(root, root, &mut on_disk, problems)?;
58 let listed: BTreeSet<&str> = manifest.files.iter().map(|f| f.path.as_str()).collect();
59 for rel in &on_disk {
60 if rel != MANIFEST_FILE && !listed.contains(rel.as_str()) {
61 problems.push(format!("unlisted file in the set: {rel}"));
62 }
63 }
64 for f in &manifest.files {
65 if !is_known_path(&f.path) {
66 problems.push(format!("manifest lists an unknown path: {}", f.path));
67 }
68 }
69
70 let has_key = listed.contains(KEY_FILE);
71 if manifest.key_included != has_key {
72 problems.push(format!(
73 "key_included is {} but the set {} keys/forge.key",
74 manifest.key_included,
75 if has_key {
76 "contains"
77 } else {
78 "does not contain"
79 }
80 ));
81 }
82 let has_db = listed.contains(DB_FILE);
83 let want_db = manifest.scope != Scope::ReposOnly;
84 if has_db != want_db || has_db != manifest.db_snapshot.is_some() {
85 problems.push(format!(
86 "scope {:?} does not match the database entries (file listed: {has_db})",
87 manifest.scope
88 ));
89 }
90 let bundles: Vec<&str> = manifest
91 .repos
92 .iter()
93 .filter_map(|r| r.bundle.as_deref())
94 .collect();
95 if manifest.scope == Scope::DbOnly && (!bundles.is_empty() || !manifest.repos.is_empty()) {
96 problems.push("a db-only set must not list repos".into());
97 }
98 let mut ids = BTreeSet::new();
99 for repo in &manifest.repos {
100 if !ids.insert(repo.id.as_str()) {
101 problems.push(format!("repo {} is listed twice", repo.id));
102 }
103 match &repo.bundle {
104 Some(b) => {
105 if *b != format!("repos/{}.bundle", repo.id) || !listed.contains(b.as_str()) {
106 problems.push(format!(
107 "repo {}: bundle {b} is not listed as a file",
108 repo.id
109 ));
110 }
111 }
112 None => {
113 if repo.refs_sha256 != empty_sha256() {
114 problems.push(format!(
115 "repo {} has no bundle but its refs are not empty",
116 repo.id
117 ));
118 }
119 }
120 }
121 // The wiki, when the set has one, follows the same rules under its own file name.
122 if let Some(wiki) = &repo.wiki {
123 match &wiki.bundle {
124 Some(b) => {
125 if *b != format!("repos/{}.wiki.bundle", repo.id)
126 || !listed.contains(b.as_str())
127 {
128 problems.push(format!(
129 "repo {}: wiki bundle {b} is not listed as a file",
130 repo.id
131 ));
132 }
133 }
134 None => {
135 if wiki.refs_sha256 != empty_sha256() {
136 problems.push(format!(
137 "repo {} has no wiki bundle but its wiki refs are not empty",
138 repo.id
139 ));
140 }
141 }
142 }
143 }
144 }
145 Ok(())
146}
147
148/// Collects regular files under `dir` as paths relative to `root`. Unexpected directories and
149/// special files are reported.
150fn walk(root: &Path, dir: &Path, out: &mut Vec<String>, problems: &mut Vec<String>) -> Result<()> {
151 for entry in std::fs::read_dir(dir).map_err(io_err)? {
152 let entry = entry.map_err(io_err)?;
153 let full = entry.path();
154 let rel = full
155 .strip_prefix(root)
156 .map_err(|e| Error::Invalid(e.to_string()))?
157 .to_string_lossy()
158 .into_owned();
159 let kind = entry.file_type().map_err(io_err)?;
160 if kind.is_dir() {
161 let top_level = dir == root;
162 if top_level && DIRS.contains(&rel.as_str()) {
163 walk(root, &full, out, problems)?;
164 } else {
165 problems.push(format!("unexpected directory in the set: {rel}"));
166 }
167 } else if kind.is_file() {
168 out.push(rel);
169 } else {
170 problems.push(format!("unexpected entry in the set: {rel}"));
171 }
172 }
173 Ok(())
174}
175
176/// Recomputes size and sha256 for every listed file. Returns the total size of the listed files.
177async fn check_files(root: &Path, manifest: &Manifest, problems: &mut Vec<String>) -> Result<u64> {
178 let mut total = 0u64;
179 for entry in &manifest.files {
180 let FileEntry {
181 path: rel,
182 bytes,
183 sha256,
184 } = entry;
185 total += bytes;
186 if !is_known_path(rel) {
187 continue;
188 }
189 let full = root.join(rel);
190 if !full.is_file() {
191 problems.push(format!("missing file: {rel}"));
192 continue;
193 }
194 let (actual_bytes, actual_sha) = tokio::task::spawn_blocking(move || hash_file(&full))
195 .await
196 .map_err(|e| Error::Io(std::io::Error::other(e)))??;
197 if actual_bytes != *bytes {
198 problems.push(format!(
199 "{rel}: size mismatch (manifest {bytes}, file {actual_bytes})"
200 ));
201 }
202 if actual_sha != *sha256 {
203 problems.push(format!(
204 "{rel}: sha256 mismatch (manifest {sha256}, file {actual_sha})"
205 ));
206 }
207 }
208 Ok(total)
209}
210
211/// Opens the DB snapshot read-only, runs `integrity_check`, and compares its schema version and
212/// repo rows with the manifest. The repo comparison runs only for full sets, because a db-only
213/// set lists no repos. Returns the snapshot's schema version when it could be read.
214async fn check_db(root: &Path, manifest: &Manifest, problems: &mut Vec<String>) -> Option<u32> {
215 let snap = root.join(DB_FILE);
216 if !snap.is_file() {
217 return None;
218 }
219 let info = match verify_snapshot(&snap).await {
220 Ok(info) => info,
221 Err(e) => {
222 problems.push(format!("database snapshot failed verification: {e}"));
223 return None;
224 }
225 };
226 if let Some(d) = &manifest.db_snapshot
227 && d.schema_version != info.schema_version
228 {
229 problems.push(format!(
230 "manifest says schema {} but the snapshot is schema {}",
231 d.schema_version, info.schema_version
232 ));
233 }
234 if manifest.scope == Scope::All {
235 let mut want: Vec<(String, String, String)> = manifest
236 .repos
237 .iter()
238 .map(|r| (r.id.clone(), r.owner.clone(), r.name.clone()))
239 .collect();
240 want.sort();
241 let mut have = info.repos.clone();
242 have.sort();
243 if want != have {
244 problems.push(format!(
245 "manifest repo list ({}) does not match the database snapshot's repos table ({})",
246 want.len(),
247 have.len()
248 ));
249 }
250 }
251 Some(info.schema_version)
252}
253
254/// Imports every bundle into its own fresh scratch repo and checks the imported refs against the
255/// manifest. Returns how many bundles passed. Scratch space is removed before returning.
256async fn check_bundles(
257 root: &Path,
258 manifest: &Manifest,
259 git: &GitEnv,
260 problems: &mut Vec<String>,
261) -> Result<usize> {
262 let bundles: Vec<(usize, &super::RepoEntry, &str)> = manifest
263 .repos
264 .iter()
265 .enumerate()
266 .filter_map(|(i, r)| r.bundle.as_deref().map(|b| (i, r, b)))
267 .collect();
268 if bundles.is_empty() {
269 return Ok(0);
270 }
271 let parent = root
272 .parent()
273 .ok_or_else(|| Error::Invalid("the backup set has no parent directory".into()))?;
274 let suffix = format!(
275 "{}-{}",
276 std::process::id(),
277 time::OffsetDateTime::now_utc().unix_timestamp_nanos()
278 );
279 let scratch_root: PathBuf = parent.join(format!(".partial-verify-{suffix}"));
280 create_private_dir(&scratch_root)?;
281
282 let mut passed = 0;
283 for (idx, repo, bundle) in bundles {
284 let scratch = scratch_root.join(format!("{idx}.git"));
285 let outcome = async {
286 forge_git::repo::init_bare(git, &scratch, "main").await?;
287 forge_git::backup::bundle_verify(git, &scratch, &root.join(bundle)).await
288 }
289 .await;
290 match outcome {
291 Ok(imported) if imported == repo.refs_sha256 => passed += 1,
292 Ok(imported) => problems.push(format!(
293 "repo {}: bundle refs {imported} do not match the manifest {}",
294 repo.id, repo.refs_sha256
295 )),
296 Err(e) => problems.push(format!("repo {}: bundle failed verification: {e}", repo.id)),
297 }
298 }
299 if let Err(e) = std::fs::remove_dir_all(&scratch_root) {
300 problems.push(format!(
301 "could not remove scratch space {}: {e}",
302 scratch_root.display()
303 ));
304 }
305 Ok(passed)
306}