forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
7.7 KiB
1//! Backup sets (phase 1, see `docs/backup-interfaces.md`).
2//!
3//! A set is a directory `forge-backup-<UTC stamp>/` holding a consistent DB snapshot, one git
4//! bundle per non-empty repo, optionally the instance key, and a `manifest.json` with a sha256 for
5//! every file. It is written under a `.partial-…` name and renamed into place only when complete.
6//! Nothing is written under `/tmp`: scratch repos live next to the output directory.
7
8mod create;
9mod fsck;
10pub(crate) mod healthcheck;
11mod restore;
12mod retention;
13pub mod scheduler;
14mod verify;
15
16use std::fs::{DirBuilder, File};
17use std::io::Read;
18use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
19use std::path::{Path, PathBuf};
20
21use forge_core::{Error, Result, Store};
22use forge_git::GitEnv;
23use serde::{Deserialize, Serialize};
24use sha2::{Digest, Sha256};
25
26pub use create::create_backup;
27pub use fsck::{FsckSummary, run_fsck};
28pub use restore::restore_backup;
29pub use scheduler::Jobs;
30pub use verify::verify_backup;
31
32/// Manifest format version. Bump when the layout changes incompatibly.
33pub const FORMAT_VERSION: u32 = 1;
34pub const MANIFEST_FILE: &str = "manifest.json";
35pub const DB_FILE: &str = "db/forge.db";
36pub const KEY_FILE: &str = "keys/forge.key";
37const BACKUP_DIR_PREFIX: &str = "forge-backup-";
38const PARTIAL_PREFIX: &str = ".partial-";
39const COPY_BUF: usize = 64 * 1024;
40
41/// What a backup set contains.
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
43#[serde(rename_all = "kebab-case")]
44pub enum Scope {
45 All,
46 ReposOnly,
47 DbOnly,
48}
49
50/// Where a backup reads from. Cheap to build from the server state: `Store` and `GitEnv` clone.
51#[derive(Clone)]
52pub struct BackupSource {
53 pub store: Store,
54 /// `data_dir` holds the repos, the instance key and the managed gitconfig.
55 pub git: GitEnv,
56 /// Recorded in the manifest. Informational only.
57 pub base_url: String,
58}
59
60#[derive(Debug, Clone)]
61pub struct BackupOptions {
62 /// Directory that receives the set. Created (0700) when missing.
63 pub out: PathBuf,
64 pub include_key: bool,
65 pub scope: Scope,
66}
67
68#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
69pub struct BackupSummary {
70 /// The finished set, `<out>/forge-backup-<stamp>`.
71 pub path: PathBuf,
72 pub scope: Scope,
73 pub files: usize,
74 pub bytes: u64,
75 pub repos: usize,
76 /// Repos that have a bundle.
77 pub bundles: usize,
78 /// Repos with no refs, recorded with `bundle: null`.
79 pub empty_repos: usize,
80 pub schema_version: Option<u32>,
81 pub key_included: bool,
82}
83
84/// Outcome of [`verify_backup`]. `problems` lists every failed check. The set is good only when it
85/// is empty.
86#[derive(Debug, Clone, PartialEq, Eq)]
87pub struct VerifyReport {
88 pub path: PathBuf,
89 pub scope: Scope,
90 pub files: usize,
91 pub bytes: u64,
92 pub repos: usize,
93 pub bundles_verified: usize,
94 pub empty_repos: usize,
95 pub schema_version: Option<u32>,
96 pub problems: Vec<String>,
97}
98
99impl VerifyReport {
100 pub fn is_ok(&self) -> bool {
101 self.problems.is_empty()
102 }
103}
104
105#[derive(Debug, Serialize, Deserialize)]
106pub struct Manifest {
107 pub format: u32,
108 pub forge_version: String,
109 pub created_at: String,
110 pub base_url: String,
111 pub scope: Scope,
112 pub db_snapshot: Option<DbSnapshot>,
113 pub key_included: bool,
114 pub files: Vec<FileEntry>,
115 pub repos: Vec<RepoEntry>,
116}
117
118#[derive(Debug, Serialize, Deserialize)]
119pub struct DbSnapshot {
120 pub method: String,
121 pub schema_version: u32,
122}
123
124#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
125pub struct FileEntry {
126 pub path: String,
127 pub bytes: u64,
128 pub sha256: String,
129}
130
131#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
132pub struct RepoEntry {
133 pub id: String,
134 pub owner: String,
135 pub name: String,
136 pub refs_sha256: String,
137 /// Relative path of the bundle, or `null` for an empty repo.
138 pub bundle: Option<String>,
139 /// The repo's wiki, when it has one (absent for a set made before wikis existed).
140 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub wiki: Option<WikiEntry>,
142}
143
144/// A wiki repo in a backup set: the same fields as a repo, with the bundle at `repos/<id>.wiki.bundle`.
145#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
146pub struct WikiEntry {
147 pub refs_sha256: String,
148 /// Relative path of the wiki bundle, or `null` when the wiki has no commits yet.
149 pub bundle: Option<String>,
150}
151
152fn io_err(e: std::io::Error) -> Error {
153 Error::Io(e)
154}
155
156/// Creates a directory with mode 0700. Fails if it already exists.
157fn create_private_dir(path: &Path) -> Result<()> {
158 DirBuilder::new().mode(0o700).create(path).map_err(io_err)
159}
160
161/// Creates `path` and missing parents with mode 0700. Existing directories are left alone.
162fn ensure_private_dir_all(path: &Path) -> Result<()> {
163 DirBuilder::new()
164 .recursive(true)
165 .mode(0o700)
166 .create(path)
167 .map_err(io_err)
168}
169
170/// Creates a new file with mode 0600 and writes `bytes` to it, then fsyncs it.
171fn write_private_file(path: &Path, bytes: &[u8]) -> Result<()> {
172 use std::io::Write;
173 let mut f = std::fs::OpenOptions::new()
174 .write(true)
175 .create_new(true)
176 .mode(0o600)
177 .open(path)
178 .map_err(io_err)?;
179 f.write_all(bytes).map_err(io_err)?;
180 f.sync_all().map_err(io_err)
181}
182
183fn set_private(path: &Path) -> Result<()> {
184 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).map_err(io_err)
185}
186
187fn fsync_dir(path: &Path) -> Result<()> {
188 File::open(path).map_err(io_err)?.sync_all().map_err(io_err)
189}
190
191/// Streams the file through sha256. Returns its size and lowercase hex digest.
192fn hash_file(path: &Path) -> Result<(u64, String)> {
193 let mut f = File::open(path).map_err(io_err)?;
194 let mut hasher = Sha256::new();
195 let mut buf = vec![0u8; COPY_BUF];
196 let mut total = 0u64;
197 loop {
198 let n = f.read(&mut buf).map_err(io_err)?;
199 if n == 0 {
200 break;
201 }
202 hasher.update(&buf[..n]);
203 total += n as u64;
204 }
205 Ok((total, hex::encode(hasher.finalize())))
206}
207
208/// `20260102T030405Z`, the UTC stamp used in set names.
209fn name_stamp(t: time::OffsetDateTime) -> String {
210 format!(
211 "{:04}{:02}{:02}T{:02}{:02}{:02}Z",
212 t.year(),
213 u8::from(t.month()),
214 t.day(),
215 t.hour(),
216 t.minute(),
217 t.second()
218 )
219}
220
221/// RFC 3339 UTC, used in the manifest.
222fn manifest_stamp(t: time::OffsetDateTime) -> String {
223 format!(
224 "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
225 t.year(),
226 u8::from(t.month()),
227 t.day(),
228 t.hour(),
229 t.minute(),
230 t.second()
231 )
232}
233
234/// A name that is unique within this process run, for partial and scratch paths.
235fn unique_suffix() -> String {
236 let nanos = time::OffsetDateTime::now_utc().unix_timestamp_nanos();
237 format!("{}-{nanos}", std::process::id())
238}
239
240/// Hex-encoded sha256 of zero bytes, the `refs_sha256` of an empty repo.
241fn empty_sha256() -> String {
242 hex::encode(Sha256::digest(b""))
243}
244
245/// Whether `rel` is a file path a set may contain: the fixed names, or a bundle under `repos/`
246/// (`<id>.bundle` for a repo, `<id>.wiki.bundle` for its wiki).
247fn is_known_path(rel: &str) -> bool {
248 if rel == DB_FILE || rel == KEY_FILE {
249 return true;
250 }
251 rel.strip_prefix("repos/")
252 .and_then(|name| name.strip_suffix(".bundle"))
253 .map(|id| id.strip_suffix(".wiki").unwrap_or(id))
254 .is_some_and(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric()))
255}
256
257/// Sort key for files in the manifest: the path as text.
258fn sorted(mut files: Vec<FileEntry>) -> Vec<FileEntry> {
259 files.sort_by(|a, b| a.path.cmp(&b.path));
260 files
261}
262
263#[cfg(test)]
264mod tests;