forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
17.4 KiB
1//! Raw-origin route `/{owner}/{repo}/-/media/{ref}/{path...}`. The literal `-` keeps it apart
2//! from raw paths, since no ref name may start with `-`. It reads the Git LFS pointer at
3//! `ref:path` and streams the object it names, when the repo links that object. Access follows
4//! the raw route in `raw.rs`: public repos are open, and private repos need a signed `?token=`.
5//! See `docs/wave-b-interfaces.md`, section 2.
6
7use forge_core::raw_token::verify_raw;
8use forge_core::{Repo, Visibility};
9use futures_util::TryStreamExt;
10use http_body::Frame;
11use http_body_util::StreamBody;
12use serde::Deserialize;
13use tokio_util::io::ReaderStream;
14use topcoat::context::{Cx, app_context};
15use topcoat::router::request::method;
16use topcoat::router::response::Response;
17use topcoat::router::{
18 Body, Method, RouteFuture, StatusCode, parse_query_params, path_param_segment,
19 path_param_segments,
20};
21
22use crate::lfs::{has_object, object_path};
23use crate::lfs_pointer;
24use crate::raw::{
25 CACHE_NONE, CACHE_PRIVATE, CACHE_PUBLIC, LINK_TTL_SECS, Lookup, Meta, OCTET, encode_path,
26 finish, lookup, now_unix, plain,
27};
28use crate::state::AppState;
29
30#[derive(Deserialize)]
31struct TokenQuery {
32 token: Option<String>,
33}
34
35/// Absolute URL of the media route for `path` at `rev` on the user-content origin. Private repos
36/// get a signed `?token=`, the same as raw URLs. The route is `/{owner}/{repo}/-/media/...`.
37pub fn media_url(state: &AppState, repo: &Repo, rev: &str, path: &str) -> String {
38 let base = state.config.user_content_base_url.trim_end_matches('/');
39 let mut url = format!(
40 "{base}/{}/{}/-/media/{}/{}",
41 encode_path(&repo.owner_name),
42 encode_path(&repo.name),
43 encode_path(rev),
44 encode_path(path)
45 );
46 if matches!(repo.visibility, Visibility::Private) {
47 let expires = now_unix().saturating_add(LINK_TTL_SECS);
48 url.push_str("?token=");
49 url.push_str(&forge_core::raw_token::sign_raw(
50 &state.key, &repo.id, expires,
51 ));
52 }
53 url
54}
55
56pub(crate) fn media_file(cx: &Cx, _body: Body) -> RouteFuture<'_> {
57 Box::pin(async move { Ok(serve(cx).await) })
58}
59
60/// Serves one LFS object. Every failure is turned into a response here, so the route never errors.
61async fn serve(cx: &Cx) -> Response {
62 let state = app_context::<AppState>(cx).clone();
63 let owner = path_param_segment(cx, "owner").to_owned();
64 let name = path_param_segment(cx, "repo").to_owned();
65 let rest = path_param_segments(cx, "rest")
66 .collect::<Vec<&str>>()
67 .join("/");
68 let is_head = method(cx) == Method::HEAD;
69 let token = parse_query_params::<TokenQuery>(cx)
70 .ok()
71 .and_then(|query| query.token);
72
73 let repo = match state.store.repo_by_owner_and_name(&owner, &name).await {
74 Ok(Some(repo)) => repo,
75 // A missing repo and a private repo without a valid token look the same.
76 Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
77 Err(e) => {
78 tracing::error!(error = %e, "media: repo lookup failed");
79 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
80 }
81 };
82 let cache = match repo.visibility {
83 Visibility::Public => CACHE_PUBLIC,
84 Visibility::Private => {
85 let valid = token
86 .as_deref()
87 .is_some_and(|t| verify_raw(&state.key, t, &repo.id, now_unix()));
88 if !valid {
89 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
90 }
91 CACHE_PRIVATE
92 }
93 };
94
95 let repo_dir = match forge_git::repo::repo_path(&state.git.data_dir, &repo.id) {
96 Ok(path) => path,
97 Err(e) => {
98 tracing::error!(error = %e, "media: repo path failed");
99 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
100 }
101 };
102 let looked = tokio::task::spawn_blocking(move || lookup(&repo_dir, &rest, None)).await;
103 // Pointers are a few hundred bytes, so they always come back inline. Anything else is not
104 // a pointer, and a symlink is never one.
105 let data = match looked {
106 Ok(Ok(Lookup::Inline {
107 symlink: false,
108 data,
109 ..
110 })) => data,
111 Ok(Ok(_)) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
112 Ok(Err(e)) => {
113 tracing::error!(error = %e, "media: lookup failed");
114 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
115 }
116 Err(e) => {
117 tracing::error!(error = %e, "media: lookup task failed");
118 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
119 }
120 };
121
122 let Some(pointer) = lfs_pointer::parse(&data) else {
123 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
124 };
125 // The repo must link the object, even when the file is on disk: objects are shared by every repo.
126 let size = match has_object(&state, &repo.id, &pointer.oid).await {
127 Ok(Some(size)) => size,
128 Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
129 Err(e) => {
130 tracing::error!(error = %e, "media: object link lookup failed");
131 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
132 }
133 };
134
135 let file = match tokio::fs::File::open(object_path(&state.config.data_dir, &pointer.oid)).await
136 {
137 Ok(file) => file,
138 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
139 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
140 }
141 Err(e) => {
142 tracing::error!(error = %e, "media: failed to open object");
143 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
144 }
145 };
146 // The Content-Length comes from the link, so a file of another size would break the response.
147 let on_disk = match file.metadata().await {
148 Ok(meta) => meta.len(),
149 Err(e) => {
150 tracing::error!(error = %e, "media: object metadata failed");
151 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
152 }
153 };
154 if on_disk != size {
155 tracing::error!(oid = %pointer.oid, on_disk, size, "media: object size differs from its link");
156 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
157 }
158
159 let meta = Meta {
160 content_type: OCTET,
161 attachment: true,
162 len: size,
163 };
164 if is_head {
165 return finish(
166 StatusCode::OK,
167 Body::empty(),
168 &pointer.oid,
169 cache,
170 Some(meta),
171 );
172 }
173 let chunks = ReaderStream::new(file).map_ok(Frame::data);
174 let body = Body::new(StreamBody::new(chunks));
175 finish(StatusCode::OK, body, &pointer.oid, cache, Some(meta))
176}
177
178#[cfg(test)]
179mod tests {
180 use super::*;
181
182 use forge_core::raw_token::sign_raw;
183 use sha2::{Digest, Sha256};
184 use topcoat::router::Router;
185 use topcoat::router::request::Request;
186
187 use crate::raw::router;
188
189 const POINTER_HEADER: &str = "version https://git-lfs.github.com/spec/v1";
190
191 struct Fixture {
192 _tmp: tempfile::TempDir,
193 state: AppState,
194 router: Router,
195 public: Repo,
196 private: Repo,
197 object: Vec<u8>,
198 oid: String,
199 }
200
201 fn sha256_hex(data: &[u8]) -> String {
202 hex::encode(Sha256::digest(data))
203 }
204
205 fn pointer_text(oid: &str, size: usize) -> Vec<u8> {
206 format!("{POINTER_HEADER}\noid sha256:{oid}\nsize {size}\n").into_bytes()
207 }
208
209 fn git(dir: &std::path::Path, args: &[&str]) {
210 let out = std::process::Command::new("git")
211 .arg("-C")
212 .arg(dir)
213 .args(["-c", "commit.gpgsign=false"])
214 .args(args)
215 .env("GIT_AUTHOR_NAME", "test")
216 .env("GIT_AUTHOR_EMAIL", "test@example.com")
217 .env("GIT_COMMITTER_NAME", "test")
218 .env("GIT_COMMITTER_EMAIL", "test@example.com")
219 .output()
220 .expect("git runs");
221 assert!(
222 out.status.success(),
223 "git {args:?} failed: {}",
224 String::from_utf8_lossy(&out.stderr)
225 );
226 }
227
228 /// Commits `files` on `main` in a work tree and pushes it to `bare`.
229 fn populate(work: &std::path::Path, bare: &std::path::Path, files: &[(&str, Vec<u8>)]) {
230 std::fs::create_dir_all(work).expect("mkdir work");
231 git(work, &["init", "-q", "-b", "main"]);
232 for (rel, contents) in files {
233 std::fs::write(work.join(rel), contents).expect("write file");
234 }
235 git(work, &["add", "-A"]);
236 git(work, &["commit", "-q", "-m", "init"]);
237 git(work, &["push", "-q", bare.to_str().expect("utf8"), "main"]);
238 }
239
240 async fn fixture() -> Fixture {
241 let tmp = tempfile::tempdir().expect("tempdir");
242 let data_dir = tmp.path().join("data");
243 std::fs::create_dir_all(&data_dir).expect("data dir");
244 let data_dir = data_dir.canonicalize().expect("canonical data dir");
245 let mut config = crate::test_support::test_config(&data_dir);
246 config.user_content_base_url = "http://127.0.0.1:7661".into();
247 let state = AppState {
248 config: std::sync::Arc::new(config),
249 ..crate::test_support::test_state(&data_dir).await
250 };
251 let store = state.store.clone();
252 let owner = store
253 .create_user("alice", None, "unused", false)
254 .await
255 .expect("user");
256 let public = store
257 .create_repo(&owner.id, "pub", "", forge_core::Visibility::Public, "main")
258 .await
259 .expect("public repo");
260 let private = store
261 .create_repo(
262 &owner.id,
263 "priv",
264 "",
265 forge_core::Visibility::Private,
266 "main",
267 )
268 .await
269 .expect("private repo");
270 for repo in [&public, &private] {
271 let bare = forge_git::repo::repo_path(&data_dir, &repo.id).expect("repo path");
272 forge_git::repo::init_bare(&state.git, &bare, "main")
273 .await
274 .expect("init bare");
275 }
276
277 let object = b"lfs object bytes\n".to_vec();
278 let oid = sha256_hex(&object);
279 let missing_oid = sha256_hex(b"never stored");
280 let stored_pointer = pointer_text(&oid, object.len());
281 let missing_pointer = pointer_text(&missing_oid, 12);
282 let files: Vec<(&str, Vec<u8>)> = vec![
283 ("big.bin", stored_pointer.clone()),
284 ("missing.bin", missing_pointer),
285 ("plain.txt", b"just text\n".to_vec()),
286 ];
287 let pub_bare = forge_git::repo::repo_path(&data_dir, &public.id).expect("path");
288 populate(&tmp.path().join("work-pub"), &pub_bare, &files);
289 let priv_bare = forge_git::repo::repo_path(&data_dir, &private.id).expect("path");
290 populate(
291 &tmp.path().join("work-priv"),
292 &priv_bare,
293 &[("big.bin", stored_pointer)],
294 );
295
296 // The object is linked to both repos and written once to the shared store.
297 for repo in [&public, &private] {
298 store
299 .insert_lfs_object(&repo.id, &oid, object.len() as u64)
300 .await
301 .expect("link object");
302 }
303 let path = object_path(&data_dir, &oid);
304 std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir objects");
305 std::fs::write(&path, &object).expect("write object");
306
307 let router = router(state.clone());
308 Fixture {
309 _tmp: tmp,
310 state,
311 router,
312 public,
313 private,
314 object,
315 oid,
316 }
317 }
318
319 async fn get(router: &Router, method: &str, uri: &str) -> Response {
320 let req = Request::builder()
321 .method(method)
322 .uri(uri)
323 .body(Body::empty())
324 .expect("request");
325 router.handle(req).await
326 }
327
328 fn header_str(res: &Response, name: &str) -> String {
329 res.headers()
330 .get(name)
331 .and_then(|v| v.to_str().ok())
332 .unwrap_or_default()
333 .to_owned()
334 }
335
336 async fn body_bytes(res: Response) -> Vec<u8> {
337 topcoat::router::to_bytes(res.into_body(), usize::MAX)
338 .await
339 .expect("body")
340 .to_vec()
341 }
342
343 #[tokio::test]
344 async fn media_serves_the_linked_object_as_a_download() {
345 let fx = fixture().await;
346 let res = get(&fx.router, "GET", "/alice/pub/-/media/main/big.bin").await;
347 assert_eq!(res.status(), StatusCode::OK);
348 assert_eq!(header_str(&res, "content-disposition"), "attachment");
349 assert_eq!(header_str(&res, "x-content-type-options"), "nosniff");
350 assert_eq!(header_str(&res, "content-type"), OCTET);
351 assert_eq!(
352 header_str(&res, "content-length"),
353 fx.object.len().to_string()
354 );
355 assert_eq!(header_str(&res, "etag"), format!("\"{}\"", fx.oid));
356 assert_eq!(body_bytes(res).await, fx.object);
357 }
358
359 #[tokio::test]
360 async fn head_has_headers_and_no_body() {
361 let fx = fixture().await;
362 let res = get(&fx.router, "HEAD", "/alice/pub/-/media/main/big.bin").await;
363 assert_eq!(res.status(), StatusCode::OK);
364 assert_eq!(
365 header_str(&res, "content-length"),
366 fx.object.len().to_string()
367 );
368 assert!(body_bytes(res).await.is_empty());
369 }
370
371 #[tokio::test]
372 async fn missing_object_is_404() {
373 let fx = fixture().await;
374 // The pointer exists, but the repo does not link its object.
375 let res = get(&fx.router, "GET", "/alice/pub/-/media/main/missing.bin").await;
376 assert_eq!(res.status(), StatusCode::NOT_FOUND);
377 }
378
379 #[tokio::test]
380 async fn non_pointer_and_missing_path_are_404() {
381 let fx = fixture().await;
382 let text = get(&fx.router, "GET", "/alice/pub/-/media/main/plain.txt").await;
383 assert_eq!(text.status(), StatusCode::NOT_FOUND);
384 let absent = get(&fx.router, "GET", "/alice/pub/-/media/main/nope.bin").await;
385 assert_eq!(absent.status(), StatusCode::NOT_FOUND);
386 let unknown_repo = get(&fx.router, "GET", "/alice/nope/-/media/main/big.bin").await;
387 assert_eq!(unknown_repo.status(), StatusCode::NOT_FOUND);
388 }
389
390 #[tokio::test]
391 async fn private_repo_needs_a_valid_token() {
392 let fx = fixture().await;
393 let path = "/alice/priv/-/media/main/big.bin";
394 let none = get(&fx.router, "GET", path).await;
395 assert_eq!(none.status(), StatusCode::NOT_FOUND);
396
397 let now = now_unix();
398 let valid = sign_raw(&fx.state.key, &fx.private.id, now + 60);
399 let ok = get(&fx.router, "GET", &format!("{path}?token={valid}")).await;
400 assert_eq!(ok.status(), StatusCode::OK);
401 assert_eq!(body_bytes(ok).await, fx.object);
402
403 let expired = sign_raw(&fx.state.key, &fx.private.id, now - 1);
404 let res = get(&fx.router, "GET", &format!("{path}?token={expired}")).await;
405 assert_eq!(res.status(), StatusCode::NOT_FOUND);
406
407 // A token signed for the public repo does not open the private one.
408 let other = sign_raw(&fx.state.key, &fx.public.id, now + 60);
409 let res = get(&fx.router, "GET", &format!("{path}?token={other}")).await;
410 assert_eq!(res.status(), StatusCode::NOT_FOUND);
411 }
412
413 #[tokio::test]
414 async fn media_url_encodes_and_signs_private_repos_only() {
415 let fx = fixture().await;
416 let public = media_url(&fx.state, &fx.public, "main", "a b/big.bin");
417 assert_eq!(
418 public,
419 "http://127.0.0.1:7661/alice/pub/-/media/main/a%20b/big.bin"
420 );
421 let private = media_url(&fx.state, &fx.private, "main", "big.bin");
422 assert!(
423 private.starts_with("http://127.0.0.1:7661/alice/priv/-/media/main/big.bin?token=")
424 );
425 let token = private
426 .split_once("?token=")
427 .map(|(_, t)| t)
428 .unwrap_or_default();
429 assert!(verify_raw(&fx.state.key, token, &fx.private.id, now_unix()));
430 }
431
432 /// A branch named `media` must not shadow the media route. The raw route serves its files,
433 /// and the URL from `media_url` still serves the linked LFS object.
434 #[tokio::test]
435 async fn branch_named_media_does_not_shadow_the_media_route() {
436 let fx = fixture().await;
437 let work = fx._tmp.path().join("work-pub");
438 let bare = forge_git::repo::repo_path(&fx.state.config.data_dir, &fx.public.id)
439 .expect("repo path");
440 git(&work, &["checkout", "-q", "-b", "media"]);
441 std::fs::write(work.join("README.md"), "# On a media branch\n").expect("write README");
442 git(&work, &["add", "-A"]);
443 git(&work, &["commit", "-q", "-m", "media branch"]);
444 git(
445 &work,
446 &["push", "-q", bare.to_str().expect("utf8"), "media"],
447 );
448
449 let readme = get(&fx.router, "GET", "/alice/pub/media/README.md").await;
450 assert_eq!(readme.status(), StatusCode::OK);
451 assert_eq!(body_bytes(readme).await, b"# On a media branch\n".to_vec());
452
453 // `-` is never a ref, so a raw path under it is not resolved.
454 let dash = get(&fx.router, "GET", "/alice/pub/-/README.md").await;
455 assert_eq!(dash.status(), StatusCode::NOT_FOUND);
456
457 let url = media_url(&fx.state, &fx.public, "main", "big.bin");
458 let path = url
459 .strip_prefix("http://127.0.0.1:7661")
460 .expect("configured base");
461 let media = get(&fx.router, "GET", path).await;
462 assert_eq!(media.status(), StatusCode::OK);
463 assert_eq!(body_bytes(media).await, fx.object);
464 }
465}