forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
17.4 KiB
1//! Raw-origin route `/{owner}/{repo}/-/media/{ref}/{path...}`. The literal `-` keeps it apart
2//! from raw paths, since no ref name may start with `-`. It reads the Git LFS pointer at
3//! `ref:path` and streams the object it names, when the repo links that object. Access follows
4//! the raw route in `raw.rs`: public repos are open, and private repos need a signed `?token=`.
5
6use forge_core::raw_token::verify_raw;
7use forge_core::{Repo, Visibility};
8use futures_util::TryStreamExt;
9use http_body::Frame;
10use http_body_util::StreamBody;
11use serde::Deserialize;
12use tokio_util::io::ReaderStream;
13use topcoat::context::{Cx, app_context};
14use topcoat::router::request::method;
15use topcoat::router::response::Response;
16use topcoat::router::{
17 Body, Method, RouteFuture, StatusCode, parse_query_params, path_param_segment,
18 path_param_segments,
19};
20
21use crate::lfs::{has_object, object_path};
22use crate::lfs_pointer;
23use crate::raw::{
24 CACHE_NONE, CACHE_PRIVATE, CACHE_PUBLIC, LINK_TTL_SECS, Lookup, Meta, OCTET, encode_path,
25 finish, lookup, now_unix, plain,
26};
27use crate::state::AppState;
28
29#[derive(Deserialize)]
30struct TokenQuery {
31 token: Option<String>,
32}
33
34/// Absolute URL of the media route for `path` at `rev` on the user-content origin. Private repos
35/// get a signed `?token=`, the same as raw URLs. The route is `/{owner}/{repo}/-/media/...`.
36pub fn media_url(state: &AppState, repo: &Repo, rev: &str, path: &str) -> String {
37 let base = state.config.user_content_base_url.trim_end_matches('/');
38 let mut url = format!(
39 "{base}/{}/{}/-/media/{}/{}",
40 encode_path(&repo.owner_name),
41 encode_path(&repo.name),
42 encode_path(rev),
43 encode_path(path)
44 );
45 if matches!(repo.visibility, Visibility::Private) {
46 let expires = now_unix().saturating_add(LINK_TTL_SECS);
47 url.push_str("?token=");
48 url.push_str(&forge_core::raw_token::sign_raw(
49 &state.key, &repo.id, expires,
50 ));
51 }
52 url
53}
54
55pub(crate) fn media_file(cx: &Cx, _body: Body) -> RouteFuture<'_> {
56 Box::pin(async move { Ok(serve(cx).await) })
57}
58
59/// Serves one LFS object. Every failure is turned into a response here, so the route never errors.
60async fn serve(cx: &Cx) -> Response {
61 let state = app_context::<AppState>(cx).clone();
62 let owner = path_param_segment(cx, "owner").to_owned();
63 let name = path_param_segment(cx, "repo").to_owned();
64 let rest = path_param_segments(cx, "rest")
65 .collect::<Vec<&str>>()
66 .join("/");
67 let is_head = method(cx) == Method::HEAD;
68 let token = parse_query_params::<TokenQuery>(cx)
69 .ok()
70 .and_then(|query| query.token);
71
72 let repo = match state.store.repo_by_owner_and_name(&owner, &name).await {
73 Ok(Some(repo)) => repo,
74 // A missing repo and a private repo without a valid token look the same.
75 Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
76 Err(e) => {
77 tracing::error!(error = %e, "media: repo lookup failed");
78 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
79 }
80 };
81 let cache = match repo.visibility {
82 Visibility::Public => CACHE_PUBLIC,
83 Visibility::Private => {
84 let valid = token
85 .as_deref()
86 .is_some_and(|t| verify_raw(&state.key, t, &repo.id, now_unix()));
87 if !valid {
88 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
89 }
90 CACHE_PRIVATE
91 }
92 };
93
94 let repo_dir = match forge_git::repo::repo_path(&state.git.data_dir, &repo.id) {
95 Ok(path) => path,
96 Err(e) => {
97 tracing::error!(error = %e, "media: repo path failed");
98 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
99 }
100 };
101 let looked = tokio::task::spawn_blocking(move || lookup(&repo_dir, &rest, None)).await;
102 // Pointers are a few hundred bytes, so they always come back inline. Anything else is not
103 // a pointer, and a symlink is never one.
104 let data = match looked {
105 Ok(Ok(Lookup::Inline {
106 symlink: false,
107 data,
108 ..
109 })) => data,
110 Ok(Ok(_)) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
111 Ok(Err(e)) => {
112 tracing::error!(error = %e, "media: lookup failed");
113 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
114 }
115 Err(e) => {
116 tracing::error!(error = %e, "media: lookup task failed");
117 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
118 }
119 };
120
121 let Some(pointer) = lfs_pointer::parse(&data) else {
122 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
123 };
124 // The repo must link the object, even when the file is on disk: objects are shared by every repo.
125 let size = match has_object(&state, &repo.id, &pointer.oid).await {
126 Ok(Some(size)) => size,
127 Ok(None) => return plain(StatusCode::NOT_FOUND, CACHE_NONE, None),
128 Err(e) => {
129 tracing::error!(error = %e, "media: object link lookup failed");
130 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
131 }
132 };
133
134 let file = match tokio::fs::File::open(object_path(&state.config.data_dir, &pointer.oid)).await
135 {
136 Ok(file) => file,
137 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
138 return plain(StatusCode::NOT_FOUND, CACHE_NONE, None);
139 }
140 Err(e) => {
141 tracing::error!(error = %e, "media: failed to open object");
142 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
143 }
144 };
145 // The Content-Length comes from the link, so a file of another size would break the response.
146 let on_disk = match file.metadata().await {
147 Ok(meta) => meta.len(),
148 Err(e) => {
149 tracing::error!(error = %e, "media: object metadata failed");
150 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
151 }
152 };
153 if on_disk != size {
154 tracing::error!(oid = %pointer.oid, on_disk, size, "media: object size differs from its link");
155 return plain(StatusCode::INTERNAL_SERVER_ERROR, CACHE_NONE, None);
156 }
157
158 let meta = Meta {
159 content_type: OCTET,
160 attachment: true,
161 len: size,
162 };
163 if is_head {
164 return finish(
165 StatusCode::OK,
166 Body::empty(),
167 &pointer.oid,
168 cache,
169 Some(meta),
170 );
171 }
172 let chunks = ReaderStream::new(file).map_ok(Frame::data);
173 let body = Body::new(StreamBody::new(chunks));
174 finish(StatusCode::OK, body, &pointer.oid, cache, Some(meta))
175}
176
177#[cfg(test)]
178mod tests {
179 use super::*;
180
181 use forge_core::raw_token::sign_raw;
182 use sha2::{Digest, Sha256};
183 use topcoat::router::Router;
184 use topcoat::router::request::Request;
185
186 use crate::raw::router;
187
188 const POINTER_HEADER: &str = "version https://git-lfs.github.com/spec/v1";
189
190 struct Fixture {
191 _tmp: tempfile::TempDir,
192 state: AppState,
193 router: Router,
194 public: Repo,
195 private: Repo,
196 object: Vec<u8>,
197 oid: String,
198 }
199
200 fn sha256_hex(data: &[u8]) -> String {
201 hex::encode(Sha256::digest(data))
202 }
203
204 fn pointer_text(oid: &str, size: usize) -> Vec<u8> {
205 format!("{POINTER_HEADER}\noid sha256:{oid}\nsize {size}\n").into_bytes()
206 }
207
208 fn git(dir: &std::path::Path, args: &[&str]) {
209 let out = std::process::Command::new("git")
210 .arg("-C")
211 .arg(dir)
212 .args(["-c", "commit.gpgsign=false"])
213 .args(args)
214 .env("GIT_AUTHOR_NAME", "test")
215 .env("GIT_AUTHOR_EMAIL", "test@example.com")
216 .env("GIT_COMMITTER_NAME", "test")
217 .env("GIT_COMMITTER_EMAIL", "test@example.com")
218 .output()
219 .expect("git runs");
220 assert!(
221 out.status.success(),
222 "git {args:?} failed: {}",
223 String::from_utf8_lossy(&out.stderr)
224 );
225 }
226
227 /// Commits `files` on `main` in a work tree and pushes it to `bare`.
228 fn populate(work: &std::path::Path, bare: &std::path::Path, files: &[(&str, Vec<u8>)]) {
229 std::fs::create_dir_all(work).expect("mkdir work");
230 git(work, &["init", "-q", "-b", "main"]);
231 for (rel, contents) in files {
232 std::fs::write(work.join(rel), contents).expect("write file");
233 }
234 git(work, &["add", "-A"]);
235 git(work, &["commit", "-q", "-m", "init"]);
236 git(work, &["push", "-q", bare.to_str().expect("utf8"), "main"]);
237 }
238
239 async fn fixture() -> Fixture {
240 let tmp = tempfile::tempdir().expect("tempdir");
241 let data_dir = tmp.path().join("data");
242 std::fs::create_dir_all(&data_dir).expect("data dir");
243 let data_dir = data_dir.canonicalize().expect("canonical data dir");
244 let mut config = crate::test_support::test_config(&data_dir);
245 config.user_content_base_url = "http://127.0.0.1:7661".into();
246 let state = AppState {
247 config: std::sync::Arc::new(config),
248 ..crate::test_support::test_state(&data_dir).await
249 };
250 let store = state.store.clone();
251 let owner = store
252 .create_user("alice", None, "unused", false)
253 .await
254 .expect("user");
255 let public = store
256 .create_repo(&owner.id, "pub", "", forge_core::Visibility::Public, "main")
257 .await
258 .expect("public repo");
259 let private = store
260 .create_repo(
261 &owner.id,
262 "priv",
263 "",
264 forge_core::Visibility::Private,
265 "main",
266 )
267 .await
268 .expect("private repo");
269 for repo in [&public, &private] {
270 let bare = forge_git::repo::repo_path(&data_dir, &repo.id).expect("repo path");
271 forge_git::repo::init_bare(&state.git, &bare, "main")
272 .await
273 .expect("init bare");
274 }
275
276 let object = b"lfs object bytes\n".to_vec();
277 let oid = sha256_hex(&object);
278 let missing_oid = sha256_hex(b"never stored");
279 let stored_pointer = pointer_text(&oid, object.len());
280 let missing_pointer = pointer_text(&missing_oid, 12);
281 let files: Vec<(&str, Vec<u8>)> = vec![
282 ("big.bin", stored_pointer.clone()),
283 ("missing.bin", missing_pointer),
284 ("plain.txt", b"just text\n".to_vec()),
285 ];
286 let pub_bare = forge_git::repo::repo_path(&data_dir, &public.id).expect("path");
287 populate(&tmp.path().join("work-pub"), &pub_bare, &files);
288 let priv_bare = forge_git::repo::repo_path(&data_dir, &private.id).expect("path");
289 populate(
290 &tmp.path().join("work-priv"),
291 &priv_bare,
292 &[("big.bin", stored_pointer)],
293 );
294
295 // The object is linked to both repos and written once to the shared store.
296 for repo in [&public, &private] {
297 store
298 .insert_lfs_object(&repo.id, &oid, object.len() as u64)
299 .await
300 .expect("link object");
301 }
302 let path = object_path(&data_dir, &oid);
303 std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir objects");
304 std::fs::write(&path, &object).expect("write object");
305
306 let router = router(state.clone());
307 Fixture {
308 _tmp: tmp,
309 state,
310 router,
311 public,
312 private,
313 object,
314 oid,
315 }
316 }
317
318 async fn get(router: &Router, method: &str, uri: &str) -> Response {
319 let req = Request::builder()
320 .method(method)
321 .uri(uri)
322 .body(Body::empty())
323 .expect("request");
324 router.handle(req).await
325 }
326
327 fn header_str(res: &Response, name: &str) -> String {
328 res.headers()
329 .get(name)
330 .and_then(|v| v.to_str().ok())
331 .unwrap_or_default()
332 .to_owned()
333 }
334
335 async fn body_bytes(res: Response) -> Vec<u8> {
336 topcoat::router::to_bytes(res.into_body(), usize::MAX)
337 .await
338 .expect("body")
339 .to_vec()
340 }
341
342 #[tokio::test]
343 async fn media_serves_the_linked_object_as_a_download() {
344 let fx = fixture().await;
345 let res = get(&fx.router, "GET", "/alice/pub/-/media/main/big.bin").await;
346 assert_eq!(res.status(), StatusCode::OK);
347 assert_eq!(header_str(&res, "content-disposition"), "attachment");
348 assert_eq!(header_str(&res, "x-content-type-options"), "nosniff");
349 assert_eq!(header_str(&res, "content-type"), OCTET);
350 assert_eq!(
351 header_str(&res, "content-length"),
352 fx.object.len().to_string()
353 );
354 assert_eq!(header_str(&res, "etag"), format!("\"{}\"", fx.oid));
355 assert_eq!(body_bytes(res).await, fx.object);
356 }
357
358 #[tokio::test]
359 async fn head_has_headers_and_no_body() {
360 let fx = fixture().await;
361 let res = get(&fx.router, "HEAD", "/alice/pub/-/media/main/big.bin").await;
362 assert_eq!(res.status(), StatusCode::OK);
363 assert_eq!(
364 header_str(&res, "content-length"),
365 fx.object.len().to_string()
366 );
367 assert!(body_bytes(res).await.is_empty());
368 }
369
370 #[tokio::test]
371 async fn missing_object_is_404() {
372 let fx = fixture().await;
373 // The pointer exists, but the repo does not link its object.
374 let res = get(&fx.router, "GET", "/alice/pub/-/media/main/missing.bin").await;
375 assert_eq!(res.status(), StatusCode::NOT_FOUND);
376 }
377
378 #[tokio::test]
379 async fn non_pointer_and_missing_path_are_404() {
380 let fx = fixture().await;
381 let text = get(&fx.router, "GET", "/alice/pub/-/media/main/plain.txt").await;
382 assert_eq!(text.status(), StatusCode::NOT_FOUND);
383 let absent = get(&fx.router, "GET", "/alice/pub/-/media/main/nope.bin").await;
384 assert_eq!(absent.status(), StatusCode::NOT_FOUND);
385 let unknown_repo = get(&fx.router, "GET", "/alice/nope/-/media/main/big.bin").await;
386 assert_eq!(unknown_repo.status(), StatusCode::NOT_FOUND);
387 }
388
389 #[tokio::test]
390 async fn private_repo_needs_a_valid_token() {
391 let fx = fixture().await;
392 let path = "/alice/priv/-/media/main/big.bin";
393 let none = get(&fx.router, "GET", path).await;
394 assert_eq!(none.status(), StatusCode::NOT_FOUND);
395
396 let now = now_unix();
397 let valid = sign_raw(&fx.state.key, &fx.private.id, now + 60);
398 let ok = get(&fx.router, "GET", &format!("{path}?token={valid}")).await;
399 assert_eq!(ok.status(), StatusCode::OK);
400 assert_eq!(body_bytes(ok).await, fx.object);
401
402 let expired = sign_raw(&fx.state.key, &fx.private.id, now - 1);
403 let res = get(&fx.router, "GET", &format!("{path}?token={expired}")).await;
404 assert_eq!(res.status(), StatusCode::NOT_FOUND);
405
406 // A token signed for the public repo does not open the private one.
407 let other = sign_raw(&fx.state.key, &fx.public.id, now + 60);
408 let res = get(&fx.router, "GET", &format!("{path}?token={other}")).await;
409 assert_eq!(res.status(), StatusCode::NOT_FOUND);
410 }
411
412 #[tokio::test]
413 async fn media_url_encodes_and_signs_private_repos_only() {
414 let fx = fixture().await;
415 let public = media_url(&fx.state, &fx.public, "main", "a b/big.bin");
416 assert_eq!(
417 public,
418 "http://127.0.0.1:7661/alice/pub/-/media/main/a%20b/big.bin"
419 );
420 let private = media_url(&fx.state, &fx.private, "main", "big.bin");
421 assert!(
422 private.starts_with("http://127.0.0.1:7661/alice/priv/-/media/main/big.bin?token=")
423 );
424 let token = private
425 .split_once("?token=")
426 .map(|(_, t)| t)
427 .unwrap_or_default();
428 assert!(verify_raw(&fx.state.key, token, &fx.private.id, now_unix()));
429 }
430
431 /// A branch named `media` must not shadow the media route. The raw route serves its files,
432 /// and the URL from `media_url` still serves the linked LFS object.
433 #[tokio::test]
434 async fn branch_named_media_does_not_shadow_the_media_route() {
435 let fx = fixture().await;
436 let work = fx._tmp.path().join("work-pub");
437 let bare = forge_git::repo::repo_path(&fx.state.config.data_dir, &fx.public.id)
438 .expect("repo path");
439 git(&work, &["checkout", "-q", "-b", "media"]);
440 std::fs::write(work.join("README.md"), "# On a media branch\n").expect("write README");
441 git(&work, &["add", "-A"]);
442 git(&work, &["commit", "-q", "-m", "media branch"]);
443 git(
444 &work,
445 &["push", "-q", bare.to_str().expect("utf8"), "media"],
446 );
447
448 let readme = get(&fx.router, "GET", "/alice/pub/media/README.md").await;
449 assert_eq!(readme.status(), StatusCode::OK);
450 assert_eq!(body_bytes(readme).await, b"# On a media branch\n".to_vec());
451
452 // `-` is never a ref, so a raw path under it is not resolved.
453 let dash = get(&fx.router, "GET", "/alice/pub/-/README.md").await;
454 assert_eq!(dash.status(), StatusCode::NOT_FOUND);
455
456 let url = media_url(&fx.state, &fx.public, "main", "big.bin");
457 let path = url
458 .strip_prefix("http://127.0.0.1:7661")
459 .expect("configured base");
460 let media = get(&fx.router, "GET", path).await;
461 assert_eq!(media.status(), StatusCode::OK);
462 assert_eq!(body_bytes(media).await, fx.object);
463 }
464}