forge
ExploreLog in

Tahinli/forge

public
0 watching
★ 0
19.5 KiB
1//! Security advisories: the OSV records read from a dump, the version ranges they name, and the
2//! lockfile parsers that turn a repo's `Cargo.lock`, `package-lock.json` and `go.sum` into
3//! dependencies. Storage is in `store/advisories.rs`; the download and the scan are in forge-web.
4//!
5//! Only the OSV fields the forge needs are read: `id`, `summary`, `aliases`, `severity`,
6//! `affected[].package`, `affected[].ranges` (SEMVER and ECOSYSTEM events) and
7//! `affected[].versions`. Git ranges are skipped. Versions are compared as semver: a leading `v`
8//! is dropped first, so Go versions and Go pseudo-versions compare as semver too.
9
10use std::collections::{BTreeMap, BTreeSet};
11use std::io::{Read, Seek};
12
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15
16use crate::error::{Error, Result};
17
18/// The OSV ecosystem names the forge knows how to scan. `[advisories] ecosystems` picks a subset.
19pub const KNOWN_ECOSYSTEMS: [&str; 3] = ["crates.io", "npm", "Go"];
20
21/// Largest single OSV record read from a dump. Real records are a few KiB.
22const MAX_RECORD_BYTES: u64 = 4 * 1024 * 1024;
23
24/// One event of a range, in the order it appears in the record.
25#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(rename_all = "snake_case")]
27pub enum Event {
28 Introduced(String),
29 Fixed(String),
30 LastAffected(String),
31 Limit(String),
32}
33
34/// One SEMVER or ECOSYSTEM range of an affected package.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
36pub struct Range {
37 pub events: Vec<Event>,
38}
39
40/// An advisory as it applies to one package in one ecosystem.
41#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
42pub struct Advisory {
43 pub id: String,
44 pub ecosystem: String,
45 pub package: String,
46 pub summary: String,
47 pub severity: String,
48 pub modified: String,
49 #[serde(default)]
50 pub aliases: Vec<String>,
51 pub ranges: Vec<Range>,
52 pub versions: Vec<String>,
53}
54
55/// A package version read from a lockfile.
56#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
57pub struct Dependency {
58 pub ecosystem: String,
59 pub package: String,
60 pub version: String,
61 pub manifest_path: String,
62}
63
64#[derive(Deserialize)]
65struct Record {
66 id: String,
67 #[serde(default)]
68 summary: String,
69 #[serde(default)]
70 aliases: Vec<String>,
71 #[serde(default)]
72 modified: String,
73 #[serde(default)]
74 severity: Vec<SeverityEntry>,
75 #[serde(default)]
76 database_specific: Value,
77 #[serde(default)]
78 affected: Vec<Affected>,
79}
80
81#[derive(Deserialize)]
82struct SeverityEntry {
83 #[serde(default)]
84 score: String,
85}
86
87#[derive(Deserialize)]
88struct Affected {
89 #[serde(default)]
90 package: Option<PackageRef>,
91 #[serde(default)]
92 ranges: Vec<RawRange>,
93 #[serde(default)]
94 versions: Vec<String>,
95}
96
97#[derive(Deserialize)]
98struct PackageRef {
99 #[serde(default)]
100 ecosystem: String,
101 #[serde(default)]
102 name: String,
103}
104
105#[derive(Deserialize)]
106struct RawRange {
107 #[serde(rename = "type", default)]
108 kind: String,
109 #[serde(default)]
110 events: Vec<RawEvent>,
111}
112
113#[derive(Deserialize)]
114struct RawEvent {
115 introduced: Option<String>,
116 fixed: Option<String>,
117 last_affected: Option<String>,
118 limit: Option<String>,
119}
120
121/// Parses one OSV record into one [`Advisory`] per affected package of `ecosystem`. Several
122/// `affected` entries for the same package are merged into one advisory.
123pub fn parse_record(json: &[u8], ecosystem: &str) -> Result<Vec<Advisory>> {
124 let record: Record = serde_json::from_slice(json)
125 .map_err(|e| Error::Invalid(format!("advisory record is not valid OSV: {e}")))?;
126 if record.id.is_empty() {
127 return Err(Error::Invalid("advisory record has no id".into()));
128 }
129 let severity = record
130 .database_specific
131 .get("severity")
132 .and_then(Value::as_str)
133 .map(str::to_owned)
134 .or_else(|| record.severity.first().map(|s| s.score.clone()))
135 .unwrap_or_default();
136 let mut merged: BTreeMap<String, Advisory> = BTreeMap::new();
137 for affected in record.affected {
138 let Some(package) = affected.package else {
139 continue;
140 };
141 if package.ecosystem != ecosystem || package.name.is_empty() {
142 continue;
143 }
144 let entry = merged
145 .entry(package.name.clone())
146 .or_insert_with(|| Advisory {
147 id: record.id.clone(),
148 ecosystem: ecosystem.to_owned(),
149 package: package.name.clone(),
150 summary: record.summary.clone(),
151 severity: severity.clone(),
152 modified: record.modified.clone(),
153 aliases: record.aliases.clone(),
154 ranges: Vec::new(),
155 versions: Vec::new(),
156 });
157 for raw in affected.ranges {
158 if !raw.kind.eq_ignore_ascii_case("SEMVER")
159 && !raw.kind.eq_ignore_ascii_case("ECOSYSTEM")
160 {
161 continue;
162 }
163 let events = raw
164 .events
165 .into_iter()
166 .filter_map(|e| {
167 if let Some(v) = e.introduced {
168 Some(Event::Introduced(v))
169 } else if let Some(v) = e.fixed {
170 Some(Event::Fixed(v))
171 } else if let Some(v) = e.last_affected {
172 Some(Event::LastAffected(v))
173 } else {
174 e.limit.map(Event::Limit)
175 }
176 })
177 .collect();
178 entry.ranges.push(Range { events });
179 }
180 entry.versions.extend(affected.versions);
181 }
182 Ok(merged.into_values().collect())
183}
184
185/// Reads every `.json` record of an OSV zip dump for `ecosystem`. A record that does not parse,
186/// or is too large, is skipped and counted in the second value.
187pub fn read_dump<R: Read + Seek>(reader: R, ecosystem: &str) -> Result<(Vec<Advisory>, usize)> {
188 let mut zip = zip::ZipArchive::new(reader)
189 .map_err(|e| Error::Invalid(format!("advisory dump is not a zip file: {e}")))?;
190 let mut advisories = Vec::new();
191 let mut skipped = 0;
192 for index in 0..zip.len() {
193 let mut file = zip
194 .by_index(index)
195 .map_err(|e| Error::Invalid(format!("advisory dump entry unreadable: {e}")))?;
196 if file.is_dir() || !file.name().is_ok_and(|name| name.ends_with(".json")) {
197 continue;
198 }
199 let mut buf = Vec::new();
200 file.by_ref()
201 .take(MAX_RECORD_BYTES + 1)
202 .read_to_end(&mut buf)?;
203 if buf.len() as u64 > MAX_RECORD_BYTES {
204 skipped += 1;
205 continue;
206 }
207 match parse_record(&buf, ecosystem) {
208 Ok(found) => advisories.extend(found),
209 Err(_) => skipped += 1,
210 }
211 }
212 Ok((advisories, skipped))
213}
214
215/// Parses a version as semver, after dropping a leading `v` (Go style).
216pub fn parse_version(raw: &str) -> Option<semver::Version> {
217 semver::Version::parse(raw.trim().trim_start_matches('v')).ok()
218}
219
220/// Whether `advisory` covers `version` of its package. A version listed in `versions` matches by
221/// text. Otherwise the version must fall inside one of the ranges.
222pub fn affects(advisory: &Advisory, version: &str) -> bool {
223 let wanted = version.trim().trim_start_matches('v');
224 if advisory
225 .versions
226 .iter()
227 .any(|v| v.trim().trim_start_matches('v') == wanted)
228 {
229 return true;
230 }
231 let Some(parsed) = parse_version(version) else {
232 return false;
233 };
234 advisory
235 .ranges
236 .iter()
237 .any(|range| range_affects(range, &parsed))
238}
239
240/// Walks the events of one range. An interval runs from an `introduced` event to the next
241/// `fixed` (exclusive) or `last_affected` (inclusive). `introduced` of `0` starts at the beginning.
242/// An interval still open at the end runs forever.
243fn range_affects(range: &Range, version: &semver::Version) -> bool {
244 // `Some(start)` while an interval is open; `start` is `None` when it began at `0`.
245 let mut open: Option<Option<semver::Version>> = None;
246 for event in &range.events {
247 match event {
248 Event::Introduced(raw) => {
249 open = Some(if raw == "0" {
250 None
251 } else {
252 // An unreadable start is treated as "from the beginning": more alerts, never fewer.
253 parse_version(raw)
254 });
255 }
256 Event::Fixed(raw) => {
257 if let Some(start) = open.take()
258 && starts_before(&start, version)
259 && parse_version(raw).is_some_and(|fixed| *version < fixed)
260 {
261 return true;
262 }
263 }
264 Event::LastAffected(raw) => {
265 if let Some(start) = open.take()
266 && starts_before(&start, version)
267 && parse_version(raw).is_some_and(|last| *version <= last)
268 {
269 return true;
270 }
271 }
272 Event::Limit(_) => {}
273 }
274 }
275 open.is_some_and(|start| starts_before(&start, version))
276}
277
278fn starts_before(start: &Option<semver::Version>, version: &semver::Version) -> bool {
279 start.as_ref().is_none_or(|s| version >= s)
280}
281
282/// Reads `Cargo.lock`. Only registry packages count: path and git packages have no `source`
283/// from crates.io and are skipped.
284pub fn parse_cargo_lock(text: &str, manifest_path: &str) -> Result<Vec<Dependency>> {
285 #[derive(Deserialize)]
286 struct Lock {
287 #[serde(default)]
288 package: Vec<Package>,
289 }
290 #[derive(Deserialize)]
291 struct Package {
292 name: String,
293 version: String,
294 #[serde(default)]
295 source: Option<String>,
296 }
297 let lock: Lock = toml::from_str(text)
298 .map_err(|e| Error::Invalid(format!("Cargo.lock is not valid TOML: {e}")))?;
299 Ok(lock
300 .package
301 .into_iter()
302 .filter(|p| {
303 p.source
304 .as_deref()
305 .is_some_and(|s| s.starts_with("registry+"))
306 })
307 .map(|p| dependency("crates.io", p.name, p.version, manifest_path))
308 .collect())
309}
310
311/// Reads `package-lock.json` in the v2 and v3 `packages` layout. Entries without a version, the
312/// root entry and linked workspace packages are skipped.
313pub fn parse_package_lock(text: &str, manifest_path: &str) -> Result<Vec<Dependency>> {
314 let doc: Value = serde_json::from_str(text)
315 .map_err(|e| Error::Invalid(format!("package-lock.json is not valid JSON: {e}")))?;
316 let Some(packages) = doc.get("packages").and_then(Value::as_object) else {
317 return Ok(Vec::new());
318 };
319 let mut out = Vec::new();
320 for (key, entry) in packages {
321 if key.is_empty() || entry.get("link").and_then(Value::as_bool) == Some(true) {
322 continue;
323 }
324 let Some(version) = entry.get("version").and_then(Value::as_str) else {
325 continue;
326 };
327 // `node_modules/a/node_modules/@s/b` names `@s/b`: the part after the last `node_modules/`.
328 let name = match key.rsplit_once("node_modules/") {
329 Some((_, name)) => name,
330 None => continue,
331 };
332 out.push(dependency(
333 "npm",
334 name.to_owned(),
335 version.to_owned(),
336 manifest_path,
337 ));
338 }
339 Ok(out)
340}
341
342/// Reads `go.sum`. Each module version is listed once, with or without its `/go.mod` hash line;
343/// the `/go.mod` lines alone are not dependencies, so they are dropped.
344pub fn parse_go_sum(text: &str, manifest_path: &str) -> Vec<Dependency> {
345 let mut seen = BTreeSet::new();
346 for line in text.lines() {
347 let mut fields = line.split_whitespace();
348 let (Some(module), Some(version)) = (fields.next(), fields.next()) else {
349 continue;
350 };
351 if version.ends_with("/go.mod") {
352 continue;
353 }
354 seen.insert((module.to_owned(), version.to_owned()));
355 }
356 seen.into_iter()
357 .map(|(module, version)| dependency("Go", module, version, manifest_path))
358 .collect()
359}
360
361fn dependency(
362 ecosystem: &str,
363 package: String,
364 version: String,
365 manifest_path: &str,
366) -> Dependency {
367 Dependency {
368 ecosystem: ecosystem.to_owned(),
369 package,
370 version,
371 manifest_path: manifest_path.to_owned(),
372 }
373}
374
375#[cfg(test)]
376mod tests {
377 use super::*;
378
379 fn adv(ranges: Vec<Range>, versions: Vec<&str>) -> Advisory {
380 Advisory {
381 id: "GHSA-test".into(),
382 ecosystem: "crates.io".into(),
383 package: "demo".into(),
384 summary: String::new(),
385 severity: String::new(),
386 modified: String::new(),
387 aliases: Vec::new(),
388 ranges,
389 versions: versions.into_iter().map(str::to_owned).collect(),
390 }
391 }
392
393 fn range(events: Vec<Event>) -> Range {
394 Range { events }
395 }
396
397 #[test]
398 fn introduced_and_fixed_is_half_open() {
399 let a = adv(
400 vec![range(vec![
401 Event::Introduced("1.0.0".into()),
402 Event::Fixed("1.2.0".into()),
403 ])],
404 vec![],
405 );
406 assert!(!affects(&a, "0.9.9"));
407 assert!(affects(&a, "1.0.0"));
408 assert!(affects(&a, "1.1.5"));
409 assert!(!affects(&a, "1.2.0"));
410 assert!(!affects(&a, "2.0.0"));
411 }
412
413 #[test]
414 fn introduced_zero_with_last_affected_is_inclusive() {
415 let a = adv(
416 vec![range(vec![
417 Event::Introduced("0".into()),
418 Event::LastAffected("0.4.7".into()),
419 ])],
420 vec![],
421 );
422 assert!(affects(&a, "0.0.1"));
423 assert!(affects(&a, "0.4.7"));
424 assert!(!affects(&a, "0.4.8"));
425 }
426
427 #[test]
428 fn open_interval_runs_forever_and_later_intervals_work() {
429 let a = adv(
430 vec![range(vec![
431 Event::Introduced("1.0.0".into()),
432 Event::Fixed("1.1.0".into()),
433 Event::Introduced("2.0.0".into()),
434 ])],
435 vec![],
436 );
437 assert!(affects(&a, "1.0.5"));
438 assert!(!affects(&a, "1.5.0"));
439 assert!(affects(&a, "9.9.9"));
440 }
441
442 #[test]
443 fn go_v_prefix_and_pseudo_versions_compare_as_semver() {
444 let a = adv(
445 vec![range(vec![
446 Event::Introduced("0".into()),
447 Event::Fixed("1.9.1".into()),
448 ])],
449 vec![],
450 );
451 assert!(affects(&a, "v1.9.0"));
452 assert!(!affects(&a, "v1.9.1"));
453 assert!(affects(&a, "v0.0.0-20210101000000-abcdef123456"));
454 }
455
456 #[test]
457 fn listed_versions_match_by_text() {
458 let a = adv(vec![], vec!["1.0.3", "v2.0.0"]);
459 assert!(affects(&a, "1.0.3"));
460 assert!(affects(&a, "2.0.0"));
461 assert!(!affects(&a, "1.0.4"));
462 }
463
464 #[test]
465 fn unparsable_version_matches_nothing_by_range() {
466 let a = adv(vec![range(vec![Event::Introduced("0".into())])], vec![]);
467 assert!(!affects(&a, "not-a-version"));
468 }
469
470 #[test]
471 fn parses_an_osv_record_and_merges_entries_for_one_package() {
472 let json = br#"{
473 "id": "GHSA-xxxx",
474 "summary": "bad things",
475 "modified": "2026-01-01T00:00:00Z",
476 "database_specific": {"severity": "HIGH"},
477 "affected": [
478 {"package": {"ecosystem": "crates.io", "name": "demo"},
479 "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.0.1"}]}]},
480 {"package": {"ecosystem": "crates.io", "name": "demo"},
481 "ranges": [{"type": "GIT", "events": [{"introduced": "0"}]},
482 {"type": "ECOSYSTEM", "events": [{"introduced": "3.0.0"}, {"last_affected": "3.0.5"}]}],
483 "versions": ["0.9.0"]},
484 {"package": {"ecosystem": "npm", "name": "other"},
485 "ranges": []}
486 ]
487 }"#;
488 let found = parse_record(json, "crates.io").expect("record parses");
489 assert_eq!(found.len(), 1);
490 let a = &found[0];
491 assert_eq!(a.id, "GHSA-xxxx");
492 assert_eq!(a.severity, "HIGH");
493 assert_eq!(a.ranges.len(), 2, "the GIT range is skipped");
494 assert!(affects(a, "0.5.0"));
495 assert!(!affects(a, "1.0.1"));
496 assert!(affects(a, "3.0.2"));
497 assert!(affects(a, "0.9.0"));
498 }
499
500 #[test]
501 fn record_without_an_id_is_refused() {
502 assert!(parse_record(br#"{"affected": []}"#, "npm").is_err());
503 assert!(parse_record(b"not json", "npm").is_err());
504 }
505
506 #[test]
507 fn reads_a_zip_dump_and_skips_bad_records() {
508 use std::io::{Cursor, Write};
509 let mut buf = Cursor::new(Vec::new());
510 {
511 let mut zip = zip::ZipWriter::new(&mut buf);
512 let opts = zip::write::SimpleFileOptions::default();
513 zip.start_file("GHSA-a.json", opts).expect("start");
514 zip.write_all(
515 br#"{"id":"GHSA-a","affected":[{"package":{"ecosystem":"npm","name":"left"},
516 "ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"2.0.0"}]}]}]}"#,
517 )
518 .expect("write");
519 zip.start_file("broken.json", opts).expect("start");
520 zip.write_all(b"{ nope").expect("write");
521 zip.start_file("readme.txt", opts).expect("start");
522 zip.write_all(b"ignored").expect("write");
523 zip.finish().expect("finish");
524 }
525 buf.set_position(0);
526 let (found, skipped) = read_dump(buf, "npm").expect("dump reads");
527 assert_eq!(skipped, 1);
528 assert_eq!(found.len(), 1);
529 assert_eq!(found[0].package, "left");
530 }
531
532 #[test]
533 fn parses_cargo_lock_registry_packages_only() {
534 let text = r#"
535version = 4
536
537[[package]]
538name = "serde"
539version = "1.0.0"
540source = "registry+https://github.com/rust-lang/crates.io-index"
541checksum = "abc"
542
543[[package]]
544name = "local"
545version = "0.1.0"
546"#;
547 let deps = parse_cargo_lock(text, "Cargo.lock").expect("parses");
548 assert_eq!(deps.len(), 1);
549 assert_eq!(deps[0].ecosystem, "crates.io");
550 assert_eq!(deps[0].package, "serde");
551 assert_eq!(deps[0].version, "1.0.0");
552 assert_eq!(deps[0].manifest_path, "Cargo.lock");
553 }
554
555 #[test]
556 fn parses_package_lock_v3_packages() {
557 let text = r#"{
558 "lockfileVersion": 3,
559 "packages": {
560 "": {"name": "app", "version": "1.0.0"},
561 "node_modules/left-pad": {"version": "1.3.0"},
562 "node_modules/a/node_modules/@scope/inner": {"version": "2.0.1"},
563 "node_modules/linked": {"link": true},
564 "packages/ws": {"version": "0.0.1"}
565 }
566 }"#;
567 let mut deps = parse_package_lock(text, "web/package-lock.json").expect("parses");
568 deps.sort();
569 let names: Vec<(&str, &str)> = deps
570 .iter()
571 .map(|d| (d.package.as_str(), d.version.as_str()))
572 .collect();
573 assert_eq!(
574 names,
575 vec![("@scope/inner", "2.0.1"), ("left-pad", "1.3.0")]
576 );
577 assert!(deps.iter().all(|d| d.ecosystem == "npm"));
578 }
579
580 #[test]
581 fn parses_go_sum_and_drops_go_mod_lines() {
582 let text = "\
583github.com/a/b v1.2.3 h1:abc=
584github.com/a/b v1.2.3/go.mod h1:def=
585github.com/c/d v0.0.0-20210101000000-abcdef123456 h1:ghi=
586github.com/c/d v0.0.0-20210101000000-abcdef123456/go.mod h1:jkl=
587";
588 let deps = parse_go_sum(text, "go.sum");
589 assert_eq!(deps.len(), 2);
590 assert_eq!(deps[0].ecosystem, "Go");
591 assert_eq!(deps[0].package, "github.com/a/b");
592 assert_eq!(deps[0].version, "v1.2.3");
593 }
594}