diff --git a/backend/.env.dist.local b/backend/.env.dist.local index 3fdede3c01..3e9c73b85d 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -199,7 +199,7 @@ OSSPCKGS_GCP_CREDENTIALS_B64=e30= # maven/all.zip 404s). The allowlist check and DB storage normalize to lowercase # internally per ADR-0001 §OSV "Ecosystem normalization", so downstream stays lowercase. OSV_BULK_BASE_URL=https://osv-vulnerabilities.storage.googleapis.com -OSV_ECOSYSTEMS=npm,Maven,cargo,NuGet +OSV_ECOSYSTEMS=npm,Maven,cargo,NuGet,RubyGems,Go OSV_TMP_DIR=/tmp/osv OSV_BATCH_SIZE=500 OSV_DERIVE_BATCH_SIZE=1000 diff --git a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts index 1c114294f8..0fa0fc515f 100644 --- a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts +++ b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts @@ -111,6 +111,56 @@ describe('compareVersion — nuget (semver)', () => { }) }) +describe('compareVersion — go (semver)', () => { + it.each([ + ['v1.2.3', 'v1.2.4', -1], + ['v1.2.4', 'v1.2.3', 1], + ['v1.2.3', 'v1.2.3', 0], + ['v1.10.0', 'v1.9.0', 1], // numeric, not lex + ['v1.0.0-alpha', 'v1.0.0', -1], // prerelease < release + ['v0.0.0-20220314234659-1baeb1ce4c0b', 'v0.0.0-20220315000000-2caec2d5d1c1', -1], // pseudo-versions + ])('compareVersion("go", %s, %s) sign = %s', (a, b, expected) => { + expect(sign(compareVersion('go', a, b))).toBe(expected) + }) + + it('returns null for unparseable go versions', () => { + expect(compareVersion('go', 'not-a-version', 'v1.0.0')).toBeNull() + }) + + it('rejects titlecase "Go" — production storage is always lowercase', () => { + expect(compareVersion('Go', 'v1.0.0', 'v2.0.0')).toBeNull() + }) +}) + +describe('compareVersion — rubygems (Gem::Version-style)', () => { + it.each([ + ['1.0.0', '1.0.1', -1], + ['1.0.1', '1.0.0', 1], + ['1.0.0', '1.0.0', 0], + ['1.10.0', '1.9.0', 1], // numeric, not lex + ['1.0', '1.0.0', 0], // missing trailing segment pads as 0 + ['1.0.pre', '1.0', -1], // prerelease sorts below the corresponding release + ['1.0.0.rc1', '1.0.0.rc2', -1], + // rack CVE-2022-30123 boundary + ['2.2.3', '2.2.3.1', -1], + ['1.0-1', '1.0.pre.1', 0], // hyphen is a prerelease marker, same as ".pre." + ['1.0.a10', '1.0.a9', 1], // letter+digit runs split: a10 -> a.10, a9 -> a.9 + ['1.99999999999999999999999999999999', '1.99999999999999999999999999999998', 1], // beyond Number.MAX_SAFE_INTEGER — must not collapse to equal + ])('compareVersion("rubygems", %s, %s) sign = %s', (a, b, expected) => { + expect(sign(compareVersion('rubygems', a, b))).toBe(expected) + }) + + it('returns null for unparseable rubygems versions', () => { + expect(compareVersion('rubygems', '', '1.0.0')).toBeNull() + expect(compareVersion('rubygems', '---', '1.0.0')).toBeNull() + expect(compareVersion('rubygems', '1..2', '1.0.0')).toBeNull() + }) + + it('rejects titlecase "RubyGems" — production storage is always lowercase', () => { + expect(compareVersion('RubyGems', '1.0.0', '2.0.0')).toBeNull() + }) +}) + describe('compareVersion — unsupported ecosystems', () => { it('returns null for ecosystems we have no comparator for', () => { expect(compareVersion('PyPI', '1.0.0', '2.0.0')).toBeNull() diff --git a/services/apps/packages_worker/src/osv/schedule.ts b/services/apps/packages_worker/src/osv/schedule.ts index 0e61307551..9511964d15 100644 --- a/services/apps/packages_worker/src/osv/schedule.ts +++ b/services/apps/packages_worker/src/osv/schedule.ts @@ -11,7 +11,7 @@ const SCHEDULE_ID = 'osv-advisories-sync' // validate the env input against this list and refuse to register the // schedule on a mismatch — better a loud startup error than a silent miss. // Add new entries here when v1 expands beyond npm + Maven. -const VALID_ECOSYSTEMS = ['npm', 'Maven', 'cargo', 'NuGet'] as const +const VALID_ECOSYSTEMS = ['npm', 'Maven', 'cargo', 'NuGet', 'RubyGems', 'Go'] as const function getEcosystems(): string[] { const raw = process.env.OSV_ECOSYSTEMS @@ -35,8 +35,27 @@ function getEcosystems(): string[] { return deduped } +function scheduleAction() { + return { + type: 'startWorkflow' as const, + workflowType: osvSync, + taskQueue: 'osv-worker', + // Headroom for npm (~1 hour today) + Maven (~5 minutes) + derive + // (~5 minutes for 600-700k packages); 4 hours leaves space for the + // upsertOne N+1 deferred fix being slower than expected. + workflowExecutionTimeout: '4 hours', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 3, + }, + args: [{ ecosystems: getEcosystems() }] as [{ ecosystems: string[] }], + } +} + // Registers the daily OSV advisory sync schedule if it doesn't already exist. -// If the schedule already exists in Temporal, we log and leave it unchanged (no update). +// If the schedule already exists in Temporal, we reconcile its action so a +// changed OSV_ECOSYSTEMS env var reaches an already-running schedule on restart. // Cron is offset from npm-registry-ingest (`15 3 * * *`) so the two large daily ingest jobs don't fight for the same DB at the same minute. export async function scheduleOsvSync(): Promise { const { temporal } = svc @@ -55,25 +74,20 @@ export async function scheduleOsvSync(): Promise { overlap: ScheduleOverlapPolicy.SKIP, catchupWindow: '1 hour', }, - action: { - type: 'startWorkflow', - workflowType: osvSync, - taskQueue: 'osv-worker', - // Headroom for npm (~1 hour today) + Maven (~5 minutes) + derive - // (~5 minutes for 600-700k packages); 4 hours leaves space for the - // upsertOne N+1 deferred fix being slower than expected. - workflowExecutionTimeout: '4 hours', - retry: { - initialInterval: '30 seconds', - backoffCoefficient: 2, - maximumAttempts: 3, - }, - args: [{ ecosystems: getEcosystems() }], - }, + action: scheduleAction(), }) } catch (err) { if (err instanceof ScheduleAlreadyRunning) { - svc.log.info(`Schedule ${SCHEDULE_ID} already registered.`) + svc.log.info(`Schedule ${SCHEDULE_ID} already exists, reconciling action.`) + const handle = temporal.schedule.getHandle(SCHEDULE_ID) + await handle.update((prev) => ({ + ...prev, + policies: { + ...prev.policies, + overlap: ScheduleOverlapPolicy.SKIP, + }, + action: scheduleAction(), + })) } else { throw err } diff --git a/services/apps/packages_worker/src/osv/versionCompare.ts b/services/apps/packages_worker/src/osv/versionCompare.ts index 1fa47c9fd5..d9971574b1 100644 --- a/services/apps/packages_worker/src/osv/versionCompare.ts +++ b/services/apps/packages_worker/src/osv/versionCompare.ts @@ -126,7 +126,76 @@ function compareMaven(a: string, b: string): number | null { return 0 } -const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget']) +// Mirrors Gem::Version::VERSION_PATTERN / ANCHORED_VERSION_PATTERN — a hyphen +// only ever introduces a single trailing prerelease group. +const RUBYGEMS_VERSION_PATTERN = + /^\s*[0-9]+(\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?\s*$/ + +// Splits a version into alternating digit/letter runs, per Gem::Version#segments. +// A hyphen is a prerelease marker (Gem::Version#initialize rewrites "-" to +// ".pre." before tokenizing), not a plain separator. Numeric runs use bigint +// so segments beyond Number.MAX_SAFE_INTEGER still compare exactly. +function toRubyGemsSegments(version: string): (bigint | string)[] { + if (!RUBYGEMS_VERSION_PATTERN.test(version)) return [] + const normalized = version.replace(/-/g, '.pre.') + const runs = normalized.match(/[0-9]+|[a-zA-Z]+/g) ?? [] + return runs.map((run) => (/^[0-9]+$/.test(run) ? BigInt(run) : run)) +} + +// Mirrors Gem::Version#canonical_segments: drop trailing zero segments, then +// drop any zero segments that sit directly before the first letter segment. +function canonicalRubyGemsSegments(segments: (bigint | string)[]): (bigint | string)[] { + const trimmed = [...segments] + while (trimmed.length > 1 && trimmed[trimmed.length - 1] === BigInt(0)) trimmed.pop() + + const firstLetterIndex = trimmed.findIndex((segment) => typeof segment === 'string') + if (firstLetterIndex === -1) return trimmed + + let zeroRunStart = firstLetterIndex + while (zeroRunStart > 0 && trimmed[zeroRunStart - 1] === BigInt(0)) zeroRunStart-- + trimmed.splice(zeroRunStart, firstLetterIndex - zeroRunStart) + + return trimmed +} + +// Compares one pair of segments the way Gem::Version#<=> does: a letter +// segment always sorts below a numeric segment, regardless of its value. +function compareRubyGemsSegment(lhs: bigint | string, rhs: bigint | string): number { + if (typeof lhs === 'string' && typeof rhs !== 'string') return -1 + if (typeof lhs !== 'string' && typeof rhs === 'string') return 1 + return lhs < rhs ? -1 : lhs > rhs ? 1 : 0 +} + +// Once the shorter side is exhausted, Gem::Version#<=> walks the longer +// side's remaining segments: a letter segment means the longer side is a +// prerelease of the shorter one (sorts lower); a nonzero number means the +// longer side sorts higher; zeros (already mostly trimmed) are skipped. +function rubyGemsTailSign(segments: (bigint | string)[], startIndex: number): number { + for (let i = startIndex; i < segments.length; i++) { + const segment = segments[i] + if (typeof segment === 'string') return -1 + if (segment !== BigInt(0)) return 1 + } + return 0 +} + +function compareRubyGems(a: string, b: string): number | null { + const aSegments = canonicalRubyGemsSegments(toRubyGemsSegments(a)) + const bSegments = canonicalRubyGemsSegments(toRubyGemsSegments(b)) + if (aSegments.length === 0 || bSegments.length === 0) return null + + const limit = Math.min(aSegments.length, bSegments.length) + for (let i = 0; i < limit; i++) { + if (aSegments[i] === bSegments[i]) continue + return compareRubyGemsSegment(aSegments[i], bSegments[i]) + } + + if (aSegments.length > bSegments.length) return rubyGemsTailSign(aSegments, limit) + if (bSegments.length > aSegments.length) return -rubyGemsTailSign(bSegments, limit) + return 0 +} + +const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget', 'go']) // Ecosystem names are stored lowercase in packages-db per ADR-0001 §OSV // "Ecosystem normalization" — 'npm', 'maven', 'cargo'. Callers (deriveCriticalFlag) @@ -134,5 +203,6 @@ const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget']) export function compareVersion(ecosystem: string, a: string, b: string): number | null { if (SEMVER_ECOSYSTEMS.has(ecosystem)) return compareSemver(a, b) if (ecosystem === 'maven') return compareMaven(a, b) + if (ecosystem === 'rubygems') return compareRubyGems(a, b) return null }