Skip to content

fix: gnore weird transient packages #4176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
themarolt merged 4 commits into from
Jun 8, 2026
+12 −0
Merged

fix: gnore weird transient packages #4176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ce1638b
fix: ignore > > chained transient packages tracked as normal packages
themarolt Jun 5, 2026
3bf900a
fix: small things
themarolt Jun 5, 2026
1feb44d
fix: comments
themarolt Jun 5, 2026
8b60a76
Merge remote-tracking branch 'origin/main' into fix/ignore-weird-tran…
themarolt Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions services/apps/packages_worker/src/deps-dev/queries/advisoriesSql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ WITH purl_map AS (
FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\`
WHERE System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
GROUP BY System, Name
)
SELECT
Expand All @@ -33,5 +34,6 @@ FROM \`bigquery-public-data.deps_dev_v1.AdvisoriesLatest\` a,
UNNEST(a.Packages) AS pkg
LEFT JOIN purl_map pm ON pm.System = pkg.System AND pm.Name = pkg.Name
WHERE pkg.System IN (${systems})
AND pkg.Name NOT LIKE '%>%'
`
}
1 change: 1 addition & 0 deletions services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ WITH purl_map AS (
FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\`
WHERE System IN ('NPM', 'GO', 'MAVEN', 'PYPI', 'NUGET', 'CARGO')
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
GROUP BY System, Name
)
SELECT
Expand Down
1 change: 1 addition & 0 deletions services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ WITH purl_map AS (
FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\`
WHERE System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
GROUP BY System, Name
),
path_computed AS (
Expand Down
4 changes: 4 additions & 0 deletions services/apps/packages_worker/src/deps-dev/queries/packagesSql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ SELECT
FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\`
WHERE System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
QUALIFY ROW_NUMBER() OVER (PARTITION BY System, Name ORDER BY UpstreamPublishedAt DESC) = 1
`
}
Expand Down Expand Up @@ -44,6 +45,7 @@ WITH today AS (
AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
AND System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
QUALIFY ROW_NUMBER() OVER (PARTITION BY System, Name ORDER BY UpstreamPublishedAt DESC) = 1
),
last_watermark AS (
Expand All @@ -53,6 +55,8 @@ last_watermark AS (
WHERE SnapshotAt >= TIMESTAMP('${watermark}')
AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
AND System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
GROUP BY System, Name
)
SELECT t.* EXCEPT(UpstreamPublishedAt)
Expand Down
4 changes: 4 additions & 0 deletions services/apps/packages_worker/src/deps-dev/queries/versionsSql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ SELECT
FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\`
WHERE System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
`
}

Expand All @@ -34,13 +35,16 @@ WITH today AS (
AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
AND System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
),
last_watermark AS (
SELECT System, Name, Version, MAX(UpstreamPublishedAt) AS UpstreamPublishedAt
FROM \`bigquery-public-data.deps_dev_v1.PackageVersions\`
WHERE SnapshotAt >= TIMESTAMP('${watermark}')
AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
AND System IN (${systems})
AND Purl IS NOT NULL
AND Name NOT LIKE '%>%'
GROUP BY System, Name, Version
)
SELECT t.*
Expand Down
Loading