diff --git a/services/apps/packages_worker/src/deps-dev/queries/advisoriesSql.ts b/services/apps/packages_worker/src/deps-dev/queries/advisoriesSql.ts index 507e88df87..9b63103f85 100644 --- a/services/apps/packages_worker/src/deps-dev/queries/advisoriesSql.ts +++ b/services/apps/packages_worker/src/deps-dev/queries/advisoriesSql.ts @@ -20,6 +20,7 @@ WITH purl_map AS ( FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\` WHERE System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' GROUP BY System, Name ) SELECT @@ -33,5 +34,6 @@ FROM \`bigquery-public-data.deps_dev_v1.AdvisoriesLatest\` a, UNNEST(a.Packages) AS pkg LEFT JOIN purl_map pm ON pm.System = pkg.System AND pm.Name = pkg.Name WHERE pkg.System IN (${systems}) + AND pkg.Name NOT LIKE '%>%' ` } diff --git a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts index ac5532289f..62a69b2671 100644 --- a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts +++ b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts @@ -9,6 +9,7 @@ WITH purl_map AS ( FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\` WHERE System IN ('NPM', 'GO', 'MAVEN', 'PYPI', 'NUGET', 'CARGO') AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' GROUP BY System, Name ) SELECT diff --git a/services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts b/services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts index dc3d0c2551..a52b65f8c0 100644 --- a/services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts +++ b/services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts @@ -5,6 +5,7 @@ WITH purl_map AS ( FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\` WHERE System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' GROUP BY System, Name ), path_computed AS ( diff --git a/services/apps/packages_worker/src/deps-dev/queries/packagesSql.ts b/services/apps/packages_worker/src/deps-dev/queries/packagesSql.ts index 0d22750f2a..5e0b7a44ed 100644 --- a/services/apps/packages_worker/src/deps-dev/queries/packagesSql.ts +++ b/services/apps/packages_worker/src/deps-dev/queries/packagesSql.ts @@ -15,6 +15,7 @@ SELECT FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\` WHERE System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' QUALIFY ROW_NUMBER() OVER (PARTITION BY System, Name ORDER BY UpstreamPublishedAt DESC) = 1 ` } @@ -44,6 +45,7 @@ WITH today AS ( AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY)) AND System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' QUALIFY ROW_NUMBER() OVER (PARTITION BY System, Name ORDER BY UpstreamPublishedAt DESC) = 1 ), last_watermark AS ( @@ -53,6 +55,8 @@ last_watermark AS ( WHERE SnapshotAt >= TIMESTAMP('${watermark}') AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY)) AND System IN (${systems}) + AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' GROUP BY System, Name ) SELECT t.* EXCEPT(UpstreamPublishedAt) diff --git a/services/apps/packages_worker/src/deps-dev/queries/versionsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/versionsSql.ts index 7595406927..c4498a9767 100644 --- a/services/apps/packages_worker/src/deps-dev/queries/versionsSql.ts +++ b/services/apps/packages_worker/src/deps-dev/queries/versionsSql.ts @@ -11,6 +11,7 @@ SELECT FROM \`bigquery-public-data.deps_dev_v1.PackageVersionsLatest\` WHERE System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' ` } @@ -34,6 +35,7 @@ WITH today AS ( AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY)) AND System IN (${systems}) AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' ), last_watermark AS ( SELECT System, Name, Version, MAX(UpstreamPublishedAt) AS UpstreamPublishedAt @@ -41,6 +43,8 @@ last_watermark AS ( WHERE SnapshotAt >= TIMESTAMP('${watermark}') AND SnapshotAt < TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY)) AND System IN (${systems}) + AND Purl IS NOT NULL + AND Name NOT LIKE '%>%' GROUP BY System, Name, Version ) SELECT t.*