Improved MySQL query performance software versions and vulnerabilities endpoints

changes/32178-software-vuln-perf

@@ -0,0 +1 @@
+Optimized MySQL queries on /api/latest/fleet/vulnerabilities and /api/latest/fleet/software/versions to improve performance for Fleet UI use cases.

server/datastore/mysql/software.go

@@ -1407,7 +1407,13 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e
 				goqu.I("software_cve").As("scv"),
 				goqu.On(goqu.I("s.id").Eq(goqu.I("scv.software_id"))),
 			)
-	} else {
+	} else if !opts.WithoutVulnerabilityDetails || opts.IncludeCVEScores {
+		// Only LEFT JOIN software_cve if we need CVE details in the list OR if we need CVE scores.
+		// When WithoutVulnerabilityDetails=true AND IncludeCVEScores=false, we can skip this join
+		// entirely in the subquery. The outer query will fetch CVEs only for the paginated results,
+		// which is much more efficient.
+		// However, if IncludeCVEScores=true, we MUST include the join because we need to select
+		// scv.resolved_in_version and other scv columns for ordering/filtering.
 		ds = ds.
 			LeftJoin(
 				goqu.I("software_cve").As("scv"),

server/datastore/mysql/vulnerabilities.go

@@ -219,41 +219,62 @@ func (ds *Datastore) SoftwareByCVE(ctx context.Context, cve string, teamID *uint
 
 func (ds *Datastore) ListVulnerabilities(ctx context.Context, opt fleet.VulnListOptions) ([]fleet.VulnerabilityWithMetadata, *fleet.PaginationMetadata, error) {
 	// Define base select statements for EE and Free versions
+	//
+	// created_at: Use MIN() to get earliest discovery date across both tables.
+	// This is important because users can sort by this field, and in production (Dogfood)
+	// data shows significant differences (>1 year) between tables.
+	// Note: created_at can be NULL in schema but never is in practice.
+	//
+	// source: Pick first match, prioritizing software_cve.
+	// This field is not exposed in the API (json:"-").
+	// Note: source can be NULL in schema but never is in practice.
 	eeSelectStmt := `
 		SELECT
-			combined.cve as cve,
-			MIN(combined.created_at) as created_at,
-			MIN(combined.source) as source,
+			vhc.cve as cve,
+			(SELECT MIN(created_at) FROM (
+				SELECT created_at FROM software_cve WHERE cve = vhc.cve
+				UNION ALL
+				SELECT created_at FROM operating_system_vulnerabilities WHERE cve = vhc.cve
+			) AS combined_dates) as created_at,
+			COALESCE(
+				(SELECT source FROM software_cve WHERE cve = vhc.cve LIMIT 1),
+				(SELECT source FROM operating_system_vulnerabilities WHERE cve = vhc.cve LIMIT 1)
+			) as source,
 			cm.cvss_score,
 			cm.epss_probability,
 			cm.cisa_known_exploit,
 			cm.published as cve_published,
 			cm.description,
 			vhc.host_count as hosts_count,
 			vhc.updated_at as hosts_count_updated_at
-		FROM (
-			SELECT cve, created_at, source FROM software_cve
-			UNION
-			SELECT cve, created_at, source FROM operating_system_vulnerabilities
-		) AS combined
-		INNER JOIN vulnerability_host_counts vhc ON vhc.cve = combined.cve
-		LEFT JOIN cve_meta cm ON cm.cve = combined.cve
+		FROM vulnerability_host_counts vhc
+		LEFT JOIN cve_meta cm ON cm.cve = vhc.cve
 		WHERE vhc.host_count > 0
+		AND (
+			EXISTS (SELECT 1 FROM software_cve WHERE cve = vhc.cve)
+			OR EXISTS (SELECT 1 FROM operating_system_vulnerabilities WHERE cve = vhc.cve)
+		)
 		`
 	freeSelectStmt := `
 		SELECT
-			combined.cve as cve,
-			MIN(combined.created_at) as created_at,
-			MIN(combined.source) as source,
+			vhc.cve as cve,
+			(SELECT MIN(created_at) FROM (
+				SELECT created_at FROM software_cve WHERE cve = vhc.cve
+				UNION ALL
+				SELECT created_at FROM operating_system_vulnerabilities WHERE cve = vhc.cve
+			) AS combined_dates) as created_at,
+			COALESCE(
+				(SELECT source FROM software_cve WHERE cve = vhc.cve LIMIT 1),
+				(SELECT source FROM operating_system_vulnerabilities WHERE cve = vhc.cve LIMIT 1)
+			) as source,
 			vhc.host_count as hosts_count,
 			vhc.updated_at as hosts_count_updated_at
-		FROM (
-			SELECT cve, created_at, source FROM software_cve
-			UNION
-			SELECT cve, created_at, source FROM operating_system_vulnerabilities
-		) AS combined
-		INNER JOIN vulnerability_host_counts vhc ON vhc.cve = combined.cve
+		FROM vulnerability_host_counts vhc
 		WHERE vhc.host_count > 0
+		AND (
+			EXISTS (SELECT 1 FROM software_cve WHERE cve = vhc.cve)
+			OR EXISTS (SELECT 1 FROM operating_system_vulnerabilities WHERE cve = vhc.cve)
+		)
 		`
 
 	// Choose the appropriate select statement based on EE or Free
@@ -281,9 +302,6 @@ func (ds *Datastore) ListVulnerabilities(ctx context.Context, opt fleet.VulnList
 		selectStmt, args = searchLike(selectStmt, args, match, "vhc.cve")
 	}
 
-	// Append group by statement
-	selectStmt += " GROUP BY cve, host_count, updated_at"
-
 	opt.ListOptions.IncludeMetadata = !(opt.ListOptions.UsesCursorPagination())
 	selectStmt, args = appendListOptionsWithCursorToSQL(selectStmt, args, &opt.ListOptions)
 
@@ -309,21 +327,20 @@ func (ds *Datastore) ListVulnerabilities(ctx context.Context, opt fleet.VulnList
 func (ds *Datastore) CountVulnerabilities(ctx context.Context, opt fleet.VulnListOptions) (uint, error) {
 	selectStmt := `
 		SELECT
-			COUNT(DISTINCT combined.cve)
-		FROM (
-			SELECT cve FROM software_cve
-			UNION
-			SELECT cve FROM operating_system_vulnerabilities
-		) AS combined
-		INNER JOIN vulnerability_host_counts vhc ON vhc.cve = combined.cve
-		LEFT JOIN cve_meta cm ON cm.cve = combined.cve
+			COUNT(DISTINCT vhc.cve)
+		FROM vulnerability_host_counts vhc
+		LEFT JOIN cve_meta cm ON cm.cve = vhc.cve
 		WHERE vhc.host_count > 0
+		AND (
+			EXISTS (SELECT 1 FROM software_cve WHERE cve = vhc.cve)
+			OR EXISTS (SELECT 1 FROM operating_system_vulnerabilities WHERE cve = vhc.cve)
+		)
 	`
 	var args []interface{}
 	if opt.TeamID == nil {
-		selectStmt += " AND global_stats = 1"
+		selectStmt += " AND vhc.global_stats = 1"
 	} else {
-		selectStmt += " AND global_stats = 0 AND vhc.team_id = ?"
+		selectStmt += " AND vhc.global_stats = 0 AND vhc.team_id = ?"
 		args = append(args, opt.TeamID)
 	}