Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

additional stats #19078

Merged
merged 26 commits into from
May 24, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
5d6cb07
additional stats
sharon-fdm May 16, 2024
b3f20f7
fix test
sharon-fdm May 16, 2024
ffd85fd
Merge branch 'main' into additional_stats
sharon-fdm May 17, 2024
f486a85
Update cmd/fleet/serve_test.go
sharon-fdm May 23, 2024
7beef00
Update serve_test.go
sharon-fdm May 23, 2024
a7c8fd2
Update serve_test.go
sharon-fdm May 23, 2024
6a98f32
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
0df924b
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
9f08269
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
8ff76bc
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
c7b9f6b
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
fbc3337
Update server/datastore/mysql/statistics.go
sharon-fdm May 23, 2024
1f544ca
Update server/fleet/statistics.go
sharon-fdm May 23, 2024
94c1d82
Apply suggestions from code review
sharon-fdm May 23, 2024
a39bd6d
PR review
sharon-fdm May 23, 2024
a52cf7a
change
sharon-fdm May 23, 2024
ad06f30
Merge branch 'main' into additional_stats
sharon-fdm May 23, 2024
bb5a97b
Merge branch 'main' into additional_stats
sharon-fdm May 24, 2024
23cc6ce
add tests
sharon-fdm May 24, 2024
279b9ac
add tests
sharon-fdm May 24, 2024
5ba4ba1
Merge branch 'main' into additional_stats
sharon-fdm May 24, 2024
ffcc1c1
refactor
sharon-fdm May 24, 2024
3327856
fix
sharon-fdm May 24, 2024
e5827e1
fix sql
sharon-fdm May 24, 2024
5be6c0a
fix test
sharon-fdm May 24, 2024
b04c256
Merge branch 'main' into additional_stats
sharon-fdm May 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion cmd/fleet/serve_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ func TestMaybeSendStatistics(t *testing.T) {
LicenseTier: "premium",
NumHostsEnrolled: 999,
NumUsers: 99,
NumSoftwareVersions: 100,
NumHostSoftwares: 101,
NumSoftwareTitles: 102,
NumHostSoftwareInstalledPaths: 103,
NumSoftwareCpes: 104,
NumSoftwareCves: 105,
NumTeams: 9,
NumPolicies: 0,
NumLabels: 3,
Expand Down Expand Up @@ -128,7 +134,7 @@ func TestMaybeSendStatistics(t *testing.T) {
require.NoError(t, err)
assert.True(t, recorded)
require.True(t, cleanedup)
assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","organization":"Fleet","numHostsEnrolled":999,"numUsers":99,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"mdmMacOsEnabled":false,"hostExpiryEnabled":false,"mdmWindowsEnabled":false,"liveQueryDisabled":false,"numWeeklyActiveUsers":111,"numWeeklyPolicyViolationDaysActual":0,"numWeeklyPolicyViolationDaysPossible":0,"hostsEnrolledByOperatingSystem":{"linux":[{"version":"1.2.3","numEnrolled":22}]},"hostsEnrolledByOrbitVersion":[],"hostsEnrolledByOsqueryVersion":[],"storedErrors":[],"numHostsNotResponding":0}`, requestBody)
assert.Equal(t, `{"anonymousIdentifier":"ident","fleetVersion":"1.2.3","licenseTier":"premium","organization":"Fleet","numHostsEnrolled":999,"numUsers":99,"NumSoftwareVersions":100,"numHostSoftwares":101,"numSoftwareTitles":102,"numHostSoftwareInstalledPaths":103,"numSoftwareCpes":104,"numSoftwareCves":105,"numTeams":9,"numPolicies":0,"numLabels":3,"softwareInventoryEnabled":true,"vulnDetectionEnabled":true,"systemUsersEnabled":true,"hostsStatusWebHookEnabled":true,"mdmMacOsEnabled":false,"hostExpiryEnabled":false,"mdmWindowsEnabled":false,"liveQueryDisabled":false,"numWeeklyActiveUsers":111,"numWeeklyPolicyViolationDaysActual":0,"numWeeklyPolicyViolationDaysPossible":0,"hostsEnrolledByOperatingSystem":{"linux":[{"version":"1.2.3","numEnrolled":22}]},"hostsEnrolledByOrbitVersion":[],"hostsEnrolledByOsqueryVersion":[],"storedErrors":[],"numHostsNotResponding":0}`, requestBody)
}

func TestMaybeSendStatisticsSkipsSendingIfNotNeeded(t *testing.T) {
Expand Down
30 changes: 30 additions & 0 deletions server/datastore/mysql/statistics.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,30 @@ func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Du
if err != nil {
return ctxerr.Wrap(ctx, err, "amount users")
}
amountSoftwares, err := amountSoftwaresDB(ctx, ds.writer(ctx))
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return ctxerr.Wrap(ctx, err, "amount software")
}
amountHostSoftwares, err := amountHostSoftwaresDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount HostSoftwares")
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
}
amountSoftwareTitles, err := amountSoftwareTitlesDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount SoftwareTitles")
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
}
amountHostSoftwareInstalledPaths, err := amountHostSoftwareInstalledPathsDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount HostSoftwareInstalledPath")
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
}
amountSoftwareCpes, err := amountSoftwareCpesDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount SoftwareCpes")
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
}
amountSoftwareCves, err := amountSoftwareCvesDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount SoftwareCves")
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
}
amountTeams, err := amountTeamsDB(ctx, ds.writer(ctx))
if err != nil {
return ctxerr.Wrap(ctx, err, "amount teams")
Expand Down Expand Up @@ -77,6 +101,12 @@ func (ds *Datastore) ShouldSendStatistics(ctx context.Context, frequency time.Du

stats.NumHostsEnrolled = amountEnrolledHosts
stats.NumUsers = amountUsers
stats.NumSoftwares = amountSoftwares
stats.NumHostSoftwares = amountHostSoftwares
stats.NumSoftwareTitles = amountSoftwareTitles
stats.NumHostSoftwareInstalledPaths = amountHostSoftwareInstalledPaths
stats.NumSoftwareCpes = amountSoftwareCpes
stats.NumSoftwareCves = amountSoftwareCves
stats.NumTeams = amountTeams
stats.NumPolicies = amountPolicies
stats.NumLabels = amountLabels
Expand Down
6 changes: 6 additions & 0 deletions server/datastore/mysql/statistics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) {
assert.Equal(t, "Fleet", stats.Organization)
assert.Equal(t, 0, stats.NumHostsEnrolled)
assert.Equal(t, 0, stats.NumUsers)
assert.Equal(t, 0, stats.NumSoftwares)
assert.Equal(t, 0, stats.NumHostSoftwares)
assert.Equal(t, 0, stats.NumSoftwareTitles)
assert.Equal(t, 0, stats.NumHostSoftwareInstalledPaths)
assert.Equal(t, 0, stats.NumSoftwareCpes)
assert.Equal(t, 0, stats.NumSoftwareCves)
assert.Equal(t, 0, stats.NumTeams)
assert.Equal(t, 0, stats.NumPolicies)
assert.Equal(t, builtinLabels, stats.NumLabels)
Expand Down
54 changes: 54 additions & 0 deletions server/datastore/mysql/users.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,60 @@ func amountUsersDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
return amount, nil
}

func amountSoftwaresDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM software`)
if err != nil {
return 0, err
}
return amount, nil
}

func amountHostSoftwaresDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM host_software`)
if err != nil {
return 0, err
}
return amount, nil
}

func amountSoftwareTitlesDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM software_titles`)
if err != nil {
return 0, err
}
return amount, nil
}

func amountHostSoftwareInstalledPathsDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM host_software_installed_paths`)
if err != nil {
return 0, err
}
return amount, nil
}

func amountSoftwareCpesDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM software_cpe`)
if err != nil {
return 0, err
}
return amount, nil
}

func amountSoftwareCvesDB(ctx context.Context, db sqlx.QueryerContext) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `SELECT count(*) FROM software_cve`)
if err != nil {
return 0, err
}
return amount, nil
}
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved

func amountActiveUsersSinceDB(ctx context.Context, db sqlx.QueryerContext, since time.Time) (int, error) {
var amount int
err := sqlx.GetContext(ctx, db, &amount, `
Expand Down
42 changes: 24 additions & 18 deletions server/fleet/statistics.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,30 @@ import (
)

type StatisticsPayload struct {
AnonymousIdentifier string `json:"anonymousIdentifier"`
FleetVersion string `json:"fleetVersion"`
LicenseTier string `json:"licenseTier"`
Organization string `json:"organization"`
NumHostsEnrolled int `json:"numHostsEnrolled"`
NumUsers int `json:"numUsers"`
NumTeams int `json:"numTeams"`
NumPolicies int `json:"numPolicies"`
NumLabels int `json:"numLabels"`
SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"`
VulnDetectionEnabled bool `json:"vulnDetectionEnabled"`
SystemUsersEnabled bool `json:"systemUsersEnabled"`
HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"`
MDMMacOsEnabled bool `json:"mdmMacOsEnabled"`
HostExpiryEnabled bool `json:"hostExpiryEnabled"`
MDMWindowsEnabled bool `json:"mdmWindowsEnabled"`
LiveQueryDisabled bool `json:"liveQueryDisabled"`
NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"`
AnonymousIdentifier string `json:"anonymousIdentifier"`
FleetVersion string `json:"fleetVersion"`
LicenseTier string `json:"licenseTier"`
Organization string `json:"organization"`
NumHostsEnrolled int `json:"numHostsEnrolled"`
NumUsers int `json:"numUsers"`
NumSoftwares int `json:"numSoftwares"`
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
NumHostSoftwares int `json:"numHostSoftwares"`
NumSoftwareTitles int `json:"numSoftwareTitles"`
NumHostSoftwareInstalledPaths int `json:"numHostSoftwareInstalledPaths"`
NumSoftwareCpes int `json:"numSoftwareCpes"`
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
NumSoftwareCves int `json:"numSoftwareCves"`
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
sharon-fdm marked this conversation as resolved.
Show resolved Hide resolved
NumTeams int `json:"numTeams"`
NumPolicies int `json:"numPolicies"`
NumLabels int `json:"numLabels"`
SoftwareInventoryEnabled bool `json:"softwareInventoryEnabled"`
VulnDetectionEnabled bool `json:"vulnDetectionEnabled"`
SystemUsersEnabled bool `json:"systemUsersEnabled"`
HostsStatusWebHookEnabled bool `json:"hostsStatusWebHookEnabled"`
MDMMacOsEnabled bool `json:"mdmMacOsEnabled"`
HostExpiryEnabled bool `json:"hostExpiryEnabled"`
MDMWindowsEnabled bool `json:"mdmWindowsEnabled"`
LiveQueryDisabled bool `json:"liveQueryDisabled"`
NumWeeklyActiveUsers int `json:"numWeeklyActiveUsers"`
// NumWeeklyPolicyViolationDaysActual is an aggregate count of actual policy violation days. One
// policy violation day is added for each policy that a host is failing as of the time the count
// is incremented. The count increments once per 24-hour interval and resets each week.
Expand Down
Loading