Fetch installed apps from iPhone/iPad devices. (#20733)

Part 2 of #19447
- iOS and iPadOS user-installed apps are loaded into Fleet
- Added an additional identifier into software_titles table to
differentiate between iOS/iPadOS apps
- Updated nano queue timestamp precision

Note: TestIntegrationsMDM/TestVPPApps fails when run as part of the
suite, but passes standalone. I'd like to proceed with merging this PR,
and figure out the issue next week.

# Checklist for submitter

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Added support on fleet's osquery simulator `cmd/osquery-perf` for
new osquery data ingestion features.
- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality

---------

Co-authored-by: Roberto Dip <rroperzh@gmail.com>

changes/19447-ios-ipados-software

@@ -1 +1,2 @@
 - iOS and iPadOS device details refetch can now be triggered with the existing `POST /api/latest/fleet/hosts/:id/refetch` endpoint.
+- iOS and iPadOS user-installed apps can be viewed in Fleet

cmd/fleet/cron.go

@@ -1305,7 +1305,12 @@ func newIPhoneIPadRefetcher(
 			}
 			logger.Log("msg", "sending commands to refetch", "count", len(uuids), "lookup-duration", time.Since(start))
 			commandUUID := fleet.RefetchCommandUUIDPrefix + uuid.NewString()
-			if err := commander.DeviceInformation(ctx, uuids, commandUUID); err != nil {
+			err = commander.InstalledApplicationList(ctx, uuids, fleet.RefetchAppsCommandUUIDPrefix+commandUUID)
+			if err != nil {
+				return ctxerr.Wrap(ctx, err, "send InstalledApplicationList commands to ios and ipados devices")
+			}
+			// DeviceInformation is last because the refetch response clears the refetch_requested flag
+			if err := commander.DeviceInformation(ctx, uuids, fleet.RefetchCommandUUIDPrefix+commandUUID); err != nil {
 				return ctxerr.Wrap(ctx, err, "send DeviceInformation commands to ios and ipados devices")
 			}
 			return nil

cmd/osquery-perf/README.md

@@ -95,3 +95,19 @@ Run the following command in the shell before running the Fleet server _and_ bef
 ``` sh
 ulimit -n 64000
 ```
+
+## Running with MDM
+
+Set up MDM on your server. To extract the SCEP challenge, you can use the [MDM asset extractor](https://github.com/fleetdm/fleet/tree/main/tools/mdm/assets).
+
+For your server, disable Apple push notifications since we will be using devices with fake UUIDs:
+
+```
+export FLEET_DEV_MDM_APPLE_DISABLE_PUSH=1
+```
+
+Example of running the agent with MDM. Note that `enroll_secret` is not needed for iPhone/iPad devices:
+
+```
+go run agent.go --os_templates ipad_13.18,iphone_14.6 --host_count 10 --mdm_scep_challenge 0d53306e-6d7a-9d14-a372-f9e53f9d62db
+```

cmd/osquery-perf/agent.go

@@ -424,6 +424,27 @@ func (n *nodeKeyManager) Add(nodekey string) {
 	}
 }
 
+type mdmAgent struct {
+	agentIndex         int
+	MDMCheckInInterval time.Duration
+	model              string
+	serverAddress      string
+	softwareCount      softwareEntityCount
+	stats              *Stats
+	strings            map[string]string
+}
+
+// stats, model, *serverURL, *mdmSCEPChallenge, *mdmCheckInInterval
+
+func (a *mdmAgent) CachedString(key string) string {
+	if val, ok := a.strings[key]; ok {
+		return val
+	}
+	val := randomString(12)
+	a.strings[key] = val
+	return val
+}
+
 type agent struct {
 	agentIndex                    int
 	softwareCount                 softwareEntityCount
@@ -1510,6 +1531,53 @@ func (a *agent) softwareMacOS() []map[string]string {
 	return software
 }
 
+func (a *mdmAgent) softwareIOSandIPadOS(source string) []fleet.Software {
+	commonSoftware := make([]map[string]string, a.softwareCount.common)
+	for i := 0; i < len(commonSoftware); i++ {
+		commonSoftware[i] = map[string]string{
+			"name":              fmt.Sprintf("Common_%d", i),
+			"version":           "0.0.1",
+			"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.common_%d", i),
+			"source":            source,
+		}
+	}
+	if a.softwareCount.commonSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.commonSoftwareUninstallProb {
+		rand.Shuffle(len(commonSoftware), func(i, j int) {
+			commonSoftware[i], commonSoftware[j] = commonSoftware[j], commonSoftware[i]
+		})
+		commonSoftware = commonSoftware[:a.softwareCount.common-a.softwareCount.commonSoftwareUninstallCount]
+	}
+	uniqueSoftware := make([]map[string]string, a.softwareCount.unique)
+	for i := 0; i < len(uniqueSoftware); i++ {
+		uniqueSoftware[i] = map[string]string{
+			"name":              fmt.Sprintf("Unique_%s_%d", a.CachedString("hostname"), i),
+			"version":           "1.1.1",
+			"bundle_identifier": fmt.Sprintf("com.fleetdm.osquery-perf.unique_%s_%d", a.CachedString("hostname"), i),
+			"source":            source,
+		}
+	}
+	if a.softwareCount.uniqueSoftwareUninstallProb > 0.0 && rand.Float64() <= a.softwareCount.uniqueSoftwareUninstallProb {
+		rand.Shuffle(len(uniqueSoftware), func(i, j int) {
+			uniqueSoftware[i], uniqueSoftware[j] = uniqueSoftware[j], uniqueSoftware[i]
+		})
+		uniqueSoftware = uniqueSoftware[:a.softwareCount.unique-a.softwareCount.uniqueSoftwareUninstallCount]
+	}
+	software := append(commonSoftware, uniqueSoftware...)
+	rand.Shuffle(len(software), func(i, j int) {
+		software[i], software[j] = software[j], software[i]
+	})
+	fleetSoftware := make([]fleet.Software, len(software))
+	for i, s := range software {
+		fleetSoftware[i] = fleet.Software{
+			Name:             s["name"],
+			Version:          s["version"],
+			BundleIdentifier: s["bundle_identifier"],
+			Source:           s["source"],
+		}
+	}
+	return fleetSoftware
+}
+
 func (a *agent) softwareVSCodeExtensions() []map[string]string {
 	commonVSCodeExtensionsSoftware := make([]map[string]string, a.softwareVSCodeExtensionsCount.common)
 	for i := 0; i < len(commonVSCodeExtensionsSoftware); i++ {
@@ -2160,48 +2228,57 @@ func (a *agent) submitLogs(results []resultLog) error {
 	return nil
 }
 
-func runAppleIDeviceMDMLoop(i int, stats *Stats, model string, serverURL string, mdmSCEPChallenge string, mdmCheckInInterval time.Duration) {
+func (a *mdmAgent) runAppleIDeviceMDMLoop(mdmSCEPChallenge string) {
 	udid := mdmtest.RandUDID()
 
 	mdmClient := mdmtest.NewTestMDMClientAppleDirect(mdmtest.AppleEnrollInfo{
 		SCEPChallenge: mdmSCEPChallenge,
-		SCEPURL:       serverURL + apple_mdm.SCEPPath,
-		MDMURL:        serverURL + apple_mdm.MDMPath,
-	}, model)
+		SCEPURL:       a.serverAddress + apple_mdm.SCEPPath,
+		MDMURL:        a.serverAddress + apple_mdm.MDMPath,
+	}, a.model)
 	mdmClient.UUID = udid
 	mdmClient.SerialNumber = mdmtest.RandSerialNumber()
-	deviceName := fmt.Sprintf("%s-%d", model, i)
-	productName := model
+	deviceName := fmt.Sprintf("%s-%d", a.model, a.agentIndex)
+	productName := a.model
+	softwareSource := "ios_apps"
+	if strings.HasPrefix(a.model, "iPad") {
+		softwareSource = "ipados_apps"
+	}
 
 	if err := mdmClient.Enroll(); err != nil {
-		log.Printf("%s MDM enroll failed: %s", model, err)
-		stats.IncrementMDMErrors()
+		log.Printf("%s MDM enroll failed: %s", a.model, err)
+		a.stats.IncrementMDMErrors()
 		return
 	}
 
-	stats.IncrementMDMEnrollments()
+	a.stats.IncrementMDMEnrollments()
 
-	mdmCheckInTicker := time.Tick(mdmCheckInInterval)
+	mdmCheckInTicker := time.Tick(a.MDMCheckInInterval)
 
 	for range mdmCheckInTicker {
 		mdmCommandPayload, err := mdmClient.Idle()
 		if err != nil {
-			log.Printf("MDM Idle request failed: %s: %s", model, err)
-			stats.IncrementMDMErrors()
+			log.Printf("MDM Idle request failed: %s: %s", a.model, err)
+			a.stats.IncrementMDMErrors()
 			continue
 		}
-		stats.IncrementMDMSessions()
+		a.stats.IncrementMDMSessions()
 
 		for mdmCommandPayload != nil {
-			stats.IncrementMDMCommandsReceived()
-			if mdmCommandPayload.Command.RequestType == "DeviceInformation" {
-				mdmCommandPayload, err = mdmClient.AcknowledgeDeviceInformation(udid, mdmCommandPayload.CommandUUID, deviceName, productName)
-			} else {
+			a.stats.IncrementMDMCommandsReceived()
+			switch mdmCommandPayload.Command.RequestType {
+			case "DeviceInformation":
+				mdmCommandPayload, err = mdmClient.AcknowledgeDeviceInformation(udid, mdmCommandPayload.CommandUUID, deviceName,
+					productName)
+			case "InstalledApplicationList":
+				software := a.softwareIOSandIPadOS(softwareSource)
+				mdmCommandPayload, err = mdmClient.AcknowledgeInstalledApplicationList(udid, mdmCommandPayload.CommandUUID, software)
+			default:
 				mdmCommandPayload, err = mdmClient.Acknowledge(mdmCommandPayload.CommandUUID)
 			}
 			if err != nil {
-				log.Printf("MDM Acknowledge request failed: %s: %s", model, err)
-				stats.IncrementMDMErrors()
+				log.Printf("MDM Acknowledge request failed: %s: %s", a.model, err)
+				a.stats.IncrementMDMErrors()
 				break
 			}
 		}
@@ -2416,7 +2493,26 @@ func main() {
 			if tmpl.Name() == "ipad_13.18.tmpl" {
 				model = "iPad 13,18"
 			}
-			go runAppleIDeviceMDMLoop(i, stats, model, *serverURL, *mdmSCEPChallenge, *mdmCheckInInterval)
+			mobileDevice := mdmAgent{
+				agentIndex:         i + 1,
+				MDMCheckInInterval: *mdmCheckInInterval,
+				model:              model,
+				serverAddress:      *serverURL,
+				softwareCount: softwareEntityCount{
+					entityCount: entityCount{
+						common: *commonSoftwareCount,
+						unique: *uniqueSoftwareCount,
+					},
+					vulnerable:                   *vulnerableSoftwareCount,
+					commonSoftwareUninstallCount: *commonSoftwareUninstallCount,
+					commonSoftwareUninstallProb:  *commonSoftwareUninstallProb,
+					uniqueSoftwareUninstallCount: *uniqueSoftwareUninstallCount,
+					uniqueSoftwareUninstallProb:  *uniqueSoftwareUninstallProb,
+				},
+				stats:   stats,
+				strings: make(map[string]string),
+			}
+			go mobileDevice.runAppleIDeviceMDMLoop(*mdmSCEPChallenge)
 			time.Sleep(sleepTime)
 			continue
 		}

pkg/mdm/mdmtest/apple.go

@@ -500,6 +500,26 @@ func (c *TestAppleMDMClient) AcknowledgeDeviceInformation(udid, cmdUUID, deviceN
 	return c.sendAndDecodeCommandResponse(payload)
 }
 
+func (c *TestAppleMDMClient) AcknowledgeInstalledApplicationList(udid, cmdUUID string, software []fleet.Software) (*mdm.Command, error) {
+	mdmSoftware := make([]map[string]interface{}, 0, len(software))
+	for _, s := range software {
+		mdmSoftware = append(mdmSoftware, map[string]interface{}{
+			"Name":         s.Name,
+			"ShortVersion": s.Version,
+			"Identifier":   s.BundleIdentifier,
+		})
+	}
+
+	payload := map[string]any{
+		"Status":                   "Acknowledged",
+		"UDID":                     udid,
+		"CommandUUID":              cmdUUID,
+		"InstalledApplicationList": mdmSoftware,
+	}
+
+	return c.sendAndDecodeCommandResponse(payload)
+}
+
 func (c *TestAppleMDMClient) GetBootstrapToken() ([]byte, error) {
 	payload := map[string]any{
 		"MessageType":  "GetBootstrapToken",

server/datastore/mysql/migrations/tables/20240725182118_AddAdditionalIdentifierToSoftwareTitles.go

@@ -0,0 +1,45 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20240725182118, Down_20240725182118)
+}
+
+func Up_20240725182118(tx *sql.Tx) error {
+	if columnExists(tx, "software_titles", "additional_identifier") {
+		return nil
+	}
+
+	_, err := tx.Exec(`
+		ALTER TABLE software_titles 
+		ADD COLUMN additional_identifier TINYINT UNSIGNED GENERATED ALWAYS AS 
+			(CASE
+				WHEN source = 'ios_apps' then 1
+				WHEN source = 'ipados_apps' then 2
+				WHEN bundle_identifier IS NOT NULL THEN 0
+				ELSE NULL
+			END) VIRTUAL`)
+	if err != nil {
+		return fmt.Errorf("adding additional_identifier to software_titles: %w", err)
+	}
+
+	_, err = tx.Exec(`ALTER TABLE software_titles DROP INDEX idx_software_titles_bundle_identifier`)
+	if err != nil {
+		return fmt.Errorf("dropping unique key for bundle_identifier in software_titles: %w", err)
+	}
+
+	_, err = tx.Exec(`ALTER TABLE software_titles ADD UNIQUE KEY idx_software_titles_bundle_identifier (bundle_identifier, additional_identifier)`)
+	if err != nil {
+		return fmt.Errorf("adding unique key for identifiers in software_titles: %w", err)
+	}
+
+	return nil
+}
+
+func Down_20240725182118(_ *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20240725182118_AddAdditionalIdentifierToSoftwareTitles_test.go

@@ -0,0 +1,86 @@
+package tables
+
+import (
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/ptr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20240725182118(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	// Data before ios and ipados apps were added
+	dataStmt := `
+	  INSERT INTO software_titles (id, name, source, browser, bundle_identifier) VALUES
+	    (1, 'Foo.app', 'apps', '', 'com.example.foo'),
+	    (2, 'Foo2.app', 'apps', '', 'com.example.foo2'),
+	    (3, 'Chrome Extension', 'chrome_extensions', 'chrome', NULL),
+	    (4, 'Microsoft Teams.exe', 'programs', '', NULL);
+	`
+
+	_, err := db.Exec(dataStmt)
+	require.NoError(t, err)
+	applyNext(t, db)
+
+	type softwareTitle struct {
+		Name                 string  `db:"name"`
+		Source               string  `db:"source"`
+		Browser              string  `db:"browser"`
+		BundleIdentifier     *string `db:"bundle_identifier"`
+		AdditionalIdentifier *uint32 `db:"additional_identifier"`
+	}
+
+	var titles []softwareTitle
+	err = db.Select(&titles, `SELECT name, source, browser, bundle_identifier, additional_identifier FROM software_titles`)
+	require.NoError(t, err)
+	zero := uint32(0)
+	expectedTitles := []softwareTitle{
+		{"Foo.app", "apps", "", ptr.String("com.example.foo"), &zero},
+		{"Foo2.app", "apps", "", ptr.String("com.example.foo2"), &zero},
+		{"Chrome Extension", "chrome_extensions", "chrome", nil, nil},
+		{"Microsoft Teams.exe", "programs", "", nil, nil},
+	}
+	assert.ElementsMatch(t, expectedTitles, titles)
+
+	// Ensure that the unique key is enforced
+	dataStmt = `
+	  INSERT INTO software_titles (id, name, source, browser, bundle_identifier) VALUES
+	    (100, 'Foo3', 'foo', '', 'com.example.foo');
+	`
+	_, err = db.Exec(dataStmt)
+	assert.ErrorContains(t, err, "Duplicate entry")
+
+	// Add ios and ipados apps
+	dataStmt = `
+	  INSERT INTO software_titles (id, name, source, browser, bundle_identifier) VALUES
+	    (5, 'Foo', 'ios_apps', '', 'com.example.foo'),
+	    (6, 'Foo', 'ipados_apps', '', 'com.example.foo'),
+	    (7, 'Bar-Pocket', 'ios_apps', '', 'com.example.bar-pocket'),
+	    (8, 'Bar', 'ipados_apps', '', 'com.example.bar');
+	`
+	_, err = db.Exec(dataStmt)
+	require.NoError(t, err)
+
+	err = db.Select(&titles, `SELECT name, source, browser, bundle_identifier, additional_identifier FROM software_titles`)
+	require.NoError(t, err)
+	one := uint32(1)
+	two := uint32(2)
+	expectedTitles = append(expectedTitles, []softwareTitle{
+		{"Foo", "ios_apps", "", ptr.String("com.example.foo"), &one},
+		{"Foo", "ipados_apps", "", ptr.String("com.example.foo"), &two},
+		{"Bar-Pocket", "ios_apps", "", ptr.String("com.example.bar-pocket"), &one},
+		{"Bar", "ipados_apps", "", ptr.String("com.example.bar"), &two},
+	}...)
+	assert.ElementsMatch(t, expectedTitles, titles)
+
+	// Ensure that the unique key is enforced
+	dataStmt = `
+	  INSERT INTO software_titles (id, name, source, browser, bundle_identifier) VALUES
+	    (200, 'Foo-Pocket', 'ipados_apps', '', 'com.example.foo');
+	`
+	_, err = db.Exec(dataStmt)
+	assert.ErrorContains(t, err, "Duplicate entry")
+
+}