x-pack/filebeat/input/entityanalytics/provider/okta: allow fine-grain…

… control of API requests

This adds support for specifying which of users/devices to collect from
the Okta API endpoints in order to reduce network costs for users who do
not need a full set of entities.

The current change does not change the behaviour of device collection of
registered owners and registered users; when the "devices" dataset is
selected there user entities will still be collected as they are
considered here as an attribute of the device, rather than a component
of the users dataset.

CHANGELOG.next.asciidoc

@@ -209,6 +209,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Reduce HTTPJSON metrics allocations. {pull}36282[36282]
 - Add support for a simplified input configuraton when running under Elastic-Agent {pull}36390[36390]
 - Make HTTPJSON response body decoding errors more informative. {pull}36481[36481]
+- Allow fine-grained control of entity analytics API requests for Okta provider. {issue}36440[36440] {pull}36492[36492]
 
 *Auditbeat*
 

x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc

@@ -545,6 +545,7 @@ Example configuration:
   enabled: true
   id: okta-1
   provider: okta
+  dataset: "all"
   sync_interval: "12h"
   update_interval: "30m"
   okta_domain: "OKTA_DOMAIN"
@@ -570,6 +571,14 @@ Whether the input should collect device and device-associated user details
 from the Okta API. Device details must be activated on the Okta account for
 this option.
 
+[float]
+===== `dataset`
+
+The datasets to collect from the API. This can be one of "all", "users" or "devices",
+or may be left empty for the default behavior which is to collect all entities.
+When the `dataset` is set to "devices", some user entity data is collected in order
+to populate the registered users and registered owner fields for each device.
+
 [float]
 ===== `sync_interval`
 

x-pack/filebeat/input/entityanalytics/provider/okta/conf.go

@@ -6,6 +6,7 @@ package okta
 
 import (
 	"errors"
+	"strings"
 	"time"
 
 	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
@@ -61,6 +62,11 @@ type conf struct {
 	// Request is the configuration for establishing
 	// HTTP requests to the API.
 	Request *requestConfig `config:"request"`
+
+	// Dataset specifies the datasets to collect from
+	// the API. It can be ""/"all", "users", or
+	// "devices".
+	Dataset string `config:"dataset"`
 }
 
 type requestConfig struct {
@@ -159,7 +165,11 @@ func (c *conf) Validate() error {
 		return errInvalidUpdateInterval
 	case c.SyncInterval <= c.UpdateInterval:
 		return errSyncBeforeUpdate
-	default:
+	}
+	switch strings.ToLower(c.Dataset) {
+	case "", "all", "users", "devices":
 		return nil
+	default:
+		return errors.New("dataset must be 'all', 'users', 'devices' or empty")
 	}
 }

x-pack/filebeat/input/entityanalytics/provider/okta/okta.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/go-retryablehttp"
@@ -243,13 +244,23 @@ func (p *oktaInput) runFullSync(inputCtx v2.Context, store *kvstore.Store, clien
 
 	ctx := ctxtool.FromCanceller(inputCtx.Cancelation)
 	p.logger.Debugf("Starting fetch...")
-	_, err = p.doFetchUsers(ctx, state, true)
-	if err != nil {
-		return err
+	switch strings.ToLower(p.cfg.Dataset) {
+	case "", "all", "users":
+		_, err = p.doFetchUsers(ctx, state, true)
+		if err != nil {
+			return err
+		}
+	default:
+		p.logger.Debugf("Skipping user collection from API: dataset=%s", p.cfg.Dataset)
 	}
-	_, err = p.doFetchDevices(ctx, state, true)
-	if err != nil {
-		return err
+	switch strings.ToLower(p.cfg.Dataset) {
+	case "", "all", "devices":
+		_, err = p.doFetchDevices(ctx, state, true)
+		if err != nil {
+			return err
+		}
+	default:
+		p.logger.Debugf("Skipping device collection from API: dataset=%s", p.cfg.Dataset)
 	}
 
 	if len(state.users) != 0 || len(state.devices) != 0 {
@@ -301,13 +312,25 @@ func (p *oktaInput) runIncrementalUpdate(inputCtx v2.Context, store *kvstore.Sto
 	}()
 
 	ctx := ctxtool.FromCanceller(inputCtx.Cancelation)
-	updatedUsers, err := p.doFetchUsers(ctx, state, false)
-	if err != nil {
-		return err
+	var updatedUsers []*User
+	switch strings.ToLower(p.cfg.Dataset) {
+	case "", "all", "users":
+		updatedUsers, err = p.doFetchUsers(ctx, state, false)
+		if err != nil {
+			return err
+		}
+	default:
+		p.logger.Debugf("Skipping user collection from API: dataset=%s", p.cfg.Dataset)
 	}
-	updatedDevices, err := p.doFetchDevices(ctx, state, false)
-	if err != nil {
-		return err
+	var updatedDevices []*Device
+	switch strings.ToLower(p.cfg.Dataset) {
+	case "", "all", "devices":
+		updatedDevices, err = p.doFetchDevices(ctx, state, false)
+		if err != nil {
+			return err
+		}
+	default:
+		p.logger.Debugf("Skipping device collection from API: dataset=%s", p.cfg.Dataset)
 	}
 
 	var tracker *kvstore.TxTracker
@@ -482,7 +505,9 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS
 
 				// Users are not stored in the state as they are in doFetchUsers. We expect
 				// them to already have been discovered/stored from that call and are stored
-				// associated with the device undecorated with discovery state.
+				// associated with the device undecorated with discovery state. Or, if the
+				// the dataset is set to "devices", then we have been asked not to care about
+				// this detail.
 				batch[i].Users = append(batch[i].Users, users...)
 
 				next, err := okta.Next(h)