From e89e16aa7841bc0d53e152c8ac8dd175b9b10902 Mon Sep 17 00:00:00 2001 From: Olivia Trewin Date: Tue, 23 Jan 2024 15:46:27 -0600 Subject: [PATCH] DLP-1223: Add support for DLP EDM and CWL datasets --- .changelog/1485.txt | 3 + dlp_dataset.go | 275 +++++++++++++++++++++++++++++ dlp_dataset_test.go | 417 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 695 insertions(+) create mode 100644 .changelog/1485.txt create mode 100644 dlp_dataset.go create mode 100644 dlp_dataset_test.go diff --git a/.changelog/1485.txt b/.changelog/1485.txt new file mode 100644 index 00000000000..e8a177e2e22 --- /dev/null +++ b/.changelog/1485.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +dlp: add support for EDM and CWL datasets +``` diff --git a/dlp_dataset.go b/dlp_dataset.go new file mode 100644 index 00000000000..f12fcb5727f --- /dev/null +++ b/dlp_dataset.go @@ -0,0 +1,275 @@ +package cloudflare + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/goccy/go-json" +) + +var ( + ErrMissingDatasetID = errors.New("missing required dataset ID") +) + +// DLPDatasetUpload represents a single upload version attached to a DLP dataset. +type DLPDatasetUpload struct { + NumCells int `json:"num_cells"` + Status string `json:"status,omitempty"` + Version int `json:"version"` +} + +// DLPDataset represents a DLP Exact Data Match dataset or Custom Word List. +type DLPDataset struct { + CreatedAt *time.Time `json:"created_at,omitempty"` + Description string `json:"description,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + NumCells int `json:"num_cells"` + Secret bool `json:"secret"` + Status string `json:"status,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + Uploads []DLPDatasetUpload `json:"uploads"` +} + +type DLPDatasetListParams struct{} + +type DLPDatasetListResponse struct { + Result []DLPDataset `json:"result"` + Response +} + +// ListDLPDatasets returns all the DLP datasets associated with an account. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-read-all +func (api *API) ListDLPDatasets(ctx context.Context, rc *ResourceContainer, params DLPDatasetListParams) ([]DLPDataset, error) { + if rc.Identifier == "" { + return nil, nil + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets", rc.Level, rc.Identifier), nil) + + res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return nil, err + } + + var dlpDatasetListResponse DLPDatasetListResponse + err = json.Unmarshal(res, &dlpDatasetListResponse) + if err != nil { + return nil, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetListResponse.Result, nil +} + +type DLPDatasetGetResponse struct { + Result DLPDataset `json:"result"` + Response +} + +// GetDLPDataset returns a DLP dataset based on the dataset ID. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-read +func (api *API) GetDLPDataset(ctx context.Context, rc *ResourceContainer, datasetID string) (DLPDataset, error) { + if rc.Identifier == "" { + return DLPDataset{}, nil + } + + if datasetID == "" { + return DLPDataset{}, ErrMissingDatasetID + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets/%s", rc.Level, rc.Identifier, datasetID), nil) + + res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return DLPDataset{}, err + } + + var dlpDatasetGetResponse DLPDatasetGetResponse + err = json.Unmarshal(res, &dlpDatasetGetResponse) + if err != nil { + return DLPDataset{}, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetGetResponse.Result, nil +} + +type DLPDatasetCreateParams struct { + Description string `json:"description,omitempty"` + Name string `json:"name"` + Secret bool `json:"secret"` +} + +type DLPDatasetCreateResult struct { + MaxCells int `json:"max_cells"` + Secret string `json:"secret"` + Version int `json:"version"` + Dataset DLPDataset `json:"dataset"` +} + +type DLPDatasetCreateResponse struct { + Result DLPDatasetCreateResult `json:"result"` + Response +} + +// CreateDLPDataset creates a DLP dataset. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-create +func (api *API) CreateDLPDataset(ctx context.Context, rc *ResourceContainer, params DLPDatasetCreateParams) (DLPDatasetCreateResult, error) { + if rc.Identifier == "" { + return DLPDatasetCreateResult{}, nil + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets", rc.Level, rc.Identifier), nil) + + res, err := api.makeRequestContext(ctx, http.MethodPost, uri, params) + if err != nil { + return DLPDatasetCreateResult{}, err + } + + var dlpDatasetCreateResponse DLPDatasetCreateResponse + err = json.Unmarshal(res, &dlpDatasetCreateResponse) + if err != nil { + return DLPDatasetCreateResult{}, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetCreateResponse.Result, nil +} + +// DeleteDLPDataset deletes a DLP dataset. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-delete +func (api *API) DeleteDLPDataset(ctx context.Context, rc *ResourceContainer, datasetID string) error { + if rc.Identifier == "" { + return ErrMissingResourceIdentifier + } + + if datasetID == "" { + return ErrMissingDatasetID + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets/%s", rc.Level, rc.Identifier, datasetID), nil) + _, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil) + return err +} + +type DLPDatasetUpdateParams struct { + DatasetID string + Description *string `json:"description,omitempty"` // nil to leave descrption as-is + Name *string `json:"name,omitempty"` // nil to leave name as-is +} + +type DLPDatasetUpdateResponse struct { + Result DLPDataset `json:"result"` + Response +} + +// UpdateDLPDataset updates the details of a DLP dataset. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-update +func (api *API) UpdateDLPDataset(ctx context.Context, rc *ResourceContainer, params DLPDatasetUpdateParams) (DLPDataset, error) { + if rc.Identifier == "" { + return DLPDataset{}, nil + } + + if params.DatasetID == "" { + return DLPDataset{}, ErrMissingDatasetID + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets/%s", rc.Level, rc.Identifier, params.DatasetID), nil) + + res, err := api.makeRequestContext(ctx, http.MethodPut, uri, params) + if err != nil { + return DLPDataset{}, err + } + + var dlpDatasetUpdateResponse DLPDatasetUpdateResponse + err = json.Unmarshal(res, &dlpDatasetUpdateResponse) + if err != nil { + return DLPDataset{}, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetUpdateResponse.Result, nil +} + +type DLPDatasetCreateUploadResult struct { + MaxCells int `json:"max_cells"` + Secret string `json:"secret"` + Version int `json:"version"` +} + +type DLPDatasetCreateUploadResponse struct { + Result DLPDatasetCreateUploadResult `json:"result"` + Response +} + +// CreateDLPDatasetUpload creates a new upload version for the specified DLP dataset. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-create-version +func (api *API) CreateDLPDatasetUpload(ctx context.Context, rc *ResourceContainer, datasetID string) (DLPDatasetCreateUploadResult, error) { + if rc.Identifier == "" { + return DLPDatasetCreateUploadResult{}, nil + } + + if datasetID == "" { + return DLPDatasetCreateUploadResult{}, ErrMissingDatasetID + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets/%s/upload", rc.Level, rc.Identifier, datasetID), nil) + + res, err := api.makeRequestContext(ctx, http.MethodPost, uri, nil) + if err != nil { + return DLPDatasetCreateUploadResult{}, err + } + + var dlpDatasetCreateUploadResponse DLPDatasetCreateUploadResponse + err = json.Unmarshal(res, &dlpDatasetCreateUploadResponse) + if err != nil { + return DLPDatasetCreateUploadResult{}, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetCreateUploadResponse.Result, nil +} + +type DLPDatasetUploadVersionParams struct { + DatasetID string + Version int + Body interface{} +} + +type DLPDatasetUploadVersionResponse struct { + Result DLPDataset `json:"result"` + Response +} + +// UploadDLPDatasetVersion uploads a new version of the specified DLP dataset. +// +// API reference: https://developers.cloudflare.com/api/operations/dlp-datasets-upload-version +func (api *API) UploadDLPDatasetVersion(ctx context.Context, rc *ResourceContainer, params DLPDatasetUploadVersionParams) (DLPDataset, error) { + if rc.Identifier == "" { + return DLPDataset{}, nil + } + + if params.DatasetID == "" { + return DLPDataset{}, ErrMissingDatasetID + } + + uri := buildURI(fmt.Sprintf("/%s/%s/dlp/datasets/%s/upload/%d", rc.Level, rc.Identifier, params.DatasetID, params.Version), nil) + + res, err := api.makeRequestContext(ctx, http.MethodPost, uri, params.Body) + if err != nil { + return DLPDataset{}, err + } + + var dlpDatasetUploadVersionResponse DLPDatasetUploadVersionResponse + err = json.Unmarshal(res, &dlpDatasetUploadVersionResponse) + if err != nil { + return DLPDataset{}, fmt.Errorf("%s: %w", errUnmarshalError, err) + } + + return dlpDatasetUploadVersionResponse.Result, nil +} diff --git a/dlp_dataset_test.go b/dlp_dataset_test.go new file mode 100644 index 00000000000..1774236ddfc --- /dev/null +++ b/dlp_dataset_test.go @@ -0,0 +1,417 @@ +package cloudflare + +import ( + "context" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListDLPDatasets(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method, "Expected method 'GET', got %s", r.Method) + w.Header().Set("content-type", "application/json") + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": [ + { + "created_at": "2019-08-24T14:15:22Z", + "description": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "num_cells": 0, + "secret": true, + "status": "empty", + "updated_at": "2019-08-24T14:15:22Z", + "uploads": [ + { + "num_cells": 0, + "status": "empty", + "version": 0 + } + ] + } + ] + }`) + } + + createdAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + updatedAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + + want := []DLPDataset{ + { + CreatedAt: &createdAt, + Description: "string", + ID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", + Name: "string", + NumCells: 0, + Secret: true, + Status: "empty", + UpdatedAt: &updatedAt, + Uploads: []DLPDatasetUpload{ + { + NumCells: 0, + Status: "empty", + Version: 0, + }, + }, + }, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets", handler) + + actual, err := client.ListDLPDatasets(context.Background(), AccountIdentifier(testAccountID), DLPDatasetListParams{}) + require.NoError(t, err) + require.Equal(t, want, actual) +} + +func TestGetDLPDataset(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method, "Expected method 'GET', got %s", r.Method) + w.Header().Set("content-type", "application/json") + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": { + "created_at": "2019-08-24T14:15:22Z", + "description": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "num_cells": 0, + "secret": true, + "status": "empty", + "updated_at": "2019-08-24T14:15:22Z", + "uploads": [ + { + "num_cells": 0, + "status": "empty", + "version": 0 + } + ] + } + }`) + } + + createdAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + updatedAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + + want := DLPDataset{ + CreatedAt: &createdAt, + Description: "string", + ID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", + Name: "string", + NumCells: 0, + Secret: true, + Status: "empty", + UpdatedAt: &updatedAt, + Uploads: []DLPDatasetUpload{ + { + NumCells: 0, + Status: "empty", + Version: 0, + }, + }, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets/497f6eca-6276-4993-bfeb-53cbbbba6f08", handler) + + actual, err := client.GetDLPDataset(context.Background(), AccountIdentifier(testAccountID), "497f6eca-6276-4993-bfeb-53cbbbba6f08") + require.NoError(t, err) + require.Equal(t, want, actual) +} + +func TestCreateDLPDataset(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method, "Expected method 'POST', got %s", r.Method) + w.Header().Set("content-type", "application/json") + + var reqBody DLPDatasetCreateParams + err := json.NewDecoder(r.Body).Decode(&reqBody) + require.Nil(t, err) + + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": { + "max_cells": 0, + "secret": "1234", + "version": 0, + "dataset": { + "created_at": "2019-08-24T14:15:22Z", + "description": "`+reqBody.Description+`", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "`+reqBody.Name+`", + "num_cells": 0, + "secret": `+fmt.Sprintf("%t", reqBody.Secret)+`, + "status": "empty", + "updated_at": "2019-08-24T14:15:22Z", + "uploads": [ + { + "num_cells": 0, + "status": "empty", + "version": 0 + } + ] + } + } + }`) + } + + createdAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + updatedAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + + want := DLPDatasetCreateResult{ + MaxCells: 0, + Secret: "1234", + Version: 0, + Dataset: DLPDataset{ + CreatedAt: &createdAt, + Description: "string", + ID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", + Name: "string", + NumCells: 0, + Secret: true, + Status: "empty", + UpdatedAt: &updatedAt, + Uploads: []DLPDatasetUpload{ + { + NumCells: 0, + Status: "empty", + Version: 0, + }, + }, + }, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets", handler) + + actual, err := client.CreateDLPDataset(context.Background(), AccountIdentifier(testAccountID), DLPDatasetCreateParams{Description: "string", Name: "string", Secret: true}) + require.NoError(t, err) + require.Equal(t, want, actual) +} + +func TestDeleteDLPDataset(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodDelete, r.Method, "Expected method 'DELETE', got %s", r.Method) + w.Header().Set("content-type", "application/json") + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": null + }`) + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets/497f6eca-6276-4993-bfeb-53cbbbba6f08", handler) + + err := client.DeleteDLPDataset(context.Background(), AccountIdentifier(testAccountID), "497f6eca-6276-4993-bfeb-53cbbbba6f08") + require.NoError(t, err) +} + +func TestUpdateDLPDataset(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPut, r.Method, "Expected method 'PUT', got %s", r.Method) + w.Header().Set("content-type", "application/json") + + var reqBody DLPDatasetUpdateParams + err := json.NewDecoder(r.Body).Decode(&reqBody) + require.Nil(t, err) + + var description string + if reqBody.Description == nil { + description = "string" + } else { + description = *reqBody.Description + } + + var name string + if reqBody.Name == nil { + name = "string" + } else { + name = *reqBody.Name + } + + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": { + "created_at": "2019-08-24T14:15:22Z", + "description": "`+description+`", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "`+name+`", + "num_cells": 0, + "secret": true, + "status": "empty", + "updated_at": "2019-08-24T14:15:22Z", + "uploads": [ + { + "num_cells": 0, + "status": "empty", + "version": 0 + } + ] + } + }`) + } + + createdAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + updatedAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + + want := DLPDataset{ + CreatedAt: &createdAt, + Description: "new_desc", + ID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", + Name: "string", + NumCells: 0, + Secret: true, + Status: "empty", + UpdatedAt: &updatedAt, + Uploads: []DLPDatasetUpload{ + { + NumCells: 0, + Status: "empty", + Version: 0, + }, + }, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets/497f6eca-6276-4993-bfeb-53cbbbba6f08", handler) + + description := "new_desc" + actual, err := client.UpdateDLPDataset(context.Background(), AccountIdentifier(testAccountID), DLPDatasetUpdateParams{Description: &description, Name: nil, DatasetID: "497f6eca-6276-4993-bfeb-53cbbbba6f08"}) + require.NoError(t, err) + require.Equal(t, want, actual) +} + +func TestCreateDLPDatasetUpload(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method, "Expected method 'POST', got %s", r.Method) + w.Header().Set("content-type", "application/json") + + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": { + "max_cells": 0, + "secret": "1234", + "version": 1 + } + }`) + } + + want := DLPDatasetCreateUploadResult{ + MaxCells: 0, + Secret: "1234", + Version: 1, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets/497f6eca-6276-4993-bfeb-53cbbbba6f08/upload", handler) + + actual, err := client.CreateDLPDatasetUpload(context.Background(), AccountIdentifier(testAccountID), "497f6eca-6276-4993-bfeb-53cbbbba6f08") + require.NoError(t, err) + require.Equal(t, want, actual) +} + +func TestUploadDLPDatasetVersion(t *testing.T) { + setup() + defer teardown() + + handler := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method, "Expected method 'POST', got %s", r.Method) + w.Header().Set("content-type", "application/json") + + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, []byte{1, 2, 3, 4}, body) + + fmt.Fprintf(w, `{ + "success": true, + "errors": [], + "messages": [], + "result": { + "created_at": "2019-08-24T14:15:22Z", + "description": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "num_cells": 5, + "secret": true, + "status": "complete", + "updated_at": "2019-08-24T14:15:22Z", + "uploads": [ + { + "num_cells": 0, + "status": "empty", + "version": 0 + }, + { + "num_cells": 5, + "status": "complete", + "version": 1 + } + ] + } + }`) + } + + createdAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + updatedAt, _ := time.Parse(time.RFC3339, "2019-08-24T14:15:22Z") + + want := DLPDataset{ + CreatedAt: &createdAt, + Description: "string", + ID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", + Name: "string", + NumCells: 5, + Secret: true, + Status: "complete", + UpdatedAt: &updatedAt, + Uploads: []DLPDatasetUpload{ + { + NumCells: 0, + Status: "empty", + Version: 0, + }, + { + NumCells: 5, + Status: "complete", + Version: 1, + }, + }, + } + + mux.HandleFunc("/accounts/"+testAccountID+"/dlp/datasets/497f6eca-6276-4993-bfeb-53cbbbba6f08/upload/1", handler) + + actual, err := client.UploadDLPDatasetVersion(context.Background(), AccountIdentifier(testAccountID), DLPDatasetUploadVersionParams{DatasetID: "497f6eca-6276-4993-bfeb-53cbbbba6f08", Version: 1, Body: []byte{1, 2, 3, 4}}) + require.NoError(t, err) + require.Equal(t, want, actual) +}