diff --git a/google/cloud/forseti/common/gcp_api/_base_repository.py b/google/cloud/forseti/common/gcp_api/_base_repository.py index 6a21ea2784..90d72e3a03 100644 --- a/google/cloud/forseti/common/gcp_api/_base_repository.py +++ b/google/cloud/forseti/common/gcp_api/_base_repository.py @@ -94,8 +94,6 @@ def _create_service_api(credentials, service_name, version, is_private_api, if use_versioned_discovery_doc: service_json = '{}_{}.json'.format(service_name, version) - else: - service_json = '{}.json'.format(service_name) service_path = os.path.join(DISCOVERY_DOCS_BASE_DIR, service_json) return _build_service_from_document( diff --git a/google/cloud/forseti/common/gcp_api/discovery_documents/securitycenter.json b/google/cloud/forseti/common/gcp_api/discovery_documents/securitycenter.json deleted file mode 100644 index a9b49e5594..0000000000 --- a/google/cloud/forseti/common/gcp_api/discovery_documents/securitycenter.json +++ /dev/null @@ -1,621 +0,0 @@ -{ - "batchPath": "batch", - "title": "Cloud Security Command Center API", - "ownerName": "Google", - "resources": { - "organizations": { - "resources": { - "assets": { - "methods": { - "search": { - "description": "Search assets within an organization.", - "request": { - "$ref": "SearchAssetsRequest" - }, - "response": { - "$ref": "SearchAssetsResponse" - }, - "parameterOrder": [ - "orgName" - ], - "httpMethod": "POST", - "parameters": { - "orgName": { - "pattern": "^organizations/[^/]+$", - "location": "path", - "description": "Name of the organization to search for assets. Its format is\n\"organizations/[organization_id]\". For example, \"organizations/1234\".", - "required": true, - "type": "string" - } - }, - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], - "flatPath": "v1alpha3/organizations/{organizationsId}/assets:search", - "path": "v1alpha3/{+orgName}/assets:search", - "id": "securitycenter.organizations.assets.search" - }, - "modify": { - "description": "Modifies the marks on the specified asset.", - "request": { - "$ref": "ModifyAssetRequest" - }, - "response": { - "$ref": "Asset" - }, - "parameterOrder": [ - "orgName", - "id" - ], - "httpMethod": "POST", - "parameters": { - "orgName": { - "description": "Organization name.", - "required": true, - "type": "string", - "pattern": "^organizations/[^/]+$", - "location": "path" - }, - "id": { - "description": "Unique identifier for the asset to be modified.", - "required": true, - "type": "string", - "location": "path" - } - }, - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], - "flatPath": "v1alpha3/organizations/{organizationsId}/assets/{id}:modify", - "path": "v1alpha3/{+orgName}/assets/{id}:modify", - "id": "securitycenter.organizations.assets.modify" - } - } - }, - "findings": { - "methods": { - "create": { - "response": { - "$ref": "Finding" - }, - "parameterOrder": [ - "orgName" - ], - "httpMethod": "POST", - "parameters": { - "orgName": { - "description": "Name of the organization to search for assets. Its format is\n\"organizations/[organization_id]\". For example, \"organizations/1234\".", - "required": true, - "type": "string", - "pattern": "^organizations/[^/]+$", - "location": "path" - } - }, - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], - "flatPath": "v1alpha3/organizations/{organizationsId}/findings", - "path": "v1alpha3/{+orgName}/findings", - "id": "securitycenter.organizations.findings.create", - "description": "Creates a finding. Creating the same finding with a later event_time will\nupdate the existing one. Cloud SCC provides the capability for users to\nsearch findings based on timestamps.", - "request": { - "$ref": "CreateFindingRequest" - } - }, - "search": { - "description": "Search findings within an organization.", - "request": { - "$ref": "SearchFindingsRequest" - }, - "httpMethod": "POST", - "parameterOrder": [ - "orgName" - ], - "response": { - "$ref": "SearchFindingsResponse" - }, - "parameters": { - "orgName": { - "location": "path", - "description": "The name of the organization to which the findings belong. Its format is\n\"organizations/[organization_id]\". For example, \"organizations/1234\".", - "required": true, - "type": "string", - "pattern": "^organizations/[^/]+$" - } - }, - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], - "flatPath": "v1alpha3/organizations/{organizationsId}/findings:search", - "id": "securitycenter.organizations.findings.search", - "path": "v1alpha3/{+orgName}/findings:search" - }, - "modify": { - "description": "Modifies the changeable parts of the specified finding.", - "request": { - "$ref": "ModifyFindingRequest" - }, - "response": { - "$ref": "Finding" - }, - "parameterOrder": [ - "orgName", - "id" - ], - "httpMethod": "POST", - "parameters": { - "id": { - "location": "path", - "description": "ID of the finding.", - "required": true, - "type": "string" - }, - "orgName": { - "description": "Organization name.", - "required": true, - "type": "string", - "pattern": "^organizations/[^/]+$", - "location": "path" - } - }, - "scopes": [ - "https://www.googleapis.com/auth/cloud-platform" - ], - "flatPath": "v1alpha3/organizations/{organizationsId}/findings/{id}:modify", - "path": "v1alpha3/{+orgName}/findings/{id}:modify", - "id": "securitycenter.organizations.findings.modify" - } - } - } - } - } - }, - "parameters": { - "key": { - "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", - "type": "string", - "location": "query" - }, - "access_token": { - "location": "query", - "description": "OAuth access token.", - "type": "string" - }, - "upload_protocol": { - "location": "query", - "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", - "type": "string" - }, - "prettyPrint": { - "location": "query", - "description": "Returns response with indentations and line breaks.", - "type": "boolean", - "default": "true" - }, - "quotaUser": { - "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", - "type": "string", - "location": "query" - }, - "uploadType": { - "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", - "type": "string", - "location": "query" - }, - "fields": { - "location": "query", - "description": "Selector specifying which fields to include in a partial response.", - "type": "string" - }, - "$.xgafv": { - "enumDescriptions": [ - "v1 error format", - "v2 error format" - ], - "location": "query", - "enum": [ - "1", - "2" - ], - "description": "V1 error format.", - "type": "string" - }, - "oauth_token": { - "description": "OAuth 2.0 token for the current user.", - "type": "string", - "location": "query" - }, - "callback": { - "description": "JSONP", - "type": "string", - "location": "query" - }, - "alt": { - "enum": [ - "json", - "media", - "proto" - ], - "type": "string", - "enumDescriptions": [ - "Responses with Content-Type of application/json", - "Media download with context-dependent Content-Type", - "Responses with Content-Type of application/x-protobuf" - ], - "location": "query", - "description": "Data format for response.", - "default": "json" - } - }, - "version": "v1alpha3", - "baseUrl": "https://securitycenter.googleapis.com/", - "kind": "discovery#restDescription", - "description": "The public Cloud Security Command Center API.", - "servicePath": "", - "basePath": "", - "id": "securitycenter:v1alpha3", - "revision": "20180625", - "documentationLink": "https://console.cloud.google.com/apis/api/securitycenter.googleapis.com/overview", - "discoveryVersion": "v1", - "version_module": true, - "schemas": { - "SourceFinding": { - "description": "Representation of a source's finding.", - "type": "object", - "properties": { - "properties": { - "description": "`Key: value` pairs provided by the source. Indexing will be provided\nfor each key.", - "type": "object", - "additionalProperties": { - "description": "Properties of the object.", - "type": "any" - } - }, - "url": { - "description": "An https url provided by the source for users to click on to see more\ninformation, used for UI navigation.", - "type": "string" - }, - "marks": { - "additionalProperties": { - "type": "string" - }, - "description": "User specified marks placed on the finding.\nNote: This field is used in responses only. Any value specified here in a\nrequest is ignored.", - "type": "object" - }, - "attributes": { - "description": "Dynamically calculated attributes provided by Google.\nFor example, first_discovered, create_time.\nNote: This field is used in responses only. Any value specified here in a\nrequest is ignored.", - "type": "object", - "additionalProperties": { - "description": "Properties of the object.", - "type": "any" - } - }, - "id": { - "description": "Required field, an organization & source unique immutable identifier\nprovided by the sources.", - "type": "string" - }, - "category": { - "description": "Required field, category of a finding, this is a custom string field. In\ngeneral, sources have logical groupings for their findings. For example,\nthe Data Loss Prevention Scanner has the following types: \"SSN\",\n\"US passport\", \"credit card number\", and so on. This field is indexed and\nused by Cloud SCC frontend for data visualization. It's also useful for\nreporting and analysis. It's recommended to populate this field\nconsistently.", - "type": "string" - }, - "sourceId": { - "description": "Required field, ID of the finding source. A source is a producer of\nsecurity findings and source IDs are namespaced under each organization.\nFor Google integrated sources, please use their official source IDs for\nbetter frontend integration. For custom sources, choose an ID that doesn't\nconflict with any existing IDs.", - "type": "string" - }, - "eventTime": { - "description": "Time when the finding was generated by the source.", - "format": "google-datetime", - "type": "string" - }, - "assetIds": { - "description": "Required field, list of IDs of affected assets. These IDs should strictly\nmap to one of the existing asset IDs in the asset inventory for the\norganization, which is populated by Cloud SCC backend. If the ID doesn't\nmap to one of the existing asset IDs, the result is a NOT_FOUND error.\nAsset types must be supported by Cloud SCC. It's best to pick the most\ngranular asset type. For example, if a file in a VM instance is affected,\npick the asset ID of the VM instance. This works best because file isn't a\nsupported asset type in Cloud SCC, and project ID is too broad.", - "type": "array", - "items": { - "type": "string" - } - } - }, - "id": "SourceFinding" - }, - "ModifyAssetRequest": { - "description": "Request message for modifying the marks on an asset.", - "type": "object", - "properties": { - "removeMarksWithKeys": { - "description": "A list of keys defining the marks to remove from the asset. There can be no\noverlaps between keys to remove and keys to add or update.", - "type": "array", - "items": { - "type": "string" - } - }, - "addOrUpdateMarks": { - "additionalProperties": { - "type": "string" - }, - "description": "Keys and values to add/update on the asset.\n\nIf a mark with the same key already exists, its value will be replaced by\nthe updated value.", - "type": "object" - } - }, - "id": "ModifyAssetRequest" - }, - "SearchFindingsRequest": { - "properties": { - "pageSize": { - "description": "The maximum number of results to return.", - "format": "int32", - "type": "integer" - }, - "referenceTime": { - "description": "The reference point used to determine the findings at a specific\npoint in time.\nQueries with the timestamp in the future are rounded down to the\ncurrent time on the server. If the value is not given, \"now\" is going to\nbe used implicitly.", - "format": "google-datetime", - "type": "string" - }, - "query": { - "description": "Expression that defines the query to apply across findings.\nThe expression is a list of one or more restrictions combined via logical\noperators `AND` and `OR`.\nParentheses are supported, and in absence of parentheses `OR` has higher\nprecedence than `AND`.\n\nRestrictions have the form `\u003cfield\u003e \u003coperator\u003e \u003cvalue\u003e` and may have a `-`\ncharacter in front of them to indicate negation. The fields can be of the\nfollowing types:\n\n* Attribute - optional `attribute.` prefix or no prefix and name.\n* Property - mandatory `property.` prefix and name.\n* Mark - mandatory `mark` prefix and name.\n\nThe supported operators are:\n\n* `=` for all value types.\n* `\u003e`, `\u003c`, `\u003e=`, `\u003c=` for integer values.\n* `:`, meaning substring matching, for strings.\n\nThe supported value types are:\n\n* string literals in quotes.\n* integer literals without quotes.\n* boolean literals `true` and `false` without quotes.\n\nFor example, `property.count = 100` is a valid query string.", - "type": "string" - }, - "orderBy": { - "description": "Expression that defines what fields and order to use for sorting.", - "type": "string" - }, - "pageToken": { - "description": "Optional pagination token returned in an earlier call.", - "type": "string" - } - }, - "id": "SearchFindingsRequest", - "description": "Request message for SearchFindings.", - "type": "object" - }, - "SearchFindingsResponse": { - "description": "Response message for SearchFindings.", - "type": "object", - "properties": { - "nextPageToken": { - "description": "Token to retrieve the next page of results, or empty if there are no more\nresults.", - "type": "string" - }, - "totalSize": { - "description": "The total number of findings irrespective of pagination.", - "format": "int32", - "type": "integer" - }, - "findings": { - "description": "Findings returned by the request.", - "type": "array", - "items": { - "$ref": "Finding" - } - }, - "referenceTime": { - "description": "Time provided for reference_time in the request.", - "format": "google-datetime", - "type": "string" - } - }, - "id": "SearchFindingsResponse" - }, - "SearchAssetsResponse": { - "description": "Response message for SearchAssets.", - "type": "object", - "properties": { - "nextPageToken": { - "description": "Token to retrieve the next page of results, or empty if there are no more\nresults.", - "type": "string" - }, - "assets": { - "description": "Assets returned by the request.", - "type": "array", - "items": { - "$ref": "Asset" - } - }, - "totalSize": { - "description": "The total number of results available.", - "format": "uint64", - "type": "string" - }, - "referenceTime": { - "description": "Time provided for reference_time in the request.", - "format": "google-datetime", - "type": "string" - }, - "compareDuration": { - "description": "Time provided for compare_duration in the request.", - "format": "google-duration", - "type": "string" - } - }, - "id": "SearchAssetsResponse" - }, - "CreateFindingRequest": { - "description": "Request message for CreatingFinding.", - "type": "object", - "properties": { - "sourceFinding": { - "description": "The source finding to be created.", - "$ref": "SourceFinding" - } - }, - "id": "CreateFindingRequest" - }, - "Finding": { - "properties": { - "properties": { - "additionalProperties": { - "description": "Properties of the object.", - "type": "any" - }, - "description": "Properties associated with the finding.", - "type": "object" - }, - "marks": { - "additionalProperties": { - "type": "string" - }, - "description": "User specified marks placed on the finding.", - "type": "object" - }, - "scannerId": { - "description": "Unique identifier for the scanner that produced the finding.", - "type": "string" - }, - "updateTime": { - "description": "Time at which this finding was last updated. This does not include updates\non user specified marks.", - "format": "google-datetime", - "type": "string" - }, - "assetId": { - "description": "Unique identifier for the asset the finding relates to.", - "type": "string" - }, - "id": { - "description": "Unique identifier for this finding. The same finding and identifier can\nappear at multiple points in time.", - "type": "string" - } - }, - "id": "Finding", - "description": "Representation of a scanner's finding.", - "type": "object" - }, - "ModifyFindingRequest": { - "description": "Request message for ModifyFinding.", - "type": "object", - "properties": { - "addOrUpdateMarks": { - "description": "Keys and values to add or update on the finding.\nIf a mark with the same key already exists, its value will be replaced by\nthe updated value.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "removeMarksWithKeys": { - "description": "A list of keys defining the marks to remove from the finding. There can be\nno overlaps between keys to remove and keys to add or update.", - "type": "array", - "items": { - "type": "string" - } - } - }, - "id": "ModifyFindingRequest" - }, - "SearchAssetsRequest": { - "description": "Request message for SearchAssets.", - "type": "object", - "properties": { - "orderBy": { - "description": "Expression that defines what fields and order to use for sorting.", - "type": "string" - }, - "compareDuration": { - "description": "When compare_duration is set, the Asset's \"state\" attribute is updated to\nindicate whether the asset was added, removed, or remained present during\nthe compare_duration period of time that precedes the reference_time. This\nis the time between (reference_time - compare_duration) and reference_time.\n\nThe state value is derived based on the presence of the asset at the two\npoints in time. Intermediate state changes between the two times don't\naffect the result. For example, the results aren't affected if the asset is\nremoved and re-created again.\n\nPossible \"state\" values when compare_duration is specified:\n\n* \"ADDED\": indicates that the asset was not present before\n compare_duration, but present at reference_time.\n* \"REMOVED\": indicates that the asset was present at the start of\n compare_duration, but not present at reference_time.\n* \"ACTIVE_AT_BOTH\": indicates that the asset was present at both the\n start and the end of the time period defined by\n compare_duration and reference_time.\n\nIf compare_duration is not specified, then the only possible state is\n\"ACTIVE\", which indicates that the asset is present at reference_time.", - "format": "google-duration", - "type": "string" - }, - "pageToken": { - "description": "Optional pagination token returned in an earlier call.", - "type": "string" - }, - "pageSize": { - "description": "The maximum number of results to return in a single response.", - "format": "int32", - "type": "integer" - }, - "referenceTime": { - "description": "Time at which to search for assets. The search will capture the state of\nassets at this point in time.\n\nNot providing a value or providing one in the future is treated as current.", - "format": "google-datetime", - "type": "string" - }, - "query": { - "description": "Expression that defines the query to apply across assets.\nThe expression is a list of one or more restrictions combined via logical\noperators `AND` and `OR`.\nParentheses are not supported, and `OR` has higher precedence than `AND`.\n\nRestrictions have the form `\u003cfield\u003e \u003coperator\u003e \u003cvalue\u003e` and may have a `-`\ncharacter in front of them to indicate negation. The fields can be of the\nfollowing types:\n\n* Attribute: optional `attribute.` prefix or no prefix and name.\n* Property: mandatory `property.` prefix and name.\n* Mark: mandatory `mark` prefix and name.\n\nThe supported operators are:\n\n* `=` for all value types.\n* `\u003e`, `\u003c`, `\u003e=`, `\u003c=` for integer values.\n* `:`, meaning substring matching, for strings.\n\nThe supported value types are:\n\n* string literals in quotes.\n* integer literals without quotes.\n* boolean literals `true` and `false` without quotes.\n\nFor example, `property.count = 100` is a valid query string.", - "type": "string" - } - }, - "id": "SearchAssetsRequest" - }, - "Asset": { - "description": "Representation of a Google Cloud Platform asset. Examples of assets include:\nApp Engine Application, Project, Bucket, and so on.", - "type": "object", - "properties": { - "properties": { - "description": "Properties associated with the asset.", - "type": "object", - "additionalProperties": { - "description": "Properties of the object.", - "type": "any" - } - }, - "parentId": { - "description": "Unique identifier for this asset's parent. For example, a Project's parent\nwould be either the organization it belongs to OR the folder it resides in.", - "type": "string" - }, - "marks": { - "additionalProperties": { - "type": "string" - }, - "description": "User specified marks placed on the asset.", - "type": "object" - }, - "state": { - "enumDescriptions": [ - "Invalid state.", - "Asset was active for the point in time.", - "Asset was not found for the point(s) in time.", - "Asset was added between the points in time.", - "Asset was removed between the points in time.", - "Asset was active at both point(s) in time." - ], - "enum": [ - "STATE_UNSPECIFIED", - "ACTIVE", - "NOT_FOUND", - "ADDED", - "REMOVED", - "ACTIVE_AT_BOTH" - ], - "description": "State of the asset.", - "type": "string" - }, - "updateTime": { - "description": "The time at which the asset was last updated, added, or deleted in\nCloud SCC.", - "format": "google-datetime", - "type": "string" - }, - "type": { - "description": "The type of asset. Examples include: APPLICATION, PROJECT, and\nORGANIZATION.", - "type": "string" - }, - "owners": { - "description": "Owners of the asset. Commonly represented as email addresses.", - "type": "array", - "items": { - "type": "string" - } - }, - "id": { - "description": "Unique identifier for this asset. This identifier persists following\nmodification, deletion, or recreation.", - "type": "string" - } - }, - "id": "Asset" - } - }, - "icons": { - "x16": "http://www.google.com/images/icons/product/search-16.gif", - "x32": "http://www.google.com/images/icons/product/search-32.gif" - }, - "protocol": "rest", - "canonicalName": "Security Command Center", - "auth": { - "oauth2": { - "scopes": { - "https://www.googleapis.com/auth/cloud-platform": { - "description": "View and manage your data across Google Cloud Platform services" - } - } - } - }, - "rootUrl": "https://securitycenter.googleapis.com/", - "ownerDomain": "google.com", - "name": "securitycenter" -} diff --git a/google/cloud/forseti/common/gcp_api/securitycenter.py b/google/cloud/forseti/common/gcp_api/securitycenter.py index e6ce57ed63..2af0a098dd 100644 --- a/google/cloud/forseti/common/gcp_api/securitycenter.py +++ b/google/cloud/forseti/common/gcp_api/securitycenter.py @@ -49,10 +49,7 @@ def __init__(self, self._findings = None self.version = version - use_versioned_discovery_doc = False - if self.version == 'v1beta1': - use_versioned_discovery_doc = True - # alpha would use the discovery_doc without version in the name + use_versioned_discovery_doc = True super(SecurityCenterRepositoryClient, self).__init__( API_NAME, versions=[self.version], @@ -91,13 +88,7 @@ def __init__(self, **kwargs): LOGGER.debug( 'Creating _SecurityCenterOrganizationsFindingsRepositoryClient') - # pylint: disable=protected-access - if kwargs.get('gcp_service')._resourceDesc.get('version') == 'v1beta1': - component = 'organizations.sources.findings' - else: - # alpha api - component = 'organizations.findings' - # pylint: enable=protected-access + component = 'organizations.sources.findings' super(_SecurityCenterOrganizationsFindingsRepository, self).__init__( key_field='name', @@ -126,13 +117,11 @@ def __init__(self, version=None): version) self.repository = SecurityCenterRepositoryClient(version=version) - def create_finding(self, finding, organization_id=None, source_id=None, - finding_id=None): + def create_finding(self, finding, source_id=None, finding_id=None): """Creates a finding in CSCC. Args: finding (dict): Forseti violation in CSCC format. - organization_id (str): The id prefixed with 'organizations/'. source_id (str): Unique ID assigned by CSCC, to the organization that the violations are originating from. finding_id (str): id hash of the CSCC finding @@ -140,47 +129,23 @@ def create_finding(self, finding, organization_id=None, source_id=None, Returns: dict: An API response containing one page of results. """ - if source_id: - # beta api - try: - LOGGER.debug('Creating finding with beta api.') - - # patch() will also create findings for new violations. - response = self.repository.findings.patch( - '{}/findings/{}'.format(source_id, finding_id), - finding - ) - LOGGER.debug('Created finding response with CSCC beta api: %s', - response) - return response - # handle 409, finding exists - except (errors.HttpError, HttpLib2Error) as e: - LOGGER.exception( - 'Unable to create CSCC finding: Resource: %s', finding) - violation_data = ( - finding.get('source_properties').get('violation_data')) - raise api_errors.ApiExecutionError(violation_data, e) - - # alpha api try: - LOGGER.debug('Creating finding with alpha api.') + LOGGER.debug('Creating finding with beta api.') - response = self.repository.findings.create( - arguments={ - 'body': {'sourceFinding': finding}, - 'orgName': organization_id - } + # patch() will also create findings for new violations. + response = self.repository.findings.patch( + '{}/findings/{}'.format(source_id, finding_id), + finding ) - LOGGER.debug('Created finding response with CSCC alpha: %s', + LOGGER.debug('Created finding response with CSCC beta api: %s', response) return response except (errors.HttpError, HttpLib2Error) as e: LOGGER.exception( 'Unable to create CSCC finding: Resource: %s', finding) - full_name = ( - finding.get('properties').get('violation_data') - .get('full_name')) - raise api_errors.ApiExecutionError(full_name, e) + violation_data = ( + finding.get('source_properties').get('violation_data')) + raise api_errors.ApiExecutionError(violation_data, e) def list_findings(self, source_id): """Lists all the findings in CSCC. @@ -207,10 +172,6 @@ def update_finding(self, finding, finding_id, source_id=None): Returns: dict: An API response containing one page of results. """ - # alpha api - if not source_id: - return {} - # beta api try: LOGGER.debug('Updated finding with beta api.') diff --git a/google/cloud/forseti/notifier/notifier.py b/google/cloud/forseti/notifier/notifier.py index da7f20ef5a..07d2e84311 100644 --- a/google/cloud/forseti/notifier/notifier.py +++ b/google/cloud/forseti/notifier/notifier.py @@ -76,7 +76,7 @@ def convert_to_timestamp(violations): return violations -# pylint: disable=too-many-branches,too-many-statements +# pylint: disable=too-many-branches def run(inventory_index_id, scanner_index_id, progress_queue, @@ -166,25 +166,12 @@ def run(inventory_index_id, if violation_configs: if violation_configs.get('cscc').get('enabled'): source_id = violation_configs.get('cscc').get('source_id') - if source_id: - # beta mode - LOGGER.debug( - 'Running CSCC notifier with beta API. source_id: ' - '%s', source_id) - (cscc_notifier.CsccNotifier(inventory_index_id) - .run(violations_as_dict, source_id=source_id)) - else: - # alpha mode - LOGGER.debug('Running CSCC notifier with alpha API.') - gcs_path = ( - violation_configs.get('cscc').get('gcs_path')) - mode = violation_configs.get('cscc').get('mode') - organization_id = ( - violation_configs.get('cscc').get( - 'organization_id')) - (cscc_notifier.CsccNotifier(inventory_index_id) - .run(violations_as_dict, gcs_path, mode, - organization_id)) + # beta mode + LOGGER.debug( + 'Running CSCC notifier with beta API. source_id: ' + '%s', source_id) + (cscc_notifier.CsccNotifier(inventory_index_id) + .run(violations_as_dict, source_id=source_id)) InventorySummary(service_config, inventory_index_id).run() diff --git a/google/cloud/forseti/notifier/notifiers/cscc_notifier.py b/google/cloud/forseti/notifier/notifiers/cscc_notifier.py index c2f2ff8e65..6fa3b24fb3 100644 --- a/google/cloud/forseti/notifier/notifiers/cscc_notifier.py +++ b/google/cloud/forseti/notifier/notifiers/cscc_notifier.py @@ -125,67 +125,38 @@ def _transform_for_api(self, violations, source_id=None): findings = [] # beta api - if source_id: - LOGGER.debug('Transforming findings with beta API. source_id: %s', - source_id) - for violation in violations: - # CSCC can't accept the full hash, so this must be shortened. - finding_id = violation.get('violation_hash')[:32] - finding = { - 'name': '{0}/findings/{1}'.format( - source_id, violation.get('violation_hash')[:32]), - 'parent': source_id, - 'resource_name': violation.get('full_name'), - 'state': 'ACTIVE', - 'category': violation.get('violation_type'), - 'event_time': violation.get('created_at_datetime'), - 'source_properties': { - 'source': 'FORSETI', - 'db_source': 'table:{}/id:{}'.format( - 'violations', violation.get('id')), - 'inventory_index_id': self.inv_index_id, - 'resource_data': ( - json.dumps(violation.get('resource_data'), - sort_keys=True)), - 'resource_id': violation.get('resource_id'), - 'resource_type': violation.get('resource_type'), - 'rule_index': violation.get('rule_index'), - 'rule_name': violation.get('rule_name'), - 'scanner_index_id': violation.get('scanner_index_id'), - 'violation_data': ( - json.dumps(violation.get('violation_data'), - sort_keys=True)) - }, - } - findings.append([finding_id, finding]) - return findings - - # alpha api LOGGER.debug('Transforming findings with beta API. source_id: %s', source_id) for violation in violations: + # CSCC can't accept the full hash, so this must be shortened. + finding_id = violation.get('violation_hash')[:32] finding = { - # CSCC can't accept the full hash, so this must be shortened. - 'id': violation.get('violation_hash')[:32], - 'assetIds': [ - violation.get('full_name') - ], - 'eventTime': violation.get('created_at_datetime'), - 'properties': { + 'name': '{0}/findings/{1}'.format( + source_id, violation.get('violation_hash')[:32]), + 'parent': source_id, + 'resource_name': violation.get('full_name'), + 'state': 'ACTIVE', + 'category': violation.get('violation_type'), + 'event_time': violation.get('created_at_datetime'), + 'source_properties': { + 'source': 'FORSETI', 'db_source': 'table:{}/id:{}'.format( 'violations', violation.get('id')), 'inventory_index_id': self.inv_index_id, - 'resource_data': violation.get('resource_data'), + 'resource_data': ( + json.dumps(violation.get('resource_data'), + sort_keys=True)), 'resource_id': violation.get('resource_id'), 'resource_type': violation.get('resource_type'), 'rule_index': violation.get('rule_index'), + 'rule_name': violation.get('rule_name'), 'scanner_index_id': violation.get('scanner_index_id'), - 'violation_data': violation.get('violation_data') + 'violation_data': ( + json.dumps(violation.get('violation_data'), + sort_keys=True)) }, - 'source_id': 'FORSETI', - 'category': violation.get('rule_name') } - findings.append(finding) + findings.append([finding_id, finding]) return findings @staticmethod @@ -226,98 +197,75 @@ def find_inactive_findings(new_findings, findings_in_cscc): return inactive_findings # pylint: disable=too-many-locals - def _send_findings_to_cscc(self, violations, organization_id=None, - source_id=None): + def _send_findings_to_cscc(self, violations, source_id=None): """Send violations to CSCC directly via the CSCC API. Args: violations (dict): Violations to be uploaded as findings. - organization_id (str): The id prefixed with 'organizations/'. source_id (str): Unique ID assigned by CSCC, to the organization that the violations are originating from. """ # beta api - if source_id: - formatted_cscc_findings = [] - LOGGER.debug('Sending findings to CSCC with beta API. source_id: ' - '%s', source_id) - new_findings = self._transform_for_api(violations, - source_id=source_id) - - client = securitycenter.SecurityCenterClient(version='v1beta1') - - paged_findings_in_cscc = client.list_findings(source_id=source_id) - - # No need to use the next page token, as the results here will - # return all the pages. - for page in paged_findings_in_cscc: - formated_findings_in_page = ( - ast.literal_eval(json.dumps(page))) - findings_in_page = formated_findings_in_page.get('findings') - for finding_data in findings_in_page: - name = finding_data.get('name') - finding_id = name[-32:] - formatted_cscc_findings.append([finding_id, finding_data]) - - inactive_findings = self.find_inactive_findings( - new_findings, - formatted_cscc_findings) - - for finding_list in new_findings: - finding_id = finding_list[0] - finding = finding_list[1] - LOGGER.debug('Creating finding CSCC:\n%s.', finding) - try: - client.create_finding(finding, source_id=source_id, - finding_id=finding_id) - LOGGER.debug('Successfully created finding in CSCC:\n%s', - finding) - except api_errors.ApiExecutionError: - LOGGER.exception('Encountered CSCC API error.') - continue - - for finding_list in inactive_findings: - finding_id = finding_list[0] - finding = finding_list[1] - LOGGER.debug('Updating finding CSCC:\n%s.', finding) - try: - client.update_finding(finding, - finding_id, - source_id=source_id) - LOGGER.debug('Successfully updated finding in CSCC:\n%s', - finding) - except api_errors.ApiExecutionError: - LOGGER.exception('Encountered CSCC API error.') - continue - - return - - # alpha api - LOGGER.debug('Sending findings to CSCC with alpha API.') - findings = self._transform_for_api(violations) - - client = securitycenter.SecurityCenterClient() - - for finding in findings: + formatted_cscc_findings = [] + LOGGER.debug('Sending findings to CSCC with beta API. source_id: ' + '%s', source_id) + new_findings = self._transform_for_api(violations, + source_id=source_id) + + client = securitycenter.SecurityCenterClient(version='v1beta1') + + paged_findings_in_cscc = client.list_findings(source_id=source_id) + + # No need to use the next page token, as the results here will + # return all the pages. + for page in paged_findings_in_cscc: + formated_findings_in_page = ( + ast.literal_eval(json.dumps(page))) + findings_in_page = formated_findings_in_page.get('findings') + for finding_data in findings_in_page: + name = finding_data.get('name') + finding_id = name[-32:] + formatted_cscc_findings.append([finding_id, finding_data]) + + inactive_findings = self.find_inactive_findings( + new_findings, + formatted_cscc_findings) + + for finding_list in new_findings: + finding_id = finding_list[0] + finding = finding_list[1] LOGGER.debug('Creating finding CSCC:\n%s.', finding) try: - client.create_finding(finding, organization_id=organization_id) + client.create_finding(finding, source_id=source_id, + finding_id=finding_id) LOGGER.debug('Successfully created finding in CSCC:\n%s', finding) except api_errors.ApiExecutionError: LOGGER.exception('Encountered CSCC API error.') continue - def run(self, violations, gcs_path=None, mode=None, organization_id=None, - source_id=None): + for finding_list in inactive_findings: + finding_id = finding_list[0] + finding = finding_list[1] + LOGGER.debug('Updating finding CSCC:\n%s.', finding) + try: + client.update_finding(finding, + finding_id, + source_id=source_id) + LOGGER.debug('Successfully updated finding in CSCC:\n%s', + finding) + except api_errors.ApiExecutionError: + LOGGER.exception('Encountered CSCC API error.') + continue + + return + + def run(self, violations, source_id=None): """Generate the temporary json file and upload to GCS. Args: violations (dict): Violations to be uploaded as findings. - gcs_path (str): The GCS bucket to upload the findings. - mode (str): The mode in which to send the CSCC notification. - organization_id (str): The id of the organization. source_id (str): Unique ID assigned by CSCC, to the organization that the violations are originating from. """ @@ -327,18 +275,6 @@ def run(self, violations, gcs_path=None, mode=None, organization_id=None, # At this point, cscc notifier is already determined to be enabled. # beta api - if source_id: - LOGGER.debug('Running CSCC with beta API. source_id: %s', source_id) - self._send_findings_to_cscc(violations, source_id=source_id) - return - - # alpha api - LOGGER.debug('Running CSCC with alpha API.') - if mode is None or mode == 'bucket': - self._send_findings_to_gcs(violations, gcs_path) - elif mode == 'api': - self._send_findings_to_cscc(violations, organization_id) - else: - LOGGER.info( - 'A valid mode for CSCC notification was not selected: %s. ' - 'Please use either "bucket" or "api" mode.', mode) + LOGGER.debug('Running CSCC with beta API. source_id: %s', source_id) + self._send_findings_to_cscc(violations, source_id=source_id) + return diff --git a/tests/common/gcp_api/securitycenter_test.py b/tests/common/gcp_api/securitycenter_test.py index 82c47e0eb2..025ded6fdf 100644 --- a/tests/common/gcp_api/securitycenter_test.py +++ b/tests/common/gcp_api/securitycenter_test.py @@ -38,34 +38,18 @@ def setUpClass(cls, mock_google_credential): """Set up.""" fake_global_configs = { 'securitycenter': {'max_calls': 1, 'period': 1.1}} - # securitycenter_alpha_api_client is alpha api client - cls.securitycenter_alpha_api_client = securitycenter.SecurityCenterClient() cls.securitycenter_beta_api_client = securitycenter.SecurityCenterClient(version='v1beta1') cls.project_id = 111111 cls.source_id = 'organizations/111/sources/222' - @mock.patch.object( - google.auth, 'default', - return_value=(mock.Mock(spec_set=credentials.Credentials), - 'test-project')) - def test_no_quota(self, mock_google_credential): - """Verify no rate limiter is used if the configuration is missing.""" - securitycenter_alpha_api_client = securitycenter.SecurityCenterClient() - self.assertEqual(None, securitycenter_alpha_api_client.repository._rate_limiter) - def test_create_findings(self): """Test create cscc findings.""" http_mocks.mock_http_response( json.dumps(fake_cscc.EXPECTED_CREATE_FINDING_RESULT)) - result = self.securitycenter_alpha_api_client.create_finding( - fake_cscc.FAKE_ALPHA_FINDING, fake_cscc.ORGANIZATION_ID) - self.assertEquals(fake_cscc.EXPECTED_CREATE_FINDING_RESULT, result) - result = self.securitycenter_beta_api_client.create_finding( 'fake finding', - fake_cscc.ORGANIZATION_ID, source_id=self.source_id ) self.assertEquals(fake_cscc.EXPECTED_CREATE_FINDING_RESULT, result) @@ -74,18 +58,11 @@ def test_create_findings_raises(self): """Test create cscc finding raises exception.""" http_mocks.mock_http_response(fake_cscc.PERMISSION_DENIED, '403') - # alpha api - with self.assertRaises(api_errors.ApiExecutionError): - self.securitycenter_alpha_api_client.create_finding( - json.loads(fake_cscc.FAKE_ALPHA_FINDING), - fake_cscc.ORGANIZATION_ID) - # beta api fake_beta_finding = {'source_properties': {'violation_data': 'foo'}} with self.assertRaises(api_errors.ApiExecutionError): self.securitycenter_beta_api_client.create_finding( fake_beta_finding, - fake_cscc.ORGANIZATION_ID, source_id=self.source_id) diff --git a/tests/notifier/notifiers/cscc_notifier_test.py b/tests/notifier/notifiers/cscc_notifier_test.py index d9b4d8c6bd..8ce6b6f8e7 100644 --- a/tests/notifier/notifiers/cscc_notifier_test.py +++ b/tests/notifier/notifiers/cscc_notifier_test.py @@ -30,7 +30,7 @@ class CsccNotifierTest(scanner_base_db.ScannerBaseDbTestCase): def setUp(self): """Setup method.""" super(CsccNotifierTest, self).setUp() - self.maxDiff=None + self.maxDiff = None def tearDown(self): """Tear down method.""" @@ -106,47 +106,6 @@ def test_can_transform_to_beta_findings_in_api_mode(self): self.assertEquals(expected_beta_findings, ast.literal_eval(json.dumps(finding_results))) - def test_can_transform_to_alpha_findings_in_api_mode(self): - - expected_alpha_findings = [ - {'assetIds': ['full_name_111'], - 'category': 'disallow_all_ports_111', - 'eventTime': '2010-08-28T10:20:30Z', - 'id': '539cfbdb1113a74ec18edf583eada77a', - 'properties': { - 'db_source': 'table:violations/id:1', - 'inventory_index_id': 'iii', - 'resource_data': 'inventory_data_111', - 'resource_id': 'fake_firewall_111', - 'resource_type': 'firewall_rule', - 'rule_index': 111, - 'scanner_index_id': 1282990830000000, - 'violation_data': '{"policy_names": ["fw-tag-match_111"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_111"]}}'}, - 'source_id': 'FORSETI'}, - {'assetIds': ['full_name_222'], - 'category': 'disallow_all_ports_222', - 'eventTime': '2010-08-28T10:20:30Z', - 'id': '3eff279ccb96799d9eb18e6b76055b22', - 'properties': { - 'db_source': 'table:violations/id:2', - 'inventory_index_id': 'iii', - 'resource_data': 'inventory_data_222', - 'resource_id': 'fake_firewall_222', - 'resource_type': 'firewall_rule', - 'rule_index': 222, - 'scanner_index_id': 1282990830000000, - 'violation_data': '{"policy_names": ["fw-tag-match_222"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_222"]}}'}, - 'source_id': 'FORSETI'}] - - violations_as_dict = self._populate_and_retrieve_violations() - - finding_results = ( - cscc_notifier.CsccNotifier('iii')._transform_for_api( - violations_as_dict) - ) - - self.assertEquals(expected_alpha_findings, finding_results) - @mock.patch('google.cloud.forseti.common.util.date_time.' 'get_utc_now_datetime') def test_can_transform_to_findings_in_bucket_mode(self, mock_get_utc_now): @@ -194,36 +153,6 @@ def test_can_transform_to_findings_in_bucket_mode(self, mock_get_utc_now): ) self.assertEquals(expected_findings, finding_results) - - @mock.patch('google.cloud.forseti.notifier.notifiers.cscc_notifier.LOGGER') - def test_modes_are_run_correctly(self, mock_logger): - - # This whole test case is for alpha API, and can be deleted - # when CSCC alpha support is removed. - - notifier = cscc_notifier.CsccNotifier(None) - - notifier._send_findings_to_gcs = mock.MagicMock() - notifier._send_findings_to_cscc = mock.MagicMock() - notifier.LOGGER = mock.MagicMock() - - self.assertEquals(0, notifier._send_findings_to_gcs.call_count) - notifier.run(None, None, None, None) - self.assertEquals(1, notifier._send_findings_to_gcs.call_count) - - notifier.run(None, None, 'bucket', None) - self.assertEquals(2, notifier._send_findings_to_gcs.call_count) - - # alpha api - self.assertEquals(0, notifier._send_findings_to_cscc.call_count) - notifier.run(None, None, 'api', None) - self.assertEquals(1, notifier._send_findings_to_cscc.call_count) - - self.assertEquals(3, mock_logger.info.call_count) - notifier.run(None, None, 'foo', None) - self.assertEquals(5, mock_logger.info.call_count) - self.assertTrue( - 'not selected' in mock_logger.info.call_args_list[4][0][0]) def test_beta_api_is_invoked_correctly(self): @@ -233,7 +162,7 @@ def test_beta_api_is_invoked_correctly(self): notifier.LOGGER = mock.MagicMock() self.assertEquals(0, notifier._send_findings_to_cscc.call_count) - notifier.run(None, None, 'api', None, source_id='111') + notifier.run(None, source_id='111') calls = notifier._send_findings_to_cscc.call_args_list call = calls[0]