Merge pull request #428 from dynatrace-oss/PCLOUDS-2765-Exclusion_par…

…am_for_projects

Added exclusion param for projects

k8s/helm-chart/dynatrace-gcp-monitor/templates/function-config.yaml

@@ -27,6 +27,7 @@ data:
   HTTPS_PROXY: {{ .Values.httpsProxy | quote }}
   {{- end }}
   SCOPING_PROJECT_SUPPORT_ENABLED: {{ .Values.scopingProjectSupportEnabled | quote }}
+  EXCLUDED_PROJECTS: {{ .Values.excludedProjects | quote }}
   KEEP_REFRESHING_EXTENSIONS_CONFIG: {{ .Values.keepRefreshingExtensionsConfig | quote }}
   {{- if or (eq .Values.deploymentType "metrics") (eq .Values.deploymentType "all") }}
   PRINT_METRIC_INGEST_INPUT: {{ .Values.printMetricIngestInput | quote }}

k8s/helm-chart/dynatrace-gcp-monitor/templates/function-deployment.yml

@@ -134,6 +134,11 @@ spec:
             configMapKeyRef:
               name: dynatrace-gcp-monitor-config
               key: SCOPING_PROJECT_SUPPORT_ENABLED
+        - name: EXCLUDED_PROJECTS
+          valueFrom:
+            configMapKeyRef:
+              name: dynatrace-gcp-monitor-config
+              key: EXCLUDED_PROJECTS
         - name:   KEEP_REFRESHING_EXTENSIONS_CONFIG
           valueFrom:
             configMapKeyRef:

k8s/helm-chart/dynatrace-gcp-monitor/values.yaml

@@ -78,8 +78,6 @@ useProxy: ""
 httpProxy: ""
 # httpsProxy: set the https proxy address. To be used in conjunction with USE_PROXY.
 httpsProxy: ""
-# scopingProjectSupportEnabled: set to true when metrics scope is configured, so metrics will be collected from all monitored projects.
-scopingProjectSupportEnabled: "false"
 # keepRefreshingExtensionsConfig: if true, will fetch enabled extensions from DT on each polling, otherwise will only fetch once during startup
 keepRefreshingExtensionsConfig: "true"
 
@@ -118,6 +116,10 @@ serviceUsageBooking: "source"
 queryInterval: 3
 #clusterName Cluster Name, should be empty as it's filed by deploy-helm.sh
 clusterName: ""
+# scopingProjectSupportEnabled: set to true when metrics scope is configured, so metrics will be collected from all monitored projects.
+scopingProjectSupportEnabled: "false"
+# excludedProjects: comma separated list of projects that will be excluded from monitoring (e.g. "project-a,project-b,project-c").
+excludedProjects: ""
 metricResources:
   requests:
     memory: "1536Mi"

src/lib/configuration/config.py

@@ -13,6 +13,10 @@ def scoping_project_support_enabled():
     return os.environ.get("SCOPING_PROJECT_SUPPORT_ENABLED", "FALSE").upper() in ["TRUE", "YES"]
 
 
+def excluded_projects():
+    return os.environ.get("EXCLUDED_PROJECTS", "")
+
+
 def project_id():
     return os.environ.get("GCP_PROJECT")
 

src/lib/gcp_apis.py

@@ -60,8 +60,8 @@ async def _get_all_disabled_apis(context: MetricsContext, project_id: str):
 
 
 # This function return all the disabled projects that Service Account has access to and all disabled apis of the enabled projects
-async def get_disabled_projects_and_disabled_apis_by_project_id(context, projects_ids) -> Tuple[List[str], Dict[str, Set[str]]]:
-    disabled_projects = []
+async def get_disabled_projects_and_disabled_apis_by_project_id(context, projects_ids) -> Tuple[Set[str], Dict[str, Set[str]]]:
+    disabled_projects = set()
     disabled_apis = {}
     tasks_to_check_if_project_is_disabled = []
 
@@ -71,7 +71,7 @@ async def get_disabled_projects_and_disabled_apis_by_project_id(context, project
     results = await asyncio.gather(*tasks_to_check_if_project_is_disabled)
     for project_id, is_project_disabled, disabled_api_set in results:
         if is_project_disabled:
-            disabled_projects.append(project_id)
+            disabled_projects.add(project_id)
         else:
             disabled_apis[project_id] = disabled_api_set
 

src/main.py

@@ -118,17 +118,19 @@ async def query_metrics(execution_id: Optional[str], services: Optional[List[GCP
 
         projects_ids = await get_all_accessible_projects(context, gcp_session, token)
 
-        disabled_projects = []
+        disabled_projects = set()
         disabled_apis_by_project_id = {}
 
         # Using metrics scope feature, checking disabled apis in every project is not needed
         if not config.scoping_project_support_enabled():
             disabled_projects, disabled_apis_by_project_id = \
                 await get_disabled_projects_and_disabled_apis_by_project_id(context, projects_ids)
 
+        disabled_projects.update(filter(None, config.excluded_projects().split(',')))
+
         if disabled_projects:
-            for disabled_project in disabled_projects:
-                projects_ids.remove(disabled_project)
+            projects_ids = [x for x in projects_ids if x not in disabled_projects]
+            context.log("Disabled projects: " + ", ".join(disabled_projects))
 
         setup_time = (time.time() - setup_start_time)
         for project_id in projects_ids: