[#1211] add type for gcp project list options and additional flexibil…

…ity to list ids in flat array

packages/app/src/App.ts

@@ -7,6 +7,7 @@ import {
   configLoader,
   EmissionRatioResult,
   EstimationResult,
+  GoogleProjectDetails,
   GroupBy,
   Logger,
   LookupTableInput,
@@ -91,8 +92,9 @@ export default class App {
         ).getDataFromBillingExportTable(startDate, endDate, grouping)
         GCPEstimatesByRegion.push(estimates)
       } else if (GCP?.projects.length) {
+        const googleProjectDetails = GCP.projects as GoogleProjectDetails[]
         // Resolve GCP Estimates asynchronously
-        for (const project of GCP.projects) {
+        for (const project of googleProjectDetails) {
           const estimates = await Promise.all(
             await new GCPAccount(
               project.id,

packages/common/src/Config.ts

@@ -4,6 +4,7 @@
 import fs from 'fs'
 import dotenv from 'dotenv'
 import { AWS_RECOMMENDATIONS_SERVICES } from './RecommendationsService'
+import { GoogleProjectDetailsOrIdList } from './Types'
 
 dotenv.config()
 
@@ -34,10 +35,7 @@ export interface CCFConfig {
     NAME?: string
     CURRENT_SERVICES?: { key: string; name: string }[]
     CURRENT_REGIONS?: string[]
-    projects?: {
-      id: string
-      name?: string
-    }[]
+    projects?: GoogleProjectDetailsOrIdList
     USE_CARBON_FREE_ENERGY_PERCENTAGE?: boolean
     INCLUDE_ESTIMATES?: boolean
     USE_BILLING_DATA?: boolean

packages/common/src/Types.ts

@@ -16,3 +16,10 @@ export type GoogleAuthClient =
   | UserRefreshClient
   | Impersonated
   | BaseExternalAccountClient
+
+export type GoogleProjectDetails = {
+  id: string
+  name?: string
+}
+
+export type GoogleProjectDetailsOrIdList = GoogleProjectDetails[] | string[]

packages/common/src/__tests__/Config.test.ts

@@ -4,6 +4,7 @@
 
 import fs from 'fs'
 import getConfig from '../Config'
+import { GoogleProjectDetails } from '../Types'
 
 describe('Config', () => {
   const withEnvironment = (name: string, value: string, test: () => void) => {
@@ -59,7 +60,7 @@ describe('Config', () => {
   })
 
   describe('Google Cloud', () => {
-    it('loads list of GCP Projects from the environment variables', () => {
+    it('loads list of GCP Projects with names and ids the environment variables', () => {
       const id = 'id'
       const secondId = 'id2'
       const name = 'project'
@@ -68,15 +69,27 @@ describe('Config', () => {
         'GCP_PROJECTS',
         `[{"id": "${id}", "name": "${name}"}, {"id": "${secondId}"}]`,
         () => {
-          const config = getConfig()
-          expect(config.GCP.projects[0].id).toBe(id)
-          expect(config.GCP.projects[0].name).toBe(name)
-          expect(config.GCP.projects[1].id).toBe(secondId)
-          expect(config.GCP.projects[1].name).toBeUndefined()
+          const configuredProjects = getConfig().GCP
+            .projects as GoogleProjectDetails[]
+          expect(configuredProjects[0].id).toBe(id)
+          expect(configuredProjects[0].name).toBe(name)
+          expect(configuredProjects[1].id).toBe(secondId)
+          expect(configuredProjects[1].name).toBeUndefined()
         },
       )
     })
 
+    it('loads list of GCP Projects with only ids from the environment variables', () => {
+      const id = 'id'
+      const secondId = 'id2'
+
+      withEnvironment('GCP_PROJECTS', `["${id}", "${secondId}"]`, () => {
+        const configuredProjects = getConfig().GCP.projects as string[]
+        expect(configuredProjects[0]).toBe(id)
+        expect(configuredProjects[1]).toBe(secondId)
+      })
+    })
+
     it('loads Google Cloud tags from environment variables', () => {
       withEnvironment('GCP_RESOURCE_TAG_NAMES', `["Environment"]`, () => {
         const config = getConfig()
@@ -110,7 +123,7 @@ describe('Config', () => {
         [true, ''],
       ])(
         `sets ${provider}_INCLUDE_ESTIMATES to %s for INCLUDE_ESTIMATES=%s`,
-        (expected: boolean, value: any) => {
+        (expected: boolean, value: string) => {
           withEnvironment(`${provider}_INCLUDE_ESTIMATES`, value, () => {
             const config = getConfig()
             expect(config[provider].INCLUDE_ESTIMATES).toBe(expected)

packages/common/src/index.ts

@@ -40,5 +40,9 @@ export {
   AWS_DEFAULT_RECOMMENDATIONS_SERVICE,
 } from './RecommendationsService'
 export * from './helpers'
-export type { GoogleAuthClient } from './Types'
+export type {
+  GoogleAuthClient,
+  GoogleProjectDetails,
+  GoogleProjectDetailsOrIdList,
+} from './Types'
 export * from './EmissionsFactors'

packages/gcp/src/__tests__/BillingExportTable.test.ts

@@ -35,6 +35,7 @@ import {
   mockQueryResultsCloudSQLSSDComputeEngineDataFlowHDD,
   mockQueryResultsComputeEngineRam,
   mockQueryResultsForProjectFilter,
+  mockQueryResultsForProjectFilterArray,
   mockQueryResultsForProjectFilterEmpty,
   mockQueryResultsForProjectFilterError,
   mockQueryResultsGPUMachineTypes,
@@ -1473,6 +1474,44 @@ describe('GCP BillingExportTable Service', () => {
     expect(result).toEqual(expectedResult)
   })
 
+  it('returns estimates for filtered projects that are an array of ids', async () => {
+    const testAccountId = 'test-account-id'
+    const testAccountIdTwo = 'test-account-id-two'
+
+    setConfig({
+      GCP: {
+        projects: [testAccountId, testAccountIdTwo],
+      },
+    })
+
+    mockJob.getQueryResults.mockResolvedValue(
+      mockQueryResultsForProjectFilterArray,
+    )
+
+    const billingExportTableService = new BillingExportTable(
+      new ComputeEstimator(),
+      new StorageEstimator(GCP_CLOUD_CONSTANTS.SSDCOEFFICIENT),
+      new StorageEstimator(GCP_CLOUD_CONSTANTS.HDDCOEFFICIENT),
+      new NetworkingEstimator(GCP_CLOUD_CONSTANTS.NETWORKING_COEFFICIENT),
+      new MemoryEstimator(GCP_CLOUD_CONSTANTS.MEMORY_COEFFICIENT),
+      new UnknownEstimator(GCP_CLOUD_CONSTANTS.ESTIMATE_UNKNOWN_USAGE_BY),
+      new EmbodiedEmissionsEstimator(
+        GCP_CLOUD_CONSTANTS.SERVER_EXPECTED_LIFESPAN,
+      ),
+      new BigQuery(),
+    )
+
+    await billingExportTableService.getEstimates(startDate, endDate, grouping)
+
+    const expectedWhereFilter = `AND project.id IN ('${testAccountId}', '${testAccountIdTwo}')`
+
+    expect(mockCreateQueryJob).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.stringContaining(expectedWhereFilter),
+      }),
+    )
+  })
+
   it('returns estimates for filtered projects when list of projects is provided', async () => {
     const testAccountId = 'test-account-id'
     const testAccountIdTwo = 'test-account-id-two'

packages/gcp/src/__tests__/fixtures/bigQuery.fixtures.ts

@@ -1006,6 +1006,39 @@ export const mockQueryResultsForProjectFilter: any[][] = [
   ],
 ]
 
+export const mockQueryResultsForProjectFilterArray: any[][] = [
+  [
+    {
+      timestamp: bigQueryDateOne,
+      accountId: accountId,
+      accountName: accountName,
+      region: 'us-east1',
+      serviceName: 'App Engine',
+      usageType: 'Cloud Datastore Storage',
+      usageUnit: 'byte-seconds',
+      usageAmount: 2.83e16,
+      cost: 5,
+      machineType: null,
+      tags: 'environment: dev',
+      labels: 'project: ccf',
+    },
+    {
+      timestamp: bigQueryDateOne,
+      accountId: accountId,
+      accountName: accountName,
+      region: 'us-east1',
+      serviceName: 'Compute Engine',
+      usageType: 'Compute optimized Core running in Americas',
+      usageUnit: 'seconds',
+      usageAmount: 80000,
+      cost: 7,
+      machineType: null,
+      tags: 'environment: prod',
+      projectLabels: 'team: thoughtworks',
+    },
+  ],
+]
+
 export const mockQueryResultsForProjectFilterError: any[][] = [
   [
     {

packages/gcp/src/lib/BillingExportTable.ts

@@ -13,6 +13,7 @@ import {
   convertBytesToGigabytes,
   EstimationResult,
   getEmissionsFactors,
+  GoogleProjectDetailsOrIdList,
   GroupBy,
   Logger,
   LookupTableInput,
@@ -91,7 +92,7 @@ export default class BillingExportTable {
   ): Promise<EstimationResult[]> {
     const gcpConfig = configLoader().GCP
     const tagNames = gcpConfig.RESOURCE_TAG_NAMES
-    const projects = gcpConfig.projects
+    const projects: GoogleProjectDetailsOrIdList = gcpConfig.projects
     const usageRows = await this.getUsage(
       start,
       end,
@@ -644,7 +645,7 @@ export default class BillingExportTable {
     end: Date,
     grouping: GroupBy,
     tagNames: string[],
-    projects: { id: string; name?: string }[],
+    projects: GoogleProjectDetailsOrIdList,
   ): Promise<RowMetadata[]> {
     const startDate = new Date(
       moment.utc(start).startOf('day') as unknown as Date,
@@ -791,21 +792,39 @@ export default class BillingExportTable {
     return parsedTags
   }
 
-  private buildProjectFilter(projects: { id: string; name?: string }[]) {
+  private buildProjectFilter(projects: GoogleProjectDetailsOrIdList): string {
+    const projectIds = this.getProjectsIdsFromList(projects)
+
+    if (!projectIds.length) return ''
+
+    const formattedProjectIds = projectIds
+      .map((project) => `'${project}'`)
+      .join(', ')
+
+    return `AND project.id IN (${formattedProjectIds})`
+  }
+
+  private getProjectsIdsFromList(
+    projects: GoogleProjectDetailsOrIdList,
+  ): string[] {
+    const projectIds = []
+
     if (!projects || !Array.isArray(projects)) {
       this.billingExportTableLogger.warn(
         'Configured list of projects is invalid. Projects must be a list of objects containing project IDs. Ignoring project filter...',
       )
-      return ''
+      return projectIds
     }
 
-    if (!projects.length) return ''
-
-    const projectIdList = projects
-      .map((project) => `'${project.id}'`)
-      .join(', ')
+    for (const project of projects) {
+      if (typeof project === 'string') {
+        projectIds.push(project)
+      } else {
+        projectIds.push(project.id)
+      }
+    }
 
-    return `AND project.id IN (${projectIdList})`
+    return projectIds
   }
 }