feat: Controls database recreation

src/commands/scan.ts

@@ -5,7 +5,7 @@ import { Opts, pluginMap, PluginType, StorageEngine } from '@cloudgraph/sdk'
 import { range } from 'lodash'
 
 import Command from './base'
-import { fileUtils } from '../utils'
+import { cleanString, fileUtils, getStoredSchema } from '../utils'
 import DgraphEngine from '../storage/dgraph'
 import { scanReport } from '../reports'
 import { loadAllData, processConnectionsBetweenEntities } from '../utils/data'
@@ -91,6 +91,14 @@ export default class Scan extends Command {
     }
   }
 
+  // Indicates when schema has new changes
+  private schemaHasChange(oldSchema: string, newSchema: string): boolean {
+    return !!Buffer.compare(
+      Buffer.from(cleanString(oldSchema), 'utf8'),
+      Buffer.from(cleanString(newSchema), 'utf8')
+    )
+  }
+
   async run(): Promise<void> {
     const { argv, flags } = await this.parse(Scan)
     const { dev: devMode } = flags as {
@@ -227,8 +235,14 @@ export default class Scan extends Command {
             if (storageEngine instanceof DgraphEngine) {
               await storageEngine.validateSchema(schema, dataFolder)
             }
-            await storageEngine.dropAll() // Delete schema before change it
-            await storageEngine.setSchema(schema)
+
+            const currentSchema = getStoredSchema(dataDir)
+
+            // Only drops and changes data when the schema changes
+            if (this.schemaHasChange(currentSchema, schema.join())) {
+              await storageEngine.dropAll() // Delete schema before change it
+              await storageEngine.setSchema(schema, { overwrite: dataDir })
+            }
           } catch (error: any) {
             this.logger.error(
               `There was an issue pushing schema for providers: ${allProviders.join(

src/reports/scan-report.ts

@@ -29,6 +29,7 @@ const servicesToIgnore = [
   /^account$/,
   /^tag$/,
   /^label$/,
+  /ruleMetadata$/,
   /^billing$/,
   /Findings$/,
 ]

src/storage/dgraph/index.ts

@@ -3,15 +3,17 @@ import {
   StorageEngineConfig,
   StorageEngine,
   GraphQLInputData,
-  GraphQLQueryData
+  GraphQLQueryData,
 } from '@cloudgraph/sdk'
+import { isEmpty } from 'lodash'
 
 import DGraphClientWrapper from './base'
 import {
   GET_SCHEMA_QUERY,
   processGQLExecutionResult,
   UPDATE_SCHEMA_QUERY,
 } from './utils'
+import { fileUtils, sleep } from '../../utils'
 
 export default class DgraphEngine
   extends DGraphClientWrapper
@@ -80,11 +82,15 @@ export default class DgraphEngine
     })
   }
 
-  async setSchema(schemas: string[]): Promise<void> {
+  async setSchema(
+    schemas: string[],
+    config?: { overwrite: string }
+  ): Promise<void> {
+    const schema = schemas.join()
     const data = {
       query: UPDATE_SCHEMA_QUERY,
       variables: {
-        schema: schemas.join(),
+        schema,
       },
     }
     try {
@@ -99,8 +105,13 @@ export default class DgraphEngine
             resData,
             errors,
           })
+
+          if (isEmpty(errors) && config?.overwrite) {
+            fileUtils.writeGraphqlSchemaToFile(`${config.overwrite}/cg`, schema)
+          }
         })
         .catch(error => Promise.reject(error))
+      sleep(3)
     } catch (error: any) {
       const {
         response: { data: resData, errors },
@@ -181,8 +192,7 @@ export default class DgraphEngine
   /**
    * Executes mutations sequentially into Dgraph
    */
-  async run(dropData = true): Promise<void> {
-    dropData && (await this.dropData())
+  async run(): Promise<void> {
     for (const mutation of this.axiosPromises) {
       try {
         await mutation()

src/utils/index.ts

@@ -150,8 +150,19 @@ export function deleteFolder(dirPath: string): void {
   fs.rmSync(dirPath, { recursive: true })
 }
 
+export const getStoredSchema = (dirPath: string): string => {
+  try {
+    const schemaPath = path.normalize(`${dirPath}/cg/schema.graphql`)
+    const schema = fs.readFileSync(schemaPath, 'utf8')
+    return schema
+  } catch (error) {
+    // Return an empty string if a schema was not found
+    return ''
+  }
+}
+
 export const sleep = (ms: number): Promise<void> =>
-  new Promise(resolve => setTimeout(resolve, ms))
+  new Promise(resolve => setTimeout(resolve, ms * 1000))
 
 export const calculateBackoff = (n: number): number => {
   const temp = Math.min(
@@ -248,3 +259,6 @@ export const getNextPort = async (port: number): Promise<string> => {
   const availablePort = await detect(port)
   return String(availablePort)
 }
+
+export const cleanString = (dirtyString: string): string =>
+  dirtyString.replace(/(\r\n|\n|\r)/gm, '').replace(/\s+/g, '')