Merge branch 'development' into feat/graph-qol

.github/workflows/build-dev.yml

@@ -11,23 +11,148 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # The type of runner that the job will run on
+  build-deeplynx-dev:
     runs-on: [ self-hosted ]
-    container:
-      image: gcr.io/kaniko-project/executor:debug
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
+    environment: development
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
+        with:
+          ref: development
+      - shell: bash
+        name: ACR build
+        env:
+          ACR_SP_USER: ${{ secrets.CI_SP_USER }}
+          ACR_SP_PASSWORD: ${{ secrets.CI_SP_PASSWORD }}
+          ACR_REGISTRY: ${{ secrets.CI_REGISTRY }}
+          ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }}
+          ACR_SP_TENANT: ${{ secrets.CI_SP_TENANT }}
+          ACR_SUBSCRIPTION: ${{ secrets.CI_ACR_SUBSCRIPTION }}
+          NODE_ENV: ${{ secrets.NODE_ENV }}
+        run: |
+          cd $GITHUB_WORKSPACE
+          az cloud set --name AzureUSGovernment
+          az login --service-principal -u $ACR_SP_USER -p $ACR_SP_PASSWORD --tenant $ACR_SP_TENANT
+          az account set --subscription $ACR_SUBSCRIPTION
+          az acr build -r $ACR_REGISTRY -f $GITHUB_WORKSPACE/Dockerfile -t $ACR_PATH:latest-dev .
+      - shell: bash
+        name: ACR Get Scan
+        env:
+          ACR_REGISTRY: ${{ secrets.CI_REGISTRY }}
+          ACR_PATH: ${{ secrets.CI_REGISTRY_PATH }}
+        run: |
+          imageDigest=$(az acr repository show -n $ACR_REGISTRY -t $ACR_PATH:latest-dev | jq --raw-output '.digest')
+          healthquery="securityresources
+          | where type == 'microsoft.security/assessments/subassessments'
+          | where id matches regex  '(.+?)/providers/Microsoft.ContainerRegistry/registries/(.+)/providers/Microsoft.Security/assessments/dbd0cb49-b563-45e7-9724-889e799fa648/'
+          | extend registryResourceId = tostring(split(id, '/providers/Microsoft.Security/assessments/')[0])
+          | extend registryResourceName = tostring(split(registryResourceId, '/providers/Microsoft.ContainerRegistry/registries/')[1])
+          | extend imageDigest = tostring(properties.additionalData.imageDigest)
+          | extend repository = tostring(properties.additionalData.repositoryName)
+          | extend scanFindingSeverity = tostring(properties.status.severity), scanStatus = tostring(properties.status.code)
+          | summarize scanFindingSeverityCount = count() by scanFindingSeverity, scanStatus, registryResourceId, registryResourceName, repository, imageDigest
+          | summarize  severitySummary = make_bag(pack(scanFindingSeverity, scanFindingSeverityCount)) by registryResourceId, registryResourceName, repository, imageDigest, scanStatus
+          | where imageDigest contains '$imageDigest'"
+          query="SecurityResources
+          | where type == 'microsoft.security/assessments'
+          | where properties.displayName contains 'Container registry images should have vulnerability findings resolved'
+          | summarize by assessmentKey=name //the ID of the assessment
+          | join kind=inner (securityresources | where type == 'microsoft.security/assessments/subassessments' | extend assessmentKey = extract('.*assessments/(.+?)/.*',1,  id)) on assessmentKey
+          | project parse_json(properties)
+          | extend description = properties.description,displayName = properties.displayName,resourceId = properties.resourceDetails.id,resourceSource = properties.resourceDetails.source,category = properties.category,severity = properties.status.severity,code = properties.status.code,timeGenerated = properties.timeGenerated,remediation = properties.remediation,impact = properties.impact,vulnId = properties.id,additionalData = properties.additionalData
+          | where resourceId contains '$imageDigest'"
+          az config set extension.use_dynamic_install=yes_without_prompt
+          count=1
+          querycount=1
+          until false; do
+              scanhealth=$(az graph query -q "$healthquery" | jq --raw-output '.data[] | .scanStatus')
+              if [[ $scanhealth = 'Healthy' ]]; then
+                  echo 'Scan returned health'
+                  break
+              elif [[ $scanhealth = 'Unhealthy' ]]; then
+                  echo "Building report with findings"
+                  rm -f scanreport.tsv
+                  echo -e 'severity\tid\tpatchable\tpublished\tregistryhost\treponame\tos\tdisplayname\tdescription\timpact\tcvetitle\tcvelink\tvendorrefrencetitle\tvendorerefrencelink\tscanner\ttype\timagedigest' >>scanreport.tsv
+                  az graph query -q "$query" | jq --raw-output '.data[] | [.severity, .properties.id, .properties.additionalData.patchable, .properties.additionalData.publishedTime, .properties.additionalData.registryHost, .properties.additionalData.repositoryName, .additionalData.imageDetails.osDetails, .displayName, '.description', .impact, .properties.additionalData.cve[].title, .properties.additionalData.cve[].link,.properties.additionalData.vendorReferences[].title, .properties.additionalData.vendorReferences[].link, .properties.additionalData.scanner,.properties.additionalData.type , .additionalData.imageDigest] | @tsv' >>scanreport.tsv
+                  break
+              elif [[ $count -eq 10 ]]; then
+                  echo "Image scan not found exiting"
+                  break
+              else
+                  echo "Scan not complete... waiting $count"
+                  sleep 30
+                  count="$((count + 1))"
+              fi
+          done
+          echo -e "run:\nreg add "HKEY_LOCAL_MACHINE\SOFTWARE\Clients\Spreadsheet\Microsoft Excel\Capabilities\FileAssociations" /v ".tsv" /t REG_SZ /d "Excel.SLK" /f\nto associate .tsv with excel. You should only need to do this once." >> SCAN_READ_ME.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Azure_Container_Scan_Result
+          path: |
+            scanreport.tsv
+            SCAN_READ_ME.txt
 
-      # Runs a set of commands using the runners shell
-      - name: Build & Push via kaniko
+      - shell: bash
+        name: Manifest env substitute
+        env:
+          CI_REGISTRY: ${{ secrets.CI_REGISTRY}}
+          CI_REGISTRY_PATH: ${{ secrets.CI_REGISTRY_PATH}}
+          DB_NAME: ${{ secrets.DB_NAME }}
+          AZURE_BLOB_CONTAINER_NAME: ${{ secrets.AZURE_BLOB_CONTAINER_NAME }}
+          CONTAINER_INVITE_URL: ${{ secrets.CONTAINER_INVITE_URL }}
+          FILE_STORAGE_METHOD: ${{ secrets.FILE_STORAGE_METHOD}}
+          SMTP_HOST: ${{ secrets.SMTP_HOST}}
+          SMTP_TLS: ${{ secrets.SMTP_TLS}}
+          EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS}}
+          EMAIL_ENABLED: ${{ secrets.EMAIL_ENABLED}}
+          EMAIL_VALIDATION_ENFORCED: ${{ secrets.EMAIL_VALIDATION_ENFORCED}}
+          ROOT_ADDRESS: ${{ secrets.ROOT_ADDRESS }}
+          ENCRYPTION_KEY_PATH: ${{ secrets.ENCRYPTION_KEY_PATH }}
+          NODE_ENV: ${{ secrets.NODE_ENV }}
+          AUTH_STRATEGY: ${{ secrets.AUTH_STRATEGY }}
+          BASE_URL: ${{ secrets.BASE_URL }}
+          SAML_ENABLED: ${{ secrets.SAML_ENABLED }}
+          SAML_ADFS_ENTRY_POINT: ${{ secrets.SAML_ADFS_ENTRY_POINT }}
+          SAML_ADFS_ISSUER: ${{ secrets.SAML_ADFS_ISSUER }}
+          SAML_ADFS_CALLBACK: ${{ secrets.SAML_ADFS_CALLBACK }}
+          SAML_ADFS_PRIVATE_CERT_PATH: ${{ secrets.SAML_ADFS_PRIVATE_CERT_PATH }}
+          SAML_ADFS_PUBLIC_CERT_PATH: ${{ secrets.SAML_ADFS_PUBLIC_CERT_PATH }}
+          SAML_ADFS_CLAIMS_EMAIL: ${{ secrets.SAML_ADFS_CLAIMS_EMAIL }}
+          SAML_ADFS_CLAIMS_NAME: ${{ secrets.SAML_ADFS_CLAIMS_NAME }}
+          PROCESS_QUEUE_NAME: ${{ secrets.PROCESS_QUEUE_NAME }}
+          EVENTS_QUEUE_NAME: ${{ secrets.EVENTS_QUEUE_NAME }}
+          DATA_SOURCES_QUEUE_NAME: ${{ secrets.DATA_SOURCES_QUEUE_NAME }}
+          CACHE_PROVIDER: ${{ secrets.CACHE_PROVIDER }}
+          CACHE_REDIS_CONNECTION_STRING: ${{ secrets.CACHE_REDIS_CONNECTION_STRING }}
+          EDGE_INSERTION_QUEUE_NAME: ${{ secrets.EDGE_INSERTION_QUEUE_NAME }}
+          QUEUE_SYSTEM: ${{ secrets.QUEUE_SYSTEM }}
+          TIMESCALEDB_ENABLED: ${{ secrets.TIMESCALEDB_ENABLED }}
+          TZ: ${{ secrets.TZ }}
+          CI_COMMIT_SHA: $GITHUB_SHA
+          NODE_EXTRA_CA_CERTS: ${{ secrets.NODE_EXTRA_CA_CERTS }}
+          RUN_JOBS: ${{ secrets.RUN_JOBS }}
+          JAZZ_HOST: ${{ secrets.JAZZ_HOST }}
+          JAZZ_IP: ${{ secrets.JAZZ_IP }}
+          ELM_HOST: ${{ secrets.ELM_HOST }}
+          ELM_IP: ${{ secrets.ELM_IP }}
+          NETWORKING_HOST: ${{ secrets.NETWORKING_HOST }}
         run: |
-          mkdir -p /kaniko/.docker
-          echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
-          /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY/$CI_REGISTRY_PATH/deep-lynx-dev:latest"
+          echo $JAZZ_HOST
+          cd $GITHUB_WORKSPACE
+          envsubst < kubernetes/development.yml > kubernetes/development_final.yml
+      - uses: azure/setup-kubectl@v3
+      - uses: azure/k8s-set-context@v3
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.KUBE_CONFIG }}
+          context: deploy-service-account
+      - uses: Azure/k8s-deploy@v4
+        with:
+          resource-group: ${{ secrets.CI_RESOURCE_GROUP }}
+          name: ${{ secrets.CLUSTER_NAME }}
+          namespace: deeplynx-dev
+          action: deploy
+          force: true
+          strategy: basic
+          manifests: |
+            kubernetes/development_final.yml

AdminWebApp/src/api/client.ts

@@ -340,6 +340,10 @@ export class Client {
         return this.get<MetatypeT>(`/containers/${containerID}/metatypes/${metatypeID}`);
     }
 
+    retrieveMetatypeByUUID(containerID: string, metatypeID: string): Promise<MetatypeT> {
+        return this.get<MetatypeT>(`/containers/${containerID}/metatypes/${metatypeID}`, {uuid: true});
+    }
+
     updateMetatype(containerID: string, metatypeID: string, metatype: any): Promise<boolean> {
         return this.put<boolean>(`/containers/${containerID}/metatypes/${metatypeID}`, metatype);
     }

AdminWebApp/src/api/types.ts

@@ -374,6 +374,15 @@ export type TypeMappingTransformationCondition = {
     subexpressions: TypeMappingTransformationSubexpression[];
 };
 
+export type EdgeConfigKeyT = {
+    id?: string; // only needed for the UI to keep track of things
+    type?: string;
+    operator?: string;
+    property?: any;
+    key?: any;
+    value?: any;
+};
+
 // Actions that can be performed when a transformation encounters an error
 export type TransformationErrorAction = 'ignore' | 'fail on required' | 'fail';
 export const TransformationErrorActions: TransformationErrorAction[] = ['ignore', 'fail on required', 'fail'];
@@ -414,6 +423,8 @@ export type TypeMappingTransformationT = {
     metatype_relationship_pair_ontology_version?: string;
     keys: TypeMappingTransformationKeyMapping[];
     archived: boolean;
+    origin_parameters?: EdgeConfigKeyT[];
+    destination_parameters?: EdgeConfigKeyT[];
 };
 
 export type TypeMappingTransformationPayloadT = {