Merge pull request #513 from lunasec-io/lunatrace-cli-docker

CLI Refactor

Former-commit-id: cd21e2f
Former-commit-id: d0d24450a84f2d2bf4ec8aaac1cf629171423cc5

go.mod

@@ -15,5 +15,5 @@ require (
 replace (
 	lunasec/lunadefend => ./go
 	lunasec/lunatrace => ./lunatrace/cli
-	lunasec/lunatrace/inventory => ./lunatrace/cli/pkg/inventory
+	lunasec/lunatrace/snapshot => ./lunatrace/cli/pkg/inventory
 )

lunatrace/bsl/backend/Dockerfile

@@ -5,7 +5,7 @@ RUN apt install -y wget
 
 RUN corepack enable
 
-COPY --from=lunasec/lunatrace-cli:v0.0.3 /lunatrace /usr/local/bin
+COPY --from=lunasec/lunatrace-cli:v0.0.4 /lunatrace /usr/local/bin
 
 COPY . /app
 WORKDIR /app

lunatrace/bsl/backend/src/cli/call-cli.ts

@@ -21,15 +21,15 @@ function importAssetArgs(assetType: LunaTraceAssetType, assetName: string, gitBr
   const baseCmdArgs = [
     '--debug',
     '--log-to-stderr',
-    'import',
+    'snapshot',
     '--skip-upload',
     '--stdout',
     '--git-branch',
     gitBranch,
     assetType,
   ];
 
-  if (assetType === 'manifest') {
+  if (assetType === 'file') {
     return [...baseCmdArgs, '--stdin', assetName];
   }
   if (assetType === 'repository') {

lunatrace/bsl/backend/src/github/projects.ts

@@ -17,18 +17,14 @@ import { Request, Response } from 'express';
 import { getServerConfig } from '../config';
 import { hasura } from '../hasura-api';
 import { CreateOrganizationsMutation } from '../hasura-api/generated';
-import { getGithubAccessTokenFromKratos } from '../kratos';
 import { ListReposAccessibleToInstallationResponseType } from '../types/github';
 import { errorResponse, logError } from '../utils/errors';
 import { tryParseInt } from '../utils/parse-int';
 import { isError, Try, tryF } from '../utils/try';
 
-import { GetUserOrganizationsQuery } from './generated';
 import { pullDataForInstallation } from './installation-populate';
 import { lunatraceOrgsFromGithubOrgs } from './organizations';
 
-import { generateGithubGraphqlClient } from './index';
-
 const serverConfig = getServerConfig();
 
 export const githubInstall = async (req: Request, res: Response) => {

lunatrace/bsl/backend/src/hasura-api/generated.ts

@@ -1,16 +1,3 @@
-/*
- * Copyright by LunaSec (owned by Refinery Labs, Inc)
- *
- * Licensed under the Business Source License v1.1 
- * (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt
- *
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
 import { GraphQLClient } from 'graphql-request';
 import * as Dom from 'graphql-request/dist/types.dom';
 import gql from 'graphql-tag';
@@ -4518,8 +4505,8 @@ export type Projects = {
   manifests_aggregate: Manifests_Aggregate;
   name: Scalars['String'];
   /** An object relationship */
-  organization: Organizations;
-  organization_id: Scalars['uuid'];
+  organization?: Maybe<Organizations>;
+  organization_id?: Maybe<Scalars['uuid']>;
   /** An array relationship */
   project_access_tokens: Array<Project_Access_Tokens>;
   /** An aggregate relationship */
@@ -7484,7 +7471,7 @@ export type GetScanReportNotifyInfoForBuildQueryVariables = Exact<{
 }>;
 
 
-export type GetScanReportNotifyInfoForBuildQuery = { __typename?: 'query_root', builds_by_pk?: { __typename?: 'builds', pull_request_id?: string | null, project?: { __typename?: 'projects', id: any, organization: { __typename?: 'organizations', installation_id?: number | null } } | null } | null };
+export type GetScanReportNotifyInfoForBuildQuery = { __typename?: 'query_root', builds_by_pk?: { __typename?: 'builds', pull_request_id?: string | null, project?: { __typename?: 'projects', id: any, organization?: { __typename?: 'organizations', installation_id?: number | null } | null } | null } | null };
 
 export type InsertBuildMutationVariables = Exact<{
   project_id: Scalars['uuid'];
@@ -7541,7 +7528,7 @@ export type UpdateManifestMutationVariables = Exact<{
 }>;
 
 
-export type UpdateManifestMutation = { __typename?: 'mutation_root', update_manifests?: { __typename?: 'manifests_mutation_response', returning: Array<{ __typename?: 'manifests', filename: string, project_id: any, project: { __typename?: 'projects', organization_id: any } }> } | null };
+export type UpdateManifestMutation = { __typename?: 'mutation_root', update_manifests?: { __typename?: 'manifests_mutation_response', returning: Array<{ __typename?: 'manifests', filename: string, project_id: any, project: { __typename?: 'projects', organization_id?: any | null } }> } | null };
 
 export type UpdateOrganizationsForUserMutationVariables = Exact<{
   organizations_for_user: Array<Organization_User_Insert_Input> | Organization_User_Insert_Input;

lunatrace/bsl/backend/src/models/scan.ts

@@ -12,6 +12,7 @@
  *
  */
 import { spawn } from 'child_process';
+import { writeFileSync } from 'fs';
 import Stream, { Readable } from 'stream';
 
 import { hasura } from '../hasura-api';
@@ -20,10 +21,7 @@ import {
   Findings_Constraint,
   Findings_Insert_Input,
   Findings_Update_Column,
-  Package_Versions_Constraint,
   Scans_Insert_Input,
-  Vulnerabilities_Constraint,
-  Vulnerability_Packages_Constraint,
 } from '../hasura-api/generated';
 import { Convert, GrypeScanReport, Match } from '../types/grype-scan-report';
 
@@ -38,9 +36,8 @@ export async function parseAndUploadScan(sbomStream: Readable, buildId: string):
 }
 
 export async function runGrypeScan(sbomStream: Readable): Promise<string> {
+  let asdf = '';
   return new Promise((resolve, reject) => {
-    const stdoutStream = new Stream.Writable();
-    // const stderrStream = new Stream.Writeable();
     const grypeCli = spawn(`lunatrace`, ['--log-to-stderr', 'scan', '--stdin', '--stdout']);
     grypeCli.on('error', reject);
     const outputBuffers: Buffer[] = [];
@@ -56,8 +53,14 @@ export async function runGrypeScan(sbomStream: Readable): Promise<string> {
       }
       resolve(Buffer.concat(outputBuffers).toString());
     });
-    sbomStream.on('data', (chunk) => grypeCli.stdin.write(chunk));
-    sbomStream.on('end', () => grypeCli.stdin.end(() => console.log('Finished passing sbom contents to grype')));
+    sbomStream.on('data', (chunk) => {
+      asdf += chunk;
+      grypeCli.stdin.write(chunk);
+    });
+    sbomStream.on('end', () => {
+      writeFileSync('/tmp/test', asdf);
+      grypeCli.stdin.end(() => console.log('Finished passing sbom contents to grype'));
+    });
     sbomStream.on('error', reject);
   });
 }
@@ -100,67 +103,67 @@ async function parseMatches(buildId: string, matches: Match[]): Promise<Findings
         Findings_Update_Column.Severity,
       ],
     },
-    data: await Promise.all(
-      matches.map(async (match): Promise<Findings_Insert_Input> => {
-        const { vulnerability, artifact } = match;
-        const details = match.matchDetails[0];
-
-        // slugs
-        const vuln_slug = vulnerability.id + ':' + vulnerability.namespace;
-        const pkg_slug = vuln_slug + ':' + match.artifact.name;
-
-        const versionConstraint = parseVersionConstraint(match.matchDetails[0].found.versionConstraint);
-
-        const version_slug = pkg_slug + ':' + (versionConstraint ? versionConstraint : '');
-
-        const slugs = {
-          vuln_slug,
-          pkg_slug,
-          version_slug,
-        };
-
-        console.debug('slugs to lookup', slugs);
-
-        const ids = await hasura.GetPackageAndVulnFromSlugs({
-          vuln_slug,
-          pkg_slug,
-          version_slug,
-        });
-
-        const vulnerability_id = ids.vulnerabilities.length === 1 ? ids.vulnerabilities[0].id : undefined;
-        const vulnerability_package_id =
-          ids.vulnerability_packages.length === 1 ? ids.vulnerability_packages[0].id : undefined;
-        const package_version_id = ids.package_versions.length === 1 ? ids.package_versions[0].id : undefined;
-
-        if ([vulnerability_id, vulnerability_package_id, package_version_id].some((id) => !id)) {
-          console.error('unable to get all required ids', {
-            slugs,
-            ids,
+    data: (
+      await Promise.all(
+        matches.map(async (match): Promise<Findings_Insert_Input | null> => {
+          const { vulnerability, artifact } = match;
+          const details = match.matchDetails[0];
+
+          // slugs
+          const vuln_slug = vulnerability.id + ':' + vulnerability.namespace;
+          const pkg_slug = vuln_slug + ':' + match.artifact.name;
+
+          const versionConstraint = parseVersionConstraint(match.matchDetails[0].found.versionConstraint);
+
+          const version_slug = pkg_slug + ':' + (versionConstraint ? versionConstraint : '');
+
+          const slugs = {
+            vuln_slug,
+            pkg_slug,
+            version_slug,
+          };
+
+          const ids = await hasura.GetPackageAndVulnFromSlugs({
+            vuln_slug,
+            pkg_slug,
+            version_slug,
           });
-          return {};
-        }
-
-        const locations = artifact.locations.map((l) => l.path);
-        return {
-          package_name: artifact.name,
-          version: artifact.version,
-          version_matcher: details.found.versionConstraint,
-          type: artifact.type,
-          locations: formatPsqlStringArray(locations),
-          language: artifact.language,
-          purl: artifact.purl,
-          severity: vulnerability.severity,
-          virtual_path: artifact.metadata ? artifact.metadata.VirtualPath : null,
-          matcher: details.matcher,
-          dedupe_slug: pkg_slug + locations.sort().join(':'),
-          fix_state: vulnerability.fix?.state || null,
-          fix_versions: vulnerability.fix?.versions ? formatPsqlStringArray(vulnerability.fix?.versions) : null,
-          build_id: buildId,
-          vulnerability_id,
-          vulnerability_package_id,
-          package_version_id,
-        };
-      })
-    ),
+
+          const vulnerability_id = ids.vulnerabilities.length === 1 ? ids.vulnerabilities[0].id : undefined;
+          const vulnerability_package_id =
+            ids.vulnerability_packages.length === 1 ? ids.vulnerability_packages[0].id : undefined;
+          const package_version_id = ids.package_versions.length === 1 ? ids.package_versions[0].id : undefined;
+
+          if ([vulnerability_id, vulnerability_package_id, package_version_id].some((id) => !id)) {
+            console.error('unable to get all required ids', {
+              slugs,
+              ids,
+            });
+            return null;
+          }
+
+          const locations = artifact.locations.map((l) => l.path);
+          return {
+            package_name: artifact.name,
+            version: artifact.version,
+            version_matcher: details.found.versionConstraint,
+            type: artifact.type,
+            locations: formatPsqlStringArray(locations),
+            language: artifact.language,
+            purl: artifact.purl,
+            severity: vulnerability.severity,
+            virtual_path: artifact.metadata ? artifact.metadata.VirtualPath : null,
+            matcher: details.matcher,
+            dedupe_slug: pkg_slug + locations.sort().join(':'),
+            fix_state: vulnerability.fix?.state || null,
+            fix_versions: vulnerability.fix?.versions ? formatPsqlStringArray(vulnerability.fix?.versions) : null,
+            build_id: buildId,
+            vulnerability_id,
+            vulnerability_package_id,
+            package_version_id,
+          };
+        })
+      )
+    ).filter((e) => e !== null) as Findings_Insert_Input[],
   };
 }

lunatrace/bsl/backend/src/sqs-handlers/generate-sbom.ts

@@ -51,7 +51,7 @@ export async function createBuildWithGenerateSbom(
   // Get manifest from s3, streaming
   const [fileStream, _fileLength] = await aws.getFileFromS3(bucketInfo.key, bucketInfo.bucketName, bucketInfo.region);
   // spawn a copy of the CLI to make an sbom, stream in the manifest
-  const gzippedSbom = generateSbomFromAsset('manifest', assetName, 'master', {
+  const gzippedSbom = generateSbomFromAsset('file', assetName, 'master', {
     inputStream: fileStream,
   });