Merge pull request #1644 from diggerhq/feat/dgctl-artefacts-support

feat/dgctl artefacts support

backend/middleware/basic.go

@@ -59,6 +59,7 @@ func HttpBasicApiAuth() gin.HandlerFunc {
 			} else {
 				setDefaultOrganisationId(c)
 				c.Set(ACCESS_LEVEL_KEY, jobToken.Type)
+				c.Set(JOB_TOKEN_KEY, jobToken.Value)
 			}
 		} else if token == os.Getenv("BEARER_AUTH_TOKEN") {
 			setDefaultOrganisationId(c)

backend/middleware/jwt.go

@@ -309,3 +309,4 @@ func CORSMiddleware() gin.HandlerFunc {
 
 const ORGANISATION_ID_KEY = "organisation_ID"
 const ACCESS_LEVEL_KEY = "access_level"
+const JOB_TOKEN_KEY = "job_token"

backend/migrations/20240729155442.sql

@@ -0,0 +1,13 @@
+-- Create "job_artefacts" table
+CREATE TABLE "public"."job_artefacts" (
+  "id" bigserial NOT NULL,
+  "created_at" timestamptz NULL,
+  "updated_at" timestamptz NULL,
+  "deleted_at" timestamptz NULL,
+  "job_token_id" bigint NULL,
+  "contents" bytea NULL,
+  PRIMARY KEY ("id"),
+  CONSTRAINT "fk_job_artefacts_job_token" FOREIGN KEY ("job_token_id") REFERENCES "public"."job_tokens" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION
+);
+-- Create index "idx_job_artefacts_deleted_at" to table: "job_artefacts"
+CREATE INDEX "idx_job_artefacts_deleted_at" ON "public"."job_artefacts" ("deleted_at");

backend/migrations/20240729155926.sql

@@ -0,0 +1,2 @@
+-- Modify "job_artefacts" table
+ALTER TABLE "public"."job_artefacts" ADD COLUMN "size" bigint NULL, ADD COLUMN "content_type" text NULL;

backend/migrations/20240729160028.sql

@@ -0,0 +1,2 @@
+-- Modify "job_artefacts" table
+ALTER TABLE "public"."job_artefacts" ADD COLUMN "filename" text NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8=
+h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -27,3 +27,6 @@ h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8=
 20240704192835.sql h1:F84HKuE3qX8hYqDCU2K9NDG4WQaSfmnk2P4OCje54qg=
 20240705144450.sql h1:AAB4GbML2Jea1MSv/+E0DLMMCI9GLqwptpvBScvU19s=
 20240709165155.sql h1:HWDuhwXD+esr9uaKHsqwltPfUEq5uLdcEzpnZ1biKgU=
+20240729155442.sql h1:s7PCALP3SgPz5y9Ya7HkDzYjeN86Q5NniW2YIkOMBrQ=
+20240729155926.sql h1:8vsDrpy/R1UDI+meIp6KoDfhS60t+ngu8aPB+uonFZ4=
+20240729160028.sql h1:snkkxhA2aEQhqBmIhN8l+nPlBhrPOZiPP+dnyhobwD8=

backend/models/artefact.go

@@ -0,0 +1,15 @@
+package models
+
+import (
+	"gorm.io/gorm"
+)
+
+type JobArtefact struct {
+	gorm.Model
+	JobTokenID  uint
+	JobToken    JobToken
+	Filename    string
+	Contents    []byte `gorm:"type:bytea"`
+	Size        int64
+	ContentType string
+}

backend/models/storage.go

@@ -1126,6 +1126,25 @@ func (db *Database) GetJobToken(tenantId any) (*JobToken, error) {
 	return token, nil
 }
 
+func (db *Database) DeleteJobTokenArtefacts(jobTokenId uint) error {
+	artefact := JobArtefact{}
+	result := db.GormDB.Where("job_token_id = ?", jobTokenId).Delete(&artefact)
+	if result.Error != nil {
+		return result.Error
+	}
+	log.Printf("DeleteJobTokenArtefacts %v has been deleted successfully\n", jobTokenId)
+	return nil
+
+}
+
+func (db *Database) GetJobArtefact(jobTokenId uint) (*JobArtefact, error) {
+	var artefact JobArtefact
+	if err := DB.GormDB.Where("job_token_id = ?", jobTokenId).First(&artefact).Error; err != nil {
+		return nil, err
+	}
+	return &artefact, nil
+}
+
 func (db *Database) CreateGithubAppInstallation(installationId int64, githubAppId int64, login string, accountId int, repoFullName string) (*GithubAppInstallation, error) {
 	installation := &GithubAppInstallation{
 		GithubInstallationId: installationId,

cli/pkg/spec/spec.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/cli/pkg/digger"
 	"github.com/diggerhq/digger/cli/pkg/usage"
+	"github.com/diggerhq/digger/cli/pkg/utils"
+	"github.com/diggerhq/digger/libs/backendapi"
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary"
@@ -14,6 +16,7 @@ import (
 	"github.com/samber/lo"
 	"log"
 	"os"
+	"os/exec"
 	"time"
 )
 
@@ -169,6 +172,45 @@ func RunSpecManualCommand(
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1)
 	}
 
+	// download zip artefact, git init and prepare for job execution
+	tempDir, err := os.MkdirTemp("", "downloaded-zip-")
+	if err != nil {
+		log.Printf("failed to create temp dir: %v", err)
+		os.Exit(1)
+	}
+
+	// downloading artefact zip , extracting, git init and then chdir to that directory for job execution
+	absoluteFileName, err := backendApi.DownloadJobArtefact(tempDir)
+	if err != nil {
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not download job artefact: %v", err), 1)
+	}
+	zipPath := *absoluteFileName
+	err = utils.ExtractZip(zipPath, tempDir)
+	if err != nil {
+		log.Printf("ExtractZip err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("artefact zip extraction err: %v", err), 1)
+
+	}
+
+	// remove the zipPath
+	err = os.Remove(zipPath)
+	if err != nil {
+		log.Printf("os.Remove err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("zip path removal err: %v", err), 1)
+	}
+
+	// Navigating to our diractory
+	err = os.Chdir(tempDir)
+	if err != nil {
+		log.Printf("Chdir err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Chdir err: %v", err), 1)
+	}
+
+	cmd := exec.Command("git", "init")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Run()
+
 	policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken)
 	if err != nil {
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1)
@@ -182,11 +224,13 @@ func RunSpecManualCommand(
 
 	jobs := []scheduler.Job{job}
 
-	fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
-	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "")
-	if err != nil {
-		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4)
-	}
+	//fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
+	//_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "")
+	//if err != nil {
+	//	usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4)
+	//}
+
+	noopBackendApi := backendapi.NoopApi{}
 
 	commentId := spec.CommentId
 	if err != nil {
@@ -199,9 +243,9 @@ func RunSpecManualCommand(
 	}
 
 	commentUpdater := comment_summary.NoopCommentUpdater{}
-	// TODO: do not require conversion to gh service
+	// do not change these placeholders as they are parsed by dgctl to stream logs
 	log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>")
-	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir)
+	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, noopBackendApi, spec.JobId, false, false, commentId, currentDir)
 	log.Printf("<========= DIGGER COMPLETED =========>")
 	if err != nil || allAppliesSuccess == false {
 		usage.ReportErrorAndExit(spec.VCS.RepoOwner, "Terraform execution failed", 1)

cli/pkg/utils/io.go

@@ -0,0 +1,52 @@
+package utils
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func ExtractZip(zipFilePath string, outDir string) error {
+
+	// Open the zip file
+	zipReader, err := zip.OpenReader(zipFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to open zip: %w", err)
+	}
+	defer zipReader.Close()
+
+	for _, file := range zipReader.File {
+		path := filepath.Join(outDir, file.Name)
+
+		if file.FileInfo().IsDir() {
+			os.MkdirAll(path, os.ModePerm)
+			continue
+		}
+
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create directory: %w", err)
+		}
+
+		dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+
+		srcFile, err := file.Open()
+		if err != nil {
+			dstFile.Close()
+			return fmt.Errorf("failed to open zip file: %w", err)
+		}
+
+		_, err = io.Copy(dstFile, srcFile)
+		srcFile.Close()
+		dstFile.Close()
+
+		if err != nil {
+			return fmt.Errorf("failed to extract file: %w", err)
+		}
+	}
+	return nil
+}

dgctl/cmd/exec.go

@@ -8,6 +8,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/diggerhq/digger/dgctl/utils"
+	"github.com/diggerhq/digger/libs/backendapi"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
 	"github.com/diggerhq/digger/libs/spec"
 	"github.com/google/go-github/v61/github"
@@ -41,18 +43,6 @@ func getRepoUsername() (string, error) {
 	return strings.TrimSpace(string(out)), err
 }
 
-func getDefaultBranch() (string, error) {
-	cmd := exec.Command("git", "symbolic-ref", "refs/remotes/origin/HEAD")
-	out, err := cmd.Output()
-	return strings.ReplaceAll(strings.TrimSpace(string(out)), "refs/remotes/origin/", ""), err
-}
-
-func getPrBranch() (string, error) {
-	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
-	out, err := cmd.Output()
-	return strings.TrimSpace(string(out)), err
-}
-
 func getRepoFullname() (string, error) {
 	// Execute 'git config --get remote.origin.url' to get the URL of the origin remote
 	cmd := exec.Command("git", "config", "--get", "remote.origin.url")
@@ -92,15 +82,13 @@ func GetUrlContents(url string) (string, error) {
 	return content, nil
 }
 
-func GetSpec(diggerUrl string, authToken string, command string, actor string, projectMarshalled string, diggerConfigMarshalled string, repoFullName string, defaultBanch string, prBranch string) ([]byte, error) {
+func GetSpec(diggerUrl string, authToken string, command string, actor string, projectMarshalled string, diggerConfigMarshalled string, repoFullName string) ([]byte, error) {
 	payload := spec.GetSpecPayload{
-		Command:       command,
-		RepoFullName:  repoFullName,
-		Actor:         actor,
-		DefaultBranch: defaultBanch,
-		PrBranch:      prBranch,
-		DiggerConfig:  diggerConfigMarshalled,
-		Project:       projectMarshalled,
+		Command:      command,
+		RepoFullName: repoFullName,
+		Actor:        actor,
+		DiggerConfig: diggerConfigMarshalled,
+		Project:      projectMarshalled,
 	}
 	u, err := url.Parse(diggerUrl)
 	if err != nil {
@@ -142,11 +130,6 @@ func GetSpec(diggerUrl string, authToken string, command string, actor string, p
 
 	return body, nil
 }
-func pushToBranch(prBranch string) error {
-	cmd := exec.Command("git", "push", "origin", prBranch)
-	_, err := cmd.Output()
-	return err
-}
 
 func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (*int64, *int64, *string, error) {
 	timeFilter := time.Now().Add(-5 * time.Minute)
@@ -234,18 +217,6 @@ var execCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		defaultBanch, err := getDefaultBranch()
-		if err != nil {
-			log.Printf("could not get default branch, please enter manually:")
-			fmt.Scanln(&defaultBanch)
-		}
-
-		prBranch, err := getPrBranch()
-		if err != nil {
-			log.Printf("could not get current branch, please enter manually:")
-			fmt.Scanln(&prBranch)
-		}
-
 		projectName := execConfig.Project
 		command := execConfig.Command
 		projectConfig := config.GetProject(projectName)
@@ -266,13 +237,33 @@ var execCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		specBytes, err := GetSpec(diggerHostname, "abc123", command, actor, string(projectMarshalled), string(configMarshalled), repoFullname, defaultBanch, prBranch)
+		specBytes, err := GetSpec(diggerHostname, "abc123", command, actor, string(projectMarshalled), string(configMarshalled), repoFullname)
 		if err != nil {
 			log.Printf("failed to get spec from backend: %v", err)
+			os.Exit(1)
 		}
 		var spec spec.Spec
 		err = json.Unmarshal(specBytes, &spec)
 
+		// attach zip archive to backend
+		backendToken := spec.Job.BackendJobToken
+		zipLocation, err := utils.ArchiveGitRepo("./")
+		if err != nil {
+			log.Printf("error archiving zip repo: %v", err)
+			os.Exit(1)
+		}
+		backendApi := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken}
+		statusCode, respBody, err := backendApi.UploadJobArtefact(zipLocation)
+		if err != nil {
+			log.Printf("could not attach zip artefact: %v", err)
+			os.Exit(1)
+		}
+		if *statusCode != 200 {
+			log.Printf("unexpected status code from backend: %v", *statusCode)
+			log.Printf("server response: %v", *respBody)
+			os.Exit(1)
+		}
+
 		token := os.Getenv("GITHUB_PAT_TOKEN")
 		if token == "" {
 			log.Printf("missing variable: GITHUB_PAT_TOKEN")
@@ -289,18 +280,19 @@ var execCmd = &cobra.Command{
 				os.Exit(1)
 			}
 		}
-		err = pushToBranch(prBranch)
+
+		repoOwner, repoName, _ := strings.Cut(repoFullname, "/")
+		repository, _, err := client.Repositories.Get(context.Background(), repoOwner, repoName)
 		if err != nil {
-			log.Printf("could not push to branchL %v", err)
-			os.Exit(1)
+			log.Fatalf("Failed to get repository: %v", err)
 		}
 
 		inputs := orchestrator_scheduler.WorkflowInput{
 			Spec:    string(specBytes),
 			RunName: fmt.Sprintf("digger %v manual run by %v", command, spec.VCS.Actor),
 		}
 		_, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), spec.VCS.RepoOwner, spec.VCS.RepoName, spec.VCS.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
-			Ref:    spec.Job.Branch,
+			Ref:    *repository.DefaultBranch,
 			Inputs: inputs.ToMap(),
 		})
 
@@ -310,7 +302,6 @@ var execCmd = &cobra.Command{
 			log.Printf("workflow has triggered successfully! waiting for results ...")
 		}
 
-		repoOwner, repoName, _ := strings.Cut(repoFullname, "/")
 		var logsUrl *string
 		var runId *int64
 		var jobId *int64