Add ability to access git over ssh (#259)

* Add ability to access git over ssh Signed-off-by: djach7 <djachimo@redhat.com> * testing permissions and file placement Signed-off-by: Ryan Cook <rcook@redhat.com> Signed-off-by: djach7 <djachimo@redhat.com> Signed-off-by: Ryan Cook <rcook@redhat.com> Co-authored-by: Ryan Cook <rcook@redhat.com>
containers · Sep 19, 2022 · dffb843 · dffb843
1 parent 5d48e21
commit dffb843
Show file tree

Hide file tree

Showing 8 changed files with 186 additions and 25 deletions.
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -1453,6 +1453,76 @@ jobs:
         if: always()
         run: sudo podman logs fetchit
 
+  ssh-validate:
+    runs-on: ubuntu-latest
+    needs: [ build, pull-and-archive ]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: pull in podman
+        uses: actions/download-artifact@v1
+        with:
+          name: podman-bins
+          path: bin
+
+      - name: replace
+        run: |
+          chmod +x bin/podman
+          sudo mv bin/podman /usr/bin/podman
+
+      - name: Enable the podman socket
+        run: sudo systemctl enable --now podman.socket
+
+      - name: pull artifact
+        uses: actions/download-artifact@v1
+        with:
+          name: fetchit-image
+          path: /tmp
+
+      - name: pull artifact
+        uses: actions/download-artifact@v1
+        with:
+          name: colors
+          path: /tmp
+
+      - name: Load the image
+        run: sudo podman load -i /tmp/fetchit.tar
+
+      - name: Load the image
+        run: sudo podman load -i /tmp/colors.tar
+
+      - name: tag the image
+        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest
+
+      - name: generate ssh assets
+        run: |
+          mkdir ~/.ssh
+          echo "${{secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          ssh-keyscan github.com > ~/.ssh/known_hosts
+          mkdir ~/.fetchit
+          mv ~/.ssh ~/.fetchit/
+          cp ./examples/ssh-config.yaml ~/.fetchit/config.yaml
+
+      - name: Start fetchit
+        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /home/runner/.fetchit:/opt/mount -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest
+
+      - name: List fetchit files
+        run: sudo ls -la ~/.fetchit
+
+      - name: List ssh files
+        run: sudo ls -la ~/.fetchit/.ssh
+
+      - name: check for file
+        run: timeout 150 bash -c "until [ -f /tmp/hello.txt ]; do sleep 2; done"
+
+      - name: check for file
+        run: timeout 150 bash -c "until [ -f /tmp/anotherfile.txt ]; do sleep 2; done" 
+
+      - name: Logs
+        if: always()
+        run: sudo podman logs fetchit
+
+
   loader-validate:
     runs-on: ubuntu-latest
     needs: [ build, pull-and-archive ]
@@ -1619,7 +1689,7 @@ jobs:
 
   push-amd-image-to-registry:
     runs-on: ubuntu-latest
-    needs: [ build, raw-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, gitsign-verify-validate, ansible-validate, loader-validate, disconnected-validate ]
+    needs: [ build, raw-validate, ssh-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, gitsign-verify-validate, ansible-validate, loader-validate, disconnected-validate ]
     if: >
       (github.event_name == 'push' || github.event_name == 'schedule') &&
       (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
@@ -1646,7 +1716,7 @@ jobs:
 
   build-arm-and-manifest-list:
     runs-on: ubuntu-latest
-    needs: [ build, raw-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, gitsign-verify-validate, ansible-validate, loader-validate, disconnected-validate ]
+    needs: [ build, raw-validate,ssh-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, gitsign-verify-validate, ansible-validate, loader-validate, disconnected-validate ]
     if: >
       (github.event_name == 'push' || github.event_name == 'schedule') &&
       (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))

diff --git a/examples/ssh-config.yaml b/examples/ssh-config.yaml
@@ -0,0 +1,11 @@
+gitAuth:
+  ssh: true
+  sshKeyFile: id_rsa
+targetConfigs:
+- url: git@github.com:containers/fetchit
+  filetransfer:
+  - name: ft-ex
+    targetPath: examples/filetransfer
+    destinationDirectory: /tmp
+    schedule: "*/1 * * * *"
+  branch: main
diff --git a/pkg/engine/apply.go b/pkg/engine/apply.go
@@ -16,6 +16,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/gobwas/glob"
 	gitsign "github.com/sigstore/gitsign/pkg/git"
 	gitsignrekor "github.com/sigstore/gitsign/pkg/rekor"
@@ -67,7 +68,9 @@ func getLatest(target *Target) (plumbing.Hash, error) {
 	}
 
 	refSpec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/heads/%s", target.branch, target.branch))
-	if err = repo.Fetch(&git.FetchOptions{
+
+	// default to using existing http method
+	fOptions := &git.FetchOptions{
 		RemoteName: "",
 		RefSpecs:   []config.RefSpec{refSpec, "HEAD:refs/heads/HEAD"},
 		Depth:      0,
@@ -80,7 +83,18 @@ func getLatest(target *Target) (plumbing.Hash, error) {
 		Force:           true,
 		InsecureSkipTLS: false,
 		CABundle:        []byte{},
-	}); err != nil && err != git.NoErrAlreadyUpToDate && !target.disconnected {
+	}
+	// if using ssh, change auth to use ssh key
+	if target.ssh {
+		logger.Infof("git clone %s ", target.url)
+		authValue, err := ssh.NewPublicKeysFromFile("git", target.sshKey, target.password)
+		if err != nil {
+			logger.Infof("generate publickeys failed: %s", err.Error())
+			return plumbing.Hash{}, err
+		}
+		fOptions.Auth = authValue
+	}
+	if err = repo.Fetch(fOptions); err != nil && err != git.NoErrAlreadyUpToDate && !target.disconnected {
 		return plumbing.Hash{}, utils.WrapErr(err, "Error fetching branch %s from remote repository %s", target.branch, target.url)
 	}
 

diff --git a/pkg/engine/config.go b/pkg/engine/config.go
@@ -29,9 +29,7 @@ type ConfigReload struct {
 	ConfigURL    string `mapstructure:"configURL"`
 	Device       string `mapstructure:"device"`
 	ConfigPath   string `mapstructure:"configPath"`
-	Pat          string `mapstructure:"pat"`
-	Username     string `mapstructure:"username"`
-	Password     string `mapstructure:"password"`
+	GitAuth      `mapstructure:",squash"`
 }
 
 func (c *ConfigReload) GetKind() string {
@@ -58,7 +56,7 @@ func (c *ConfigReload) Process(ctx, conn context.Context, skew int) {
 	// CheckForConfigUpdates downloads & places config file in defaultConfigPath
 	// if the downloaded config file differs from what's currently on the system.
 	if envURL != "" {
-		restart := checkForConfigUpdates(envURL, true, false, c.Pat, c.Username, c.Password)
+		restart := checkForConfigUpdates(envURL, true, false, c.PAT, c.Username, c.Password)
 		if !restart {
 			return
 		}

diff --git a/pkg/engine/fetchit.go b/pkg/engine/fetchit.go
@@ -13,6 +13,7 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -36,6 +37,10 @@ type Fetchit struct {
 	// conn holds podman client
 	conn               context.Context
 	volume             string
+	ssh                bool
+	sshKey             string
+	username           string
+	password           string
 	pat                string
 	restartFetchit     bool
 	scheduler          *gocron.Scheduler
@@ -85,7 +90,7 @@ func (fc *FetchitConfig) Restart() {
 	fetchit.RunTargets()
 }
 
-func populateConfig(v *viper.Viper) (*FetchitConfig, bool, error) {
+func readConfig(v *viper.Viper) (*FetchitConfig, bool, error) {
 	config := newFetchitConfig()
 	configDir := filepath.Dir(defaultConfigPath)
 	configName := filepath.Base(defaultConfigPath)
@@ -136,6 +141,30 @@ func (fc *FetchitConfig) populateFetchit(config *FetchitConfig) *Fetchit {
 			config.TargetConfigs = append(config.TargetConfigs, reload)
 		}
 	}
+
+	// Check for GitAuth field
+	if config.GitAuth != nil {
+		// Check for SSH usage
+		if config.GitAuth.SSH {
+                	if err := os.Setenv("SSH_KNOWN_HOSTS", "/opt/mount/.ssh/known_hosts"); err != nil {
+                        	cobra.CheckErr(err)
+                	}
+			keyPath := defaultSSHKey
+			// Check for unique ssh key file
+			if config.GitAuth.SSHKeyFile != "" {
+				keyPath = filepath.Join("/opt", "mount", ".ssh", config.GitAuth.SSHKeyFile)
+			}
+			if err := checkForPrivateKey(keyPath); err != nil {
+				cobra.CheckErr(err)
+			}
+			fetchit.ssh = true
+			fetchit.sshKey = keyPath
+		}
+		fetchit.username = config.GitAuth.Username
+		fetchit.password = config.GitAuth.Password
+		fetchit.pat = config.GitAuth.PAT
+	}
+
 	if config.Prune != nil {
 		prune := &TargetConfig{
 			prune: config.Prune,
@@ -173,7 +202,7 @@ func isLocalConfig(v *viper.Viper) (*FetchitConfig, bool, error) {
 		logger.Infof("Local config file not found: %v", err)
 		return nil, false, err
 	}
-	return populateConfig(v)
+	return readConfig(v)
 }
 
 // Initconfig reads in config file and env variables if set.
@@ -185,9 +214,6 @@ func (fc *FetchitConfig) InitConfig(initial bool) *Fetchit {
 	var isLocal, exists bool
 	var config *FetchitConfig
 	envURL := os.Getenv("FETCHIT_CONFIG_URL")
-	pat := ""
-	username := ""
-	password := ""
 
 	// user will pass path on local system, but it must be mounted at the defaultConfigPath in fetchit pod
 	// regardless of where the config file is on the host, fetchit will read the configFile from within
@@ -205,15 +231,15 @@ func (fc *FetchitConfig) InitConfig(initial bool) *Fetchit {
 		// Only run this from initial startup and only after trying to populate the config from a local file.
 		// because CheckForConfigUpdates also runs with each processConfig, so if !initial this is already done
 		// If configURL is passed in, a config file on disk has priority on the initial run.
-		_ = checkForConfigUpdates(envURL, false, true, pat, username, password)
+		_ = checkForConfigUpdates(envURL, false, true, "", "", "")
 	}
 
 	// if config is not yet populated, fc.CheckForConfigUpdates has placed the config
 	// downloaded from URL to the defaultconfigPath
 	if !isLocal {
 		// If not initial run, only way to get here is if already determined need for reload
 		// with an updated config placed in defaultConfigPath.
-		config, exists, err = populateConfig(v)
+		config, exists, err = readConfig(v)
 		if config == nil || !exists || err != nil {
 			if err != nil {
 				cobra.CheckErr(fmt.Errorf("Could not populate config, tried %s in fetchit pod and also URL: %s. Ensure local config is mounted or served from a URL and try again.", defaultConfigPath, envURL))
@@ -229,16 +255,19 @@ func (fc *FetchitConfig) InitConfig(initial bool) *Fetchit {
 	return fc.populateFetchit(config)
 }
 
+// Takes target from user and converts it for internal use
 func getMethodTargetScheds(targetConfigs []*TargetConfig, fetchit *Fetchit) *Fetchit {
 	for _, tc := range targetConfigs {
 		tc.mu.Lock()
 		defer tc.mu.Unlock()
 		internalTarget := &Target{
 			url:          tc.Url,
 			device:       tc.Device,
-			pat:          tc.Pat,
-			username:     tc.Username,
-			password:     tc.Password,
+			pat:          fetchit.pat,
+			ssh:          fetchit.ssh,
+			sshKey:       fetchit.sshKey,
+			username:     fetchit.username,
+			password:     fetchit.password,
 			branch:       tc.Branch,
 			disconnected: tc.Disconnected,
 		}
@@ -368,22 +397,36 @@ func getClone(target *Target) error {
 	} else if !os.IsNotExist(err) {
 		return err
 	}
-
 	if !exists {
 		logger.Infof("git clone %s %s --recursive", target.url, target.branch)
 		if target.pat != "" {
 			target.username = "fetchit"
 			target.password = target.pat
 		}
-		_, err = git.PlainClone(absPath, false, &git.CloneOptions{
+		// default to using existing http method
+		cOptions := &git.CloneOptions{
 			Auth: &githttp.BasicAuth{
 				Username: target.username, // the value of this field should not matter when using a PAT
 				Password: target.password,
 			},
 			URL:           target.url,
 			ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", target.branch)),
 			SingleBranch:  true,
-		})
+		}
+		// if using ssh, change auth to use ssh key
+		if target.ssh {
+			logger.Infof("git clone %s ", target.url)
+			authValue, err := ssh.NewPublicKeysFromFile("git", target.sshKey, target.password)
+			if err != nil {
+				logger.Infof("generate publickeys failed: %s", err.Error())
+				return err
+			}
+			cOptions.Auth = authValue
+		}
+		if err := cOptions.Validate(); err != nil {
+			return err
+		}
+		_, err = git.PlainClone(absPath, false, cOptions)
 		if err != nil {
 			return err
 		}

diff --git a/pkg/engine/filetransfer.go b/pkg/engine/filetransfer.go
@@ -42,7 +42,7 @@ func (ft *FileTransfer) Process(ctx, conn context.Context, skew int) {
 
 		err = zeroToCurrent(ctx, conn, ft, target, nil)
 		if err != nil {
-			logger.Errorf("Error moving to current: %v", err)
+			logger.Errorf("Error moving to current: %v target url is: %s ", err, target.url)
 			return
 		}
 	}

diff --git a/pkg/engine/gitauth.go b/pkg/engine/gitauth.go
@@ -0,0 +1,25 @@
+package engine
+
+import (
+	"os"
+	"path/filepath"
+)
+
+var defaultSSHKey = filepath.Join("/opt", "mount", ".ssh", "id_rsa")
+
+// Basic type needed for ssh authentication
+type GitAuth struct {
+	SSH        bool   `mapstructure:"ssh"`
+	SSHKeyFile string `mapstructure:"sshKeyFile"`
+	Username   string `mapstructure:"username"`
+	Password   string `mapstructure:"password"`
+	PAT        string `mapstructure:"pat"`
+}
+
+// Checks to see if private key exists on given path
+func checkForPrivateKey(path string) error {
+	if _, err := os.Stat(path); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/pkg/engine/types.go b/pkg/engine/types.go
@@ -20,6 +20,7 @@ type Method interface {
 
 // FetchitConfig requires necessary objects to process targets
 type FetchitConfig struct {
+	GitAuth          *GitAuth          `mapstructure:"gitAuth"`
 	TargetConfigs    []*TargetConfig   `mapstructure:"targetConfigs"`
 	ConfigReload     *ConfigReload     `mapstructure:"configReload"`
 	Prune            *Prune            `mapstructure:"prune"`
@@ -32,9 +33,6 @@ type FetchitConfig struct {
 type TargetConfig struct {
 	Name              string             `mapstructure:"name"`
 	Url               string             `mapstructure:"url"`
-	Pat               string             `mapstructure:"pat"`
-	Username          string             `mapstructure:"username"`
-	Password          string             `mapstructure:"password"`
 	Device            string             `mapstructure:"device"`
 	Disconnected      bool               `mapstructure:"disconnected"`
 	VerifyCommitsInfo *VerifyCommitsInfo `mapstructure:"verifyCommitsInfo"`
@@ -52,6 +50,8 @@ type TargetConfig struct {
 }
 
 type Target struct {
+	ssh             bool
+	sshKey          string
 	url             string
 	pat             string
 	username        string