Skip to content

Commit

Permalink
pkg/gitutil: init
Browse files Browse the repository at this point in the history
This refactors the code we're using to manage temporary git repositories
into a utility package.
  • Loading branch information
jzelinskie committed Sep 19, 2018
1 parent f98ff58 commit c2d887f
Show file tree
Hide file tree
Showing 4 changed files with 152 additions and 80 deletions.
41 changes: 3 additions & 38 deletions ext/vulnsrc/alpine/alpine.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -20,7 +20,6 @@ import (
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"

Expand All @@ -31,7 +30,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)

const (
Expand All @@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er

// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
if err != nil {
return
}
Expand Down Expand Up @@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
return
}

func (u *updater) pullRepository() (commit string, err error) {
// If the repository doesn't exist, clone it.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
return "", vulnsrc.ErrFilesystem
}

cmd := exec.Command("git", "clone", secdbGitURL, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}

commit = strings.TrimSpace(string(out))
return
}

type secDBFile struct {
Distro string `yaml:"distroversion"`
Packages []struct {
Expand Down
3 changes: 0 additions & 3 deletions ext/vulnsrc/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ var (
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")

// ErrGitFailure is returned when a fetcher fails to interact with git.
ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")

updatersM sync.RWMutex
updaters = make(map[string]Updater)
)
Expand Down
42 changes: 3 additions & 39 deletions ext/vulnsrc/ubuntu/ubuntu.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -21,9 +21,7 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"

Expand All @@ -33,7 +31,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)

const (
Expand Down Expand Up @@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er

// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag)
if err != nil {
return resp, err
}
Expand Down Expand Up @@ -150,40 +148,6 @@ func (u *updater) Clean() {
}
}

func (u *updater) pullRepository() (commit string, err error) {
// Determine whether we should branch or pull.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
// Create a temporary folder to store the repository.
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return "", vulnsrc.ErrFilesystem
}
cmd := exec.Command("git", "clone", trackerURI, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}

commit = strings.TrimSpace(string(out))
return
}

func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
modifiedCVE := make(map[string]struct{})
for _, dirName := range []string{"active", "retired"} {
Expand Down
146 changes: 146 additions & 0 deletions pkg/gitutil/gitutil.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gitutil implements an easy way to update a git repository to a local
// temporary directory.
package gitutil

import (
"errors"
"io/ioutil"
"os"
"os/exec"
"strings"

log "github.com/sirupsen/logrus"
)

// ErrFailedClone is returned when a git clone is unsuccessful.
var ErrFailedClone = errors.New("failed to clone git repository")

// ErrFailedRevParse is returned when a git rev-parse is unsuccessful.
var ErrFailedRevParse = errors.New("failed to rev-parse git repository")

// ErrFailedPull is returned when a git pull is unsuccessful.
var ErrFailedPull = errors.New("failed to pull git repository")

// pull performs a git pull on the provided path and returns the commit SHA
// for the HEAD reference.
func pull(path string) (head string, err error) {
// Prepare a command to pull the repository.
cmd := exec.Command("git", "pull")
cmd.Dir = path

// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedPull
return
}

return revParseHead(path)
}

// CloneOrPull performs a git pull if there is a git repository located at
// repoPath. Otherwise, it performs a git clone to that path.
//
// If repoPath is left empty, a temporary directory is generated with the
// provided prefix and returned.
func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) {
// Create a temporary directory if the path is unspecified.
if repoPath == "" {
path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
if err != nil {
return
}
} else {
path = repoPath
}

if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) {
head, err = clone(remote, path)
return
}

head, err = pull(path)
return
}

// clone performs a git clone to the provided path and returns the commit SHA
// for the HEAD reference.
func clone(remote, path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.WithField("remote", remote).Error("attempted to git clone repository to empty path")
err = ErrFailedClone
return
}

// Prepare a command to clone the repository.
cmd := exec.Command("git", "clone", remote, ".")
cmd.Dir = path

// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"remote": remote,
"path": path,
"output": string(commandOutput),
}).Error("failed to git clone repository")

err = os.RemoveAll(path)
if err != nil {
log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone")
}
err = ErrFailedClone
return
}

return revParseHead(path)
}

// revParseHead performs a git rev-parse HEAD on the provided path and returns
// the commit SHA for the HEAD reference.
func revParseHead(path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.Error("attempted to rev-parse repository with empty path")
err = ErrFailedRevParse
return
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = path

var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedRevParse
return
}

head = strings.TrimSpace(string(commandOutput))
return
}

0 comments on commit c2d887f

Please sign in to comment.