Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pkg/gitutil: init #621

Merged
merged 1 commit into from
Sep 26, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 3 additions & 38 deletions ext/vulnsrc/alpine/alpine.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -20,7 +20,6 @@ import (
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"

Expand All @@ -31,7 +30,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)

const (
Expand All @@ -53,7 +52,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er

// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(secdbGitURL, u.repositoryLocalPath, updaterFlag)
if err != nil {
return
}
Expand Down Expand Up @@ -183,40 +182,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
return
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow.
Very duplication.
So Git.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

func (u *updater) pullRepository() (commit string, err error) {
// If the repository doesn't exist, clone it.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
return "", vulnsrc.ErrFilesystem
}

cmd := exec.Command("git", "clone", secdbGitURL, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone alpine-secdb repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}

commit = strings.TrimSpace(string(out))
return
}

type secDBFile struct {
Distro string `yaml:"distroversion"`
Packages []struct {
Expand Down
3 changes: 0 additions & 3 deletions ext/vulnsrc/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ var (
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")

// ErrGitFailure is returned when a fetcher fails to interact with git.
ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")

updatersM sync.RWMutex
updaters = make(map[string]Updater)
)
Expand Down
42 changes: 3 additions & 39 deletions ext/vulnsrc/ubuntu/ubuntu.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2017 clair authors
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -21,9 +21,7 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"

Expand All @@ -33,7 +31,7 @@ import (
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/gitutil"
)

const (
Expand Down Expand Up @@ -89,7 +87,7 @@ func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, er

// Pull the master branch.
var commit string
commit, err = u.pullRepository()
u.repositoryLocalPath, commit, err = gitutil.CloneOrPull(trackerURI, u.repositoryLocalPath, updaterFlag)
if err != nil {
return resp, err
}
Expand Down Expand Up @@ -150,40 +148,6 @@ func (u *updater) Clean() {
}
}

func (u *updater) pullRepository() (commit string, err error) {
// Determine whether we should branch or pull.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
// Create a temporary folder to store the repository.
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return "", vulnsrc.ErrFilesystem
}
cmd := exec.Command("git", "clone", trackerURI, ".")
cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil {
u.Clean()
log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
return "", commonerr.ErrCouldNotDownload
}
} else {
// The repository already exists and it needs to be refreshed via a pull.
cmd := exec.Command("git", "pull")
cmd.Dir = u.repositoryLocalPath
if _, err := cmd.CombinedOutput(); err != nil {
return "", vulnsrc.ErrGitFailure
}
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = u.repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil {
return "", vulnsrc.ErrGitFailure
}

commit = strings.TrimSpace(string(out))
return
}

func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
modifiedCVE := make(map[string]struct{})
for _, dirName := range []string{"active", "retired"} {
Expand Down
146 changes: 146 additions & 0 deletions pkg/gitutil/gitutil.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// Copyright 2018 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gitutil implements an easy way to update a git repository to a local
// temporary directory.
package gitutil

import (
"errors"
"io/ioutil"
"os"
"os/exec"
"strings"

log "github.com/sirupsen/logrus"
)

// ErrFailedClone is returned when a git clone is unsuccessful.
var ErrFailedClone = errors.New("failed to clone git repository")

// ErrFailedRevParse is returned when a git rev-parse is unsuccessful.
var ErrFailedRevParse = errors.New("failed to rev-parse git repository")

// ErrFailedPull is returned when a git pull is unsuccessful.
var ErrFailedPull = errors.New("failed to pull git repository")

// pull performs a git pull on the provided path and returns the commit SHA
// for the HEAD reference.
func pull(path string) (head string, err error) {
// Prepare a command to pull the repository.
cmd := exec.Command("git", "pull")
cmd.Dir = path

// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedPull
return
}

return revParseHead(path)
}

// CloneOrPull performs a git pull if there is a git repository located at
// repoPath. Otherwise, it performs a git clone to that path.
//
// If repoPath is left empty, a temporary directory is generated with the
// provided prefix and returned.
func CloneOrPull(remote, repoPath, tempDirPrefix string) (path, head string, err error) {
// Create a temporary directory if the path is unspecified.
if repoPath == "" {
path, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
if err != nil {
return
}
} else {
path = repoPath
}

if _, pathExists := os.Stat(path); os.IsNotExist(pathExists) {
head, err = clone(remote, path)
return
}

head, err = pull(path)
return
}

// clone performs a git clone to the provided path and returns the commit SHA
// for the HEAD reference.
func clone(remote, path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.WithField("remote", remote).Error("attempted to git clone repository to empty path")
err = ErrFailedClone
return
}

// Prepare a command to clone the repository.
cmd := exec.Command("git", "clone", remote, ".")
cmd.Dir = path

// Execute the command.
var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"remote": remote,
"path": path,
"output": string(commandOutput),
}).Error("failed to git clone repository")

err = os.RemoveAll(path)
if err != nil {
log.WithError(err).WithField("path", path).Warn("failed to remove directory of failed clone")
}
err = ErrFailedClone
return
}

return revParseHead(path)
}

// revParseHead performs a git rev-parse HEAD on the provided path and returns
// the commit SHA for the HEAD reference.
func revParseHead(path string) (head string, err error) {
// Handle an invalid path.
if path == "" {
log.Error("attempted to rev-parse repository with empty path")
err = ErrFailedRevParse
return
}

cmd := exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = path

var commandOutput []byte
commandOutput, err = cmd.CombinedOutput()
if err != nil {
log.WithError(err).WithFields(log.Fields{
"path": path,
"output": string(commandOutput),
}).Error("failed to git rev-parse repository")
err = ErrFailedRevParse
return
}

head = strings.TrimSpace(string(commandOutput))
return
}