feat(python): add pypa support

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/hashicorp/go-getter v1.5.6
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed
@@ -21,7 +22,6 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.5.1
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aquasecurity/vuln-list-update/nvd"
 	oracleoval "github.com/aquasecurity/vuln-list-update/oracle/oval"
 	"github.com/aquasecurity/vuln-list-update/photon"
+	"github.com/aquasecurity/vuln-list-update/pypa"
 	redhatoval "github.com/aquasecurity/vuln-list-update/redhat/oval"
 	"github.com/aquasecurity/vuln-list-update/redhat/securitydataapi"
 	susecvrf "github.com/aquasecurity/vuln-list-update/suse/cvrf"
@@ -156,6 +157,12 @@ func run() error {
 			return xerrors.Errorf("error in Photon update: %w", err)
 		}
 		commitMsg = "Photon Security Advisories"
+	case "pypa":
+		p := pypa.NewPypa()
+		if err := p.Update(); err != nil {
+			return xerrors.Errorf("error in Pypa update: %w", err)
+		}
+		commitMsg = "Pypa Security Advisories"
 	case "ghsa":
 		src := oauth2.StaticTokenSource(
 			&oauth2.Token{AccessToken: githubToken},

pypa/pypa.go

@@ -0,0 +1,125 @@
+package pypa
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/aquasecurity/vuln-list-update/types"
+	"github.com/aquasecurity/vuln-list-update/utils"
+	"github.com/cheggaaa/pb"
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	pypaDir            = "pypa"
+	securityTrackerURL = "https://github.com/pypa/advisory-db/archive/refs/heads/main.zip"
+	retry              = 3
+	yamlExt            = ".yaml"
+)
+
+type options struct {
+	url   string
+	dir   string
+	retry int
+}
+type option func(*options)
+
+type Pypa struct {
+	opts  *options
+	AppFs afero.Fs
+}
+
+func WithURL(url string) option {
+	return func(opts *options) { opts.url = url }
+}
+
+func WithDir(dir string) option {
+	return func(opts *options) { opts.dir = dir }
+}
+
+func WithRetry(retry int) option {
+	return func(opts *options) { opts.retry = retry }
+}
+
+func NewPypa(opts ...option) Pypa {
+	o := &options{
+		url:   securityTrackerURL,
+		dir:   filepath.Join(utils.VulnListDir(), pypaDir),
+		retry: retry,
+	}
+
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return Pypa{
+		opts:  o,
+		AppFs: afero.NewOsFs(),
+	}
+
+}
+
+func (pypa *Pypa) Update() error {
+	dir, err := utils.DownloadToTempDir(context.Background(), pypa.opts.url)
+
+	if err != nil {
+		return xerrors.Errorf("failed to download %s: %w", pypa.opts.url, err)
+	}
+
+	vulnDir := filepath.Join(dir, "advisory-db-main/vulns")
+
+	yamlFiles, err := getYamlFiles(vulnDir)
+
+	if err != nil {
+		return xerrors.Errorf("failed to find vulnerability files in the directory %s: %w", vulnDir, err)
+	}
+
+	bar := pb.StartNew(len(yamlFiles))
+
+	for _, file := range yamlFiles {
+		data, err := ioutil.ReadFile(file)
+
+		if err != nil {
+			return xerrors.Errorf("unable to read %s: %w", file, err)
+		}
+
+		osv := &types.Osv{}
+
+		err = yaml.Unmarshal(data, osv)
+
+		if err != nil {
+			return xerrors.Errorf("unable to parse yaml %s: %w", file, err)
+		}
+
+		if err := utils.WriteJSON(pypa.AppFs, pypa.opts.dir, fmt.Sprintf("%s.json", osv.Id), osv); err != nil {
+			return xerrors.Errorf("failed to write file: %w", err)
+		}
+
+		bar.Increment()
+	}
+	bar.Finish()
+	return nil
+}
+func getYamlFiles(vulnDir string) ([]string, error) {
+	yamlFiles := make([]string, 0)
+
+	err := filepath.WalkDir(vulnDir,
+		func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !d.IsDir() && filepath.Ext(path) == yamlExt {
+				yamlFiles = append(yamlFiles, path)
+			}
+
+			return nil
+		})
+	return yamlFiles, err
+
+}

pypa/pypa_test.go

@@ -0,0 +1,78 @@
+package pypa
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Update(t *testing.T) {
+	tests := []struct {
+		name          string
+		inputArchive  string
+		expectedFiles []string
+		wantErr       string
+	}{
+		{
+			name:          "happy path",
+			inputArchive:  "testdata/pypa.zip",
+			expectedFiles: []string{"PYSEC-2005-1.json", "PYSEC-2006-1.json", "PYSEC-2006-2.json"},
+		},
+		{
+			name:    "sad path, unable to download archive",
+			wantErr: "connection refused",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				b, err := os.ReadFile(tt.inputArchive)
+				require.NoError(t, err)
+
+				w.Write(b)
+			}))
+
+			defer ts.Close()
+
+			// Intentionally close to induce network errors
+			if tt.inputArchive == "" {
+				ts.Close()
+			}
+
+			dir := t.TempDir()
+			c := NewPypa(WithURL(ts.URL+"/"+tt.inputArchive), WithDir(filepath.Join(dir)), WithRetry(0))
+			err := c.Update()
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			} else {
+				require.NoError(t, err)
+			}
+
+			entries, err := os.ReadDir("testdata/golden")
+			require.NoError(t, err)
+
+			for _, e := range entries {
+				if e.IsDir() {
+					continue
+				}
+
+				filePath := e.Name()
+				gotJSON, err := os.ReadFile(filepath.Join(dir, filePath))
+				require.NoError(t, err)
+
+				wantJSON, err := os.ReadFile(filepath.Join("testdata", "golden", filepath.Base(filePath)))
+				require.NoError(t, err)
+
+				assert.JSONEq(t, string(wantJSON), string(gotJSON))
+			}
+		})
+	}
+}

pypa/testdata/golden/PYSEC-2005-1.json

@@ -0,0 +1,59 @@
+{
+  "id": "PYSEC-2005-1",
+  "modified": "2021-07-16T01:31:33.917972Z",
+  "published": "2005-12-31T05:00:00Z",
+  "aliases": [
+    "CVE-2005-4644"
+  ],
+  "package": {
+    "ecosystem": "PyPI",
+    "name": "trac"
+  },
+  "details": "Cross-site scripting (XSS) vulnerability in the HTML WikiProcessor in Edgewall Trac 0.9.2 allows remote attackers to inject arbitrary web script or HTML via javascript in the SRC attribute of an IMG tag.",
+  "affects": {
+    "ranges": [
+      {
+        "type": "ECOSYSTEM",
+        "fixed": "0.10"
+      }
+    ],
+    "versions": [
+      "0.8.4",
+      "0.9"
+    ]
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "http://projects.edgewall.com/trac/ticket/2473"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.securityfocus.com/bid/16198"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18465"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.debian.org/security/2006/dsa-951"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18555"
+    },
+    {
+      "type": "WEB",
+      "url": "http://trac.edgewall.org/ticket/2473"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.vupen.com/english/advisories/2006/0226"
+    },
+    {
+      "type": "WEB",
+      "url": "https://exchange.xforce.ibmcloud.com/vulnerabilities/24183"
+    }
+  ]
+}

pypa/testdata/golden/PYSEC-2006-1.json

@@ -0,0 +1,66 @@
+{
+  "id": "PYSEC-2006-1",
+  "modified": "2021-07-05T00:01:17.388273Z",
+  "published": "2006-02-22T02:02:00Z",
+  "aliases": [
+    "CVE-2006-0847"
+  ],
+  "package": {
+    "ecosystem": "PyPI",
+    "name": "cherrypy"
+  },
+  "details": "Directory traversal vulnerability in the staticfilter component in CherryPy before 2.1.1 allows remote attackers to read arbitrary files via \"..\" sequences in unspecified vectors.",
+  "affects": {
+    "ranges": [
+      {
+        "type": "ECOSYSTEM",
+        "fixed": "2.1.1"
+      }
+    ],
+    "versions": [
+      "0.10",
+      "2.0.0-final",
+      "2.0.0b",
+      "2.1.0",
+      "2.1.0-beta"
+    ]
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "http://sourceforge.net/project/shownotes.php?release_id=384316\u0026group_id=56099"
+    },
+    {
+      "type": "WEB",
+      "url": "http://groups.google.com/group/cherrypy-announce/browse_thread/thread/92b2972f774fe6df/2f63afc9433dc306#2f63afc9433dc306"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.securityfocus.com/bid/16760"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.cherrypy.org/"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18944"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.gentoo.org/security/en/glsa/glsa-200605-16.xml"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/20344"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.vupen.com/english/advisories/2006/0677"
+    },
+    {
+      "type": "WEB",
+      "url": "https://exchange.xforce.ibmcloud.com/vulnerabilities/24809"
+    }
+  ]
+}