feat(python): add pypa support

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/hashicorp/go-getter v1.5.6
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed
@@ -21,7 +22,6 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.5.1
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aquasecurity/vuln-list-update/nvd"
 	oracleoval "github.com/aquasecurity/vuln-list-update/oracle/oval"
 	"github.com/aquasecurity/vuln-list-update/photon"
+	"github.com/aquasecurity/vuln-list-update/pypa"
 	redhatoval "github.com/aquasecurity/vuln-list-update/redhat/oval"
 	"github.com/aquasecurity/vuln-list-update/redhat/securitydataapi"
 	susecvrf "github.com/aquasecurity/vuln-list-update/suse/cvrf"
@@ -156,6 +157,12 @@ func run() error {
 			return xerrors.Errorf("error in Photon update: %w", err)
 		}
 		commitMsg = "Photon Security Advisories"
+	case "pypa":
+		p := pypa.NewPypa()
+		if err := p.Update(); err != nil {
+			return xerrors.Errorf("error in PyPA update: %w", err)
+		}
+		commitMsg = "PyPA Security Advisories"
 	case "ghsa":
 		src := oauth2.StaticTokenSource(
 			&oauth2.Token{AccessToken: githubToken},

pypa/pypa.go

@@ -0,0 +1,117 @@
+package pypa
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/aquasecurity/vuln-list-update/types"
+	"github.com/aquasecurity/vuln-list-update/utils"
+	"github.com/cheggaaa/pb"
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	pypaDir            = "pypa"
+	securityTrackerURL = "https://github.com/pypa/advisory-db/archive/refs/heads/main.zip"
+	yamlExt            = ".yaml"
+)
+
+type options struct {
+	url string
+	dir string
+}
+type option func(*options)
+
+type PyPA struct {
+	opts  *options
+	AppFs afero.Fs
+}
+
+func WithURL(url string) option {
+	return func(opts *options) { opts.url = url }
+}
+
+func WithDir(dir string) option {
+	return func(opts *options) { opts.dir = dir }
+}
+
+func NewPypa(opts ...option) PyPA {
+	o := &options{
+		url: securityTrackerURL,
+		dir: filepath.Join(utils.VulnListDir(), pypaDir),
+	}
+
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return PyPA{
+		opts: o,
+	}
+
+}
+
+func (pypa *PyPA) Update() error {
+	dir, err := utils.DownloadToTempDir(context.Background(), pypa.opts.url)
+
+	if err != nil {
+		return xerrors.Errorf("failed to download %s: %w", pypa.opts.url, err)
+	}
+
+	vulnDir := filepath.Join(dir, "advisory-db-main", "vulns")
+
+	yamlFiles, err := getYamlFiles(vulnDir)
+
+	if err != nil {
+		return xerrors.Errorf("failed to find vulnerability files in the directory %s: %w", vulnDir, err)
+	}
+
+	bar := pb.StartNew(len(yamlFiles))
+
+	for _, file := range yamlFiles {
+		data, err := os.ReadFile(file)
+
+		if err != nil {
+			return xerrors.Errorf("unable to read %s: %w", file, err)
+		}
+
+		osv := &types.Osv{}
+
+		err = yaml.Unmarshal(data, osv)
+
+		if err != nil {
+			return xerrors.Errorf("unable to parse yaml %s: %w", file, err)
+		}
+
+		if err := utils.WriteJSON(afero.NewOsFs(), filepath.Join(pypa.opts.dir, osv.Package.Name), fmt.Sprintf("%s.json", osv.Id), osv); err != nil {
+			return xerrors.Errorf("failed to write file: %w", err)
+		}
+
+		bar.Increment()
+	}
+	bar.Finish()
+	return nil
+}
+func getYamlFiles(vulnDir string) ([]string, error) {
+	yamlFiles := make([]string, 0)
+
+	err := filepath.WalkDir(vulnDir,
+		func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !d.IsDir() && filepath.Ext(path) == yamlExt {
+				yamlFiles = append(yamlFiles, path)
+			}
+
+			return nil
+		})
+	return yamlFiles, err
+
+}

pypa/pypa_test.go

@@ -0,0 +1,78 @@
+package pypa_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aquasecurity/vuln-list-update/pypa"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Update(t *testing.T) {
+	type fstruct struct {
+		pkg  string
+		name string
+	}
+	tests := []struct {
+		name         string
+		inputArchive string
+		wantFiles    []fstruct
+		wantErr      string
+	}{
+		{
+			name:         "happy path",
+			inputArchive: "testdata/pypa.zip",
+			wantFiles: []fstruct{{
+				"trac", "PYSEC-2005-1.json"}, {"cherrypy", "PYSEC-2006-1.json"}, {"trac", "PYSEC-2006-2.json"}},
+		},
+		{
+			name:    "sad path, unable to download archive",
+			wantErr: "connection refused",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				b, err := os.ReadFile(tt.inputArchive)
+				require.NoError(t, err)
+
+				w.Write(b)
+			}))
+
+			defer ts.Close()
+
+			// Intentionally close to induce network errors
+			if tt.inputArchive == "" {
+				ts.Close()
+			}
+
+			dir := t.TempDir()
+			c := pypa.NewPypa(pypa.WithURL(ts.URL+"/"+tt.inputArchive), pypa.WithDir(filepath.Join(dir)))
+			err := c.Update()
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+
+			for _, f := range tt.wantFiles {
+
+				filePath := filepath.Join(f.pkg, f.name)
+				gotJSON, err := os.ReadFile(filepath.Join(dir, filePath))
+				require.NoError(t, err)
+
+				wantJSON, err := os.ReadFile(filepath.Join("testdata", "golden", filePath))
+				require.NoError(t, err)
+
+				assert.JSONEq(t, string(wantJSON), string(gotJSON))
+			}
+		})
+	}
+}

pypa/testdata/golden/cherrypy/PYSEC-2006-1.json

@@ -0,0 +1,66 @@
+{
+  "id": "PYSEC-2006-1",
+  "modified": "2021-07-05T00:01:17.388273Z",
+  "published": "2006-02-22T02:02:00Z",
+  "aliases": [
+    "CVE-2006-0847"
+  ],
+  "package": {
+    "ecosystem": "PyPI",
+    "name": "cherrypy"
+  },
+  "details": "Directory traversal vulnerability in the staticfilter component in CherryPy before 2.1.1 allows remote attackers to read arbitrary files via \"..\" sequences in unspecified vectors.",
+  "affects": {
+    "ranges": [
+      {
+        "type": "ECOSYSTEM",
+        "fixed": "2.1.1"
+      }
+    ],
+    "versions": [
+      "0.10",
+      "2.0.0-final",
+      "2.0.0b",
+      "2.1.0",
+      "2.1.0-beta"
+    ]
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "http://sourceforge.net/project/shownotes.php?release_id=384316\u0026group_id=56099"
+    },
+    {
+      "type": "WEB",
+      "url": "http://groups.google.com/group/cherrypy-announce/browse_thread/thread/92b2972f774fe6df/2f63afc9433dc306#2f63afc9433dc306"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.securityfocus.com/bid/16760"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.cherrypy.org/"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18944"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.gentoo.org/security/en/glsa/glsa-200605-16.xml"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/20344"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.vupen.com/english/advisories/2006/0677"
+    },
+    {
+      "type": "WEB",
+      "url": "https://exchange.xforce.ibmcloud.com/vulnerabilities/24809"
+    }
+  ]
+}

pypa/testdata/golden/trac/PYSEC-2005-1.json

@@ -0,0 +1,59 @@
+{
+  "id": "PYSEC-2005-1",
+  "modified": "2021-07-16T01:31:33.917972Z",
+  "published": "2005-12-31T05:00:00Z",
+  "aliases": [
+    "CVE-2005-4644"
+  ],
+  "package": {
+    "ecosystem": "PyPI",
+    "name": "trac"
+  },
+  "details": "Cross-site scripting (XSS) vulnerability in the HTML WikiProcessor in Edgewall Trac 0.9.2 allows remote attackers to inject arbitrary web script or HTML via javascript in the SRC attribute of an IMG tag.",
+  "affects": {
+    "ranges": [
+      {
+        "type": "ECOSYSTEM",
+        "fixed": "0.10"
+      }
+    ],
+    "versions": [
+      "0.8.4",
+      "0.9"
+    ]
+  },
+  "references": [
+    {
+      "type": "WEB",
+      "url": "http://projects.edgewall.com/trac/ticket/2473"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.securityfocus.com/bid/16198"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18465"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.debian.org/security/2006/dsa-951"
+    },
+    {
+      "type": "WEB",
+      "url": "http://secunia.com/advisories/18555"
+    },
+    {
+      "type": "WEB",
+      "url": "http://trac.edgewall.org/ticket/2473"
+    },
+    {
+      "type": "WEB",
+      "url": "http://www.vupen.com/english/advisories/2006/0226"
+    },
+    {
+      "type": "WEB",
+      "url": "https://exchange.xforce.ibmcloud.com/vulnerabilities/24183"
+    }
+  ]
+}