Skip to content

Commit

Permalink
Support parsing source packages in featurefmt
Browse files Browse the repository at this point in the history
  • Loading branch information
@KeyboardNerd
KeyboardNerd committed Feb 13, 2019
1 parent 0a5e87b commit bd4a4ec
Show file tree
Hide file tree
Showing 7 changed files with 612 additions and 384 deletions.
8 changes: 4 additions & 4 deletions database/pgsql/testutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ import (
// int keys must be the consistent with the database ID.
var (
realFeatures = map[int]database.Feature{
1: {"ourchat", "0.5", "ourchat", "0.5", "dpkg"},
2: {"openssl", "1.0", "openssl", "1.0", "dpkg"},
3: {"openssl", "2.0", "openssl", "2.0", "dpkg"},
4: {"fake", "2.0", "fake", "2.0", "rpm"},
1: {"ourchat", "0.5", "dpkg", "binary"},
2: {"openssl", "1.0", "dpkg", "binary"},
3: {"openssl", "2.0", "dpkg", "binary"},
4: {"fake", "2.0", "rpm", "binary"},
}

realNamespaces = map[int]database.Namespace{
Expand Down
2 changes: 2 additions & 0 deletions ext/featurefmt/apk/apk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
line := scanner.Text()
if len(line) < 2 {
if valid(&pkg) {
pkg.Type = database.BinaryPackage
packages.Add(pkg)
pkg = database.Feature{VersionFormat: dpkg.ParserName}
}
Expand All @@ -81,6 +82,7 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)

// in case of no terminal line
if valid(&pkg) {
pkg.Type = database.BinaryPackage
packages.Add(pkg)
}

Expand Down
22 changes: 11 additions & 11 deletions ext/featurefmt/apk/apk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@ func TestAPKFeatureDetection(t *testing.T) {
"valid case",
map[string]string{"lib/apk/db/installed": "apk/testdata/valid"},
[]database.Feature{
{"musl", "1.1.14-r10", "", "", dpkg.ParserName},
{"busybox", "1.24.2-r9", "", "", dpkg.ParserName},
{"alpine-baselayout", "3.0.3-r0", "", "", dpkg.ParserName},
{"alpine-keys", "1.1-r0", "", "", dpkg.ParserName},
{"zlib", "1.2.8-r2", "", "", dpkg.ParserName},
{"libcrypto1.0", "1.0.2h-r1", "", "", dpkg.ParserName},
{"libssl1.0", "1.0.2h-r1", "", "", dpkg.ParserName},
{"apk-tools", "2.6.7-r0", "", "", dpkg.ParserName},
{"scanelf", "1.1.6-r0", "", "", dpkg.ParserName},
{"musl-utils", "1.1.14-r10", "", "", dpkg.ParserName},
{"libc-utils", "0.7-r0", "", "", dpkg.ParserName},
{"apk-tools", "2.6.7-r0", "dpkg", "binary"},
{"musl", "1.1.14-r10", "dpkg", "binary"},
{"libssl1.0", "1.0.2h-r1", "dpkg", "binary"},
{"libc-utils", "0.7-r0", "dpkg", "binary"},
{"busybox", "1.24.2-r9", "dpkg", "binary"},
{"scanelf", "1.1.6-r0", "dpkg", "binary"},
{"alpine-keys", "1.1-r0", "dpkg", "binary"},
{"libcrypto1.0", "1.0.2h-r1", "dpkg", "binary"},
{"zlib", "1.2.8-r2", "dpkg", "binary"},
{"musl-utils", "1.1.14-r10", "dpkg", "binary"},
{"alpine-baselayout", "3.0.3-r0", "dpkg", "binary"},
},
},
} {
Expand Down
121 changes: 74 additions & 47 deletions ext/featurefmt/dpkg/dpkg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,12 @@ var (

type lister struct{}

func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{})
}

func valid(pkg *database.Feature) bool {
return pkg.Name != "" && pkg.Version != ""
func (l lister) RequiredFilenames() []string {
return []string{"var/lib/dpkg/status"}
}

func addSourcePackage(pkg *database.Feature) {
if pkg.SourceName == "" {
pkg.SourceName = pkg.Name
}

if pkg.SourceVersion == "" {
pkg.SourceVersion = pkg.Version
}
func init() {
featurefmt.RegisterLister("dpkg", "1.0", &lister{})
}

func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error) {
Expand All @@ -61,21 +51,45 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
return []database.Feature{}, nil
}

packages := mapset.NewSet()
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}

binary, source := parseDpkgDB(scanner)
if binary != nil {
packages.Add(*binary)
}

if source != nil {
packages.Add(*source)
}
}

return database.ConvertFeatureSetToFeatures(packages), nil
}

// parseDpkgDB consumes the status file scanner exactly one package info, until
// EOF or empty space, and generate the parsed packages from it.
func parseDpkgDB(scanner *bufio.Scanner) (binaryPackage *database.Feature, sourcePackage *database.Feature) {
var (
pkg = database.Feature{VersionFormat: dpkg.ParserName}
pkgs = mapset.NewSet()
err error
name string
version string
sourceName string
sourceVersion string
)

scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
// Package line
// Defines the name of the package
for {
line := strings.TrimSpace(scanner.Text())
if line == "" {
break
}

pkg.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = ""
if strings.HasPrefix(line, "Package: ") {
name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
} else if strings.HasPrefix(line, "Source: ") {
// Source line (Optional)
// Gives the name of the source package
Expand All @@ -87,40 +101,53 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.Feature, error)
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}

pkg.SourceName = md["name"]
sourceName = md["name"]
if md["version"] != "" {
version := md["version"]
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.SourceVersion = version
}
sourceVersion = md["version"]
}
} else if strings.HasPrefix(line, "Version: ") {
// Version line
// Defines the version of the package
// This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter
version := strings.TrimPrefix(line, "Version: ")
if err = versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithField("version", string(line[1])).Warning("could not parse package version. skipping")
} else {
pkg.Version = version
}
} else if line == "" {
pkg = database.Feature{VersionFormat: dpkg.ParserName}
version = strings.TrimPrefix(line, "Version: ")
}

if valid(&pkg) {
addSourcePackage(&pkg)
pkgs.Add(pkg)
if !scanner.Scan() {
break
}
}

return database.ConvertFeatureSetToFeatures(pkgs), nil
}
if name != "" && version != "" {
if err := versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package")
} else {
binaryPackage = &database.Feature{name, version, dpkg.ParserName, database.BinaryPackage}
}
}

func (l lister) RequiredFilenames() []string {
return []string{"var/lib/dpkg/status"}
// Source version and names are computed from binary package names and versions
// in dpkg.
// Source package name:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n338
// Source package version:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n355
if sourceName == "" {
sourceName = name
}

if sourceVersion == "" {
sourceVersion = version
}

if sourceName != "" && sourceVersion != "" {
if err := versionfmt.Valid(dpkg.ParserName, version); err != nil {
log.WithError(err).WithFields(log.Fields{"name": name, "version": version}).Warning("skipped unparseable package")
} else {
sourcePackage = &database.Feature{sourceName, sourceVersion, dpkg.ParserName, database.SourcePackage}
}
}

return
}
Loading

0 comments on commit bd4a4ec

Please sign in to comment.