Skip to content

Commit

Permalink
refactor opensuse and sle updater fetchers
Browse files Browse the repository at this point in the history 
I just took the common code and extract it to a new class called Oval.
We can use this also in rhel code.

I left some common code on some Parse* methods cause before merging
that I wanted to take a look at the rhel code.
  • Loading branch information
@jordimassaguerpla
jordimassaguerpla committed Jun 30, 2016
1 parent 5609458 commit c5833fc
Show file tree
Hide file tree
Showing 5 changed files with 417 additions and 593 deletions.
341 changes: 50 additions & 291 deletions updater/fetchers/opensuse/opensuse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,329 +15,88 @@
package opensuse

import (
"bufio"
"encoding/xml"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
"strings"

"github.com/coreos/clair/database"
"strconv"
"github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/updater/fetchers/oval"
"github.com/coreos/pkg/capnslog"
)

const (
ovalURI = "http://ftp.suse.com/pub/projects/security/oval/"
updaterFlag = "opensuseUpdater"
distMinVersion = 13.1
)

var (
ignoredCriterions = []string{}
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/sle")

critSystem = regexp.MustCompile(`openSUSE [^0-9]*(\d+\.\d+)[^0-9]* is installed`)
critPackage = regexp.MustCompile(`(.*)-(.*\-[\d\.]+) is installed`)
distRegexp = regexp.MustCompile(`opensuse.[^0-9]*(\d+\.\d+).xml`)

log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/opensuse")
)

type oval struct {
Definitions []definition `xml:"definitions>definition"`
}
func init() {
opensuse_info := &OpenSUSE_Info{}

type definition struct {
Title string `xml:"metadata>title"`
Description string `xml:"metadata>description"`
References []reference `xml:"metadata>reference"`
Criteria criteria `xml:"criteria"`
updater.RegisterFetcher(opensuse_info.DistName(),
&oval.OvalFetcher{Os_info: opensuse_info})
}

type reference struct {
Source string `xml:"source,attr"`
URI string `xml:"ref_url,attr"`
type OpenSUSE_Info struct {
}

type criteria struct {
Operator string `xml:"operator,attr"`
Criterias []*criteria `xml:"criteria"`
Criterions []criterion `xml:"criterion"`
func (f *OpenSUSE_Info) CritSystem() *regexp.Regexp {
return regexp.MustCompile(`openSUSE [^0-9]*(\d+\.\d+)[^0-9]* is installed`)
}

type criterion struct {
Comment string `xml:"comment,attr"`
func (f *OpenSUSE_Info) CritPackage() *regexp.Regexp {
return regexp.MustCompile(`(.*)-(.*\-[\d\.]+) is installed`)
}

func init() {
updater.RegisterFetcher("openSUSE", &OpenSUSEFetcher{})
func (f *OpenSUSE_Info) DistRegexp() *regexp.Regexp {
return regexp.MustCompile(`opensuse.[^0-9]*(\d+\.\d+).xml`)
}

// OpenSUSEFetcher implements updater.Fetcher.
type OpenSUSEFetcher struct{}

// FetchUpdate gets vulnerability updates from the OVAL definitions.
func (f *OpenSUSEFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
log.Info("fetching openSUSE vulnerabilities")

// TODO: Skip already loaded records

// flagValue, err := datastore.GetKeyValue(updaterFlag)

// if err != nil {
// return resp, err
// }

r, err := http.Get(ovalURI)

if err != nil {
log.Errorf("could not download openSUSE's update list: %s", err)
return resp, cerrors.ErrCouldNotDownload
}

var distList []string
scanner := bufio.NewScanner(r.Body)

for scanner.Scan() {
line := scanner.Text()
r := distRegexp.FindStringSubmatch(line)

if len(r) == 2 {
distVersion, _ := strconv.ParseFloat(r[1], 32)
if distVersion >= distMinVersion {
distList = append(distList, r[0])
}
}
}

for _, distFile := range distList {
r, err := http.Get(ovalURI + distFile)

if err != nil {
log.Errorf("could not download openSUSE's update file: %s", err)
return resp, cerrors.ErrCouldNotDownload
}

vs, err := parseOval(r.Body)

if err != nil {
return resp, err
}

for _, v := range vs {
resp.Vulnerabilities = append(resp.Vulnerabilities, v)
}
}

return resp, nil
func (f *OpenSUSE_Info) OvalURI() string {
return "http://ftp.suse.com/pub/projects/security/oval/"
}

// Clean deletes any allocated resources.
func (f *OpenSUSEFetcher) Clean() {}

func parseOval(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
var ov oval
err = xml.NewDecoder(ovalReader).Decode(&ov)

if err != nil {
log.Errorf("could not decode openSUSE's XML: %s", err)
return vulnerabilities, cerrors.ErrCouldNotParse
}

for _, definition := range ov.Definitions {
pkgs := toFeatureVersions(definition.Criteria)

if len(pkgs) > 0 {
vulnerability := database.Vulnerability{
Name: name(definition),
Link: link(definition),
Severity: priority(definition),
Description: description(definition),
}

for _, p := range pkgs {
vulnerability.FixedIn = append(vulnerability.FixedIn, p)
}

vulnerabilities = append(vulnerabilities, vulnerability)
}
}

return
func (f *OpenSUSE_Info) UpdaterFlag() string {
return "opensuseUpdater"
}

func toFeatureVersions(criteria criteria) []database.FeatureVersion {
featureVersionParameters := make(map[string]database.FeatureVersion)
possibilities := getPossibilities(criteria)

for _, criterions := range possibilities {
var (
featureVersion database.FeatureVersion
osVersion string
err error
)

for _, c := range criterions {
systemMatch := critSystem.FindStringSubmatch(c.Comment)

if len(systemMatch) == 2 {
osVersion = systemMatch[1]

// log.Infof("got %s as system version from: %s", osVersion, c.Comment)

continue
}

packageMatch := critPackage.FindStringSubmatch(c.Comment)

if len(packageMatch) == 3 {
featureVersion.Feature.Name = packageMatch[1]
featureVersion.Version, err = types.NewVersion(packageMatch[2])

if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", packageMatch[2], err.Error())
}

// log.Infof("got %s = %s as pkg version from: %s", packageMatch[1], packageMatch[2], c.Comment)

continue
}

log.Warningf("could not parse criteria: '%s'.", c.Comment)
}

if osVersion == "" {
log.Warning(criterions)
}

featureVersion.Feature.Namespace.Name = fmt.Sprintf("opensuse:%s", osVersion)

if featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion
} else {
log.Warningf("could not determine a valid package from criterions: %v", criterions)
}
}

var featureVersionParametersArray []database.FeatureVersion

for _, fv := range featureVersionParameters {
featureVersionParametersArray = append(featureVersionParametersArray, fv)
}

return featureVersionParametersArray
func (f *OpenSUSE_Info) DistMinVersion() float64 {
return 13.1
}

func getCriterions(node criteria) [][]criterion {
var criterions []criterion

for _, c := range node.Criterions {
ignored := false

for _, ignoredItem := range ignoredCriterions {
if strings.Contains(c.Comment, ignoredItem) {
ignored = true
break
}
}

if !ignored {
criterions = append(criterions, c)
}
}

if node.Operator == "AND" {
return [][]criterion{criterions}
} else if node.Operator == "OR" {
var possibilities [][]criterion

for _, c := range criterions {
possibilities = append(possibilities, []criterion{c})
}

return possibilities
}

return [][]criterion{}
func (f *OpenSUSE_Info) DistName() string {
return "opensuse"
}

func getPossibilities(node criteria) [][]criterion {
if len(node.Criterias) == 0 {
return getCriterions(node)
}

var possibilitiesToCompose [][][]criterion

for _, criteria := range node.Criterias {
possibilitiesToCompose = append(possibilitiesToCompose, getPossibilities(*criteria))
}

if len(node.Criterions) > 0 {
possibilitiesToCompose = append(possibilitiesToCompose, getCriterions(node))
}

var possibilities [][]criterion

if node.Operator == "AND" {
for _, possibility := range possibilitiesToCompose[0] {
possibilities = append(possibilities, possibility)
}
func (f *OpenSUSE_Info) ParseOsVersion(comment string) string {
systemMatch := f.CritSystem().FindStringSubmatch(comment)
if len(systemMatch) < 2 {
return ""
}
osVersion := systemMatch[1]
if len(systemMatch) == 4 && systemMatch[3] != "" {
sp:= systemMatch[3]
osVersion = fmt.Sprintf("%s.%s", osVersion, sp)
}

for _, possibilityGroup := range possibilitiesToCompose[1:] {
var newPossibilities [][]criterion
// log.Infof("got %s as system version from: %s", osVersion, c.Comment)
return osVersion

for _, possibility := range possibilities {
for _, possibilityInGroup := range possibilityGroup {
var p []criterion

p = append(p, possibility...)
p = append(p, possibilityInGroup...)

newPossibilities = append(newPossibilities, p)
}
}

possibilities = newPossibilities
}
} else if node.Operator == "OR" {
for _, possibilityGroup := range possibilitiesToCompose {
for _, possibility := range possibilityGroup {
possibilities = append(possibilities, possibility)
}
}
}

return possibilities
}

func description(def definition) (desc string) {
desc = strings.Replace(def.Description, "\n\n\n", " ", -1)
desc = strings.Replace(desc, "\n\n", " ", -1)
desc = strings.Replace(desc, "\n", " ", -1)
func (f *OpenSUSE_Info) ParsePackageNameVersion(comment string) (string, string) {
packageMatch := f.CritPackage().FindStringSubmatch(comment)

return
if len(packageMatch) != 3 {
return "", ""
}
name := packageMatch[1]
version:= packageMatch[2]
return name, version
}

func name(def definition) string {
return strings.TrimSpace(def.Title)
}

func link(def definition) (link string) {
for _, reference := range def.References {
if reference.Source == "CVE" {
link = reference.URI
break
}
}

return
}
func (f *OpenSUSE_Info) ParseFilenameDistVersion(line string) (string, float64) {
r := f.DistRegexp().FindStringSubmatch(line)
if len(r) == 2 {
distVersion, _ := strconv.ParseFloat(r[1], 32)
return r[0], distVersion
}
return "", 0
}

func priority(def definition) types.Priority {
// The OVAL files provided by SUSE doesn't include any priority/severity yet.
return types.Unknown
}
Loading

0 comments on commit c5833fc

Please sign in to comment.