Skip to content

Commit

Permalink
feat(java): add support for fetching packages from repos mentioned in…
Browse files Browse the repository at this point in the history
… pom.xml (#6171)

Co-authored-by: DmitriyLewen <dmitriy.lewen@smartforce.io>
  • Loading branch information
renypaul and DmitriyLewen committed Feb 22, 2024
1 parent cf0f0d0 commit ce81c05
Show file tree
Hide file tree
Showing 8 changed files with 286 additions and 96 deletions.
22 changes: 20 additions & 2 deletions pkg/dependency/parser/java/pom/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type parser struct {
localRepository string
remoteRepositories []string
offline bool
servers []Server
}

func NewParser(filePath string, opts ...option) types.Parser {
Expand All @@ -78,6 +79,7 @@ func NewParser(filePath string, opts ...option) types.Parser {
localRepository: localRepository,
remoteRepositories: o.remoteRepos,
offline: o.offline,
servers: s.Servers,
}
}

Expand Down Expand Up @@ -312,7 +314,7 @@ func (p *parser) analyze(pom *pom, opts analysisOptions) (analysisResult, error)
}

// Update remoteRepositories
p.remoteRepositories = utils.UniqueStrings(append(p.remoteRepositories, pom.repositories()...))
p.remoteRepositories = utils.UniqueStrings(append(pom.repositories(p.servers), p.remoteRepositories...))

// Parent
parent, err := p.parseParent(pom.filePath, pom.content.Parent)
Expand Down Expand Up @@ -586,6 +588,10 @@ func (p *parser) openPom(filePath string) (*pom, error) {
}, nil
}
func (p *parser) tryRepository(groupID, artifactID, version string) (*pom, error) {
if version == "" {
return nil, xerrors.Errorf("Version missing for %s:%s", groupID, artifactID)
}

// Generate a proper path to the pom.xml
// e.g. com.fasterxml.jackson.core, jackson-annotations, 2.10.0
// => com/fasterxml/jackson/core/jackson-annotations/2.10.0/jackson-annotations-2.10.0.pom
Expand Down Expand Up @@ -644,8 +650,20 @@ func fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
paths = append([]string{repoURL.Path}, paths...)
repoURL.Path = path.Join(paths...)

resp, err := http.Get(repoURL.String())
client := &http.Client{}
req, err := http.NewRequest("GET", repoURL.String(), http.NoBody)
if err != nil {
log.Logger.Debugf("Request failed for %s%s", repoURL.Host, repoURL.Path)
return nil, nil
}
if repoURL.User != nil {
password, _ := repoURL.User.Password()
req.SetBasicAuth(repoURL.User.Username(), password)
}

resp, err := client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
log.Logger.Debugf("Failed to fetch from %s%s", repoURL.Host, repoURL.Path)
return nil, nil
}
defer resp.Body.Close()
Expand Down
2 changes: 1 addition & 1 deletion pkg/dependency/parser/java/pom/parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1226,7 +1226,7 @@ func TestPom_Parse(t *testing.T) {
var remoteRepos []string
if tt.local {
// for local repository
t.Setenv("MAVEN_HOME", "testdata")
t.Setenv("MAVEN_HOME", "testdata/settings/global")
} else {
// for remote repository
h := http.FileServer(http.Dir(filepath.Join("testdata", "repository")))
Expand Down
65 changes: 45 additions & 20 deletions pkg/dependency/parser/java/pom/pom.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (
"fmt"
"io"
"maps"
"net/url"
"reflect"
"strings"

"github.com/samber/lo"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy/pkg/dependency/parser/log"
"github.com/aquasecurity/trivy/pkg/dependency/parser/types"
"github.com/aquasecurity/trivy/pkg/dependency/parser/utils"
)
Expand Down Expand Up @@ -113,12 +115,31 @@ func (p pom) licenses() []string {
})
}

func (p pom) repositories() []string {
func (p pom) repositories(servers []Server) []string {
var urls []string
for _, rep := range p.content.Repositories.Repository {
if rep.Releases.Enabled != "false" {
urls = append(urls, rep.URL)
// Add only enabled repositories
if rep.Releases.Enabled == "false" && rep.Snapshots.Enabled == "false" {
continue
}

repoURL, err := url.Parse(rep.URL)
if err != nil {
log.Logger.Debugf("Unable to parse remote repository url: %s", err)
continue
}

// Get the credentials from settings.xml based on matching server id
// with the repository id from pom.xml and use it for accessing the repository url
for _, server := range servers {
if rep.ID == server.ID && server.Username != "" && server.Password != "" {
repoURL.User = url.UserPassword(server.Username, server.Password)
break
}
}

log.Logger.Debugf("Adding repository %s: %s", rep.ID, rep.URL)
urls = append(urls, repoURL.String())
}
return urls
}
Expand All @@ -139,23 +160,7 @@ type pomXML struct {
Dependencies pomDependencies `xml:"dependencies"`
} `xml:"dependencyManagement"`
Dependencies pomDependencies `xml:"dependencies"`
Repositories struct {
Text string `xml:",chardata"`
Repository []struct {
Text string `xml:",chardata"`
ID string `xml:"id"`
Name string `xml:"name"`
URL string `xml:"url"`
Releases struct {
Text string `xml:",chardata"`
Enabled string `xml:"enabled"`
} `xml:"releases"`
Snapshots struct {
Text string `xml:",chardata"`
Enabled string `xml:"enabled"`
} `xml:"snapshots"`
} `xml:"repository"`
} `xml:"repositories"`
Repositories pomRepositories `xml:"repositories"`
}

type pomParent struct {
Expand Down Expand Up @@ -350,3 +355,23 @@ func findDep(name string, depManagement []pomDependency) (pomDependency, bool) {
return item.Name() == name
})
}

type pomRepositories struct {
Text string `xml:",chardata"`
Repository []pomRepository `xml:"repository"`
}

type pomRepository struct {
Text string `xml:",chardata"`
ID string `xml:"id"`
Name string `xml:"name"`
URL string `xml:"url"`
Releases struct {
Text string `xml:",chardata"`
Enabled string `xml:"enabled"`
} `xml:"releases"`
Snapshots struct {
Text string `xml:",chardata"`
Enabled string `xml:"enabled"`
} `xml:"snapshots"`
}
49 changes: 42 additions & 7 deletions pkg/dependency/parser/java/pom/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,59 @@ import (
"golang.org/x/net/html/charset"
)

type Server struct {
ID string `xml:"id"`
Username string `xml:"username"`
Password string `xml:"password"`
}

type settings struct {
LocalRepository string `xml:"localRepository"`
LocalRepository string `xml:"localRepository"`
Servers []Server `xml:"servers>server"`
}

// serverFound checks that servers already contain server.
// Maven compares servers by ID only.
func serverFound(servers []Server, id string) bool {
for _, server := range servers {
if server.ID == id {
return true
}
}
return false
}

func readSettings() settings {
s := settings{}

userSettingsPath := filepath.Join(os.Getenv("HOME"), ".m2", "settings.xml")
userSettings, err := openSettings(userSettingsPath)
if err == nil && userSettings.LocalRepository != "" {
return userSettings
if err == nil {
s = userSettings
}

globalSettingsPath := filepath.Join(os.Getenv("MAVEN_HOME"), "conf", "settings.xml")
// Some package managers use this path by default
mavenHome := "/usr/share/maven"
if mHome := os.Getenv("MAVEN_HOME"); mHome != "" {
mavenHome = mHome
}
globalSettingsPath := filepath.Join(mavenHome, "conf", "settings.xml")
globalSettings, err := openSettings(globalSettingsPath)
if err == nil && globalSettings.LocalRepository != "" {
return globalSettings
if err == nil {
// We need to merge global and user settings. User settings being dominant.
// https://maven.apache.org/settings.html#quick-overview
if s.LocalRepository == "" {
s.LocalRepository = globalSettings.LocalRepository
}
// Maven checks user servers first, then global servers
for _, server := range globalSettings.Servers {
if !serverFound(s.Servers, server.ID) {
s.Servers = append(s.Servers, server)
}
}
}

return settings{}
return s
}

func openSettings(filePath string) (settings, error) {
Expand Down
136 changes: 136 additions & 0 deletions pkg/dependency/parser/java/pom/settings_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package pom

import (
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
)

func Test_ReadSettings(t *testing.T) {
tests := []struct {
name string
envs map[string]string
wantSettings settings
}{
{
name: "happy path with only global settings",
envs: map[string]string{
"HOME": "",
"MAVEN_HOME": filepath.Join("testdata", "settings", "global"),
},
wantSettings: settings{
LocalRepository: "testdata/repository",
Servers: []Server{
{
ID: "global-server",
},
{
ID: "server-with-credentials",
Username: "test-user",
Password: "test-password-from-global",
},
{
ID: "server-with-name-only",
Username: "test-user-only",
},
},
},
},
{
name: "happy path with only user settings",
envs: map[string]string{
"HOME": filepath.Join("testdata", "settings", "user"),
"MAVEN_HOME": "NOT_EXISTING_PATH",
},
wantSettings: settings{
LocalRepository: "testdata/user/repository",
Servers: []Server{
{
ID: "user-server",
},
{
ID: "server-with-credentials",
Username: "test-user",
Password: "test-password",
},
{
ID: "server-with-name-only",
Username: "test-user-only",
},
},
},
},
{
// $ mvn help:effective-settings
//[INFO] ------------------< org.apache.maven:standalone-pom >-------------------
//[INFO] --- maven-help-plugin:3.4.0:effective-settings (default-cli) @ standalone-pom ---
//Effective user-specific configuration settings:
//
//<?xml version="1.0" encoding="UTF-8"?>
//<settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd">
// <localRepository>/root/testdata/user/repository</localRepository>
// <servers>
// <server>
// <id>user-server</id>
// </server>
// <server>
// <username>test-user</username>
// <password>***</password>
// <id>server-with-credentials</id>
// </server>
// <server>
// <username>test-user-only</username>
// <id>server-with-name-only</id>
// </server>
// <server>
// <id>global-server</id>
// </server>
// </servers>
//</settings>
name: "happy path with global and user settings",
envs: map[string]string{
"HOME": filepath.Join("testdata", "settings", "user"),
"MAVEN_HOME": filepath.Join("testdata", "settings", "global"),
},
wantSettings: settings{
LocalRepository: "testdata/user/repository",
Servers: []Server{
{
ID: "user-server",
},
{
ID: "server-with-credentials",
Username: "test-user",
Password: "test-password",
},
{
ID: "server-with-name-only",
Username: "test-user-only",
},
{
ID: "global-server",
},
},
},
},
{
name: "without settings",
envs: map[string]string{
"HOME": "",
"MAVEN_HOME": "NOT_EXISTING_PATH",
},
wantSettings: settings{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
for env, settingsDir := range tt.envs {
t.Setenv(env, settingsDir)
}

gotSettings := readSettings()
require.Equal(t, tt.wantSettings, gotSettings)
})
}
}

0 comments on commit ce81c05

Please sign in to comment.