From 1f8fca1fc77b989bb4e3ba820b297464dbdd825f Mon Sep 17 00:00:00 2001
From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com>
Date: Wed, 19 Jun 2024 13:47:42 +0600
Subject: [PATCH] feat(java): add support for `maven-metadata.xml` files for
remote snapshot repositories. (#6950)
---
pkg/dependency/parser/java/pom/metadata.go | 17 ++++
pkg/dependency/parser/java/pom/parse.go | 80 ++++++++++++++++++-
pkg/dependency/parser/java/pom/parse_test.go | 61 +++++++++++++-
... => example-dependency-1.2.3-SNAPSHOT.pom} | 0
...e-dependency-2.17.0-20240312.035235-10.pom | 23 ++++++
.../2.17.0-SNAPSHOT/maven-metadata.xml | 35 ++++++++
.../snapshot/with-maven-metadata/pom.xml | 20 +++++
7 files changed, 231 insertions(+), 5 deletions(-)
create mode 100644 pkg/dependency/parser/java/pom/metadata.go
rename pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/{example-dependency-1.2.3.pom => example-dependency-1.2.3-SNAPSHOT.pom} (100%)
create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom
create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml
create mode 100644 pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml
diff --git a/pkg/dependency/parser/java/pom/metadata.go b/pkg/dependency/parser/java/pom/metadata.go
new file mode 100644
index 000000000000..0a35e9e4f556
--- /dev/null
+++ b/pkg/dependency/parser/java/pom/metadata.go
@@ -0,0 +1,17 @@
+package pom
+
+type Metadata struct {
+ GroupId string `xml:"groupId"`
+ ArtifactId string `xml:"artifactId"`
+ Versioning Versioning `xml:"versioning"`
+ Version string `xml:"version"`
+}
+
+type Versioning struct {
+ SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"`
+}
+
+type SnapshotVersion struct {
+ Extension string `xml:"extension"`
+ Value string `xml:"value"`
+}
diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go
index bf8df2ad1c0a..141ebbee0f7c 100644
--- a/pkg/dependency/parser/java/pom/parse.go
+++ b/pkg/dependency/parser/java/pom/parse.go
@@ -14,6 +14,7 @@ import (
multierror "github.com/hashicorp/go-multierror"
"github.com/samber/lo"
+ "golang.org/x/exp/slices"
"golang.org/x/net/html/charset"
"golang.org/x/xerrors"
@@ -48,6 +49,12 @@ func WithReleaseRemoteRepos(repos []string) option {
}
}
+func WithSnapshotRemoteRepos(repos []string) option {
+ return func(opts *options) {
+ opts.snapshotRemoteRepos = repos
+ }
+}
+
type Parser struct {
logger *log.Logger
rootPath string
@@ -648,7 +655,18 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (
// try all remoteRepositories
for _, repo := range remoteRepos {
- fetched, err := p.fetchPOMFromRemoteRepository(repo, paths)
+ repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths`
+ if snapshot {
+ pomFileName, err := p.fetchPomFileNameFromMavenMetadata(repo, repoPaths)
+ if err != nil {
+ return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err)
+ }
+ // Use file name from `maven-metadata.xml` if it exists
+ if pomFileName != "" {
+ repoPaths[len(repoPaths)-1] = pomFileName
+ }
+ }
+ fetched, err := p.fetchPOMFromRemoteRepository(repo, repoPaths)
if err != nil {
return nil, xerrors.Errorf("fetch repository error: %w", err)
} else if fetched == nil {
@@ -659,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (
return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories")
}
-func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
+func (p *Parser) remoteRepoRequest(repo string, paths []string) (*http.Request, error) {
repoURL, err := url.Parse(repo)
if err != nil {
p.logger.Error("URL parse error", log.String("repo", repo))
@@ -670,7 +688,6 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
repoURL.Path = path.Join(paths...)
logger := p.logger.With(log.String("host", repoURL.Host), log.String("path", repoURL.Path))
- client := &http.Client{}
req, err := http.NewRequest("GET", repoURL.String(), http.NoBody)
if err != nil {
logger.Debug("HTTP request failed")
@@ -681,9 +698,54 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
req.SetBasicAuth(repoURL.User.Username(), password)
}
+ return req, nil
+}
+
+// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file.
+func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) (string, error) {
+ // Overwrite pom file name to `maven-metadata.xml`
+ mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths`
+ mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml")
+
+ req, err := p.remoteRepoRequest(repo, mavenMetadataPaths)
+ if err != nil {
+ return "", xerrors.Errorf("unable to create request for maven-metadata.xml file")
+ }
+
+ client := &http.Client{}
resp, err := client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
- logger.Debug("Failed to fetch")
+ p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
+ return "", nil
+ }
+ defer resp.Body.Close()
+
+ mavenMetadata, err := parseMavenMetadata(resp.Body)
+ if err != nil {
+ return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err)
+ }
+
+ var pomFileName string
+ for _, sv := range mavenMetadata.Versioning.SnapshotVersions {
+ if sv.Extension == "pom" {
+ // mavenMetadataPaths[len(mavenMetadataPaths)-3] is always artifactID
+ pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value)
+ }
+ }
+
+ return pomFileName, nil
+}
+
+func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
+ req, err := p.remoteRepoRequest(repo, paths)
+ if err != nil {
+ return nil, xerrors.Errorf("unable to create request for pom file")
+ }
+
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil || resp.StatusCode != http.StatusOK {
+ p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
return nil, nil
}
defer resp.Body.Close()
@@ -709,6 +771,16 @@ func parsePom(r io.Reader) (*pomXML, error) {
return parsed, nil
}
+func parseMavenMetadata(r io.Reader) (*Metadata, error) {
+ parsed := &Metadata{}
+ decoder := xml.NewDecoder(r)
+ decoder.CharsetReader = charset.NewReaderLabel
+ if err := decoder.Decode(parsed); err != nil {
+ return nil, xerrors.Errorf("xml decode error: %w", err)
+ }
+ return parsed, nil
+}
+
func packageID(name, version string) string {
return dependency.ID(ftypes.Pom, name, version)
}
diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go
index 1207f32adcf7..15740d599eb9 100644
--- a/pkg/dependency/parser/java/pom/parse_test.go
+++ b/pkg/dependency/parser/java/pom/parse_test.go
@@ -143,6 +143,13 @@ func TestPom_Parse(t *testing.T) {
},
},
},
+ {
+ ID: "org.example:example-api:2.0.0",
+ Name: "org.example:example-api",
+ Version: "2.0.0",
+ Licenses: []string{"The Apache Software License, Version 2.0"},
+ Relationship: ftypes.RelationshipIndirect,
+ },
},
wantDeps: []ftypes.Dependency{
{
@@ -151,6 +158,58 @@ func TestPom_Parse(t *testing.T) {
"org.example:example-dependency:1.2.3-SNAPSHOT",
},
},
+ {
+ ID: "org.example:example-dependency:1.2.3-SNAPSHOT",
+ DependsOn: []string{
+ "org.example:example-api:2.0.0",
+ },
+ },
+ },
+ },
+ {
+ name: "snapshot repository with maven-metadata.xml",
+ inputFile: filepath.Join("testdata", "snapshot", "with-maven-metadata", "pom.xml"),
+ local: false,
+ want: []ftypes.Package{
+ {
+ ID: "com.example:happy:1.0.0",
+ Name: "com.example:happy",
+ Version: "1.0.0",
+ Relationship: ftypes.RelationshipRoot,
+ },
+ {
+ ID: "org.example:example-dependency:2.17.0-SNAPSHOT",
+ Name: "org.example:example-dependency",
+ Version: "2.17.0-SNAPSHOT",
+ Relationship: ftypes.RelationshipDirect,
+ Locations: ftypes.Locations{
+ {
+ StartLine: 14,
+ EndLine: 18,
+ },
+ },
+ },
+ {
+ ID: "org.example:example-api:2.0.0",
+ Name: "org.example:example-api",
+ Version: "2.0.0",
+ Licenses: []string{"The Apache Software License, Version 2.0"},
+ Relationship: ftypes.RelationshipIndirect,
+ },
+ },
+ wantDeps: []ftypes.Dependency{
+ {
+ ID: "com.example:happy:1.0.0",
+ DependsOn: []string{
+ "org.example:example-dependency:2.17.0-SNAPSHOT",
+ },
+ },
+ {
+ ID: "org.example:example-dependency:2.17.0-SNAPSHOT",
+ DependsOn: []string{
+ "org.example:example-api:2.0.0",
+ },
+ },
},
},
{
@@ -1404,7 +1463,7 @@ func TestPom_Parse(t *testing.T) {
remoteRepos = []string{ts.URL}
}
- p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
+ p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
gotPkgs, gotDeps, err := p.Parse(f)
if tt.wantErr != "" {
diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom
similarity index 100%
rename from pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom
rename to pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom
diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom
new file mode 100644
index 000000000000..0ecec117f873
--- /dev/null
+++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom
@@ -0,0 +1,23 @@
+
+
+
+ 4.0.0
+
+ org.example
+ example-dependency
+ 2.17.0-SNAPSHOT
+
+ jar
+ Example API Dependency
+ The example API
+
+
+
+ org.example
+ example-api
+ 2.0.0
+
+
+
+
\ No newline at end of file
diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml
new file mode 100644
index 000000000000..258de9db99c5
--- /dev/null
+++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml
@@ -0,0 +1,35 @@
+
+ org.example
+ example-dependency
+
+ 20240312035235
+
+ 20240312.035235
+ 10
+
+
+
+ sources
+ jar
+ 2.17.0-20240312.035235-10
+ 20240312035235
+
+
+ module
+ 2.17.0-20240312.035235-10
+ 20240312035235
+
+
+ jar
+ 2.17.0-20240312.035235-10
+ 20240312035235
+
+
+ pom
+ 2.17.0-20240312.035235-10
+ 20240312035235
+
+
+
+ 2.17.0-SNAPSHOT
+
\ No newline at end of file
diff --git a/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml
new file mode 100644
index 000000000000..7e4c2144d417
--- /dev/null
+++ b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml
@@ -0,0 +1,20 @@
+
+ 4.0.0
+
+ com.example
+ happy
+ 1.0.0
+
+ happy
+ Example
+
+
+
+
+ org.example
+ example-dependency
+ 2.17.0-SNAPSHOT
+
+
+