From 1f8fca1fc77b989bb4e3ba820b297464dbdd825f Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:47:42 +0600 Subject: [PATCH] feat(java): add support for `maven-metadata.xml` files for remote snapshot repositories. (#6950) --- pkg/dependency/parser/java/pom/metadata.go | 17 ++++ pkg/dependency/parser/java/pom/parse.go | 80 ++++++++++++++++++- pkg/dependency/parser/java/pom/parse_test.go | 61 +++++++++++++- ... => example-dependency-1.2.3-SNAPSHOT.pom} | 0 ...e-dependency-2.17.0-20240312.035235-10.pom | 23 ++++++ .../2.17.0-SNAPSHOT/maven-metadata.xml | 35 ++++++++ .../snapshot/with-maven-metadata/pom.xml | 20 +++++ 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 pkg/dependency/parser/java/pom/metadata.go rename pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/{example-dependency-1.2.3.pom => example-dependency-1.2.3-SNAPSHOT.pom} (100%) create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom create mode 100644 pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml create mode 100644 pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml diff --git a/pkg/dependency/parser/java/pom/metadata.go b/pkg/dependency/parser/java/pom/metadata.go new file mode 100644 index 000000000000..0a35e9e4f556 --- /dev/null +++ b/pkg/dependency/parser/java/pom/metadata.go @@ -0,0 +1,17 @@ +package pom + +type Metadata struct { + GroupId string `xml:"groupId"` + ArtifactId string `xml:"artifactId"` + Versioning Versioning `xml:"versioning"` + Version string `xml:"version"` +} + +type Versioning struct { + SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"` +} + +type SnapshotVersion struct { + Extension string `xml:"extension"` + Value string `xml:"value"` +} diff --git a/pkg/dependency/parser/java/pom/parse.go b/pkg/dependency/parser/java/pom/parse.go index bf8df2ad1c0a..141ebbee0f7c 100644 --- a/pkg/dependency/parser/java/pom/parse.go +++ b/pkg/dependency/parser/java/pom/parse.go @@ -14,6 +14,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/samber/lo" + "golang.org/x/exp/slices" "golang.org/x/net/html/charset" "golang.org/x/xerrors" @@ -48,6 +49,12 @@ func WithReleaseRemoteRepos(repos []string) option { } } +func WithSnapshotRemoteRepos(repos []string) option { + return func(opts *options) { + opts.snapshotRemoteRepos = repos + } +} + type Parser struct { logger *log.Logger rootPath string @@ -648,7 +655,18 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( // try all remoteRepositories for _, repo := range remoteRepos { - fetched, err := p.fetchPOMFromRemoteRepository(repo, paths) + repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths` + if snapshot { + pomFileName, err := p.fetchPomFileNameFromMavenMetadata(repo, repoPaths) + if err != nil { + return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err) + } + // Use file name from `maven-metadata.xml` if it exists + if pomFileName != "" { + repoPaths[len(repoPaths)-1] = pomFileName + } + } + fetched, err := p.fetchPOMFromRemoteRepository(repo, repoPaths) if err != nil { return nil, xerrors.Errorf("fetch repository error: %w", err) } else if fetched == nil { @@ -659,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) ( return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories") } -func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) { +func (p *Parser) remoteRepoRequest(repo string, paths []string) (*http.Request, error) { repoURL, err := url.Parse(repo) if err != nil { p.logger.Error("URL parse error", log.String("repo", repo)) @@ -670,7 +688,6 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom repoURL.Path = path.Join(paths...) logger := p.logger.With(log.String("host", repoURL.Host), log.String("path", repoURL.Path)) - client := &http.Client{} req, err := http.NewRequest("GET", repoURL.String(), http.NoBody) if err != nil { logger.Debug("HTTP request failed") @@ -681,9 +698,54 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom req.SetBasicAuth(repoURL.User.Username(), password) } + return req, nil +} + +// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file. +func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) (string, error) { + // Overwrite pom file name to `maven-metadata.xml` + mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths` + mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml") + + req, err := p.remoteRepoRequest(repo, mavenMetadataPaths) + if err != nil { + return "", xerrors.Errorf("unable to create request for maven-metadata.xml file") + } + + client := &http.Client{} resp, err := client.Do(req) if err != nil || resp.StatusCode != http.StatusOK { - logger.Debug("Failed to fetch") + p.logger.Debug("Failed to fetch", log.String("url", req.URL.String())) + return "", nil + } + defer resp.Body.Close() + + mavenMetadata, err := parseMavenMetadata(resp.Body) + if err != nil { + return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err) + } + + var pomFileName string + for _, sv := range mavenMetadata.Versioning.SnapshotVersions { + if sv.Extension == "pom" { + // mavenMetadataPaths[len(mavenMetadataPaths)-3] is always artifactID + pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value) + } + } + + return pomFileName, nil +} + +func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) { + req, err := p.remoteRepoRequest(repo, paths) + if err != nil { + return nil, xerrors.Errorf("unable to create request for pom file") + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + p.logger.Debug("Failed to fetch", log.String("url", req.URL.String())) return nil, nil } defer resp.Body.Close() @@ -709,6 +771,16 @@ func parsePom(r io.Reader) (*pomXML, error) { return parsed, nil } +func parseMavenMetadata(r io.Reader) (*Metadata, error) { + parsed := &Metadata{} + decoder := xml.NewDecoder(r) + decoder.CharsetReader = charset.NewReaderLabel + if err := decoder.Decode(parsed); err != nil { + return nil, xerrors.Errorf("xml decode error: %w", err) + } + return parsed, nil +} + func packageID(name, version string) string { return dependency.ID(ftypes.Pom, name, version) } diff --git a/pkg/dependency/parser/java/pom/parse_test.go b/pkg/dependency/parser/java/pom/parse_test.go index 1207f32adcf7..15740d599eb9 100644 --- a/pkg/dependency/parser/java/pom/parse_test.go +++ b/pkg/dependency/parser/java/pom/parse_test.go @@ -143,6 +143,13 @@ func TestPom_Parse(t *testing.T) { }, }, }, + { + ID: "org.example:example-api:2.0.0", + Name: "org.example:example-api", + Version: "2.0.0", + Licenses: []string{"The Apache Software License, Version 2.0"}, + Relationship: ftypes.RelationshipIndirect, + }, }, wantDeps: []ftypes.Dependency{ { @@ -151,6 +158,58 @@ func TestPom_Parse(t *testing.T) { "org.example:example-dependency:1.2.3-SNAPSHOT", }, }, + { + ID: "org.example:example-dependency:1.2.3-SNAPSHOT", + DependsOn: []string{ + "org.example:example-api:2.0.0", + }, + }, + }, + }, + { + name: "snapshot repository with maven-metadata.xml", + inputFile: filepath.Join("testdata", "snapshot", "with-maven-metadata", "pom.xml"), + local: false, + want: []ftypes.Package{ + { + ID: "com.example:happy:1.0.0", + Name: "com.example:happy", + Version: "1.0.0", + Relationship: ftypes.RelationshipRoot, + }, + { + ID: "org.example:example-dependency:2.17.0-SNAPSHOT", + Name: "org.example:example-dependency", + Version: "2.17.0-SNAPSHOT", + Relationship: ftypes.RelationshipDirect, + Locations: ftypes.Locations{ + { + StartLine: 14, + EndLine: 18, + }, + }, + }, + { + ID: "org.example:example-api:2.0.0", + Name: "org.example:example-api", + Version: "2.0.0", + Licenses: []string{"The Apache Software License, Version 2.0"}, + Relationship: ftypes.RelationshipIndirect, + }, + }, + wantDeps: []ftypes.Dependency{ + { + ID: "com.example:happy:1.0.0", + DependsOn: []string{ + "org.example:example-dependency:2.17.0-SNAPSHOT", + }, + }, + { + ID: "org.example:example-dependency:2.17.0-SNAPSHOT", + DependsOn: []string{ + "org.example:example-api:2.0.0", + }, + }, }, }, { @@ -1404,7 +1463,7 @@ func TestPom_Parse(t *testing.T) { remoteRepos = []string{ts.URL} } - p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) + p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline)) gotPkgs, gotDeps, err := p.Parse(f) if tt.wantErr != "" { diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom similarity index 100% rename from pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom rename to pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3-SNAPSHOT.pom diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom new file mode 100644 index 000000000000..0ecec117f873 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom @@ -0,0 +1,23 @@ + + + + 4.0.0 + + org.example + example-dependency + 2.17.0-SNAPSHOT + + jar + Example API Dependency + The example API + + + + org.example + example-api + 2.0.0 + + + + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml new file mode 100644 index 000000000000..258de9db99c5 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml @@ -0,0 +1,35 @@ + + org.example + example-dependency + + 20240312035235 + + 20240312.035235 + 10 + + + + sources + jar + 2.17.0-20240312.035235-10 + 20240312035235 + + + module + 2.17.0-20240312.035235-10 + 20240312035235 + + + jar + 2.17.0-20240312.035235-10 + 20240312035235 + + + pom + 2.17.0-20240312.035235-10 + 20240312035235 + + + + 2.17.0-SNAPSHOT + \ No newline at end of file diff --git a/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml new file mode 100644 index 000000000000..7e4c2144d417 --- /dev/null +++ b/pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + com.example + happy + 1.0.0 + + happy + Example + + + + + org.example + example-dependency + 2.17.0-SNAPSHOT + + +