Skip to content

Commit

Permalink
feat(java): add support for maven-metadata.xml files for remote sna…
Browse files Browse the repository at this point in the history 
…pshot repositories. (#6950)
  • Loading branch information
@DmitriyLewen
DmitriyLewen committed Jun 19, 2024
1 parent 2d85a00 commit 1f8fca1
Show file tree
Hide file tree
Showing 7 changed files with 231 additions and 5 deletions.
17 changes: 17 additions & 0 deletions pkg/dependency/parser/java/pom/metadata.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package pom

type Metadata struct {
GroupId string `xml:"groupId"`
ArtifactId string `xml:"artifactId"`
Versioning Versioning `xml:"versioning"`
Version string `xml:"version"`
}

type Versioning struct {
SnapshotVersions []SnapshotVersion `xml:"snapshotVersions>snapshotVersion"`
}

type SnapshotVersion struct {
Extension string `xml:"extension"`
Value string `xml:"value"`
}
80 changes: 76 additions & 4 deletions pkg/dependency/parser/java/pom/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

multierror "github.com/hashicorp/go-multierror"
"github.com/samber/lo"
"golang.org/x/exp/slices"
"golang.org/x/net/html/charset"
"golang.org/x/xerrors"

Expand Down Expand Up @@ -48,6 +49,12 @@ func WithReleaseRemoteRepos(repos []string) option {
}
}

func WithSnapshotRemoteRepos(repos []string) option {
return func(opts *options) {
opts.snapshotRemoteRepos = repos
}
}

type Parser struct {
logger *log.Logger
rootPath string
Expand Down Expand Up @@ -648,7 +655,18 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (

// try all remoteRepositories
for _, repo := range remoteRepos {
fetched, err := p.fetchPOMFromRemoteRepository(repo, paths)
repoPaths := slices.Clone(paths) // Clone slice to avoid overwriting last element of `paths`
if snapshot {
pomFileName, err := p.fetchPomFileNameFromMavenMetadata(repo, repoPaths)
if err != nil {
return nil, xerrors.Errorf("fetch maven-metadata.xml error: %w", err)
}
// Use file name from `maven-metadata.xml` if it exists
if pomFileName != "" {
repoPaths[len(repoPaths)-1] = pomFileName
}
}
fetched, err := p.fetchPOMFromRemoteRepository(repo, repoPaths)
if err != nil {
return nil, xerrors.Errorf("fetch repository error: %w", err)
} else if fetched == nil {
Expand All @@ -659,7 +677,7 @@ func (p *Parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (
return nil, xerrors.Errorf("the POM was not found in remote remoteRepositories")
}

func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
func (p *Parser) remoteRepoRequest(repo string, paths []string) (*http.Request, error) {
repoURL, err := url.Parse(repo)
if err != nil {
p.logger.Error("URL parse error", log.String("repo", repo))
Expand All @@ -670,7 +688,6 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
repoURL.Path = path.Join(paths...)

logger := p.logger.With(log.String("host", repoURL.Host), log.String("path", repoURL.Path))
client := &http.Client{}
req, err := http.NewRequest("GET", repoURL.String(), http.NoBody)
if err != nil {
logger.Debug("HTTP request failed")
Expand All @@ -681,9 +698,54 @@ func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom
req.SetBasicAuth(repoURL.User.Username(), password)
}

return req, nil
}

// fetchPomFileNameFromMavenMetadata fetches `maven-metadata.xml` file to detect file name of pom file.
func (p *Parser) fetchPomFileNameFromMavenMetadata(repo string, paths []string) (string, error) {
// Overwrite pom file name to `maven-metadata.xml`
mavenMetadataPaths := slices.Clone(paths[:len(paths)-1]) // Clone slice to avoid shadow overwriting last element of `paths`
mavenMetadataPaths = append(mavenMetadataPaths, "maven-metadata.xml")

req, err := p.remoteRepoRequest(repo, mavenMetadataPaths)
if err != nil {
return "", xerrors.Errorf("unable to create request for maven-metadata.xml file")
}

client := &http.Client{}
resp, err := client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
logger.Debug("Failed to fetch")
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
return "", nil
}
defer resp.Body.Close()

mavenMetadata, err := parseMavenMetadata(resp.Body)
if err != nil {
return "", xerrors.Errorf("failed to parse maven-metadata.xml file: %w", err)
}

var pomFileName string
for _, sv := range mavenMetadata.Versioning.SnapshotVersions {
if sv.Extension == "pom" {
// mavenMetadataPaths[len(mavenMetadataPaths)-3] is always artifactID
pomFileName = fmt.Sprintf("%s-%s.pom", mavenMetadataPaths[len(mavenMetadataPaths)-3], sv.Value)
}
}

return pomFileName, nil
}

func (p *Parser) fetchPOMFromRemoteRepository(repo string, paths []string) (*pom, error) {
req, err := p.remoteRepoRequest(repo, paths)
if err != nil {
return nil, xerrors.Errorf("unable to create request for pom file")
}

client := &http.Client{}
resp, err := client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
p.logger.Debug("Failed to fetch", log.String("url", req.URL.String()))
return nil, nil
}
defer resp.Body.Close()
Expand All @@ -709,6 +771,16 @@ func parsePom(r io.Reader) (*pomXML, error) {
return parsed, nil
}

func parseMavenMetadata(r io.Reader) (*Metadata, error) {
parsed := &Metadata{}
decoder := xml.NewDecoder(r)
decoder.CharsetReader = charset.NewReaderLabel
if err := decoder.Decode(parsed); err != nil {
return nil, xerrors.Errorf("xml decode error: %w", err)
}
return parsed, nil
}

func packageID(name, version string) string {
return dependency.ID(ftypes.Pom, name, version)
}
Expand Down
61 changes: 60 additions & 1 deletion pkg/dependency/parser/java/pom/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,13 @@ func TestPom_Parse(t *testing.T) {
},
},
},
{
ID: "org.example:example-api:2.0.0",
Name: "org.example:example-api",
Version: "2.0.0",
Licenses: []string{"The Apache Software License, Version 2.0"},
Relationship: ftypes.RelationshipIndirect,
},
},
wantDeps: []ftypes.Dependency{
{
Expand All @@ -151,6 +158,58 @@ func TestPom_Parse(t *testing.T) {
"org.example:example-dependency:1.2.3-SNAPSHOT",
},
},
{
ID: "org.example:example-dependency:1.2.3-SNAPSHOT",
DependsOn: []string{
"org.example:example-api:2.0.0",
},
},
},
},
{
name: "snapshot repository with maven-metadata.xml",
inputFile: filepath.Join("testdata", "snapshot", "with-maven-metadata", "pom.xml"),
local: false,
want: []ftypes.Package{
{
ID: "com.example:happy:1.0.0",
Name: "com.example:happy",
Version: "1.0.0",
Relationship: ftypes.RelationshipRoot,
},
{
ID: "org.example:example-dependency:2.17.0-SNAPSHOT",
Name: "org.example:example-dependency",
Version: "2.17.0-SNAPSHOT",
Relationship: ftypes.RelationshipDirect,
Locations: ftypes.Locations{
{
StartLine: 14,
EndLine: 18,
},
},
},
{
ID: "org.example:example-api:2.0.0",
Name: "org.example:example-api",
Version: "2.0.0",
Licenses: []string{"The Apache Software License, Version 2.0"},
Relationship: ftypes.RelationshipIndirect,
},
},
wantDeps: []ftypes.Dependency{
{
ID: "com.example:happy:1.0.0",
DependsOn: []string{
"org.example:example-dependency:2.17.0-SNAPSHOT",
},
},
{
ID: "org.example:example-dependency:2.17.0-SNAPSHOT",
DependsOn: []string{
"org.example:example-api:2.0.0",
},
},
},
},
{
Expand Down Expand Up @@ -1404,7 +1463,7 @@ func TestPom_Parse(t *testing.T) {
remoteRepos = []string{ts.URL}
}

p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithSnapshotRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))

gotPkgs, gotDeps, err := p.Parse(f)
if tt.wantErr != "" {
Expand Down
0 ...2.3-SNAPSHOT/example-dependency-1.2.3.pom → ...HOT/example-dependency-1.2.3-SNAPSHOT.pom
View file
File renamed without changes.
23 changes: 23 additions & 0 deletions ...ample/example-dependency/2.17.0-SNAPSHOT/example-dependency-2.17.0-20240312.035235-10.pom
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>

<groupId>org.example</groupId>
<artifactId>example-dependency</artifactId>
<version>2.17.0-SNAPSHOT</version>

<packaging>jar</packaging>
<name>Example API Dependency</name>
<description>The example API</description>

<dependencies>
<dependency>
<groupId>org.example</groupId>
<artifactId>example-api</artifactId>
<version>2.0.0</version>
</dependency>
</dependencies>

</project>
35 changes: 35 additions & 0 deletions ...pom/testdata/repository/org/example/example-dependency/2.17.0-SNAPSHOT/maven-metadata.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<metadata modelVersion="1.1.0">
<groupId>org.example</groupId>
<artifactId>example-dependency</artifactId>
<versioning>
<lastUpdated>20240312035235</lastUpdated>
<snapshot>
<timestamp>20240312.035235</timestamp>
<buildNumber>10</buildNumber>
</snapshot>
<snapshotVersions>
<snapshotVersion>
<classifier>sources</classifier>
<extension>jar</extension>
<value>2.17.0-20240312.035235-10</value>
<updated>20240312035235</updated>
</snapshotVersion>
<snapshotVersion>
<extension>module</extension>
<value>2.17.0-20240312.035235-10</value>
<updated>20240312035235</updated>
</snapshotVersion>
<snapshotVersion>
<extension>jar</extension>
<value>2.17.0-20240312.035235-10</value>
<updated>20240312035235</updated>
</snapshotVersion>
<snapshotVersion>
<extension>pom</extension>
<value>2.17.0-20240312.035235-10</value>
<updated>20240312035235</updated>
</snapshotVersion>
</snapshotVersions>
</versioning>
<version>2.17.0-SNAPSHOT</version>
</metadata>
20 changes: 20 additions & 0 deletions pkg/dependency/parser/java/pom/testdata/snapshot/with-maven-metadata/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.example</groupId>
<artifactId>happy</artifactId>
<version>1.0.0</version>

<name>happy</name>
<description>Example</description>


<dependencies>
<dependency>
<groupId>org.example</groupId>
<artifactId>example-dependency</artifactId>
<version>2.17.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

0 comments on commit 1f8fca1

Please sign in to comment.