[GH-113] Download of specific artifacts.

CHANGELOG.md

@@ -5,7 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-### Changed
+None
+
+## [3.5.0] - 2020-03-29
+### Added
+- Download of specific artifacts using a regex
+- Instructions added to README how to use this new feature
+
+### Improved
+- Section about how to clone an old Nexus3 repository
+
+### Fixed
+- Three code smells that were reported by SonarCloud
+
+## [3.4.0] - 2020-03-25
+### Added
+- Upload of zip artifacts
+
+## Changed
 - Create files for testing rather than storing them in git
 
 ## [3.3.3] - 2019-12-12
@@ -142,7 +159,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Download all artifacts from a certain Nexus3 repository.
 
-[Unreleased]: https://github.com/030/n3dr/compare/3.3.3...HEAD
+[Unreleased]: https://github.com/030/n3dr/compare/3.5.0...HEAD
+[3.5.0]: https://github.com/030/n3dr/compare/3.4.0...3.5.0
+[3.4.0]: https://github.com/030/n3dr/compare/3.3.3...3.4.0
 [3.3.3]: https://github.com/030/n3dr/compare/3.3.2...3.3.3
 [3.3.2]: https://github.com/030/n3dr/compare/3.3.1...3.3.2
 [3.3.1]: https://github.com/030/n3dr/compare/3.3.0...3.3.1

README.md

@@ -121,6 +121,23 @@ Note: a new folder will be created for every repository:
 * download/maven-public
 * download/maven-releases
 
+## Backup only certain artifacts
+
+It is possible to only download artifacts that match a regular expression. If
+one would like to download all artifacts from 'some/group42' then one could do
+that as follows:
+
+```
+./n3dr-linux backup -u admin -n http://localhost:8081 -r maven-releases -x 'some/group42'
+```
+
+If one would like to deploy is while download from all repositories then use
+the `-x` option as well:
+
+```
+./n3dr-linux repositories -u admin -n http://localhost:8081 -b -x 'some/group42'
+```
+
 ## Add all downloaded archives to a ZIP archive
 
 In order to add all archives to a zip archive, one has to use the --zip or -z flag.
@@ -134,16 +151,19 @@ running the following command:
 ./n3dr-linux upload -u admin -n http://localhost:8081 -r maven-public
 ```
 
-## "Clone" the content of a repository in a different Nexus 3 server in a different repository 
+## "Clone" a Nexus3 repository
 
-These are the basic steps to "clone" and eventually rename the content of a
-repository from one nexus3 server to another one:
+Suppose that one has created a new Nexus3 repository, e.g. NexusNEW and that
+one would like to copy the content of the old repository, e.g. NexusOLD, then
+these basic steps could be issued to "clone" NexusOLD:
 
 ```
-n3dr backup -u <source-nexus3-user> -n <source-nexus3-server-url> -r <repo-source-name>
+n3dr backup -u <old-nexus3-user> -n <old-nexus3-server-url> \
+-r <old-repo-source-name>
 cd download
-mv <repo-source-name> <repo-target-name>
-n3dr upload -u <target-nexus3-user> -n <target-nexus3-server-url> -r <repo-target-name>
+mv <old-repo-source-name> <new-repo-target-name>
+n3dr upload -u <new-target-nexus3-user> -n <new-target-nexus3-server-url> \
+-r <new-repo-target-name>
 ```
 
 ## Rationale for N3DR

cli/backup.go

@@ -100,6 +100,7 @@ func (n Nexus3) continuationTokenRecursion(t string) ([]string, error) {
 }
 
 func createArtifact(d string, f string, content string) error {
+	log.Debug("Create artifact: '" + d + "/" + f + "'")
 	err := os.MkdirAll(d, os.ModePerm)
 	if err != nil {
 		return err
@@ -177,6 +178,7 @@ func (n Nexus3) downloadURLs() ([]interface{}, error) {
 
 			jq := gojsonq.New().JSONString(json)
 			downloadURLsInterface := jq.From("items").Pluck("downloadUrl")
+			log.Debug("DownloadURLs: " + fmt.Sprintf("%v", downloadURLsInterface))
 
 			downloadURLsInterfaceArray := downloadURLsInterface.([]interface{})
 			downloadURLsInterfaceArrayAll = append(downloadURLsInterfaceArrayAll, downloadURLsInterfaceArray...)
@@ -189,7 +191,7 @@ func (n Nexus3) downloadURLs() ([]interface{}, error) {
 }
 
 // StoreArtifactsOnDisk downloads all artifacts from nexus and saves them on disk
-func (n Nexus3) StoreArtifactsOnDisk() error {
+func (n Nexus3) StoreArtifactsOnDisk(regex string) error {
 	urls, err := n.downloadURLs()
 	if err != nil {
 		return err
@@ -200,13 +202,20 @@ func (n Nexus3) StoreArtifactsOnDisk() error {
 		log.Info("Backing up artifacts '" + n.Repository + "'")
 		bar := pb.StartNew(len(urls))
 		for _, downloadURL := range urls {
-			u := fmt.Sprint(downloadURL)
+			url := fmt.Sprint(downloadURL)
 
-			// Exclude download of md5 and sha1 files as these are unavailable
-			// unless the metadata.xml is opened first
-			if !(filepath.Ext(u) == ".md5" || filepath.Ext(u) == ".sha1") {
-				if err := n.downloadArtifact(fmt.Sprint(downloadURL)); err != nil {
-					return err
+			log.Debug("Only download artifacts that match the regex: '" + regex + "'")
+			r, err := regexp.Compile(regex)
+			if err != nil {
+				return err
+			}
+			if r.MatchString(url) {
+				// Exclude download of md5 and sha1 files as these are unavailable
+				// unless the metadata.xml is opened first
+				if !(filepath.Ext(url) == ".md5" || filepath.Ext(url) == ".sha1") {
+					if err := n.downloadArtifact(fmt.Sprint(downloadURL)); err != nil {
+						return err
+					}
 				}
 			}
 

cli/backup_test.go

@@ -49,7 +49,7 @@ func TestDownloadURLs(t *testing.T) {
 }
 
 func TestStoreArtifactsOnDisk(t *testing.T) {
-	if err := n.StoreArtifactsOnDisk(); err != nil {
+	if err := n.StoreArtifactsOnDisk(".*"); err != nil {
 		log.Fatal(err)
 	}
 

cli/repositories.go

@@ -43,15 +43,15 @@ func (n Nexus3) CountRepositories() error {
 }
 
 // Downloads retrieves artifacts from all repositories
-func (n Nexus3) Downloads() error {
+func (n Nexus3) Downloads(regex string) error {
 	repos, err := n.repositoriesSlice()
 	if err != nil {
 		return err
 	}
 
 	for _, name := range repos {
 		n := Nexus3{URL: n.URL, User: n.User, Pass: n.Pass, Repository: name.(string), APIVersion: n.APIVersion, ZIP: n.ZIP}
-		if err := n.StoreArtifactsOnDisk(); err != nil {
+		if err := n.StoreArtifactsOnDisk(regex); err != nil {
 			return err
 		}
 	}

cli/upload.go

@@ -99,7 +99,7 @@ func (n Nexus3) Upload() error {
 		return err
 	}
 	for i, path := range foldersWithPOMStringSlice {
-		log.Info(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'")
+		log.Debug(strconv.Itoa(i) + " Detecting artifacts in folder '" + path + "'")
 		sb, err := pomDirs(path)
 		if err != nil {
 			return err

cmd/backup.go

@@ -22,6 +22,8 @@ import (
 	"github.com/spf13/viper"
 )
 
+var regex string
+
 // backupCmd represents the backup command
 var backupCmd = &cobra.Command{
 	Use:   "backup",
@@ -36,7 +38,7 @@ reside in a certain Nexus3 repository`,
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		n := cli.Nexus3{URL: n3drURL, User: n3drUser, Pass: viper.GetString("n3drPass"), Repository: n3drRepo, APIVersion: apiVersion, ZIP: zip}
-		if err := n.StoreArtifactsOnDisk(); err != nil {
+		if err := n.StoreArtifactsOnDisk(regex); err != nil {
 			log.Fatal(err)
 		}
 
@@ -55,4 +57,5 @@ func init() {
 	}
 
 	rootCmd.AddCommand(backupCmd)
+	backupCmd.Flags().StringVarP(&regex, "regex", "x", ".*", "only download artifacts that match a regular expression, e.g. 'some/group42'")
 }

cmd/repositories.go

@@ -61,7 +61,7 @@ download artifacts from all repositories`,
 			}
 		}
 		if backup {
-			err := n.Downloads()
+			err := n.Downloads(regex)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -75,4 +75,5 @@ func init() {
 	repositoriesCmd.Flags().BoolVarP(&names, "names", "a", false, "print all repository names")
 	repositoriesCmd.Flags().BoolVarP(&count, "count", "c", false, "count the number of repositories")
 	repositoriesCmd.Flags().BoolVarP(&backup, "backup", "b", false, "backup artifacts from all repositories")
+	repositoriesCmd.Flags().StringVarP(&regex, "regex", "x", ".*", "only download artifacts that match a regular expression, e.g. 'some/group42'")
 }

integration-tests.sh

@@ -35,7 +35,7 @@ readiness(){
     until docker logs nexus | grep 'Started Sonatype Nexus OSS'
     do
         echo "Nexus unavailable"
-        sleep 10
+        sleep 3
     done
 }
 
@@ -52,10 +52,10 @@ password(){
 }
 
 artifact(){
-    mkdir -p "maven-releases/file${1}/file${1}/1.0.0"
-    echo someContent > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.jar"
-    echo someContentZIP > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.zip"
-    echo -e "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>file${1}</groupId>\n<artifactId>file${1}</artifactId>\n<version>1.0.0</version>\n</project>" > "maven-releases/file${1}/file${1}/1.0.0/file${1}-1.0.0.pom"
+    mkdir -p "maven-releases/some/group${1}/file${1}/1.0.0"
+    echo someContent > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.jar"
+    echo someContentZIP > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.zip"
+    echo -e "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>some.group${1}</groupId>\n<artifactId>file${1}</artifactId>\n<version>1.0.0</version>\n</project>" > "maven-releases/some/group${1}/file${1}/1.0.0/file${1}-1.0.0.pom"
 }
 
 files(){
@@ -75,15 +75,42 @@ backup(){
 
     if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
         count_downloads 300
-        test_zip 132
+        test_zip 140
     else
         count_downloads 400
-        test_zip 168
+        test_zip 180
     fi
 
     cleanup_downloads
 }
 
+regex(){
+    echo "Testing backup regex..."
+    $TOOL backup -n http://localhost:9999 -u admin -p $PASSWORD -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42'
+    $TOOL backup -n http://localhost:9999 -u admin -p $PASSWORD -r maven-releases -v "${NEXUS_API_VERSION}" -x 'some/group42' -z
+    if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
+        count_downloads 3
+        test_zip 4
+    else
+        count_downloads 4
+        test_zip 4
+    fi
+    cleanup_downloads
+
+
+    echo -e "\nTesting repositories regex..."
+    $TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v "${NEXUS_API_VERSION}" -b -x 'some/group42'
+    $TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v "${NEXUS_API_VERSION}" -b -x 'some/group42' -z
+    if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
+        count_downloads 6
+        test_zip 4
+    else
+        count_downloads 8
+        test_zip 8
+    fi
+    cleanup_downloads
+}
+
 repositories(){
     local cmd="$TOOL repositories -n http://localhost:9999 -u admin -p $PASSWORD -v ${NEXUS_API_VERSION}"
 
@@ -95,10 +122,10 @@ repositories(){
 
     if [ "${NEXUS_VERSION}" == "3.9.0" ]; then
         count_downloads 600
-        test_zip 256
+        test_zip 272
     else
         count_downloads 800
-        test_zip 336
+        test_zip 356
     fi
 
     cleanup_downloads
@@ -141,6 +168,7 @@ main(){
     upload
     backup
     repositories
+    regex
     bats --tap tests.bats
 }