Draft: Feature: Add Pull Snapshot API

api/router.go

@@ -153,6 +153,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	}
 
 	{
+
 		api.GET("/publish", apiPublishList)
 		api.POST("/publish", apiPublishRepoOrSnapshot)
 		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
@@ -169,6 +170,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
 		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
 		api.POST("/snapshots/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/pull", apiSnapshotsPull)
 	}
 
 	{

api/snapshot.go

@@ -3,11 +3,13 @@
 import (
 	"fmt"
 	"net/http"
+	"sort"
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
@@ -475,3 +477,165 @@
 		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
 	})
 }
+
+// POST /api/snapshots/pull
+func apiSnapshotsPull(c *gin.Context) {
+	var (
+		err                 error
+		destinationSnapshot *deb.Snapshot
+	)
+
+	var body struct {
+		Source        string   `binding:"required"`
+		To            string   `binding:"required"`
+		Destination   string   `binding:"required"`
+		Queries       []string `binding:"required"`
+		Architectures []string
+	}
+
+	if err = c.BindJSON(&body); err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, err)
+		return
+	}
+
+	allMatches := c.Request.URL.Query().Get("all-matches") == "1"
+	dryRun := c.Request.URL.Query().Get("dry-run") == "1"
+	noDeps := c.Request.URL.Query().Get("no-deps") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <To> snapshot
+	toSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.To)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	// Load <Source> snapshot
+	sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, body.To, body.Destination)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// convert snapshots to package list
+		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		toPackageList.PrepareIndex()
+		sourcePackageList.PrepareIndex()
+
+		var architecturesList []string
+
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = toPackageList.Architectures(false)
+		}
+
+		architecturesList = append(architecturesList, body.Architectures...)
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+		var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+		for _, arch := range architecturesList {
+			archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+		}
+
+		queries := make([]deb.PackageQuery, len(body.Queries))
+		for i, q := range body.Queries {
+			queries[i], err = query.Parse(q)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// Add architecture filter
+			queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+		}
+
+		// Filter with dependencies as requested
+		destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		destinationPackageList.PrepareIndex()
+
+		removedPackages := []string{}
+		addedPackages := []string{}
+		alreadySeen := map[string]bool{}
+
+		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+			key := pkg.Architecture + "_" + pkg.Name
+			_, seen := alreadySeen[key]
+
+			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+			if !noRemove && !seen {
+				// Remove all packages with the same name and architecture
+				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+				for _, p := range packageSearchResults {
+					toPackageList.Remove(p)
+					removedPackages = append(removedPackages, p.String())
+				}
+			}
+
+			// If !allMatches, add only first matching name-arch package
+			if !seen || allMatches {
+				toPackageList.Add(pkg)
+				addedPackages = append(addedPackages, pkg.String())
+			}
+
+			alreadySeen[key] = true
+
+			return nil
+		})
+		alreadySeen = nil
+
+		if dryRun {
+			response := struct {
+				AddedPackages   []string `json:"added_packages"`
+				RemovedPackages []string `json:"removed_packages"`
+			}{
+				AddedPackages:   addedPackages,
+				RemovedPackages: removedPackages,
+			}
+
+			return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
+		}
+
+		// Create <destination> snapshot
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
+	})
+
+}

system/t12_api/snapshots.py

@@ -7,6 +7,7 @@ class SnapshotsAPITestCreateShowEmpty(APITest):
     """
     GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages
     """
+
     def check(self):
         snapshot_name = self.random_name()
         snapshot_desc = {'Description': 'fun snapshot',
@@ -35,6 +36,7 @@ class SnapshotsAPITestCreateFromRefs(APITest):
     GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages,
     GET /api/snapshots
     """
+
     def check(self):
         snapshot_name = self.random_name()
         snapshot_desc = {'Description': 'fun snapshot',
@@ -95,6 +97,7 @@ class SnapshotsAPITestCreateFromRepo(APITest):
     """
     POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name
     """
+
     def check(self):
         repo_name = self.random_name()
         snapshot_name = self.random_name()
@@ -138,6 +141,7 @@ class SnapshotsAPITestCreateUpdate(APITest):
     """
     POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name
     """
+
     def check(self):
         snapshot_name = self.random_name()
         snapshot_desc = {'Description': 'fun snapshot',
@@ -170,6 +174,7 @@ class SnapshotsAPITestCreateDelete(APITest):
     """
     POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name
     """
+
     def check(self):
         snapshot_name = self.random_name()
         snapshot_desc = {'Description': 'fun snapshot',
@@ -218,6 +223,7 @@ class SnapshotsAPITestSearch(APITest):
     """
     POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name
     """
+
     def check(self):
 
         repo_name = self.random_name()
@@ -251,6 +257,7 @@ class SnapshotsAPITestDiff(APITest):
     """
     GET /api/snapshot/:name/diff/:name2
     """
+
     def check(self):
         repos = [self.random_name() for x in range(2)]
         snapshots = [self.random_name() for x in range(2)]
@@ -291,7 +298,7 @@ def check(self):
 
 class SnapshotsAPITestMerge(APITest):
     """
-    POST /api/snapshots, GET /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name
+    POST /api/snapshots, POST /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name
     """
 
     def check(self):
@@ -384,3 +391,91 @@ def check(self):
             resp.json()["error"], "no-remove and latest are mutually exclusive"
         )
         self.check_equal(resp.status_code, 400)
+
+
+class SnapshotsAPITestPull(APITest):
+    """
+    POST /api/snapshots/pull, POST /api/snapshots, GET /api/snapshots/:name/packages?name=:package_name
+    """
+
+    def check(self):
+        repo_with_libboost = self.random_name()
+        empty_repo = self.random_name()
+        snapshot_repo_with_libboost = self.random_name()
+        snapshot_empty_repo = self.random_name()
+
+        # create repo with file in it and snapshot of it
+        self.check_equal(self.post("/api/repos", json={"Name": repo_with_libboost}).status_code, 201)
+
+        dir_name = self.random_name()
+        self.check_equal(self.upload(f"/api/files/{dir_name}",
+                         "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200)
+        self.check_equal(self.post(f"/api/repos/{repo_with_libboost}/file/{dir_name}").status_code, 200)
+
+        resp = self.post(f"/api/repos/{repo_with_libboost}/snapshots", json={'Name': snapshot_repo_with_libboost})
+        self.check_equal(resp.status_code, 201)
+
+        # create empty repo and snapshot of it
+        self.check_equal(self.post("/api/repos", json={"Name": empty_repo}).status_code, 201)
+
+        resp = self.post(f"/api/repos/{empty_repo}/snapshots", json={'Name': snapshot_empty_repo})
+        self.check_equal(resp.status_code, 201)
+
+        # pull libboost from repo_with_libboost to empty_repo, save into snapshot_pull_libboost
+        snapshot_pull_libboost = self.random_name()
+        resp = self.post("/api/snapshots/pull", json={
+            'Source': snapshot_repo_with_libboost,
+            'To': snapshot_empty_repo,
+            'Destination': snapshot_pull_libboost,
+            'Queries': [
+                'libboost-program-options-dev'
+            ],
+            'Architectures': [
+                'amd64'
+                'i386'
+            ]
+        })
+        self.check_equal(resp.status_code, 201)
+        self.check_subset({
+            'Name': snapshot_pull_libboost,
+            'SourceKind': 'snapshot',
+            'Description': f"Pulled into '{snapshot_empty_repo}' with '{snapshot_repo_with_libboost}' as source, pull request was: 'libboost-program-options-dev'",
+        }, resp.json())
+
+        # check that snapshot_pull_libboost contains libboost
+        resp = self.get(f"/api/snapshots/{snapshot_pull_libboost}/packages?name=libboost-program-options-dev")
+        self.check_equal(resp.status_code, 200)
+
+        # pull from non-existing source
+        non_existing_source = self.random_name()
+        destination = self.random_name()
+        resp = self.post("/api/snapshots/pull", json={
+            'Source': non_existing_source,
+            'To': snapshot_empty_repo,
+            'Destination': destination,
+            'Queries': [
+                'Name (~ *)'
+            ],
+            'Architectures': [
+                'all',
+            ]
+        })
+        self.check_equal(resp.status_code, 404)
+        self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_source} not found")
+
+        # pull to non-existing snapshot
+        non_existing_snapshot = self.random_name()
+        destination = self.random_name()
+        resp = self.post("/api/snapshots/pull", json={
+            'Source': non_existing_snapshot,
+            'To': snapshot_empty_repo,
+            'Destination': destination,
+            'Queries': [
+                'Name (~ *)'
+            ],
+            'Architectures': [
+                'all',
+            ]
+        })
+        self.check_equal(resp.status_code, 404)
+        self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_snapshot} not found")