diff --git a/api/router.go b/api/router.go index ddd30e976..a5ad5b96e 100644 --- a/api/router.go +++ b/api/router.go @@ -153,6 +153,7 @@ func Router(c *ctx.AptlyContext) http.Handler { } { + api.GET("/publish", apiPublishList) api.POST("/publish", apiPublishRepoOrSnapshot) api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) @@ -169,6 +170,7 @@ func Router(c *ctx.AptlyContext) http.Handler { api.DELETE("/snapshots/:name", apiSnapshotsDrop) api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) api.POST("/snapshots/merge", apiSnapshotsMerge) + api.POST("/snapshots/pull", apiSnapshotsPull) } { diff --git a/api/snapshot.go b/api/snapshot.go index dfad75c34..83c3cfbb9 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -3,11 +3,13 @@ package api import ( "fmt" "net/http" + "sort" "strings" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/database" "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/query" "github.com/aptly-dev/aptly/task" "github.com/gin-gonic/gin" ) @@ -475,3 +477,165 @@ func apiSnapshotsMerge(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil }) } + +// POST /api/snapshots/pull +func apiSnapshotsPull(c *gin.Context) { + var ( + err error + destinationSnapshot *deb.Snapshot + ) + + var body struct { + Source string `binding:"required"` + To string `binding:"required"` + Destination string `binding:"required"` + Queries []string `binding:"required"` + Architectures []string + } + + if err = c.BindJSON(&body); err != nil { + AbortWithJSONError(c, http.StatusBadRequest, err) + return + } + + allMatches := c.Request.URL.Query().Get("all-matches") == "1" + dryRun := c.Request.URL.Query().Get("dry-run") == "1" + noDeps := c.Request.URL.Query().Get("no-deps") == "1" + noRemove := c.Request.URL.Query().Get("no-remove") == "1" + + collectionFactory := context.NewCollectionFactory() + + // Load snapshot + toSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.To) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, err) + return + } + err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, err) + return + } + + // Load snapshot + sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, err) + return + } + err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, err) + return + } + + resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())} + taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, body.To, body.Destination) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + // convert snapshots to package list + toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress()) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress()) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + toPackageList.PrepareIndex() + sourcePackageList.PrepareIndex() + + var architecturesList []string + + if len(context.ArchitecturesList()) > 0 { + architecturesList = context.ArchitecturesList() + } else { + architecturesList = toPackageList.Architectures(false) + } + + architecturesList = append(architecturesList, body.Architectures...) + sort.Strings(architecturesList) + + if len(architecturesList) == 0 { + err := fmt.Errorf("unable to determine list of architectures, please specify explicitly") + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + // Build architecture query: (arch == "i386" | arch == "amd64" | ...) + var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""} + for _, arch := range architecturesList { + archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery} + } + + queries := make([]deb.PackageQuery, len(body.Queries)) + for i, q := range body.Queries { + queries[i], err = query.Parse(q) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + // Add architecture filter + queries[i] = &deb.AndQuery{L: queries[i], R: archQuery} + } + + // Filter with dependencies as requested + destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress()) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + destinationPackageList.PrepareIndex() + + removedPackages := []string{} + addedPackages := []string{} + alreadySeen := map[string]bool{} + + destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error { + key := pkg.Architecture + "_" + pkg.Name + _, seen := alreadySeen[key] + + // If we haven't seen such name-architecture pair and were instructed to remove, remove it + if !noRemove && !seen { + // Remove all packages with the same name and architecture + packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true) + for _, p := range packageSearchResults { + toPackageList.Remove(p) + removedPackages = append(removedPackages, p.String()) + } + } + + // If !allMatches, add only first matching name-arch package + if !seen || allMatches { + toPackageList.Add(pkg) + addedPackages = append(addedPackages, pkg.String()) + } + + alreadySeen[key] = true + + return nil + }) + alreadySeen = nil + + if dryRun { + response := struct { + AddedPackages []string `json:"added_packages"` + RemovedPackages []string `json:"removed_packages"` + }{ + AddedPackages: addedPackages, + RemovedPackages: removedPackages, + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil + } + + // Create snapshot + destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList, + fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", "))) + + err = collectionFactory.SnapshotCollection().Add(destinationSnapshot) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil + }) + +} diff --git a/system/t12_api/snapshots.py b/system/t12_api/snapshots.py index 0d12066ad..152594ebe 100644 --- a/system/t12_api/snapshots.py +++ b/system/t12_api/snapshots.py @@ -7,6 +7,7 @@ class SnapshotsAPITestCreateShowEmpty(APITest): """ GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages """ + def check(self): snapshot_name = self.random_name() snapshot_desc = {'Description': 'fun snapshot', @@ -35,6 +36,7 @@ class SnapshotsAPITestCreateFromRefs(APITest): GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages, GET /api/snapshots """ + def check(self): snapshot_name = self.random_name() snapshot_desc = {'Description': 'fun snapshot', @@ -95,6 +97,7 @@ class SnapshotsAPITestCreateFromRepo(APITest): """ POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name """ + def check(self): repo_name = self.random_name() snapshot_name = self.random_name() @@ -138,6 +141,7 @@ class SnapshotsAPITestCreateUpdate(APITest): """ POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name """ + def check(self): snapshot_name = self.random_name() snapshot_desc = {'Description': 'fun snapshot', @@ -170,6 +174,7 @@ class SnapshotsAPITestCreateDelete(APITest): """ POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name """ + def check(self): snapshot_name = self.random_name() snapshot_desc = {'Description': 'fun snapshot', @@ -218,6 +223,7 @@ class SnapshotsAPITestSearch(APITest): """ POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name """ + def check(self): repo_name = self.random_name() @@ -251,6 +257,7 @@ class SnapshotsAPITestDiff(APITest): """ GET /api/snapshot/:name/diff/:name2 """ + def check(self): repos = [self.random_name() for x in range(2)] snapshots = [self.random_name() for x in range(2)] @@ -291,7 +298,7 @@ def check(self): class SnapshotsAPITestMerge(APITest): """ - POST /api/snapshots, GET /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name + POST /api/snapshots, POST /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name """ def check(self): @@ -384,3 +391,91 @@ def check(self): resp.json()["error"], "no-remove and latest are mutually exclusive" ) self.check_equal(resp.status_code, 400) + + +class SnapshotsAPITestPull(APITest): + """ + POST /api/snapshots/pull, POST /api/snapshots, GET /api/snapshots/:name/packages?name=:package_name + """ + + def check(self): + repo_with_libboost = self.random_name() + empty_repo = self.random_name() + snapshot_repo_with_libboost = self.random_name() + snapshot_empty_repo = self.random_name() + + # create repo with file in it and snapshot of it + self.check_equal(self.post("/api/repos", json={"Name": repo_with_libboost}).status_code, 201) + + dir_name = self.random_name() + self.check_equal(self.upload(f"/api/files/{dir_name}", + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + self.check_equal(self.post(f"/api/repos/{repo_with_libboost}/file/{dir_name}").status_code, 200) + + resp = self.post(f"/api/repos/{repo_with_libboost}/snapshots", json={'Name': snapshot_repo_with_libboost}) + self.check_equal(resp.status_code, 201) + + # create empty repo and snapshot of it + self.check_equal(self.post("/api/repos", json={"Name": empty_repo}).status_code, 201) + + resp = self.post(f"/api/repos/{empty_repo}/snapshots", json={'Name': snapshot_empty_repo}) + self.check_equal(resp.status_code, 201) + + # pull libboost from repo_with_libboost to empty_repo, save into snapshot_pull_libboost + snapshot_pull_libboost = self.random_name() + resp = self.post("/api/snapshots/pull", json={ + 'Source': snapshot_repo_with_libboost, + 'To': snapshot_empty_repo, + 'Destination': snapshot_pull_libboost, + 'Queries': [ + 'libboost-program-options-dev' + ], + 'Architectures': [ + 'amd64' + 'i386' + ] + }) + self.check_equal(resp.status_code, 201) + self.check_subset({ + 'Name': snapshot_pull_libboost, + 'SourceKind': 'snapshot', + 'Description': f"Pulled into '{snapshot_empty_repo}' with '{snapshot_repo_with_libboost}' as source, pull request was: 'libboost-program-options-dev'", + }, resp.json()) + + # check that snapshot_pull_libboost contains libboost + resp = self.get(f"/api/snapshots/{snapshot_pull_libboost}/packages?name=libboost-program-options-dev") + self.check_equal(resp.status_code, 200) + + # pull from non-existing source + non_existing_source = self.random_name() + destination = self.random_name() + resp = self.post("/api/snapshots/pull", json={ + 'Source': non_existing_source, + 'To': snapshot_empty_repo, + 'Destination': destination, + 'Queries': [ + 'Name (~ *)' + ], + 'Architectures': [ + 'all', + ] + }) + self.check_equal(resp.status_code, 404) + self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_source} not found") + + # pull to non-existing snapshot + non_existing_snapshot = self.random_name() + destination = self.random_name() + resp = self.post("/api/snapshots/pull", json={ + 'Source': non_existing_snapshot, + 'To': snapshot_empty_repo, + 'Destination': destination, + 'Queries': [ + 'Name (~ *)' + ], + 'Architectures': [ + 'all', + ] + }) + self.check_equal(resp.status_code, 404) + self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_snapshot} not found")