Introduce option multi-dist to the publish commands

This change makes it possible to publish multiple distributions
with packages named the same but with different content by changing
structure of the generated pool hierarchy. The option not enabled
by default as this changes the structure of the output which could
break the expectations of other tools.

AUTHORS

@@ -59,3 +59,4 @@ List of contributors, in chronological order:
 * Paul Cacheux (https://github.com/paulcacheux)
 * Nic Waller (https://github.com/sf-nwaller)
 * iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)

api/publish.go

@@ -101,6 +101,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Architectures        []string
 		Signing              SigningOptions
 		AcquireByHash        *bool
+		MultiDist            bool
 	}
 
 	if c.Bind(&b) != nil {
@@ -226,7 +227,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
-		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite)
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, b.MultiDist)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
@@ -257,6 +258,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			Name      string `binding:"required"`
 		}
 		AcquireByHash *bool
+		MultiDist     bool
 	}
 
 	if c.Bind(&b) != nil {
@@ -341,7 +343,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	resources = append(resources, string(published.Key()))
 	taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

cmd/publish_repo.go

@@ -51,6 +51,7 @@ Example:
 	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -116,6 +116,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	origin := context.Flags().Lookup("origin").Value.String()
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
 	if err != nil {
@@ -165,7 +166,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -242,6 +243,7 @@ Example:
 	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -14,6 +14,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
 	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
@@ -100,7 +101,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -161,6 +162,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -17,6 +17,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	distribution := args[0]
 	param := "."
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
 	if len(args) == 2 {
 		param = args[1]
@@ -64,7 +65,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -119,6 +120,7 @@ Example:
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

completion.d/aptly

@@ -503,7 +503,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))

deb/publish.go

@@ -544,7 +544,7 @@ func (p *PublishedRepo) GetCodename() string {
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
-	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, multiDist bool) error {
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
 
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
@@ -656,7 +656,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						if err2 != nil {
 							return err2
 						}
-						relPath = filepath.Join("pool", component, poolDir)
+						if multiDist {
+							relPath = filepath.Join("pool", p.Distribution, component, poolDir)
+						} else {
+							relPath = filepath.Join("pool", component, poolDir)
+						}
+
 					} else {
 						if p.Distribution == aptly.DistributionFocal {
 							relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "legacy-images")

deb/publish_test.go

@@ -193,6 +193,58 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
 	c.Check(err, IsNil)
 }
 
+func (s *PublishedRepoSuite) TestMultiDistPool(c *C) {
+	repo, err := NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Assert(err, IsNil)
+	err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, true)
+	c.Assert(err, IsNil)
+
+	publishedStorage := files.NewPublishedStorage(s.root, "", "")
+
+	c.Check(repo.Architectures, DeepEquals, []string{"i386"})
+
+	rf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Origin"], Equals, "ppa squeeze")
+	c.Check(st["Components"], Equals, "main")
+	c.Check(st["Architectures"], Equals, "i386")
+
+	pf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(pf, false, false)
+
+	for i := 0; i < 3; i++ {
+		st, err = cfr.ReadStanza()
+		c.Assert(err, IsNil)
+
+		c.Check(st["Filename"], Equals, "pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	}
+
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+	c.Assert(st, IsNil)
+
+	drf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(drf, true, false)
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Archive"], Equals, "squeeze")
+	c.Check(st["Architecture"], Equals, "i386")
+
+	_, err = os.Stat(filepath.Join(publishedStorage.PublicPath(), "ppa/pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
+	c.Assert(err, IsNil)
+
+}
+
 func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 
 	for _, t := range []struct {
@@ -308,7 +360,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, false)
 	c.Assert(err, IsNil)
 
 	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
@@ -355,31 +407,31 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
-	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/binary-i386/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
-	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
-	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)