Improve rubygems package registry (#31357)

To make it work with Bundler:
https://guides.rubygems.org/rubygems-org-compact-index-api/

It only adds 2 new API endpoints and improves some tests, existing logic
is not changed.

routers/api/packages/api.go

@@ -588,6 +588,8 @@ func CommonRoutes() *web.Route {
 			r.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease)
 			r.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification)
 			r.Get("/gems/{filename}", rubygems.DownloadPackageFile)
+			r.Get("/info/{packagename}", rubygems.GetPackageInfo)
+			r.Get("/versions", rubygems.GetAllPackagesVersions)
 			r.Group("/api/v1/gems", func() {
 				r.Post("/", rubygems.UploadPackageFile)
 				r.Delete("/yank", rubygems.DeletePackage)

routers/api/packages/rubygems/rubygems.go

@@ -6,6 +6,7 @@ package rubygems
 import (
 	"compress/gzip"
 	"compress/zlib"
+	"crypto/md5"
 	"errors"
 	"fmt"
 	"io"
@@ -227,12 +228,7 @@ func UploadPackageFile(ctx *context.Context) {
 		return
 	}
 
-	var filename string
-	if rp.Metadata.Platform == "" || rp.Metadata.Platform == "ruby" {
-		filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", rp.Name, rp.Version))
-	} else {
-		filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", rp.Name, rp.Version, rp.Metadata.Platform))
-	}
+	filename := makeGemFullFileName(rp.Name, rp.Version, rp.Metadata.Platform)
 
 	_, _, err = packages_service.CreatePackageAndAddFile(
 		ctx,
@@ -300,6 +296,136 @@ func DeletePackage(ctx *context.Context) {
 	}
 }
 
+// GetPackageInfo returns a custom text based format for the single rubygem with a line for each version of the rubygem
+// ref: https://guides.rubygems.org/rubygems-org-compact-index-api/
+func GetPackageInfo(ctx *context.Context) {
+	packageName := ctx.Params("packagename")
+	versions, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, packageName)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(versions) == 0 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+	infoContent, err := makePackageInfo(ctx, versions)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	ctx.PlainText(http.StatusOK, infoContent)
+}
+
+// GetAllPackagesVersions returns a custom text based format containing information about all versions of all rubygems.
+// ref: https://guides.rubygems.org/rubygems-org-compact-index-api/
+func GetAllPackagesVersions(ctx *context.Context) {
+	packages, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	out := &strings.Builder{}
+	out.WriteString("---\n")
+	for _, pkg := range packages {
+		versions, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, pkg.Name)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+		if len(versions) == 0 {
+			continue
+		}
+
+		info, err := makePackageInfo(ctx, versions)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+
+		// format: RUBYGEM [-]VERSION_PLATFORM[,VERSION_PLATFORM],...] MD5
+		_, _ = fmt.Fprintf(out, "%s ", pkg.Name)
+		for i, v := range versions {
+			sep := util.Iif(i == len(versions)-1, "", ",")
+			_, _ = fmt.Fprintf(out, "%s%s", v.Version, sep)
+		}
+		_, _ = fmt.Fprintf(out, " %x\n", md5.Sum([]byte(info)))
+	}
+
+	ctx.PlainText(http.StatusOK, out.String())
+}
+
+func writePackageVersionRequirements(prefix string, reqs []rubygems_module.VersionRequirement, out *strings.Builder) {
+	out.WriteString(prefix)
+	if len(reqs) == 0 {
+		reqs = []rubygems_module.VersionRequirement{{Restriction: ">=", Version: "0"}}
+	}
+	for i, req := range reqs {
+		sep := util.Iif(i == 0, "", "&")
+		_, _ = fmt.Fprintf(out, "%s%s %s", sep, req.Restriction, req.Version)
+	}
+}
+
+func makePackageVersionDependency(ctx *context.Context, version *packages_model.PackageVersion) (string, error) {
+	// format: VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...]
+	// DEPENDENCY: GEM:CONSTRAINT[&CONSTRAINT]
+	// REQUIREMENT: KEY:VALUE (always contains "checksum")
+	pd, err := packages_model.GetPackageDescriptor(ctx, version)
+	if err != nil {
+		return "", err
+	}
+
+	metadata := pd.Metadata.(*rubygems_module.Metadata)
+	fullFilename := makeGemFullFileName(pd.Package.Name, version.Version, metadata.Platform)
+	file, err := packages_model.GetFileForVersionByName(ctx, version.ID, fullFilename, "")
+	if err != nil {
+		return "", err
+	}
+	blob, err := packages_model.GetBlobByID(ctx, file.BlobID)
+	if err != nil {
+		return "", err
+	}
+
+	buf := &strings.Builder{}
+	buf.WriteString(version.Version)
+	buf.WriteByte(' ')
+	for i, dep := range metadata.RuntimeDependencies {
+		sep := util.Iif(i == 0, "", ",")
+		writePackageVersionRequirements(fmt.Sprintf("%s%s:", sep, dep.Name), dep.Version, buf)
+	}
+	_, _ = fmt.Fprintf(buf, "|checksum:%s", blob.HashSHA256)
+	if len(metadata.RequiredRubyVersion) != 0 {
+		writePackageVersionRequirements(",ruby:", metadata.RequiredRubyVersion, buf)
+	}
+	if len(metadata.RequiredRubygemsVersion) != 0 {
+		writePackageVersionRequirements(",rubygems:", metadata.RequiredRubygemsVersion, buf)
+	}
+	return buf.String(), nil
+}
+
+func makePackageInfo(ctx *context.Context, versions []*packages_model.PackageVersion) (string, error) {
+	ret := "---\n"
+	for _, v := range versions {
+		dep, err := makePackageVersionDependency(ctx, v)
+		if err != nil {
+			return "", err
+		}
+		ret += dep + "\n"
+	}
+	return ret, nil
+}
+
+func makeGemFullFileName(gemName, version, platform string) string {
+	var basename string
+	if platform == "" || platform == "ruby" {
+		basename = fmt.Sprintf("%s-%s", gemName, version)
+	} else {
+		basename = fmt.Sprintf("%s-%s-%s", gemName, version, platform)
+	}
+	return strings.ToLower(basename) + ".gem"
+}
+
 func getVersionsByFilename(ctx *context.Context, filename string) ([]*packages_model.PackageVersion, error) {
 	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
 		OwnerID:         ctx.Package.Owner.ID,