Teach PkgServer to serve multiple registry flavors

The storage server now exports two different "flavors" of registry
declarations, `eager` and `conservative`:

- `eager` registries are published as soon as possible, and should track
upstream git repositories quite closely, (in general, it should be less
than two minutes behind the state of the upstream git repository).  The
downside is that the storage server may not have processed some of the
resources in the registry, so if you are a user behind a restrictive
firewall and cannot download resources off of arbitrary servers, we
suggest you use the `conservative` registry.

- `conservative` registries are published only once the StorageServer
has at least tried to process all resources (packages, artifacts,
etc...) within a registry.  It does not guarantee that all resources are
downloadable (after all, there are some package versions that contain
links to artifacts that simply do not exist) but it gives a "best
effort" treehash to use.  The downside is that in the event that a large
number of new resources have been added to the registry recently, it may
take a little while for the storage server to process them all, so the
`conservative` registry flavor may lag behind real-time by a few hours.

Pkg clients that ask for `/registries` are, by default, served a
`conservative` registry, unless one of the following two conditions
hold:

- The Pkg client declares itself to be running CI (via one of the CI
  variables transmitted in the `Julia-Ci-Variables` header, see [0] for
  more), in which case we assume you likely want `eager`.

- The Pkg client transmits a `Julia-Registry-Preference` header set to
  one of the valid flavors, which currently includes `eager` and
  `conservative`.

Now that [1] has been merged into `Pkg.jl`, users can easily set their
own preference header by simply defining the environment mapping
`JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager`.

[0] https://github.com/JuliaLang/Pkg.jl/blob/bb180de2802a8320241e10721c1ac4eefd3eb27d/src/PlatformEngines.jl#L200-L213

[1] JuliaLang/Pkg.jl#2770

Author: staticfloat

src/PkgServer.jl

@@ -22,8 +22,8 @@ include("dynamic.jl")
 mutable struct RegistryMeta
     # Upstream registry URL (e.g. "https://github.com/JuliaRegistries/General")
     upstream_url::String
-    # The latest hash we know about for this registry
-    latest_hash::Union{Nothing,String}
+    # The hash (for each flavor) for each registry
+    hashes::Dict{String,String}
 
     function RegistryMeta(url::String)
         # Check to ensure this path actually exists
@@ -36,7 +36,7 @@ mutable struct RegistryMeta
         if !endswith(url, ".git") && url_exists(git_url)
             url = git_url
         end
-        return new(url, nothing)
+        return new(url, Dict{String,String}())
     end
 end
 
@@ -99,7 +99,8 @@ function start(;kwargs...)
 
     # Update registries first thing
     @info("Performing initial registry update")
-    update_registries()
+    registry_flavor_list = ("eager", "conservative")
+    update_registries.(registry_flavor_list)
     global last_registry_update = now()
 
     # Experimental.@sync throws if _any_ of the tasks fail
@@ -109,7 +110,7 @@ function start(;kwargs...)
                 sleep(1)
                 @try_printerror begin
                     forget_failures()
-                    update_registries()
+                    update_registries.(registry_flavor_list)
                     last_registry_update = now()
                 end
             end
@@ -166,8 +167,17 @@ function start(;kwargs...)
                 return
             end
 
-            if resource  == "/registries"
-                open(joinpath(config.root, "static", "registries")) do io
+            if occursin(r"^/registries(\.[a-z]+)?$", resource)
+                # If they're asking for just "/registries", inspect headers to figure
+                # out which registry flavor they actually want, and if none is given,
+                # give them `conservative` by default, unless they are self-reporting
+                # as a CI bot, in which case we'll always point them to `eager`.
+                if resource == "/registries"
+                    ci = any([v == "t" for (k, v) in filter(!isempty, split(HTTP.header(http, "Julia-CI-Variables", ""), ";"))])
+                    flavor = HTTP.header(http, "Julia-Registry-Preference", ci ? "eager" : "conservative")
+                    resource = "/registries.$(flavor)"
+                end
+                open(joinpath(config.root, "static", basename(resource))) do io
                     serve_file(http, io, "text/plain")
                 end
                 return

src/resource.jl

@@ -10,17 +10,17 @@ const resource_re = Regex("""
 """, "x")
 
 """
-    get_registries(server)
+    get_registries(server, flavor)
 
 Interrogate a storage server for a list of registries, match the response against the
 registries we are paying attention to, return dict mapping from registry UUID to its
 latest treehash.
 """
-function get_registries(server::AbstractString)
+function get_registries(server::AbstractString, flavor::AbstractString)
     regs = Dict{String,String}()
-    response = HTTP.get("$server/registries", status_exception = false)
+    response = HTTP.get("$server/registries.$(flavor)", status_exception = false)
     if response.status != 200
-        @error("Failure to fetch /registries", server, response.status)
+        @error("Failure to fetch /registries.$(flavor)", server, response.status)
         return regs
     end
     for line in eachline(IOBuffer(response.body))
@@ -30,7 +30,7 @@ function get_registries(server::AbstractString)
             uuid in keys(config.registries) || continue
             regs[uuid] = hash
         else
-            @error("invalid response", server, resource="registries", line)
+            @error("invalid response", server, resource="registries.$(flavor)", line)
         end
     end
     return regs
@@ -126,12 +126,12 @@ function verify_registry_hash(uuid::AbstractString, hash::AbstractString)
     return url === nothing || url_exists(url)
 end
 
-function update_registries()
+function update_registries(flavor::String)
     # collect current registry hashes from servers
     regs = Dict(uuid => Dict{String,Vector{String}}() for uuid in keys(config.registries))
     servers = Dict(uuid => Vector{String}() for uuid in keys(config.registries))
     for server in config.storage_servers
-        for (uuid, hash) in get_registries(server)
+        for (uuid, hash) in get_registries(server, flavor)
             push!(get!(regs[uuid], hash, String[]), server)
             push!(servers[uuid], server)
         end
@@ -173,29 +173,33 @@ function update_registries()
                 wait(dl_state.dl_task)
             end
 
-            if config.registries[uuid].latest_hash != hash
+            if get(config.registries[uuid].hashes, flavor, "") != hash
                 @info("new current registry hash", uuid, hash, hash_servers)
                 changed = true
             end
 
             # we've got a new registry hash to serve
-            config.registries[uuid].latest_hash = hash
+            config.registries[uuid].hashes[flavor] = hash
             break
         end
     end
 
-    # write new registry info to file
-    registries_path = joinpath(config.root, "static", "registries")
-    if changed || !isfile(registries_path)
-        new_registries = joinpath(config.root, "temp", "registries.tmp." * randstring())
-        mkpath(dirname(new_registries))
-        open(new_registries, "w") do io
-            for uuid in sort!(collect(keys(config.registries)))
-                println(io, "/registry/$(uuid)/$(config.registries[uuid].latest_hash)")
+    # write new registry info to file, for each flavor we publish
+    for flavor in ("eager", "conservative")
+        registries_path = joinpath(config.root, "static", "registries.$(flavor)")
+        if changed || !isfile(registries_path)
+            new_registries = joinpath(config.root, "temp", "registries.$(flavor).tmp." * randstring())
+            mkpath(dirname(new_registries))
+            open(new_registries, "w") do io
+                for uuid in sort!(collect(keys(config.registries)))
+                    if haskey(config.registries[uuid].hashes, flavor)
+                        println(io, "/registry/$(uuid)/$(config.registries[uuid].hashes[flavor])")
+                    end
+                end
             end
+            mkpath(dirname(registries_path))
+            mv(new_registries, registries_path; force=true)
         end
-        mkpath(dirname(registries_path))
-        mv(new_registries, registries_path; force=true)
     end
     return changed
 end

test/tests.jl

@@ -52,12 +52,15 @@ while true
 end
 
 @testset "Direct HTTP requests" begin
-    # Test that we get a sensical answer for /registries
-    response = HTTP.get("$(server_url)/registries")
-    @test response.status == 200
-    registry_url = chomp(String(response.body))
-    @test match(r"^/registry/([a-z0-9]+-){4}[a-z0-9]+/[a-z0-9]+$", registry_url) != nothing
-    registry_uuid, registry_treehash = split(registry_url, "/")[3:4]
+    # Test that we get a sensical answer for /registries{,.eager,.conservative}
+    local registry_uuid, registry_treehash
+    for registry_flavor in ("registries", "registries.eager", "registries.conservative")
+        response = HTTP.get("$(server_url)/$(registry_flavor)")
+        @test response.status == 200
+        registry_url = chomp(String(response.body))
+        @test match(r"^/registry/([a-z0-9]+-){4}[a-z0-9]+/[a-z0-9]+$", registry_url) !== nothing
+        registry_uuid, registry_treehash = split(registry_url, "/")[3:4]
+    end
 
     # Test asking for that registry directly, unpacking it and verifying the treehash
     mktemp() do tarball_path, tarball_io
@@ -68,7 +71,8 @@ end
     end
 
     # Verify that these files exist within the cache
-    @test isfile(joinpath(cache_dir, "..", "static", "registries"))
+    @test isfile(joinpath(cache_dir, "..", "static", "registries.eager"))
+    @test isfile(joinpath(cache_dir, "..", "static", "registries.conservative"))
     @test isfile(joinpath(cache_dir, "registry", registry_uuid, registry_treehash))
 
     # Next, hit the `/meta` endpoint, ensure that the version it reports matches with what we expect: