feat(pop): generate popularity on startup if url is not set

Signed-off-by: Maximilian Huber <maximilian.huber@tngtech.com>

README.md

@@ -88,6 +88,7 @@ variables:
   (defaults to 60)
 * `NIX_POPULARITY_URL`: URL to a file containing popularity data for
   the package set (see `popcount/`)
+* `NIXERY_COMPUTE_POPULARITY`: if `NIX_POPULARITY_URL` is not specified, and if the channel is set, the popularity data is computed by default. Setting this flag to `false` allows to disable that.
 
 If the `GOOGLE_APPLICATION_CREDENTIALS` environment variable is set to a service
 account key, Nixery will also use this key to create [signed URLs][] for layers

config/config.go

@@ -47,12 +47,13 @@ const (
 
 // Config holds the Nixery configuration options.
 type Config struct {
-	Port    string    // Port on which to launch HTTP server
-	Pkgs    PkgSource // Source for Nix package set
-	Timeout string    // Timeout for a single Nix builder (seconds)
-	WebDir  string    // Directory with static web assets
-	PopUrl  string    // URL to the Nix package popularity count
-	Backend Backend   // Storage backend to use for Nixery
+	Port    string          // Port on which to launch HTTP server
+	Pkgs    PkgSource       // Source for Nix package set
+	Timeout string          // Timeout for a single Nix builder (seconds)
+	WebDir  string          // Directory with static web assets
+	PopUrl  string          // URL to the Nix package popularity count
+	GeneratePopularity bool // whether to generate popularity if no url is specified
+	Backend Backend         // Storage backend to use for Nixery
 }
 
 func FromEnv() (Config, error) {
@@ -79,6 +80,7 @@ func FromEnv() (Config, error) {
 		Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
 		WebDir:  getConfig("WEB_DIR", "Static web file dir", ""),
 		PopUrl:  os.Getenv("NIX_POPULARITY_URL"),
+		GeneratePopularity: os.Getenv("NIXERY_COMPUTE_POPULARITY") != "false",
 		Backend: b,
 	}, nil
 }

default.nix

@@ -81,6 +81,7 @@ in rec {
   nixery-bin = writeShellScriptBin "nixery" ''
     export WEB_DIR="${nixery-book}"
     export PATH="${nixery-prepare-image}/bin:$PATH"
+    export PATH="${nixery-popcount}/bin:$PATH"
     exec ${nixery-server}/bin/nixery
   '';
 

main.go

@@ -31,6 +31,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"os/exec"
 	"io/ioutil"
 	"net/http"
 	"regexp"
@@ -87,6 +89,38 @@ func downloadPopularity(url string) (builder.Popularity, error) {
 	return pop, nil
 }
 
+func computePopularity() (builder.Popularity, error) {
+	channel := os.Getenv("NIXERY_CHANNEL")
+	if channel == "" {
+		return nil, nil
+	}
+
+	file, err := ioutil.TempFile(os.TempDir(), "popularity.*.json")
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(file.Name())
+
+	cmd := exec.Command("nixery-popcount", channel, file.Name())
+	_, err = cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	j, err := ioutil.ReadFile(file.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	var pop builder.Popularity
+	err = json.Unmarshal(j, &pop)
+	if err != nil {
+		return nil, err
+	}
+
+	return pop, nil
+}
+
 // Error format corresponding to the registry protocol V2 specification. This
 // allows feeding back errors to clients in a way that can be presented to
 // users.
@@ -264,6 +298,12 @@ func main() {
 			log.WithError(err).WithField("popURL", cfg.PopUrl).
 				Fatal("failed to fetch popularity information")
 		}
+	} else if cfg.GeneratePopularity {
+		pop, err = computePopularity()
+		if err != nil {
+			log.WithError(err).
+				Fatal("failed to compute popularity information")
+		}
 	}
 
 	state := builder.State{

popcount/popcount.go

@@ -224,6 +224,7 @@ func main() {
 		log.Fatalf("Nix channel must be specified as first argument")
 	}
 
+
 	err := os.MkdirAll("popcache", 0755)
 	if err != nil {
 		log.Fatalf("Failed to create 'popcache' directory in current folder: %s\n", err)
@@ -281,7 +282,12 @@ func main() {
 	}
 
 	bytes, _ := json.Marshal(counts)
-	outfile := fmt.Sprintf("popularity-%s-%s.json", meta.name, meta.commit)
+	var outfile string
+	if len(os.Args) >= 3 {
+		outfile = os.Args[2]
+	} else {
+		outfile = fmt.Sprintf("popularity-%s-%s.json", meta.name, meta.commit)
+	}
 	err = ioutil.WriteFile(outfile, bytes, 0644)
 	if err != nil {
 		log.Fatalf("Failed to write output to '%s': %s\n", outfile, err)