main: Use configuration file instead of flags and simplify app extens…

…ion.

Clair will now use a YAML configuration file instead of command line
arguments as the number of parameters grows.

Also, Clair now exposes a Boot() func that allows everyone to easily
create their own project and load dynamically their own fetchers/updaters.

Dockerfile

@@ -5,14 +5,15 @@ RUN apt-get update && apt-get install -y bzr rpm && apt-get autoremove -y && apt
 
 RUN mkdir         /db
 VOLUME            /db
+VOLUME            /config
 
 EXPOSE 6060 6061
 
 ADD .         /go/src/github.com/coreos/clair/
 WORKDIR       /go/src/github.com/coreos/clair/
 
 ENV GO15VENDOREXPERIMENT 1
-RUN go install -v
+RUN go install -v github.com/coreos/clair/cmd/clair
 RUN go test $(go list ./... | grep -v /vendor/) # https://github.com/golang/go/issues/11659
 
 ENTRYPOINT ["clair"]

api/api.go

@@ -25,66 +25,68 @@ import (
 	"github.com/coreos/pkg/capnslog"
 	"github.com/tylerb/graceful"
 
+	"github.com/coreos/clair/config"
 	"github.com/coreos/clair/utils"
 	httputils "github.com/coreos/clair/utils/http"
 )
 
 var log = capnslog.NewPackageLogger("github.com/coreos/clair", "api")
 
-// Config represents the configuration for the Main API.
-type Config struct {
-	Port                      int
-	TimeOut                   time.Duration
-	CertFile, KeyFile, CAFile string
-}
-
-// RunMain launches the main API, which exposes every possible interactions
+// Run launches the main API, which exposes every possible interactions
 // with clair.
-func RunMain(conf *Config, st *utils.Stopper) {
-	log.Infof("starting API on port %d.", conf.Port)
-	defer func() {
-		log.Info("API stopped")
-		st.End()
-	}()
+func Run(config *config.APIConfig, st *utils.Stopper) {
+	defer st.End()
+
+	// Do not run the API service if there is no config.
+	if config == nil {
+		log.Infof("main API service is disabled.")
+		return
+	}
+	log.Infof("starting main API on port %d.", config.Port)
 
-	tlsConfig, err := httputils.LoadTLSClientConfigForServer(conf.CAFile)
+	tlsConfig, err := httputils.LoadTLSClientConfigForServer(config.CAFile)
 	if err != nil {
 		log.Fatalf("could not initialize client cert authentification: %s\n", err)
 	}
 	if tlsConfig != nil {
-		log.Info("api configured with client certificate authentification")
+		log.Info("main API configured with client certificate authentification")
 	}
 
 	srv := &graceful.Server{
 		Timeout:          0,    // Already handled by our TimeOut middleware
 		NoSignalHandling: true, // We want to use our own Stopper
 		Server: &http.Server{
-			Addr:      ":" + strconv.Itoa(conf.Port),
+			Addr:      ":" + strconv.Itoa(config.Port),
 			TLSConfig: tlsConfig,
-			Handler:   NewVersionRouter(conf.TimeOut),
+			Handler:   NewVersionRouter(config.Timeout),
 		},
 	}
-	listenAndServeWithStopper(srv, st, conf.CertFile, conf.KeyFile)
+	listenAndServeWithStopper(srv, st, config.CertFile, config.KeyFile)
+	log.Info("main API stopped")
 }
 
 // RunHealth launches the Health API, which only exposes a method to fetch
-// clair's health without any security or authentification mechanism.
-func RunHealth(port int, st *utils.Stopper) {
-	log.Infof("starting Health API on port %d.", port)
-	defer func() {
-		log.Info("Health API stopped")
-		st.End()
-	}()
+// Clair's health without any security or authentification mechanism.
+func RunHealth(config *config.APIConfig, st *utils.Stopper) {
+	defer st.End()
+
+	// Do not run the API service if there is no config.
+	if config == nil {
+		log.Infof("health API service is disabled.")
+		return
+	}
+	log.Infof("starting health API on port %d.", config.HealthPort)
 
 	srv := &graceful.Server{
 		Timeout:          10 * time.Second, // Interrupt health checks when stopping
 		NoSignalHandling: true,             // We want to use our own Stopper
 		Server: &http.Server{
-			Addr:    ":" + strconv.Itoa(port),
+			Addr:    ":" + strconv.Itoa(config.HealthPort),
 			Handler: NewHealthRouter(),
 		},
 	}
 	listenAndServeWithStopper(srv, st, "", "")
+	log.Info("health API stopped")
 }
 
 // listenAndServeWithStopper wraps graceful.Server's

api/logic/layers.go

@@ -52,7 +52,7 @@ func POSTLayers(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	httputils.WriteHTTP(w, http.StatusCreated, struct{ Version string }{Version: strconv.Itoa(worker.Version)})
 }
 
-// DeleteLayer deletes the specified layer and any child layers that are
+// DELETELayers deletes the specified layer and any child layers that are
 // dependent on the specified layer.
 func DELETELayers(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 	err := database.DeleteLayer(p.ByName("id"))

api/wrappers/log.go

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package logic implements all the available API methods.
-// Every methods are documented in docs/API.md.
-
 // Package wrappers contains httprouter.Handle wrappers that are used in the API.
 package wrappers
 

api/wrappers/timeout.go

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package logic implements all the available API methods.
-// Every methods are documented in docs/API.md.
-
 package wrappers
 
 import (

clair.go

@@ -0,0 +1,74 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package clair implements the ability to boot Clair with your own imports
+// that can dynamically register additional functionality.
+package clair
+
+import (
+	"math/rand"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/coreos/clair/api"
+	"github.com/coreos/clair/config"
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/notifier"
+	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/pkg/capnslog"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "main")
+
+// Boot starts Clair. By exporting this function, anyone can import their own
+// custom fetchers/updaters into their own package and then call clair.Boot.
+func Boot(config *config.Config) {
+	rand.Seed(time.Now().UnixNano())
+	st := utils.NewStopper()
+
+	// Open database
+	err := database.Open(config.Database)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer database.Close()
+
+	// Start notifier
+	st.Begin()
+	notifier := notifier.New(config.Notifier)
+	go notifier.Serve(st)
+
+	// Start API
+	st.Begin()
+	go api.Run(config.API, st)
+	st.Begin()
+	go api.RunHealth(config.API, st)
+
+	// Start updater
+	st.Begin()
+	go updater.Run(config.Updater, st)
+
+	// Wait for interruption and shutdown gracefully.
+	waitForSignals(os.Interrupt)
+	log.Info("Received interruption, gracefully stopping ...")
+	st.Stop()
+}
+
+func waitForSignals(signals ...os.Signal) {
+	interrupts := make(chan os.Signal, 1)
+	signal.Notify(interrupts, signals...)
+	<-interrupts
+}

cmd/clair/main.go

@@ -0,0 +1,77 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"runtime/pprof"
+	"strings"
+
+	"github.com/coreos/clair"
+	"github.com/coreos/clair/config"
+
+	"github.com/coreos/pkg/capnslog"
+
+	// Register components
+	_ "github.com/coreos/clair/updater/fetchers"
+	_ "github.com/coreos/clair/worker/detectors/os"
+	_ "github.com/coreos/clair/worker/detectors/packages"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair/cmd/clair", "main")
+
+func main() {
+	// Parse command-line arguments
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+	flagConfigPath := flag.String("config", "", "Load configuration from the specified file.")
+	flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.")
+	flagLogLevel := flag.String("log-level", "info", "Define the logging level.")
+	flag.Parse()
+	// Load configuration
+	config, err := config.Load(*flagConfigPath)
+	if err != nil {
+		log.Fatalf("failed to load configuration: %s", err)
+	}
+
+	// Initialize logging system
+	logLevel, err := capnslog.ParseLevel(strings.ToUpper(*flagLogLevel))
+	capnslog.SetGlobalLogLevel(logLevel)
+	capnslog.SetFormatter(capnslog.NewPrettyFormatter(os.Stdout, false))
+
+	// Enable CPU Profiling if specified
+	if *flagCPUProfilePath != "" {
+		startCPUProfiling(*flagCPUProfilePath)
+		defer stopCPUProfiling()
+	}
+
+	clair.Boot(config)
+}
+
+func startCPUProfiling(path string) {
+	f, err := os.Create(path)
+	if err != nil {
+		log.Fatalf("failed to create profile file: %s", err)
+	}
+	defer f.Close()
+
+	pprof.StartCPUProfile(f)
+	log.Info("started CPU profiling")
+}
+
+func stopCPUProfiling() {
+	pprof.StopCPUProfile()
+	log.Info("stopped CPU profiling")
+}

config.example.yaml

@@ -0,0 +1,37 @@
+# The values specified here are the default values that Clair uses if no configuration file
+# is specified or if the keys are not defined.
+---
+database:
+  # Database backend.
+  #
+  # Possible values are "bolt", "leveldb", "memstore", "mongo", "sql".
+  #
+  # When running multiple instances is not desired, using BoltDB backend is the best choice as it is
+  # lightning fast. However, using PostgreSQL enables running multiple instances concurrently.
+  # The default is just an ephemeral database.
+  type: memstore
+  # Path to the database.
+  #
+  # Can be a file or a connection string.
+  path:
+api:
+  # Port on which the main API and the health API will listen on.
+  port: 6060
+  healthport: 6061
+  # Maximum time that API requests may take before they time-out with a HTTP 503 error.
+  timeout: 900s
+  # Paths to certificates to secure the main API with TLS and client certificate auth.
+  cafile:
+  keyfile:
+  certfile:
+updater:
+  # Frequency at which the vulnerability updater will run.
+  # Use 0 to disable the updater entirely.
+  interval: 2h
+notifier:
+  # HTTP endpoint that will receive notifications with POST requests.
+  endpoint:
+  # Path to certificates to call the endpoint securely with TLS and client certificate auth.
+  cafile:
+  keyfile:
+  certfile: