Simpler config

Dockerfile

@@ -1,10 +1,10 @@
 FROM golang:alpine
 
-
-ENV PATH="${PATH}:/tmp/git-annex.linux"
+RUN mkdir /git-annex
+ENV PATH="${PATH}:/git-annex/git-annex.linux"
 RUN apk add --no-cache git openssh curl
-RUN curl -Lo /tmp/git-annex-standalone-amd64.tar.gz https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
-RUN cd /tmp && tar -xzf git-annex-standalone-amd64.tar.gz && rm git-annex-standalone-amd64.tar.gz
+RUN curl -Lo /git-annex/git-annex-standalone-amd64.tar.gz https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
+RUN cd /git-annex && tar -xzf git-annex-standalone-amd64.tar.gz && rm git-annex-standalone-amd64.tar.gz
 RUN apk del --no-cache curl
 
 RUN go version
@@ -24,5 +24,5 @@ RUN go build
 VOLUME ["/doidata"]
 VOLUME ["/gindoid/config"]
 
-ENTRYPOINT ./gindoid --debug --knownhosts=/gindoid/config/hostkey --target=/doidata --templates=/tmpl --key=$tokenkey --port=10443 --sendmail --mailtofile=/gindoid/config/emails --mailserver=$mailserver --oauthserver=$authserver --source=$ginserver
+ENTRYPOINT ./gindoid --debug
 EXPOSE 10443

cmd/gindoid/dstorage.go

@@ -35,39 +35,44 @@ func (ls LocalStorage) Put(job DOIJob) error {
 	target := job.Name
 	dReq := &job.Request
 
-	//todo do this better
 	to := filepath.Join(ls.Path, target)
 	tmpDir := filepath.Join(to, tmpdir)
 	ls.prepDir(target, dReq.DOIInfo)
 	ds := ls.GetDataSource()
 
+	preperrors := make([]string, 0, 5)
+
 	if out, err := ds.CloneRepository(source, tmpDir, &job.Key, ls.KnownHosts); err != nil {
 		log.WithFields(log.Fields{
 			"source": lpStorage,
 			"error":  err,
 			"out":    out,
 			"target": target,
 		}).Error("Repository cloning failed")
+		preperrors = append(preperrors, fmt.Sprintf("Failed to clone repository '%s': %s", source, err))
+	} else {
+		fSize, err := ls.zip(target)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"source": lpStorage,
+				"error":  err,
+				"target": target,
+			}).Error("Could not zip the data")
+			preperrors = append(preperrors, fmt.Sprintf("Failed to create the zip file: %s", err))
+		}
+		// +1 to report something with small datasets
+		dReq.DOIInfo.FileSize = fSize/(1024*1000) + 1
 	}
-	fSize, err := ls.zip(target)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"source": lpStorage,
-			"error":  err,
-			"target": target,
-		}).Error("Could not zip the data")
-	}
-	// +1 to report something with small datasets
-	dReq.DOIInfo.FileSize = fSize/(1024*1000) + 1
 	ls.createIndexFile(target, dReq)
 
-	fp, _ := os.Create(filepath.Join(to, "doi.xml"))
+	fp, err := os.Create(filepath.Join(to, "doi.xml"))
 	if err != nil {
 		log.WithFields(log.Fields{
 			"source": lpStorage,
 			"error":  err,
 			"target": target,
-		}).Error("Could not create parse the metadata template")
+		}).Error("Could not create the metadata template")
+		preperrors = append(preperrors, fmt.Sprintf("Failed to create the XML metadata template: %s", err))
 	}
 	defer fp.Close()
 	// No registering. But the XML is provided with everything
@@ -79,7 +84,8 @@ func (ls LocalStorage) Put(job DOIJob) error {
 			"source": lpStorage,
 			"error":  err,
 			"target": target,
-		}).Error("Could not create the metadata file")
+		}).Error("Could not parse the metadata file")
+		preperrors = append(preperrors, fmt.Sprintf("Failed to parse the XML metadata: %s", err))
 	}
 	_, err = fp.Write([]byte(data))
 	if err != nil {
@@ -88,7 +94,9 @@ func (ls LocalStorage) Put(job DOIJob) error {
 			"error":  err,
 			"target": target,
 		}).Error("Could not write to the metadata file")
+		preperrors = append(preperrors, fmt.Sprintf("Failed to write the metadata XML file: %s", err))
 	}
+	dReq.ErrorMessages = preperrors
 	ls.sendMaster(dReq)
 	return err
 }
@@ -110,6 +118,10 @@ func (ls *LocalStorage) zip(target string) (int64, error) {
 	}
 	defer fp.Close()
 	err = libgin.MakeZip(fp, filepath.Join(to, tmpdir))
+	if err != nil {
+		log.Errorf("MakeZip failed: %s", err)
+		return 0, err
+	}
 	stat, _ := fp.Stat()
 	return stat.Size(), err
 }
@@ -154,7 +166,7 @@ func (ls LocalStorage) createIndexFile(target string, info *DOIReq) error {
 }
 
 func (ls *LocalStorage) prepDir(target string, info *DOIRegInfo) error {
-	err := os.Mkdir(filepath.Join(ls.Path, target), os.ModePerm)
+	err := os.MkdirAll(filepath.Join(ls.Path, target), os.ModePerm)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"source": lpStorage,
@@ -190,7 +202,6 @@ func (ls LocalStorage) getSCP(dReq *DOIReq) string {
 	return fmt.Sprintf("%s/%s/doi.xml", ls.SCPURL, dReq.DOIInfo.UUID)
 }
 func (ls LocalStorage) sendMaster(dReq *DOIReq) error {
-
 	urljoin := func(a, b string) string {
 		fallback := fmt.Sprintf("%s/%s (fallback URL join)", a, b)
 		base, err := url.Parse(a)
@@ -211,6 +222,14 @@ func (ls LocalStorage) sendMaster(dReq *DOIReq) error {
 	uuid := dReq.DOIInfo.UUID
 	doitarget := urljoin(ls.HTTPBase, uuid)
 
+	errorlist := ""
+	if len(dReq.ErrorMessages) > 0 {
+		errorlist = "The following errors occurred during the dataset preparation\n"
+		for idx, msg := range dReq.ErrorMessages {
+			errorlist = fmt.Sprintf("%s	%d. %s\n", errorlist, idx+1, msg)
+		}
+	}
+
 	subject := fmt.Sprintf("New DOI registration request: %s", repopath)
 
 	body := `A new DOI registration request has been received.
@@ -221,7 +240,9 @@ func (ls LocalStorage) sendMaster(dReq *DOIReq) error {
 	DOI XML: %s
 	DOI target URL: %s
 	UUID: %s
+
+%s
 `
-	body = fmt.Sprintf(body, repopath, userlogin, useremail, xmlurl, doitarget, uuid)
+	body = fmt.Sprintf(body, repopath, userlogin, useremail, xmlurl, doitarget, uuid, errorlist)
 	return ls.MServer.SendMail(subject, body)
 }

cmd/gindoid/gindoi.go

@@ -54,12 +54,13 @@ type DOIUser struct {
 }
 
 type DOIReq struct {
-	URI        string
-	User       DOIUser
-	OAuthLogin string
-	Token      string
-	Message    template.HTML
-	DOIInfo    *DOIRegInfo
+	URI           string
+	User          DOIUser
+	OAuthLogin    string
+	Token         string
+	Message       template.HTML
+	DOIInfo       *DOIRegInfo
+	ErrorMessages []string
 }
 
 type OAuthIdentity struct {

cmd/gindoid/main.go

@@ -12,35 +12,24 @@ import (
 
 const usage = `gindoid: DOI service for preparing GIN repositories for publication
 Usage:
-  gindoid [--maxworkers=<n> --maxqueue=<n> --port=<port> --source=<url> --gitsource=<url>
-           --oauthserver=<url> --target=<dir> --storeurl=<url> --mailserver=<host:port> --mailfrom=<address>
-           --mailtofile=<path> --doibase=<prefix> --sendmail --debug --templates=<path> --xmlurl=<url>
-           --knownhosts=<path>] --key=<key>
-
-Options:
-  --maxworkers=<n>                 The number of workers to start [default: 3]
-  --maxqueue=<n>                   The size of the job queue [default: 100]
-  --port=<port>                    The server port [default: 8083]
-  --source=<url>                   The server address from which data can be read [default: https://web.gin.g-node.org]
-  --gitsource=<url>                The git server address from which data can be cloned [default: ssh://git@gin.g-node.org]
-  --oauthserver=<url>              The server of the repo service [default: https://web.gin.g-node.org]
-  --target=<dir>                   The location for long term storage [default: data]
-  --storeurl=<url>                 The base URL for storage [default: http://doid.gin.g-node.org/]
-  --mailserver=<host:port>         The mail server address (:and port) [default: localhost:25]
-  --mailfrom=<address>             The mail from address [default: no-reply@g-node.org]
-  --mailtofile=<path>              A file containing email addresses (one per line) to notify of new requests
-  --doibase=<prefix>               The DOI prefix [default: 10.12751/g-node.]
-  --sendmail                       Whether mail notifications should really be sent (otherwise just print them)
-  --debug                          Whether debug messages shall be printed
-  --templates=<path>               Path to the templates [default: tmpl]
-  --xmlurl=<url>                   URI of the datacite XML [default: gin.g-node.org:/data/doid]
-  --knownhosts=<path>              Path to SSH known hosts file [default: .ssh/known_hosts]
-  --key=<key>                      Key used to decrypt token
+  gindoid [--debug]
+
+  --debug              Print debug messages
 `
 
 // TODO: Make non-global
 var doibase string
 
+// reqconf (require configuration) returns the value of a configuration env variable and exits with an error if it is not set.
+func reqconf(key string) string {
+	value, ok := os.LookupEnv(key)
+	if !ok {
+		log.Errorf("Configuration environment variable '%s' is not set", key)
+		os.Exit(-1)
+	}
+	return value
+}
+
 func main() {
 	args, err := docopt.Parse(usage, nil, true, "gin doi 0.1a", false)
 	if err != nil {
@@ -57,19 +46,19 @@ func main() {
 	log.Debug("Starting up")
 
 	// Setup data source
-	ginurl := args["--source"].(string)
-	giturl := args["--gitsource"].(string)
+	ginurl := reqconf("ginurl")
+	giturl := reqconf("giturl")
 	log.Debugf("gin: %s -- git: %s", ginurl, giturl)
 	ds := DataSource{GinURL: ginurl, GinGitURL: giturl}
 
-	doibase = args["--doibase"].(string)
+	doibase = reqconf("doibase")
 	log.Debugf("doibase: %s", doibase)
 
 	// Setup storage
-	mailserver := args["--mailserver"].(string)
-	mailfrom := args["--mailfrom"].(string)
-	sendmail := args["--sendmail"].(bool)
-	mailtofile := args["--mailtofile"].(string)
+	mailserver := reqconf("mailserver")
+	mailfrom := reqconf("mailfrom")
+	sendmail := true // TODO: Remove option
+	mailtofile := reqconf("mailtofile")
 	mServer := MailServer{
 		Address:   mailserver,
 		From:      mailfrom,
@@ -78,11 +67,11 @@ func main() {
 	}
 	log.Debugf("Mail configuration: %+v", mServer)
 
-	target := args["--target"].(string)
-	storeurl := args["--storeurl"].(string)
-	templates := args["--templates"].(string)
-	xmlurl := args["--xmlurl"].(string)
-	knownhosts := args["--knownhosts"].(string)
+	target := reqconf("target")
+	storeurl := reqconf("storeurl")
+	templates := reqconf("templates")
+	xmlurl := reqconf("xmlurl")
+	knownhosts := reqconf("knownhosts")
 	storage := LocalStorage{
 		Path:         target,
 		Source:       ds,
@@ -95,26 +84,26 @@ func main() {
 	log.Debugf("LocalStorage configuration: %+v", storage)
 
 	// setup authentication
-	oAuthAddress := args["--oauthserver"].(string)
+	oAuthAddress := reqconf("oauthserver")
 	op := OAuthProvider{
 		URI:      fmt.Sprintf("%s/api/v1/user", oAuthAddress),
 		TokenURL: "",
 		KeyURL:   fmt.Sprintf("%s/api/v1/user/keys", oAuthAddress),
 	}
 	log.Debugf("OAuth configuration: %+v", op)
 
-	key := args["--key"].(string)
+	key := reqconf("key")
 
 	// Create the job queue.
-	maxQ, err := strconv.Atoi(args["--maxqueue"].(string))
+	maxQ, err := strconv.Atoi(reqconf("maxqueue"))
 	if err != nil {
 		log.Printf("Error while parsing maxqueue flag: %s", err.Error())
 		log.Print("Using default")
 		maxQ = 100
 	}
 	jobQueue := make(chan DOIJob, maxQ)
 	// Start the dispatcher.
-	maxW, err := strconv.Atoi(args["--maxworkers"].(string))
+	maxW, err := strconv.Atoi(reqconf("maxworkers"))
 	if err != nil {
 		log.Printf("Error while parsing maxworkers flag: %s", err.Error())
 		log.Print("Using default")
@@ -137,7 +126,7 @@ func main() {
 	http.Handle("/assets/",
 		http.StripPrefix("/assets/", http.FileServer(http.Dir("/assets"))))
 
-	port := args["--port"].(string)
+	port := reqconf("port")
 	fmt.Printf("Listening for connections on port %s\n", port)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
 }