From 4ed560158a54df7b5c52ca4289cc0a07ebc73093 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Mon, 17 Jul 2017 13:45:02 -0500 Subject: [PATCH 01/13] go fmt --- shock-server/auth/cache.go | 4 ++-- shock-server/auth/globus/globus.go | 2 +- shock-server/conf/conf.go | 14 ++++---------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/shock-server/auth/cache.go b/shock-server/auth/cache.go index 44a9084e..dbfcacdb 100644 --- a/shock-server/auth/cache.go +++ b/shock-server/auth/cache.go @@ -1,8 +1,8 @@ package auth import ( + "github.com/MG-RAST/Shock/shock-server/conf" "github.com/MG-RAST/Shock/shock-server/user" - "github.com/MG-RAST/Shock/shock-server/conf" "sync" "time" ) @@ -36,7 +36,7 @@ func (c *cache) add(header string, u *user.User) { c.m[header] = cacheValue{ expires: time.Now().Add(time.Duration(conf.AUTH_CACHE_TIMEOUT) * time.Minute), //expires: time.Now().Add(1 * time.Minute), - user: u, + user: u, } return } diff --git a/shock-server/auth/globus/globus.go b/shock-server/auth/globus/globus.go index ed4c279e..3f3fe3bd 100644 --- a/shock-server/auth/globus/globus.go +++ b/shock-server/auth/globus/globus.go @@ -140,7 +140,7 @@ func clientId(t string) string { Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } req, err := http.NewRequest("GET", conf.AUTH_GLOBUS_TOKEN_URL, nil) - //logger.Error("URL: " + conf.AUTH_GLOBUS_TOKEN_URL) + //logger.Error("URL: " + conf.AUTH_GLOBUS_TOKEN_URL) if err != nil { logger.Error("Failed contact with auth server") diff --git a/shock-server/conf/conf.go b/shock-server/conf/conf.go index f62a021c..3d8390d9 100644 --- a/shock-server/conf/conf.go +++ b/shock-server/conf/conf.go @@ -49,11 +49,9 @@ var ( // Runtime - EXPIRE_WAIT int // wait time for reaper in minutes - GOMAXPROCS string - MAX_REVISIONS int // max number of node revisions to keep; values < 0 mean keep all - - + EXPIRE_WAIT int // wait time for reaper in minutes + GOMAXPROCS string + MAX_REVISIONS int // max number of node revisions to keep; values < 0 mean keep all // Logs LOG_PERF bool // Indicates whether performance logs should be stored @@ -183,7 +181,7 @@ func Print() { } else { fmt.Printf("##### Log rotation disabled #####\n\n") } - fmt.Printf("##### Expiration #####\nexpire_wait:\t%d minutes\n\n", EXPIRE_WAIT) + fmt.Printf("##### Expiration #####\nexpire_wait:\t%d minutes\n\n", EXPIRE_WAIT) fmt.Printf("##### Max Revisions #####\nmax_revisions:\t%d\n\n", MAX_REVISIONS) fmt.Printf("API_PORT: %d\n", API_PORT) } @@ -226,7 +224,6 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "") c_store.AddInt(&MAX_REVISIONS, 3, "Runtime", "max_revisions", "", "") - c_store.AddBool(&LOG_PERF, false, "Log", "perf_log", "", "") c_store.AddBool(&LOG_ROTATE, true, "Log", "rotate", "", "") @@ -312,9 +309,6 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.Parse() - - return } - From 2f960d781cbff2a3bf502bae62e3e79b5fd42fdd Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 18 Jul 2017 10:52:34 -0500 Subject: [PATCH 02/13] first pass use generic oauth service --- shock-server/auth/auth.go | 6 +-- shock-server/auth/globus/globus.go | 2 +- .../auth/{mgrast/mgrast.go => oauth/oauth.go} | 39 +++++++++------ shock-server/conf/conf.go | 49 +++++++++++-------- shock-server/main.go | 6 ++- 5 files changed, 61 insertions(+), 41 deletions(-) rename shock-server/auth/{mgrast/mgrast.go => oauth/oauth.go} (64%) diff --git a/shock-server/auth/auth.go b/shock-server/auth/auth.go index 37d3c4c6..74676022 100644 --- a/shock-server/auth/auth.go +++ b/shock-server/auth/auth.go @@ -5,7 +5,7 @@ import ( "errors" //"github.com/MG-RAST/Shock/shock-server/auth/basic" "github.com/MG-RAST/Shock/shock-server/auth/globus" - "github.com/MG-RAST/Shock/shock-server/auth/mgrast" + "github.com/MG-RAST/Shock/shock-server/auth/oauth" "github.com/MG-RAST/Shock/shock-server/conf" e "github.com/MG-RAST/Shock/shock-server/errors" "github.com/MG-RAST/Shock/shock-server/user" @@ -21,8 +21,8 @@ func Initialize() { if conf.AUTH_GLOBUS_TOKEN_URL != "" && conf.AUTH_GLOBUS_PROFILE_URL != "" { authMethods = append(authMethods, globus.Auth) } - if conf.AUTH_MGRAST_OAUTH_URL != "" { - authMethods = append(authMethods, mgrast.Auth) + if len(conf.AUTH_OAUTH) > 0 { + authMethods = append(authMethods, oauth.Auth) } } diff --git a/shock-server/auth/globus/globus.go b/shock-server/auth/globus/globus.go index 3f3fe3bd..3bce6abc 100644 --- a/shock-server/auth/globus/globus.go +++ b/shock-server/auth/globus/globus.go @@ -42,7 +42,7 @@ func authHeaderType(header string) string { // user func Auth(header string) (usr *user.User, err error) { switch authHeaderType(header) { - case "globus-goauthtoken", "oauth": + case "globus-goauthtoken", "globus", "goauth": return fetchProfile(strings.Split(header, " ")[1]) case "basic": if username, password, err := basic.DecodeHeader(header); err == nil { diff --git a/shock-server/auth/mgrast/mgrast.go b/shock-server/auth/oauth/oauth.go similarity index 64% rename from shock-server/auth/mgrast/mgrast.go rename to shock-server/auth/oauth/oauth.go index fe596167..6e5fbc48 100644 --- a/shock-server/auth/mgrast/mgrast.go +++ b/shock-server/auth/oauth/oauth.go @@ -1,5 +1,5 @@ // Package globus implements MG-RAST OAuth authentication -package mgrast +package oauth import ( "crypto/tls" @@ -20,6 +20,7 @@ type resErr struct { type credentials struct { Uname string `json:"login"` + Name string `json:"name"` Fname string `json:"firstname"` Lname string `json:"lastname"` Email string `json:"email"` @@ -33,29 +34,35 @@ func authHeaderType(header string) string { return "" } -// Auth takes the request authorization header and returns -// user +// Auth takes the request authorization header and returns user +// bearer token "oauth" returns default url (first item in auth_oauth_url conf value) func Auth(header string) (*user.User, error) { - switch authHeaderType(header) { - case "mgrast", "oauth": - return authToken(strings.Split(header, " ")[1]) - case "basic": - return nil, errors.New("This authentication method does not support username/password authentication. Please use your MG-RAST token.") - default: - return nil, errors.New("Invalid authentication header.") + bearer := authHeaderType(header) + if bearer == "" { + return nil, errors.New("Invalid authentication header, missing bearer token.") + } + oauth_url, found_url := conf.AUTH_OAUTH[bearer] + if bearer == "basic" { + return nil, errors.New("This authentication method does not support username/password authentication. Please use your OAuth token.") + } else if bearer == "oauth" { + return authToken(strings.Split(header, " ")[1], conf.OAUTH_DEFAULT) + } else if found_url { + return authToken(strings.Split(header, " ")[1], oauth_url) + } else { + return nil, errors.New("Invalid authentication header, unknown bearer token: " + bearer) } } // authToken validiates token by fetching user information. -func authToken(t string) (u *user.User, err error) { +func authToken(token string, url string) (u *user.User, err error) { client := &http.Client{ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, } - req, err := http.NewRequest("GET", conf.AUTH_MGRAST_OAUTH_URL, nil) + req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - req.Header.Add("Auth", t) + req.Header.Add("Auth", token) if resp, err := client.Do(req); err == nil { defer resp.Body.Close() if resp.StatusCode == http.StatusOK { @@ -69,7 +76,11 @@ func authToken(t string) (u *user.User, err error) { return nil, errors.New(e.InvalidAuth) } u.Username = c.Uname - u.Fullname = c.Fname + " " + c.Lname + if c.Name != "" { + u.Fullname = c.Name + } else { + u.Fullname = c.Fname + " " + c.Lname + } u.Email = c.Email if err = u.SetMongoInfo(); err != nil { return nil, err diff --git a/shock-server/conf/conf.go b/shock-server/conf/conf.go index 3d8390d9..d2408b27 100644 --- a/shock-server/conf/conf.go +++ b/shock-server/conf/conf.go @@ -37,8 +37,11 @@ var ( AUTH_BASIC bool AUTH_GLOBUS_TOKEN_URL string AUTH_GLOBUS_PROFILE_URL string - AUTH_MGRAST_OAUTH_URL string + AUTH_OAUTH_URL_STR string + AUTH_OAUTH_BEARER_STR string AUTH_CACHE_TIMEOUT int + AUTH_OAUTH = make(map[string]string) + OAUTH_DEFAULT string // first value in AUTH_OAUTH_URL_STR // Default Chunksize for size virtual index CHUNK_SIZE int64 = 1048576 @@ -48,7 +51,6 @@ var ( LOG_OUTPUT string // Runtime - EXPIRE_WAIT int // wait time for reaper in minutes GOMAXPROCS string MAX_REVISIONS int // max number of node revisions to keep; values < 0 mean keep all @@ -127,14 +129,14 @@ func Initialize() (err error) { c_store, err := getConfiguration(c) // from config file and command line arguments if err != nil { - err = fmt.Errorf("ERROR: error reading conf file: %v\n", err) + err = errors.New("ERROR: error reading conf file: " + err.Error()) return } // ####### at this point configuration variables are set ######## if FAKE_VAR == false { - err = fmt.Errorf("ERROR: config was not parsed\n") + err = errors.New("ERROR: config was not parsed") return } if PRINT_HELP || SHOW_HELP { @@ -154,13 +156,16 @@ func Bool(s string) bool { // Print prints the configuration loads to stdout func Print() { fmt.Printf("####### Anonymous ######\nread:\t%v\nwrite:\t%v\ndelete:\t%v\n\n", ANON_READ, ANON_WRITE, ANON_DELETE) - if (AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "") || AUTH_MGRAST_OAUTH_URL != "" { + if (AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "") || len(AUTH_OAUTH) > 0 { fmt.Printf("##### Auth #####\n") if AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "" { fmt.Printf("type:\tglobus\ntoken_url:\t%s\nprofile_url:\t%s\n\n", AUTH_GLOBUS_TOKEN_URL, AUTH_GLOBUS_PROFILE_URL) } - if AUTH_MGRAST_OAUTH_URL != "" { - fmt.Printf("type:\tmgrast\noauth_url:\t%s\n\n", AUTH_MGRAST_OAUTH_URL) + if len(AUTH_OAUTH) > 0 { + for b, u := range AUTH_OAUTH { + fmt.Printf("bearer: %s\turl: %s\n", b, u) + } + fmt.Printf("\n") } } fmt.Printf("##### Admin #####\nusers:\t%s\n\n", ADMIN_USERS) @@ -216,9 +221,24 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddBool(&AUTH_BASIC, false, "Auth", "basic", "", "") c_store.AddString(&AUTH_GLOBUS_TOKEN_URL, "", "Auth", "globus_token_url", "", "") c_store.AddString(&AUTH_GLOBUS_PROFILE_URL, "", "Auth", "globus_profile_url", "", "") - c_store.AddString(&AUTH_MGRAST_OAUTH_URL, "", "Auth", "mgrast_oauth_url", "", "") + c_store.AddString(&AUTH_OAUTH_URL_STR, "", "Auth", "oauth_urls", "", "") + c_store.AddString(&AUTH_OAUTH_BEARER_STR, "", "Auth", "oauth_bearers", "", "") c_store.AddInt(&AUTH_CACHE_TIMEOUT, 60, "Auth", "cache_timeout", "", "") + // parse OAuth settings if used + if AUTH_OAUTH_URL_STR != "" && AUTH_OAUTH_BEARER_STR != "" { + ou := strings.Split(AUTH_OAUTH_URL_STR, ",") + ob := strings.Split(AUTH_OAUTH_BEARER_STR, ",") + if len(ou) != len(ob) { + err = errors.New("ERROR: number of items in oauth_urls and oauth_bearers are not the same") + return + } + for i := range ob { + AUTH_OAUTH[ob[i]] = ou[i] + } + OAUTH_DEFAULT = ou[0] // first url is default for "oauth" bearer token + } + // Runtime c_store.AddInt(&EXPIRE_WAIT, 60, "Runtime", "expire_wait", "", "") c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "") @@ -230,10 +250,6 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { // Mongodb c_store.AddString(&MONGODB_ATTRIBUTE_INDEXES, "", "Mongodb", "attribute_indexes", "", "") c_store.AddString(&MONGODB_DATABASE, "ShockDB", "Mongodb", "database", "", "") - - //MONGODB_HOSTS, _ = c.String("Mongodb", "hosts") - //MONGODB_PASSWORD, _ = c.String("Mongodb", "password") - //MONGODB_USER, _ = c.String("Mongodb", "user") c_store.AddString(&MONGODB_HOSTS, "mongo", "Mongodb", "hosts", "", "") c_store.AddString(&MONGODB_PASSWORD, "", "Mongodb", "password", "", "") c_store.AddString(&MONGODB_USER, "", "Mongodb", "user", "", "") @@ -270,12 +286,6 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { } // Paths - //PATH_SITE, _ = c.String("Paths", "site") - //PATH_DATA, _ = c.String("Paths", "data") - //PATH_LOGS, _ = c.String("Paths", "logs") - //PATH_LOCAL, _ = c.String("Paths", "local_paths") - //PATH_PIDFILE, _ = c.String("Paths", "pidfile") - c_store.AddString(&PATH_SITE, "/usr/local/shock/site", "Paths", "site", "", "") c_store.AddString(&PATH_DATA, "/usr/local/shock", "Paths", "data", "", "") c_store.AddString(&PATH_LOGS, "/var/log/shock", "Paths", "logs", "", "") @@ -283,11 +293,8 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddString(&PATH_PIDFILE, "", "Paths", "pidfile", "", "") // SSL - //SSL, _ = c.Bool("SSL", "enable") c_store.AddBool(&SSL, false, "SSL", "enable", "", "") if SSL { - //SSL_KEY, _ = c.String("SSL", "key") - //SSL_CERT, _ = c.String("SSL", "cert") c_store.AddString(&SSL_KEY, "", "SSL", "key", "", "") c_store.AddString(&SSL_CERT, "", "SSL", "cert", "", "") } diff --git a/shock-server/main.go b/shock-server/main.go index fb3c7c46..40fd0c5d 100644 --- a/shock-server/main.go +++ b/shock-server/main.go @@ -141,8 +141,10 @@ func mapRoutes() { if conf.AUTH_GLOBUS_TOKEN_URL != "" && conf.AUTH_GLOBUS_PROFILE_URL != "" { auth = append(auth, "globus") } - if conf.AUTH_MGRAST_OAUTH_URL != "" { - auth = append(auth, "mgrast") + if len(conf.AUTH_OAUTH) > 0 { + for b := range conf.AUTH_OAUTH { + auth = append(auth, b) + } } r := resource{ From 5d45ac998d65a95b31b1e97469806e240f72fd05 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Tue, 18 Jul 2017 12:40:51 -0500 Subject: [PATCH 03/13] fix cong init and error passing --- shock-server/conf/conf.go | 51 ++++++++++++++++++++------------------- shock-server/main.go | 24 ++++++++---------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/shock-server/conf/conf.go b/shock-server/conf/conf.go index d2408b27..4fef62ad 100644 --- a/shock-server/conf/conf.go +++ b/shock-server/conf/conf.go @@ -108,8 +108,7 @@ func Initialize() (err error) { } else if i+1 < len(os.Args) { CONFIG_FILE = os.Args[i+1] } else { - err = errors.New("ERROR: parsing command options, missing conf file") - return + return errors.New("parsing command options, missing conf file") } } } @@ -118,8 +117,7 @@ func Initialize() (err error) { if CONFIG_FILE != "" { c, err = config.ReadDefault(CONFIG_FILE) if err != nil { - err = errors.New("ERROR: error reading conf file: " + err.Error()) - return + return errors.New("error reading conf file: " + err.Error()) } fmt.Printf("read %s\n", CONFIG_FILE) } else { @@ -129,15 +127,18 @@ func Initialize() (err error) { c_store, err := getConfiguration(c) // from config file and command line arguments if err != nil { - err = errors.New("ERROR: error reading conf file: " + err.Error()) - return + return errors.New("error reading conf file: " + err.Error()) } // ####### at this point configuration variables are set ######## + err = parseConfiguration() + if err != nil { + return errors.New("error parsing conf file: " + err.Error()) + } + if FAKE_VAR == false { - err = errors.New("ERROR: config was not parsed") - return + return errors.New("config was not parsed") } if PRINT_HELP || SHOW_HELP { c_store.PrintHelp() @@ -162,6 +163,7 @@ func Print() { fmt.Printf("type:\tglobus\ntoken_url:\t%s\nprofile_url:\t%s\n\n", AUTH_GLOBUS_TOKEN_URL, AUTH_GLOBUS_PROFILE_URL) } if len(AUTH_OAUTH) > 0 { + fmt.Printf("type:\toauth\n") for b, u := range AUTH_OAUTH { fmt.Printf("bearer: %s\turl: %s\n", b, u) } @@ -176,7 +178,7 @@ func Print() { } else { fmt.Printf("##### SSL disabled #####\n\n") } - fmt.Printf("##### Mongodb #####\nhost(s):\t%s\ndatabase:\t%s\n\n", MONGODB_HOSTS, MONGODB_DATABASE) + fmt.Printf("##### Mongodb #####\nhost(s):\t%s\ndatabase:\t%s\nattribute_indexes:\t%s\n\n", MONGODB_HOSTS, MONGODB_DATABASE, MONGODB_ATTRIBUTE_INDEXES) fmt.Printf("##### Address #####\nip:\t%s\nport:\t%d\n\n", API_IP, API_PORT) if LOG_PERF { fmt.Printf("##### PerfLog enabled #####\n\n") @@ -195,9 +197,7 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store = NewCS(c) // Admin - //ADMIN_EMAIL, _ = c.String("Admin", "email") c_store.AddString(&ADMIN_EMAIL, "", "Admin", "email", "", "") - //ADMIN_USERS, _ = c.String("Admin", "users") c_store.AddString(&ADMIN_USERS, "", "Admin", "users", "", "") if ADMIN_USERS != "" { for _, name := range strings.Split(ADMIN_USERS, ",") { @@ -225,20 +225,6 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddString(&AUTH_OAUTH_BEARER_STR, "", "Auth", "oauth_bearers", "", "") c_store.AddInt(&AUTH_CACHE_TIMEOUT, 60, "Auth", "cache_timeout", "", "") - // parse OAuth settings if used - if AUTH_OAUTH_URL_STR != "" && AUTH_OAUTH_BEARER_STR != "" { - ou := strings.Split(AUTH_OAUTH_URL_STR, ",") - ob := strings.Split(AUTH_OAUTH_BEARER_STR, ",") - if len(ou) != len(ob) { - err = errors.New("ERROR: number of items in oauth_urls and oauth_bearers are not the same") - return - } - for i := range ob { - AUTH_OAUTH[ob[i]] = ou[i] - } - OAUTH_DEFAULT = ou[0] // first url is default for "oauth" bearer token - } - // Runtime c_store.AddInt(&EXPIRE_WAIT, 60, "Runtime", "expire_wait", "", "") c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "") @@ -317,5 +303,20 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.Parse() return +} +func parseConfiguration() (err error) { + // parse OAuth settings if used + if AUTH_OAUTH_URL_STR != "" && AUTH_OAUTH_BEARER_STR != "" { + ou := strings.Split(AUTH_OAUTH_URL_STR, ",") + ob := strings.Split(AUTH_OAUTH_BEARER_STR, ",") + if len(ou) != len(ob) { + return errors.New("number of items in oauth_urls and oauth_bearers are not the same") + } + for i := range ob { + AUTH_OAUTH[ob[i]] = ou[i] + } + OAUTH_DEFAULT = ou[0] // first url is default for "oauth" bearer token + } + return } diff --git a/shock-server/main.go b/shock-server/main.go index 40fd0c5d..18b0d6c1 100644 --- a/shock-server/main.go +++ b/shock-server/main.go @@ -183,7 +183,8 @@ func main() { // init config err = conf.Initialize() if err != nil { - fmt.Errorf("Err@db.Initialize: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@conf.Initialize: %s\n", err.Error()) + os.Exit(1) } // init logging system @@ -193,19 +194,16 @@ func main() { if conf.ANON_WRITE { warnstr := "Warning: anonymous write is activated, only use for development !!!!" logger.Info(warnstr) - fmt.Errorf("%s\n", warnstr) } - if conf.ANON_DELETE { warnstr := "Warning: anonymous delete is activated, only use for development !!!!" logger.Info(warnstr) - fmt.Errorf("%s\n", warnstr) } // init database err = db.Initialize() if err != nil { - fmt.Fprintf(os.Stderr, "Err@db.Initialize: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@db.Initialize: %s\n", err.Error()) logger.Error("Err@db.Initialize: " + err.Error()) os.Exit(1) } @@ -217,13 +215,13 @@ func main() { node.InitReaper() err = versions.Initialize() if err != nil { - fmt.Fprintf(os.Stderr, "Err@versions.Initialize: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@versions.Initialize: %s\n", err.Error()) logger.Error("Err@versions.Initialize: " + err.Error()) os.Exit(1) } err = versions.RunVersionUpdates() if err != nil { - fmt.Fprintf(os.Stderr, "Err@versions.RunVersionUpdates: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@versions.RunVersionUpdates: %s\n", err.Error()) logger.Error("Err@versions.RunVersionUpdates: " + err.Error()) os.Exit(1) } @@ -231,25 +229,24 @@ func main() { // Note: configured version numbers are configured in conf.go but are NOT user configurable by design err = versions.PushVersionsToDatabase() if err != nil { - fmt.Fprintf(os.Stderr, "Err@versions.PushVersionsToDatabase: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@versions.PushVersionsToDatabase: %s\n", err.Error()) logger.Error("Err@versions.PushVersionsToDatabase: " + err.Error()) os.Exit(1) } printLogo() conf.Print() if err := versions.Print(); err != nil { - fmt.Fprintf(os.Stderr, "Err@versions.Print: %v\n", err) + fmt.Fprintf(os.Stderr, "Err@versions.Print: %s\n", err.Error()) logger.Error("Err@versions.Print: " + err.Error()) os.Exit(1) } // check if necessary directories exist or created for _, path := range []string{conf.PATH_SITE, conf.PATH_DATA, conf.PATH_LOGS, conf.PATH_DATA + "/temp"} { - err = os.MkdirAll(path, 0777) if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - logger.Errorf("error createing directory %s: %v", err) + fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error()) + logger.Errorf("error createing directory %s: %s", path, err.Error()) os.Exit(1) } @@ -260,7 +257,7 @@ func main() { fmt.Println("####### Reloading #######") err := reload(conf.RELOAD) if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error()) logger.Error("ERROR: " + err.Error()) os.Exit(1) } @@ -343,7 +340,6 @@ func main() { go func() { for _ = range c { // sig is a ^C, handle it - // stop the HTTP server fmt.Fprintln(os.Stderr, "Stopping the server...") listener.Close() From 94647a85e13f6bb6cc5d0e128812e3820bd129cf Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Wed, 19 Jul 2017 14:02:32 -0500 Subject: [PATCH 04/13] fix filename collisions --- shock-server/node/archive/archive.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/shock-server/node/archive/archive.go b/shock-server/node/archive/archive.go index 53e26525..2384c372 100644 --- a/shock-server/node/archive/archive.go +++ b/shock-server/node/archive/archive.go @@ -214,12 +214,20 @@ func ArchiveReader(format string, files []*file.FileInfo) (outReader io.ReadClos if format == "tar" { tWriter := tar.NewWriter(pWriter) go func() { + fileNames := map[string]int{} for _, f := range files { - fHdr := &tar.Header{Name: f.Name, Mode: 0660, ModTime: f.ModTime, Size: f.Size} + fileName := f.Name + if num, ok := fileNames[f.Name]; ok { + fileName = fmt.Sprintf("%s.%d", fileName, num+1) + fileNames[f.Name] = num + 1 + } else { + fileNames[f.Name] = 1 + } + fHdr := &tar.Header{Name: fileName, Mode: 0660, ModTime: f.ModTime, Size: f.Size} tWriter.WriteHeader(fHdr) io.Copy(tWriter, f.Body) if f.Checksum != "" { - cHdr := &tar.Header{Name: f.Name + ".md5", Mode: 0660, ModTime: f.ModTime, Size: int64(len(f.Checksum))} + cHdr := &tar.Header{Name: fileName + ".md5", Mode: 0660, ModTime: f.ModTime, Size: int64(len(f.Checksum))} tWriter.WriteHeader(cHdr) io.Copy(tWriter, bytes.NewBufferString(f.Checksum)) } @@ -230,13 +238,21 @@ func ArchiveReader(format string, files []*file.FileInfo) (outReader io.ReadClos } else if format == "zip" { zWriter := zip.NewWriter(pWriter) go func() { + fileNames := map[string]int{} for _, f := range files { - zHdr := &zip.FileHeader{Name: f.Name, UncompressedSize64: uint64(f.Size)} + fileName := f.Name + if num, ok := fileNames[f.Name]; ok { + fileName = fmt.Sprintf("%s.%d", fileName, num+1) + fileNames[f.Name] = num + 1 + } else { + fileNames[f.Name] = 1 + } + zHdr := &zip.FileHeader{Name: fileName, UncompressedSize64: uint64(f.Size)} zHdr.SetModTime(f.ModTime) zFile, _ := zWriter.CreateHeader(zHdr) io.Copy(zFile, f.Body) if f.Checksum != "" { - cHdr := &zip.FileHeader{Name: f.Name + ".md5", UncompressedSize64: uint64(len(f.Checksum))} + cHdr := &zip.FileHeader{Name: fileName + ".md5", UncompressedSize64: uint64(len(f.Checksum))} cHdr.SetModTime(f.ModTime) zSum, _ := zWriter.CreateHeader(cHdr) io.Copy(zSum, bytes.NewBufferString(f.Checksum)) From 3e8ace070eb2dd92879fa53216f3661c7178e664 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Wed, 19 Jul 2017 14:03:11 -0500 Subject: [PATCH 05/13] enable multi-node archive download based on ID list --- shock-server/controller/node/create.go | 77 ++++++++++++++++++++++++++ shock-server/util/util.go | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/shock-server/controller/node/create.go b/shock-server/controller/node/create.go index 0228904a..d3f668e7 100644 --- a/shock-server/controller/node/create.go +++ b/shock-server/controller/node/create.go @@ -5,11 +5,17 @@ import ( e "github.com/MG-RAST/Shock/shock-server/errors" "github.com/MG-RAST/Shock/shock-server/logger" "github.com/MG-RAST/Shock/shock-server/node" + "github.com/MG-RAST/Shock/shock-server/node/archive" + "github.com/MG-RAST/Shock/shock-server/preauth" "github.com/MG-RAST/Shock/shock-server/request" "github.com/MG-RAST/Shock/shock-server/responder" "github.com/MG-RAST/Shock/shock-server/user" + "github.com/MG-RAST/Shock/shock-server/util" "github.com/MG-RAST/golib/stretchr/goweb/context" + mgo "gopkg.in/mgo.v2" "net/http" + "strings" + "time" ) // POST: /node @@ -29,6 +35,7 @@ func (cr *NodeController) Create(ctx context.Context) error { } // Parse uploaded form + // all POSTed files writen to temp dir params, files, err := request.ParseMultipartForm(ctx.HttpRequest()) // clean up temp dir !! defer node.RemoveAllFormFiles(files) @@ -71,6 +78,76 @@ func (cr *NodeController) Create(ctx context.Context) error { } } + // special case, create preauth download url from list of ids + if _, hasDownloadUrl := params["download_url"]; hasDownloadUrl { + if idStr, hasIds := params["ids"]; hasIds { + idList := strings.Split(idStr, ",") + // validate id list + var nodeIds []string + var totalBytes int64 + for _, id := range idList { + // check if node exists + n, err := node.Load(id) + if err != nil { + if err == mgo.ErrNotFound { + return responder.RespondWithError(ctx, http.StatusNotFound, e.NodeNotFound) + } else { + // In theory the db connection could be lost between + // checking user and load but seems unlikely. + err_msg := "Err@node_Read:LoadNode: " + id + ":" + err.Error() + logger.Error(err_msg) + return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) + } + } + // check ACLs + rights := n.Acl.Check(u.Uuid) + prights := n.Acl.Check("public") + if rights["read"] == false && u.Admin == false && n.Acl.Owner != u.Uuid && prights["read"] == false { + return responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth) + } + if n.HasFile() { + nodeIds = append(nodeIds, n.Id) + totalBytes += n.File.Size + } + } + if len(nodeIds) == 0 { + return responder.RespondWithError(ctx, http.StatusBadRequest, "err:@node_Create download url: no available files found") + } + // add options - set defaults first + options := map[string]string{} + options["archive"] = "zip" // default is zip + if af, ok := params["archive_format"]; ok { + if archive.IsValidToArchive(af) { + options["archive"] = af + } + } + preauthId := util.RandString(20) + if fn, ok := params["file_name"]; ok { + options["filename"] = fn + } else { + options["filename"] = preauthId + } + if !strings.HasSuffix(options["filename"], options["archive"]) { + options["filename"] = options["filename"] + "." + options["archive"] + } + // set preauth + if p, err := preauth.New(preauthId, "download", nodeIds, options); err != nil { + err_msg := "err:@node_Create download_url: " + err.Error() + logger.Error(err_msg) + return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) + } else { + data := preauth.PreAuthResponse{ + Url: util.ApiUrl(ctx) + "/preauth/" + p.Id, + ValidTill: p.ValidTill.Format(time.ANSIC), + Format: options["archive"], + Filename: options["filename"], + Files: len(nodeIds), + Size: totalBytes, + } + return responder.RespondWithData(ctx, data) + } + } + } // special case, creates multiple nodes if archiveId, hasArchiveNode := params["unpack_node"]; hasArchiveNode { ns, err := node.CreateNodesFromArchive(u, params, files, archiveId) diff --git a/shock-server/util/util.go b/shock-server/util/util.go index cd07e1a0..649b9571 100644 --- a/shock-server/util/util.go +++ b/shock-server/util/util.go @@ -15,7 +15,7 @@ const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890" // Arrays to check for valid param and file form names for node creation and updating, and also acl modification. // Note: indexing and querying do not use functions that use these arrays and thus we don't have to include those field names. -var validParams = []string{"action", "all", "archive_format", "attributes_str", "clear_revisions", "copy_attributes", "copy_data", "copy_indexes", "compression", "delete", "expiration", "file_name", "format", "ids", "index_name", "linkage", "operation", "owner", "parent_index", "parent_node", "parts", "path", "preserve_acls", "priority", "read", "remove_expiration", "source", "tags", "type", "unpack_node", "upload_url", "users", "write"} +var validParams = []string{"action", "all", "archive_format", "attributes_str", "clear_revisions", "copy_attributes", "copy_data", "copy_indexes", "compression", "delete", "download_url", "expiration", "file_name", "format", "ids", "index_name", "linkage", "operation", "owner", "parent_index", "parent_node", "parts", "path", "preserve_acls", "priority", "read", "remove_expiration", "source", "tags", "type", "unpack_node", "upload_url", "users", "write"} var validFiles = []string{"attributes", "subset_indices", "upload", "gzip", "bzip2"} var ValidUpload = []string{"upload", "gzip", "bzip2"} From 9c9e73b6c6d5e4a88e9e084cd6bb8f17a787c147 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Wed, 19 Jul 2017 14:03:23 -0500 Subject: [PATCH 06/13] cleanup --- shock-server/controller/node/index/index.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/shock-server/controller/node/index/index.go b/shock-server/controller/node/index/index.go index 696ccf41..4af1c267 100644 --- a/shock-server/controller/node/index/index.go +++ b/shock-server/controller/node/index/index.go @@ -280,10 +280,6 @@ func IndexTypedRequest(ctx context.Context) { CreatedOn: time.Now(), } - //if idxType == "chunkrecord" { - // idxInfo.AvgUnitSize = conf.CHUNK_SIZE - //} - if idxType == "subset" { idxType = subsetName idxInfo.AvgUnitSize = subsetSize / count From c40c4dbcedbb10f99449c3563c475beddbb5268f Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Wed, 19 Jul 2017 15:01:15 -0500 Subject: [PATCH 07/13] fix chunkrecord to find proper record start --- shock-server/node/file/format/fasta/fasta.go | 4 ++-- shock-server/node/file/format/fastq/fastq.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shock-server/node/file/format/fasta/fasta.go b/shock-server/node/file/format/fasta/fasta.go index 6a2332b4..f659a3cb 100644 --- a/shock-server/node/file/format/fasta/fasta.go +++ b/shock-server/node/file/format/fasta/fasta.go @@ -123,11 +123,11 @@ func (self *Reader) SeekChunk(offSet int64) (n int64, err error) { if n, err := r.Read(buf); err != nil { return int64(n), err } - if pos := bytes.LastIndex(buf, []byte(">")); pos == -1 { + if pos := bytes.LastIndex(buf, []byte("\n>")); pos == -1 { indexPos, err := self.SeekChunk(offSet + winSize) return (winSize + indexPos), err } else { - return conf.CHUNK_SIZE - winSize + int64(pos), nil + return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } return } diff --git a/shock-server/node/file/format/fastq/fastq.go b/shock-server/node/file/format/fastq/fastq.go index ba823d1d..3adf4395 100644 --- a/shock-server/node/file/format/fastq/fastq.go +++ b/shock-server/node/file/format/fastq/fastq.go @@ -194,11 +194,11 @@ func (self *Reader) SeekChunk(offSet int64) (n int64, err error) { if n, err := r.Read(buf); err != nil { return int64(n), err } - if pos := bytes.LastIndex(buf, []byte("@")); pos == -1 { + if pos := bytes.LastIndex(buf, []byte("\n@")); pos == -1 { indexPos, err := self.SeekChunk(offSet + winSize) return (winSize + indexPos), err } else { - return conf.CHUNK_SIZE - winSize + int64(pos), nil + return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } return } From 718edbf4e222cbb8af889dac8dafa8573ee7d02a Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Thu, 20 Jul 2017 10:23:18 -0500 Subject: [PATCH 08/13] chunkrecord idx match both /n and /r to be consistant with record idx --- shock-server/node/file/format/fasta/fasta.go | 15 +++++++++++---- shock-server/node/file/format/fastq/fastq.go | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/shock-server/node/file/format/fasta/fasta.go b/shock-server/node/file/format/fasta/fasta.go index f659a3cb..db7c2197 100644 --- a/shock-server/node/file/format/fasta/fasta.go +++ b/shock-server/node/file/format/fasta/fasta.go @@ -121,15 +121,22 @@ func (self *Reader) SeekChunk(offSet int64) (n int64, err error) { r := io.NewSectionReader(self.f, offSet+conf.CHUNK_SIZE-winSize, winSize) buf := make([]byte, winSize) if n, err := r.Read(buf); err != nil { + // EOF reached return int64(n), err } - if pos := bytes.LastIndex(buf, []byte("\n>")); pos == -1 { + // recursivly extend by window size until start of new record found + // try both /n and /r + var pos int + pos = bytes.LastIndex(buf, []byte("\n>")) + if pos == -1 { + pos = bytes.LastIndex(buf, []byte("\r>")) + } + if pos == -1 { indexPos, err := self.SeekChunk(offSet + winSize) return (winSize + indexPos), err - } else { - return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } - return + // done, start new record found + return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } // Rewind the reader. diff --git a/shock-server/node/file/format/fastq/fastq.go b/shock-server/node/file/format/fastq/fastq.go index 3adf4395..b1476013 100644 --- a/shock-server/node/file/format/fastq/fastq.go +++ b/shock-server/node/file/format/fastq/fastq.go @@ -192,15 +192,22 @@ func (self *Reader) SeekChunk(offSet int64) (n int64, err error) { r := io.NewSectionReader(self.f, offSet+conf.CHUNK_SIZE-winSize, winSize) buf := make([]byte, winSize) if n, err := r.Read(buf); err != nil { + // EOF reached return int64(n), err } - if pos := bytes.LastIndex(buf, []byte("\n@")); pos == -1 { + // recursivly extend by window size until start of new record found + // try both /n and /r + var pos int + pos = bytes.LastIndex(buf, []byte("\n@")) + if pos == -1 { + pos = bytes.LastIndex(buf, []byte("\r@")) + } + if pos == -1 { indexPos, err := self.SeekChunk(offSet + winSize) return (winSize + indexPos), err - } else { - return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } - return + // done, start new record found + return conf.CHUNK_SIZE - winSize + int64(pos+1), nil } // Rewind the reader. From 6b2024f17338fe326c6d4599877cea0484f20416 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Fri, 21 Jul 2017 11:23:23 -0500 Subject: [PATCH 09/13] return index object with build request --- shock-server/controller/node/index/index.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shock-server/controller/node/index/index.go b/shock-server/controller/node/index/index.go index 4af1c267..7bc71860 100644 --- a/shock-server/controller/node/index/index.go +++ b/shock-server/controller/node/index/index.go @@ -105,9 +105,9 @@ func IndexTypedRequest(ctx context.Context) { query := ctx.HttpRequest().URL.Query() _, forceRebuild := query["force_rebuild"] - if _, has := n.Indexes[idxType]; has { + if v, has := n.Indexes[idxType]; has { if idxType == "size" { - responder.RespondOK(ctx) + responder.RespondWithData(ctx, map[string]interface{}{idxType: v}) return } else if !forceRebuild { responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.") @@ -308,7 +308,7 @@ func IndexTypedRequest(ctx context.Context) { logger.Perf("END indexing: " + nid) } - responder.RespondOK(ctx) + responder.RespondWithData(ctx, map[string]interface{}{idxType: idxInfo}) default: responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") From 2da824e2f528661315a1d4a63e67d05a04a4f69b Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Fri, 21 Jul 2017 13:41:10 -0500 Subject: [PATCH 10/13] undo index output, awe does not support --- shock-server/controller/node/index/index.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shock-server/controller/node/index/index.go b/shock-server/controller/node/index/index.go index 7bc71860..4af1c267 100644 --- a/shock-server/controller/node/index/index.go +++ b/shock-server/controller/node/index/index.go @@ -105,9 +105,9 @@ func IndexTypedRequest(ctx context.Context) { query := ctx.HttpRequest().URL.Query() _, forceRebuild := query["force_rebuild"] - if v, has := n.Indexes[idxType]; has { + if _, has := n.Indexes[idxType]; has { if idxType == "size" { - responder.RespondWithData(ctx, map[string]interface{}{idxType: v}) + responder.RespondOK(ctx) return } else if !forceRebuild { responder.RespondWithError(ctx, http.StatusBadRequest, "This index already exists, please add the parameter 'force_rebuild=1' to force a rebuild of the existing index.") @@ -308,7 +308,7 @@ func IndexTypedRequest(ctx context.Context) { logger.Perf("END indexing: " + nid) } - responder.RespondWithData(ctx, map[string]interface{}{idxType: idxInfo}) + responder.RespondOK(ctx) default: responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.") From 7fe8b31ae759584aab280ea8f92c93de0b00888e Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Fri, 21 Jul 2017 15:24:59 -0500 Subject: [PATCH 11/13] golang version --- Dockerfile | 2 +- Dockerfile.old | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a159642b..39dc597f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # docker build -t mgrast/shock . # docker run --rm --name test -ti mgrast/shock /bin/ash -FROM golang:1.7.5-alpine +FROM golang:1.7.6-alpine ENV DIR=/go/src/github.com/MG-RAST/Shock WORKDIR /go/bin diff --git a/Dockerfile.old b/Dockerfile.old index b4c829b5..412404f2 100644 --- a/Dockerfile.old +++ b/Dockerfile.old @@ -1,6 +1,6 @@ # creates statically compiled shock-server binary: /go/bin/shock-server -FROM golang:1.7.5-alpine +FROM golang:1.7.6-alpine RUN apk update && apk add git make gcc libc-dev cyrus-sasl-dev From 828da6ce8fbc4bc48a248dfb7e94306c3cdda213 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Fri, 21 Jul 2017 15:34:40 -0500 Subject: [PATCH 12/13] sync defaults in config template and conf.go, update error handling --- shock-server.conf.template | 55 +++++++++++++++++++------------------- shock-server/conf/conf.go | 21 +++++++++++---- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/shock-server.conf.template b/shock-server.conf.template index 442f1e13..94bf81f2 100644 --- a/shock-server.conf.template +++ b/shock-server.conf.template @@ -1,10 +1,3 @@ -[Address] -# IP and port for api -# Note: use of port 80 may require root access -# 0.0.0.0 will bind Shock to all IP's -api-ip=0.0.0.0 -api-port=7445 - [Admin] # Email address is displayed at base URL email=admin@host.com @@ -15,36 +8,52 @@ users= # Controls an anonymous user's ability to read/write # values: true/false read=true -write=false -delete=false +write=true +delete=true -[Auth] -# defaults to local user management with basis auth -#basic=true -# comment line above and uncomment below to use Globus Online as auth provider -globus_token_url=https://nexus.api.globusonline.org/goauth/token?grant_type=client_credentials -globus_profile_url=https://nexus.api.globusonline.org/users -cache_timeout = 60 -#mgrast_oauth_url=https://api.metagenomics.anl.gov/user/authenticate +[Address] +# IP and port for api +# Note: use of port 80 may require root access +# 0.0.0.0 will bind Shock to all IP's +api-ip=0.0.0.0 +api-port=7445 [External] # URL displayed at base URL api-url=http://localhost +[Auth] +# defaults to local user management with basic auth +basic=false +globus_token_url= +globus_profile_url= +oauth_urls= +oauth_bearers= +cache_timeout=60 + +[Runtime] +# wait time in minutes before expiration reaper runs +expire_wait=60 +# golang setting: The GOMAXPROCS variable limits the number of operating system threads that can execute user-level Go code simultaneously. +GOMAXPROCS= +# maximum number of most recent revisions to keep. 0 means keep none, -1 means keep all +max_revisions=3 + [Log] # Logs performance of some Shock operations perf_log=false rotate=true +logoutput=both [Mongodb] # Mongodb configuration # Hostnames and ports hosts=host1[,host2:port,...,hostN] # attribute_indexes defines a list of fields in the Node's attribute struct that will be indexed -hosts=localhost +attribute_indexes= database=ShockDB +hosts=localhost user= password= -attribute_indexes= [Paths] # site directory should contain the documentation files @@ -58,14 +67,6 @@ local_paths= # pidfile should be the path to a file that Shock can use to store the server's process ID pidfile= -[Runtime] -# maximum number of most recent revisions to keep. 0 means keep none, -1 means keep all -max_revisions=3 -# wait time in minutes before expiration reaper runs -expire_wait=60 -# golang setting: The GOMAXPROCS variable limits the number of operating system threads that can execute user-level Go code simultaneously. -GOMAXPROCS= - [SSL] enable=false #key= diff --git a/shock-server/conf/conf.go b/shock-server/conf/conf.go index 4fef62ad..55182c1e 100644 --- a/shock-server/conf/conf.go +++ b/shock-server/conf/conf.go @@ -17,6 +17,8 @@ type idxOpts struct { sparse bool } +var LOG_OUTPUTS = [3]string{"file", "console", "both"} + var ( // Admin ADMIN_EMAIL string @@ -48,7 +50,6 @@ var ( // Config File CONFIG_FILE string - LOG_OUTPUT string // Runtime EXPIRE_WAIT int // wait time for reaper in minutes @@ -58,6 +59,7 @@ var ( // Logs LOG_PERF bool // Indicates whether performance logs should be stored LOG_ROTATE bool // Indicates whether logs should be rotated daily + LOG_OUTPUT string // Mongo information MONGODB_HOSTS string @@ -230,8 +232,10 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "") c_store.AddInt(&MAX_REVISIONS, 3, "Runtime", "max_revisions", "", "") + // Log c_store.AddBool(&LOG_PERF, false, "Log", "perf_log", "", "") c_store.AddBool(&LOG_ROTATE, true, "Log", "rotate", "", "") + c_store.AddString(&LOG_OUTPUT, "both", "Log", "logoutput", "console, file or both", "") // Mongodb c_store.AddString(&MONGODB_ATTRIBUTE_INDEXES, "", "Mongodb", "attribute_indexes", "", "") @@ -285,10 +289,7 @@ func getConfiguration(c *config.Config) (c_store *Config_store, err error) { c_store.AddString(&SSL_CERT, "", "SSL", "cert", "", "") } - // Log - c_store.AddString(&LOG_OUTPUT, "console", "Log", "logoutput", "console, file or both", "") - - //Other + // Other - thses option are CLI only c_store.AddString(&RELOAD, "", "Other", "reload", "path or url to shock data. WARNING this will drop all current data.", "") gopath := os.Getenv("GOPATH") c_store.AddString(&CONFIG_FILE, gopath+"/src/github.com/MG-RAST/Shock/shock-server.conf.template", "Other", "conf", "path to config file", "") @@ -318,5 +319,15 @@ func parseConfiguration() (err error) { } OAUTH_DEFAULT = ou[0] // first url is default for "oauth" bearer token } + // validate LOG_OUTPUT + vaildLogout := false + for _, logout := range LOG_OUTPUTS { + if LOG_OUTPUT == logout { + vaildLogout = true + } + } + if !vaildLogout { + return errors.New("invalid option for logoutput, use one of: file, console, both") + } return } From 91cd02847fc944760a54769044d6338247281040 Mon Sep 17 00:00:00 2001 From: Travis Harrison Date: Fri, 21 Jul 2017 15:34:51 -0500 Subject: [PATCH 13/13] update release notes and version --- RELEASE_NOTES.txt | 9 ++++++++- VERSION | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index f945cdbf..8f9b61d9 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,8 +1,15 @@ +# v0.9.22 +- update to golang 1.7.6 +- added ability to download multiple files (.tar or .zip format) from a list (POST) +- auth update. now support multiple oauth services at once. requires config file update +- update error handling in config init / parse +- bug fix: chunkrecord regex + # v0.9.21 - graceful error handling of missing .bson file - add more to preauth return: file size, options used -- added ability to download multiple files (.tar or .zip format) from a query +- added ability to download multiple files (.tar or .zip format) from a query (GET) # v0.9.20 diff --git a/VERSION b/VERSION index fc6d9f6b..938145fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.21 \ No newline at end of file +0.9.22 \ No newline at end of file