[GOAL2-558] Update S3 Usage (#124)

* Update to support use of Algorand's AWS S3 Service.

* Removed updatekey.json file and references to it.

* Support overriding default upload bucket and region with environment
variables S3_UPLOAD_BUCKET and S3_REGION.

* Added require for S3_UPLOAD_BUCKET environment variable in
create_and_deploy_recipe.sh script.

* Removed redundant s3Helper.go from logging, and consolidated with existing
file in util/s3.

* Fixed lint issue in s3Helper.go

* Check for empty bucket name.
Use consistent naming for bucket name constants.

* Removed redundant updater/s3Helper.go.
Replaced remaining references to S3_UPLOAD_ID and S3_UPLOAD_SECRET with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
Removed temporary recipe geo-test-small from testdata.

* Simplified S3Helper upload and download methods.

* Updated logic for checking for bucket specification which can be through
a flag or through S3_UPLOAD_BUCKET environment variable.

* Added -b bucket flags to nodecfg apply and get commands.

* Added unit test for s3Helper.go

* Updated to handle public/private scenarios

Removed obsolete fileIterator.
Fixed scripts and rationalized usage of S3_UPLOAD_BUCKET
Fixed anonymous credentials
Logging now requires credentials - S3 doesn't allow anonymous MultiPartUploads

* More fixes with more testing

Standardized use of S3_RELEASE_BUCKET for builds; S3_UPLOAD_BUCKET only used for uploading logs.

* Fix build, fix release_packages

* Fix usage of RELEASE_BUCKET

* Fixed prompt_nightly

* Fix s3 uploads test

* Stop uploading files from gen/

* Fixed script syntax error

* Fixed  to support overriding bucket

* Fixed algonet/nodecfg support for new bucket strategy

* addressed PR feedback

* Reviewed release naming and updated

We no longer prefix with "pending_" and instead move from the build bucket to the release bucket.

* Fix syntax error

.travis.yml

@@ -16,9 +16,9 @@ stages:
   - name: build_release
     if: branch =~ /^rel\// AND type != pull_request
   - name: deploy
-    if: branch =~ /^rel\/nightly/ AND type != pull_request
+    if: (branch =~ /^rel\/nightly/ OR branch =~ /^rel\/dev/) AND type != pull_request
   - name: release
-    if: branch =~ /^rel\/nightly/ AND type != pull_request
+    if: (branch != rel/stable AND branch =~ /^rel\//) AND type != pull_request
 
 jobs:
   include:
@@ -83,7 +83,7 @@ addons:
   artifacts:
     s3_region: "us-east-1"
     paths:
-      - $(git ls-files -o | grep -v crypto/libsodium-fork | grep -v crypto/lib/ | tr "\n" ":")
+      - $(git ls-files -o | grep -v crypto/libsodium-fork | grep -v crypto/lib/ | grep -v ^gen/ | tr "\n" ":")
 
 notifications:
   slack:

Makefile

@@ -170,16 +170,14 @@ build-race: build
 		GOBIN=$(GOPATH)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" $(SOURCES) && \
 		GOBIN=$(GOPATH)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -ldflags="$(GOLDFLAGS)" $(SOURCES_RACE)
 
-NONGO_BIN_FILES=$(GOPATH)/bin/find-nodes.sh $(GOPATH)/bin/update.sh $(GOPATH)/bin/updatekey.json $(GOPATH)/bin/COPYING
+NONGO_BIN_FILES=$(GOPATH)/bin/find-nodes.sh $(GOPATH)/bin/update.sh $(GOPATH)/bin/COPYING
 
 NONGO_BIN: $(NONGO_BIN_FILES)
 
 $(GOPATH)/bin/find-nodes.sh: scripts/find-nodes.sh
 
 $(GOPATH)/bin/update.sh: cmd/updater/update.sh
 
-$(GOPATH)/bin/updatekey.json: installer/updatekey.json
-
 $(GOPATH)/bin/COPYING: COPYING
 
 $(GOPATH)/bin/%:

cmd/goal/logging.go

@@ -24,11 +24,13 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/logging"
 )
 
 var (
-	nodeName string
+	nodeName   string
+	logChannel string
 )
 
 func init() {
@@ -38,6 +40,8 @@ func init() {
 
 	// Enable Logging : node name
 	enableCmd.Flags().StringVarP(&nodeName, "name", "n", "", "Friendly-name to use for node")
+
+	loggingSendCmd.Flags().StringVarP(&logChannel, "channel", "c", "", "Release channel for log file source")
 }
 
 var loggingCmd = &cobra.Command{
@@ -115,13 +119,19 @@ var loggingSendCmd = &cobra.Command{
 		}
 		timestamp := time.Now().UTC().Format("20060102150405")
 
+		if logChannel == "" {
+			logChannel = config.GetCurrentVersion().Channel
+		}
+
+		targetFolder := filepath.Join("channel", logChannel)
+
 		modifier := ""
 		counter := uint(1)
 		onDataDirs(func(dataDir string) {
 			dirname := filepath.Base(dataDir)
 			name := basename + cfg.GUID + "_" + dirname + "-" + timestamp + modifier + ".tar.gz"
 
-			for err := range logging.CollectAndUploadData(dataDir, name) {
+			for err := range logging.CollectAndUploadData(dataDir, name, targetFolder) {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 			}
 			modifier = fmt.Sprintf("-%d", counter)

cmd/nodecfg/apply.go

@@ -34,6 +34,7 @@ var applyHostName string
 var applyRootDir string
 var applyRootNodeDir string
 var applyPublicAddress string
+var nodeConfigBucket string
 
 func init() {
 	applyCmd.Flags().StringVarP(&applyChannel, "channel", "c", "", "Channel for the nodes we are configuring")
@@ -47,6 +48,8 @@ func init() {
 	applyCmd.Flags().StringVarP(&applyRootNodeDir, "rootnodedir", "n", "", "The root directory for node directories")
 
 	applyCmd.Flags().StringVarP(&applyPublicAddress, "publicaddress", "a", "", "The public address to use if registering Relay or for Metrics")
+
+	applyCmd.Flags().StringVarP(&nodeConfigBucket, "bucket", "b", "", "S3 bucket to get node configuration from.")
 }
 
 var applyCmd = &cobra.Command{
@@ -104,7 +107,7 @@ func doApply(rootDir string, rootNodeDir, channel string, hostName string, dnsNa
 		}
 		defer os.RemoveAll(rootDir)
 
-		if err = downloadAndExtractConfigPackage(channel, rootDir); err != nil {
+		if err = downloadAndExtractConfigPackage(channel, rootDir, nodeConfigBucket); err != nil {
 			return err
 		}
 	}

cmd/nodecfg/download.go

@@ -19,15 +19,16 @@ package main
 import (
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 
 	"github.com/algorand/go-algorand/util/s3"
 	"github.com/algorand/go-algorand/util/tar"
 )
 
-func downloadAndExtractConfigPackage(channel string, targetDir string) (err error) {
+func downloadAndExtractConfigPackage(channel string, targetDir string, configBucket string) (err error) {
 	fmt.Fprintf(os.Stdout, "Downloading latest configuration file for '%s'...\n", channel)
-	packageFile, err := downloadConfigPackage(channel, targetDir)
+	packageFile, err := downloadConfigPackage(channel, targetDir, configBucket)
 	if err != nil {
 		return fmt.Errorf("error downloading config package for channel '%s': %v", channel, err)
 	}
@@ -37,14 +38,18 @@ func downloadAndExtractConfigPackage(channel string, targetDir string) (err erro
 	return extractConfigPackage(packageFile, targetDir)
 }
 
-func downloadConfigPackage(channelName string, targetDir string) (packageFile string, err error) {
-	s3, err := s3.MakeS3SessionForDownload()
+func downloadConfigPackage(channelName string, targetDir string, configBucket string) (packageFile string, err error) {
+	if configBucket == "" {
+		configBucket = s3.GetS3ReleaseBucket()
+	}
+	fmt.Fprintf(os.Stdout, "Downloading configuration package for channel '%s' from bucket '%s'\n", channelName, configBucket)
+	s3Session, err := s3.MakeS3SessionForDownloadWithBucket(configBucket)
 	if err != nil {
 		return
 	}
 
 	prefix := fmt.Sprintf("config_%s", channelName)
-	version, name, err := s3.GetLatestVersion(prefix)
+	version, name, err := s3Session.GetLatestPackageFilesVersion(channelName, prefix)
 	if err != nil {
 		return
 	}
@@ -53,14 +58,15 @@ func downloadConfigPackage(channelName string, targetDir string) (packageFile st
 		return
 	}
 
-	packageFile = filepath.Join(targetDir, name)
+	fileName := path.Base(name)
+	packageFile = filepath.Join(targetDir, fileName)
 	file, err := os.Create(packageFile)
 	if err != nil {
 		return
 	}
 	defer file.Close()
 
-	if err = s3.DownloadFile(name, file); err != nil {
+	if err = s3Session.DownloadFile(name, file); err != nil {
 		err = fmt.Errorf("error downloading file: %v", err)
 		return
 	}

cmd/nodecfg/get.go

@@ -29,12 +29,15 @@ import (
 
 var getChannel string
 var getRootDir string
+var configBucket string
 
 func init() {
 	getCmd.Flags().StringVarP(&getChannel, "channel", "c", "", "Channel for the nodes we are configuring")
 	getCmd.MarkFlagRequired("channel")
 
 	getCmd.Flags().StringVarP(&getRootDir, "rootdir", "r", "", "The rootdir containing the node configuration files")
+
+	getCmd.Flags().StringVarP(&configBucket, "bucket", "b", "", "S3 bucket to get configuration from.")
 }
 
 var getCmd = &cobra.Command{
@@ -68,5 +71,5 @@ func doGet(channel, rootDir string) (err error) {
 	if err = os.Mkdir(rootDir, 0700); err != nil {
 		return
 	}
-	return downloadAndExtractConfigPackage(channel, rootDir)
+	return downloadAndExtractConfigPackage(channel, rootDir, configBucket)
 }