S3 sync works

.gitignore

@@ -1,5 +1,6 @@
 foobar
 scarr
-
+.DS_STore
 scarr.yml
-.vscode
+.vscode
+test

src/acm.go

@@ -6,6 +6,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/acm"
 	"os"
+	"time"
 )
 
 func amcService() *acm.ACM {
@@ -22,7 +23,6 @@ func getAcmCertificateARN(domain string) *string {
 
 	for _, certSummary := range listResult.CertificateSummaryList {
 		if *certSummary.DomainName == domain {
-			fmt.Println("Found cert:", certSummary)
 			return certSummary.CertificateArn
 		}
 	}
@@ -38,32 +38,64 @@ func createACMCertificate(domain string) *string {
 		ValidationMethod:        aws.String("DNS"),
 	})
 	dieOnError(err, "Failed to request ACM certificate")
-	fmt.Println("requestResult=", requestResult)
 	setACMDNS(*requestResult.CertificateArn, domain)
 
 	// TODO: wait until the acm cert lists as validated (might need to re-trigger something?)
 	return requestResult.CertificateArn
 }
 
-func setACMDNS(certificateARN string, domain string) {
+func getCertificateValidation(certificateARN string) *acm.DomainValidation {
 	service := amcService()
 	describeResult, err := service.DescribeCertificate(&acm.DescribeCertificateInput{
 		CertificateArn: &certificateARN,
 	})
 	dieOnError(err, "Failed to describe ACM certificate")
-	fmt.Println("Got detailed cert info")
-	domainValidation := describeResult.Certificate.DomainValidationOptions[0]
-	// TODO: getting an error here on first run.  Apparently domainValidation.ValidationStatus is nil?
-	// Not sure what I should check at this point then.
-	if *domainValidation.ValidationStatus == "PENDING_VALIDATION" {
-		fmt.Println("One of the cert's validation thingys are still waiting on dns.  Creating...")
+	return describeResult.Certificate.DomainValidationOptions[0]
+}
+
+func setACMDNS(certificateARN string, domain string) {
+
+	var domainValidation *acm.DomainValidation
+	// Right after certificate creation, validation status seems to be nil.  Wait a bit.
+	for i := 0; i < 5; i++ {
+		domainValidation := getCertificateValidation(certificateARN)
+		if domainValidation.ValidationStatus != nil {
+			break
+		} else {
+			time.Sleep(5 * time.Second)
+		}
+	}
+	// TODO: read up on go memory management so I don't have to do this again here to avoid segfaults
+	domainValidation = getCertificateValidation(certificateARN)
+
+	if *(domainValidation.ValidationStatus) == "PENDING_VALIDATION" {
+		fmt.Print("not yet valid; creating validation dns records...")
 		dns := domainValidation.ResourceRecord
-		createDNSRecord(domain, *dns.Name, *dns.Type, *dns.Value)
+		// If the dns record already exists, we're just waiting for validation so don't try to recreate it.
+		if !dnsRecordExists(getHostedZone(domain), *dns.Name, *dns.Type) {
+			createDNSRecord(domain, *dns.Name, *dns.Type, dns.Value, nil)
+		}
+
+		fmt.Print("waiting for validation (takes up to a few hours - feel free to ctrl-c and restart scarr later)...")
+		time.Sleep(5 * time.Second)
+
+		maxTries := 60 * 3
+		for i := 0; i < maxTries; i++ {
+			if *getCertificateValidation(certificateARN).ValidationStatus != "PENDING_VALIDATION" {
+				setACMDNS(certificateARN, domain)
+				break
+			}
+			if i == (maxTries - 1) {
+				fmt.Println("\nTimed out waiting for ACM certificate to validate.")
+				os.Exit(1)
+			}
+			time.Sleep(60 * time.Second)
+		}
 	} else if *domainValidation.ValidationStatus == "FAILED" {
 		fmt.Println("Err!  Cert validation failed!")
 		os.Exit(1)
 	} else {
-		fmt.Println("Certificate already validated")
+		fmt.Print("Certificate already validated")
 	}
 
 }

src/cloudfront.go

@@ -15,7 +15,9 @@ func cloudFrontService() *cloudfront.CloudFront {
 	return cloudfront.New(sess)
 }
 
-func cloudFrontExists(s3Url string) bool {
+// Returns cloudfrontDomain, distId
+func getCloudfront(s3Bucket string) (*string, *string) {
+	s3Domain := s3Bucket + ".s3.amazonaws.com"
 	service := cloudFrontService()
 	result, err := service.ListDistributions(&cloudfront.ListDistributionsInput{})
 	dieOnError(err, "Failed getting distribution list")
@@ -27,26 +29,24 @@ func cloudFrontExists(s3Url string) bool {
 
 	for _, dist := range result.DistributionList.Items {
 		for _, origin := range dist.Origins.Items {
-			if *origin.DomainName == s3Url {
+			if *origin.DomainName == s3Domain {
 				// s3Url looks like:
 				// voyage-found.s3-website-us-west-1.amazonaws.com
-				return true
+				return dist.DomainName, dist.Id
 			}
 		}
 	}
-	return false
+	return nil, nil
 }
 
-func createCloudFront(s3Url string, bucketName string, certificateArn string, domain string) {
+func createCloudFront(s3Url string, bucketName string, certificateArn string, domain string) *string {
 	fmt.Println("Creating cloudfront")
-	fmt.Println("s3Url=", s3Url)
-	fmt.Println("bucketName=", bucketName)
-	fmt.Println("certificateArn=", certificateArn)
-	fmt.Println("domain=", domain)
+
 	// Taking a break from this function to go set up ACM, since we'll need that ID
 	service := cloudFrontService()
 	originID := "S3-" + bucketName
 	// s3DomainName := bucketName + ".s3.amazonaws.com"
+	s3Domain := bucketName + ".s3.amazonaws.com"
 
 	aliases := cloudfront.Aliases{
 		Items:    aws.StringSlice([]string{domain}),
@@ -78,13 +78,7 @@ func createCloudFront(s3Url string, bucketName string, certificateArn string, do
 		ViewerProtocolPolicy: aws.String("redirect-to-https"),
 	}
 
-	s3Domain := bucketName + ".s3.amazonaws.com"
-
-	fmt.Println("s3Domain=", s3Domain)
 	origin := cloudfront.Origin{
-		// S3OriginConfig: &cloudfront.S3OriginConfig{
-		// 	OriginAccessIdentity: aws.String(""),
-		// },
 		CustomOriginConfig: &cloudfront.CustomOriginConfig{
 			HTTPPort:             aws.Int64(80),
 			HTTPSPort:            aws.Int64(443),
@@ -135,9 +129,35 @@ func createCloudFront(s3Url string, bucketName string, certificateArn string, do
 
 	dieOnError(err, "Failed to create cloudfront distribution")
 
-	fmt.Println("Waiting for distribution to finish...")
+	fmt.Print("Waiting for distribution to finish (20-40 minutes)...")
 	service.WaitUntilDistributionDeployed(&cloudfront.GetDistributionInput{
 		Id: createResult.Distribution.Id,
 	})
-	fmt.Println("Distribution finished!")
+	fmt.Println(" Done")
+	return createResult.Distribution.DomainName
+}
+
+func createCloudfrontInvalidation(s3Bucket string, paths []string) {
+	_, distributionID := getCloudfront(s3Bucket)
+	service := cloudFrontService()
+	callerReference := time.Now().Format(time.RFC850)
+	fmt.Print("Invalidating cache...")
+	invalidationResult, err := service.CreateInvalidation(&cloudfront.CreateInvalidationInput{
+		DistributionId: distributionID,
+		InvalidationBatch: &cloudfront.InvalidationBatch{
+			CallerReference: &callerReference,
+			Paths: &cloudfront.Paths{
+				Items:    aws.StringSlice(paths),
+				Quantity: aws.Int64(int64(len(paths))),
+			},
+		},
+	})
+	dieOnError(err, "Failed to create Invalidation")
+
+	fmt.Print("waiting (5-10 minutes)...")
+	service.WaitUntilInvalidationCompleted(&cloudfront.GetInvalidationInput{
+		DistributionId: distributionID,
+		Id:             invalidationResult.Invalidation.Id,
+	})
+	fmt.Println(" done")
 }

src/deploy.go

@@ -30,6 +30,7 @@ type configType struct {
 	Name          string             `yaml:"name"`
 	Region        string             `yaml:"region"`
 	DomainContact contactDetailsType `yaml:"domainContact"`
+	Exclude       []string           `yaml:"exclude"`
 }
 
 func dieOnError(err error, message string) {
@@ -64,10 +65,11 @@ func getConfig() configType {
 }
 
 func ensureDomainRegistered(config configType) {
+	fmt.Printf("Checking domain %v registration...", config.Domain)
 
 	domainDetail := getDomainDetails(config.Domain)
 	if domainDetail == nil {
-		fmt.Println("Couldn't find domain '" + config.Domain + "' in your route53 account.")
+		fmt.Println("\nNot registered in our Route53")
 
 		// Not clear if there's a good way to detect this
 		// if isRegistering(config.Domain) {
@@ -82,7 +84,7 @@ But it *is* available to register.  For current prices, see the document linked
 https://aws.amazon.com/route53/pricing/
 				`)
 			if strings.HasSuffix(config.Domain, ".com") {
-				fmt.Println("But as of April 2018, .com TLDs were $12/yr")
+				fmt.Println("(As of April 2018, .com TLDs were $12/yr)")
 			}
 			if confirm("Register that domain?") {
 				registerDomain(config.Domain, config.DomainContact)
@@ -98,57 +100,77 @@ this (you'll have to manage your own domain + dns setup then)
 			os.Exit(1)
 		}
 	} else {
-		fmt.Println("Domain exists!")
-		//fmt.Println("domainDetail=", domainDetail)
+		fmt.Println("Looks good!")
 	}
 }
 func ensureS3BucketExists(s3BucketName string, region string) {
+	fmt.Printf("Checking bucket %v...", s3BucketName)
 	if !bucketExists(s3BucketName, region) {
-		fmt.Println("S3 Bucket doesn't exist - creating it.")
+		fmt.Print(" bucket doesn't exist; creating it now...")
 		createBucket(s3BucketName, region)
 	} else {
-		fmt.Println("Bucket exists")
+		fmt.Print(" bucket already exists.")
 	}
 
 	if !bucketIsWorldReadable(s3BucketName, region) {
 		// We could _make_ this bucket world-readable, but that'd be bad if it turns out to have sensitive info in it.
-		fmt.Println("Bucket is not world-readable.  You should fix this (or delete the bucket and let us re-create it).")
+		fmt.Println("\nBucket is not world-readable.  You should fix this (or delete the bucket and let us re-create it).")
 		os.Exit(1)
 	}
+	fmt.Println(" Done")
 	ensureBucketIsWebsite(s3BucketName, region)
 }
 
 func ensureACMCertificate(domain string) string {
+	fmt.Printf("Checking ACM cert for %v...", domain)
 	certificateArn := getAcmCertificateARN(domain)
 	if certificateArn == nil {
-		fmt.Println("ACM certificate for this domain doesn't exist - creating one.")
+		fmt.Print("doesn't exist; creating...")
 		certificateArn = createACMCertificate(domain)
 	} else {
 		// Ensure it's DNS is set up
-		fmt.Println("Certificate already exists - ensuring it's validated")
+		fmt.Print("already exists; ensuring it's validated...")
 		setACMDNS(*certificateArn, domain)
 	}
+	fmt.Println("done.")
 	return *certificateArn
 }
-func ensureCloudFrontExists(certificateArn string, s3Url string, s3Bucket string, domain string) {
-	if !cloudFrontExists(s3Url) {
+func ensureCloudFrontExists(certificateArn string, s3Url string, s3Bucket string, domain string) string {
+	cloudfrontDomain, _ := getCloudfront(s3Bucket)
+	if cloudfrontDomain == nil {
 		fmt.Println("CloudFront distribution does not exist; creating")
-		createCloudFront(s3Url, s3Bucket, certificateArn, domain)
+		cloudfrontDomain = createCloudFront(s3Url, s3Bucket, certificateArn, domain)
 	}
+	return *cloudfrontDomain
+}
+func ensureDomainPointingToCloudfront(cloudfrontDomain string, mainDomain string) {
+	hostedZoneID := getHostedZone(mainDomain)
+	if dnsRecordExists(hostedZoneID, mainDomain, "A") {
+		fmt.Println("Domain has a (hopefully-correct) alias already configured")
+	} else {
+		fmt.Println("Creating A-record alias to domain")
+		createAliasRecord(mainDomain, mainDomain, cloudfrontDomain)
+	}
+
+	// TODO: set up an alias or redirect from www to apex
+}
+
+func invalidateCloudfront(s3Bucket string, pathsToInvalidate []string) {
+	// TODO: actually invalidate what's passed in
+	createCloudfrontInvalidation(s3Bucket, []string{"/*"})
 }
 
 func runDeploy() {
 	config := getConfig()
 	s3Bucket := config.Name + "-bucket"
 	s3Url := s3Bucket + ".s3-website-" + config.Region + ".amazonaws.com"
-	fmt.Println("s3Url=", s3Url)
 
 	ensureDomainRegistered(config)
 	certArn := ensureACMCertificate(config.Domain)
 	ensureS3BucketExists(s3Bucket, config.Region)
-	ensureCloudFrontExists(certArn, s3Url, s3Bucket, config.Domain)
-
-	//changed_files = sync_to_s3(s3Bucket)
-	//invalidate(changed_files)
+	cloudfrontDomain := ensureCloudFrontExists(certArn, s3Url, s3Bucket, config.Domain)
+	ensureDomainPointingToCloudfront(cloudfrontDomain, config.Domain)
 
+	changedFiles := s3Sync(config.Region, s3Bucket, &config.Exclude)
+	invalidateCloudfront(s3Bucket, changedFiles)
 }

src/init.go

@@ -28,6 +28,10 @@ domainContact:
   phoneNumber: 'fillmein'
   state: 'fillmein'
   zipCode: 'fillmein'
+
+# A list of regexes to be run against paths in the current directory.  Any file path matching any of these regexes will not be synced to s3
+exclude:
+  - "scarr\\.yml"
 `
 
 func generateConfig(domain string, name string, region string) string {