Merge branch 'master' into feature/checkpoint-state-to-s3

.circleci/config.yml

@@ -7,6 +7,7 @@ jobs:
         environment:
           GOPATH: /go
           MONGO_TEST_URL: localhost:27017
+          ETCD_TEST_URL: http://localhost:2379
       - image: mongo:3.0.2
       - image: quay.io/coreos/etcd:v2.3.7
     steps:

README.md

@@ -17,6 +17,28 @@ govendor sync
 
 before building and running the project locally.
 
+To test the project, use:
+
+```
+govendor test -v -race +local
+```
+
+There are MongoDB and Etcd integration tests, which require local running instances of MongoDB and Etcd. These can be skipped (along with long running tests) by using the command:
+
+```
+govendor test -v -race -short +local
+```
+
+To connect to a MongoDB instance, please use the environment variable `MONGO_TEST_URL` i.e. `export MONGO_TEST_URL=localhost:27017`. To connect to an Etcd instance, please use the environment variable `ETCD_TEST_URL` i.e. `export ETCD_TEST_URL=http://localhost:2379`.
+
+For running the Carousel locally, please see the command line arguments that need to be set using:
+
+```
+./publish-carousel --help
+```
+
+Please note that you must connect to the **Primary** Mongo instance if you are connecting to one of our UPP clusters.
+
 # Code Structure
 
 ## Packages

blacklist/test_blacklist.txt

@@ -1,3 +1,13 @@
+--SYNTHETIC REQUEST UUIDS
+--pre-prod-us
+fbe8b93a-67f0-47e0-b55c-f7b811284b47
+f8aeb712-b008-468b-bc30-a5ddd92d2d5f
+614a9bf2-4639-408b-8493-635dd996224e
+--dynpub
+39c7a6a5-e820-433c-b5aa-48d62405f033
+806a34c0-0438-4a5f-bcb1-da7d8a27b9bf
+342473aa-fbdb-4120-b66f-59e1034aa363
+--END SYNTHETIC REQUEST IDS
 2565e15c-6fd3-11de-b835-00144feabdc0
 233152d0-8055-11de-bf04-00144feabdc0
 626d75fa-92c0-11de-b63b-00144feabdc0

carousel_blacklist.txt

@@ -1,3 +1,33 @@
+--SYNTHETIC REQUEST UUIDS
+--prod-uk
+38bb47a2-80e6-11e6-a078-d6ccddbc3d35
+4f2f97ea-b8ec-11e4-b8e6-00144feab7de
+c94a3a57-3c99-423c-a6bd-ed8c4c10a3c3
+--prod-us
+6c984298-12bc-11e6-bb40-c30e3bfcf63b
+6442247f-6e94-4cbd-aaca-906158bae38c
+43eabb23-88a2-4e62-a4dc-6e1e8f737c1f
+--pre-prod uk
+2ca63ad2-63b8-49a0-ab01-4f6a80de6ec0
+85a30235-2a52-485d-a492-6a59055a453f
+da41b1f8-289b-47c1-a640-e1cca7afa811
+--pre-prod-us
+fbe8b93a-67f0-47e0-b55c-f7b811284b47
+f8aeb712-b008-468b-bc30-a5ddd92d2d5f
+614a9bf2-4639-408b-8493-635dd996224e
+--dynpub
+39c7a6a5-e820-433c-b5aa-48d62405f033
+806a34c0-0438-4a5f-bcb1-da7d8a27b9bf
+342473aa-fbdb-4120-b66f-59e1034aa363
+--semantic
+dd2615a8-a424-48fe-a024-c19312465021
+a648c244-b3af-4391-9e11-f9e234bcb591
+a6b4c15b-dfbe-48e0-a42f-c80df24fb280
+--xp
+70ff2011-db31-4808-ad5b-102773034c67
+3abe5917-3da9-49e1-acc7-7ea77bc22841
+fa525e88-ab9d-457d-b36e-4723445f3bde
+--END SYNTHETIC REQUEST IDS
 2565e15c-6fd3-11de-b835-00144feabdc0
 233152d0-8055-11de-bf04-00144feabdc0
 626d75fa-92c0-11de-b63b-00144feabdc0

cms/cms_notifier.go

@@ -50,7 +50,7 @@ func (c *cmsNotifier) Notify(origin string, tid string, content *native.Content,
 	req.Header.Add("X-Native-Hash", hash)
 	req.Header.Add("X-Origin-System-Id", origin)
 
-	log.WithField("transaction_id", tid).WithField("nativeHash", hash).Info("Calling CMS notifier...")
+	log.WithField("transaction_id", tid).WithField("nativeHash", hash).Info("Calling CMS notifier.")
 
 	if err != nil {
 		return err

etcd/watcher_test.go

@@ -2,19 +2,30 @@ package etcd
 
 import (
 	"context"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
 	etcdClient "github.com/coreos/etcd/client"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestWatch(t *testing.T) {
+func startEtcd(t *testing.T) (Watcher, error) {
 	if testing.Short() {
 		t.Skip("Skipping etcd integration test")
 	}
 
-	watcher, err := NewEtcdWatcher([]string{"http://localhost:2379"})
+	etcdURL := os.Getenv("ETCD_TEST_URL")
+	if strings.TrimSpace(etcdURL) == "" {
+		t.Fatal("Please set the environment variable ETCD_TEST_URL to run etcd integration tests (e.g. ETCD_TEST_URL=http://localhost:2379). Alternatively, run tests with `-short` to skip them.")
+	}
+
+	return NewEtcdWatcher([]string{etcdURL})
+}
+
+func TestWatch(t *testing.T) {
+	watcher, err := startEtcd(t)
 	assert.NoError(t, err)
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*4)
@@ -39,11 +50,7 @@ func TestWatch(t *testing.T) {
 }
 
 func TestWatchCallbackPanics(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping etcd integration test")
-	}
-
-	watcher, err := NewEtcdWatcher([]string{"http://localhost:2379"})
+	watcher, err := startEtcd(t)
 	assert.NoError(t, err)
 
 	cfg := etcdClient.Config{

native/mocks.go

@@ -51,9 +51,9 @@ type MockReader struct {
 	mock.Mock
 }
 
-func (m *MockReader) Get(collection string, uuid string) (*Content, string, error) {
+func (m *MockReader) Get(collection string, uuid string) (*Content, error) {
 	args := m.Called(collection, uuid)
-	return args.Get(0).(*Content), args.String(1), args.Error(2)
+	return args.Get(0).(*Content), args.Error(1)
 }
 
 type MockDBIter struct {

native/native_reader.go

@@ -1,9 +1,7 @@
 package native
 
-import "encoding/json"
-
 type Reader interface {
-	Get(collection string, uuid string) (*Content, string, error)
+	Get(collection string, uuid string) (*Content, error)
 }
 
 type MongoReader struct {
@@ -14,29 +12,19 @@ func NewMongoNativeReader(mongo DB) Reader {
 	return &MongoReader{mongo}
 }
 
-func (m *MongoReader) Get(collection string, uuid string) (*Content, string, error) {
+func (m *MongoReader) Get(collection string, uuid string) (*Content, error) {
 	tx, err := m.mongo.Open()
 
 	if err != nil {
-		return nil, "", err
+		return nil, err
 	}
 
 	defer tx.Close()
 
 	content, err := tx.ReadNativeContent(collection, uuid)
 	if err != nil {
-		return nil, "", err
-	}
-
-	data, err := json.Marshal(content.Body)
-	if err != nil {
-		return nil, "", err
-	}
-
-	hash, err := Hash(data)
-	if err != nil {
-		return nil, "", err
+		return nil, err
 	}
 
-	return content, hash, nil
+	return content, nil
 }

native/native_reader_test.go

@@ -22,13 +22,12 @@ func TestNativeReaderGet(t *testing.T) {
 	mockTx.On("Close")
 	mockTx.On("ReadNativeContent", testCollection, testUUID).Return(&testContent, nil)
 
-	actual, hash, err := reader.Get(testCollection, testUUID)
+	actual, err := reader.Get(testCollection, testUUID)
 	assert.NoError(t, err)
 
 	mockDb.AssertExpectations(t)
 	mockTx.AssertExpectations(t)
 
-	assert.Equal(t, "5cdd15a873608087be07a41b7f1a04e96d3a66fe7a9b0faac71f8d05", hash)
 	assert.Equal(t, "application/vnd.expect-this", actual.ContentType)
 }
 
@@ -43,7 +42,7 @@ func TestNativeReaderMongoOpenFails(t *testing.T) {
 
 	mockDb.On("Open").Return(mockTx, errors.New("mongo broke mate"))
 
-	_, _, err := reader.Get(testCollection, testUUID)
+	_, err := reader.Get(testCollection, testUUID)
 	assert.Error(t, err)
 
 	mockDb.AssertExpectations(t)
@@ -63,31 +62,7 @@ func TestNativeReaderMongoReadFails(t *testing.T) {
 	mockTx.On("Close")
 	mockTx.On("ReadNativeContent", testCollection, testUUID).Return(&Content{}, errors.New("couldn't find it for ya"))
 
-	_, _, err := reader.Get(testCollection, testUUID)
-	assert.Error(t, err)
-
-	mockDb.AssertExpectations(t)
-	mockTx.AssertExpectations(t)
-}
-
-func TestNativeReaderJSONMarshalFails(t *testing.T) {
-	mockDb := new(MockDB)
-	mockTx := new(MockTX)
-
-	testCollection := "testing-123"
-	testUUID := "fake-uuid"
-
-	testBody := make(map[string]interface{})
-	testBody["errrr"] = func() {}
-	testContent := Content{Body: testBody, ContentType: "application/vnd.expect-this"}
-
-	reader := NewMongoNativeReader(mockDb)
-
-	mockDb.On("Open").Return(mockTx, nil)
-	mockTx.On("Close")
-	mockTx.On("ReadNativeContent", testCollection, testUUID).Return(&testContent, nil)
-
-	_, _, err := reader.Get(testCollection, testUUID)
+	_, err := reader.Get(testCollection, testUUID)
 	assert.Error(t, err)
 
 	mockDb.AssertExpectations(t)

resources/healthchecks.go

@@ -98,14 +98,24 @@ func pingMongo(db native.DB) func() (string, error) {
 		}
 
 		defer tx.Close()
+		err = tx.Ping()
 
-		return "", tx.Ping()
+		if err != nil {
+			return "", err
+		}
+
+		return "OK", nil
 	}
 }
 
 func pingS3(svc s3.ReadWriter) func() (string, error) {
 	return func() (string, error) {
-		return "", svc.Ping()
+		err := svc.Ping()
+		if err != nil {
+			return "", err
+		}
+
+		return "OK", nil
 	}
 }
 
@@ -124,13 +134,18 @@ func unhealthyCycles(sched scheduler.Scheduler) func() (string, error) {
 			return "", errors.New("The following cycles are unhealthy! " + toJSON(unhealthyIDs))
 		}
 
-		return "", nil
+		return "No unhealthy cycles.", nil
 	}
 }
 
 func cmsNotifierGTG(notifier cms.Notifier) func() (string, error) {
 	return func() (string, error) {
-		return "", notifier.GTG()
+		err := notifier.GTG()
+		if err != nil {
+			return "", err
+		}
+
+		return "OK", nil
 	}
 }
 
@@ -168,6 +183,10 @@ func unhealthyClusters(sched scheduler.Scheduler, upServices ...cluster.Service)
 
 func configHealthcheck(err error) func() (string, error) {
 	return func() (string, error) {
-		return "", err
+		if err != nil {
+			return "", err
+		}
+
+		return "OK", nil
 	}
 }