Merging master into feature/cleanup-concorded-concepts

health/healthcheck_test.go

@@ -349,6 +349,9 @@ func (m *AuthorServiceMock) IsGTG() error {
 	return args.Error(0)
 }
 
+func (m *AuthorServiceMock) RefreshAuthorIdentifiers() {
+}
+
 func parseHealthcheck(healthcheckJSON string) ([]fthealth.CheckResult, error) {
 	result := &struct {
 		Checks []fthealth.CheckResult `json:"checks"`

main.go

@@ -99,11 +99,18 @@ func main() {
 
 	pubClusterCredKey := app.String(cli.StringOpt{
 		Name:   "publish-cluster-credentials",
-		Value:  "",
+		Value:  "dummyUser:dummyValue",
 		Desc:   "The ETCD key value that specifies the credentials for connection to the publish cluster in the form user:pass",
 		EnvVar: "PUBLISH_CLUSTER_CREDENTIALS",
 	})
 
+	authorRefreshInterval := app.Int(cli.IntOpt{
+		Name:   "author-refresh-interval",
+		Value:  60,
+		Desc:   "the time interval between author identefier list refreshes in minutes",
+		EnvVar: "AUTHOR_REFRESH_INTERVAL",
+	})
+
 	accessConfig := service.NewAccessConfig(*accessKey, *secretKey, *esEndpoint)
 
 	log.SetLevel(log.InfoLevel)
@@ -121,10 +128,9 @@ func main() {
 					log.Infof("connected to ElasticSearch")
 					ecc <- ec
 					return
-				} else {
-					log.Errorf("could not connect to ElasticSearch: %s", err.Error())
-					time.Sleep(time.Minute)
 				}
+				log.Errorf("could not connect to ElasticSearch: %s", err.Error())
+				time.Sleep(time.Minute)
 			}
 		}()
 
@@ -134,15 +140,14 @@ func main() {
 		esService := service.NewEsService(ecc, *indexName, &bulkProcessorConfig)
 
 		allowedConceptTypes := strings.Split(*elasticsearchWhitelistedConceptTypes, ",")
-		authorService, err := service.NewAuthorService(*pubClusterReadURL, *pubClusterCredKey, &http.Client{Timeout: time.Second * 30})
+		authorService, err := service.NewAuthorService(*pubClusterReadURL, *pubClusterCredKey, time.Duration(*authorRefreshInterval)*time.Minute, &http.Client{Timeout: time.Second * 30})
 		if err != nil {
 			log.Errorf("Could not retrieve author list, error=[%s]\n", err)
-			//TODO we need to stop writing until we have authors
-			return
+			os.Exit(1)
 		}
-
 		handler := resources.NewHandler(esService, authorService, allowedConceptTypes)
 		defer handler.Close()
+		authorService.RefreshAuthorIdentifiers()
 
 		//create health service
 		healthService := health.NewHealthService(esService, authorService)

resources/handler_test.go

@@ -437,6 +437,10 @@ func (service *dummyAuthorService) LoadAuthorIdentifiers() error {
 	return nil
 }
 
+func (service *dummyAuthorService) RefreshAuthorIdentifiers() {
+
+}
+
 func (service *dummyAuthorService) IsFTAuthor(UUID string) bool {
 	return service.isAuthor
 }

service/author_service.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
+	"time"
 
 	transactionid "github.com/Financial-Times/transactionid-utils-go"
 	log "github.com/Sirupsen/logrus"
@@ -21,6 +23,7 @@ type AuthorUUID struct {
 
 type AuthorService interface {
 	LoadAuthorIdentifiers() error
+	RefreshAuthorIdentifiers()
 	IsFTAuthor(UUID string) bool
 	IsGTG() error
 }
@@ -30,16 +33,18 @@ type curatedAuthorService struct {
 	httpClient             *http.Client
 	serviceURL             string
 	authorUUIDs            map[string]struct{}
+	authorRefreshInterval  time.Duration
+	authorLock             *sync.RWMutex
 	publishClusterUser     string
 	publishClusterpassword string
 }
 
-func NewAuthorService(serviceURL string, pubClusterKey string, client *http.Client) (AuthorService, error) {
+func NewAuthorService(serviceURL string, pubClusterKey string, authorRefreshInterval time.Duration, client *http.Client) (AuthorService, error) {
 	if len(pubClusterKey) == 0 {
 		return nil, fmt.Errorf("credentials missing credentials, author service cannot make request to author transformer")
 	}
 	credentials := strings.Split(pubClusterKey, ":")
-	cas := &curatedAuthorService{client, serviceURL, nil, credentials[0], credentials[1]}
+	cas := &curatedAuthorService{client, serviceURL, nil, authorRefreshInterval, &sync.RWMutex{}, credentials[0], credentials[1]}
 	return cas, cas.LoadAuthorIdentifiers()
 }
 
@@ -65,7 +70,7 @@ func (as *curatedAuthorService) LoadAuthorIdentifiers() error {
 		return fmt.Errorf("A non-200 error code from v1 authors transformer! Status: %v", resp.StatusCode)
 	}
 
-	as.authorUUIDs = make(map[string]struct{})
+	authorUUIDsTmp := make(map[string]struct{})
 
 	scan := bufio.NewScanner(resp.Body)
 	for scan.Scan() {
@@ -74,15 +79,37 @@ func (as *curatedAuthorService) LoadAuthorIdentifiers() error {
 		if err != nil {
 			return err
 		}
-		as.authorUUIDs[id.UUID] = struct{}{}
+		authorUUIDsTmp[id.UUID] = struct{}{}
 	}
+	as.authorLock.Lock()
+	defer as.authorLock.Unlock()
+	as.authorUUIDs = authorUUIDsTmp
 	log.Infof("Found %v authors", len(as.authorUUIDs))
 
 	return nil
 }
 
+func (as *curatedAuthorService) RefreshAuthorIdentifiers() {
+	ticker := time.NewTicker(as.authorRefreshInterval)
+	go func() {
+		for range ticker.C {
+			err := as.LoadAuthorIdentifiers()
+			if err != nil { //log and use the map in memory
+				log.Errorf("Error on author identifier list refresh attempt %v", err)
+			} else {
+				log.Infof("Author identifier list has been refreshed")
+			}
+
+		}
+	}()
+
+}
+
 func (as *curatedAuthorService) IsFTAuthor(uuid string) bool {
+	as.authorLock.RLock()
+	defer as.authorLock.RUnlock()
 	_, found := as.authorUUIDs[uuid]
+
 	return found
 }
 

service/author_service_test.go

@@ -1,21 +1,26 @@
 package service
 
 import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
 	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
 )
 
 var expectedAuthorUUIDs = map[string]struct{}{
 	"2916ded0-6d1f-4449-b54c-3805da729c1d": struct{}{},
 	"ddc22d37-624a-4a3d-88e5-ba508e38c8ba": struct{}{},
 }
 
+var authorIds = "{\"ID\":\"2916ded0-6d1f-4449-b54c-3805da729c1d\"}\n{\"ID\":\"ddc22d37-624a-4a3d-88e5-ba508e38c8ba\"}"
+
 func (m *mockAuthorTransformerServer) startMockAuthorTransformerServer(t *testing.T) *httptest.Server {
+	m.countLock = &sync.RWMutex{}
 	r := mux.NewRouter()
 	r.HandleFunc(authorTransformerIdsPath, func(w http.ResponseWriter, req *http.Request) {
 		ua := req.Header.Get("User-Agent")
@@ -25,11 +30,12 @@ func (m *mockAuthorTransformerServer) startMockAuthorTransformerServer(t *testin
 		contentType := req.Header.Get("Content-Type")
 		user, password, _ := req.BasicAuth()
 
-		w.WriteHeader(m.Ids(contentType, user, password))
-
-		authorIds := "{\"ID\":\"2916ded0-6d1f-4449-b54c-3805da729c1d\"}\n{\"ID\":\"ddc22d37-624a-4a3d-88e5-ba508e38c8ba\"}"
-		w.Write([]byte(authorIds))
-
+		status, resp := m.Ids(contentType, user, password)
+		w.WriteHeader(status)
+		w.Write([]byte(resp))
+		m.countLock.Lock()
+		defer m.countLock.Unlock()
+		m.count++
 	}).Methods("GET")
 
 	r.HandleFunc(gtgPath, func(w http.ResponseWriter, req *http.Request) {
@@ -41,28 +47,28 @@ func (m *mockAuthorTransformerServer) startMockAuthorTransformerServer(t *testin
 
 func TestLoadAuthorIdentifiersResponseSuccess(t *testing.T) {
 	m := new(mockAuthorTransformerServer)
-	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
 
 	testServer := m.startMockAuthorTransformerServer(t)
 	defer testServer.Close()
 
-	as, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
 
 	for expectedUUID := range expectedAuthorUUIDs {
-		assert.True(t, as.IsFTAuthor(expectedUUID), "The UUID should belong to an author")
+		assert.True(t, as.IsFTAuthor(expectedUUID), "The UUID is not in the author uuid map")
 	}
 	m.AssertExpectations(t)
 }
 
 func TestLoadAuthorIdentifiersResponseNot200(t *testing.T) {
 	m := new(mockAuthorTransformerServer)
-	m.On("Ids", "application/json", "username", "password").Return(http.StatusBadRequest)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusBadRequest, "")
 
 	testServer := m.startMockAuthorTransformerServer(t)
 	defer testServer.Close()
 
-	_, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	_, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.Error(t, err)
 	m.AssertExpectations(t)
 
@@ -73,7 +79,7 @@ func TestLoadAuthorIdentifiersRequestError(t *testing.T) {
 	testServer := m.startMockAuthorTransformerServer(t)
 	defer testServer.Close()
 
-	_, err := NewAuthorService("#:", "username:password", &http.Client{})
+	_, err := NewAuthorService("#:", "username:password", time.Minute*60, &http.Client{})
 	assert.Error(t, err)
 	m.AssertExpectations(t)
 
@@ -84,7 +90,7 @@ func TestLoadAuthorIdentifiersResponseError(t *testing.T) {
 	testServer := m.startMockAuthorTransformerServer(t)
 	testServer.Close()
 
-	_, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	_, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.Error(t, err)
 	m.AssertExpectations(t)
 }
@@ -94,16 +100,65 @@ func TestLoadAuthorServiceMissingRequestCredentials(t *testing.T) {
 	testServer := m.startMockAuthorTransformerServer(t)
 	testServer.Close()
 
-	_, err := NewAuthorService(testServer.URL, "", &http.Client{})
+	_, err := NewAuthorService(testServer.URL, "", time.Minute*60, &http.Client{})
 	assert.Error(t, err)
 	m.AssertExpectations(t)
 }
 
+func TestRefreshAuthorIdentifiersResponseSuccess(t *testing.T) {
+	m := new(mockAuthorTransformerServer)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
+	testServer := m.startMockAuthorTransformerServer(t)
+	defer testServer.Close()
+
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Millisecond*10, &http.Client{})
+
+	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
+	for expectedUUID := range expectedAuthorUUIDs {
+		assert.True(t, as.IsFTAuthor(expectedUUID), "The UUID is not in the author uuid map")
+	}
+
+	as.RefreshAuthorIdentifiers()
+
+	time.Sleep(time.Millisecond * 35)
+	m.countLock.RLock()
+	defer m.countLock.RUnlock()
+	assert.True(t, m.count >= 4, "author list is not being refreshed correctly, number of calls made: "+strconv.Itoa(m.count))
+	m.AssertExpectations(t)
+}
+
+func TestRefreshAuthorIdentifiersWithErrorContinues(t *testing.T) {
+	m := new(mockAuthorTransformerServer)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
+	m.ExpectedCalls = make([]*mock.Call, 0)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusBadRequest, "")
+	m.ExpectedCalls = make([]*mock.Call, 0)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
+	testServer := m.startMockAuthorTransformerServer(t)
+	defer testServer.Close()
+
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Millisecond*10, &http.Client{})
+
+	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
+	for expectedUUID := range expectedAuthorUUIDs {
+		assert.True(t, as.IsFTAuthor(expectedUUID), "The UUID is not in the author uuid map")
+	}
+	as.RefreshAuthorIdentifiers()
+
+	time.Sleep(time.Millisecond * 35)
+	m.countLock.RLock()
+	defer m.countLock.RUnlock()
+	assert.True(t, m.count >= 4, "author list is not being refreshed correctly, number of calls made:"+strconv.Itoa(m.count))
+
+	m.AssertExpectations(t)
+}
+
 func TestIsFTAuthorTrue(t *testing.T) {
 	testService := &curatedAuthorService{
 		httpClient:  nil,
 		serviceURL:  "url",
 		authorUUIDs: expectedAuthorUUIDs,
+		authorLock:  &sync.RWMutex{},
 	}
 	isAuthor := testService.IsFTAuthor("2916ded0-6d1f-4449-b54c-3805da729c1d")
 	assert.True(t, isAuthor)
@@ -114,57 +169,60 @@ func TestIsIsFTAuthorFalse(t *testing.T) {
 		httpClient:  nil,
 		serviceURL:  "url",
 		authorUUIDs: expectedAuthorUUIDs,
+		authorLock:  &sync.RWMutex{},
 	}
 	isAuthor := testService.IsFTAuthor("61346cf7-008b-49e0-945a-832a90cd60ac")
 	assert.False(t, isAuthor)
 }
 
 func TestIsGTG(t *testing.T) {
 	m := new(mockAuthorTransformerServer)
-	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
 	m.On("GTG").Return(http.StatusOK)
 
 	testServer := m.startMockAuthorTransformerServer(t)
 	defer testServer.Close()
 
-	as, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
 	assert.NoError(t, as.IsGTG(), "No GTG errors")
 }
 
 func TestIsNotGTG(t *testing.T) {
 	m := new(mockAuthorTransformerServer)
-	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
 	m.On("GTG").Return(http.StatusServiceUnavailable)
 
 	testServer := m.startMockAuthorTransformerServer(t)
 	defer testServer.Close()
 
-	as, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
 	assert.EqualError(t, as.IsGTG(), "gtg endpoint returned a non-200 status: 503", "GTG should return 503")
 }
 
 func TestGTGConnectionError(t *testing.T) {
 	m := new(mockAuthorTransformerServer)
-	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK)
+	m.On("Ids", "application/json", "username", "password").Return(http.StatusOK, authorIds)
 	m.On("GTG").Return(http.StatusServiceUnavailable)
 
 	testServer := m.startMockAuthorTransformerServer(t)
 
-	as, err := NewAuthorService(testServer.URL, "username:password", &http.Client{})
+	as, err := NewAuthorService(testServer.URL, "username:password", time.Minute*60, &http.Client{})
 	assert.NoError(t, err, "Creation of a new Author sevice should not return an error")
 	testServer.Close()
 	assert.Error(t, as.IsGTG(), "GTG should return a connection error")
 }
 
 type mockAuthorTransformerServer struct {
 	mock.Mock
+	count     int
+	countLock *sync.RWMutex
 }
 
-func (m *mockAuthorTransformerServer) Ids(contentType string, user string, password string) int {
+func (m *mockAuthorTransformerServer) Ids(contentType string, user string, password string) (int, string) {
 	args := m.Called(contentType, user, password)
-	return args.Int(0)
+	return args.Int(0), args.String(1)
 }
 
 func (m *mockAuthorTransformerServer) GTG() int {