Merge pull request #52 from Financial-Times/feature/UPPSF-4569-write-…

…concepts-in-separate-folders

Feature/uppsf 4569 write concepts in separate folders

README.md

@@ -78,7 +78,7 @@ For complete API specification see [S3 Read/Write API Endpoint](https://docs.goo
 Any payload can be written via the PUT using a unique UUID to identify this payload within the S3 bucket
 
 ```sh
-curl -H 'Content-Type: application/json' -X PUT -d '{"tags":["tag1","tag2"],"question":"Which band?","answers":[{"id":"a0","answer":"Answer1"},{"id":"a1","answer":"answer2"}]}' http://localhost:8080/bcac6326-dd23-4b6a-9dfa-c2fbeb9737d9
+curl -H 'Content-Type: application/json' -X PUT -d '{"tags":["tag1","tag2"],"question":"Which band?","answers":[{"id":"a0","answer":"Answer1"},{"id":"a1","answer":"answer2"}]}' http://localhost:8080/123e4567-e89b-12d3-a456-426655440000
 ```
 
 The `Content-Type` is important as that will be what the file will be stored as.
@@ -90,6 +90,17 @@ The reason we do this is so that it becomes easier to manage/browser for content
 It is also good practice to do this as it means that files get put into different partitions.
 This is important if you're writing and pulling content from S3 as it means that content will get written/read from different partitions on S3.
 
+If `bucket_prefix` is present, concepts will always be written in the following format `<bucket_prefix>/123e4567/e89b/12d3/a456/426655440000`.
+
+However if `bucket_prefix` is not present, there is additional `path` parameter functionality for writing/reading concepts to/from a specific partition. A sample request would look like:
+
+```sh
+curl -H 'Content-Type: application/json' -X PUT -d '{"tags":["tag1","tag2"],"question":"Which band?","answers":[{"id":"a0","answer":"Answer1"},{"id":"a1","answer":"answer2"}]}' http://localhost:8080/123e4567-e89b-12d3-a456-426655440000?path=TestDirectory
+```
+
+when the parameter is present and the content is uploaded, the key generated for the item is converted from
+`123e4567-e89b-12d3-a456-426655440000` to `TestDirectory/123e4567/e89b/12d3/a456/426655440000`.
+
 ### GET /UUID
 
 This internal read should return what was written to S3
@@ -100,8 +111,20 @@ If not found, you'll get a 404 response.
 curl http://localhost:8080/bcac6326-dd23-4b6a-9dfa-c2fbeb9737d9
 ```
 
+Getting what was written in specific directory
+
+```sh
+curl http://localhost:8080/bcac6326-dd23-4b6a-9dfa-c2fbeb9737d9?path=TestDirectory
+```
+
 ### DELETE /UUID
 
+To delete something from specific directory the `path` parameter should be appended to the request as follows:
+
+```sh
+curl .../bcac6326-dd23-4b6a-9dfa-c2fbeb9737d9?path=TestDirectory
+```
+
 Will return 204
 
 ## Utility endpoints
@@ -110,6 +133,12 @@ Will return 204
 
 Streams all payloads in a given bucket
 
+To stream the payload a specific directory the `path` parameter should be appended to the request as follows:
+
+```sh
+curl .../bcac6326-dd23-4b6a-9dfa-c2fbeb9737d9?path=TestDirectory
+```
+
 ### GET /__ids
 
 Streams all ids in a given bucket
@@ -129,6 +158,17 @@ The return payload will look like:
 ...
 ```
 
+If there were concepts also in a directory `TestDirectory` the payload will look like:
+
+```sh
+{"ID":"dcfa65d6-3849-445e-ac6a-15bc5a17e954"}
+{"ID":"2136f8ad-e94e-45cb-b616-336f38533214"}
+{"ID":"c9f5337d-0435-477e-b0f5-bd35ff3a4b48"}
+{"ID":"TestDirectory-7f84a70b-7085-4309-aa8e-304b3759f49f"}
+{"ID":"TestDirectory-99a0537a-3635-479b-92f7-ba10b63e2f87"}
+...
+```
+
 ### Admin endpoints
 
 Healthchecks: [http://localhost:8080/__health](http://localhost:8080/__health)  

service/handlers_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -87,6 +86,23 @@ func TestWriteHandlerNewContentReturnsCreated(t *testing.T) {
 	assert.Equal(t, "{\"message\":\"Created concept record in store\"}", rec.Body.String())
 }
 
+func TestWriteHandlerNewContentSpecificDirectoryReturnsCreated(t *testing.T) {
+	log := logger.NewUPPLogger("handlers_test", "Debug")
+	r := mux.NewRouter()
+	mw := &mockWriter{writeStatus: CREATED}
+	mr := &mockReader{log: log}
+	Handlers(r, NewWriterHandler(mw, mr, log), ReaderHandler{}, ExpectedResourcePath)
+
+	rec := httptest.NewRecorder()
+	r.ServeHTTP(rec, newRequestWithPathParameter("PUT", withExpectedResourcePath("/22f53313-85c6-46b2-94e7-cfde9322f26c"), "PAYLOAD"))
+
+	assert.Equal(t, 201, rec.Code)
+	assert.Equal(t, "PAYLOAD", mw.payload)
+	assert.Equal(t, "22f53313-85c6-46b2-94e7-cfde9322f26c", mw.uuid)
+	assert.Equal(t, ExpectedContentType, mw.ct)
+	assert.Equal(t, "{\"message\":\"Created concept record in store\"}", rec.Body.String())
+}
+
 func TestWriteHandlerUpdateContentReturnsOK(t *testing.T) {
 	log := logger.NewUPPLogger("handlers_test", "Debug")
 	r := mux.NewRouter()
@@ -104,6 +120,23 @@ func TestWriteHandlerUpdateContentReturnsOK(t *testing.T) {
 	assert.Equal(t, "{\"message\":\"Updated concept record in store\"}", rec.Body.String())
 }
 
+func TestWriteHandlerUpdateSpecificDirectoryContentReturnsOK(t *testing.T) {
+	log := logger.NewUPPLogger("handlers_test", "Debug")
+	r := mux.NewRouter()
+	mw := &mockWriter{writeStatus: UPDATED}
+	mr := &mockReader{log: log}
+	Handlers(r, NewWriterHandler(mw, mr, log), ReaderHandler{}, ExpectedResourcePath)
+
+	rec := httptest.NewRecorder()
+	r.ServeHTTP(rec, newRequestWithPathParameter("PUT", withExpectedResourcePath("/89d15f70-640d-11e4-9803-0800200c9a66"), "PAYLOAD"))
+
+	assert.Equal(t, 200, rec.Code)
+	assert.Equal(t, "PAYLOAD", mw.payload)
+	assert.Equal(t, "89d15f70-640d-11e4-9803-0800200c9a66", mw.uuid)
+	assert.Equal(t, ExpectedContentType, mw.ct)
+	assert.Equal(t, "{\"message\":\"Updated concept record in store\"}", rec.Body.String())
+}
+
 func TestWriteHandlerAlreadyExistsReturnsNotModified(t *testing.T) {
 	log := logger.NewUPPLogger("handlers_test", "Debug")
 	r := mux.NewRouter()
@@ -121,6 +154,23 @@ func TestWriteHandlerAlreadyExistsReturnsNotModified(t *testing.T) {
 	assert.Equal(t, "", rec.Body.String())
 }
 
+func TestWriteHandlerAlreadyExistsSpecificDirectoryReturnsNotModified(t *testing.T) {
+	log := logger.NewUPPLogger("handlers_test", "Debug")
+	r := mux.NewRouter()
+	mw := &mockWriter{writeStatus: UNCHANGED}
+	mr := &mockReader{log: log}
+	Handlers(r, NewWriterHandler(mw, mr, log), ReaderHandler{}, ExpectedResourcePath)
+
+	rec := httptest.NewRecorder()
+	r.ServeHTTP(rec, newRequestWithPathParameter("PUT", withExpectedResourcePath("/89d15f70-640d-11e4-9803-0800200c9a66"), "PAYLOAD"))
+
+	assert.Equal(t, 304, rec.Code)
+	assert.Equal(t, "PAYLOAD", mw.payload)
+	assert.Equal(t, "89d15f70-640d-11e4-9803-0800200c9a66", mw.uuid)
+	assert.Equal(t, ExpectedContentType, mw.ct)
+	assert.Equal(t, "", rec.Body.String())
+}
+
 func TestWriterHandlerFailReadingBody(t *testing.T) {
 	log := logger.NewUPPLogger("handlers_test", "Debug")
 	r := mux.NewRouter()
@@ -161,6 +211,20 @@ func TestWriterHandlerDeleteReturnsOK(t *testing.T) {
 	assert.Empty(t, rec.Body.String())
 }
 
+func TestWriterHandlerDeleteSpecificDirectoryReturnsOK(t *testing.T) {
+	log := logger.NewUPPLogger("handlers_test", "Debug")
+	r := mux.NewRouter()
+	mw := &mockWriter{}
+	mr := &mockReader{log: log}
+	Handlers(r, NewWriterHandler(mw, mr, log), ReaderHandler{}, ExpectedResourcePath)
+
+	rec := httptest.NewRecorder()
+	r.ServeHTTP(rec, newRequestWithPathParameter("DELETE", withExpectedResourcePath("/22f53313-85c6-46b2-94e7-cfde9322f26c"), ""))
+	assert.Equal(t, "22f53313-85c6-46b2-94e7-cfde9322f26c", mw.uuid)
+	assert.Equal(t, 204, rec.Code)
+	assert.Empty(t, rec.Body.String())
+}
+
 func TestWriterHandlerDeleteFailsReturns503(t *testing.T) {
 	log := logger.NewUPPLogger("handlers_test", "Debug")
 	r := mux.NewRouter()
@@ -264,7 +328,6 @@ func TestHandleGetAllFailsReturnsServiceUnavailable(t *testing.T) {
 }
 
 func assertRequestAndResponseFromRouter(t testing.TB, r *mux.Router, url string, expectedStatus int, expectedBody string, expectedContentType string) *httptest.ResponseRecorder {
-
 	rec := httptest.NewRecorder()
 	r.ServeHTTP(rec, newRequest("GET", url, ""))
 	assert.Equal(t, expectedStatus, rec.Code)
@@ -279,7 +342,6 @@ func assertRequestAndResponseFromRouter(t testing.TB, r *mux.Router, url string,
 }
 
 func assertRequestAndResponse(t testing.TB, url string, expectedStatus int, expectedBody string) *httptest.ResponseRecorder {
-
 	rec := httptest.NewRecorder()
 	http.DefaultServeMux.ServeHTTP(rec, newRequest("GET", url, ""))
 	assert.Equal(t, expectedStatus, rec.Code)
@@ -313,6 +375,24 @@ func newRequestBodyFail(method, url string) *http.Request {
 	return req
 }
 
+func newRequestWithPathParameter(method, url string, body string) *http.Request {
+	var payload io.Reader
+	if body != "" {
+		payload = bytes.NewReader([]byte(body))
+	}
+	req, err := http.NewRequest(method, url, payload)
+	values := req.URL.Query()
+	values.Set("path", "testDirectory")
+	req.URL.RawQuery = values.Encode()
+	req.Header = map[string][]string{
+		"Content-Type": {ExpectedContentType},
+	}
+	if err != nil {
+		panic(err)
+	}
+	return req
+}
+
 func newRequest(method, url string, body string) *http.Request {
 	var payload io.Reader
 	if body != "" {
@@ -339,15 +419,15 @@ type mockReader struct {
 	log         *logger.UPPLogger
 }
 
-func (r *mockReader) Get(uuid string) (bool, io.ReadCloser, *string, error) {
+func (r *mockReader) Get(uuid string, path string) (bool, io.ReadCloser, *string, error) {
 	r.Lock()
 	defer r.Unlock()
 	r.log.Infof("Got request for uuid: %v", uuid)
 	r.uuid = uuid
 	var body io.ReadCloser
 
 	if r.payload != "" {
-		body = ioutil.NopCloser(strings.NewReader(r.payload))
+		body = io.NopCloser(strings.NewReader(r.payload))
 	}
 
 	if r.rc != nil {
@@ -374,7 +454,7 @@ func (r *mockReader) processPipe() (io.PipeReader, error) {
 	return *pv, r.returnError
 }
 
-func (r *mockReader) GetAll() (io.PipeReader, error) {
+func (r *mockReader) GetAll(path string) (io.PipeReader, error) {
 	return r.processPipe()
 }
 
@@ -390,10 +470,10 @@ type mockWriter struct {
 	deleteError error
 	ct          string
 	tid         string
-	writeStatus status
+	writeStatus Status
 }
 
-func (mw *mockWriter) Delete(uuid string, tid string) error {
+func (mw *mockWriter) Delete(uuid string, path string, tid string) error {
 	mw.Lock()
 	defer mw.Unlock()
 	mw.uuid = uuid
@@ -403,7 +483,7 @@ func (mw *mockWriter) Delete(uuid string, tid string) error {
 	return mw.deleteError
 }
 
-func (mw *mockWriter) Write(uuid string, b *[]byte, ct string, tid string, ignoreHash bool) (status, error) {
+func (mw *mockWriter) Write(uuid string, path string, b *[]byte, ct string, tid string, ignoreHash bool) (Status, error) {
 	mw.Lock()
 	defer mw.Unlock()
 	mw.uuid = uuid