Skip to content

Commit

Permalink
Add x-ignore-hash header
Browse files Browse the repository at this point in the history
  • Loading branch information
@geoah
geoah committed Apr 19, 2018
1 parent 16eb6b9 commit ec3de38
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 15 deletions.
2 changes: 1 addition & 1 deletion service/handlers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ func (mw *mockWriter) Delete(uuid string, tid string) error {
return mw.deleteError
}

func (mw *mockWriter) Write(uuid string, b *[]byte, ct string, tid string) (status, error) {
func (mw *mockWriter) Write(uuid string, b *[]byte, ct string, tid string, ignoreHash bool) (status, error) {
mw.Lock()
defer mw.Unlock()
mw.uuid = uuid
Expand Down
11 changes: 6 additions & 5 deletions service/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (r *S3QProcessor) ProcessMsg(m consumer.Message) {
uuid = m.Headers["Message-Id"]
}

writeStatus, err := r.Write(uuid, &b, ct, tid)
writeStatus, err := r.Write(uuid, &b, ct, tid, false)
if err != nil {
logger.WithError(err).WithTransactionID(tid).WithUUID(uuid).Error("Failed to write")
return
Expand Down Expand Up @@ -291,7 +291,7 @@ func (r *S3Reader) listObjects(keys chan<- *string) error {
}

type Writer interface {
Write(uuid string, b *[]byte, contentType string, transactionId string) (status, error)
Write(uuid string, b *[]byte, contentType string, transactionId string, ignoreHash bool) (status, error)
Delete(uuid string, transactionId string) error
}

Expand Down Expand Up @@ -328,7 +328,7 @@ func (w *S3Writer) Delete(uuid string, tid string) error {
return nil
}

func (w *S3Writer) Write(uuid string, b *[]byte, ct string, tid string) (status, error) {
func (w *S3Writer) Write(uuid string, b *[]byte, ct string, tid string, ignoreHash bool) (status, error) {
params := &s3.PutObjectInput{
Bucket: aws.String(w.bucketName),
Key: aws.String(getKey(w.bucketPrefix, uuid)),
Expand All @@ -347,7 +347,7 @@ func (w *S3Writer) Write(uuid string, b *[]byte, ct string, tid string) (status,
status, newHash, err := w.compareObjectToStore(uuid, b, tid)
if err != nil {
return status, err
} else if w.onlyUpdatesEnabled && status == UNCHANGED {
} else if w.onlyUpdatesEnabled && !ignoreHash && status == UNCHANGED {
logger.WithTransactionID(tid).WithUUID(uuid).Debug("Concept has not been updated since last upload, record was skipped")
return status, nil
}
Expand Down Expand Up @@ -430,8 +430,9 @@ func (w *WriterHandler) HandleWrite(rw http.ResponseWriter, r *http.Request) {
return
}

ignoreHash, _ := strconv.ParseBool(r.Header.Get("X-Ignore-Hash"))
ct := r.Header.Get("Content-Type")
writeStatus, _ := w.writer.Write(uuid, &bs, ct, tid)
writeStatus, _ := w.writer.Write(uuid, &bs, ct, tid, ignoreHash)

switch writeStatus {
case INTERNAL_ERROR:
Expand Down
42 changes: 33 additions & 9 deletions service/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func TestWritingToS3(t *testing.T) {
p := []byte("PAYLOAD")
ct := expectedContentType
var err error
_, err = w.Write(expectedUUID, &p, ct, expectedTransactionId)
_, err = w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
Expand Down Expand Up @@ -169,7 +169,7 @@ func TestWritingToS3WithTransactionID(t *testing.T) {

w, s := getWriter()

_, err := w.Write(expectedUUID, &[]byte{}, "", mw.tid)
_, err := w.Write(expectedUUID, &[]byte{}, "", mw.tid, false)

assert.NoError(t, err)
assert.Equal(t, expectedTransactionId, *s.putObjectInput.Metadata[transactionid.TransactionIDKey])
Expand All @@ -188,7 +188,7 @@ func TestWritingToS3WithNewTransactionID(t *testing.T) {

w, s := getWriter()

_, err := w.Write(expectedUUID, &[]byte{}, "", mw.tid)
_, err := w.Write(expectedUUID, &[]byte{}, "", mw.tid, false)

assert.NoError(t, err)
assert.Equal(t, mw.tid, *s.putObjectInput.Metadata[transactionid.TransactionIDKey])
Expand All @@ -198,7 +198,7 @@ func TestWritingToS3WithNoContentType(t *testing.T) {
w, s := getWriter()
p := []byte("PAYLOAD")
var err error
_, err = w.Write(expectedUUID, &p, "", expectedTransactionId)
_, err = w.Write(expectedUUID, &p, "", expectedTransactionId, false)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
Expand All @@ -218,7 +218,7 @@ func TestWritingToS3WithOnlyUpdatesAllowed_SuccessWhenHashIsOutOfDate(t *testing
p := []byte("PAYLOAD")
ct := expectedContentType
var err error
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId)
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
Expand All @@ -239,7 +239,7 @@ func TestWritingToS3WithOnlyUpdatesAllowed_ObjectHasNoCurrentObjectHash(t *testi
p := []byte("PAYLOAD")
ct := expectedContentType
var err error
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId)
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
Expand All @@ -263,18 +263,42 @@ func TestWritingToS3WithOnlyUpdatesAllowed_NoObjectWrittenForObjectWithExistingH
existingHashString := fmt.Sprint(existingHash)
w, s := getWriterOnlyUpdates(existingHashString)
ct := expectedContentType
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId)
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.NoError(t, err)
assert.Empty(t, s.putObjectInput)
assert.Equal(t, UNCHANGED, writeStatus, "Object should have existed prior to write and was unchanged")
}

func TestWritingToS3WithOnlyUpdatesAllowed_ObjectWrittenForObjectWithExistingHashWithIgnoreHash(t *testing.T) {
var err error
p := []byte("PAYLOAD")
existingHash, err := hashstructure.Hash(&p, nil)
assert.NoError(t, err)
existingHashString := fmt.Sprint(existingHash)
w, s := getWriterOnlyUpdates(existingHashString)
ct := expectedContentType
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, true)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
assert.Equal(t, "testBucket", *s.putObjectInput.Bucket)
assert.Equal(t, expectedContentType, *s.putObjectInput.ContentType)
assert.Equal(t, UNCHANGED, writeStatus, "Object should have existed prior to write with with hash metadata and was updated")

rs := s.putObjectInput.Body
assert.NotNil(t, rs)
ba, err := ioutil.ReadAll(rs)
assert.NoError(t, err)
body := string(ba[:])
assert.Equal(t, "PAYLOAD", body)
}

func TestWritingToS3WithOnlyUpdatesAllowed_SuccessForNewObject(t *testing.T) {
w, s := getWriterNoExistingObject()
p := []byte("PAYLOAD")
ct := expectedContentType
var err error
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId)
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.NoError(t, err)
assert.NotEmpty(t, s.putObjectInput)
assert.Equal(t, "test/prefix/123e4567/e89b/12d3/a456/426655440000", *s.putObjectInput.Key)
Expand All @@ -295,7 +319,7 @@ func TestFailingToWriteToS3(t *testing.T) {
p := []byte("PAYLOAD")
ct := expectedContentType
s.s3error = errors.New("S3 error")
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId)
writeStatus, err := w.Write(expectedUUID, &p, ct, expectedTransactionId, false)
assert.Error(t, err)
assert.Equal(t, SERVICE_UNAVAILABLE, writeStatus, "Write should have returned an error with status unavailable")
}
Expand Down

0 comments on commit ec3de38

Please sign in to comment.