From 21da5e68d88341ce85f511c6554aeb2d235f574c Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Mon, 12 Sep 2016 14:18:08 +0200 Subject: [PATCH 1/6] failing test for case attachments --- .../apps/case/tests/test_multimedia.py | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py index b49e2f68180d..a0eeb3e8be30 100644 --- a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py +++ b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py @@ -2,6 +2,7 @@ import uuid import os import hashlib +from unittest.case import skip from django.conf import settings from django.template import Template, Context @@ -14,13 +15,14 @@ from casexml.apps.case.tests.util import TEST_DOMAIN_NAME from casexml.apps.case.xml import V2 from corehq.apps.receiverwrapper.util import submit_form_locally +from corehq.blobs import get_blob_db +from corehq.blobs.tests.util import TemporaryS3BlobDB from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors from couchforms.models import XFormInstance from dimagi.utils.parsing import json_format_datetime from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.tests.utils import FormProcessorTestUtils, run_with_all_backends -from corehq.util.test_utils import TestFileMixin - +from corehq.util.test_utils import TestFileMixin, trap_extra_setup TEST_CASE_ID = "EOL9FIAKIQWOFXFOH0QAMWU64" CREATE_XFORM_ID = "6RGAZTETE3Z2QC0PE2DKM88MO" @@ -300,3 +302,34 @@ def test_sync_log_invalidation_bug(self): # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[], sync_token=sync_log._id) + + +@skip("Case attachments don't work with S3 API yet") +class CaseMultimediaS3DBTest(BaseCaseMultimediaTest): + """ + Tests new attachments for cases and case properties + Spec: https://github.com/dimagi/commcare/wiki/CaseAttachmentAPI + """ + + def setUp(self): + super(CaseMultimediaS3DBTest, self).setUp() + with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"): + config = settings.S3_BLOB_DB_SETTINGS + self.s3db = TemporaryS3BlobDB(config) + assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db) + + def tearDown(self): + self.s3db.close() + super(CaseMultimediaS3DBTest, self).tearDown() + + @run_with_all_backends + def test_case_attachment(self): + single_attach = 'fruity_file' + xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach]) + + self.assertEqual(1, len(case.case_attachments)) + self.assertTrue(single_attach in case.case_attachments) + self.assertEqual( + self._calc_file_hash(single_attach), + hashlib.md5(case.get_attachment(single_attach)).hexdigest() + ) From 9b5e7bceb3227d53788209c70dd5d4f21ae40819 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 13 Sep 2016 12:06:51 +0200 Subject: [PATCH 2/6] remove test skip --- corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py index a0eeb3e8be30..d8e14960fb5c 100644 --- a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py +++ b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py @@ -2,7 +2,6 @@ import uuid import os import hashlib -from unittest.case import skip from django.conf import settings from django.template import Template, Context @@ -304,7 +303,6 @@ def test_sync_log_invalidation_bug(self): sync_token=sync_log._id) -@skip("Case attachments don't work with S3 API yet") class CaseMultimediaS3DBTest(BaseCaseMultimediaTest): """ Tests new attachments for cases and case properties From d46dfa653e44b44d7519711356a526efb59de981 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 13 Sep 2016 12:08:19 +0200 Subject: [PATCH 3/6] better unicode printing of form attachments for debugging --- corehq/form_processor/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/corehq/form_processor/models.py b/corehq/form_processor/models.py index 5b6698b3d3fc..7a8da7a21dc2 100644 --- a/corehq/form_processor/models.py +++ b/corehq/form_processor/models.py @@ -462,6 +462,19 @@ class XFormAttachmentSQL(AbstractAttachment, IsImageMixin): related_name=AttachmentMixin.ATTACHMENTS_RELATED_NAME, related_query_name="attachment" ) + def __unicode__(self): + return unicode( + "XFormAttachmentSQL(" + "attachment_id='{a.attachment_id}', " + "form_id='{a.form_id}', " + "name='{a.name}', " + "content_type='{a.content_type}', " + "content_length='{a.content_length}', " + "md5='{a.md5}', " + "blob_id='{a.blob_id}', " + "properties='{a.properties}', " + ).format(a=self) + class Meta: db_table = XFormAttachmentSQL_DB_TABLE app_label = "form_processor" From 69c20ef3fdd9a8ff5dd4ff3c07131c6ddb3ece6a Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 13 Sep 2016 12:27:20 +0200 Subject: [PATCH 4/6] tweak for readability --- corehq/form_processor/backends/sql/update_strategy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/corehq/form_processor/backends/sql/update_strategy.py b/corehq/form_processor/backends/sql/update_strategy.py index 0bd282997529..feba0530693b 100644 --- a/corehq/form_processor/backends/sql/update_strategy.py +++ b/corehq/form_processor/backends/sql/update_strategy.py @@ -157,14 +157,13 @@ def _apply_attachments_action(self, attachment_action, xform): new_attachment = CaseAttachmentSQL.from_case_update(att) if new_attachment.is_present: form_attachment = xform.get_attachment_meta(att.attachment_src) - new_attachment.update_from_attachment(form_attachment) - if identifier in current_attachments: existing_attachment = current_attachments[identifier] - existing_attachment.update_from_attachment(new_attachment) + existing_attachment.update_from_attachment(form_attachment) existing_attachment.copy_content(form_attachment) self.case.track_update(existing_attachment) else: + new_attachment.update_from_attachment(form_attachment) new_attachment.copy_content(form_attachment) new_attachment.case = self.case self.case.track_create(new_attachment) From 7d039ed7c1de8a86c71d60a169e5da404d41d831 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 13 Sep 2016 12:32:34 +0200 Subject: [PATCH 5/6] reference form attachments in case - don't copy --- .../backends/sql/update_strategy.py | 6 ++---- corehq/form_processor/models.py | 16 +++------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/corehq/form_processor/backends/sql/update_strategy.py b/corehq/form_processor/backends/sql/update_strategy.py index feba0530693b..e04db367838f 100644 --- a/corehq/form_processor/backends/sql/update_strategy.py +++ b/corehq/form_processor/backends/sql/update_strategy.py @@ -159,12 +159,10 @@ def _apply_attachments_action(self, attachment_action, xform): form_attachment = xform.get_attachment_meta(att.attachment_src) if identifier in current_attachments: existing_attachment = current_attachments[identifier] - existing_attachment.update_from_attachment(form_attachment) - existing_attachment.copy_content(form_attachment) + existing_attachment.from_form_attachment(form_attachment) self.case.track_update(existing_attachment) else: - new_attachment.update_from_attachment(form_attachment) - new_attachment.copy_content(form_attachment) + new_attachment.from_form_attachment(form_attachment) new_attachment.case = self.case self.case.track_create(new_attachment) elif identifier in current_attachments: diff --git a/corehq/form_processor/models.py b/corehq/form_processor/models.py index 7a8da7a21dc2..a8034b77774e 100644 --- a/corehq/form_processor/models.py +++ b/corehq/form_processor/models.py @@ -869,14 +869,15 @@ class CaseAttachmentSQL(AbstractAttachment, CaseAttachmentMixin): attachment_src = models.TextField(null=True) attachment_from = models.TextField(null=True) - def update_from_attachment(self, attachment): + def from_form_attachment(self, attachment): """ - Update fields in this attachment with fields from anaother attachment + Update fields in this attachment with fields from another attachment :param attachment: XFormAttachmentSQL or CaseAttachmentSQL object """ self.content_length = attachment.content_length self.blob_id = attachment.blob_id + self.blob_bucket = attachment._blobdb_bucket() self.md5 = attachment.md5 self.content_type = attachment.content_type self.properties = attachment.properties @@ -891,17 +892,6 @@ def update_from_attachment(self, attachment): self.attachment_src = attachment.attachment_src self.attachment_from = attachment.attachment_from - def copy_content(self, attachment): - if self.is_saved(): - deleted = self.delete_content() - if not deleted: - logging.warn( - "Case attachment content not deleted. bucket=%s, blob_id=%s", - self._blobdb_bucket(), self.blob_id - ) - content = attachment.read_content(stream=True) - self.write_content(content) - @classmethod def from_case_update(cls, attachment): if attachment.attachment_src: From 995cb484179289529c7aa27ed48c585f6a582f93 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 13 Sep 2016 15:30:44 +0200 Subject: [PATCH 6/6] move out of context --- .../ex-submodules/casexml/apps/case/tests/test_multimedia.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py index d8e14960fb5c..45db9c8503ef 100644 --- a/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py +++ b/corehq/ex-submodules/casexml/apps/case/tests/test_multimedia.py @@ -313,8 +313,9 @@ def setUp(self): super(CaseMultimediaS3DBTest, self).setUp() with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"): config = settings.S3_BLOB_DB_SETTINGS - self.s3db = TemporaryS3BlobDB(config) - assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db) + + self.s3db = TemporaryS3BlobDB(config) + assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db) def tearDown(self): self.s3db.close()