Merge branch 'a342594055523082_complaints_off' into dev

setup.py

@@ -45,7 +45,7 @@
 }
 
 setup(name='openprocurement.api',
-      version='2.3.59',
+      version='2.3.60.1',
       description='openprocurement.api',
       long_description=README,
       classifiers=[

src/openprocurement/api/migration.py

@@ -8,7 +8,7 @@
 
 
 LOGGER = logging.getLogger(__name__)
-SCHEMA_VERSION = 23
+SCHEMA_VERSION = 24
 SCHEMA_DOC = 'openprocurement_schema'
 
 
@@ -716,3 +716,43 @@ def __init__(self, registry):
             docs = []
     if docs:
         registry.db.update(docs)
+
+
+def from23to24(registry):
+
+    def update_doc_url(document):
+        doc_id = document['id']
+        doc_url = document['url']
+        if doc_url.startswith(registry.docservice_url):
+            return False
+        if 'documents/' + doc_id not in doc_url:
+            pre, post = doc_url.split('documents')
+            doc_key = post.split('?')[1]
+            doc_url = pre + 'documents/' + doc_id + '?' + doc_key
+            document['url'] = doc_url
+            return True
+
+    results = registry.db.iterview('tenders/all', 2 ** 10, include_docs=True)
+    docs = []
+    count = 0
+    for i in results:
+        changed = False
+        doc = i.doc
+        for bid in doc.get('bids', []):
+            for document_type in ['documents', 'financialDocuments', 'eligibilityDocuments', 'qualificationDocuments']:
+                for document in bid.get(document_type, []):
+                    if update_doc_url(document):
+                        changed = True
+        if changed:
+            doc['dateModified'] = get_now().isoformat()
+            docs.append(doc)
+        if len(docs) >= 2 ** 7:
+            registry.db.update(docs)
+            count += len(docs)
+            docs = []
+    if docs:
+        registry.db.update(docs)
+        count += len(docs)
+
+    LOGGER.info("Migrated {} objects.".format(count))
+    LOGGER.info("Migration complete.")

src/openprocurement/api/tests/bidder.py

@@ -980,6 +980,7 @@ def test_put_tender_bidder_document_json(self):
         self.assertEqual(response.content_type, 'application/json')
         self.assertEqual('test description', response.json["data"]["description"])
         self.assertEqual(doc_id, response.json["data"]["id"])
+        self.assertIn(self.bid_id + '/documents/' + doc_id, response.json["data"]["url"])
         key = response.json["data"]["url"].split('?')[-1]
 
         response = self.app.get('/tenders/{}/bids/{}/documents/{}?{}&acc_token={}'.format(

src/openprocurement/api/tests/migration.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-from openprocurement.api.models import Tender
+from openprocurement.api.models import Tender, get_now
 from openprocurement.api.migration import migrate_data, get_db_schema_version, set_db_schema_version, SCHEMA_VERSION
 from openprocurement.api.tests.base import BaseWebTest, test_tender_data
 from email.header import Header
@@ -763,6 +763,85 @@ def test_migrate_from22to23(self):
         self.assertIn('KeyID=', migrated_item['documents'][0]['url'])
         self.assertIn('Signature=', migrated_item['documents'][0]['url'])
 
+    def test_migrate_from23to24(self):
+        set_db_schema_version(self.db, 23)
+        self.app.app.registry.docservice_url = 'http://localhost.ds'
+        u = Tender(test_tender_data)
+        u.tenderID = "UA-X"
+        u.dateModified = get_now().isoformat()
+        u.store(self.db)
+        tender_raw = self.db.get(u.id)
+        date_modified_before = tender_raw['dateModified']
+        bid = {"id": "1b4da15470e84c4d948a5d1660d29776"}
+        bid["documents"] = [
+            {
+                # non-ds url. should be fixed (correct id in url) by migrator
+                "id": "1801ca2749bd40b0944e58adc3e09c46",
+                "title": "name.txt",
+                "documentOf": "tender",
+                "url": "/tenders/{}/bids/{}/documents/63073ea0da17414db9f13e1a8c347e11?download=a65ef5c688884931aed1a472620d3a00".format(u.id, bid['id']),
+                "datePublished": "2016-06-01T00:00:00+03:00",
+                "dateModified": "2016-06-01T00:00:00+03:00",
+                "format": "text/plain",
+                "language": "uk",
+            },
+            {
+                # non-ds url. should be fixed (correct id in url) by migrator
+                "id": "f3e5470b76f84c66a89fd52ed871f645",
+                "title": "name.txt",
+                "documentOf": "tender",
+                "url": "/tenders/{}/bids/{}/documents/512bd84155b145b99e0ac80894fe2b8f?download=d48723d7b4014599ac8d94fb0ac958b4".format(u.id, bid['id']),
+                "datePublished": "2016-06-01T00:00:00+03:00",
+                "dateModified": "2016-06-01T00:00:00+03:00",
+                "format": "text/plain",
+            },
+            {
+                # ds url. should NOT be rewrited by migrator
+                "id": "352d774608d749c996fc0e798ffef433",
+                "title": "name.txt",
+                "documentOf": "tender",
+                "url": "http://localhost.ds/get/b893bf5d2fb44a26bd6896178afe5953?KeyID=i_am_ds_url_lalalalala",  # DS url
+                "datePublished": "2016-06-01T00:00:00+03:00",
+                "dateModified": "2016-06-01T00:00:00+03:00",
+                "format": "text/plain",
+            }
+        ]
+
+        # dirty eligibility documents container simulation
+        bid["eligibilityDocuments"] = [
+            {
+                # non-ds url. should be fixed (correct id in url) by migrator
+                "id": "73e728784f924518b07f16e34750df1b",
+                "title": "name.txt",
+                "documentOf": "tender",
+                "url": "/tenders/{}/bids/{}/eligibility_documents/a109ca6b04d24bd5bccf57917a65e835?download=5443af5e910f46debe412fc36e69f1ad".format(u.id, bid['id']),
+                "datePublished": "2016-06-01T00:00:00+03:00",
+                "dateModified": "2016-06-01T00:00:00+03:00",
+                "format": "text/plain",
+                "language": "uk",
+                "confidentiality": "buyerOnly"  # documents for which url is not rewrited to ds url
+            }
+        ]
+
+        tender_raw['bids'] = [bid,]
+
+        _id, _rev = self.db.save(tender_raw)
+
+        migrate_data(self.app.app.registry, 24)
+        migrated= self.db.get(u.id)
+        migrated_bid = migrated['bids'][0]
+
+        self.assertGreater(migrated['dateModified'], date_modified_before)
+
+        # url should be corrected
+        self.assertIn("/tenders/{}/bids/1b4da15470e84c4d948a5d1660d29776/documents/1801ca2749bd40b0944e58adc3e09c46?download=a65ef5c688884931aed1a472620d3a00".format(u.id), migrated_bid['documents'][0]['url'])
+        self.assertIn("/tenders/{}/bids/1b4da15470e84c4d948a5d1660d29776/documents/f3e5470b76f84c66a89fd52ed871f645?download=d48723d7b4014599ac8d94fb0ac958b4".format(u.id), migrated_bid['documents'][1]['url'])
+        self.assertIn("/tenders/{}/bids/1b4da15470e84c4d948a5d1660d29776/eligibility_documents/73e728784f924518b07f16e34750df1b?download=5443af5e910f46debe412fc36e69f1ad".format(u.id), migrated_bid['eligibilityDocuments'][0]['url'])
+
+        # url remained the same as before migration
+        self.assertIn("http://localhost.ds/get/b893bf5d2fb44a26bd6896178afe5953?KeyID=i_am_ds_url_lalalalala", migrated_bid['documents'][2]['url'])
+
+
 
 def suite():
     suite = unittest.TestSuite()

src/openprocurement/api/utils.py

@@ -103,13 +103,18 @@ def generate_docservice_url(request, doc_id, temporary=True, prefix=None):
 def upload_file(request, blacklisted_fields=DOCUMENT_BLACKLISTED_FIELDS, whitelisted_fields=DOCUMENT_WHITELISTED_FIELDS):
     first_document = request.validated['documents'][-1] if 'documents' in request.validated and request.validated['documents'] else None
     if 'data' in request.validated and request.validated['data']:
-        document = check_document(request, request.validated['document'], 'body', {})
+        document = request.validated['document']
+        check_document(request, document, 'body')
+
         if first_document:
             for attr_name in type(first_document)._fields:
                 if attr_name in whitelisted_fields:
                     setattr(document, attr_name, getattr(first_document, attr_name))
                 elif attr_name not in blacklisted_fields and attr_name not in request.validated['json_data']:
                     setattr(document, attr_name, getattr(first_document, attr_name))
+
+        document_route = request.matched_route.name.replace("collection_", "")
+        document = update_document_url(request, document, document_route, {})
         return document
     if request.content_type == 'multipart/form-data':
         data = request.validated['file']
@@ -380,7 +385,7 @@ def check_bids(request):
     check_ignored_claim(tender)
 
 
-def check_document(request, document, document_container, route_kwargs):
+def check_document(request, document, document_container):
     url = document.url
     parsed_url = urlparse(url)
     parsed_query = dict(parse_qsl(parsed_url.query))
@@ -416,14 +421,31 @@ def check_document(request, document, document_container, route_kwargs):
         request.errors.add(document_container, 'url', "Document url invalid.")
         request.errors.status = 422
         raise error_handler(request.errors)
+
+
+def update_document_url(request, document, document_route, route_kwargs):
+    key = urlparse(document.url).path.split('/')[-1]
+    route_kwargs.update({'_route_name': document_route,
+                         'document_id': document.id,
+                         '_query': {'download': key}})
+    document_path = request.current_route_path(**route_kwargs)
+    document.url = '/' + '/'.join(document_path.split('/')[3:])
+    return document
+
+
+def check_document_batch(request, document, document_container, route_kwargs):
+    check_document(request, document, document_container)
+
     document_route = request.matched_route.name.replace("collection_", "")
+    # Following piece of code was written by leits, so no one knows how it works
+    # and why =)
+    # To redefine document_route to get appropriate real document route when bid
+    # is created with documents? I hope so :)
     if "Documents" not in document_route:
         specified_document_route_end = (document_container.lower().rsplit('documents')[0] + ' documents').lstrip().title()
         document_route = ' '.join([document_route[:-1], specified_document_route_end])
-    route_kwargs.update({'_route_name': document_route, 'document_id': document.id, '_query': {'download': key}})
-    document_path = request.current_route_path(**route_kwargs)
-    document.url = '/' + '/'.join(document_path.split('/')[3:])
-    return document
+
+    return update_document_url(request, document, document_route, route_kwargs)
 
 
 def check_complaint_status(request, complaint, now=None):
@@ -605,7 +627,8 @@ def check_tender_status(request):
             tender.status = 'unsuccessful'
         if tender.contracts and tender.contracts[-1].status == 'active':
             tender.status = 'complete'
-    check_ignored_claim(tender)
+    if tender.procurementMethodType == "belowThreshold":
+        check_ignored_claim(tender)
 
 
 def add_next_award(request):

src/openprocurement/api/validation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from openprocurement.api.models import get_now, SANDBOX_MODE
 from schematics.exceptions import ModelValidationError, ModelConversionError
-from openprocurement.api.utils import apply_data_patch, update_logging_context, check_document
+from openprocurement.api.utils import apply_data_patch, update_logging_context, check_document_batch
 
 
 def validate_json_data(request):
@@ -331,6 +331,6 @@ def validate_bid_documents(request):
             document = model(document)
             document.validate()
             route_kwargs = {'bid_id': request.validated['bid'].id}
-            document = check_document(request, document, doc_type, route_kwargs)
+            document = check_document_batch(request, document, doc_type, route_kwargs)
             documents[doc_type].append(document)
     return documents