[Fix] Add cloudsearch delete docs

RyanSept · Mar 24, 2017 · 9f47fb4 · 9f47fb4
1 parent 7f783ed
commit 9f47fb4
Show file tree

Hide file tree

Showing 6 changed files with 5 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1 @@
 *.pyc
-delete_*.json
diff --git a/delete_clinical_officers.json b/delete_clinical_officers.json
diff --git a/delete_doctors.json b/delete_doctors.json
diff --git a/delete_foreign_doctors.json b/delete_foreign_doctors.json
diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py
@@ -118,8 +118,8 @@ def archive_data(self, payload):
 
                 # archive historical data
                 date = datetime.today().strftime('%Y%m%d')
-                s3_client.copy_object(Bucket="cfa-healthtools-ke",
-                                      CopySource="data/foreign_doctors.json",
+                self.s3.copy_object(Bucket="cfa-healthtools-ke",
+                                      CopySource="cfa-healthtools-ke/" + self.s3_key,
                                       Key=self.s3_historical_record_key.format(
                                           date)
                                       )

diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py
@@ -30,7 +30,6 @@ def test_it_scrapes_clinical_officers_page(self):
             "http://clinicalofficerscouncil.org/online-services/retention/?currpage=1")[0]
         self.assertTrue(len(entries[0]["fields"]) == 7)
 
-    @unittest.skip('a')
     def test_it_scrapes_whole_doctors_site(self):
         all_entries = self.doctors_scraper.scrape_site()
         self.assertTrue(len(all_entries) > 0)
@@ -39,7 +38,6 @@ def test_it_scrapes_whole_foreign_doctors_site(self):
         all_entries = self.foreign_doctors_scraper.scrape_site()
         self.assertTrue(len(all_entries) > 0)
 
-    @unittest.skip('a')
     def test_it_scrapes_whole_clinical_officers_site(self):
         all_entries = self.clinical_officers_scraper.scrape_site()
         self.assertTrue(len(all_entries) > 0)
@@ -62,7 +60,6 @@ def test_clinical_officers_scraper_uploads_to_cloudsearch(self):
             response = self.clinical_officers_scraper.upload_data(data)
             self.assertEqual(response.get('status'), "success")
 
-    @unittest.skip('a')
     def test_doctors_scraper_archives_to_s3(self):
         self.doctors_scraper.s3_key = "test/doctors.json"
         with open(TEST_DIR + "/dummy_files/doctors.json", "r") as my_file:
@@ -73,9 +70,6 @@ def test_doctors_scraper_archives_to_s3(self):
             Key=self.doctors_scraper.s3_key
         )['Body'].read()
         self.assertEqual(uploaded_data, data)
-        self.doctors_scraper.s3.delete_object(
-            Bucket="cfa-healthtools-ke",
-            Key=self.doctors_scraper.s3_key)
 
     def test_foreign_doctors_scraper_archives_to_s3(self):
         self.foreign_doctors_scraper.s3_key = "test/foreign_doctors.json"
@@ -87,9 +81,6 @@ def test_foreign_doctors_scraper_archives_to_s3(self):
             Key=self.foreign_doctors_scraper.s3_key
         )['Body'].read()
         self.assertEqual(uploaded_data, data)
-        self.foreign_doctors_scraper.s3.delete_object(
-            Bucket="cfa-healthtools-ke",
-            Key=self.foreign_doctors_scraper.s3_key)
 
     def test_clinical_officers_scraper_archives_to_s3(self):
         self.clinical_officers_scraper.s3_key = "test/clinical_officers.json"
@@ -101,9 +92,6 @@ def test_clinical_officers_scraper_archives_to_s3(self):
             Key=self.clinical_officers_scraper.s3_key
         )['Body'].read()
         self.assertEqual(uploaded_data, data)
-        self.clinical_officers_scraper.s3.delete_object(
-            Bucket="cfa-healthtools-ke",
-            Key=self.clinical_officers_scraper.s3_key)
 
     def test_foreign_doctors_scraper_deletes_cloudsearch_docs(self):
         response = self.foreign_doctors_scraper.delete_cloudsearch_docs()