From 7c059c0ba747ca4a63742e37b376f96459768c59 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Wed, 14 Jun 2017 16:13:38 +0300 Subject: [PATCH 01/18] Add support for elastic search cloud service Fix tests to run with elastic search Refactor tests to remove redundant tests --- healthtools/config.py | 17 +++- healthtools/scrapers/base_scraper.py | 91 +++++++++++++------ healthtools/scrapers/clinical_officers.py | 37 ++++---- healthtools/scrapers/doctors.py | 32 ++++--- healthtools/scrapers/foreign_doctors.py | 32 ++++--- .../tests/dummy_files/clinical_officers.json | 2 +- healthtools/tests/dummy_files/doctors.json | 2 +- .../tests/dummy_files/foreign_doctors.json | 2 +- healthtools/tests/test_scrapers.py | 49 ++++------ requirements.txt | 41 ++++++--- scraper.py | 7 +- 11 files changed, 185 insertions(+), 127 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index ca87fe7..157788b 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -5,8 +5,8 @@ "DOCTORS": "http://medicalboard.co.ke/online-services/retention/?currpage={}", "FOREIGN_DOCTORS": "http://medicalboard.co.ke/online-services/foreign-doctors-license-register/?currpage={}", "CLINICAL_OFFICERS": "http://clinicalofficerscouncil.org/online-services/retention/?currpage={}", - "TOKEN_URL" : "http://api.kmhfl.health.go.ke/o/token/" -} + "TOKEN_URL": "http://api.kmhfl.health.go.ke/o/token/" + } AWS = { "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY_ID"), @@ -17,8 +17,17 @@ # Clinical document endpoint "cloudsearch_cos_endpoint": "http://doc-cfa-healthtools-ke-cos-nhxtw3w5goufkzram4er7sciz4.eu-west-1.cloudsearch.amazonaws.com/", # Health facilities endpoint - "cloudsearch_health_faciities_endpoint":"https://doc-health-facilities-ke-65ftd7ksxazyatw5fiv5uyaiqi.eu-west-1.cloudsearch.amazonaws.com", + "cloudsearch_health_faciities_endpoint": "https://doc-health-facilities-ke-65ftd7ksxazyatw5fiv5uyaiqi.eu-west-1.cloudsearch.amazonaws.com", -} + } +ES = { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT"), + "user": os.getenv("ES_USER"), + "pass": os.getenv("ES_PASS"), + "index": "healthtools", + "doctors_type": "doctors", + "cos_type": "clinical-officers" + } TEST_DIR = os.getcwd() + "/healthtools/tests" diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 71d21ec..5d61b43 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -1,7 +1,8 @@ from bs4 import BeautifulSoup from cStringIO import StringIO from datetime import datetime -from healthtools.config import AWS +from elasticsearch import Elasticsearch +from healthtools.config import AWS, ES import requests import boto3 import re @@ -14,7 +15,6 @@ def __init__(self): self.num_pages_to_scrape = None self.site_url = None self.fields = None - self.cloudsearch = None self.s3_key = None self.document_id = 0 # id for each entry, to be incremented self.delete_file = None # contains docs to be deleted after scrape @@ -23,7 +23,10 @@ def __init__(self): "aws_access_key_id": AWS["aws_access_key_id"], "aws_secret_access_key": AWS["aws_secret_access_key"], "region_name": AWS["region_name"], - }) + }) + self.es_client = Elasticsearch([ + "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) + ]) def scrape_site(self): ''' @@ -45,8 +48,7 @@ def scrape_site(self): print "There's something wrong with the site. Proceeding to the next scraper." return - entries = scraped_page[0] - delete_docs = scraped_page[1] + entries, delete_docs = scraped_page all_results.extend(entries) delete_batch.extend(delete_docs) @@ -54,14 +56,15 @@ def scrape_site(self): skipped_pages += 1 print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) continue - print "{{{0}}} - Scraper completed. {1} documents retrieved.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),len(all_results)) + print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(all_results)) if all_results: all_results_json = json.dumps(all_results) delete_batch = json.dumps(delete_batch) - self.delete_cloudsearch_docs() - self.upload_data(all_results_json) + self.delete_elasticsearch_docs() + self.upload_data(all_results) self.archive_data(all_results_json) # store delete operations for next scrape @@ -92,10 +95,17 @@ def scrape_page(self, page_url): columns.append(self.document_id) entry = dict(zip(self.fields, columns)) - entry = self.format_for_cloudsearch(entry) + meta, entry = self.format_for_elasticsearch(entry) + entries.append(meta) entries.append(entry) - delete_batch.append({"type": "delete", "id": entry["id"]}) + delete_batch.append({ + "delete": + { + "_index": ES['index'], + "_type": meta['index']['_type'], + "_id": entry["id"] + }}) self.document_id += 1 return entries, delete_batch except Exception as err: @@ -108,12 +118,11 @@ def scrape_page(self, page_url): def upload_data(self, payload): ''' - Upload data to AWS Cloud Search + Upload data to Elastic Search ''' try: - response = self.cloudsearch.upload_documents( - documents=payload, contentType="application/json" - ) + # bulk index the data and use refresh to ensure that our data will be immediately available + response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) @@ -140,31 +149,45 @@ def archive_data(self, payload): print "{{{0}}} - Archived data has been updated.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) return else: - print "{{{0}}} - Data Scraped does not differ from archived data.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "{{{0}}} - Data Scraped does not differ from archived data.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) - def delete_cloudsearch_docs(self): + def delete_elasticsearch_docs(self): ''' - Delete documents that were uploaded to cloudsearch in the last scrape + Delete documents that were uploaded to elasticsearch in the last scrape ''' try: + # get the type to use with the index depending on the calling method + _type = ES['cos_type'] \ + if 'clinical' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower() \ + else ES['doctors_type'] # get documents to be deleted delete_docs = self.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.delete_file)['Body'].read() - # delete - response = self.cloudsearch.upload_documents( - documents=delete_docs, contentType="application/json" - ) + try: + response = self.es_client.bulk(index=ES['index'], body=delete_docs, refresh=True) + except: + # incase records are saved in cloudsearch's format, reformat for elasticsearch deletion + delete_records = [] + for record in json.loads(delete_docs): + delete_records.append({ + "delete": { + "_index": ES['index'], + "_type": _type, + "_id": record['delete']["_id"] + }}) + response = self.es_client.bulk(index=ES['index'], body=delete_records) return response except Exception as err: if "NoSuchKey" in err: - print "ERROR - delete_cloudsearch_docs() - no delete file present" + print "ERROR - delete_elasticsearch_docs() - no delete file present" return - print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + print "ERROR - delete_elasticsearch_docs() - {} - {}".format(type(self).__name__, str(err)) def get_total_number_of_pages(self): ''' @@ -177,7 +200,7 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ + print "ERROR: **get_total_page_numbers()** - url: {} - err: {}". \ format(self.site_url, str(err)) return @@ -189,8 +212,18 @@ def make_soup(self, url): soup = BeautifulSoup(response.content, "html.parser") return soup - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' - return {"id": entry["id"], "type": "add", "fields": entry} + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": "index", + "_type": "type", + "_id": "id" + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/clinical_officers.py b/healthtools/scrapers/clinical_officers.py index 3cbbe53..c9766c4 100644 --- a/healthtools/scrapers/clinical_officers.py +++ b/healthtools/scrapers/clinical_officers.py @@ -1,7 +1,6 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS +from healthtools.config import SITES, ES from datetime import datetime -import boto3 class ClinicalOfficersScraper(Scraper): @@ -15,24 +14,30 @@ def __init__(self): self.fields = [ "name", "reg_date", "reg_no", "valid_dates", "address", "qualifications", "id", - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_cos_endpoint"] - }) + ] self.s3_key = "data/clinical_officers.json" self.s3_historical_record_key = "data/archive/clinical_officers-{}.json" self.delete_file = "data/delete_clinical_officers.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' - date_obj = datetime.strptime(entry['reg_date'], "%d-%m-%y %H:%M") + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ + try: + date_obj = datetime.strptime(entry['reg_date'], "%Y-%m-%d") + except: + date_obj = datetime.strptime(entry['reg_date'], "%d-%m-%Y") entry['reg_date'] = datetime.strftime( date_obj, "%Y-%m-%dT%H:%M:%S.000Z") - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": ES['cos_type'], + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/doctors.py b/healthtools/scrapers/doctors.py index fce7bb6..dcb3199 100644 --- a/healthtools/scrapers/doctors.py +++ b/healthtools/scrapers/doctors.py @@ -1,7 +1,6 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS +from healthtools.config import SITES, ES from datetime import datetime -import boto3 class DoctorsScraper(Scraper): @@ -15,23 +14,18 @@ def __init__(self): self.fields = [ "name", "reg_date", "reg_no", "postal_address", "qualifications", "speciality", "sub_speciality", "id", - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_doctors_endpoint"] - }) + ] self.s3_key = "data/doctors.json" self.s3_historical_record_key = "data/archive/doctors-{}.json" self.delete_file = "data/delete_doctors.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ try: date_obj = datetime.strptime(entry['reg_date'], "%Y-%m-%d") except: @@ -39,4 +33,12 @@ def format_for_cloudsearch(self, entry): entry['reg_date'] = datetime.strftime( date_obj, "%Y-%m-%dT%H:%M:%S.000Z") entry["facility"] = entry["practice_type"] = "-" - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": ES['doctors_type'], + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/foreign_doctors.py b/healthtools/scrapers/foreign_doctors.py index babf48f..89347f3 100644 --- a/healthtools/scrapers/foreign_doctors.py +++ b/healthtools/scrapers/foreign_doctors.py @@ -1,6 +1,5 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS -import boto3 +from healthtools.config import SITES, ES class ForeignDoctorsScraper(Scraper): @@ -14,22 +13,25 @@ def __init__(self): self.fields = [ "name", "reg_no", "postal_address", "qualifications", "facility", "practice_type", "id" - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_doctors_endpoint"] - }) + ] self.s3_key = "data/foreign_doctors.json" self.s3_historical_record_key = "data/archive/foreign_doctors-{}.json" self.delete_file = "data/delete_foreign_doctors.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ entry["reg_date"] = "0000-01-01T00:00:00.000Z" entry["reg_no"] = entry["speciality"] = entry["sub_speciality"] = "-" - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": ES['doctors_type'], + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/tests/dummy_files/clinical_officers.json b/healthtools/tests/dummy_files/clinical_officers.json index 6910ab6..b316910 100644 --- a/healthtools/tests/dummy_files/clinical_officers.json +++ b/healthtools/tests/dummy_files/clinical_officers.json @@ -1 +1 @@ -[{"fields": {"name": "JACOB KILIMO KISANG", "reg_no": "1 / Ra00302/17", "valid_dates": "2023-01-17 / 2031-01-19", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "PO BOX 1881", "id": 0}, "type": "add", "id": 0}, {"fields": {"name": "FRANCIS KINGE MATHERI", "reg_no": "18 / Ra01477/16", "valid_dates": "2030-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-07-08T00:00:00.000Z", "address": "P.O BOX 656 KIAMBU", "id": 1}, "type": "add", "id": 1}, {"fields": {"name": "GLADYS OKAKAH KOYENGO", "reg_no": "19 / Ra00811/15", "valid_dates": "2018-11-15 / 2030-11-17", "qualifications": "DIP", "reg_date": "1989-04-26T00:00:00.000Z", "address": "P.O BOX 20715 NAIROBI", "id": 2}, "type": "add", "id": 2}, {"fields": {"name": "STEPHEN OKEYO ABEBE", "reg_no": "20 / R02062/16", "valid_dates": "2018-08-16 / 2031-08-18", "qualifications": "DIP", "reg_date": "1989-03-16T00:00:00.000Z", "address": "P.O. BOX 1 KADONGO", "id": 3}, "type": "add", "id": 3}, {"fields": {"name": "JOSEPH OGEKA MAORE", "reg_no": "30 / Ra01287/16", "valid_dates": "2014-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O BOX 153 KISII", "id": 4}, "type": "add", "id": 4}, {"fields": {"name": "HAMISI ALI FAJIN", "reg_no": "37 / Ra01032/15", "valid_dates": "2008-12-15 / 2031-12-17", "qualifications": "DIP", "reg_date": "1989-01-10T00:00:00.000Z", "address": "P.O BOX 57 MALINDI", "id": 5}, "type": "add", "id": 5}, {"fields": {"name": "GEORGE AYIEKO JOWI", "reg_no": "40 / Ra01823/16", "valid_dates": "2012-05-16 / 2031-05-18", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O. BOX 538 MTWAPA", "id": 6}, "type": "add", "id": 6}, {"fields": {"name": "STEPHEN KIPLANGAT KITTUR", "reg_no": "48 / R02318/15", "valid_dates": "2013-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1987-08-25T00:00:00.000Z", "address": "P.O BOX 5036 ELDORET", "id": 7}, "type": "add", "id": 7}, {"fields": {"name": "ROBERT MUNGE KIMUNDUU", "reg_no": "54 / R02261/15", "valid_dates": "2013-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1988-02-02T00:00:00.000Z", "address": "P.O BOX 304 MACHAKOS", "id": 8}, "type": "add", "id": 8}, {"fields": {"name": "CHRISANTHUS MAKORI ORINA", "reg_no": "55 / Ra00375/15", "valid_dates": "2029-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 3685 KISII", "id": 9}, "type": "add", "id": 9}, {"fields": {"name": "EZEKIEL WALUFU WALUFU", "reg_no": "76 / R02849/15", "valid_dates": "2021-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1987-03-19T00:00:00.000Z", "address": "P.O BOX 802 MUMIAS", "id": 10}, "type": "add", "id": 10}, {"fields": {"name": "ELIKANAH KEBAGENDI OMWENGA", "reg_no": "78 / Ra00147/15", "valid_dates": "2001-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-08-16T00:00:00.000Z", "address": "P.O BOX 252 80100 MOMBASA", "id": 11}, "type": "add", "id": 11}, {"fields": {"name": "DANIEL OMBACHI MOGENI", "reg_no": "97 / Ra00643/16", "valid_dates": "2003-02-16 / 2028-02-18", "qualifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 235 NYAMIRA", "id": 12}, "type": "add", "id": 12}, {"fields": {"name": "BENSON GICHUHI NG\\'ERI", "reg_no": "107 / Ra00293/15", "valid_dates": "2015-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 15383 NAKURU", "id": 13}, "type": "add", "id": 13}, {"fields": {"name": "ANDIVA HERMAN KAGHALI", "reg_no": "112 / Ra00758/16", "valid_dates": "2009-02-16 / 2028-02-18", "qualifications": "DIP", "reg_date": "1989-05-12T00:00:00.000Z", "address": "P.O. BOX 119 50309 TIRIKI", "id": 14}, "type": "add", "id": 14}, {"fields": {"name": "DABASO GARAMBODA JILLO", "reg_no": "121 / Ra00245/17", "valid_dates": "2019-01-17 / 2031-01-19", "qualifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O BOX 753 GARISSA", "id": 15}, "type": "add", "id": 15}, {"fields": {"name": "SIYAD MOHAMED OMAR", "reg_no": "122 / Ra02079/16", "valid_dates": "2015-06-16 / 2030-06-18", "qualifications": "DIP", "reg_date": "1989-01-06T00:00:00.000Z", "address": "P.O BOX 253 GARISSA", "id": 16}, "type": "add", "id": 16}, {"fields": {"name": "DAVID MAINA NJURU", "reg_no": "125 / Ra02390/16", "valid_dates": "2019-07-16 / 2031-07-18", "qualifications": "DIP", "reg_date": "1989-10-12T00:00:00.000Z", "address": "P.O BOX 20435 00200 NAIROBI", "id": 17}, "type": "add", "id": 17}, {"fields": {"name": "ALLEN PETERSON MBELA", "reg_no": "138 / Ra00187/16", "valid_dates": "2013-01-16 / 2031-01-18", "qualifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O. BOX 47 MWATATE", "id": 18}, "type": "add", "id": 18}, {"fields": {"name": "LINCOLN MURAI NJUGUNA", "reg_no": "139 / Ra00572/15", "valid_dates": "2001-11-15 / 2030-11-17", "qualifications": "DIP", "reg_date": "1988-02-12T00:00:00.000Z", "address": "P.O BOX 125 KILGORIS", "id": 19}, "type": "add", "id": 19}, {"fields": {"name": "KITHUSI PETER KIITI", "reg_no": "141 / Ra01441/16", "valid_dates": "2024-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O BOX 1473 KANGUNDO", "id": 20}, "type": "add", "id": 20}, {"fields": {"name": "MICHAEL GITARI KAMAU", "reg_no": "147 / R03501/15", "valid_dates": "2016-07-15 / 2031-07-17", "qualifications": "DIP", "reg_date": "1989-01-20T00:00:00.000Z", "address": "P.O BOX 701 THIKA", "id": 21}, "type": "add", "id": 21}, {"fields": {"name": "JAMES NYORO WAMWEA", "reg_no": "163 / Ra01474/16", "valid_dates": "2029-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2055-03-15T00:00:00.000Z", "address": "P.O BOX 265 LIMURU", "id": 22}, "type": "add", "id": 22}, {"fields": {"name": "GITHAIGA NJUMA ONESMUS", "reg_no": "168 / Ra01203/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-01-25T00:00:00.000Z", "address": "P.O. BOX 171 MUKURWEINI", "id": 23}, "type": "add", "id": 23}, {"fields": {"name": "JUSTUS KARIGU KIBARUA", "reg_no": "171 / Ra01199/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2001-01-31T10:46:00.000Z", "address": "P.O. BOX 2 GIAKANJA", "id": 24}, "type": "add", "id": 24}, {"fields": {"name": "KILLIAN ANSELM MWOLOI", "reg_no": "178 / Ra00200/16", "valid_dates": "2013-01-16 / 2031-01-18", "qualifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O. BOX 152 KIBWEZI", "id": 25}, "type": "add", "id": 25}, {"fields": {"name": "CHARLES GATHUMA BIRINGI", "reg_no": "182 / Ra01202/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2011-03-15T11:34:00.000Z", "address": "P.O BOX 920 NYERI", "id": 26}, "type": "add", "id": 26}, {"fields": {"name": "ANTONY KITSAO CHIRAHU", "reg_no": "198 / Ra00540/15", "valid_dates": "2021-10-15 / 2031-10-17", "qualifications": "DIP", "reg_date": "1989-01-31T00:00:00.000Z", "address": "P.O BOX 49 MOMBASA", "id": 27}, "type": "add", "id": 27}, {"fields": {"name": "JULIUS MUTAMBUKI KILIKU", "reg_no": "206 / Ra01512/16", "valid_dates": "2001-05-16 / 2031-05-18", "qualifications": "DIP", "reg_date": "2011-03-17T14:20:00.000Z", "address": "P.O BOX 350 90119 MATUU", "id": 28}, "type": "add", "id": 28}, {"fields": {"name": "STEPHEN ODUOR JURE", "reg_no": "207 / Ra00326/15", "valid_dates": "2018-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-07-12T00:00:00.000Z", "address": "P.O BOX 370 KISUMU", "id": 29}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "clinical-officers", "_id": 0, "_index": "healthtools"}}, {"name": "JACOB KILIMO KISANG", "reg_no": "1 / Ra00302/17", "valid_dates": "2017-01-23 / 2019-01-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "PO BOX 1881", "id": 0}, {"index": {"_type": "clinical-officers", "_id": 1, "_index": "healthtools"}}, {"name": "ISAAC NNDA NYABAYO", "reg_no": "2 / Ra02803/17", "valid_dates": "2017-05-31 / 2019-05-31", "qalifications": "DIP", "reg_date": "2011-04-28T00:00:00.000Z", "address": "P.O BOX 456 KISII", "id": 1}, {"index": {"_type": "clinical-officers", "_id": 2, "_index": "healthtools"}}, {"name": "FRANCIS KINGE MATHERI", "reg_no": "18 / Ra01477/16", "valid_dates": "2016-03-30 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-07-08T00:00:00.000Z", "address": "P.O BOX 656 KIAMB", "id": 2}, {"index": {"_type": "clinical-officers", "_id": 3, "_index": "healthtools"}}, {"name": "GLADYS OKAKAH KOYENGO", "reg_no": "19 / Ra00811/15", "valid_dates": "2015-11-18 / 2017-11-30", "qalifications": "DIP", "reg_date": "1989-04-26T00:00:00.000Z", "address": "P.O BOX 20715 NAIROBI", "id": 3}, {"index": {"_type": "clinical-officers", "_id": 4, "_index": "healthtools"}}, {"name": "STEPHEN OKEYO ABEBE", "reg_no": "20 / R02062/16", "valid_dates": "2016-08-18 / 2018-08-31", "qalifications": "DIP", "reg_date": "1989-03-16T00:00:00.000Z", "address": "P.O. BOX 1 KADONGO", "id": 4}, {"index": {"_type": "clinical-officers", "_id": 5, "_index": "healthtools"}}, {"name": "DANIEL ONGWENYI MOGAMBI", "reg_no": "22 / R01412/17", "valid_dates": "2017-05-09 / 2019-05-31", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O BOX 4315 ELDORET", "id": 5}, {"index": {"_type": "clinical-officers", "_id": 6, "_index": "healthtools"}}, {"name": "JOSEPH OGEKA MAORE", "reg_no": "30 / Ra01287/16", "valid_dates": "2016-03-14 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O BOX 153 KISII", "id": 6}, {"index": {"_type": "clinical-officers", "_id": 7, "_index": "healthtools"}}, {"name": "HAMISI ALI FAJIN", "reg_no": "37 / Ra01032/15", "valid_dates": "2015-12-08 / 2017-12-31", "qalifications": "DIP", "reg_date": "1989-01-10T00:00:00.000Z", "address": "P.O BOX 57 MALINDI", "id": 7}, {"index": {"_type": "clinical-officers", "_id": 8, "_index": "healthtools"}}, {"name": "GEORGE AYIEKO JOWI", "reg_no": "40 / Ra01823/16", "valid_dates": "2016-05-12 / 2018-05-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O. BOX 538 MTWAPA", "id": 8}, {"index": {"_type": "clinical-officers", "_id": 9, "_index": "healthtools"}}, {"name": "STEPHEN KIPLAGAT KITTR", "reg_no": "48 / Ra02696/17", "valid_dates": "2017-05-23 / 2019-05-31", "qalifications": "DIP", "reg_date": "1987-08-25T00:00:00.000Z", "address": "P.O BOX 5036 ELDORET", "id": 9}, {"index": {"_type": "clinical-officers", "_id": 10, "_index": "healthtools"}}, {"name": "ROBERT MNGE KIMND", "reg_no": "54 / Ra02229/17", "valid_dates": "2017-05-01 / 2019-05-31", "qalifications": "DIP", "reg_date": "1988-02-02T00:00:00.000Z", "address": "P.O BOX 304 MACHAKOS", "id": 10}, {"index": {"_type": "clinical-officers", "_id": 11, "_index": "healthtools"}}, {"name": "CHRISANTHS MAKORI ORINA", "reg_no": "55 / Ra00375/15", "valid_dates": "2015-09-29 / 2017-09-30", "qalifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 3685 KISII", "id": 11}, {"index": {"_type": "clinical-officers", "_id": 12, "_index": "healthtools"}}, {"name": "EZEKIEL WALF WALF", "reg_no": "76 / Ra02666/17", "valid_dates": "2017-05-19 / 2019-05-31", "qalifications": "DIP", "reg_date": "1987-03-19T00:00:00.000Z", "address": "P.O BOX 802 MMIAS", "id": 12}, {"index": {"_type": "clinical-officers", "_id": 13, "_index": "healthtools"}}, {"name": "ELIKANAH KEBAGENDI OMWENGA", "reg_no": "78 / Ra00147/15", "valid_dates": "2015-09-01 / 2017-09-30", "qalifications": "DIP", "reg_date": "1989-08-16T00:00:00.000Z", "address": "P.O BOX 252 80100 MOMBASA", "id": 13}, {"index": {"_type": "clinical-officers", "_id": 14, "_index": "healthtools"}}, {"name": "EVANS NGK HEZRON", "reg_no": "87 / Ra02613/17", "valid_dates": "2017-05-16 / 2019-05-31", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 14907 NAKR", "id": 14}, {"index": {"_type": "clinical-officers", "_id": 15, "_index": "healthtools"}}, {"name": "DANIEL OMBACHI MOGENI", "reg_no": "97 / Ra00643/16", "valid_dates": "2016-02-03 / 2018-02-28", "qalifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 235 NYAMIRA", "id": 15}, {"index": {"_type": "clinical-officers", "_id": 16, "_index": "healthtools"}}, {"name": "BENSON GICHHI NG'ERI", "reg_no": "107 / Ra00293/15", "valid_dates": "2015-09-15 / 2017-09-30", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 15383 NAKR", "id": 16}, {"index": {"_type": "clinical-officers", "_id": 17, "_index": "healthtools"}}, {"name": "ANDIVA HERMAN KAGHALI", "reg_no": "112 / Ra00758/16", "valid_dates": "2016-02-09 / 2018-02-28", "qalifications": "DIP", "reg_date": "1989-05-12T00:00:00.000Z", "address": "P.O. BOX 119 50309 TIRIKI", "id": 17}, {"index": {"_type": "clinical-officers", "_id": 18, "_index": "healthtools"}}, {"name": "DABASO GARAMBODA JILLO", "reg_no": "121 / Ra00245/17", "valid_dates": "2017-01-19 / 2019-01-31", "qalifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O BOX 753 GARISSA", "id": 18}, {"index": {"_type": "clinical-officers", "_id": 19, "_index": "healthtools"}}, {"name": "SIYAD MOHAMED OMAR", "reg_no": "122 / Ra02079/16", "valid_dates": "2016-06-15 / 2018-06-30", "qalifications": "DIP", "reg_date": "1989-01-06T00:00:00.000Z", "address": "P.O BOX 253 GARISSA", "id": 19}, {"index": {"_type": "clinical-officers", "_id": 20, "_index": "healthtools"}}, {"name": "DAVID MAINA NJR", "reg_no": "125 / Ra02390/16", "valid_dates": "2016-07-19 / 2018-07-31", "qalifications": "DIP", "reg_date": "1989-10-12T00:00:00.000Z", "address": "P.O BOX 20435 00200 NAIROBI", "id": 20}, {"index": {"_type": "clinical-officers", "_id": 21, "_index": "healthtools"}}, {"name": "ALLEN PETERSON MBELA", "reg_no": "138 / Ra00187/16", "valid_dates": "2016-01-13 / 2018-01-31", "qalifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O. BOX 47 MWATATE", "id": 21}, {"index": {"_type": "clinical-officers", "_id": 22, "_index": "healthtools"}}, {"name": "LINCOLN MRAI NJGNA", "reg_no": "139 / Ra00572/15", "valid_dates": "2015-11-01 / 2017-11-30", "qalifications": "DIP", "reg_date": "1988-02-12T00:00:00.000Z", "address": "P.O BOX 125 KILGORIS", "id": 22}, {"index": {"_type": "clinical-officers", "_id": 23, "_index": "healthtools"}}, {"name": "KITHSI PETER KIITI", "reg_no": "141 / Ra01441/16", "valid_dates": "2016-03-24 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O BOX 1473 KANGNDO", "id": 23}, {"index": {"_type": "clinical-officers", "_id": 24, "_index": "healthtools"}}, {"name": "MICHAEL GITARI KAMA", "reg_no": "147 / R03501/15", "valid_dates": "2015-07-16 / 2017-07-31", "qalifications": "DIP", "reg_date": "1989-01-20T00:00:00.000Z", "address": "P.O BOX 701 THIKA", "id": 24}, {"index": {"_type": "clinical-officers", "_id": 25, "_index": "healthtools"}}, {"name": "JSTS MEME MGWIKA", "reg_no": "152 / Ra02087/17", "valid_dates": "2017-03-31 / 2019-03-31", "qalifications": "DIP", "reg_date": "1989-01-22T00:00:00.000Z", "address": "P.O. BOX 1681 MER", "id": 25}, {"index": {"_type": "clinical-officers", "_id": 26, "_index": "healthtools"}}, {"name": "JAMES NYORO WAMWEA", "reg_no": "163 / Ra01474/16", "valid_dates": "2016-03-29 / 2018-03-31", "qalifications": "DIP", "reg_date": "1955-03-15T00:00:00.000Z", "address": "P.O BOX 265 LIMR", "id": 26}, {"index": {"_type": "clinical-officers", "_id": 27, "_index": "healthtools"}}, {"name": "GITHAIGA NJMA ONESMS", "reg_no": "168 / Ra01203/16", "valid_dates": "2016-03-08 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-01-25T00:00:00.000Z", "address": "P.O. BOX 171 MKRWEINI", "id": 27}, {"index": {"_type": "clinical-officers", "_id": 28, "_index": "healthtools"}}, {"name": "JSTS KARIG KIBARA", "reg_no": "171 / Ra01199/16", "valid_dates": "2016-03-08 / 2018-03-31", "qalifications": "DIP", "reg_date": "2001-01-31T00:00:00.000Z", "address": "P.O. BOX 2 GIAKANJA", "id": 28}, {"index": {"_type": "clinical-officers", "_id": 29, "_index": "healthtools"}}, {"name": "KILLIAN ANSELM MWOLOI", "reg_no": "178 / Ra00200/16", "valid_dates": "2016-01-13 / 2018-01-31", "qalifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O. BOX 152 KIBWEZI", "id": 29}] diff --git a/healthtools/tests/dummy_files/doctors.json b/healthtools/tests/dummy_files/doctors.json index 7d3dd66..b042c2d 100644 --- a/healthtools/tests/dummy_files/doctors.json +++ b/healthtools/tests/dummy_files/doctors.json @@ -1 +1 @@ -[{"fields": {"name": "Dr. DEVANI JAYENDRA KUMAR", "postal_address": "P.O.Box 43376-00100 NAIROBI", "facility": "-", "reg_no": "B021", "practice_type": "-", "qualifications": "BDS(Bombay) 1966 LDS RCS(Edin) 1968", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 0, "speciality": "NONE"}, "type": "add", "id": 0}, {"fields": {"name": "Dr. VARSHANI NARAN SHIVJI", "postal_address": "P.O.Box 75977-00200 NAIROBI", "facility": "-", "reg_no": "B025", "practice_type": "-", "qualifications": "BDS(Gujarat) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 1, "speciality": "NONE"}, "type": "add", "id": 1}, {"fields": {"name": "Dr. NATHWANI JITENDRA PRABHUDAS", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B029", "practice_type": "-", "qualifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 2, "speciality": "NONE"}, "type": "add", "id": 2}, {"fields": {"name": "Dr. SHAH RASIK RAISHI", "postal_address": "P.O.Box 31597-00600 NAIROBI", "facility": "-", "reg_no": "B033", "practice_type": "-", "qualifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 3, "speciality": "NONE"}, "type": "add", "id": 3}, {"fields": {"name": "Dr. VERJEE NIZAR JAFFERALI", "postal_address": "P.O.Box 59745-00200 NAIROBI", "facility": "-", "reg_no": "B037", "practice_type": "-", "qualifications": "BDS(London) 1967 LDS RCS(Eng) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 4, "speciality": "NONE"}, "type": "add", "id": 4}, {"fields": {"name": "Dr. PATEL NIRA PANKAJ", "postal_address": "P.O.Box 14105-00800 NAIROBI", "facility": "-", "reg_no": "B049", "practice_type": "-", "qualifications": "BDS(Mysore) 1976 MRSH(london) 1977 FRSH(Lond) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 5, "speciality": "NONE"}, "type": "add", "id": 5}, {"fields": {"name": "Dr. SANGHANI PRAFUL MANGALJI", "postal_address": "P.O.Box 45071-00100 NAIROBI", "facility": "-", "reg_no": "B051", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 6, "speciality": "NONE"}, "type": "add", "id": 6}, {"fields": {"name": "Dr. SHAH RAMESH MEGHJI", "postal_address": "P.O.Box 66102-00800 NAIROBI", "facility": "-", "reg_no": "B052", "practice_type": "-", "qualifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 7, "speciality": "NONE"}, "type": "add", "id": 7}, {"fields": {"name": "Dr. SHAH AMRITLAL PREMCHAND", "postal_address": "P.O.Box 14191-00800 NAIROBI", "facility": "-", "reg_no": "B059", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 8, "speciality": "NONE"}, "type": "add", "id": 8}, {"fields": {"name": "Dr. KANYI EVA CHAMKOVA", "postal_address": "P.O.Box 727-10100 NYERI", "facility": "-", "reg_no": "B065", "practice_type": "-", "qualifications": "MD Stomatology (Komensky) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 9, "speciality": "NONE"}, "type": "add", "id": 9}, {"fields": {"name": "Dr. JANDU PARVIN SINGH", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B071", "practice_type": "-", "qualifications": "BDS(Bombay) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 10, "speciality": "NONE"}, "type": "add", "id": 10}, {"fields": {"name": "Dr. PATEL CHANDRAKANT BHAILALBHAI", "postal_address": "P.O.Box 81111-80100 MOMBASA", "facility": "-", "reg_no": "B072", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 11, "speciality": "NONE"}, "type": "add", "id": 11}, {"fields": {"name": "Dr. DESAI ASHOK NARENDRA", "postal_address": "P.O.Box 47605-00100 NAIROBI", "facility": "-", "reg_no": "B075", "practice_type": "-", "qualifications": "BDS(Dundee) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 12, "speciality": "NONE"}, "type": "add", "id": 12}, {"fields": {"name": "Dr. DESAI PARIMAL VITHALBHAI", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B080", "practice_type": "-", "qualifications": "BDS(Bombay) 1971", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 13, "speciality": "NONE"}, "type": "add", "id": 13}, {"fields": {"name": "Dr. PATEL MANDAKINI JAYENDRA", "postal_address": "P.O.Box 49545-00100 NAIROBI", "facility": "-", "reg_no": "B081", "practice_type": "-", "qualifications": "BDS(Bombay) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 14, "speciality": "NONE"}, "type": "add", "id": 14}, {"fields": {"name": "Dr. PATEL JAIMAN CHUNIBHAI", "postal_address": "P.O.Box 84420-80100 MOMBASA", "facility": "-", "reg_no": "B085", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 15, "speciality": "NONE"}, "type": "add", "id": 15}, {"fields": {"name": "Dr. GITATA MUTHONI GITHUNGO", "postal_address": "P.O.Box 1769-00100 NAIROBI", "facility": "-", "reg_no": "B086", "practice_type": "-", "qualifications": "DDS(Nashville) 1973", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 16, "speciality": "NONE"}, "type": "add", "id": 16}, {"fields": {"name": "Dr. MAGON SWARNLATA", "postal_address": "P.O.Box 81431-80100 MOMBASA", "facility": "-", "reg_no": "B098", "practice_type": "-", "qualifications": "BDS(Punjabi) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 17, "speciality": "NONE"}, "type": "add", "id": 17}, {"fields": {"name": "Dr. PATEL PANKAJKUMAR RAOJIBHAI", "postal_address": "P.O.Box 43917-00100 NAIROBI", "facility": "-", "reg_no": "B114", "practice_type": "-", "qualifications": "BDS(Bombay) 1974 MRSH(London) 1975 FRSH(London) 1981 FADI(U.S.A) 1983", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 18, "speciality": "NONE"}, "type": "add", "id": 18}, {"fields": {"name": "Dr. NATHWANI BAKULA JITENDRA", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B118", "practice_type": "-", "qualifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 19, "speciality": "NONE"}, "type": "add", "id": 19}, {"fields": {"name": "Dr. WAGAIYU CHRISTOPHER KIBURI GICHURU", "postal_address": "P.O.Box 54197-00200 NAIROBI", "facility": "-", "reg_no": "B123", "practice_type": "-", "qualifications": "BDS(London) 1977 MSc(London) 1980 FDS.FRCS(Edin) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 20, "speciality": "CONSERVATIVE DENTISTRY"}, "type": "add", "id": 20}, {"fields": {"name": "Dr. VAGHELA SATISH MAGANLAL", "postal_address": "P.O.Box 80198-80100 MOMBASA", "facility": "-", "reg_no": "B143", "practice_type": "-", "qualifications": "BDS(Gujarat) 1978", "reg_date": "1979-11-28T00:00:00.000Z", "sub_speciality": "NONE", "id": 21, "speciality": "NONE"}, "type": "add", "id": 21}, {"fields": {"name": "Dr. NJINO MICHAEL", "postal_address": "P.O.Box 13166-00200 NAIROBI", "facility": "-", "reg_no": "B144", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 22, "speciality": "NONE"}, "type": "add", "id": 22}, {"fields": {"name": "Dr. NDUNGU MICHAEL J.K.", "postal_address": "P.O.Box 47263-00100 NAIROBI", "facility": "-", "reg_no": "B145", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 23, "speciality": "NONE"}, "type": "add", "id": 23}, {"fields": {"name": "Dr. KIBUGI EDWIN WAMBUGU", "postal_address": "P.O.Box 54193-00200 NAIROBI", "facility": "-", "reg_no": "B150", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978 MSc(London) 1983", "reg_date": "1980-02-29T00:00:00.000Z", "sub_speciality": "NONE", "id": 24, "speciality": "PROSTHODONTICS"}, "type": "add", "id": 24}, {"fields": {"name": "Dr. GRIFFITHS PETER DAVID", "postal_address": "P.O.Box 25672-00603 NAIROBI", "facility": "-", "reg_no": "B153", "practice_type": "-", "qualifications": "BDS(Birmingham) 1979", "reg_date": "1980-05-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 25, "speciality": "NONE"}, "type": "add", "id": 25}, {"fields": {"name": "Dr. AWORI MARTIN WANDERA", "postal_address": "P.O.Box 19630-00202 NAIROBI", "facility": "-", "reg_no": "B168", "practice_type": "-", "qualifications": "BDS(Kerala) 1979 DDPH RCS(England) 1983 MIHE(UK) 1983", "reg_date": "1981-03-31T00:00:00.000Z", "sub_speciality": "NONE", "id": 26, "speciality": "PUBLIC HEALTH"}, "type": "add", "id": 26}, {"fields": {"name": "Dr. MUMENYA ROBINSON KOGI WAMBUGU", "postal_address": "P.O.Box 19320-00202 NAIROBI", "facility": "-", "reg_no": "B174", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979 FDS RCS(England) 1991 MSc(Oral Surgery)(London) 1991", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 27, "speciality": "ORAL AND MAXILLOFACIAL SURGERY"}, "type": "add", "id": 27}, {"fields": {"name": "Dr. JANI SAILESH GUNVANTRY", "postal_address": "P.O.Box 45640-00100 NAIROBI", "facility": "-", "reg_no": "B175", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 28, "speciality": "NONE"}, "type": "add", "id": 28}, {"fields": {"name": "Dr. CHINDIA MARK LUBISIA", "postal_address": "P.O.Box 19676-00202 NAIROBI", "facility": "-", "reg_no": "B176", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979 FFDFCSI(Ireland) 1983 MSc(London) 1989", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 29, "speciality": "ORAL AND MAXILLOFACIAL SURGERY"}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "doctors", "_id": 0, "_index": "healthtools"}}, {"name": "Dr. DEVANI JAYENDRA KMAR", "postal_address": "P.O.Box 43376-00100 NAIROBI", "facility": "-", "reg_no": "B021", "practice_type": "-", "qalifications": "BDS(Bombay) 1966 LDS RCS(Edin) 1968", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 0, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 1, "_index": "healthtools"}}, {"name": "Dr. VARSHANI NARAN SHIVJI", "postal_address": "P.O.Box 75977-00200 NAIROBI", "facility": "-", "reg_no": "B025", "practice_type": "-", "qalifications": "BDS(Gjarat) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 1, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 2, "_index": "healthtools"}}, {"name": "Dr. NATHWANI JITENDRA PRABHDAS", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B029", "practice_type": "-", "qalifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 2, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 3, "_index": "healthtools"}}, {"name": "Dr. SHAH RASIK RAISHI", "postal_address": "P.O.Box 31597-00600 NAIROBI", "facility": "-", "reg_no": "B033", "practice_type": "-", "qalifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 3, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 4, "_index": "healthtools"}}, {"name": "Dr. VERJEE NIZAR JAFFERALI", "postal_address": "P.O.Box 59745-00200 NAIROBI", "facility": "-", "reg_no": "B037", "practice_type": "-", "qalifications": "BDS(London) 1967 LDS RCS(Eng) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 4, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 5, "_index": "healthtools"}}, {"name": "Dr. PATEL NIRA PANKAJ", "postal_address": "P.O.Box 14105-00800 NAIROBI", "facility": "-", "reg_no": "B049", "practice_type": "-", "qalifications": "BDS(Mysore) 1976 MRSH(london) 1977 FRSH(Lond) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 5, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 6, "_index": "healthtools"}}, {"name": "Dr. SANGHANI PRAFL MANGALJI", "postal_address": "P.O.Box 45071-00100 NAIROBI", "facility": "-", "reg_no": "B051", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 6, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 7, "_index": "healthtools"}}, {"name": "Dr. SHAH RAMESH MEGHJI", "postal_address": "P.O.Box 66102-00800 NAIROBI", "facility": "-", "reg_no": "B052", "practice_type": "-", "qalifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 7, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 8, "_index": "healthtools"}}, {"name": "Dr. SHAH AMRITLAL PREMCHAND", "postal_address": "P.O.Box 14191-00800 NAIROBI", "facility": "-", "reg_no": "B059", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 8, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 9, "_index": "healthtools"}}, {"name": "Dr. KANYI EVA CHAMKOVA", "postal_address": "P.O.Box 727-10100 NYERI", "facility": "-", "reg_no": "B065", "practice_type": "-", "qalifications": "MD Stomatology (Komensky) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 9, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 10, "_index": "healthtools"}}, {"name": "Dr. JAND PARVIN SINGH", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B071", "practice_type": "-", "qalifications": "BDS(Bombay) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 10, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 11, "_index": "healthtools"}}, {"name": "Dr. PATEL CHANDRAKANT BHAILALBHAI", "postal_address": "P.O.Box 81111-80100 MOMBASA", "facility": "-", "reg_no": "B072", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 11, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 12, "_index": "healthtools"}}, {"name": "Dr. DESAI ASHOK NARENDRA", "postal_address": "P.O.Box 47605-00100 NAIROBI", "facility": "-", "reg_no": "B075", "practice_type": "-", "qalifications": "BDS(Dndee) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 12, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 13, "_index": "healthtools"}}, {"name": "Dr. DESAI PARIMAL VITHALBHAI", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B080", "practice_type": "-", "qalifications": "BDS(Bombay) 1971", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 13, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 14, "_index": "healthtools"}}, {"name": "Dr. PATEL MANDAKINI JAYENDRA", "postal_address": "P.O.Box 49545-00100 NAIROBI", "facility": "-", "reg_no": "B081", "practice_type": "-", "qalifications": "BDS(Bombay) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 14, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 15, "_index": "healthtools"}}, {"name": "Dr. PATEL JAIMAN CHNIBHAI", "postal_address": "P.O.Box 84420-80100 MOMBASA", "facility": "-", "reg_no": "B085", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 15, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 16, "_index": "healthtools"}}, {"name": "Dr. GITATA MTHONI GITHNGO", "postal_address": "P.O.Box 1769-00100 NAIROBI", "facility": "-", "reg_no": "B086", "practice_type": "-", "qalifications": "DDS(Nashville) 1973", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 16, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 17, "_index": "healthtools"}}, {"name": "Dr. MAGON SWARNLATA", "postal_address": "P.O.Box 81431-80100 MOMBASA", "facility": "-", "reg_no": "B098", "practice_type": "-", "qalifications": "BDS(Pnjabi) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 17, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 18, "_index": "healthtools"}}, {"name": "Dr. PATEL PANKAJKMAR RAOJIBHAI", "postal_address": "P.O.Box 43917-00100 NAIROBI", "facility": "-", "reg_no": "B114", "practice_type": "-", "qalifications": "BDS(Bombay) 1974 MRSH(London) 1975 FRSH(London) 1981 FADI(.S.A) 1983", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 18, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 19, "_index": "healthtools"}}, {"name": "Dr. NATHWANI BAKLA JITENDRA", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B118", "practice_type": "-", "qalifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 19, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 20, "_index": "healthtools"}}, {"name": "Dr. WAGAIY CHRISTOPHER KIBRI GICHR", "postal_address": "P.O.Box 54197-00200 NAIROBI", "facility": "-", "reg_no": "B123", "practice_type": "-", "qalifications": "BDS(London) 1977 MSc(London) 1980 FDS.FRCS(Edin) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 20, "speciality": "CONSERVATIVE DENTISTRY"}, {"index": {"_type": "doctors", "_id": 21, "_index": "healthtools"}}, {"name": "Dr. VAGHELA SATISH MAGANLAL", "postal_address": "P.O.Box 80198-80100 MOMBASA", "facility": "-", "reg_no": "B143", "practice_type": "-", "qalifications": "BDS(Gjarat) 1978", "reg_date": "1979-11-28T00:00:00.000Z", "sb_speciality": "NONE", "id": 21, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 22, "_index": "healthtools"}}, {"name": "Dr. NJINO MICHAEL", "postal_address": "P.O.Box 13166-00200 NAIROBI", "facility": "-", "reg_no": "B144", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 22, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 23, "_index": "healthtools"}}, {"name": "Dr. NDNG MICHAEL J.K.", "postal_address": "P.O.Box 47263-00100 NAIROBI", "facility": "-", "reg_no": "B145", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 23, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 24, "_index": "healthtools"}}, {"name": "Dr. KIBGI EDWIN WAMBG", "postal_address": "P.O.Box 54193-00200 NAIROBI", "facility": "-", "reg_no": "B150", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978 MSc(London) 1983", "reg_date": "1980-02-29T00:00:00.000Z", "sb_speciality": "NONE", "id": 24, "speciality": "PROSTHODONTICS"}, {"index": {"_type": "doctors", "_id": 25, "_index": "healthtools"}}, {"name": "Dr. GRIFFITHS PETER DAVID", "postal_address": "P.O.Box 25672-00603 NAIROBI", "facility": "-", "reg_no": "B153", "practice_type": "-", "qalifications": "BDS(Birmingham) 1979", "reg_date": "1980-05-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 25, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 26, "_index": "healthtools"}}, {"name": "Dr. AWORI MARTIN WANDERA", "postal_address": "P.O.Box 19630-00202 NAIROBI", "facility": "-", "reg_no": "B168", "practice_type": "-", "qalifications": "BDS(Kerala) 1979 DDPH RCS(England) 1983 MIHE(K) 1983", "reg_date": "1981-03-31T00:00:00.000Z", "sb_speciality": "NONE", "id": 26, "speciality": "PBLIC HEALTH"}, {"index": {"_type": "doctors", "_id": 27, "_index": "healthtools"}}, {"name": "Prof. MACIGO FRANCIS GITHA", "postal_address": "P.O.Box 2917-00100 NAIROBI", "facility": "-", "reg_no": "B173", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979 MPH(Nairobi) 1990", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 27, "speciality": "PBLIC HEALTH"}, {"index": {"_type": "doctors", "_id": 28, "_index": "healthtools"}}, {"name": "Dr. MMENYA ROBINSON KOGI WAMBG", "postal_address": "P.O.Box 19320-00202 NAIROBI", "facility": "-", "reg_no": "B174", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979 FDS RCS(England) 1991 MSc(Oral Srgery)(London) 1991", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 28, "speciality": "ORAL AND MAXILLOFACIAL SRGERY"}, {"index": {"_type": "doctors", "_id": 29, "_index": "healthtools"}}, {"name": "Dr. JANI SAILESH GNVANTRY", "postal_address": "P.O.Box 45640-00100 NAIROBI", "facility": "-", "reg_no": "B175", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 29, "speciality": "NONE"}] diff --git a/healthtools/tests/dummy_files/foreign_doctors.json b/healthtools/tests/dummy_files/foreign_doctors.json index bbcb7bd..c458e10 100644 --- a/healthtools/tests/dummy_files/foreign_doctors.json +++ b/healthtools/tests/dummy_files/foreign_doctors.json @@ -1 +1 @@ -[{"fields": {"name": "DR NARAYAN VIJAYA KUMAR", "postal_address": "P.O BOX 39173 00623 NAIROBI", "facility": "CANCER CARE KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(CALCUTTA)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 0, "speciality": "-"}, "type": "add", "id": 0}, {"fields": {"name": "DR SMITHSON HELEN CLAIRE", "postal_address": "P.O BOX 63 60600 MAUA", "facility": "MAUA METHODIST HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(NEW CASTLE)1984", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 1, "speciality": "-"}, "type": "add", "id": 1}, {"fields": {"name": "DR ADAM \u00a0MARY", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(ARIZONA)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 2, "speciality": "-"}, "type": "add", "id": 2}, {"fields": {"name": "DR AHSAN \u00a0SABBIR", "postal_address": "P.O BOX 32557 00600 NAIROBI", "facility": "TAIBA MEDICAL CENTRE", "reg_no": "-", "practice_type": "Dental", "qualifications": "BDS(DHAKA)2011", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 3, "speciality": "-"}, "type": "add", "id": 3}, {"fields": {"name": "DR AKANKSHA ARORA SHARMA", "postal_address": "P.O BOX 5587 00506 NAIROBI", "facility": "MEDICROSS LTD", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(INDORE)2007", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 4, "speciality": "-"}, "type": "add", "id": 4}, {"fields": {"name": "DR ANNA ELIZABETH MONROE", "postal_address": "P.O BOX", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(STANFORD)2009", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 5, "speciality": "-"}, "type": "add", "id": 5}, {"fields": {"name": "DR ARIMPUR MARY PAUL", "postal_address": "P.O BOX 49682 00100 NAIROBI", "facility": "NAZARETH HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(CALCUTTA)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 6, "speciality": "-"}, "type": "add", "id": 6}, {"fields": {"name": "DR ASOKAN \u00a0LINI", "postal_address": "P.O BOX 30577 00100 NAIROBI", "facility": "COLUMBIA AFRICA HEALTHCARE LTD", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(M.S.RAMAIAH)1998", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 7, "speciality": "-"}, "type": "add", "id": 7}, {"fields": {"name": "DR ATKINSON SARAH HELEN", "postal_address": "P.O BOX 230 80108 KILIFI", "facility": "KEMRI WELLCOME TRUST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(LONDON)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 8, "speciality": "-"}, "type": "add", "id": 8}, {"fields": {"name": "DR AUGUST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 9, "speciality": "-"}, "type": "add", "id": 9}, {"fields": {"name": "DR AUGUST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 10, "speciality": "-"}, "type": "add", "id": 10}, {"fields": {"name": "DR BADANO \u00a0SIMONA", "postal_address": "P.O BOX 88 20318 NORTH KINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(GENEVA)2001", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 11, "speciality": "-"}, "type": "add", "id": 11}, {"fields": {"name": "DR BAL RAJPREET KAUR", "postal_address": "P.O BOX 46206 00100 NAIROBI", "facility": "THE AGA KHAN UNIVERSITY HOSPITAL - NAIROBI", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(NORTH WESTERN)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 12, "speciality": "-"}, "type": "add", "id": 12}, {"fields": {"name": "DR BARBOSA SIERRA DIEGO", "postal_address": "P.O BOX 18 \u00a0LODWAR", "facility": "LODWAR COUNTY AND REFERRAL HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MADRID)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 13, "speciality": "-"}, "type": "add", "id": 13}, {"fields": {"name": "DR BASHIR MOHAMMED AHMED", "postal_address": "P.O BOX 530 40100 KISUMU", "facility": "AGA KHAN HOSPITAL KISUMU", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBCHB(KAMPALA)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 14, "speciality": "-"}, "type": "add", "id": 14}, {"fields": {"name": "DR BEGEMANN HERALD OTTFRIED", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(HAMBURG)1980", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 15, "speciality": "-"}, "type": "add", "id": 15}, {"fields": {"name": "DR BEKHEET TOFELES \u00a0MARIAN MORRIS", "postal_address": "P.O BOX 21570 00505 NAIROBI", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBCH(AIN SHAMS)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 16, "speciality": "-"}, "type": "add", "id": 16}, {"fields": {"name": "DR BERKLEY JAMES ALEXANDER", "postal_address": "P.O BOX 428 80108 KILIFI", "facility": "KEMRI/WELLCOME TRUST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS NEWCASTLE UPON TYNE)", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 17, "speciality": "-"}, "type": "add", "id": 17}, {"fields": {"name": "DR JORG \u00a0BERLING", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FREIBURG)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 18, "speciality": "-"}, "type": "add", "id": 18}, {"fields": {"name": "DR BHATT MEGHNA MEHUL", "postal_address": "P.O BOX 30270 00100 NAIROBI", "facility": "AGA KHAN UNIVERSITY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(DEEMED)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 19, "speciality": "-"}, "type": "add", "id": 19}, {"fields": {"name": "DR BOEDEMANN \u00a0MELANIE", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 20, "speciality": "-"}, "type": "add", "id": 20}, {"fields": {"name": "DR BOEVE \u00a0NORMAN", "postal_address": "P.O BOX 20 0220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MICHIGAN)1965", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 21, "speciality": "-"}, "type": "add", "id": 21}, {"fields": {"name": "DR JENNY \u00a0BOYD", "postal_address": "P.O BOX 39 20400 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(TENNESSEE)2003", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 22, "speciality": "-"}, "type": "add", "id": 22}, {"fields": {"name": "DR BROWN ROGER KEVIN", "postal_address": "P.O BOX 21171 00505 NAIROBI", "facility": "AFRICAN INLAND MISSION INTERNATIONAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(ILLINOIS)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 23, "speciality": "-"}, "type": "add", "id": 23}, {"fields": {"name": "DR BUDACH RUTH MAGDALENE", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE AGLICAN CHURCH OF KENYA - MOTHERS MERCY HOME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1978", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 24, "speciality": "-"}, "type": "add", "id": 24}, {"fields": {"name": "DR BURGERT STEPHEN LOUIS", "postal_address": "P.O BOX 39 \u00a0BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MAYO)1979", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 25, "speciality": "-"}, "type": "add", "id": 25}, {"fields": {"name": "DR CARBONE \u00a0MARCO", "postal_address": "P.O BOX 88 20318 NORTHKINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(GENOVA)1993", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 26, "speciality": "-"}, "type": "add", "id": 26}, {"fields": {"name": "DR CARTER ROBERT ALLEN", "postal_address": "P.O BOX 20 \u00a000220 KIJABE", "facility": "A I C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(INDIANA)1981", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 27, "speciality": "-"}, "type": "add", "id": 27}, {"fields": {"name": "DR CHANG SANG HO", "postal_address": "P.O BOX 20954 00202 NAIROBI", "facility": "CHRISTIAN MEDICAL & DENTAL ASSOCIATION OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(SEOUL)1977", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 28, "speciality": "-"}, "type": "add", "id": 28}, {"fields": {"name": "DR COLLIS FABIAN BERNARD", "postal_address": "P.O BOX 45 00902 KIKUYU", "facility": "P.C.E.A KIKUYU HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MSCH MBCHA(SAME)1999", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 29, "speciality": "-"}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "doctors", "_id": 0, "_index": "healthtools"}}, {"name": "DR NARAYAN VIJAYA KMAR", "postal_address": "P.O BOX 39173 00623 NAIROBI", "facility": "CANCER CARE KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(CALCTTA)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 0, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 1, "_index": "healthtools"}}, {"name": "DR SMITHSON HELEN CLAIRE", "postal_address": "P.O BOX 63 60600 MAA", "facility": "MAA METHODIST HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(NEW CASTLE)1984", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 1, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 2, "_index": "healthtools"}}, {"name": "DR ADAM MARY", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(ARIZONA)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 2, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 3, "_index": "healthtools"}}, {"name": "DR AHSAN SABBIR", "postal_address": "P.O BOX 32557 00600 NAIROBI", "facility": "TAIBA MEDICAL CENTRE", "reg_no": "-", "practice_type": "Dental", "qalifications": "BDS(DHAKA)2011", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 3, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 4, "_index": "healthtools"}}, {"name": "DR AKANKSHA ARORA SHARMA", "postal_address": "P.O BOX 5587 00506 NAIROBI", "facility": "MEDICROSS LTD", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(INDORE)2007", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 4, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 5, "_index": "healthtools"}}, {"name": "DR ANNA ELIZABETH MONROE", "postal_address": "P.O BOX", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(STANFORD)2009", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 5, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 6, "_index": "healthtools"}}, {"name": "DR ARIMPR MARY PAL", "postal_address": "P.O BOX 49682 00100 NAIROBI", "facility": "NAZARETH HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(CALCTTA)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 6, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 7, "_index": "healthtools"}}, {"name": "DR ASOKAN LINI", "postal_address": "P.O BOX 30577 00100 NAIROBI", "facility": "COLMBIA AFRICA HEALTHCARE LTD", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(M.S.RAMAIAH)1998", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 7, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 8, "_index": "healthtools"}}, {"name": "DR ATKINSON SARAH HELEN", "postal_address": "P.O BOX 230 80108 KILIFI", "facility": "KEMRI WELLCOME TRST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(LONDON)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 8, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 9, "_index": "healthtools"}}, {"name": "DR AGST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 9, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 10, "_index": "healthtools"}}, {"name": "DR AGST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 10, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 11, "_index": "healthtools"}}, {"name": "DR BADANO SIMONA", "postal_address": "P.O BOX 88 20318 NORTH KINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(GENEVA)2001", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 11, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 12, "_index": "healthtools"}}, {"name": "DR BAL RAJPREET KAR", "postal_address": "P.O BOX 46206 00100 NAIROBI", "facility": "THE AGA KHAN NIVERSITY HOSPITAL - NAIROBI", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(NORTH WESTERN)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 12, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 13, "_index": "healthtools"}}, {"name": "DR BARBOSA SIERRA DIEGO", "postal_address": "P.O BOX 18 LODWAR", "facility": "LODWAR CONTY AND REFERRAL HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MADRID)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 13, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 14, "_index": "healthtools"}}, {"name": "DR BASHIR MOHAMMED AHMED", "postal_address": "P.O BOX 530 40100 KISM", "facility": "AGA KHAN HOSPITAL KISM", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBCHB(KAMPALA)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 14, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 15, "_index": "healthtools"}}, {"name": "DR BEGEMANN HERALD OTTFRIED", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(HAMBRG)1980", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 15, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 16, "_index": "healthtools"}}, {"name": "DR BEKHEET TOFELES MARIAN MORRIS", "postal_address": "P.O BOX 21570 00505 NAIROBI", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBCH(AIN SHAMS)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 16, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 17, "_index": "healthtools"}}, {"name": "DR BERKLEY JAMES ALEXANDER", "postal_address": "P.O BOX 428 80108 KILIFI", "facility": "KEMRI/WELLCOME TRST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS NEWCASTLE PON TYNE)", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 17, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 18, "_index": "healthtools"}}, {"name": "DR JORG BERLING", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FREIBRG)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 18, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 19, "_index": "healthtools"}}, {"name": "DR BHATT MEGHNA MEHL", "postal_address": "P.O BOX 30270 00100 NAIROBI", "facility": "AGA KHAN NIVERSITY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(DEEMED)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 19, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 20, "_index": "healthtools"}}, {"name": "DR BOEDEMANN MELANIE", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 20, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 21, "_index": "healthtools"}}, {"name": "DR BOEVE NORMAN", "postal_address": "P.O BOX 20 0220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MICHIGAN)1965", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 21, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 22, "_index": "healthtools"}}, {"name": "DR JENNY BOYD", "postal_address": "P.O BOX 39 20400 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(TENNESSEE)2003", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 22, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 23, "_index": "healthtools"}}, {"name": "DR BROWN ROGER KEVIN", "postal_address": "P.O BOX 21171 00505 NAIROBI", "facility": "AFRICAN INLAND MISSION INTERNATIONAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(ILLINOIS)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 23, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 24, "_index": "healthtools"}}, {"name": "DR BDACH RTH MAGDALENE", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE AGLICAN CHRCH OF KENYA - MOTHERS MERCY HOME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1978", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 24, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 25, "_index": "healthtools"}}, {"name": "DR BRGERT STEPHEN LOIS", "postal_address": "P.O BOX 39 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MAYO)1979", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 25, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 26, "_index": "healthtools"}}, {"name": "DR CARBONE MARCO", "postal_address": "P.O BOX 88 20318 NORTHKINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(GENOVA)1993", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 26, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 27, "_index": "healthtools"}}, {"name": "DR CARTER ROBERT ALLEN", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A I C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(INDIANA)1981", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 27, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 28, "_index": "healthtools"}}, {"name": "DR CHANG SANG HO", "postal_address": "P.O BOX 20954 00202 NAIROBI", "facility": "CHRISTIAN MEDICAL & DENTAL ASSOCIATION OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(SEOL)1977", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 28, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 29, "_index": "healthtools"}}, {"name": "DR COLLIS FABIAN BERNARD", "postal_address": "P.O BOX 45 00902 KIKY", "facility": "P.C.E.A KIKY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MSCH MBCHA(SAME)1999", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 29, "speciality": "-"}] diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index c406256..7b15213 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -1,4 +1,5 @@ import unittest +import json from healthtools.scrapers.doctors import DoctorsScraper from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper @@ -18,47 +19,38 @@ def test_it_gets_the_total_number_of_pages(self): def test_it_scrapes_doctors_page(self): entries = self.doctors_scraper.scrape_page( "http://medicalboard.co.ke/online-services/retention/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 10) + self.assertTrue(len(entries) == 60) def test_it_scrapes_foreign_doctors_page(self): entries = self.foreign_doctors_scraper.scrape_page( "http://medicalboard.co.ke/online-services/foreign-doctors-license-register/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 10) + self.assertTrue(len(entries) == 60) def test_it_scrapes_clinical_officers_page(self): entries = self.clinical_officers_scraper.scrape_page( "http://clinicalofficerscouncil.org/online-services/retention/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 7) + self.assertTrue(len(entries) == 60) - def test_it_scrapes_whole_doctors_site(self): - all_entries = self.doctors_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_it_scrapes_whole_foreign_doctors_site(self): - all_entries = self.foreign_doctors_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_it_scrapes_whole_clinical_officers_site(self): - all_entries = self.clinical_officers_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_doctors_scraper_uploads_to_cloudsearch(self): + def test_doctors_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/doctors.json", "r") as my_file: data = my_file.read() - response = self.doctors_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.doctors_scraper.delete_elasticsearch_docs() + response = self.doctors_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) - def test_foreign_doctors_scraper_uploads_to_cloudsearch(self): + def test_foreign_doctors_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/foreign_doctors.json", "r") as my_file: data = my_file.read() - response = self.foreign_doctors_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.foreign_doctors_scraper.delete_elasticsearch_docs() + response = self.foreign_doctors_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) - def test_clinical_officers_scraper_uploads_to_cloudsearch(self): + def test_clinical_officers_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/clinical_officers.json", "r") as my_file: data = my_file.read() - response = self.clinical_officers_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.clinical_officers_scraper.delete_elasticsearch_docs() + response = self.clinical_officers_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) def test_doctors_scraper_archives_to_s3(self): self.doctors_scraper.s3_key = "test/doctors.json" @@ -68,7 +60,7 @@ def test_doctors_scraper_archives_to_s3(self): uploaded_data = self.doctors_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.doctors_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) def test_foreign_doctors_scraper_archives_to_s3(self): @@ -79,7 +71,7 @@ def test_foreign_doctors_scraper_archives_to_s3(self): uploaded_data = self.foreign_doctors_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.foreign_doctors_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) def test_clinical_officers_scraper_archives_to_s3(self): @@ -90,9 +82,6 @@ def test_clinical_officers_scraper_archives_to_s3(self): uploaded_data = self.clinical_officers_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.clinical_officers_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) - def test_foreign_doctors_scraper_deletes_cloudsearch_docs(self): - response = self.foreign_doctors_scraper.delete_cloudsearch_docs() - self.assertEqual(response.get("status"), "success") diff --git a/requirements.txt b/requirements.txt index 0ec8ef6..294151d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,29 @@ -beautifulsoup4==4.5.3 -boto3==1.4.4 -botocore==1.5.27 -bs4==0.0.1 -docutils==0.13.1 -futures==3.0.5 -jmespath==0.9.2 -nose==1.3.7 -python-dateutil==2.6.0 -requests==2.13.0 -s3transfer==0.1.10 -six==1.10.0 +appdirs (1.4.3) +beautifulsoup4 (4.5.3) +boto3 (1.4.4) +botocore (1.5.27) +bs4 (0.0.1) +certifi (2017.4.17) +click (6.7) +docutils (0.13.1) +elasticsearch (5.4.0) +Flask (0.12.1) +futures (3.0.5) +gunicorn (19.7.1) +itsdangerous (0.24) +Jinja2 (2.9.6) +jmespath (0.9.2) +MarkupSafe (1.0) +nose (1.3.7) +packaging (16.8) +pip (9.0.1) +pyparsing (2.2.0) +python-dateutil (2.6.0) +python-memcached (1.58) +requests (2.13.0) +s3transfer (0.1.10) +setuptools (36.0.1) +six (1.10.0) +urllib3 (1.21.1) +Werkzeug (0.12.1) +wheel (0.29.0) diff --git a/scraper.py b/scraper.py index ab948c5..e4388cc 100644 --- a/scraper.py +++ b/scraper.py @@ -3,14 +3,15 @@ from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper from healthtools.scrapers.health_facilities import HealthFacilitiesScraper if __name__ == "__main__": - healthfacilities_scraper = HealthFacilitiesScraper() + # healthfacilities_scraper = HealthFacilitiesScraper() doctors_scraper = DoctorsScraper() foreign_doctors_scraper = ForeignDoctorsScraper() clinical_officers_scraper = ClinicalOfficersScraper() - healthfacilities_scraper.scrape_data() + # get back to healthfacilities after indexing docs and co's to elastic search + # healthfacilities_scraper.scrape_data() # scraping you softly with these bots... + clinical_officers_result = clinical_officers_scraper.scrape_site() doctors_result = doctors_scraper.scrape_site() if doctors_result: foreign_doctors_scraper.document_id = len(doctors_result) + 1 foreign_docs_result = foreign_doctors_scraper.scrape_site() - clinical_officers_result = clinical_officers_scraper.scrape_site() From 82d7e49f2f42da68ab7a4be1d4ff401030b54cf6 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 15 Jun 2017 12:37:57 +0300 Subject: [PATCH 02/18] Add support for health facilities to elastic search Add tests for the health facilities scraper --- healthtools/config.py | 1 - healthtools/scrapers/base_scraper.py | 29 +++-- healthtools/scrapers/clinical_officers.py | 2 +- healthtools/scrapers/doctors.py | 2 +- healthtools/scrapers/foreign_doctors.py | 2 +- healthtools/scrapers/health_facilities.py | 119 +++++++----------- .../tests/dummy_files/health_facilities.json | 1 + healthtools/tests/test_scrapers.py | 26 +++- scraper.py | 5 +- 9 files changed, 97 insertions(+), 90 deletions(-) create mode 100644 healthtools/tests/dummy_files/health_facilities.json diff --git a/healthtools/config.py b/healthtools/config.py index 157788b..f6a4b05 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -26,7 +26,6 @@ "user": os.getenv("ES_USER"), "pass": os.getenv("ES_PASS"), "index": "healthtools", - "doctors_type": "doctors", "cos_type": "clinical-officers" } diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 5d61b43..ba64644 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -161,9 +161,12 @@ def delete_elasticsearch_docs(self): ''' try: # get the type to use with the index depending on the calling method - _type = ES['cos_type'] \ - if 'clinical' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower() \ - else ES['doctors_type'] + if 'clinical' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower(): + _type = 'clinical-officers' + elif 'doctors' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower(): + _type = 'doctors' + else: + _type = 'health-facilities' # get documents to be deleted delete_docs = self.s3.get_object( Bucket="cfa-healthtools-ke", @@ -175,12 +178,20 @@ def delete_elasticsearch_docs(self): # incase records are saved in cloudsearch's format, reformat for elasticsearch deletion delete_records = [] for record in json.loads(delete_docs): - delete_records.append({ - "delete": { - "_index": ES['index'], - "_type": _type, - "_id": record['delete']["_id"] - }}) + try: + delete_records.append({ + "delete": { + "_index": ES['index'], + "_type": _type, + "_id": record['delete']["_id"] + }}) + except: + delete_records.append({ + "delete": { + "_index": ES['index'], + "_type": _type, + "_id": record["id"] + }}) response = self.es_client.bulk(index=ES['index'], body=delete_records) return response except Exception as err: diff --git a/healthtools/scrapers/clinical_officers.py b/healthtools/scrapers/clinical_officers.py index c9766c4..1e3aaa1 100644 --- a/healthtools/scrapers/clinical_officers.py +++ b/healthtools/scrapers/clinical_officers.py @@ -36,7 +36,7 @@ def format_for_elasticsearch(self, entry): meta_dict = { "index": { "_index": ES['index'], - "_type": ES['cos_type'], + "_type": 'clinical-officers', "_id": entry['id'] } } diff --git a/healthtools/scrapers/doctors.py b/healthtools/scrapers/doctors.py index dcb3199..c4d7cf9 100644 --- a/healthtools/scrapers/doctors.py +++ b/healthtools/scrapers/doctors.py @@ -37,7 +37,7 @@ def format_for_elasticsearch(self, entry): meta_dict = { "index": { "_index": ES['index'], - "_type": ES['doctors_type'], + "_type": 'doctors', "_id": entry['id'] } } diff --git a/healthtools/scrapers/foreign_doctors.py b/healthtools/scrapers/foreign_doctors.py index 89347f3..9f66288 100644 --- a/healthtools/scrapers/foreign_doctors.py +++ b/healthtools/scrapers/foreign_doctors.py @@ -30,7 +30,7 @@ def format_for_elasticsearch(self, entry): meta_dict = { "index": { "_index": ES['index'], - "_type": ES['doctors_type'], + "_type": 'doctors', "_id": entry['id'] } } diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index b3b3a0d..3ee130a 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -1,39 +1,10 @@ import json from cStringIO import StringIO from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import AWS +from healthtools.config import ES, AWS import requests -import boto3 from datetime import datetime -health_facilities_template = """ - { - "type": "add", - "id": "%s", - "fields": { - "name": "%s", - "facility_type_name": "%s", - "approved": "%s", - "sub_county_name": "%s", - "service_names": "%s", - "county_name": "%s", - "open_public_holidays": "%s", - "keph_level_name": "%s", - "open_whole_day": "%s", - "owner_name": "%s", - "constituency_name": "%s", - "regulatory_body_name": "%s", - "operation_status_name": "%s", - "open_late_night": "%s", - "open_weekends": "%s", - "ward_name": "%s" - } - }""" -delete_template = """ - { - "type": "delete", - "id" : "%s" - }""" TOKEN_URL = 'http://api.kmhfl.health.go.ke/o/token/' SEARCH_URL = 'http://api.kmhfl.health.go.ke/api/facilities/material/?page_size=100000&' \ 'fields=id,regulatory_status_name,facility_type_name,facility_type_parent,owner_name,owner_type_name,' \ @@ -57,13 +28,8 @@ def __init__(self): self.s3_key = "data/health_facilities.json" self.s3_historical_record_key = "data/archive/health_facilities-{}.json" self.delete_file = "data/delete_health_facilities.json" - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_health_faciities_endpoint"] - }) + self.payload = [] + self.delete_data = [] def get_token(self): print "[Health Facilities Scraper]" @@ -74,64 +40,71 @@ def get_token(self): 'grant_type': 'password', 'client_id': 'xMddOofHI0jOKboVxdoKAXWKpkEQAP0TuloGpfj5', 'client_secret': 'PHrUzCRFm9558DGa6Fh1hEvSCh3C9Lijfq8sbCMZhZqmANYV5ZP04mUXGJdsrZLXuZG4VCmvjShdKHwU6IRmPQld5LDzvJoguEP8AAXGJhrqfLnmtFXU3x2FO1nWLxUx' - } + } r = requests.post(TOKEN_URL, data=data, headers=headers) self.access_token = json.loads(r.text)['access_token'] + def upload(self, payload): + return self.upload_data(payload) + def get_data(self): try: print "{{{0}}} - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) headers = {'Authorization': 'Bearer ' + self.access_token} r = requests.get(SEARCH_URL, headers=headers) data = r.json() - payload = '' - delete_payload = '' for i, record in enumerate(data['results']): - payload += self.index_for_cloudsearch(record) + ',' - delete_payload += self.delete_payload(record) + ',' - payload = '[%s]' % payload[:-1] #remove last comma - delete_payload = '[%s]' % delete_payload[:-1] #remove last comma - self.delete_cloudsearch_docs() #delete cloudsearch data - self.upload_data(payload) #upload data to cloudsearch - print "{{{0}}} - Scraper completed. {1} documents retrieved.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),len(data['results'])) + meta, elastic_data = self.index_for_elasticsearch(record) + self.payload.append(meta) + self.payload.append(elastic_data) + self.delete_data.append(self.delete_payload(record)) + self.delete_elasticsearch_docs() # delete elasticsearch data + self.upload(self.payload) # upload data to elasticsearch + print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(data['results'])) # Push the data to s3 - self.archive_data(payload) + self.archive_data(json.dumps(self.payload)) # Push the delete payload to s3 - delete_file = StringIO(delete_payload) + delete_file = StringIO(json.dumps(self.delete_data)) self.s3.upload_fileobj( delete_file, "cfa-healthtools-ke", self.delete_file) print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) - except Exception, err: - print "ERROR IN - index_for_search() - %s" % (err) + except Exception as err: + print "ERROR IN - index_for_search() Health Facilities Scraper - %s" % err - def index_for_cloudsearch(self, record): - return health_facilities_template % ( - record['code'], - record['name'].replace("\"", "'"), - record['facility_type_name'], - record['approved'], - record['sub_county_name'], - record['service_names'], - record['county_name'], - record['open_public_holidays'], - record['keph_level_name'], - record['open_whole_day'], - record['owner_name'], - record['constituency_name'], - record['regulatory_body_name'], - record['operation_status_name'], - record['open_late_night'], - record['open_weekends'], - record['ward_name'].decode("string_escape").replace('\\', ''), - ) + def index_for_elasticsearch(self, record): + meta_data = {"index": { + "_index": ES['index'], + "_type": 'health-facilities', + "_id": record['code'] + }} + health_facilities = { + "id": record['code'], + "name": record['name'].replace("\"", "'"), + "facility_type_name": record['facility_type_name'], + "approved": record['approved'], + "sub_county_name": record['sub_county_name'], + "service_names": record['service_names'], + "county_name": record['county_name'], + "open_public_holidays": record['open_public_holidays'], + "keph_level_name": record['keph_level_name'], + "open_whole_day": record['open_whole_day'], + "owner_name": record['owner_name'], + "constituency_name": record['constituency_name'], + "regulatory_body_name": record['regulatory_body_name'], + "operation_status_name": record['operation_status_name'], + "open_late_night": record['open_late_night'], + "open_weekends": record['open_weekends'], + "ward_name": record['ward_name'].decode("string_escape").replace('\\', '') + } + return meta_data, health_facilities def delete_payload(self, record): - return delete_template %(record['code']) + return {"delete": {"_index": ES['index'], "_type": "health-facilities", "_id": record['code']}} def scrape_data(self): self.get_token() self.get_data() - diff --git a/healthtools/tests/dummy_files/health_facilities.json b/healthtools/tests/dummy_files/health_facilities.json new file mode 100644 index 0000000..c43ab36 --- /dev/null +++ b/healthtools/tests/dummy_files/health_facilities.json @@ -0,0 +1 @@ +[{"index": {"_type": "health-facilities", "_id": 16188, "_index": "healthtools"}}, {"sub_county_name": "kaloleni", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mariakani Community Health Care Services", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "KALOLENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16188, "ward_name": "MARIAKANI"}, {"index": {"_type": "health-facilities", "_id": 10359, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ithe-Kahuno Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10359, "ward_name": "AGUTHI-GAAKI"}, {"index": {"_type": "health-facilities", "_id": 14898, "_index": "healthtools"}}, {"sub_county_name": "kapseret", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kipkenyo Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KAPSERET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "UASIN GISH", "open_late_night": "False", "keph_level_name": "Level 2", "id": 14898, "ward_name": "KIPKENYO"}, {"index": {"_type": "health-facilities", "_id": 10652, "_index": "healthtools"}}, {"sub_county_name": "kieni west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Labura/Babito Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": ["Short Term", "Natural", "Long Term", "Condom Distribution & STI Prevention", "HIV Testing Services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10652, "ward_name": "MWIYOGO/ENDARASHA"}, {"index": {"_type": "health-facilities", "_id": 13655, "_index": "healthtools"}}, {"sub_county_name": "bondo", "facility_type_name": "Basic primary health care facility", "owner_name": "Ministry of Health", "name": "Kapiyo Health Centre", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "BONDO", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "SIAYA", "open_late_night": "False", "keph_level_name": "Level 3", "id": 13655, "ward_name": "WEST SAKWA"}, {"index": {"_type": "health-facilities", "_id": 10278, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Gituamba Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10278, "ward_name": "GITUAMBA"}, {"index": {"_type": "health-facilities", "_id": 10616, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kinunga Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10616, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 10800, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mutitu Gikondi Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10800, "ward_name": "GIKONDI"}, {"index": {"_type": "health-facilities", "_id": 10042, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Atlas Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10042, "ward_name": "IRIA-INI"}, {"index": {"_type": "health-facilities", "_id": 10730, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mihuti Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10730, "ward_name": "RUGI"}, {"index": {"_type": "health-facilities", "_id": 11019, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga north/mwea west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ssema Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11019, "ward_name": "MUTITHI"}, {"index": {"_type": "health-facilities", "_id": 10679, "_index": "healthtools"}}, {"sub_county_name": "ndaragwa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Manguo Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NDARAGWA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYANDARUA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10679, "ward_name": "KIRITTA"}, {"index": {"_type": "health-facilities", "_id": 11125, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Tunuku Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11125, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 12124, "_index": "healthtools"}}, {"sub_county_name": "tharaka north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kageni Med Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "THARAKA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "THARAKA-NITHI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12124, "ward_name": "MUKOTHIMA"}, {"index": {"_type": "health-facilities", "_id": 10605, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kimahuri Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10605, "ward_name": "KABAR"}, {"index": {"_type": "health-facilities", "_id": 10393, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kabuti Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GICHUG", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10393, "ward_name": "BARAGWI"}, {"index": {"_type": "health-facilities", "_id": 10358, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ithare Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GICHUG", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10358, "ward_name": "KABARE"}, {"index": {"_type": "health-facilities", "_id": 16182, "_index": "healthtools"}}, {"sub_county_name": "rabai", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Khadija Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "RABAI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16182, "ward_name": "MWAWESA"}, {"index": {"_type": "health-facilities", "_id": 10741, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mt Kenya Narumoru Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10741, "ward_name": "NAROMORU/KIAMATHAGA"}, {"index": {"_type": "health-facilities", "_id": 10849, "_index": "healthtools"}}, {"sub_county_name": "mathioya", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "New Kihoya Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KANGEMA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10849, "ward_name": "RWATHIA"}, {"index": {"_type": "health-facilities", "_id": 11820, "_index": "healthtools"}}, {"sub_county_name": "magarini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "St Mary's Medical Clinic (Malindi)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "MAGARINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11820, "ward_name": "AD"}, {"index": {"_type": "health-facilities", "_id": 10047, "_index": "healthtools"}}, {"sub_county_name": "kihar", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Baraka Medical Clinic (Muranga North)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIHAR", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10047, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 13460, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Zakma Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13460, "ward_name": "GALBET"}, {"index": {"_type": "health-facilities", "_id": 13326, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Dertu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13326, "ward_name": "GALBET"}, {"index": {"_type": "health-facilities", "_id": 10168, "_index": "healthtools"}}, {"sub_county_name": "kieni west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Emmanuel Medical Clinic (Nyeri North)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": ["Short Term", "Natural", "Long Term", "Condom Distribution & STI Prevention", "HIV Testing Services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10168, "ward_name": "MUGUNDA"}, {"index": {"_type": "health-facilities", "_id": 14989, "_index": "healthtools"}}, {"sub_county_name": "pokot north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kopeeto Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "KACHELIBA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "WEST POKOT", "open_late_night": "False", "keph_level_name": "Level 2", "id": 14989, "ward_name": "ALALE"}, {"index": {"_type": "health-facilities", "_id": 12346, "_index": "healthtools"}}, {"sub_county_name": "kitui west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kiseveni Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KITUI WEST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12346, "ward_name": "KAUWI"}, {"index": {"_type": "health-facilities", "_id": 15661, "_index": "healthtools"}}, {"sub_county_name": "turkana central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Kenya Episcopal Conference-Catholic Secretariat", "name": "St Monica Nakwamekwi Dispensary", "regulatory_body_name": "Kenya MPDB - Institution", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TURKANA CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "TURKANA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 15661, "ward_name": "LODWAR TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 10416, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kahara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10416, "ward_name": "MUKURWE-INI CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 10160, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ebenezer Medical Clinic (Nyeri South)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10160, "ward_name": "RWARE"}, {"index": {"_type": "health-facilities", "_id": 10501, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kariumba Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10501, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 10002, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Aberdare Medical & Surgical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10002, "ward_name": "IRIA-INI"}, {"index": {"_type": "health-facilities", "_id": 12218, "_index": "healthtools"}}, {"sub_county_name": "mwingi central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Katethya Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWINGI CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12218, "ward_name": "CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 11088, "_index": "healthtools"}}, {"sub_county_name": "kihar", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Tewas Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIHAR", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11088, "ward_name": "WANG"}, {"index": {"_type": "health-facilities", "_id": 10795, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Muthuthiini Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10795, "ward_name": "GIKONDI"}, {"index": {"_type": "health-facilities", "_id": 16447, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Safi Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16447, "ward_name": "NYANGUTI"}, {"index": {"_type": "health-facilities", "_id": 10206, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Gatamu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10206, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 11168, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Wanduta Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11168, "ward_name": "GITHOBOKONI"}, {"index": {"_type": "health-facilities", "_id": 10108, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Diana Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10108, "ward_name": "NAROMORU/KIAMATHAGA"}, {"index": {"_type": "health-facilities", "_id": 10754, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mukarara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10754, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 12089, "_index": "healthtools"}}, {"sub_county_name": "mwala", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Imani Yako Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWALA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MACHAKOS", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12089, "ward_name": "WAMUNY"}, {"index": {"_type": "health-facilities", "_id": 10611, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kimondo Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10611, "ward_name": "MUKURWE-INI CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 10835, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ndonyero Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10835, "ward_name": "MANGU"}, {"index": {"_type": "health-facilities", "_id": 11137, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Upendo Medical Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11137, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 13377, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Jaribu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13377, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 12160, "_index": "healthtools"}}, {"sub_county_name": "kitui central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kamandio Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KITUI CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12160, "ward_name": "MIAMBANI"}, {"index": {"_type": "health-facilities", "_id": 11944, "_index": "healthtools"}}, {"sub_county_name": "mwingi central", "facility_type_name": "Basic primary health care facility", "owner_name": "Private Practice - Clinical Officer", "name": "Salem Medical Services", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWINGI CENTRAL", "approved": "True", "service_names": ["Short Term", "Integrated Management of Childhood Illnesses", "Postnatal care services", "Class B", "Focused Antenatal Care", "Stand Alone - Retail services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 3", "id": 11944, "ward_name": "CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 11780, "_index": "healthtools"}}, {"sub_county_name": "GARSEN", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Sera Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "True", "open_weekends": "False", "constituency_name": "GARSEN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "TANA RIVER", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11780, "ward_name": "GARSEN NORTH"}, {"index": {"_type": "health-facilities", "_id": 12397, "_index": "healthtools"}}, {"sub_county_name": "mwala", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kwakala Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWALA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MACHAKOS", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12397, "ward_name": "WAMUNY"}, {"index": {"_type": "health-facilities", "_id": 11181, "_index": "healthtools"}}, {"sub_county_name": "laikipia west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Wellness Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "LAIKIPIA WEST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "LAIKIPIA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11181, "ward_name": "IGWAMITI"}, {"index": {"_type": "health-facilities", "_id": 10908, "_index": "healthtools"}}, {"sub_county_name": "mathioya", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Obeys Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MATHIOYA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10908, "ward_name": "KIR"}, {"index": {"_type": "health-facilities", "_id": 10106, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Delina Health Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10106, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 10074, "_index": "healthtools"}}, {"sub_county_name": "thika town", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Boore Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "THIKA TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10074, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 10710, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mayos Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10710, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 10707, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - General Practitioner", "name": "Mathingira Medical Clinic", "regulatory_body_name": "Kenya MPDB - Private Practice", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10707, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 10882, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Njika Wega Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10882, "ward_name": "RUGI"}, {"index": {"_type": "health-facilities", "_id": 10022, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Alpha Family Health Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 3", "id": 10022, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 10297, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga north/mwea west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Good Samaritan Health Services (Kirinyaga)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10297, "ward_name": "MUTITHI"}, {"index": {"_type": "health-facilities", "_id": 10879, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Enterprise (Institution)", "name": "Njambi Nursing Home", "regulatory_body_name": "Kenya MPDB - Institution", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10879, "ward_name": "GATHIGIRIRI"}, {"index": {"_type": "health-facilities", "_id": 13888, "_index": "healthtools"}}, {"sub_county_name": "kasipul", "facility_type_name": "Basic primary health care facility", "owner_name": "Private Practice - General Practitioner", "name": "Nyalenda Nursing Home", "regulatory_body_name": "Kenya MPDB - Private Practice", "open_whole_day": "True", "open_weekends": "True", "constituency_name": "KASIPUL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "HOMA BAY", "open_late_night": "False", "keph_level_name": "Level 3", "id": 13888, "ward_name": "CENTRAL KASIPUL"}, {"index": {"_type": "health-facilities", "_id": 13396, "_index": "healthtools"}}, {"sub_county_name": "mandera east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Libehiya Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MANDERA EAST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MANDERA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13396, "ward_name": "ARABIA"}, {"index": {"_type": "health-facilities", "_id": 11256, "_index": "healthtools"}}, {"sub_county_name": "magarini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Bombi Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "MAGARINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11256, "ward_name": "AD"}, {"index": {"_type": "health-facilities", "_id": 10325, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Hope Medical Clinic (Kirinyaga)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NDIA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10325, "ward_name": "KIINE"}, {"index": {"_type": "health-facilities", "_id": 10680, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Manjo Health Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10680, "ward_name": "MUKURWE-INI WEST"}, {"index": {"_type": "health-facilities", "_id": 10356, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Itara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10356, "ward_name": "MUKURWE-INI WEST"}] diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index 7b15213..8585f62 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -3,6 +3,7 @@ from healthtools.scrapers.doctors import DoctorsScraper from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper +from healthtools.scrapers.health_facilities import HealthFacilitiesScraper from healthtools.config import TEST_DIR @@ -11,6 +12,7 @@ def setUp(self): self.doctors_scraper = DoctorsScraper() self.foreign_doctors_scraper = ForeignDoctorsScraper() self.clinical_officers_scraper = ClinicalOfficersScraper() + self.health_facilities_scraper = HealthFacilitiesScraper() def test_it_gets_the_total_number_of_pages(self): self.doctors_scraper.get_total_number_of_pages() @@ -43,7 +45,7 @@ def test_foreign_doctors_scraper_uploads_to_elasticsearch(self): data = my_file.read() self.foreign_doctors_scraper.delete_elasticsearch_docs() response = self.foreign_doctors_scraper.upload_data(json.loads(data)) - self.assertEqual(response['items'][0]['index']['status'], 201) + self.assertEqual(response['items'][0]['index']['status'], 200) def test_clinical_officers_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/clinical_officers.json", "r") as my_file: @@ -52,6 +54,13 @@ def test_clinical_officers_scraper_uploads_to_elasticsearch(self): response = self.clinical_officers_scraper.upload_data(json.loads(data)) self.assertEqual(response['items'][0]['index']['status'], 201) + def test_health_facilities_scraper_uploads_to_elasticsearch(self): + with open(TEST_DIR + "/dummy_files/health_facilities.json", "r") as my_file: + data = my_file.read() + self.health_facilities_scraper.delete_elasticsearch_docs() + response = self.health_facilities_scraper.upload(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) + def test_doctors_scraper_archives_to_s3(self): self.doctors_scraper.s3_key = "test/doctors.json" with open(TEST_DIR + "/dummy_files/doctors.json", "r") as my_file: @@ -63,6 +72,18 @@ def test_doctors_scraper_archives_to_s3(self): )['Body'].read() self.assertEqual(uploaded_data, data) + # get test/health_facilities.json key for this test + # def test_health_facilities_scraper_archives_to_s3(self): + # self.health_facilities_scraper.s3_key = "test/health_facilities.json" + # with open(TEST_DIR + "/dummy_files/health_facilities.json", "r") as my_file: + # data = my_file.read() + # self.health_facilities_scraper.archive_data(data) + # uploaded_data = self.health_facilities_scraper.s3.get_object( + # Bucket="cfa-healthtools-ke", + # Key=self.health_facilities_scraper.s3_key + # )['Body'].read() + # self.assertEqual(uploaded_data, data) + def test_foreign_doctors_scraper_archives_to_s3(self): self.foreign_doctors_scraper.s3_key = "test/foreign_doctors.json" with open(TEST_DIR + "/dummy_files/foreign_doctors.json", "r") as my_file: @@ -85,3 +106,6 @@ def test_clinical_officers_scraper_archives_to_s3(self): )['Body'].read() self.assertEqual(uploaded_data, data) + def test_health_facilities_scraper_gets_token(self): + self.health_facilities_scraper.get_token() + self.assertIsNotNone(self.health_facilities_scraper.access_token) diff --git a/scraper.py b/scraper.py index e4388cc..e42d1cd 100644 --- a/scraper.py +++ b/scraper.py @@ -3,12 +3,11 @@ from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper from healthtools.scrapers.health_facilities import HealthFacilitiesScraper if __name__ == "__main__": - # healthfacilities_scraper = HealthFacilitiesScraper() + healthfacilities_scraper = HealthFacilitiesScraper() doctors_scraper = DoctorsScraper() foreign_doctors_scraper = ForeignDoctorsScraper() clinical_officers_scraper = ClinicalOfficersScraper() - # get back to healthfacilities after indexing docs and co's to elastic search - # healthfacilities_scraper.scrape_data() + healthfacilities_scraper.scrape_data() # scraping you softly with these bots... clinical_officers_result = clinical_officers_scraper.scrape_site() doctors_result = doctors_scraper.scrape_site() From e59d6bac5ce56e624eed793c981f28e1a9e1fe7a Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 15 Jun 2017 15:09:48 +0300 Subject: [PATCH 03/18] Convert to save to local elastic search and not cloud service --- healthtools/scrapers/base_scraper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index ba64644..d5f32ee 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -24,8 +24,13 @@ def __init__(self): "aws_secret_access_key": AWS["aws_secret_access_key"], "region_name": AWS["region_name"], }) + # client host for elastic cloud + # self.es_client = Elasticsearch([ + # "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) + # ]) self.es_client = Elasticsearch([ - "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) + {"host": "localhost", + "port": 9200} ]) def scrape_site(self): From 05bbd87f4950a4ae9467095957a469423bfbb5c6 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 15 Jun 2017 15:21:26 +0300 Subject: [PATCH 04/18] Add CircleCi integration --- circle.yml | 10 +++++++++ requirements.txt | 55 +++++++++++++++++++++++------------------------- 2 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 circle.yml diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..734554e --- /dev/null +++ b/circle.yml @@ -0,0 +1,10 @@ +machine: + python: + version: 2.7.5 +dependencies: + pre: + - pip install -r requirements.txt + +test: + override: + - nosetests --nocapture \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 294151d..95a4474 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,26 @@ -appdirs (1.4.3) -beautifulsoup4 (4.5.3) -boto3 (1.4.4) -botocore (1.5.27) -bs4 (0.0.1) -certifi (2017.4.17) -click (6.7) -docutils (0.13.1) -elasticsearch (5.4.0) -Flask (0.12.1) -futures (3.0.5) -gunicorn (19.7.1) -itsdangerous (0.24) -Jinja2 (2.9.6) -jmespath (0.9.2) -MarkupSafe (1.0) -nose (1.3.7) -packaging (16.8) -pip (9.0.1) -pyparsing (2.2.0) -python-dateutil (2.6.0) -python-memcached (1.58) -requests (2.13.0) -s3transfer (0.1.10) -setuptools (36.0.1) -six (1.10.0) -urllib3 (1.21.1) -Werkzeug (0.12.1) -wheel (0.29.0) +appdirs==1.4.3 +beautifulsoup4==4.5.3 +boto3==1.4.4 +botocore==1.5.27 +bs4==0.0.1 +certifi==2017.4.17 +click==6.7 +docutils==0.13.1 +elasticsearch==5.4.0 +Flask==0.12.1 +futures==3.0.5 +gunicorn==19.7.1 +itsdangerous==0.24 +Jinja2==2.9.6 +jmespath==0.9.2 +MarkupSafe==1.0 +nose==1.3.7 +packaging==16.8 +pyparsing==2.2.0 +python-dateutil==2.6.0 +python-memcached==1.58 +requests==2.13.0 +s3transfer==0.1.10 +six==1.10.0 +urllib3==1.21.1 +Werkzeug==0.12.1 From 8329b4fb50e9cf8dfe68315c771bed38d5505a50 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Fri, 16 Jun 2017 10:05:08 +0300 Subject: [PATCH 05/18] Use elastic cloud search host --- circle.yml | 2 +- healthtools/scrapers/base_scraper.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/circle.yml b/circle.yml index 734554e..c3c93da 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: python: - version: 2.7.5 + version: 2.7.13 dependencies: pre: - pip install -r requirements.txt diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index d5f32ee..e07fee0 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -25,13 +25,13 @@ def __init__(self): "region_name": AWS["region_name"], }) # client host for elastic cloud - # self.es_client = Elasticsearch([ - # "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) - # ]) self.es_client = Elasticsearch([ - {"host": "localhost", - "port": 9200} + "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) ]) + # self.es_client = Elasticsearch([ + # {"host": "localhost", + # "port": 9200} + # ]) def scrape_site(self): ''' From 4e703be02d04c3321ee99002713607efc4f07483 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Mon, 19 Jun 2017 13:48:29 +0300 Subject: [PATCH 06/18] Index to elasticsearch AWS Service --- healthtools/config.py | 5 +---- healthtools/scrapers/base_scraper.py | 26 +++++++++++++++-------- healthtools/scrapers/health_facilities.py | 2 +- healthtools/scrapers/serializer.py | 18 ++++++++++++++++ requirements.txt | 4 ++++ 5 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 healthtools/scrapers/serializer.py diff --git a/healthtools/config.py b/healthtools/config.py index f6a4b05..acfedaa 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -23,10 +23,7 @@ ES = { "host": os.getenv("ES_HOST"), "port": os.getenv("ES_PORT"), - "user": os.getenv("ES_USER"), - "pass": os.getenv("ES_PASS"), - "index": "healthtools", - "cos_type": "clinical-officers" + "index": "healthtools" } TEST_DIR = os.getcwd() + "/healthtools/tests" diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index e07fee0..e6eee4b 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -1,8 +1,10 @@ from bs4 import BeautifulSoup from cStringIO import StringIO from datetime import datetime -from elasticsearch import Elasticsearch +from elasticsearch import Elasticsearch, RequestsHttpConnection +from requests_aws4auth import AWS4Auth from healthtools.config import AWS, ES +from serializer import JSONSerializerPython2 import requests import boto3 import re @@ -24,14 +26,18 @@ def __init__(self): "aws_secret_access_key": AWS["aws_secret_access_key"], "region_name": AWS["region_name"], }) - # client host for elastic cloud - self.es_client = Elasticsearch([ - "https://{}:{}@{}:{}".format(ES['user'], ES['pass'], ES['host'], ES['port']) - ]) - # self.es_client = Elasticsearch([ - # {"host": "localhost", - # "port": 9200} - # ]) + # set up authentication credentials + awsauth = AWS4Auth(AWS["aws_access_key_id"], AWS["aws_secret_access_key"], AWS["region_name"], 'es') + # client host for aws elastic search service + self.es_client = Elasticsearch( + hosts=ES['host'], + port=443, + http_auth=awsauth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + serializer=JSONSerializerPython2() + ) def scrape_site(self): ''' @@ -243,3 +249,5 @@ def format_for_elasticsearch(self, entry): } } return meta_dict, entry + + diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index 3ee130a..3df747d 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -1,7 +1,7 @@ import json from cStringIO import StringIO from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import ES, AWS +from healthtools.config import ES import requests from datetime import datetime diff --git a/healthtools/scrapers/serializer.py b/healthtools/scrapers/serializer.py new file mode 100644 index 0000000..926d786 --- /dev/null +++ b/healthtools/scrapers/serializer.py @@ -0,0 +1,18 @@ +import json +from elasticsearch import serializer, compat, exceptions + + +class JSONSerializerPython2(serializer.JSONSerializer): + """Override elasticsearch library serializer to ensure it encodes utf characters during json dump. + See original at: https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/serializer.py#L42 + A description of how ensure_ascii encodes unicode characters to ensure they can be sent across the wire + as ascii can be found here: https://docs.python.org/2/library/json.html#basic-usage + """ + def dumps(self, data): + # don't serialize strings + if isinstance(data, compat.string_types): + return data + try: + return json.dumps(data, default=self.default, ensure_ascii=True) + except (ValueError, TypeError) as e: + raise exceptions.SerializationError(data, e) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 95a4474..a27a1d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ appdirs==1.4.3 +backports.ssl-match-hostname==3.5.0.1 beautifulsoup4==4.5.3 boto3==1.4.4 botocore==1.5.27 @@ -20,7 +21,10 @@ pyparsing==2.2.0 python-dateutil==2.6.0 python-memcached==1.58 requests==2.13.0 +requests-aws4auth==0.9 s3transfer==0.1.10 six==1.10.0 +slackclient==1.0.6 urllib3==1.21.1 +websocket-client==0.40.0 Werkzeug==0.12.1 From 4c4e2f05c9b3df84880f5a37f3de6f704dd359e7 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Tue, 20 Jun 2017 11:20:35 +0300 Subject: [PATCH 07/18] Shift data indexing to Elasticsearch from cloudsearch (#1) * Add support for elastic search cloud service Fix tests to run with elastic search Refactor tests to remove redundant tests * Add support for health facilities to elastic search Add tests for the health facilities scraper * Convert to save to local elastic search and not cloud service * Add CircleCi integration * Use elastic cloud search host * Index to elasticsearch AWS Service --- circle.yml | 10 ++ healthtools/config.py | 13 +- healthtools/scrapers/base_scraper.py | 115 ++++++++++++----- healthtools/scrapers/clinical_officers.py | 37 +++--- healthtools/scrapers/doctors.py | 32 ++--- healthtools/scrapers/foreign_doctors.py | 32 ++--- healthtools/scrapers/health_facilities.py | 119 +++++++----------- healthtools/scrapers/serializer.py | 18 +++ .../tests/dummy_files/clinical_officers.json | 2 +- healthtools/tests/dummy_files/doctors.json | 2 +- .../tests/dummy_files/foreign_doctors.json | 2 +- .../tests/dummy_files/health_facilities.json | 1 + healthtools/tests/test_scrapers.py | 73 ++++++----- requirements.txt | 18 +++ scraper.py | 2 +- 15 files changed, 290 insertions(+), 186 deletions(-) create mode 100644 circle.yml create mode 100644 healthtools/scrapers/serializer.py create mode 100644 healthtools/tests/dummy_files/health_facilities.json diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..c3c93da --- /dev/null +++ b/circle.yml @@ -0,0 +1,10 @@ +machine: + python: + version: 2.7.13 +dependencies: + pre: + - pip install -r requirements.txt + +test: + override: + - nosetests --nocapture \ No newline at end of file diff --git a/healthtools/config.py b/healthtools/config.py index ca87fe7..acfedaa 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -5,8 +5,8 @@ "DOCTORS": "http://medicalboard.co.ke/online-services/retention/?currpage={}", "FOREIGN_DOCTORS": "http://medicalboard.co.ke/online-services/foreign-doctors-license-register/?currpage={}", "CLINICAL_OFFICERS": "http://clinicalofficerscouncil.org/online-services/retention/?currpage={}", - "TOKEN_URL" : "http://api.kmhfl.health.go.ke/o/token/" -} + "TOKEN_URL": "http://api.kmhfl.health.go.ke/o/token/" + } AWS = { "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY_ID"), @@ -17,8 +17,13 @@ # Clinical document endpoint "cloudsearch_cos_endpoint": "http://doc-cfa-healthtools-ke-cos-nhxtw3w5goufkzram4er7sciz4.eu-west-1.cloudsearch.amazonaws.com/", # Health facilities endpoint - "cloudsearch_health_faciities_endpoint":"https://doc-health-facilities-ke-65ftd7ksxazyatw5fiv5uyaiqi.eu-west-1.cloudsearch.amazonaws.com", + "cloudsearch_health_faciities_endpoint": "https://doc-health-facilities-ke-65ftd7ksxazyatw5fiv5uyaiqi.eu-west-1.cloudsearch.amazonaws.com", -} + } +ES = { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT"), + "index": "healthtools" + } TEST_DIR = os.getcwd() + "/healthtools/tests" diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 71d21ec..e6eee4b 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -1,7 +1,10 @@ from bs4 import BeautifulSoup from cStringIO import StringIO from datetime import datetime -from healthtools.config import AWS +from elasticsearch import Elasticsearch, RequestsHttpConnection +from requests_aws4auth import AWS4Auth +from healthtools.config import AWS, ES +from serializer import JSONSerializerPython2 import requests import boto3 import re @@ -14,7 +17,6 @@ def __init__(self): self.num_pages_to_scrape = None self.site_url = None self.fields = None - self.cloudsearch = None self.s3_key = None self.document_id = 0 # id for each entry, to be incremented self.delete_file = None # contains docs to be deleted after scrape @@ -23,7 +25,19 @@ def __init__(self): "aws_access_key_id": AWS["aws_access_key_id"], "aws_secret_access_key": AWS["aws_secret_access_key"], "region_name": AWS["region_name"], - }) + }) + # set up authentication credentials + awsauth = AWS4Auth(AWS["aws_access_key_id"], AWS["aws_secret_access_key"], AWS["region_name"], 'es') + # client host for aws elastic search service + self.es_client = Elasticsearch( + hosts=ES['host'], + port=443, + http_auth=awsauth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + serializer=JSONSerializerPython2() + ) def scrape_site(self): ''' @@ -45,8 +59,7 @@ def scrape_site(self): print "There's something wrong with the site. Proceeding to the next scraper." return - entries = scraped_page[0] - delete_docs = scraped_page[1] + entries, delete_docs = scraped_page all_results.extend(entries) delete_batch.extend(delete_docs) @@ -54,14 +67,15 @@ def scrape_site(self): skipped_pages += 1 print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) continue - print "{{{0}}} - Scraper completed. {1} documents retrieved.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),len(all_results)) + print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(all_results)) if all_results: all_results_json = json.dumps(all_results) delete_batch = json.dumps(delete_batch) - self.delete_cloudsearch_docs() - self.upload_data(all_results_json) + self.delete_elasticsearch_docs() + self.upload_data(all_results) self.archive_data(all_results_json) # store delete operations for next scrape @@ -92,10 +106,17 @@ def scrape_page(self, page_url): columns.append(self.document_id) entry = dict(zip(self.fields, columns)) - entry = self.format_for_cloudsearch(entry) + meta, entry = self.format_for_elasticsearch(entry) + entries.append(meta) entries.append(entry) - delete_batch.append({"type": "delete", "id": entry["id"]}) + delete_batch.append({ + "delete": + { + "_index": ES['index'], + "_type": meta['index']['_type'], + "_id": entry["id"] + }}) self.document_id += 1 return entries, delete_batch except Exception as err: @@ -108,12 +129,11 @@ def scrape_page(self, page_url): def upload_data(self, payload): ''' - Upload data to AWS Cloud Search + Upload data to Elastic Search ''' try: - response = self.cloudsearch.upload_documents( - documents=payload, contentType="application/json" - ) + # bulk index the data and use refresh to ensure that our data will be immediately available + response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) @@ -140,31 +160,56 @@ def archive_data(self, payload): print "{{{0}}} - Archived data has been updated.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) return else: - print "{{{0}}} - Data Scraped does not differ from archived data.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "{{{0}}} - Data Scraped does not differ from archived data.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) - def delete_cloudsearch_docs(self): + def delete_elasticsearch_docs(self): ''' - Delete documents that were uploaded to cloudsearch in the last scrape + Delete documents that were uploaded to elasticsearch in the last scrape ''' try: + # get the type to use with the index depending on the calling method + if 'clinical' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower(): + _type = 'clinical-officers' + elif 'doctors' in re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__).lower(): + _type = 'doctors' + else: + _type = 'health-facilities' # get documents to be deleted delete_docs = self.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.delete_file)['Body'].read() - # delete - response = self.cloudsearch.upload_documents( - documents=delete_docs, contentType="application/json" - ) + try: + response = self.es_client.bulk(index=ES['index'], body=delete_docs, refresh=True) + except: + # incase records are saved in cloudsearch's format, reformat for elasticsearch deletion + delete_records = [] + for record in json.loads(delete_docs): + try: + delete_records.append({ + "delete": { + "_index": ES['index'], + "_type": _type, + "_id": record['delete']["_id"] + }}) + except: + delete_records.append({ + "delete": { + "_index": ES['index'], + "_type": _type, + "_id": record["id"] + }}) + response = self.es_client.bulk(index=ES['index'], body=delete_records) return response except Exception as err: if "NoSuchKey" in err: - print "ERROR - delete_cloudsearch_docs() - no delete file present" + print "ERROR - delete_elasticsearch_docs() - no delete file present" return - print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + print "ERROR - delete_elasticsearch_docs() - {} - {}".format(type(self).__name__, str(err)) def get_total_number_of_pages(self): ''' @@ -177,7 +222,7 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ + print "ERROR: **get_total_page_numbers()** - url: {} - err: {}". \ format(self.site_url, str(err)) return @@ -189,8 +234,20 @@ def make_soup(self, url): soup = BeautifulSoup(response.content, "html.parser") return soup - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' - return {"id": entry["id"], "type": "add", "fields": entry} + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": "index", + "_type": "type", + "_id": "id" + } + } + return meta_dict, entry + + diff --git a/healthtools/scrapers/clinical_officers.py b/healthtools/scrapers/clinical_officers.py index 3cbbe53..1e3aaa1 100644 --- a/healthtools/scrapers/clinical_officers.py +++ b/healthtools/scrapers/clinical_officers.py @@ -1,7 +1,6 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS +from healthtools.config import SITES, ES from datetime import datetime -import boto3 class ClinicalOfficersScraper(Scraper): @@ -15,24 +14,30 @@ def __init__(self): self.fields = [ "name", "reg_date", "reg_no", "valid_dates", "address", "qualifications", "id", - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_cos_endpoint"] - }) + ] self.s3_key = "data/clinical_officers.json" self.s3_historical_record_key = "data/archive/clinical_officers-{}.json" self.delete_file = "data/delete_clinical_officers.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' - date_obj = datetime.strptime(entry['reg_date'], "%d-%m-%y %H:%M") + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ + try: + date_obj = datetime.strptime(entry['reg_date'], "%Y-%m-%d") + except: + date_obj = datetime.strptime(entry['reg_date'], "%d-%m-%Y") entry['reg_date'] = datetime.strftime( date_obj, "%Y-%m-%dT%H:%M:%S.000Z") - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": 'clinical-officers', + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/doctors.py b/healthtools/scrapers/doctors.py index fce7bb6..c4d7cf9 100644 --- a/healthtools/scrapers/doctors.py +++ b/healthtools/scrapers/doctors.py @@ -1,7 +1,6 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS +from healthtools.config import SITES, ES from datetime import datetime -import boto3 class DoctorsScraper(Scraper): @@ -15,23 +14,18 @@ def __init__(self): self.fields = [ "name", "reg_date", "reg_no", "postal_address", "qualifications", "speciality", "sub_speciality", "id", - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_doctors_endpoint"] - }) + ] self.s3_key = "data/doctors.json" self.s3_historical_record_key = "data/archive/doctors-{}.json" self.delete_file = "data/delete_doctors.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ try: date_obj = datetime.strptime(entry['reg_date'], "%Y-%m-%d") except: @@ -39,4 +33,12 @@ def format_for_cloudsearch(self, entry): entry['reg_date'] = datetime.strftime( date_obj, "%Y-%m-%dT%H:%M:%S.000Z") entry["facility"] = entry["practice_type"] = "-" - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": 'doctors', + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/foreign_doctors.py b/healthtools/scrapers/foreign_doctors.py index babf48f..9f66288 100644 --- a/healthtools/scrapers/foreign_doctors.py +++ b/healthtools/scrapers/foreign_doctors.py @@ -1,6 +1,5 @@ from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import SITES, AWS -import boto3 +from healthtools.config import SITES, ES class ForeignDoctorsScraper(Scraper): @@ -14,22 +13,25 @@ def __init__(self): self.fields = [ "name", "reg_no", "postal_address", "qualifications", "facility", "practice_type", "id" - ] - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_doctors_endpoint"] - }) + ] self.s3_key = "data/foreign_doctors.json" self.s3_historical_record_key = "data/archive/foreign_doctors-{}.json" self.delete_file = "data/delete_foreign_doctors.json" - def format_for_cloudsearch(self, entry): - ''' - Format entry into cloudsearch ready document - ''' + def format_for_elasticsearch(self, entry): + """ + Format entry into elasticsearch ready document + :param entry: the data to be formatted + :return: dictionaries of the entry's metadata and the formatted entry + """ entry["reg_date"] = "0000-01-01T00:00:00.000Z" entry["reg_no"] = entry["speciality"] = entry["sub_speciality"] = "-" - return {"id": entry["id"], "type": "add", "fields": entry} + # all bulk data need meta data describing the data + meta_dict = { + "index": { + "_index": ES['index'], + "_type": 'doctors', + "_id": entry['id'] + } + } + return meta_dict, entry diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index b3b3a0d..3df747d 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -1,39 +1,10 @@ import json from cStringIO import StringIO from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import AWS +from healthtools.config import ES import requests -import boto3 from datetime import datetime -health_facilities_template = """ - { - "type": "add", - "id": "%s", - "fields": { - "name": "%s", - "facility_type_name": "%s", - "approved": "%s", - "sub_county_name": "%s", - "service_names": "%s", - "county_name": "%s", - "open_public_holidays": "%s", - "keph_level_name": "%s", - "open_whole_day": "%s", - "owner_name": "%s", - "constituency_name": "%s", - "regulatory_body_name": "%s", - "operation_status_name": "%s", - "open_late_night": "%s", - "open_weekends": "%s", - "ward_name": "%s" - } - }""" -delete_template = """ - { - "type": "delete", - "id" : "%s" - }""" TOKEN_URL = 'http://api.kmhfl.health.go.ke/o/token/' SEARCH_URL = 'http://api.kmhfl.health.go.ke/api/facilities/material/?page_size=100000&' \ 'fields=id,regulatory_status_name,facility_type_name,facility_type_parent,owner_name,owner_type_name,' \ @@ -57,13 +28,8 @@ def __init__(self): self.s3_key = "data/health_facilities.json" self.s3_historical_record_key = "data/archive/health_facilities-{}.json" self.delete_file = "data/delete_health_facilities.json" - self.cloudsearch = boto3.client( - "cloudsearchdomain", **{ - "aws_access_key_id": AWS["aws_access_key_id"], - "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], - "endpoint_url": AWS["cloudsearch_health_faciities_endpoint"] - }) + self.payload = [] + self.delete_data = [] def get_token(self): print "[Health Facilities Scraper]" @@ -74,64 +40,71 @@ def get_token(self): 'grant_type': 'password', 'client_id': 'xMddOofHI0jOKboVxdoKAXWKpkEQAP0TuloGpfj5', 'client_secret': 'PHrUzCRFm9558DGa6Fh1hEvSCh3C9Lijfq8sbCMZhZqmANYV5ZP04mUXGJdsrZLXuZG4VCmvjShdKHwU6IRmPQld5LDzvJoguEP8AAXGJhrqfLnmtFXU3x2FO1nWLxUx' - } + } r = requests.post(TOKEN_URL, data=data, headers=headers) self.access_token = json.loads(r.text)['access_token'] + def upload(self, payload): + return self.upload_data(payload) + def get_data(self): try: print "{{{0}}} - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) headers = {'Authorization': 'Bearer ' + self.access_token} r = requests.get(SEARCH_URL, headers=headers) data = r.json() - payload = '' - delete_payload = '' for i, record in enumerate(data['results']): - payload += self.index_for_cloudsearch(record) + ',' - delete_payload += self.delete_payload(record) + ',' - payload = '[%s]' % payload[:-1] #remove last comma - delete_payload = '[%s]' % delete_payload[:-1] #remove last comma - self.delete_cloudsearch_docs() #delete cloudsearch data - self.upload_data(payload) #upload data to cloudsearch - print "{{{0}}} - Scraper completed. {1} documents retrieved.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),len(data['results'])) + meta, elastic_data = self.index_for_elasticsearch(record) + self.payload.append(meta) + self.payload.append(elastic_data) + self.delete_data.append(self.delete_payload(record)) + self.delete_elasticsearch_docs() # delete elasticsearch data + self.upload(self.payload) # upload data to elasticsearch + print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(data['results'])) # Push the data to s3 - self.archive_data(payload) + self.archive_data(json.dumps(self.payload)) # Push the delete payload to s3 - delete_file = StringIO(delete_payload) + delete_file = StringIO(json.dumps(self.delete_data)) self.s3.upload_fileobj( delete_file, "cfa-healthtools-ke", self.delete_file) print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) - except Exception, err: - print "ERROR IN - index_for_search() - %s" % (err) + except Exception as err: + print "ERROR IN - index_for_search() Health Facilities Scraper - %s" % err - def index_for_cloudsearch(self, record): - return health_facilities_template % ( - record['code'], - record['name'].replace("\"", "'"), - record['facility_type_name'], - record['approved'], - record['sub_county_name'], - record['service_names'], - record['county_name'], - record['open_public_holidays'], - record['keph_level_name'], - record['open_whole_day'], - record['owner_name'], - record['constituency_name'], - record['regulatory_body_name'], - record['operation_status_name'], - record['open_late_night'], - record['open_weekends'], - record['ward_name'].decode("string_escape").replace('\\', ''), - ) + def index_for_elasticsearch(self, record): + meta_data = {"index": { + "_index": ES['index'], + "_type": 'health-facilities', + "_id": record['code'] + }} + health_facilities = { + "id": record['code'], + "name": record['name'].replace("\"", "'"), + "facility_type_name": record['facility_type_name'], + "approved": record['approved'], + "sub_county_name": record['sub_county_name'], + "service_names": record['service_names'], + "county_name": record['county_name'], + "open_public_holidays": record['open_public_holidays'], + "keph_level_name": record['keph_level_name'], + "open_whole_day": record['open_whole_day'], + "owner_name": record['owner_name'], + "constituency_name": record['constituency_name'], + "regulatory_body_name": record['regulatory_body_name'], + "operation_status_name": record['operation_status_name'], + "open_late_night": record['open_late_night'], + "open_weekends": record['open_weekends'], + "ward_name": record['ward_name'].decode("string_escape").replace('\\', '') + } + return meta_data, health_facilities def delete_payload(self, record): - return delete_template %(record['code']) + return {"delete": {"_index": ES['index'], "_type": "health-facilities", "_id": record['code']}} def scrape_data(self): self.get_token() self.get_data() - diff --git a/healthtools/scrapers/serializer.py b/healthtools/scrapers/serializer.py new file mode 100644 index 0000000..926d786 --- /dev/null +++ b/healthtools/scrapers/serializer.py @@ -0,0 +1,18 @@ +import json +from elasticsearch import serializer, compat, exceptions + + +class JSONSerializerPython2(serializer.JSONSerializer): + """Override elasticsearch library serializer to ensure it encodes utf characters during json dump. + See original at: https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/serializer.py#L42 + A description of how ensure_ascii encodes unicode characters to ensure they can be sent across the wire + as ascii can be found here: https://docs.python.org/2/library/json.html#basic-usage + """ + def dumps(self, data): + # don't serialize strings + if isinstance(data, compat.string_types): + return data + try: + return json.dumps(data, default=self.default, ensure_ascii=True) + except (ValueError, TypeError) as e: + raise exceptions.SerializationError(data, e) \ No newline at end of file diff --git a/healthtools/tests/dummy_files/clinical_officers.json b/healthtools/tests/dummy_files/clinical_officers.json index 6910ab6..b316910 100644 --- a/healthtools/tests/dummy_files/clinical_officers.json +++ b/healthtools/tests/dummy_files/clinical_officers.json @@ -1 +1 @@ -[{"fields": {"name": "JACOB KILIMO KISANG", "reg_no": "1 / Ra00302/17", "valid_dates": "2023-01-17 / 2031-01-19", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "PO BOX 1881", "id": 0}, "type": "add", "id": 0}, {"fields": {"name": "FRANCIS KINGE MATHERI", "reg_no": "18 / Ra01477/16", "valid_dates": "2030-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-07-08T00:00:00.000Z", "address": "P.O BOX 656 KIAMBU", "id": 1}, "type": "add", "id": 1}, {"fields": {"name": "GLADYS OKAKAH KOYENGO", "reg_no": "19 / Ra00811/15", "valid_dates": "2018-11-15 / 2030-11-17", "qualifications": "DIP", "reg_date": "1989-04-26T00:00:00.000Z", "address": "P.O BOX 20715 NAIROBI", "id": 2}, "type": "add", "id": 2}, {"fields": {"name": "STEPHEN OKEYO ABEBE", "reg_no": "20 / R02062/16", "valid_dates": "2018-08-16 / 2031-08-18", "qualifications": "DIP", "reg_date": "1989-03-16T00:00:00.000Z", "address": "P.O. BOX 1 KADONGO", "id": 3}, "type": "add", "id": 3}, {"fields": {"name": "JOSEPH OGEKA MAORE", "reg_no": "30 / Ra01287/16", "valid_dates": "2014-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O BOX 153 KISII", "id": 4}, "type": "add", "id": 4}, {"fields": {"name": "HAMISI ALI FAJIN", "reg_no": "37 / Ra01032/15", "valid_dates": "2008-12-15 / 2031-12-17", "qualifications": "DIP", "reg_date": "1989-01-10T00:00:00.000Z", "address": "P.O BOX 57 MALINDI", "id": 5}, "type": "add", "id": 5}, {"fields": {"name": "GEORGE AYIEKO JOWI", "reg_no": "40 / Ra01823/16", "valid_dates": "2012-05-16 / 2031-05-18", "qualifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O. BOX 538 MTWAPA", "id": 6}, "type": "add", "id": 6}, {"fields": {"name": "STEPHEN KIPLANGAT KITTUR", "reg_no": "48 / R02318/15", "valid_dates": "2013-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1987-08-25T00:00:00.000Z", "address": "P.O BOX 5036 ELDORET", "id": 7}, "type": "add", "id": 7}, {"fields": {"name": "ROBERT MUNGE KIMUNDUU", "reg_no": "54 / R02261/15", "valid_dates": "2013-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1988-02-02T00:00:00.000Z", "address": "P.O BOX 304 MACHAKOS", "id": 8}, "type": "add", "id": 8}, {"fields": {"name": "CHRISANTHUS MAKORI ORINA", "reg_no": "55 / Ra00375/15", "valid_dates": "2029-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 3685 KISII", "id": 9}, "type": "add", "id": 9}, {"fields": {"name": "EZEKIEL WALUFU WALUFU", "reg_no": "76 / R02849/15", "valid_dates": "2021-04-15 / 2030-04-17", "qualifications": "DIP", "reg_date": "1987-03-19T00:00:00.000Z", "address": "P.O BOX 802 MUMIAS", "id": 10}, "type": "add", "id": 10}, {"fields": {"name": "ELIKANAH KEBAGENDI OMWENGA", "reg_no": "78 / Ra00147/15", "valid_dates": "2001-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-08-16T00:00:00.000Z", "address": "P.O BOX 252 80100 MOMBASA", "id": 11}, "type": "add", "id": 11}, {"fields": {"name": "DANIEL OMBACHI MOGENI", "reg_no": "97 / Ra00643/16", "valid_dates": "2003-02-16 / 2028-02-18", "qualifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 235 NYAMIRA", "id": 12}, "type": "add", "id": 12}, {"fields": {"name": "BENSON GICHUHI NG\\'ERI", "reg_no": "107 / Ra00293/15", "valid_dates": "2015-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 15383 NAKURU", "id": 13}, "type": "add", "id": 13}, {"fields": {"name": "ANDIVA HERMAN KAGHALI", "reg_no": "112 / Ra00758/16", "valid_dates": "2009-02-16 / 2028-02-18", "qualifications": "DIP", "reg_date": "1989-05-12T00:00:00.000Z", "address": "P.O. BOX 119 50309 TIRIKI", "id": 14}, "type": "add", "id": 14}, {"fields": {"name": "DABASO GARAMBODA JILLO", "reg_no": "121 / Ra00245/17", "valid_dates": "2019-01-17 / 2031-01-19", "qualifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O BOX 753 GARISSA", "id": 15}, "type": "add", "id": 15}, {"fields": {"name": "SIYAD MOHAMED OMAR", "reg_no": "122 / Ra02079/16", "valid_dates": "2015-06-16 / 2030-06-18", "qualifications": "DIP", "reg_date": "1989-01-06T00:00:00.000Z", "address": "P.O BOX 253 GARISSA", "id": 16}, "type": "add", "id": 16}, {"fields": {"name": "DAVID MAINA NJURU", "reg_no": "125 / Ra02390/16", "valid_dates": "2019-07-16 / 2031-07-18", "qualifications": "DIP", "reg_date": "1989-10-12T00:00:00.000Z", "address": "P.O BOX 20435 00200 NAIROBI", "id": 17}, "type": "add", "id": 17}, {"fields": {"name": "ALLEN PETERSON MBELA", "reg_no": "138 / Ra00187/16", "valid_dates": "2013-01-16 / 2031-01-18", "qualifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O. BOX 47 MWATATE", "id": 18}, "type": "add", "id": 18}, {"fields": {"name": "LINCOLN MURAI NJUGUNA", "reg_no": "139 / Ra00572/15", "valid_dates": "2001-11-15 / 2030-11-17", "qualifications": "DIP", "reg_date": "1988-02-12T00:00:00.000Z", "address": "P.O BOX 125 KILGORIS", "id": 19}, "type": "add", "id": 19}, {"fields": {"name": "KITHUSI PETER KIITI", "reg_no": "141 / Ra01441/16", "valid_dates": "2024-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O BOX 1473 KANGUNDO", "id": 20}, "type": "add", "id": 20}, {"fields": {"name": "MICHAEL GITARI KAMAU", "reg_no": "147 / R03501/15", "valid_dates": "2016-07-15 / 2031-07-17", "qualifications": "DIP", "reg_date": "1989-01-20T00:00:00.000Z", "address": "P.O BOX 701 THIKA", "id": 21}, "type": "add", "id": 21}, {"fields": {"name": "JAMES NYORO WAMWEA", "reg_no": "163 / Ra01474/16", "valid_dates": "2029-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2055-03-15T00:00:00.000Z", "address": "P.O BOX 265 LIMURU", "id": 22}, "type": "add", "id": 22}, {"fields": {"name": "GITHAIGA NJUMA ONESMUS", "reg_no": "168 / Ra01203/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "1989-01-25T00:00:00.000Z", "address": "P.O. BOX 171 MUKURWEINI", "id": 23}, "type": "add", "id": 23}, {"fields": {"name": "JUSTUS KARIGU KIBARUA", "reg_no": "171 / Ra01199/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2001-01-31T10:46:00.000Z", "address": "P.O. BOX 2 GIAKANJA", "id": 24}, "type": "add", "id": 24}, {"fields": {"name": "KILLIAN ANSELM MWOLOI", "reg_no": "178 / Ra00200/16", "valid_dates": "2013-01-16 / 2031-01-18", "qualifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O. BOX 152 KIBWEZI", "id": 25}, "type": "add", "id": 25}, {"fields": {"name": "CHARLES GATHUMA BIRINGI", "reg_no": "182 / Ra01202/16", "valid_dates": "2008-03-16 / 2031-03-18", "qualifications": "DIP", "reg_date": "2011-03-15T11:34:00.000Z", "address": "P.O BOX 920 NYERI", "id": 26}, "type": "add", "id": 26}, {"fields": {"name": "ANTONY KITSAO CHIRAHU", "reg_no": "198 / Ra00540/15", "valid_dates": "2021-10-15 / 2031-10-17", "qualifications": "DIP", "reg_date": "1989-01-31T00:00:00.000Z", "address": "P.O BOX 49 MOMBASA", "id": 27}, "type": "add", "id": 27}, {"fields": {"name": "JULIUS MUTAMBUKI KILIKU", "reg_no": "206 / Ra01512/16", "valid_dates": "2001-05-16 / 2031-05-18", "qualifications": "DIP", "reg_date": "2011-03-17T14:20:00.000Z", "address": "P.O BOX 350 90119 MATUU", "id": 28}, "type": "add", "id": 28}, {"fields": {"name": "STEPHEN ODUOR JURE", "reg_no": "207 / Ra00326/15", "valid_dates": "2018-09-15 / 2030-09-17", "qualifications": "DIP", "reg_date": "1989-07-12T00:00:00.000Z", "address": "P.O BOX 370 KISUMU", "id": 29}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "clinical-officers", "_id": 0, "_index": "healthtools"}}, {"name": "JACOB KILIMO KISANG", "reg_no": "1 / Ra00302/17", "valid_dates": "2017-01-23 / 2019-01-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "PO BOX 1881", "id": 0}, {"index": {"_type": "clinical-officers", "_id": 1, "_index": "healthtools"}}, {"name": "ISAAC NNDA NYABAYO", "reg_no": "2 / Ra02803/17", "valid_dates": "2017-05-31 / 2019-05-31", "qalifications": "DIP", "reg_date": "2011-04-28T00:00:00.000Z", "address": "P.O BOX 456 KISII", "id": 1}, {"index": {"_type": "clinical-officers", "_id": 2, "_index": "healthtools"}}, {"name": "FRANCIS KINGE MATHERI", "reg_no": "18 / Ra01477/16", "valid_dates": "2016-03-30 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-07-08T00:00:00.000Z", "address": "P.O BOX 656 KIAMB", "id": 2}, {"index": {"_type": "clinical-officers", "_id": 3, "_index": "healthtools"}}, {"name": "GLADYS OKAKAH KOYENGO", "reg_no": "19 / Ra00811/15", "valid_dates": "2015-11-18 / 2017-11-30", "qalifications": "DIP", "reg_date": "1989-04-26T00:00:00.000Z", "address": "P.O BOX 20715 NAIROBI", "id": 3}, {"index": {"_type": "clinical-officers", "_id": 4, "_index": "healthtools"}}, {"name": "STEPHEN OKEYO ABEBE", "reg_no": "20 / R02062/16", "valid_dates": "2016-08-18 / 2018-08-31", "qalifications": "DIP", "reg_date": "1989-03-16T00:00:00.000Z", "address": "P.O. BOX 1 KADONGO", "id": 4}, {"index": {"_type": "clinical-officers", "_id": 5, "_index": "healthtools"}}, {"name": "DANIEL ONGWENYI MOGAMBI", "reg_no": "22 / R01412/17", "valid_dates": "2017-05-09 / 2019-05-31", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O BOX 4315 ELDORET", "id": 5}, {"index": {"_type": "clinical-officers", "_id": 6, "_index": "healthtools"}}, {"name": "JOSEPH OGEKA MAORE", "reg_no": "30 / Ra01287/16", "valid_dates": "2016-03-14 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O BOX 153 KISII", "id": 6}, {"index": {"_type": "clinical-officers", "_id": 7, "_index": "healthtools"}}, {"name": "HAMISI ALI FAJIN", "reg_no": "37 / Ra01032/15", "valid_dates": "2015-12-08 / 2017-12-31", "qalifications": "DIP", "reg_date": "1989-01-10T00:00:00.000Z", "address": "P.O BOX 57 MALINDI", "id": 7}, {"index": {"_type": "clinical-officers", "_id": 8, "_index": "healthtools"}}, {"name": "GEORGE AYIEKO JOWI", "reg_no": "40 / Ra01823/16", "valid_dates": "2016-05-12 / 2018-05-31", "qalifications": "DIP", "reg_date": "1989-01-05T00:00:00.000Z", "address": "P.O. BOX 538 MTWAPA", "id": 8}, {"index": {"_type": "clinical-officers", "_id": 9, "_index": "healthtools"}}, {"name": "STEPHEN KIPLAGAT KITTR", "reg_no": "48 / Ra02696/17", "valid_dates": "2017-05-23 / 2019-05-31", "qalifications": "DIP", "reg_date": "1987-08-25T00:00:00.000Z", "address": "P.O BOX 5036 ELDORET", "id": 9}, {"index": {"_type": "clinical-officers", "_id": 10, "_index": "healthtools"}}, {"name": "ROBERT MNGE KIMND", "reg_no": "54 / Ra02229/17", "valid_dates": "2017-05-01 / 2019-05-31", "qalifications": "DIP", "reg_date": "1988-02-02T00:00:00.000Z", "address": "P.O BOX 304 MACHAKOS", "id": 10}, {"index": {"_type": "clinical-officers", "_id": 11, "_index": "healthtools"}}, {"name": "CHRISANTHS MAKORI ORINA", "reg_no": "55 / Ra00375/15", "valid_dates": "2015-09-29 / 2017-09-30", "qalifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 3685 KISII", "id": 11}, {"index": {"_type": "clinical-officers", "_id": 12, "_index": "healthtools"}}, {"name": "EZEKIEL WALF WALF", "reg_no": "76 / Ra02666/17", "valid_dates": "2017-05-19 / 2019-05-31", "qalifications": "DIP", "reg_date": "1987-03-19T00:00:00.000Z", "address": "P.O BOX 802 MMIAS", "id": 12}, {"index": {"_type": "clinical-officers", "_id": 13, "_index": "healthtools"}}, {"name": "ELIKANAH KEBAGENDI OMWENGA", "reg_no": "78 / Ra00147/15", "valid_dates": "2015-09-01 / 2017-09-30", "qalifications": "DIP", "reg_date": "1989-08-16T00:00:00.000Z", "address": "P.O BOX 252 80100 MOMBASA", "id": 13}, {"index": {"_type": "clinical-officers", "_id": 14, "_index": "healthtools"}}, {"name": "EVANS NGK HEZRON", "reg_no": "87 / Ra02613/17", "valid_dates": "2017-05-16 / 2019-05-31", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 14907 NAKR", "id": 14}, {"index": {"_type": "clinical-officers", "_id": 15, "_index": "healthtools"}}, {"name": "DANIEL OMBACHI MOGENI", "reg_no": "97 / Ra00643/16", "valid_dates": "2016-02-03 / 2018-02-28", "qalifications": "DIP", "reg_date": "1988-01-28T00:00:00.000Z", "address": "P.O. BOX 235 NYAMIRA", "id": 15}, {"index": {"_type": "clinical-officers", "_id": 16, "_index": "healthtools"}}, {"name": "BENSON GICHHI NG'ERI", "reg_no": "107 / Ra00293/15", "valid_dates": "2015-09-15 / 2017-09-30", "qalifications": "DIP", "reg_date": "1989-01-11T00:00:00.000Z", "address": "P.O. BOX 15383 NAKR", "id": 16}, {"index": {"_type": "clinical-officers", "_id": 17, "_index": "healthtools"}}, {"name": "ANDIVA HERMAN KAGHALI", "reg_no": "112 / Ra00758/16", "valid_dates": "2016-02-09 / 2018-02-28", "qalifications": "DIP", "reg_date": "1989-05-12T00:00:00.000Z", "address": "P.O. BOX 119 50309 TIRIKI", "id": 17}, {"index": {"_type": "clinical-officers", "_id": 18, "_index": "healthtools"}}, {"name": "DABASO GARAMBODA JILLO", "reg_no": "121 / Ra00245/17", "valid_dates": "2017-01-19 / 2019-01-31", "qalifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O BOX 753 GARISSA", "id": 18}, {"index": {"_type": "clinical-officers", "_id": 19, "_index": "healthtools"}}, {"name": "SIYAD MOHAMED OMAR", "reg_no": "122 / Ra02079/16", "valid_dates": "2016-06-15 / 2018-06-30", "qalifications": "DIP", "reg_date": "1989-01-06T00:00:00.000Z", "address": "P.O BOX 253 GARISSA", "id": 19}, {"index": {"_type": "clinical-officers", "_id": 20, "_index": "healthtools"}}, {"name": "DAVID MAINA NJR", "reg_no": "125 / Ra02390/16", "valid_dates": "2016-07-19 / 2018-07-31", "qalifications": "DIP", "reg_date": "1989-10-12T00:00:00.000Z", "address": "P.O BOX 20435 00200 NAIROBI", "id": 20}, {"index": {"_type": "clinical-officers", "_id": 21, "_index": "healthtools"}}, {"name": "ALLEN PETERSON MBELA", "reg_no": "138 / Ra00187/16", "valid_dates": "2016-01-13 / 2018-01-31", "qalifications": "DIP", "reg_date": "1989-01-17T00:00:00.000Z", "address": "P.O. BOX 47 MWATATE", "id": 21}, {"index": {"_type": "clinical-officers", "_id": 22, "_index": "healthtools"}}, {"name": "LINCOLN MRAI NJGNA", "reg_no": "139 / Ra00572/15", "valid_dates": "2015-11-01 / 2017-11-30", "qalifications": "DIP", "reg_date": "1988-02-12T00:00:00.000Z", "address": "P.O BOX 125 KILGORIS", "id": 22}, {"index": {"_type": "clinical-officers", "_id": 23, "_index": "healthtools"}}, {"name": "KITHSI PETER KIITI", "reg_no": "141 / Ra01441/16", "valid_dates": "2016-03-24 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O BOX 1473 KANGNDO", "id": 23}, {"index": {"_type": "clinical-officers", "_id": 24, "_index": "healthtools"}}, {"name": "MICHAEL GITARI KAMA", "reg_no": "147 / R03501/15", "valid_dates": "2015-07-16 / 2017-07-31", "qalifications": "DIP", "reg_date": "1989-01-20T00:00:00.000Z", "address": "P.O BOX 701 THIKA", "id": 24}, {"index": {"_type": "clinical-officers", "_id": 25, "_index": "healthtools"}}, {"name": "JSTS MEME MGWIKA", "reg_no": "152 / Ra02087/17", "valid_dates": "2017-03-31 / 2019-03-31", "qalifications": "DIP", "reg_date": "1989-01-22T00:00:00.000Z", "address": "P.O. BOX 1681 MER", "id": 25}, {"index": {"_type": "clinical-officers", "_id": 26, "_index": "healthtools"}}, {"name": "JAMES NYORO WAMWEA", "reg_no": "163 / Ra01474/16", "valid_dates": "2016-03-29 / 2018-03-31", "qalifications": "DIP", "reg_date": "1955-03-15T00:00:00.000Z", "address": "P.O BOX 265 LIMR", "id": 26}, {"index": {"_type": "clinical-officers", "_id": 27, "_index": "healthtools"}}, {"name": "GITHAIGA NJMA ONESMS", "reg_no": "168 / Ra01203/16", "valid_dates": "2016-03-08 / 2018-03-31", "qalifications": "DIP", "reg_date": "1989-01-25T00:00:00.000Z", "address": "P.O. BOX 171 MKRWEINI", "id": 27}, {"index": {"_type": "clinical-officers", "_id": 28, "_index": "healthtools"}}, {"name": "JSTS KARIG KIBARA", "reg_no": "171 / Ra01199/16", "valid_dates": "2016-03-08 / 2018-03-31", "qalifications": "DIP", "reg_date": "2001-01-31T00:00:00.000Z", "address": "P.O. BOX 2 GIAKANJA", "id": 28}, {"index": {"_type": "clinical-officers", "_id": 29, "_index": "healthtools"}}, {"name": "KILLIAN ANSELM MWOLOI", "reg_no": "178 / Ra00200/16", "valid_dates": "2016-01-13 / 2018-01-31", "qalifications": "DIP", "reg_date": "1989-02-21T00:00:00.000Z", "address": "P.O. BOX 152 KIBWEZI", "id": 29}] diff --git a/healthtools/tests/dummy_files/doctors.json b/healthtools/tests/dummy_files/doctors.json index 7d3dd66..b042c2d 100644 --- a/healthtools/tests/dummy_files/doctors.json +++ b/healthtools/tests/dummy_files/doctors.json @@ -1 +1 @@ -[{"fields": {"name": "Dr. DEVANI JAYENDRA KUMAR", "postal_address": "P.O.Box 43376-00100 NAIROBI", "facility": "-", "reg_no": "B021", "practice_type": "-", "qualifications": "BDS(Bombay) 1966 LDS RCS(Edin) 1968", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 0, "speciality": "NONE"}, "type": "add", "id": 0}, {"fields": {"name": "Dr. VARSHANI NARAN SHIVJI", "postal_address": "P.O.Box 75977-00200 NAIROBI", "facility": "-", "reg_no": "B025", "practice_type": "-", "qualifications": "BDS(Gujarat) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 1, "speciality": "NONE"}, "type": "add", "id": 1}, {"fields": {"name": "Dr. NATHWANI JITENDRA PRABHUDAS", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B029", "practice_type": "-", "qualifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 2, "speciality": "NONE"}, "type": "add", "id": 2}, {"fields": {"name": "Dr. SHAH RASIK RAISHI", "postal_address": "P.O.Box 31597-00600 NAIROBI", "facility": "-", "reg_no": "B033", "practice_type": "-", "qualifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 3, "speciality": "NONE"}, "type": "add", "id": 3}, {"fields": {"name": "Dr. VERJEE NIZAR JAFFERALI", "postal_address": "P.O.Box 59745-00200 NAIROBI", "facility": "-", "reg_no": "B037", "practice_type": "-", "qualifications": "BDS(London) 1967 LDS RCS(Eng) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 4, "speciality": "NONE"}, "type": "add", "id": 4}, {"fields": {"name": "Dr. PATEL NIRA PANKAJ", "postal_address": "P.O.Box 14105-00800 NAIROBI", "facility": "-", "reg_no": "B049", "practice_type": "-", "qualifications": "BDS(Mysore) 1976 MRSH(london) 1977 FRSH(Lond) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 5, "speciality": "NONE"}, "type": "add", "id": 5}, {"fields": {"name": "Dr. SANGHANI PRAFUL MANGALJI", "postal_address": "P.O.Box 45071-00100 NAIROBI", "facility": "-", "reg_no": "B051", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 6, "speciality": "NONE"}, "type": "add", "id": 6}, {"fields": {"name": "Dr. SHAH RAMESH MEGHJI", "postal_address": "P.O.Box 66102-00800 NAIROBI", "facility": "-", "reg_no": "B052", "practice_type": "-", "qualifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 7, "speciality": "NONE"}, "type": "add", "id": 7}, {"fields": {"name": "Dr. SHAH AMRITLAL PREMCHAND", "postal_address": "P.O.Box 14191-00800 NAIROBI", "facility": "-", "reg_no": "B059", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 8, "speciality": "NONE"}, "type": "add", "id": 8}, {"fields": {"name": "Dr. KANYI EVA CHAMKOVA", "postal_address": "P.O.Box 727-10100 NYERI", "facility": "-", "reg_no": "B065", "practice_type": "-", "qualifications": "MD Stomatology (Komensky) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 9, "speciality": "NONE"}, "type": "add", "id": 9}, {"fields": {"name": "Dr. JANDU PARVIN SINGH", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B071", "practice_type": "-", "qualifications": "BDS(Bombay) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 10, "speciality": "NONE"}, "type": "add", "id": 10}, {"fields": {"name": "Dr. PATEL CHANDRAKANT BHAILALBHAI", "postal_address": "P.O.Box 81111-80100 MOMBASA", "facility": "-", "reg_no": "B072", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 11, "speciality": "NONE"}, "type": "add", "id": 11}, {"fields": {"name": "Dr. DESAI ASHOK NARENDRA", "postal_address": "P.O.Box 47605-00100 NAIROBI", "facility": "-", "reg_no": "B075", "practice_type": "-", "qualifications": "BDS(Dundee) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 12, "speciality": "NONE"}, "type": "add", "id": 12}, {"fields": {"name": "Dr. DESAI PARIMAL VITHALBHAI", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B080", "practice_type": "-", "qualifications": "BDS(Bombay) 1971", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 13, "speciality": "NONE"}, "type": "add", "id": 13}, {"fields": {"name": "Dr. PATEL MANDAKINI JAYENDRA", "postal_address": "P.O.Box 49545-00100 NAIROBI", "facility": "-", "reg_no": "B081", "practice_type": "-", "qualifications": "BDS(Bombay) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 14, "speciality": "NONE"}, "type": "add", "id": 14}, {"fields": {"name": "Dr. PATEL JAIMAN CHUNIBHAI", "postal_address": "P.O.Box 84420-80100 MOMBASA", "facility": "-", "reg_no": "B085", "practice_type": "-", "qualifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 15, "speciality": "NONE"}, "type": "add", "id": 15}, {"fields": {"name": "Dr. GITATA MUTHONI GITHUNGO", "postal_address": "P.O.Box 1769-00100 NAIROBI", "facility": "-", "reg_no": "B086", "practice_type": "-", "qualifications": "DDS(Nashville) 1973", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 16, "speciality": "NONE"}, "type": "add", "id": 16}, {"fields": {"name": "Dr. MAGON SWARNLATA", "postal_address": "P.O.Box 81431-80100 MOMBASA", "facility": "-", "reg_no": "B098", "practice_type": "-", "qualifications": "BDS(Punjabi) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 17, "speciality": "NONE"}, "type": "add", "id": 17}, {"fields": {"name": "Dr. PATEL PANKAJKUMAR RAOJIBHAI", "postal_address": "P.O.Box 43917-00100 NAIROBI", "facility": "-", "reg_no": "B114", "practice_type": "-", "qualifications": "BDS(Bombay) 1974 MRSH(London) 1975 FRSH(London) 1981 FADI(U.S.A) 1983", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 18, "speciality": "NONE"}, "type": "add", "id": 18}, {"fields": {"name": "Dr. NATHWANI BAKULA JITENDRA", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B118", "practice_type": "-", "qualifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 19, "speciality": "NONE"}, "type": "add", "id": 19}, {"fields": {"name": "Dr. WAGAIYU CHRISTOPHER KIBURI GICHURU", "postal_address": "P.O.Box 54197-00200 NAIROBI", "facility": "-", "reg_no": "B123", "practice_type": "-", "qualifications": "BDS(London) 1977 MSc(London) 1980 FDS.FRCS(Edin) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sub_speciality": "NONE", "id": 20, "speciality": "CONSERVATIVE DENTISTRY"}, "type": "add", "id": 20}, {"fields": {"name": "Dr. VAGHELA SATISH MAGANLAL", "postal_address": "P.O.Box 80198-80100 MOMBASA", "facility": "-", "reg_no": "B143", "practice_type": "-", "qualifications": "BDS(Gujarat) 1978", "reg_date": "1979-11-28T00:00:00.000Z", "sub_speciality": "NONE", "id": 21, "speciality": "NONE"}, "type": "add", "id": 21}, {"fields": {"name": "Dr. NJINO MICHAEL", "postal_address": "P.O.Box 13166-00200 NAIROBI", "facility": "-", "reg_no": "B144", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 22, "speciality": "NONE"}, "type": "add", "id": 22}, {"fields": {"name": "Dr. NDUNGU MICHAEL J.K.", "postal_address": "P.O.Box 47263-00100 NAIROBI", "facility": "-", "reg_no": "B145", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 23, "speciality": "NONE"}, "type": "add", "id": 23}, {"fields": {"name": "Dr. KIBUGI EDWIN WAMBUGU", "postal_address": "P.O.Box 54193-00200 NAIROBI", "facility": "-", "reg_no": "B150", "practice_type": "-", "qualifications": "BDS(Nairobi) 1978 MSc(London) 1983", "reg_date": "1980-02-29T00:00:00.000Z", "sub_speciality": "NONE", "id": 24, "speciality": "PROSTHODONTICS"}, "type": "add", "id": 24}, {"fields": {"name": "Dr. GRIFFITHS PETER DAVID", "postal_address": "P.O.Box 25672-00603 NAIROBI", "facility": "-", "reg_no": "B153", "practice_type": "-", "qualifications": "BDS(Birmingham) 1979", "reg_date": "1980-05-10T00:00:00.000Z", "sub_speciality": "NONE", "id": 25, "speciality": "NONE"}, "type": "add", "id": 25}, {"fields": {"name": "Dr. AWORI MARTIN WANDERA", "postal_address": "P.O.Box 19630-00202 NAIROBI", "facility": "-", "reg_no": "B168", "practice_type": "-", "qualifications": "BDS(Kerala) 1979 DDPH RCS(England) 1983 MIHE(UK) 1983", "reg_date": "1981-03-31T00:00:00.000Z", "sub_speciality": "NONE", "id": 26, "speciality": "PUBLIC HEALTH"}, "type": "add", "id": 26}, {"fields": {"name": "Dr. MUMENYA ROBINSON KOGI WAMBUGU", "postal_address": "P.O.Box 19320-00202 NAIROBI", "facility": "-", "reg_no": "B174", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979 FDS RCS(England) 1991 MSc(Oral Surgery)(London) 1991", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 27, "speciality": "ORAL AND MAXILLOFACIAL SURGERY"}, "type": "add", "id": 27}, {"fields": {"name": "Dr. JANI SAILESH GUNVANTRY", "postal_address": "P.O.Box 45640-00100 NAIROBI", "facility": "-", "reg_no": "B175", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 28, "speciality": "NONE"}, "type": "add", "id": 28}, {"fields": {"name": "Dr. CHINDIA MARK LUBISIA", "postal_address": "P.O.Box 19676-00202 NAIROBI", "facility": "-", "reg_no": "B176", "practice_type": "-", "qualifications": "BDS(Nairobi) 1979 FFDFCSI(Ireland) 1983 MSc(London) 1989", "reg_date": "1981-04-13T00:00:00.000Z", "sub_speciality": "NONE", "id": 29, "speciality": "ORAL AND MAXILLOFACIAL SURGERY"}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "doctors", "_id": 0, "_index": "healthtools"}}, {"name": "Dr. DEVANI JAYENDRA KMAR", "postal_address": "P.O.Box 43376-00100 NAIROBI", "facility": "-", "reg_no": "B021", "practice_type": "-", "qalifications": "BDS(Bombay) 1966 LDS RCS(Edin) 1968", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 0, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 1, "_index": "healthtools"}}, {"name": "Dr. VARSHANI NARAN SHIVJI", "postal_address": "P.O.Box 75977-00200 NAIROBI", "facility": "-", "reg_no": "B025", "practice_type": "-", "qalifications": "BDS(Gjarat) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 1, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 2, "_index": "healthtools"}}, {"name": "Dr. NATHWANI JITENDRA PRABHDAS", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B029", "practice_type": "-", "qalifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 2, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 3, "_index": "healthtools"}}, {"name": "Dr. SHAH RASIK RAISHI", "postal_address": "P.O.Box 31597-00600 NAIROBI", "facility": "-", "reg_no": "B033", "practice_type": "-", "qalifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 3, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 4, "_index": "healthtools"}}, {"name": "Dr. VERJEE NIZAR JAFFERALI", "postal_address": "P.O.Box 59745-00200 NAIROBI", "facility": "-", "reg_no": "B037", "practice_type": "-", "qalifications": "BDS(London) 1967 LDS RCS(Eng) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 4, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 5, "_index": "healthtools"}}, {"name": "Dr. PATEL NIRA PANKAJ", "postal_address": "P.O.Box 14105-00800 NAIROBI", "facility": "-", "reg_no": "B049", "practice_type": "-", "qalifications": "BDS(Mysore) 1976 MRSH(london) 1977 FRSH(Lond) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 5, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 6, "_index": "healthtools"}}, {"name": "Dr. SANGHANI PRAFL MANGALJI", "postal_address": "P.O.Box 45071-00100 NAIROBI", "facility": "-", "reg_no": "B051", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 6, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 7, "_index": "healthtools"}}, {"name": "Dr. SHAH RAMESH MEGHJI", "postal_address": "P.O.Box 66102-00800 NAIROBI", "facility": "-", "reg_no": "B052", "practice_type": "-", "qalifications": "BDS(Bombay) 1974", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 7, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 8, "_index": "healthtools"}}, {"name": "Dr. SHAH AMRITLAL PREMCHAND", "postal_address": "P.O.Box 14191-00800 NAIROBI", "facility": "-", "reg_no": "B059", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 8, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 9, "_index": "healthtools"}}, {"name": "Dr. KANYI EVA CHAMKOVA", "postal_address": "P.O.Box 727-10100 NYERI", "facility": "-", "reg_no": "B065", "practice_type": "-", "qalifications": "MD Stomatology (Komensky) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 9, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 10, "_index": "healthtools"}}, {"name": "Dr. JAND PARVIN SINGH", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B071", "practice_type": "-", "qalifications": "BDS(Bombay) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 10, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 11, "_index": "healthtools"}}, {"name": "Dr. PATEL CHANDRAKANT BHAILALBHAI", "postal_address": "P.O.Box 81111-80100 MOMBASA", "facility": "-", "reg_no": "B072", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 11, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 12, "_index": "healthtools"}}, {"name": "Dr. DESAI ASHOK NARENDRA", "postal_address": "P.O.Box 47605-00100 NAIROBI", "facility": "-", "reg_no": "B075", "practice_type": "-", "qalifications": "BDS(Dndee) 1975", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 12, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 13, "_index": "healthtools"}}, {"name": "Dr. DESAI PARIMAL VITHALBHAI", "postal_address": "P.O.Box 14198-00800 NAIROBI", "facility": "-", "reg_no": "B080", "practice_type": "-", "qalifications": "BDS(Bombay) 1971", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 13, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 14, "_index": "healthtools"}}, {"name": "Dr. PATEL MANDAKINI JAYENDRA", "postal_address": "P.O.Box 49545-00100 NAIROBI", "facility": "-", "reg_no": "B081", "practice_type": "-", "qalifications": "BDS(Bombay) 1966", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 14, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 15, "_index": "healthtools"}}, {"name": "Dr. PATEL JAIMAN CHNIBHAI", "postal_address": "P.O.Box 84420-80100 MOMBASA", "facility": "-", "reg_no": "B085", "practice_type": "-", "qalifications": "BDS(Bombay) 1972", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 15, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 16, "_index": "healthtools"}}, {"name": "Dr. GITATA MTHONI GITHNGO", "postal_address": "P.O.Box 1769-00100 NAIROBI", "facility": "-", "reg_no": "B086", "practice_type": "-", "qalifications": "DDS(Nashville) 1973", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 16, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 17, "_index": "healthtools"}}, {"name": "Dr. MAGON SWARNLATA", "postal_address": "P.O.Box 81431-80100 MOMBASA", "facility": "-", "reg_no": "B098", "practice_type": "-", "qalifications": "BDS(Pnjabi) 1967", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 17, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 18, "_index": "healthtools"}}, {"name": "Dr. PATEL PANKAJKMAR RAOJIBHAI", "postal_address": "P.O.Box 43917-00100 NAIROBI", "facility": "-", "reg_no": "B114", "practice_type": "-", "qalifications": "BDS(Bombay) 1974 MRSH(London) 1975 FRSH(London) 1981 FADI(.S.A) 1983", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 18, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 19, "_index": "healthtools"}}, {"name": "Dr. NATHWANI BAKLA JITENDRA", "postal_address": "P.O.Box 63501-00619 NAIROBI", "facility": "-", "reg_no": "B118", "practice_type": "-", "qalifications": "BDS(Bombay) 1976", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 19, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 20, "_index": "healthtools"}}, {"name": "Dr. WAGAIY CHRISTOPHER KIBRI GICHR", "postal_address": "P.O.Box 54197-00200 NAIROBI", "facility": "-", "reg_no": "B123", "practice_type": "-", "qalifications": "BDS(London) 1977 MSc(London) 1980 FDS.FRCS(Edin) 1982", "reg_date": "1978-01-01T00:00:00.000Z", "sb_speciality": "NONE", "id": 20, "speciality": "CONSERVATIVE DENTISTRY"}, {"index": {"_type": "doctors", "_id": 21, "_index": "healthtools"}}, {"name": "Dr. VAGHELA SATISH MAGANLAL", "postal_address": "P.O.Box 80198-80100 MOMBASA", "facility": "-", "reg_no": "B143", "practice_type": "-", "qalifications": "BDS(Gjarat) 1978", "reg_date": "1979-11-28T00:00:00.000Z", "sb_speciality": "NONE", "id": 21, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 22, "_index": "healthtools"}}, {"name": "Dr. NJINO MICHAEL", "postal_address": "P.O.Box 13166-00200 NAIROBI", "facility": "-", "reg_no": "B144", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 22, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 23, "_index": "healthtools"}}, {"name": "Dr. NDNG MICHAEL J.K.", "postal_address": "P.O.Box 47263-00100 NAIROBI", "facility": "-", "reg_no": "B145", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978", "reg_date": "1979-12-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 23, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 24, "_index": "healthtools"}}, {"name": "Dr. KIBGI EDWIN WAMBG", "postal_address": "P.O.Box 54193-00200 NAIROBI", "facility": "-", "reg_no": "B150", "practice_type": "-", "qalifications": "BDS(Nairobi) 1978 MSc(London) 1983", "reg_date": "1980-02-29T00:00:00.000Z", "sb_speciality": "NONE", "id": 24, "speciality": "PROSTHODONTICS"}, {"index": {"_type": "doctors", "_id": 25, "_index": "healthtools"}}, {"name": "Dr. GRIFFITHS PETER DAVID", "postal_address": "P.O.Box 25672-00603 NAIROBI", "facility": "-", "reg_no": "B153", "practice_type": "-", "qalifications": "BDS(Birmingham) 1979", "reg_date": "1980-05-10T00:00:00.000Z", "sb_speciality": "NONE", "id": 25, "speciality": "NONE"}, {"index": {"_type": "doctors", "_id": 26, "_index": "healthtools"}}, {"name": "Dr. AWORI MARTIN WANDERA", "postal_address": "P.O.Box 19630-00202 NAIROBI", "facility": "-", "reg_no": "B168", "practice_type": "-", "qalifications": "BDS(Kerala) 1979 DDPH RCS(England) 1983 MIHE(K) 1983", "reg_date": "1981-03-31T00:00:00.000Z", "sb_speciality": "NONE", "id": 26, "speciality": "PBLIC HEALTH"}, {"index": {"_type": "doctors", "_id": 27, "_index": "healthtools"}}, {"name": "Prof. MACIGO FRANCIS GITHA", "postal_address": "P.O.Box 2917-00100 NAIROBI", "facility": "-", "reg_no": "B173", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979 MPH(Nairobi) 1990", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 27, "speciality": "PBLIC HEALTH"}, {"index": {"_type": "doctors", "_id": 28, "_index": "healthtools"}}, {"name": "Dr. MMENYA ROBINSON KOGI WAMBG", "postal_address": "P.O.Box 19320-00202 NAIROBI", "facility": "-", "reg_no": "B174", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979 FDS RCS(England) 1991 MSc(Oral Srgery)(London) 1991", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 28, "speciality": "ORAL AND MAXILLOFACIAL SRGERY"}, {"index": {"_type": "doctors", "_id": 29, "_index": "healthtools"}}, {"name": "Dr. JANI SAILESH GNVANTRY", "postal_address": "P.O.Box 45640-00100 NAIROBI", "facility": "-", "reg_no": "B175", "practice_type": "-", "qalifications": "BDS(Nairobi) 1979", "reg_date": "1981-04-13T00:00:00.000Z", "sb_speciality": "NONE", "id": 29, "speciality": "NONE"}] diff --git a/healthtools/tests/dummy_files/foreign_doctors.json b/healthtools/tests/dummy_files/foreign_doctors.json index bbcb7bd..c458e10 100644 --- a/healthtools/tests/dummy_files/foreign_doctors.json +++ b/healthtools/tests/dummy_files/foreign_doctors.json @@ -1 +1 @@ -[{"fields": {"name": "DR NARAYAN VIJAYA KUMAR", "postal_address": "P.O BOX 39173 00623 NAIROBI", "facility": "CANCER CARE KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(CALCUTTA)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 0, "speciality": "-"}, "type": "add", "id": 0}, {"fields": {"name": "DR SMITHSON HELEN CLAIRE", "postal_address": "P.O BOX 63 60600 MAUA", "facility": "MAUA METHODIST HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(NEW CASTLE)1984", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 1, "speciality": "-"}, "type": "add", "id": 1}, {"fields": {"name": "DR ADAM \u00a0MARY", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(ARIZONA)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 2, "speciality": "-"}, "type": "add", "id": 2}, {"fields": {"name": "DR AHSAN \u00a0SABBIR", "postal_address": "P.O BOX 32557 00600 NAIROBI", "facility": "TAIBA MEDICAL CENTRE", "reg_no": "-", "practice_type": "Dental", "qualifications": "BDS(DHAKA)2011", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 3, "speciality": "-"}, "type": "add", "id": 3}, {"fields": {"name": "DR AKANKSHA ARORA SHARMA", "postal_address": "P.O BOX 5587 00506 NAIROBI", "facility": "MEDICROSS LTD", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(INDORE)2007", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 4, "speciality": "-"}, "type": "add", "id": 4}, {"fields": {"name": "DR ANNA ELIZABETH MONROE", "postal_address": "P.O BOX", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(STANFORD)2009", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 5, "speciality": "-"}, "type": "add", "id": 5}, {"fields": {"name": "DR ARIMPUR MARY PAUL", "postal_address": "P.O BOX 49682 00100 NAIROBI", "facility": "NAZARETH HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(CALCUTTA)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 6, "speciality": "-"}, "type": "add", "id": 6}, {"fields": {"name": "DR ASOKAN \u00a0LINI", "postal_address": "P.O BOX 30577 00100 NAIROBI", "facility": "COLUMBIA AFRICA HEALTHCARE LTD", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(M.S.RAMAIAH)1998", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 7, "speciality": "-"}, "type": "add", "id": 7}, {"fields": {"name": "DR ATKINSON SARAH HELEN", "postal_address": "P.O BOX 230 80108 KILIFI", "facility": "KEMRI WELLCOME TRUST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS(LONDON)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 8, "speciality": "-"}, "type": "add", "id": 8}, {"fields": {"name": "DR AUGUST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 9, "speciality": "-"}, "type": "add", "id": 9}, {"fields": {"name": "DR AUGUST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 10, "speciality": "-"}, "type": "add", "id": 10}, {"fields": {"name": "DR BADANO \u00a0SIMONA", "postal_address": "P.O BOX 88 20318 NORTH KINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(GENEVA)2001", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 11, "speciality": "-"}, "type": "add", "id": 11}, {"fields": {"name": "DR BAL RAJPREET KAUR", "postal_address": "P.O BOX 46206 00100 NAIROBI", "facility": "THE AGA KHAN UNIVERSITY HOSPITAL - NAIROBI", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(NORTH WESTERN)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 12, "speciality": "-"}, "type": "add", "id": 12}, {"fields": {"name": "DR BARBOSA SIERRA DIEGO", "postal_address": "P.O BOX 18 \u00a0LODWAR", "facility": "LODWAR COUNTY AND REFERRAL HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MADRID)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 13, "speciality": "-"}, "type": "add", "id": 13}, {"fields": {"name": "DR BASHIR MOHAMMED AHMED", "postal_address": "P.O BOX 530 40100 KISUMU", "facility": "AGA KHAN HOSPITAL KISUMU", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBCHB(KAMPALA)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 14, "speciality": "-"}, "type": "add", "id": 14}, {"fields": {"name": "DR BEGEMANN HERALD OTTFRIED", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(HAMBURG)1980", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 15, "speciality": "-"}, "type": "add", "id": 15}, {"fields": {"name": "DR BEKHEET TOFELES \u00a0MARIAN MORRIS", "postal_address": "P.O BOX 21570 00505 NAIROBI", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBCH(AIN SHAMS)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 16, "speciality": "-"}, "type": "add", "id": 16}, {"fields": {"name": "DR BERKLEY JAMES ALEXANDER", "postal_address": "P.O BOX 428 80108 KILIFI", "facility": "KEMRI/WELLCOME TRUST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MBBS NEWCASTLE UPON TYNE)", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 17, "speciality": "-"}, "type": "add", "id": 17}, {"fields": {"name": "DR JORG \u00a0BERLING", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FREIBURG)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 18, "speciality": "-"}, "type": "add", "id": 18}, {"fields": {"name": "DR BHATT MEGHNA MEHUL", "postal_address": "P.O BOX 30270 00100 NAIROBI", "facility": "AGA KHAN UNIVERSITY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "BDS(DEEMED)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 19, "speciality": "-"}, "type": "add", "id": 19}, {"fields": {"name": "DR BOEDEMANN \u00a0MELANIE", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE ANGLICAN CHURCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 20, "speciality": "-"}, "type": "add", "id": 20}, {"fields": {"name": "DR BOEVE \u00a0NORMAN", "postal_address": "P.O BOX 20 0220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MICHIGAN)1965", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 21, "speciality": "-"}, "type": "add", "id": 21}, {"fields": {"name": "DR JENNY \u00a0BOYD", "postal_address": "P.O BOX 39 20400 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(TENNESSEE)2003", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 22, "speciality": "-"}, "type": "add", "id": 22}, {"fields": {"name": "DR BROWN ROGER KEVIN", "postal_address": "P.O BOX 21171 00505 NAIROBI", "facility": "AFRICAN INLAND MISSION INTERNATIONAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(ILLINOIS)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 23, "speciality": "-"}, "type": "add", "id": 23}, {"fields": {"name": "DR BUDACH RUTH MAGDALENE", "postal_address": "P.O BOX 886 00900 KIAMBU", "facility": "THE AGLICAN CHURCH OF KENYA - MOTHERS MERCY HOME", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(FRANKFURT)1978", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 24, "speciality": "-"}, "type": "add", "id": 24}, {"fields": {"name": "DR BURGERT STEPHEN LOUIS", "postal_address": "P.O BOX 39 \u00a0BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(MAYO)1979", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 25, "speciality": "-"}, "type": "add", "id": 25}, {"fields": {"name": "DR CARBONE \u00a0MARCO", "postal_address": "P.O BOX 88 20318 NORTHKINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(GENOVA)1993", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 26, "speciality": "-"}, "type": "add", "id": 26}, {"fields": {"name": "DR CARTER ROBERT ALLEN", "postal_address": "P.O BOX 20 \u00a000220 KIJABE", "facility": "A I C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(INDIANA)1981", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 27, "speciality": "-"}, "type": "add", "id": 27}, {"fields": {"name": "DR CHANG SANG HO", "postal_address": "P.O BOX 20954 00202 NAIROBI", "facility": "CHRISTIAN MEDICAL & DENTAL ASSOCIATION OF KENYA", "reg_no": "-", "practice_type": "Medical", "qualifications": "MD(SEOUL)1977", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 28, "speciality": "-"}, "type": "add", "id": 28}, {"fields": {"name": "DR COLLIS FABIAN BERNARD", "postal_address": "P.O BOX 45 00902 KIKUYU", "facility": "P.C.E.A KIKUYU HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qualifications": "MSCH MBCHA(SAME)1999", "reg_date": "0000-01-01T00:00:00.000Z", "sub_speciality": "-", "id": 29, "speciality": "-"}, "type": "add", "id": 29}] \ No newline at end of file +[{"index": {"_type": "doctors", "_id": 0, "_index": "healthtools"}}, {"name": "DR NARAYAN VIJAYA KMAR", "postal_address": "P.O BOX 39173 00623 NAIROBI", "facility": "CANCER CARE KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(CALCTTA)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 0, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 1, "_index": "healthtools"}}, {"name": "DR SMITHSON HELEN CLAIRE", "postal_address": "P.O BOX 63 60600 MAA", "facility": "MAA METHODIST HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(NEW CASTLE)1984", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 1, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 2, "_index": "healthtools"}}, {"name": "DR ADAM MARY", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(ARIZONA)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 2, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 3, "_index": "healthtools"}}, {"name": "DR AHSAN SABBIR", "postal_address": "P.O BOX 32557 00600 NAIROBI", "facility": "TAIBA MEDICAL CENTRE", "reg_no": "-", "practice_type": "Dental", "qalifications": "BDS(DHAKA)2011", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 3, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 4, "_index": "healthtools"}}, {"name": "DR AKANKSHA ARORA SHARMA", "postal_address": "P.O BOX 5587 00506 NAIROBI", "facility": "MEDICROSS LTD", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(INDORE)2007", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 4, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 5, "_index": "healthtools"}}, {"name": "DR ANNA ELIZABETH MONROE", "postal_address": "P.O BOX", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(STANFORD)2009", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 5, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 6, "_index": "healthtools"}}, {"name": "DR ARIMPR MARY PAL", "postal_address": "P.O BOX 49682 00100 NAIROBI", "facility": "NAZARETH HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(CALCTTA)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 6, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 7, "_index": "healthtools"}}, {"name": "DR ASOKAN LINI", "postal_address": "P.O BOX 30577 00100 NAIROBI", "facility": "COLMBIA AFRICA HEALTHCARE LTD", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(M.S.RAMAIAH)1998", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 7, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 8, "_index": "healthtools"}}, {"name": "DR ATKINSON SARAH HELEN", "postal_address": "P.O BOX 230 80108 KILIFI", "facility": "KEMRI WELLCOME TRST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS(LONDON)1997", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 8, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 9, "_index": "healthtools"}}, {"name": "DR AGST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 9, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 10, "_index": "healthtools"}}, {"name": "DR AGST BERGER THOMAS", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1983", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 10, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 11, "_index": "healthtools"}}, {"name": "DR BADANO SIMONA", "postal_address": "P.O BOX 88 20318 NORTH KINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(GENEVA)2001", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 11, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 12, "_index": "healthtools"}}, {"name": "DR BAL RAJPREET KAR", "postal_address": "P.O BOX 46206 00100 NAIROBI", "facility": "THE AGA KHAN NIVERSITY HOSPITAL - NAIROBI", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(NORTH WESTERN)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 12, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 13, "_index": "healthtools"}}, {"name": "DR BARBOSA SIERRA DIEGO", "postal_address": "P.O BOX 18 LODWAR", "facility": "LODWAR CONTY AND REFERRAL HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MADRID)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 13, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 14, "_index": "healthtools"}}, {"name": "DR BASHIR MOHAMMED AHMED", "postal_address": "P.O BOX 530 40100 KISM", "facility": "AGA KHAN HOSPITAL KISM", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBCHB(KAMPALA)2010", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 14, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 15, "_index": "healthtools"}}, {"name": "DR BEGEMANN HERALD OTTFRIED", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(HAMBRG)1980", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 15, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 16, "_index": "healthtools"}}, {"name": "DR BEKHEET TOFELES MARIAN MORRIS", "postal_address": "P.O BOX 21570 00505 NAIROBI", "facility": "COPTIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBCH(AIN SHAMS)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 16, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 17, "_index": "healthtools"}}, {"name": "DR BERKLEY JAMES ALEXANDER", "postal_address": "P.O BOX 428 80108 KILIFI", "facility": "KEMRI/WELLCOME TRST RESEARCH PROGRAMME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MBBS NEWCASTLE PON TYNE)", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 17, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 18, "_index": "healthtools"}}, {"name": "DR JORG BERLING", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FREIBRG)1985", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 18, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 19, "_index": "healthtools"}}, {"name": "DR BHATT MEGHNA MEHL", "postal_address": "P.O BOX 30270 00100 NAIROBI", "facility": "AGA KHAN NIVERSITY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "BDS(DEEMED)2002", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 19, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 20, "_index": "healthtools"}}, {"name": "DR BOEDEMANN MELANIE", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE ANGLICAN CHRCH OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 20, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 21, "_index": "healthtools"}}, {"name": "DR BOEVE NORMAN", "postal_address": "P.O BOX 20 0220 KIJABE", "facility": "A.I.C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MICHIGAN)1965", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 21, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 22, "_index": "healthtools"}}, {"name": "DR JENNY BOYD", "postal_address": "P.O BOX 39 20400 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(TENNESSEE)2003", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 22, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 23, "_index": "healthtools"}}, {"name": "DR BROWN ROGER KEVIN", "postal_address": "P.O BOX 21171 00505 NAIROBI", "facility": "AFRICAN INLAND MISSION INTERNATIONAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(ILLINOIS)1982", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 23, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 24, "_index": "healthtools"}}, {"name": "DR BDACH RTH MAGDALENE", "postal_address": "P.O BOX 886 00900 KIAMB", "facility": "THE AGLICAN CHRCH OF KENYA - MOTHERS MERCY HOME", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(FRANKFRT)1978", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 24, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 25, "_index": "healthtools"}}, {"name": "DR BRGERT STEPHEN LOIS", "postal_address": "P.O BOX 39 BOMET", "facility": "TENWEK HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(MAYO)1979", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 25, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 26, "_index": "healthtools"}}, {"name": "DR CARBONE MARCO", "postal_address": "P.O BOX 88 20318 NORTHKINANGOP", "facility": "NORTH KINANGOP CATHOLIC HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(GENOVA)1993", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 26, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 27, "_index": "healthtools"}}, {"name": "DR CARTER ROBERT ALLEN", "postal_address": "P.O BOX 20 00220 KIJABE", "facility": "A I C KIJABE HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(INDIANA)1981", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 27, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 28, "_index": "healthtools"}}, {"name": "DR CHANG SANG HO", "postal_address": "P.O BOX 20954 00202 NAIROBI", "facility": "CHRISTIAN MEDICAL & DENTAL ASSOCIATION OF KENYA", "reg_no": "-", "practice_type": "Medical", "qalifications": "MD(SEOL)1977", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 28, "speciality": "-"}, {"index": {"_type": "doctors", "_id": 29, "_index": "healthtools"}}, {"name": "DR COLLIS FABIAN BERNARD", "postal_address": "P.O BOX 45 00902 KIKY", "facility": "P.C.E.A KIKY HOSPITAL", "reg_no": "-", "practice_type": "Medical", "qalifications": "MSCH MBCHA(SAME)1999", "reg_date": "0000-01-01T00:00:00.000Z", "sb_speciality": "-", "id": 29, "speciality": "-"}] diff --git a/healthtools/tests/dummy_files/health_facilities.json b/healthtools/tests/dummy_files/health_facilities.json new file mode 100644 index 0000000..c43ab36 --- /dev/null +++ b/healthtools/tests/dummy_files/health_facilities.json @@ -0,0 +1 @@ +[{"index": {"_type": "health-facilities", "_id": 16188, "_index": "healthtools"}}, {"sub_county_name": "kaloleni", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mariakani Community Health Care Services", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "KALOLENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16188, "ward_name": "MARIAKANI"}, {"index": {"_type": "health-facilities", "_id": 10359, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ithe-Kahuno Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10359, "ward_name": "AGUTHI-GAAKI"}, {"index": {"_type": "health-facilities", "_id": 14898, "_index": "healthtools"}}, {"sub_county_name": "kapseret", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kipkenyo Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KAPSERET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "UASIN GISH", "open_late_night": "False", "keph_level_name": "Level 2", "id": 14898, "ward_name": "KIPKENYO"}, {"index": {"_type": "health-facilities", "_id": 10652, "_index": "healthtools"}}, {"sub_county_name": "kieni west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Labura/Babito Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": ["Short Term", "Natural", "Long Term", "Condom Distribution & STI Prevention", "HIV Testing Services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10652, "ward_name": "MWIYOGO/ENDARASHA"}, {"index": {"_type": "health-facilities", "_id": 13655, "_index": "healthtools"}}, {"sub_county_name": "bondo", "facility_type_name": "Basic primary health care facility", "owner_name": "Ministry of Health", "name": "Kapiyo Health Centre", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "BONDO", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "SIAYA", "open_late_night": "False", "keph_level_name": "Level 3", "id": 13655, "ward_name": "WEST SAKWA"}, {"index": {"_type": "health-facilities", "_id": 10278, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Gituamba Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10278, "ward_name": "GITUAMBA"}, {"index": {"_type": "health-facilities", "_id": 10616, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kinunga Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10616, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 10800, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mutitu Gikondi Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10800, "ward_name": "GIKONDI"}, {"index": {"_type": "health-facilities", "_id": 10042, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Atlas Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10042, "ward_name": "IRIA-INI"}, {"index": {"_type": "health-facilities", "_id": 10730, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mihuti Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10730, "ward_name": "RUGI"}, {"index": {"_type": "health-facilities", "_id": 11019, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga north/mwea west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ssema Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11019, "ward_name": "MUTITHI"}, {"index": {"_type": "health-facilities", "_id": 10679, "_index": "healthtools"}}, {"sub_county_name": "ndaragwa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Manguo Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NDARAGWA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYANDARUA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10679, "ward_name": "KIRITTA"}, {"index": {"_type": "health-facilities", "_id": 11125, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Tunuku Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11125, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 12124, "_index": "healthtools"}}, {"sub_county_name": "tharaka north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kageni Med Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "THARAKA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "THARAKA-NITHI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12124, "ward_name": "MUKOTHIMA"}, {"index": {"_type": "health-facilities", "_id": 10605, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kimahuri Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10605, "ward_name": "KABAR"}, {"index": {"_type": "health-facilities", "_id": 10393, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kabuti Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GICHUG", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10393, "ward_name": "BARAGWI"}, {"index": {"_type": "health-facilities", "_id": 10358, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ithare Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GICHUG", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10358, "ward_name": "KABARE"}, {"index": {"_type": "health-facilities", "_id": 16182, "_index": "healthtools"}}, {"sub_county_name": "rabai", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Khadija Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "RABAI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16182, "ward_name": "MWAWESA"}, {"index": {"_type": "health-facilities", "_id": 10741, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mt Kenya Narumoru Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10741, "ward_name": "NAROMORU/KIAMATHAGA"}, {"index": {"_type": "health-facilities", "_id": 10849, "_index": "healthtools"}}, {"sub_county_name": "mathioya", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "New Kihoya Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KANGEMA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10849, "ward_name": "RWATHIA"}, {"index": {"_type": "health-facilities", "_id": 11820, "_index": "healthtools"}}, {"sub_county_name": "magarini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "St Mary's Medical Clinic (Malindi)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "MAGARINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11820, "ward_name": "AD"}, {"index": {"_type": "health-facilities", "_id": 10047, "_index": "healthtools"}}, {"sub_county_name": "kihar", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Baraka Medical Clinic (Muranga North)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIHAR", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10047, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 13460, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Zakma Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13460, "ward_name": "GALBET"}, {"index": {"_type": "health-facilities", "_id": 13326, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Dertu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13326, "ward_name": "GALBET"}, {"index": {"_type": "health-facilities", "_id": 10168, "_index": "healthtools"}}, {"sub_county_name": "kieni west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Emmanuel Medical Clinic (Nyeri North)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": ["Short Term", "Natural", "Long Term", "Condom Distribution & STI Prevention", "HIV Testing Services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10168, "ward_name": "MUGUNDA"}, {"index": {"_type": "health-facilities", "_id": 14989, "_index": "healthtools"}}, {"sub_county_name": "pokot north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kopeeto Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "KACHELIBA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "WEST POKOT", "open_late_night": "False", "keph_level_name": "Level 2", "id": 14989, "ward_name": "ALALE"}, {"index": {"_type": "health-facilities", "_id": 12346, "_index": "healthtools"}}, {"sub_county_name": "kitui west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kiseveni Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KITUI WEST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12346, "ward_name": "KAUWI"}, {"index": {"_type": "health-facilities", "_id": 15661, "_index": "healthtools"}}, {"sub_county_name": "turkana central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Kenya Episcopal Conference-Catholic Secretariat", "name": "St Monica Nakwamekwi Dispensary", "regulatory_body_name": "Kenya MPDB - Institution", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TURKANA CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "TURKANA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 15661, "ward_name": "LODWAR TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 10416, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kahara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10416, "ward_name": "MUKURWE-INI CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 10160, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ebenezer Medical Clinic (Nyeri South)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10160, "ward_name": "RWARE"}, {"index": {"_type": "health-facilities", "_id": 10501, "_index": "healthtools"}}, {"sub_county_name": "nyeri central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kariumba Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NYERI TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10501, "ward_name": "KAMAKWA/MUKARO"}, {"index": {"_type": "health-facilities", "_id": 10002, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Aberdare Medical & Surgical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10002, "ward_name": "IRIA-INI"}, {"index": {"_type": "health-facilities", "_id": 12218, "_index": "healthtools"}}, {"sub_county_name": "mwingi central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Katethya Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWINGI CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12218, "ward_name": "CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 11088, "_index": "healthtools"}}, {"sub_county_name": "kihar", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Tewas Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIHAR", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11088, "ward_name": "WANG"}, {"index": {"_type": "health-facilities", "_id": 10795, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Muthuthiini Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10795, "ward_name": "GIKONDI"}, {"index": {"_type": "health-facilities", "_id": 16447, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Safi Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 16447, "ward_name": "NYANGUTI"}, {"index": {"_type": "health-facilities", "_id": 10206, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Gatamu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10206, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 11168, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Wanduta Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11168, "ward_name": "GITHOBOKONI"}, {"index": {"_type": "health-facilities", "_id": 10108, "_index": "healthtools"}}, {"sub_county_name": "kieni east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Diana Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KIENI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10108, "ward_name": "NAROMORU/KIAMATHAGA"}, {"index": {"_type": "health-facilities", "_id": 10754, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mukarara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10754, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 12089, "_index": "healthtools"}}, {"sub_county_name": "mwala", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Imani Yako Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWALA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MACHAKOS", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12089, "ward_name": "WAMUNY"}, {"index": {"_type": "health-facilities", "_id": 10611, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Kimondo Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10611, "ward_name": "MUKURWE-INI CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 10835, "_index": "healthtools"}}, {"sub_county_name": "gatundu north", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Ndonyero Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GATUNDU NORTH", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10835, "ward_name": "MANGU"}, {"index": {"_type": "health-facilities", "_id": 11137, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Upendo Medical Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11137, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 13377, "_index": "healthtools"}}, {"sub_county_name": "garissa", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Jaribu Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "GARISSA TOWNSHIP", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "GARISSA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13377, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 12160, "_index": "healthtools"}}, {"sub_county_name": "kitui central", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kamandio Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "KITUI CENTRAL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12160, "ward_name": "MIAMBANI"}, {"index": {"_type": "health-facilities", "_id": 11944, "_index": "healthtools"}}, {"sub_county_name": "mwingi central", "facility_type_name": "Basic primary health care facility", "owner_name": "Private Practice - Clinical Officer", "name": "Salem Medical Services", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWINGI CENTRAL", "approved": "True", "service_names": ["Short Term", "Integrated Management of Childhood Illnesses", "Postnatal care services", "Class B", "Focused Antenatal Care", "Stand Alone - Retail services", "Outpatient"], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KITUI", "open_late_night": "False", "keph_level_name": "Level 3", "id": 11944, "ward_name": "CENTRAL"}, {"index": {"_type": "health-facilities", "_id": 11780, "_index": "healthtools"}}, {"sub_county_name": "GARSEN", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Sera Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "True", "open_weekends": "False", "constituency_name": "GARSEN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "TANA RIVER", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11780, "ward_name": "GARSEN NORTH"}, {"index": {"_type": "health-facilities", "_id": 12397, "_index": "healthtools"}}, {"sub_county_name": "mwala", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Kwakala Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWALA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MACHAKOS", "open_late_night": "False", "keph_level_name": "Level 2", "id": 12397, "ward_name": "WAMUNY"}, {"index": {"_type": "health-facilities", "_id": 11181, "_index": "healthtools"}}, {"sub_county_name": "laikipia west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Clinical Officer", "name": "Wellness Medical Clinic", "regulatory_body_name": "Clinical Officers Council", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "LAIKIPIA WEST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "LAIKIPIA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11181, "ward_name": "IGWAMITI"}, {"index": {"_type": "health-facilities", "_id": 10908, "_index": "healthtools"}}, {"sub_county_name": "mathioya", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Obeys Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MATHIOYA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MURANG'A", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10908, "ward_name": "KIR"}, {"index": {"_type": "health-facilities", "_id": 10106, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Delina Health Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10106, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 10074, "_index": "healthtools"}}, {"sub_county_name": "thika town", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Boore Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "THIKA TOWN", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIAMB", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10074, "ward_name": "TOWNSHIP"}, {"index": {"_type": "health-facilities", "_id": 10710, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Mayos Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10710, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 10707, "_index": "healthtools"}}, {"sub_county_name": "nyeri south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - General Practitioner", "name": "Mathingira Medical Clinic", "regulatory_body_name": "Kenya MPDB - Private Practice", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "Nyeri South", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10707, "ward_name": "KARIMA"}, {"index": {"_type": "health-facilities", "_id": 10882, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Njika Wega Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10882, "ward_name": "RUGI"}, {"index": {"_type": "health-facilities", "_id": 10022, "_index": "healthtools"}}, {"sub_county_name": "tet", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Alpha Family Health Care", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "TET", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 3", "id": 10022, "ward_name": "WAMAGANA"}, {"index": {"_type": "health-facilities", "_id": 10297, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga north/mwea west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Good Samaritan Health Services (Kirinyaga)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10297, "ward_name": "MUTITHI"}, {"index": {"_type": "health-facilities", "_id": 10879, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga south", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Enterprise (Institution)", "name": "Njambi Nursing Home", "regulatory_body_name": "Kenya MPDB - Institution", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MWEA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10879, "ward_name": "GATHIGIRIRI"}, {"index": {"_type": "health-facilities", "_id": 13888, "_index": "healthtools"}}, {"sub_county_name": "kasipul", "facility_type_name": "Basic primary health care facility", "owner_name": "Private Practice - General Practitioner", "name": "Nyalenda Nursing Home", "regulatory_body_name": "Kenya MPDB - Private Practice", "open_whole_day": "True", "open_weekends": "True", "constituency_name": "KASIPUL", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "HOMA BAY", "open_late_night": "False", "keph_level_name": "Level 3", "id": 13888, "ward_name": "CENTRAL KASIPUL"}, {"index": {"_type": "health-facilities", "_id": 13396, "_index": "healthtools"}}, {"sub_county_name": "mandera east", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Libehiya Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MANDERA EAST", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "MANDERA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 13396, "ward_name": "ARABIA"}, {"index": {"_type": "health-facilities", "_id": 11256, "_index": "healthtools"}}, {"sub_county_name": "magarini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Ministry of Health", "name": "Bombi Dispensary", "regulatory_body_name": "Ministry of Health", "open_whole_day": "False", "open_weekends": "True", "constituency_name": "MAGARINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KILIFI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 11256, "ward_name": "AD"}, {"index": {"_type": "health-facilities", "_id": 10325, "_index": "healthtools"}}, {"sub_county_name": "kirinyaga west", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Hope Medical Clinic (Kirinyaga)", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "NDIA", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "KIRINYAGA", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10325, "ward_name": "KIINE"}, {"index": {"_type": "health-facilities", "_id": 10680, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Manjo Health Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10680, "ward_name": "MUKURWE-INI WEST"}, {"index": {"_type": "health-facilities", "_id": 10356, "_index": "healthtools"}}, {"sub_county_name": "mukurweini", "facility_type_name": "Dispensaries and clinic-out patient only", "owner_name": "Private Practice - Nurse / Midwifery", "name": "Itara Medical Clinic", "regulatory_body_name": "Nursing Council of Kenya (Private Practice)", "open_whole_day": "False", "open_weekends": "False", "constituency_name": "MUKURWEINI", "approved": "True", "service_names": [], "operation_status_name": "Operational", "open_public_holidays": "False", "county_name": "NYERI", "open_late_night": "False", "keph_level_name": "Level 2", "id": 10356, "ward_name": "MUKURWE-INI WEST"}] diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index c406256..8585f62 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -1,7 +1,9 @@ import unittest +import json from healthtools.scrapers.doctors import DoctorsScraper from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper +from healthtools.scrapers.health_facilities import HealthFacilitiesScraper from healthtools.config import TEST_DIR @@ -10,6 +12,7 @@ def setUp(self): self.doctors_scraper = DoctorsScraper() self.foreign_doctors_scraper = ForeignDoctorsScraper() self.clinical_officers_scraper = ClinicalOfficersScraper() + self.health_facilities_scraper = HealthFacilitiesScraper() def test_it_gets_the_total_number_of_pages(self): self.doctors_scraper.get_total_number_of_pages() @@ -18,47 +21,45 @@ def test_it_gets_the_total_number_of_pages(self): def test_it_scrapes_doctors_page(self): entries = self.doctors_scraper.scrape_page( "http://medicalboard.co.ke/online-services/retention/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 10) + self.assertTrue(len(entries) == 60) def test_it_scrapes_foreign_doctors_page(self): entries = self.foreign_doctors_scraper.scrape_page( "http://medicalboard.co.ke/online-services/foreign-doctors-license-register/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 10) + self.assertTrue(len(entries) == 60) def test_it_scrapes_clinical_officers_page(self): entries = self.clinical_officers_scraper.scrape_page( "http://clinicalofficerscouncil.org/online-services/retention/?currpage=1")[0] - self.assertTrue(len(entries[0]["fields"]) == 7) + self.assertTrue(len(entries) == 60) - def test_it_scrapes_whole_doctors_site(self): - all_entries = self.doctors_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_it_scrapes_whole_foreign_doctors_site(self): - all_entries = self.foreign_doctors_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_it_scrapes_whole_clinical_officers_site(self): - all_entries = self.clinical_officers_scraper.scrape_site() - self.assertTrue(len(all_entries) > 0) - - def test_doctors_scraper_uploads_to_cloudsearch(self): + def test_doctors_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/doctors.json", "r") as my_file: data = my_file.read() - response = self.doctors_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.doctors_scraper.delete_elasticsearch_docs() + response = self.doctors_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) - def test_foreign_doctors_scraper_uploads_to_cloudsearch(self): + def test_foreign_doctors_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/foreign_doctors.json", "r") as my_file: data = my_file.read() - response = self.foreign_doctors_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.foreign_doctors_scraper.delete_elasticsearch_docs() + response = self.foreign_doctors_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 200) - def test_clinical_officers_scraper_uploads_to_cloudsearch(self): + def test_clinical_officers_scraper_uploads_to_elasticsearch(self): with open(TEST_DIR + "/dummy_files/clinical_officers.json", "r") as my_file: data = my_file.read() - response = self.clinical_officers_scraper.upload_data(data) - self.assertEqual(response.get('status'), "success") + self.clinical_officers_scraper.delete_elasticsearch_docs() + response = self.clinical_officers_scraper.upload_data(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) + + def test_health_facilities_scraper_uploads_to_elasticsearch(self): + with open(TEST_DIR + "/dummy_files/health_facilities.json", "r") as my_file: + data = my_file.read() + self.health_facilities_scraper.delete_elasticsearch_docs() + response = self.health_facilities_scraper.upload(json.loads(data)) + self.assertEqual(response['items'][0]['index']['status'], 201) def test_doctors_scraper_archives_to_s3(self): self.doctors_scraper.s3_key = "test/doctors.json" @@ -68,9 +69,21 @@ def test_doctors_scraper_archives_to_s3(self): uploaded_data = self.doctors_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.doctors_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) + # get test/health_facilities.json key for this test + # def test_health_facilities_scraper_archives_to_s3(self): + # self.health_facilities_scraper.s3_key = "test/health_facilities.json" + # with open(TEST_DIR + "/dummy_files/health_facilities.json", "r") as my_file: + # data = my_file.read() + # self.health_facilities_scraper.archive_data(data) + # uploaded_data = self.health_facilities_scraper.s3.get_object( + # Bucket="cfa-healthtools-ke", + # Key=self.health_facilities_scraper.s3_key + # )['Body'].read() + # self.assertEqual(uploaded_data, data) + def test_foreign_doctors_scraper_archives_to_s3(self): self.foreign_doctors_scraper.s3_key = "test/foreign_doctors.json" with open(TEST_DIR + "/dummy_files/foreign_doctors.json", "r") as my_file: @@ -79,7 +92,7 @@ def test_foreign_doctors_scraper_archives_to_s3(self): uploaded_data = self.foreign_doctors_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.foreign_doctors_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) def test_clinical_officers_scraper_archives_to_s3(self): @@ -90,9 +103,9 @@ def test_clinical_officers_scraper_archives_to_s3(self): uploaded_data = self.clinical_officers_scraper.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.clinical_officers_scraper.s3_key - )['Body'].read() + )['Body'].read() self.assertEqual(uploaded_data, data) - def test_foreign_doctors_scraper_deletes_cloudsearch_docs(self): - response = self.foreign_doctors_scraper.delete_cloudsearch_docs() - self.assertEqual(response.get("status"), "success") + def test_health_facilities_scraper_gets_token(self): + self.health_facilities_scraper.get_token() + self.assertIsNotNone(self.health_facilities_scraper.access_token) diff --git a/requirements.txt b/requirements.txt index 0ec8ef6..a27a1d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,30 @@ +appdirs==1.4.3 +backports.ssl-match-hostname==3.5.0.1 beautifulsoup4==4.5.3 boto3==1.4.4 botocore==1.5.27 bs4==0.0.1 +certifi==2017.4.17 +click==6.7 docutils==0.13.1 +elasticsearch==5.4.0 +Flask==0.12.1 futures==3.0.5 +gunicorn==19.7.1 +itsdangerous==0.24 +Jinja2==2.9.6 jmespath==0.9.2 +MarkupSafe==1.0 nose==1.3.7 +packaging==16.8 +pyparsing==2.2.0 python-dateutil==2.6.0 +python-memcached==1.58 requests==2.13.0 +requests-aws4auth==0.9 s3transfer==0.1.10 six==1.10.0 +slackclient==1.0.6 +urllib3==1.21.1 +websocket-client==0.40.0 +Werkzeug==0.12.1 diff --git a/scraper.py b/scraper.py index ab948c5..e42d1cd 100644 --- a/scraper.py +++ b/scraper.py @@ -9,8 +9,8 @@ clinical_officers_scraper = ClinicalOfficersScraper() healthfacilities_scraper.scrape_data() # scraping you softly with these bots... + clinical_officers_result = clinical_officers_scraper.scrape_site() doctors_result = doctors_scraper.scrape_site() if doctors_result: foreign_doctors_scraper.document_id = len(doctors_result) + 1 foreign_docs_result = foreign_doctors_scraper.scrape_site() - clinical_officers_result = clinical_officers_scraper.scrape_site() From be4db2da4004d2f7acb3f53ea424c355bc12353b Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Tue, 20 Jun 2017 12:08:37 +0300 Subject: [PATCH 08/18] Add slack integration --- healthtools/config.py | 4 +++ healthtools/scrapers/base_scraper.py | 45 +++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index acfedaa..47a0221 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -27,3 +27,7 @@ } TEST_DIR = os.getcwd() + "/healthtools/tests" + +SLACK = { + "token": os.getenv("SLACK_API_TOKEN") + } diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index e6eee4b..cce409b 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -3,8 +3,9 @@ from datetime import datetime from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth -from healthtools.config import AWS, ES from serializer import JSONSerializerPython2 +from healthtools.config import AWS, ES, SLACK +from slackclient import SlackClient import requests import boto3 import re @@ -38,6 +39,7 @@ def __init__(self): connection_class=RequestsHttpConnection, serializer=JSONSerializerPython2() ) + self.slack_client = SlackClient(SLACK['token']) def scrape_site(self): ''' @@ -65,6 +67,11 @@ def scrape_site(self): delete_batch.extend(delete_docs) except Exception as err: skipped_pages += 1 + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) + ) print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) continue print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( @@ -121,6 +128,11 @@ def scrape_page(self, page_url): return entries, delete_batch except Exception as err: if self.retries >= 5: + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text=" @channel check this out: ```ERROR: Failed to scrape data from page {} -- {}```".format(page_url, str(err)) + ) print "ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err)) return err else: @@ -136,6 +148,11 @@ def upload_data(self, payload): response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) + ) print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) def archive_data(self, payload): @@ -164,6 +181,11 @@ def archive_data(self, payload): datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) + ) print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) def delete_elasticsearch_docs(self): @@ -207,9 +229,19 @@ def delete_elasticsearch_docs(self): return response except Exception as err: if "NoSuchKey" in err: - print "ERROR - delete_elasticsearch_docs() - no delete file present" + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR - delete_cloudsearch_docs() - no delete file present" + ) + print "ERROR - delete_cloudsearch_docs() - no delete file present" return - print "ERROR - delete_elasticsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + ) + print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) def get_total_number_of_pages(self): ''' @@ -222,7 +254,12 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - print "ERROR: **get_total_page_numbers()** - url: {} - err: {}". \ + self.slack_client.api_call( + "chat.postMessage", + channel="#c4a", + text="ERROR: **get_total_page_numbers()** - url: {} - err: {}".format(self.site_url, str(err)) + ) + print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ format(self.site_url, str(err)) return From ddfd6f8a17e385371d53436f08c153b46ffecbb9 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Tue, 20 Jun 2017 16:02:05 +0300 Subject: [PATCH 09/18] Refractor slack integration to use webhook's url and not test tokens --- healthtools/config.py | 11 +---- healthtools/scrapers/base_scraper.py | 63 +++++++++++++++------------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index 47a0221..413e2f8 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -11,14 +11,7 @@ AWS = { "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY_ID"), "aws_secret_access_key": os.getenv("MORPH_AWS_SECRET_KEY"), - "region_name": os.getenv("MORPH_AWS_REGION"), - # Doctors document endpoint - "cloudsearch_doctors_endpoint": "http://doc-cfa-healthtools-ke-doctors-m34xee6byjmzcgzmovevkjpffy.eu-west-1.cloudsearch.amazonaws.com/", - # Clinical document endpoint - "cloudsearch_cos_endpoint": "http://doc-cfa-healthtools-ke-cos-nhxtw3w5goufkzram4er7sciz4.eu-west-1.cloudsearch.amazonaws.com/", - # Health facilities endpoint - "cloudsearch_health_faciities_endpoint": "https://doc-health-facilities-ke-65ftd7ksxazyatw5fiv5uyaiqi.eu-west-1.cloudsearch.amazonaws.com", - + "region_name": os.getenv("MORPH_AWS_REGION") } ES = { "host": os.getenv("ES_HOST"), @@ -29,5 +22,5 @@ TEST_DIR = os.getcwd() + "/healthtools/tests" SLACK = { - "token": os.getenv("SLACK_API_TOKEN") + "url": os.getenv("WEBHOOK_URL") } diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index cce409b..b721fbe 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -5,7 +5,6 @@ from requests_aws4auth import AWS4Auth from serializer import JSONSerializerPython2 from healthtools.config import AWS, ES, SLACK -from slackclient import SlackClient import requests import boto3 import re @@ -39,7 +38,6 @@ def __init__(self): connection_class=RequestsHttpConnection, serializer=JSONSerializerPython2() ) - self.slack_client = SlackClient(SLACK['token']) def scrape_site(self): ''' @@ -67,10 +65,10 @@ def scrape_site(self): delete_batch.extend(delete_docs) except Exception as err: skipped_pages += 1 - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) + requests.post( + SLACK['url'], + data=json.dumps({"text": "```ERROR: scrape_site() - source: {} - page: {} - {}```".format(url, page_num, err)}), + headers={'Content-Type': 'application/json'} ) print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) continue @@ -128,10 +126,11 @@ def scrape_page(self, page_url): return entries, delete_batch except Exception as err: if self.retries >= 5: - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text=" @channel check this out: ```ERROR: Failed to scrape data from page {} -- {}```".format(page_url, str(err)) + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR: Failed to scrape data from page {} -- {}```".format(page_url, str(err))}), + headers={'Content-Type': 'application/json'} ) print "ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err)) return err @@ -148,10 +147,11 @@ def upload_data(self, payload): response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR - upload_data() - {} - {}```".format(type(self).__name__, str(err))}), + headers={'Content-Type': 'application/json'} ) print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) @@ -181,10 +181,11 @@ def archive_data(self, payload): datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR - archive_data() - {} - {}```".format(self.s3_key, str(err))}), + headers={'Content-Type': 'application/json'} ) print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) @@ -229,17 +230,18 @@ def delete_elasticsearch_docs(self): return response except Exception as err: if "NoSuchKey" in err: - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR - delete_cloudsearch_docs() - no delete file present" + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR - delete_cloudsearch_docs() - no delete file present```"}), + headers={'Content-Type': 'application/json'} ) print "ERROR - delete_cloudsearch_docs() - no delete file present" return - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + requests.post( + SLACK['url'], + data=json.dumps({"text": "```ERROR - delete_cloudsearch_docs() - {} - {}```".format(type(self).__name__, str(err))}), + headers={'Content-Type': 'application/json'} ) print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) @@ -254,10 +256,11 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - self.slack_client.api_call( - "chat.postMessage", - channel="#c4a", - text="ERROR: **get_total_page_numbers()** - url: {} - err: {}".format(self.site_url, str(err)) + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR: **get_total_page_numbers()** - url: {} - err: {}```".format(self.site_url, str(err))}), + headers={'Content-Type': 'application/json'} ) print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ format(self.site_url, str(err)) From c762f18c574be719cb758829d51c60876192e38e Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Tue, 20 Jun 2017 16:29:22 +0300 Subject: [PATCH 10/18] Add slack error notification to health facilities scrapper Add documentation to README Add tests for slack notification --- README.md | 2 ++ healthtools/scrapers/health_facilities.py | 8 +++++++- healthtools/tests/test_scrapers.py | 23 ++++++++++------------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 58797f4..295c392 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ You can set the required environment variables like so $ export MORPH_AWS_REGION= $ export MORPH_AWS_ACCESS_KEY_ID= $ export MORPH_AWS_SECRET_KEY= +$ export ES_HOST='' +$ export WEBHOOK_URL='' ``` You can now run the scrapers `$ python scraper.py` (It might take a while and you might need to change the endpoints in config.py if you haven't authorization for them) diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index 3df747d..af3506e 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -1,7 +1,7 @@ import json from cStringIO import StringIO from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import ES +from healthtools.config import ES, SLACK import requests from datetime import datetime @@ -73,6 +73,12 @@ def get_data(self): print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: + requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```ERROR IN - index_for_search() Health Facilities Scraper - %s```" % err}), + headers={'Content-Type': 'application/json'} + ) print "ERROR IN - index_for_search() Health Facilities Scraper - %s" % err def index_for_elasticsearch(self, record): diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index 8585f62..e55c282 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -1,10 +1,11 @@ import unittest +import requests import json from healthtools.scrapers.doctors import DoctorsScraper from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper from healthtools.scrapers.health_facilities import HealthFacilitiesScraper -from healthtools.config import TEST_DIR +from healthtools.config import TEST_DIR, SLACK class TestDoctorsScraper(unittest.TestCase): @@ -72,18 +73,6 @@ def test_doctors_scraper_archives_to_s3(self): )['Body'].read() self.assertEqual(uploaded_data, data) - # get test/health_facilities.json key for this test - # def test_health_facilities_scraper_archives_to_s3(self): - # self.health_facilities_scraper.s3_key = "test/health_facilities.json" - # with open(TEST_DIR + "/dummy_files/health_facilities.json", "r") as my_file: - # data = my_file.read() - # self.health_facilities_scraper.archive_data(data) - # uploaded_data = self.health_facilities_scraper.s3.get_object( - # Bucket="cfa-healthtools-ke", - # Key=self.health_facilities_scraper.s3_key - # )['Body'].read() - # self.assertEqual(uploaded_data, data) - def test_foreign_doctors_scraper_archives_to_s3(self): self.foreign_doctors_scraper.s3_key = "test/foreign_doctors.json" with open(TEST_DIR + "/dummy_files/foreign_doctors.json", "r") as my_file: @@ -109,3 +98,11 @@ def test_clinical_officers_scraper_archives_to_s3(self): def test_health_facilities_scraper_gets_token(self): self.health_facilities_scraper.get_token() self.assertIsNotNone(self.health_facilities_scraper.access_token) + + def test_scrapper_sends_slack_notification(self): + response = requests.post( + SLACK['url'], + data=json.dumps({"text": "*This test is passing* :tada:"}), + headers={'Content-Type': 'application/json'} + ) + self.assertEqual(response.status_code, 200) From 792af4465607fc32605ced2595dcd2186da22283 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Wed, 21 Jun 2017 12:47:04 +0300 Subject: [PATCH 11/18] Refactor code for sending slack messages to a method --- healthtools/config.py | 2 +- healthtools/scrapers/base_scraper.py | 56 +++++++---------------- healthtools/scrapers/health_facilities.py | 9 +--- healthtools/tests/test_scrapers.py | 8 ++-- 4 files changed, 22 insertions(+), 53 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index 413e2f8..21c71f3 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -11,7 +11,7 @@ AWS = { "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY_ID"), "aws_secret_access_key": os.getenv("MORPH_AWS_SECRET_KEY"), - "region_name": os.getenv("MORPH_AWS_REGION") + "region_name": os.getenv("MORPH_AWS_REGION", 'eu-west-1') } ES = { "host": os.getenv("ES_HOST"), diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index b721fbe..3a768b2 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -65,11 +65,7 @@ def scrape_site(self): delete_batch.extend(delete_docs) except Exception as err: skipped_pages += 1 - requests.post( - SLACK['url'], - data=json.dumps({"text": "```ERROR: scrape_site() - source: {} - page: {} - {}```".format(url, page_num, err)}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err)) print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) continue print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( @@ -126,12 +122,7 @@ def scrape_page(self, page_url): return entries, delete_batch except Exception as err: if self.retries >= 5: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR: Failed to scrape data from page {} -- {}```".format(page_url, str(err))}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err))) print "ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err)) return err else: @@ -147,12 +138,7 @@ def upload_data(self, payload): response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR - upload_data() - {} - {}```".format(type(self).__name__, str(err))}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err))) print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) def archive_data(self, payload): @@ -181,12 +167,7 @@ def archive_data(self, payload): datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR - archive_data() - {} - {}```".format(self.s3_key, str(err))}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR - archive_data() - {} - {}".format(self.s3_key, str(err))) print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) def delete_elasticsearch_docs(self): @@ -230,19 +211,10 @@ def delete_elasticsearch_docs(self): return response except Exception as err: if "NoSuchKey" in err: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR - delete_cloudsearch_docs() - no delete file present```"}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR - delete_cloudsearch_docs() - no delete file present") print "ERROR - delete_cloudsearch_docs() - no delete file present" return - requests.post( - SLACK['url'], - data=json.dumps({"text": "```ERROR - delete_cloudsearch_docs() - {} - {}```".format(type(self).__name__, str(err))}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err))) print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) def get_total_number_of_pages(self): @@ -256,12 +228,7 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR: **get_total_page_numbers()** - url: {} - err: {}```".format(self.site_url, str(err))}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR: **get_total_page_numbers()** - url: {} - err: {}".format(self.site_url, str(err))) print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ format(self.site_url, str(err)) return @@ -290,4 +257,13 @@ def format_for_elasticsearch(self, entry): } return meta_dict, entry + def post_to_slack(self, message): + """post messages to slack""" + response = requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```{}```".format(message)}), + headers={'Content-Type': 'application/json'} + ) + return response diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index af3506e..31c58b9 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -1,7 +1,7 @@ import json from cStringIO import StringIO from healthtools.scrapers.base_scraper import Scraper -from healthtools.config import ES, SLACK +from healthtools.config import ES import requests from datetime import datetime @@ -73,12 +73,7 @@ def get_data(self): print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: - requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```ERROR IN - index_for_search() Health Facilities Scraper - %s```" % err}), - headers={'Content-Type': 'application/json'} - ) + self.post_to_slack("ERROR IN - index_for_search() Health Facilities Scraper - {}" % err) print "ERROR IN - index_for_search() Health Facilities Scraper - %s" % err def index_for_elasticsearch(self, record): diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index e55c282..1a49df4 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -1,6 +1,7 @@ import unittest import requests import json +from healthtools.scrapers.base_scraper import Scraper from healthtools.scrapers.doctors import DoctorsScraper from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper @@ -10,6 +11,7 @@ class TestDoctorsScraper(unittest.TestCase): def setUp(self): + self.base_scraper = Scraper() self.doctors_scraper = DoctorsScraper() self.foreign_doctors_scraper = ForeignDoctorsScraper() self.clinical_officers_scraper = ClinicalOfficersScraper() @@ -100,9 +102,5 @@ def test_health_facilities_scraper_gets_token(self): self.assertIsNotNone(self.health_facilities_scraper.access_token) def test_scrapper_sends_slack_notification(self): - response = requests.post( - SLACK['url'], - data=json.dumps({"text": "*This test is passing* :tada:"}), - headers={'Content-Type': 'application/json'} - ) + response = self.base_scraper.post_to_slack("Tests are passing") self.assertEqual(response.status_code, 200) From 698975c15c6eb70157aae2a814fdc90ad347cc0d Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 22 Jun 2017 10:10:54 +0300 Subject: [PATCH 12/18] Refactor error printing method --- healthtools/scrapers/base_scraper.py | 29 ++++++++++------------- healthtools/scrapers/health_facilities.py | 3 +-- healthtools/tests/test_scrapers.py | 2 +- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 3a768b2..29c6fb9 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -65,8 +65,7 @@ def scrape_site(self): delete_batch.extend(delete_docs) except Exception as err: skipped_pages += 1 - self.post_to_slack("ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err)) - print "ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err) + self.print_error("ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err)) continue print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(all_results)) @@ -122,8 +121,7 @@ def scrape_page(self, page_url): return entries, delete_batch except Exception as err: if self.retries >= 5: - self.post_to_slack("ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err))) - print "ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err)) + self.print_error("ERROR: Failed to scrape data from page {} -- {}".format(page_url, str(err))) return err else: self.retries += 1 @@ -138,8 +136,7 @@ def upload_data(self, payload): response = self.es_client.bulk(index=ES['index'], body=payload, refresh=True) return response except Exception as err: - self.post_to_slack("ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err))) - print "ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err)) + self.print_error("ERROR - upload_data() - {} - {}".format(type(self).__name__, str(err))) def archive_data(self, payload): ''' @@ -167,8 +164,7 @@ def archive_data(self, payload): datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: - self.post_to_slack("ERROR - archive_data() - {} - {}".format(self.s3_key, str(err))) - print "ERROR - archive_data() - {} - {}".format(self.s3_key, str(err)) + self.print_error("ERROR - archive_data() - {} - {}".format(self.s3_key, str(err))) def delete_elasticsearch_docs(self): ''' @@ -211,11 +207,9 @@ def delete_elasticsearch_docs(self): return response except Exception as err: if "NoSuchKey" in err: - self.post_to_slack("ERROR - delete_cloudsearch_docs() - no delete file present") - print "ERROR - delete_cloudsearch_docs() - no delete file present" + self.print_error("ERROR - delete_cloudsearch_docs() - no delete file present") return - self.post_to_slack("ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err))) - print "ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err)) + self.print_error("ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err))) def get_total_number_of_pages(self): ''' @@ -228,9 +222,7 @@ def get_total_number_of_pages(self): pattern = re.compile("(\d+) pages?") self.num_pages_to_scrape = int(pattern.search(text).group(1)) except Exception as err: - self.post_to_slack("ERROR: **get_total_page_numbers()** - url: {} - err: {}".format(self.site_url, str(err))) - print "ERROR: **get_total_page_numbers()** - url: {} - err: {}".\ - format(self.site_url, str(err)) + self.print_error("ERROR: **get_total_page_numbers()** - url: {} - err: {}".format(self.site_url, str(err))) return def make_soup(self, url): @@ -257,8 +249,11 @@ def format_for_elasticsearch(self, entry): } return meta_dict, entry - def post_to_slack(self, message): - """post messages to slack""" + def print_error(self, message): + """ + post messages to slack and print them on the terminal + """ + print(message) response = requests.post( SLACK['url'], data=json.dumps( diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index 31c58b9..618ca27 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -73,8 +73,7 @@ def get_data(self): print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: - self.post_to_slack("ERROR IN - index_for_search() Health Facilities Scraper - {}" % err) - print "ERROR IN - index_for_search() Health Facilities Scraper - %s" % err + self.print_error("ERROR IN - index_for_search() Health Facilities Scraper - {}" % err) def index_for_elasticsearch(self, record): meta_data = {"index": { diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index 1b60fcd..2c785ed 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -101,6 +101,6 @@ def test_health_facilities_scraper_gets_token(self): self.assertIsNotNone(self.health_facilities_scraper.access_token) def test_scrapper_sends_slack_notification(self): - response = self.base_scraper.post_to_slack("Tests are passing") + response = self.base_scraper.print_error("Tests are passing") self.assertEqual(response.status_code, 200) From 9a26c335b0695adeaf24d3af236e0ce90b55e430 Mon Sep 17 00:00:00 2001 From: David Lemayian Date: Thu, 22 Jun 2017 11:59:17 +0300 Subject: [PATCH 13/18] Merge branch 'master' into develop # Conflicts: # healthtools/config.py # healthtools/scrapers/base_scraper.py # scraper.py --- healthtools/config.py | 15 ++++++++------- healthtools/scrapers/base_scraper.py | 2 +- healthtools/scrapers/health_facilities.py | 2 +- scraper.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index 5f0e74d..ef468b5 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -6,20 +6,21 @@ "FOREIGN_DOCTORS": "http://medicalboard.co.ke/online-services/foreign-doctors-license-register/?currpage={}", "CLINICAL_OFFICERS": "http://clinicalofficerscouncil.org/online-services/retention/?currpage={}", "TOKEN_URL": "http://api.kmhfl.health.go.ke/o/token/" - } +} AWS = { - "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY_ID"), + "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY"), "aws_secret_access_key": os.getenv("MORPH_AWS_SECRET_KEY"), "region_name": os.getenv("MORPH_AWS_REGION", 'eu-west-1') - } +} + ES = { "host": os.getenv("ES_HOST"), "index": "healthtools" - } - -TEST_DIR = os.getcwd() + "/healthtools/tests" +} SLACK = { "url": os.getenv("WEBHOOK_URL") - } +} + +TEST_DIR = os.getcwd() + "/healthtools/tests" diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 29c6fb9..dc15f09 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -49,7 +49,7 @@ def scrape_site(self): skipped_pages = 0 print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__)) - print "{{{0}}} - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "[{0}] - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) for page_num in range(1, self.num_pages_to_scrape + 1): url = self.site_url.format(page_num) try: diff --git a/healthtools/scrapers/health_facilities.py b/healthtools/scrapers/health_facilities.py index 618ca27..e1fd6da 100644 --- a/healthtools/scrapers/health_facilities.py +++ b/healthtools/scrapers/health_facilities.py @@ -49,7 +49,7 @@ def upload(self, payload): def get_data(self): try: - print "{{{0}}} - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "[{0}] - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) headers = {'Authorization': 'Bearer ' + self.access_token} r = requests.get(SEARCH_URL, headers=headers) data = r.json() diff --git a/scraper.py b/scraper.py index e42d1cd..1809e44 100644 --- a/scraper.py +++ b/scraper.py @@ -2,15 +2,20 @@ from healthtools.scrapers.foreign_doctors import ForeignDoctorsScraper from healthtools.scrapers.clinical_officers import ClinicalOfficersScraper from healthtools.scrapers.health_facilities import HealthFacilitiesScraper + if __name__ == "__main__": - healthfacilities_scraper = HealthFacilitiesScraper() + + # Initialize the Scrapers doctors_scraper = DoctorsScraper() foreign_doctors_scraper = ForeignDoctorsScraper() clinical_officers_scraper = ClinicalOfficersScraper() - healthfacilities_scraper.scrape_data() + healthfacilities_scraper = HealthFacilitiesScraper() + # scraping you softly with these bots... - clinical_officers_result = clinical_officers_scraper.scrape_site() doctors_result = doctors_scraper.scrape_site() if doctors_result: foreign_doctors_scraper.document_id = len(doctors_result) + 1 foreign_docs_result = foreign_doctors_scraper.scrape_site() + + clinical_officers_result = clinical_officers_scraper.scrape_site() + healthfacilities_result = healthfacilities_scraper.scrape_data() From 7193987e341ecc0d158cf21d642deb9768ca7772 Mon Sep 17 00:00:00 2001 From: David Lemayian Date: Thu, 22 Jun 2017 12:12:16 +0300 Subject: [PATCH 14/18] Change order of logs. --- healthtools/config.py | 3 ++- healthtools/scrapers/base_scraper.py | 21 +++++++++++---------- healthtools/tests/test_scrapers.py | 1 - 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/healthtools/config.py b/healthtools/config.py index ef468b5..2ecf5be 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -11,7 +11,8 @@ AWS = { "aws_access_key_id": os.getenv("MORPH_AWS_ACCESS_KEY"), "aws_secret_access_key": os.getenv("MORPH_AWS_SECRET_KEY"), - "region_name": os.getenv("MORPH_AWS_REGION", 'eu-west-1') + "region_name": os.getenv("MORPH_AWS_REGION", "eu-west-1"), + "s3_bucket": os.getenv("S3_BUCKET", "cfa-healthtools-ke") } ES = { diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index dc15f09..1975ea5 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -24,7 +24,7 @@ def __init__(self): self.s3 = boto3.client("s3", **{ "aws_access_key_id": AWS["aws_access_key_id"], "aws_secret_access_key": AWS["aws_secret_access_key"], - "region_name": AWS["region_name"], + "region_name": AWS["region_name"] }) # set up authentication credentials awsauth = AWS4Auth(AWS["aws_access_key_id"], AWS["aws_secret_access_key"], AWS["region_name"], 'es') @@ -43,13 +43,15 @@ def scrape_site(self): ''' Scrape the whole site ''' - self.get_total_number_of_pages() + print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__)) + print "[{0}] Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + all_results = [] delete_batch = [] skipped_pages = 0 - print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__)) - print "[{0}] - Started Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + self.get_total_number_of_pages() + for page_num in range(1, self.num_pages_to_scrape + 1): url = self.site_url.format(page_num) try: @@ -67,7 +69,7 @@ def scrape_site(self): skipped_pages += 1 self.print_error("ERROR: scrape_site() - source: {} - page: {} - {}".format(url, page_num, err)) continue - print "{{{0}}} - Scraper completed. {1} documents retrieved.".format( + print "[{0}] - Scraper completed. {1} documents retrieved.".format( datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(all_results)) if all_results: @@ -157,10 +159,10 @@ def archive_data(self, payload): CopySource="cfa-healthtools-ke/" + self.s3_key, Key=self.s3_historical_record_key.format( date)) - print "{{{0}}} - Archived data has been updated.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "[{0}] - Archived data has been updated.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) return else: - print "{{{0}}} - Data Scraped does not differ from archived data.".format( + print "[{0}] - Data Scraped does not differ from archived data.".format( datetime.now().strftime('%Y-%m-%d %H:%M:%S')) except Exception as err: @@ -207,9 +209,9 @@ def delete_elasticsearch_docs(self): return response except Exception as err: if "NoSuchKey" in err: - self.print_error("ERROR - delete_cloudsearch_docs() - no delete file present") + self.print_error("ERROR - delete_elasticsearch_docs() - no delete file present") return - self.print_error("ERROR - delete_cloudsearch_docs() - {} - {}".format(type(self).__name__, str(err))) + self.print_error("ERROR - delete_elasticsearch_docs() - {} - {}".format(type(self).__name__, str(err))) def get_total_number_of_pages(self): ''' @@ -261,4 +263,3 @@ def print_error(self, message): headers={'Content-Type': 'application/json'} ) return response - diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index 2c785ed..f9a31f5 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -103,4 +103,3 @@ def test_health_facilities_scraper_gets_token(self): def test_scrapper_sends_slack_notification(self): response = self.base_scraper.print_error("Tests are passing") self.assertEqual(response.status_code, 200) - From ade3c0bcb51ba46146b9174833f980766cc92102 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 22 Jun 2017 12:29:29 +0300 Subject: [PATCH 15/18] Add timestamp whilst printing error messages on terminal --- healthtools/scrapers/base_scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 29c6fb9..99d5fea 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -253,7 +253,7 @@ def print_error(self, message): """ post messages to slack and print them on the terminal """ - print(message) + print('{{{0}}} - '.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + message) response = requests.post( SLACK['url'], data=json.dumps( From d784d3916892936f032b735ca069279b5af5dbdd Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 22 Jun 2017 14:12:07 +0300 Subject: [PATCH 16/18] Refactor code to allow to use elastic search locally Update README to reflect changes --- README.md | 16 +++++++++--- healthtools/config.py | 4 +-- healthtools/scrapers/base_scraper.py | 38 ++++++++++++++++------------ healthtools/tests/test_scrapers.py | 7 +++-- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 295c392..3ddc5a7 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,21 @@ You can set the required environment variables like so $ export MORPH_AWS_REGION= $ export MORPH_AWS_ACCESS_KEY_ID= $ export MORPH_AWS_SECRET_KEY= -$ export ES_HOST='' -$ export WEBHOOK_URL='' +$ export ES_HOST= (DO NOT SET THIS IF YOU WOULD LIKE TO USE ELASTIC SEARCH LOCALLY ON YOUR MACHINE) +$ export WEBHOOK_URL= (DO NOT SET THIS IF YOU DON'T WANT TO POST ERROR MESSAGES ON SLACK) ``` +**If you want to use elasticsearch locally on your machine use the following instructions to set it up** + +For linux and windows users, follow instructions from this [link](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html) + +For mac users run `brew install elasticsearch` on your terminal + +**If you want to post messages on slack** + +Set up `Incoming Webhooks` [here](https://slack.com/signin?redir=%2Fservices%2Fnew%2Fincoming-webhook) and set the global environment for the `WEBHOOK_URL` + +You can now run the scrapers `$ python scraper.py` (It might take a while) -You can now run the scrapers `$ python scraper.py` (It might take a while and you might need to change the endpoints in config.py if you haven't authorization for them) ## Running the tests diff --git a/healthtools/config.py b/healthtools/config.py index 5f0e74d..c5bef1d 100644 --- a/healthtools/config.py +++ b/healthtools/config.py @@ -14,12 +14,12 @@ "region_name": os.getenv("MORPH_AWS_REGION", 'eu-west-1') } ES = { - "host": os.getenv("ES_HOST"), + "host": os.getenv("ES_HOST", None), "index": "healthtools" } TEST_DIR = os.getcwd() + "/healthtools/tests" SLACK = { - "url": os.getenv("WEBHOOK_URL") + "url": os.getenv("WEBHOOK_URL", None) } diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 99d5fea..2db8920 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -29,15 +29,18 @@ def __init__(self): # set up authentication credentials awsauth = AWS4Auth(AWS["aws_access_key_id"], AWS["aws_secret_access_key"], AWS["region_name"], 'es') # client host for aws elastic search service - self.es_client = Elasticsearch( - hosts=ES['host'], - port=443, - http_auth=awsauth, - use_ssl=True, - verify_certs=True, - connection_class=RequestsHttpConnection, - serializer=JSONSerializerPython2() - ) + if ES['host']: + self.es_client = Elasticsearch( + hosts=ES['host'], + port=443, + http_auth=awsauth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + serializer=JSONSerializerPython2() + ) + else: + self.es_client = Elasticsearch('127.0.0.1') def scrape_site(self): ''' @@ -251,14 +254,17 @@ def format_for_elasticsearch(self, entry): def print_error(self, message): """ - post messages to slack and print them on the terminal + print error messages in the terminal + if slack webhook is set up post the errors to slack """ print('{{{0}}} - '.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + message) - response = requests.post( - SLACK['url'], - data=json.dumps( - {"text": "```{}```".format(message)}), - headers={'Content-Type': 'application/json'} - ) + response = None + if SLACK['url']: + response = requests.post( + SLACK['url'], + data=json.dumps( + {"text": "```{}```".format(message)}), + headers={'Content-Type': 'application/json'} + ) return response diff --git a/healthtools/tests/test_scrapers.py b/healthtools/tests/test_scrapers.py index 2c785ed..bce03c7 100644 --- a/healthtools/tests/test_scrapers.py +++ b/healthtools/tests/test_scrapers.py @@ -100,7 +100,10 @@ def test_health_facilities_scraper_gets_token(self): self.health_facilities_scraper.get_token() self.assertIsNotNone(self.health_facilities_scraper.access_token) - def test_scrapper_sends_slack_notification(self): + def test_scrapper_prints_notification(self): response = self.base_scraper.print_error("Tests are passing") - self.assertEqual(response.status_code, 200) + if SLACK['url']: + self.assertEqual(response.status_code, 200) + else: + self.assertIsNone(response) From f9a84ae71dd3b7038573a6a666041dfbe79b1751 Mon Sep 17 00:00:00 2001 From: Denis Gathondu Date: Thu, 22 Jun 2017 14:40:38 +0300 Subject: [PATCH 17/18] Update circle ci file --- README.md | 4 ++++ circle.yml | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ddc5a7..5692c31 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,14 @@ For mac users run `brew install elasticsearch` on your terminal Set up `Incoming Webhooks` [here](https://slack.com/signin?redir=%2Fservices%2Fnew%2Fincoming-webhook) and set the global environment for the `WEBHOOK_URL` +If you set up elasticsearch locally run it `$ elasticsearch` + You can now run the scrapers `$ python scraper.py` (It might take a while) ## Running the tests +_**make sure if you use elasticsearch locally, it's running**_ Use nosetests to run tests (with stdout) like this: ```$ nosetests --nocapture``` + diff --git a/circle.yml b/circle.yml index c3c93da..881155c 100644 --- a/circle.yml +++ b/circle.yml @@ -1,9 +1,16 @@ machine: python: - version: 2.7.13 + version: 2.7.5 + java: + version: openjdk8 dependencies: pre: - pip install -r requirements.txt + post: + - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.2.tar.gz + - tar -xzf elasticsearch-5.4.2.tar.gz + - elasticsearch-5.4.2/bin/elasticsearch: {background: true} + - sleep 10 && wget --waitretry=5 --retry-connrefused -v http://127.0.0.1:9200/ test: override: From 7f9bc3e2c5dd861959004a7eabd334d42c053799 Mon Sep 17 00:00:00 2001 From: David Lemayian Date: Thu, 22 Jun 2017 16:34:19 +0300 Subject: [PATCH 18/18] Small changes. --- README.md | 7 +++---- healthtools/scrapers/base_scraper.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 295c392..2dbdfa1 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,9 @@ Install the dependencies by running `$ pip install requirements.txt` You can set the required environment variables like so ``` -$ export MORPH_AWS_REGION= -$ export MORPH_AWS_ACCESS_KEY_ID= -$ export MORPH_AWS_SECRET_KEY= -$ export ES_HOST='' +$ export MORPH_AWS_ACCESS_KEY_ID='' +$ export MORPH_AWS_SECRET_KEY='' +$ export ES_HOST='' # e.g localhost:9200 $ export WEBHOOK_URL='' ``` diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index 1975ea5..7e408b9 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -25,7 +25,7 @@ def __init__(self): "aws_access_key_id": AWS["aws_access_key_id"], "aws_secret_access_key": AWS["aws_secret_access_key"], "region_name": AWS["region_name"] - }) + }) # set up authentication credentials awsauth = AWS4Auth(AWS["aws_access_key_id"], AWS["aws_secret_access_key"], AWS["region_name"], 'es') # client host for aws elastic search service @@ -37,7 +37,7 @@ def __init__(self): verify_certs=True, connection_class=RequestsHttpConnection, serializer=JSONSerializerPython2() - ) + ) def scrape_site(self): ''' @@ -85,7 +85,7 @@ def scrape_site(self): self.s3.upload_fileobj( delete_file, "cfa-healthtools-ke", self.delete_file) - print "{{{0}}} - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + print "[{0}] - Completed Scraper.".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) return all_results @@ -180,10 +180,12 @@ def delete_elasticsearch_docs(self): _type = 'doctors' else: _type = 'health-facilities' + # get documents to be deleted delete_docs = self.s3.get_object( Bucket="cfa-healthtools-ke", Key=self.delete_file)['Body'].read() + # delete try: response = self.es_client.bulk(index=ES['index'], body=delete_docs, refresh=True) @@ -197,14 +199,16 @@ def delete_elasticsearch_docs(self): "_index": ES['index'], "_type": _type, "_id": record['delete']["_id"] - }}) + } + }) except: delete_records.append({ "delete": { "_index": ES['index'], "_type": _type, "_id": record["id"] - }}) + } + }) response = self.es_client.bulk(index=ES['index'], body=delete_records) return response except Exception as err: @@ -259,7 +263,8 @@ def print_error(self, message): response = requests.post( SLACK['url'], data=json.dumps( - {"text": "```{}```".format(message)}), + {"text": "```{}```".format(message)} + ), headers={'Content-Type': 'application/json'} - ) + ) return response