Changed DB structure for PharmaDoc table to use full pharmacy list

README.md

@@ -18,14 +18,15 @@ Open Data BC Hackathon 2018 Project. ineedmymeds crowdsources information about
 
 ## Setting up the database
 
-There are two open source databases that we use. Download them here:
+The open source datasets that we use. Download them here:
     * BC After Hours Pharmacies [List](https://catalogue.data.gov.bc.ca/dataset/after-hours-pharmacies-in-bc/resource/681b4fa5-13a5-4273-a189-fc101f0f8356)
     * Free Drugs [Database](https://packages.debian.org/sid/freemedforms-freedata)
 
 ```shell
 cat create_db.sql | sqlite3 ineedmy.db
-python import_drugs.py /usr/share/freemedforms/datapacks/appinstalled/drugs/master.db ./ineedmy.db
-python import_pharmacies.py hlbcpharmaciesfeb282018.csv ineedmy.db  
+(venv) python data/import_drugs.py /usr/share/freemedforms/datapacks/appinstalled/drugs/master.db ./ineedmy.db
+(venv) python data/pharmacies_scraper.py
+python data/import_all_pharmacies.py data/pharmacies_list.csv ineedmy.db  
 ```
 
 ## Installing Dependencies

create_db.sql

@@ -3,9 +3,6 @@ BEGIN TRANSACTION;
 CREATE TABLE IF NOT EXISTS "DrugDoc" (
 	`doc`	BLOB NOT NULL
 );
-CREATE TABLE `PharmaDoc` (
-	`doc`	BLOB NOT NULL
-);
 CREATE TABLE `DrugNames` (
 	`name`	TEXT NOT NULL UNIQUE,
 	`drug_id`	INTEGER NOT NULL
@@ -19,6 +16,15 @@ CREATE TABLE `PharmaLoc` (
 	`lat`	REAL,
 	`long`	REAL
 );
+CREATE TABLE `PharmaDoc` ( 
+        `name` TEXT NOT NULL, 
+        `address` TEXT NOT NULL, 
+        `phone` REAL NOT NULL, 
+        `fax` REAL, 
+        `manager` TEXT, 
+        `latitude` REAL, 
+        `longitude` REAL 
+);
 CREATE TABLE IF NOT EXISTS "DrugRequests" (
 	`drug_id`	INTEGER NOT NULL,
 	`when_requested`	INTEGER NOT NULL,

data/import_all_pharmacies.py

@@ -0,0 +1,27 @@
+import csv
+import sys
+import json
+import sqlite3
+
+# Usage:
+# python import_pharmacies.py /path/to/pharmacies_list.csv
+#   (from http://www.bcpharmacists.org/list-community-pharmacies)
+#   /path/to/ineedmy.db (for example ./ineedmy.db)
+
+if __name__ == '__main__':
+    tgt_conn = sqlite3.connect(sys.argv[2])
+
+    with open(sys.argv[1], 'r') as csvfile:
+        csvreader = csv.reader(csvfile)
+        next(csvreader) #skip header
+        for row in csvreader:
+            name = row[0]
+            address = row[1].replace('\n', ' ')
+            manager = row[2]
+            phone = row[3]
+            fax = row[4]
+
+            cur = tgt_conn.cursor()
+            cur.execute('INSERT INTO PharmaDoc (name, address, phone, fax, manager) VALUES(?, ?, ?, ?, ?)', ((name, address, phone, fax, manager)))
+
+    tgt_conn.commit()

data/pharmacies_scraper.py

@@ -0,0 +1,47 @@
+import re
+import requests
+from bs4 import BeautifulSoup
+import csv
+import time
+
+
+def pharmacy_list():
+
+    url = "http://www.bcpharmacists.org/list-community-pharmacies"
+    page = requests.get(url)
+    soup = BeautifulSoup(page.content, 'html.parser')
+
+
+    table = soup.find_all('table')[0]
+    table_rows = table.find_all('tr')
+    del table_rows[0]
+
+    with open('pharmacies_list.csv', 'w') as csvfile:
+        fieldnames = ['name', 'address', 'manager', 'phone', 'fax']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for index, table_row in enumerate(table_rows, 0):
+            cells = table_row.findAll('td')
+            #Finding name
+            name = cells[0].text.strip()
+
+            #Finding address
+            address = cells[1].text.strip()
+
+            #Manager
+            manager = cells[2].text.strip()
+
+            #Phone
+            raw_phone = cells[3].text.strip()
+            phone = re.sub("\D", "", raw_phone)
+
+            #Fax
+            raw_fax = cells[4].text.strip()
+            fax = re.sub("\D", "", raw_fax)
+
+            writer.writerow({'name': name, 'address': address, 'manager': manager, 'phone': phone, 'fax': fax})
+
+
+
+pharmacy_list()