Merge pull request #1 from OrangeCardinal/foia

Adding code for Freedom of Information Act

README.md

@@ -3,9 +3,14 @@ Attempt at building the most complete collection of government data related func
 
 
 ## How to Use
-1. Set the environment variable
+1. Create a api key @ api.data.gov ()
 
-2. 
+2. Set the environment variable API_DATA_GOV_KEY 
+Various government services require this key, as such it must be set for all sources to work. 
+
+3. Set the environment variable GOVERNMENT_DOWNLOAD_DIRECTORY
+Various calls will download data as requested and will be written into the directory specified. 
+
 ## Contributors
 Who|Github Contact|What|
 ---|---|---|

foia_samples.py

@@ -0,0 +1,23 @@
+from government.usa.foia.foia import FreedomOfInformationAct
+
+foia = FreedomOfInformationAct()
+
+#
+dataset = foia.download_data_set(2016)
+
+
+exit()
+
+# Display All the Federal Agencies that are available via this API currently
+for ac in foia.agency_components():
+    print(ac)
+
+
+# This sample shows how to get all the various agency components, and then how to get each agencies
+# request form
+foia = FreedomOfInformationAct()
+for ac in foia.agency_components():
+    ac_request_form = foia.request_form(ac.id)
+
+
+

government/base/api.py

@@ -0,0 +1,25 @@
+import certifi
+import urllib3
+
+class API(object):
+
+    def get_response(self, url):
+        http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
+        response = http.request('GET', url)
+        return response
+
+
+
+    def get_file(self, url, path):
+
+        http = urllib3.PoolManager()
+        r = http.request('GET', url, preload_content=False)
+
+        with open(path, 'wb') as out:
+            while True:
+                data = r.read(1024)
+                if not data:
+                    break
+                out.write(data)
+
+        r.release_conn()

international/imf/imf.py → government/international/imf/imf.py

@@ -1,6 +1,6 @@
-from base.api import API
-from international.imf.imf_data_structure import IMF_DataStructure
-from international.imf.imf_data_set import IMF_DataSet
+from government.base import API
+from government.international.imf.imf_data_structure import IMF_DataStructure
+from government.international.imf.imf_data_set import IMF_DataSet
 import json
 
 class IMF(API):

usa/bureau_labor_statistics.py → government/usa/bureau_labor_statistics.py

@@ -1,4 +1,4 @@
-from base.api import API
+from government.base import API
 import json
 
 class BureauOfLaborStatistics(API):

government/usa/foia/agency_component.py

@@ -0,0 +1,25 @@
+class AgencyComponent(object):
+    def __init__(self, data):
+        self.data = data
+        self.id = self.data['id']
+        for name, val in self.data['attributes'].items():
+            setattr(self, name, val)
+
+
+
+    def __str__(self):
+        readable_desc = "<AgencyComponent>\n"
+        readable_desc += "{0}\n{1}\n".format(self.title,self.id)
+        readable_desc += "Email: {0}\n".format(self.email)
+        readable_desc += "Telephone: {0}\n".format(self.telephone)
+
+        readable_desc += "Update Schedule\n"
+        readable_desc += "Scheduled Transition Date: {0}\n".format(self.scheduled_transition_date)
+        readable_desc += "Scheduled Publication: {0}\n".format(self.scheduled_publication)
+        readable_desc += "Scheduled Moderation State: {0}\n".format(self.scheduled_moderation_state)
+
+        readable_desc += "Reading Rooms\n"
+        for room in self.reading_rooms:
+            readable_desc += "{0}\n".format(room['uri'])
+
+        return readable_desc

government/usa/foia/foia.py

@@ -0,0 +1,60 @@
+import json
+import os
+from government.base.api import API
+from government.usa.foia.agency_component import AgencyComponent
+
+class FreedomOfInformationAct(API):
+    """
+    Wrapper for the API Endpoints
+
+    References
+    https://www.foia.gov/developer/agency-api/
+    """
+
+    def agency_components(self):
+        """
+        Gets a list of Agency Components
+        :return:
+        """
+
+        api_key = os.environ['API_DATA_GOV_KEY']
+
+
+        # Make the API Call parse the json data received
+        url = "https://api.foia.gov/api/agency_components?api_key={0}".format(api_key)
+        response = self.get_response(url)
+        json_response = json.loads(response.data)
+
+        # Format the data as warranted
+        processed_data = []
+        for agency_component in json_response['data']:
+            processed_data.append(AgencyComponent(agency_component))
+
+        return processed_data
+
+    def download_data_set(self, year):
+        """
+        Downloads a compressed archive of xml data into the requested for a given year if available
+
+        :param year: Year to get data set for
+        :return: Path to Zip File | None
+        """
+        zipname  = "{0}-FOIASetFull.zip".format(year)
+        filename = "{0}\{1}".format(os.environ['GOVERNMENT_DATA_DOWNLOAD_DIRECTORY'],zipname)
+        url = "https://www.foia.gov/{0}".format(zipname)
+        self.get_file(url, filename)
+        print(filename)
+        return filename
+
+
+
+    def request_form(self, agency_id):
+        api_key = os.environ['API_DATA_GOV_KEY']
+        url = "https://api.foia.gov/api/agency_components/{0}/request_form?api_key={1}".format(agency_id, api_key)
+        print(url)
+        response = self.get_response(url)
+        json_response = json.loads(response.data)
+        print(json_response)
+
+
+

test.py

@@ -1,4 +1,4 @@
-from international.imf.imf import IMF
+from government.international.imf.imf import IMF
 
 #bls = BureauOfLaborStatistics()
 #popular_series = bls.popular_series()