Merge pull request #19 from NYU-Chicago-data-facility/fr5_project_dat…

…abase_on_rds Fr5 project database on rds
Coleridge-Initiative-org · Mar 18, 2019 · 7c55db2 · 7c55db2
2 parents af16c7b + 4acf376
commit 7c55db2
Show file tree

Hide file tree

Showing 77 changed files with 44,529 additions and 437 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -2,6 +2,7 @@
 omit =
         scripts/*
         deploy/*
+        data/*
         data_facility/wsgi.py
         */test_*
 [xml]

diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+*.swp
 __pycache__/
 *.py[cod]
 *$py.class

diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, The Coleridge Initiative @NYU
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Makefile b/Makefile
@@ -36,12 +36,12 @@ code-check:
 	pylint --load-plugins pylint_django data_facility data_facility_admin
 
 test:
-	docker-compose exec web coverage run --source='.' manage.py test --noinput -v2 --parallel 2
+	docker-compose exec web coverage run --source='.' manage.py test --settings=data_facility.test_settings --noinput -v2 --parallel 1
 	docker-compose exec web coverage xml
 	docker-compose exec web coverage report
 
 test-quick:
-	docker-compose exec web coverage run --source='.' manage.py test --keepdb --noinput -v2 --parallel 2
+	docker-compose exec web coverage run --source='.' manage.py test --settings=data_facility.test_settings --keepdb --noinput -v2 --parallel 1
 	docker-compose exec web coverage xml
 	docker-compose exec web coverage report
 

diff --git a/README.md b/README.md
@@ -94,3 +94,16 @@ make docs
 ## Release
 
 Please check the [RELEASE documentation](documentation/RELEASE.md) when preparing a release.  
+
+
+# Keycloak Integration
+
+Postman Collections: https://www.getpostman.com/collections/1b72e4b52b9e5ad42d3c
+
+Getting KeyCloak PubKey (It needs to have the "----BEGIN..." And "-----END...") (PEM format): https://www.getpostman.com/collections/1b72e4b52b9e5ad42d3c
+
+## References:
+https://zapier.com/engineering/apikey-oauth-jwt/
+https://www.digitalocean.com/community/tutorials/an-introduction-to-oauth-2
+http://getblimp.github.io/django-rest-framework-jwt/#additional-settings
+https://auth0.com/blog/building-modern-applications-with-django-and-vuejs/
diff --git a/data/datasets/detailed_metadata/dataset-adrf-000005.json b/data/datasets/detailed_metadata/dataset-adrf-000005.json
diff --git a/data/datasets/detailed_metadata/dataset-adrf-000006.json b/data/datasets/detailed_metadata/dataset-adrf-000006.json
diff --git a/data/datasets/example_gmeta.json b/data/datasets/example_gmeta.json
@@ -0,0 +1,61 @@
+{
+    "gmeta": [
+        {
+            "adrf-000005": {
+                "mimetype": "application/json",
+                "content": {
+                    "temporal_coverage_end": "2000",
+                    "files_total": 1,
+                    "data_classification": "Public",
+                    "access_actions_required": "No further actions required to access this dataset",
+                    "geographical_coverage": [
+                        "Illinois"
+                    ],
+                    "keywords": [
+                        "census",
+                        "demographics",
+                        "geospatial",
+                        "population",
+                        "socioeconomics"
+                    ],
+                    "dataset_id": "adrf-000005",
+                    "category": "Demographic Characteristics",
+                    "dataset_version": 1,
+                    "title": "Decennial Census Illinois Profile of General Population and Housing Characteristics: 2000",
+                    "data_usage_policy": "This dataset is intended for public access and use.",
+                    "data_steward_organization": "Center for Urban Science and Progress (CUSP), NYU",
+                    "data_steward": "Drew Gordon",
+                    "access_requirements": "No access restrictions on this dataset",
+                    "description": "The Demographic Profile Summary File (SF1) contains 100 percent data asked of all people and about every housing unit on topics such as sex, age, race, Hispanic or Latino origin, household relationship, householdtype, group quarters population, housing occupancy, and housing tenure.",
+                    "source_url": "https://factfinder.census.gov/faces/nav/jsf/pages/download_center.xhtml",
+                    "geographical_unit": [
+                        "Census Tract"
+                    ],
+                    "related_articles": [
+                        "rc-000005",
+                        "rc-000006",
+                        "rc-000066",
+                        "rc-000068"
+                    ],
+                    "data_provider": "US Census Bureau",
+                    "dataset_documentation": [
+                        "DEC_00_SF1_DP1_metadata.csv",
+                        "DEC_00_SF1_DP1.txt",
+                        "2000_census_demographic_profile_summary_file.pdf"
+                    ],
+                    "dataset_version_date": 1488823173,
+                    "source_archive": "American FactFinder",
+                    "dataset_citation": "US Census Bureau, 2000, \"Illinois Profile of General Population and Housing Characteristics\", https://factfinder.census.gov/faces/nav/jsf/pages/download_center.xhtml#none, US Census Bureau American FactFinder [Distributor], 1 [Version]",
+                    "reference_url": "https://www.census.gov/prod/cen2000/doc/ProfileTD.pdf",
+                    "temporal_coverage_start": "1990",
+                    "file_names": [
+                        "adrf-000005.csv"
+                    ]
+                },
+                "visible_to": [
+                    "public"
+                ]
+            }
+        }
+    ]
+}
diff --git a/data/datasets/search_metadata/dataset-adrf-000001.json b/data/datasets/search_metadata/dataset-adrf-000001.json
@@ -0,0 +1,64 @@
+{
+    "gmeta": [
+        {
+            "adrf-000001": {
+                "mimetype": "application/json",
+                "content": {
+                    "temporal_coverage_end": "2015",
+                    "files_total": 1,
+                    "data_classification": "Restricted",
+                    "access_actions_required": "This is ADRF course data. In order to access, you must have signed the appropriate NDA and agreed to the ADRF terms of usage. These data are located in the course database. Field specific metadata are a sample only.",
+                    "geographical_coverage": [
+                        "Illinois"
+                    ],
+                    "keywords": [
+                        "criminal justice system",
+                        "inmates",
+                        "jail",
+                        "correctional facilities",
+                        "health",
+                        "employment",
+                        "demographics",
+                        "offenses"
+                    ],
+                    "dataset_id": "adrf-000001",
+                    "category": "Inmate Populations",
+                    "dataset_version": 1,
+                    "title": "Illinois Department of Corrections (DOC) Inmate Admissions 1990-2015",
+                    "data_usage_policy": "This dataset is restricted to ADRF course participants who have approval and have agreed to the terms of use provided by the Illinois Department of Corrections (IDOC). The user shall not attempt to re-identify individuals represented in the data and agree to not disclose any data contained here to any unauthorized parties. Any analyses and data products derived from the data must be reviewed by IDOC before export from this system.",
+                    "data_steward_organization": "Center for Urban Science and Progress (CUSP), NYU",
+                    "data_steward": "Julia Lane",
+                    "access_requirements": "You must be enrolled in an ADRF class to access these data.",
+                    "description": "Detailed transactional data of each time a person was admitted to an Illinois Department of Corrections (DOC) facility from 1990 to 2015. Variables include demographic, charges, sentencing, conduct, security level, health and mental health status, gang affiliation. Data are collected using correctional facility data and court records. See data dictionary documentation for detailed description of all fields.\n\nThe summary of this dataset\u2019s fields is a sample of records only. Please access the full dataset in course database table: ildoc_admit",
+                    "source_url": "",
+                    "geographical_unit": [
+                        "State",
+                        "Zip Code"
+                    ],
+                    "related_articles": [
+                        "rc-000001",
+                        "rc-000003",
+                        "rc-000005",
+                        "rc-000009",
+                        "rc-000010"
+                    ],
+                    "data_provider": "Illinois Department of Corrections",
+                    "dataset_documentation": [
+                        "idoc_admissions_data_dictionary.pdf"
+                    ],
+                    "dataset_version_date": 1488823197,
+                    "source_archive": "",
+                    "dataset_citation": "Illinois Department of Corrections, 2016, \"Illinois Department of Corrections (DOC) Inmate Admissions 1990-2015\", Applied Data Analytics Course [Distributor], 1 [Version]",
+                    "reference_url": "https://www.illinois.gov/idoc/Pages/default.aspx",
+                    "temporal_coverage_start": "1990",
+                    "file_names": [
+                        "adrf-000001.csv"
+                    ]
+                },
+                "visible_to": [
+                    "dataset-adrf-000001"
+                ]
+            }
+        }
+    ]
+}
diff --git a/data/datasets/search_metadata/dataset-adrf-000002.json b/data/datasets/search_metadata/dataset-adrf-000002.json
@@ -0,0 +1,64 @@
+{
+    "gmeta": [
+        {
+            "adrf-000002": {
+                "mimetype": "application/json",
+                "content": {
+                    "temporal_coverage_end": "2015",
+                    "files_total": 1,
+                    "data_classification": "Restricted",
+                    "access_actions_required": "This is ADRF course data. In order to access, you must have signed the appropriate NDA and agreed to the ADRF terms of usage. These data are located in the course database. Field specific metadata are a sample only.",
+                    "geographical_coverage": [
+                        "Illinois"
+                    ],
+                    "keywords": [
+                        "criminal justice system",
+                        "inmates",
+                        "jail",
+                        "correctional facilities",
+                        "health",
+                        "employment",
+                        "demographics",
+                        "offenses"
+                    ],
+                    "dataset_id": "adrf-000002",
+                    "category": "Inmate Populations",
+                    "dataset_version": 1,
+                    "title": "Illinois Department of Corrections (DOC) Inmate Exits - 1990-2015",
+                    "data_usage_policy": "This dataset is restricted to ADRF course participants who have approval and have agreed to the terms of use provided by the Illinois Department of Corrections (IDOC). The user shall not attempt to re-identify individuals represented in the data and agree to not disclose any data contained here to any unauthorized parties. Any analyses and data products derived from the data must be reviewed by IDOC before export from this system.",
+                    "data_steward_organization": "Center for Urban Science and Progress (CUSP), NYU",
+                    "data_steward": "Julia Lane",
+                    "access_requirements": "You must be enrolled in an ADRF class to access these data.",
+                    "description": "Detailed transactional data of each time an inmate was released from an Illinois Department of Corrections (DOC) facility from 1990 to 2015. Variables include demographic, residence, charges, sentencing, conduct, security level, health and mental health status, gang affiliation. Data are collected using correctional facility data and court records. See data dictionary documentation for detailed description of all fields.\n\nThe summary of this dataset\u2019s fields is a sample of records only. Please access the full dataset in course database table: ildoc_exit",
+                    "source_url": "",
+                    "geographical_unit": [
+                        "State",
+                        "Zip Code"
+                    ],
+                    "related_articles": [
+                        "rc-000001",
+                        "rc-000003",
+                        "rc-000005",
+                        "rc-000009",
+                        "rc-000010"
+                    ],
+                    "data_provider": "Illinois Department of Corrections",
+                    "dataset_documentation": [
+                        "idoc_exits_data_dictionary.pdf"
+                    ],
+                    "dataset_version_date": 1488823197,
+                    "source_archive": "",
+                    "dataset_citation": "Illinois Department of Corrections, 2016, \"Illinois Department of Corrections (DOC) Inmate Exits - 1990-2015\", Applied Data Analytics Course [Distributor], 1 [Version]",
+                    "reference_url": "https://www.illinois.gov/idoc/Pages/default.aspx",
+                    "temporal_coverage_start": "1990",
+                    "file_names": [
+                        "adrf-000002.csv"
+                    ]
+                },
+                "visible_to": [
+                    "dataset-adrf-000002"
+                ]
+            }
+        }
+    ]
+}
diff --git a/data/datasets/search_metadata/dataset-adrf-000003.json b/data/datasets/search_metadata/dataset-adrf-000003.json
@@ -0,0 +1,61 @@
+{
+    "gmeta": [
+        {
+            "adrf-000003": {
+                "mimetype": "application/json",
+                "content": {
+                    "temporal_coverage_end": "2015",
+                    "files_total": 1,
+                    "data_classification": "Restricted",
+                    "access_actions_required": "This is ADRF course data. In order to access, you must have signed the appropriate NDA and agreed to the ADRF terms of usage. These data are located in the course database. Field specific metadata are a sample only.",
+                    "geographical_coverage": [
+                        "Illinois"
+                    ],
+                    "keywords": [
+                        "income",
+                        "welfare",
+                        "poverty",
+                        "socioeconomics"
+                    ],
+                    "dataset_id": "adrf-000003",
+                    "category": "Economic Characteristics",
+                    "dataset_version": 1,
+                    "title": "Illinois Department of Employment Security (IDES) Unemployment Insurance (UI) wage records - 2005-2015",
+                    "data_usage_policy": "This dataset is restricted to ADRF course participants who have approval and have agreed to the terms of use provided by the Illinois Department of Economic Security (IDES). All data provided here are protected under the provisions of Title 13, U.S. Code and users accessing them are subject to the provisions of 820 ILCS 405/1900. The user shall not attempt to re-identify individuals represented in the data and agree not to disclose any data contained here to any unauthorized parties. Any analyses and data products derived from the data must be reviewed by IDES before export from this system.",
+                    "data_steward_organization": "Center for Urban Science and Progress (CUSP), NYU",
+                    "data_steward": "Julia Lane",
+                    "access_requirements": "You must be enrolled in an ADRF class to access these data.",
+                    "description": "Quarterly records of wages for every job held by each person in the state of Illinois from 2005 to 2015. This data is derived from the Illinois Dept. of Employment Security (IDES) Unemployment Insurance (UI) wage file that the Local Employment Dynamics (LED) state partners supply to the Census department for use in producing Quarterly Workforce Indicators (QWI). See attached data dictionary for field details.\n\nThe summary of this dataset\u2019s fields is a sample of records only. Please access the full dataset in course database table: il_wage",
+                    "source_url": "",
+                    "geographical_unit": [
+                        "State"
+                    ],
+                    "related_articles": [
+                        "rc-000001",
+                        "rc-000002",
+                        "rc-000064",
+                        "rc-000065",
+                        "rc-000066",
+                        "rc-000067",
+                        "rc-000068"
+                    ],
+                    "data_provider": "Illinois Department of Employment Security, US Census Bureau",
+                    "dataset_documentation": [
+                        "ides_wage_data_dictionary.pdf"
+                    ],
+                    "dataset_version_date": 1488823197,
+                    "source_archive": "",
+                    "dataset_citation": "Illinois Department of Economic Security, US Census Bureau, 2016, \"Illinois Department of Employment Security (IDES) Unemployment Insurance (UI) wage records - 2005-2015\", Applied Data Analytics Course [Distributor], 1 [Version]",
+                    "reference_url": "http://www.ides.illinois.gov/Pages/Unemployment%20Insurance.aspx",
+                    "temporal_coverage_start": "2005",
+                    "file_names": [
+                        "adrf-000003.csv"
+                    ]
+                },
+                "visible_to": [
+                    "dataset-adrf-000003"
+                ]
+            }
+        }
+    ]
+}