diff --git a/README.md b/README.md index b89bc08..1d7d47b 100644 --- a/README.md +++ b/README.md @@ -141,21 +141,45 @@ git checkout v2.0-dev pip install -U -r requirements.txt ``` +Make sure you have the `Postgres` database running. See [Ubuntu's help page](https://help.ubuntu.com/community/PostgreSQL) on setting it up. + +After that create the database and user with from your postgres interface. + +``` +CREATE USER cookcountyjail WITH PASSWORD 'walblgadb;lgall'; + +CREATE DATABASE cookcountyjail_v2_0_dev; + +GRANT ALL PRIVILEGES ON DATABASE cookcountyjail_v2_0_dev to cookcountyjail; + +``` + #Usage +Set up the database + +``` +python -3 ./manage.py upgrade + +``` + First get some data: ``` python -m scripts.scraper + ``` Then see it on the development server: ``` -python -3 manage.py -sdb -python -3 manage.py +python -3 ./manage.py runserver + ``` +The `-3` portion on all of these is optional. It just tells Python to wine about code +that wouldn't work on Python 3. You can ommit it when it's distracting you. + #Testing Whenever adding new functionality write tests. The first test file written, diff --git a/ccj/app.py b/ccj/app.py index 0461b30..e4c2c34 100644 --- a/ccj/app.py +++ b/ccj/app.py @@ -7,13 +7,16 @@ from flask.ext.sqlalchemy import SQLAlchemy from flask.ext import restful as rest +from json import loads from os import getcwd -from datetime import datetime +from datetime import datetime, date from ccj.models.daily_population import DailyPopulation as DPC from ccj.models.version_info import VersionInfo from ccj import config +from rest_api import CcjApi, get_or_create + STARTUP_TIME = datetime.now() app = Flask(__name__) @@ -21,8 +24,7 @@ db = SQLAlchemy(app) -from ccj.models.models import Person, Charge, Housing, CourtBuilding, CourtRoom -from rest_api import CcjApi +from ccj.models.models import Person, ChargeDescription, Statute, Housing, CourtBuilding, CourtRoom api = CcjApi(app, db) @@ -73,8 +75,51 @@ def version(): return VersionInfo(STARTUP_TIME).fetch(all_version_info=('all' in args and args['all'] == '1')) class Process(rest.Resource): + """ + The Proccess route handles all database input. + Scraper instances make POST requests to to this + route and it takes care of saving the data. + + In the near future scrapers might be whitelisted. + + The process route expects all data to be in a 'data' + key in the request's body. The data must be valid JSON. + + Example data: + + { + + } + + """ + def post(self): - return {"status": "saved"} + + data = None + + try: + data = loads(request.form['data']) + + except ValueError: + return {"message": "Error parsing json data", "status": 500}, 500 + + # the Person's attributes + phash = data.get("hash") + gender = data.get("gender") + race = data.get("race") + + new_person, person = get_or_create(db.session, Person, hash=phash) + person.gender = gender + person.race = race + + if new_person: + person.date_created = date.today() + + db.session.add(person) + db.session.commit() + + return {"message": "saved", "status": 200} + api.full_resource(env_info, "/os_env_info") api.full_resource(daily_population, "/daily_population") @@ -83,9 +128,9 @@ def post(self): api.full_class_resource(Process, "/process") api.less_resource(Person) -api.less_resource(Charge) +api.less_resource(ChargeDescription) +api.less_resource(Statute) api.less_resource(Housing) api.less_resource(CourtBuilding) api.less_resource(CourtRoom) - diff --git a/ccj/config.py b/ccj/config.py index 319191d..db0cf93 100644 --- a/ccj/config.py +++ b/ccj/config.py @@ -11,22 +11,14 @@ def get_db_uri(): - if use_postgres(): - db_config = { - 'dialect': 'postgresql', - 'db_user': 'cookcountyjail', - 'db_name': 'cookcountyjail_v2_0_dev', - 'pw': 'walblgadb;lgall', - 'host': 'localhost', - } - template = '%(dialect)s://%(db_user)s:%(pw)s@%(host)s/%(db_name)s' - else: - db_config = { - 'dialect': 'sqlite', - 'abs_path_to_db': '%s/ccj.db' % _basedir - } - template = '%(dialect)s:///%(abs_path_to_db)s' - + db_config = { + 'dialect': 'postgresql', + 'db_user': 'cookcountyjail', + 'db_name': 'cookcountyjail_v2_0_dev', + 'pw': 'walblgadb;lgall', + 'host': 'localhost', + } + template = '%(dialect)s://%(db_user)s:%(pw)s@%(host)s/%(db_name)s' return (template % db_config) diff --git a/ccj/models/migrations/versions/117bd134663c_.py b/ccj/models/migrations/versions/117bd134663c_.py index 22aa3ac..91d23e3 100644 --- a/ccj/models/migrations/versions/117bd134663c_.py +++ b/ccj/models/migrations/versions/117bd134663c_.py @@ -27,7 +27,7 @@ def upgrade(): op.create_table('person', sa.Column('id', sa.Integer(), nullable=False), sa.Column('hash', sa.Unicode(length=64), nullable=True), - sa.Column('gender', sa.Enum('M', 'F'), nullable=True), + sa.Column('gender', sa.Enum('M', 'F', name='pgender'), nullable=True), sa.Column('race', sa.Unicode(), nullable=True), sa.Column('date_created', sa.Date(), nullable=True), sa.PrimaryKeyConstraint('id'), diff --git a/ccj/models/migrations/versions/45e4c0464b06_.py b/ccj/models/migrations/versions/45e4c0464b06_.py new file mode 100644 index 0000000..2f49f0f --- /dev/null +++ b/ccj/models/migrations/versions/45e4c0464b06_.py @@ -0,0 +1,48 @@ +"""empty message + +Revision ID: 45e4c0464b06 +Revises: c52485120f6 +Create Date: 2014-05-13 21:40:39.342602 + +""" + +# revision identifiers, used by Alembic. +revision = '45e4c0464b06' +down_revision = 'c52485120f6' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.alter_column('charge_description', 'description', + existing_type=sa.VARCHAR(), + nullable=False) + op.create_unique_constraint(None, 'charge_description', ['description']) + op.add_column('court_room', sa.Column('court_building_id', sa.Integer(), nullable=True)) + op.alter_column('person', 'hash', + existing_type=sa.VARCHAR(length=64), + nullable=False) + op.alter_column('statue', 'citation', + existing_type=sa.VARCHAR(), + nullable=False) + op.create_unique_constraint(None, 'statue', ['citation']) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'statue') + op.alter_column('statue', 'citation', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('person', 'hash', + existing_type=sa.VARCHAR(length=64), + nullable=True) + op.drop_column('court_room', 'court_building_id') + op.drop_constraint(None, 'charge_description') + op.alter_column('charge_description', 'description', + existing_type=sa.VARCHAR(), + nullable=True) + ### end Alembic commands ### diff --git a/ccj/models/migrations/versions/46ccc3cb06c4_.py b/ccj/models/migrations/versions/46ccc3cb06c4_.py new file mode 100644 index 0000000..3ab21fc --- /dev/null +++ b/ccj/models/migrations/versions/46ccc3cb06c4_.py @@ -0,0 +1,39 @@ +"""empty message + +Revision ID: 46ccc3cb06c4 +Revises: 45e4c0464b06 +Create Date: 2014-05-15 10:37:39.673677 + +""" + +# revision identifiers, used by Alembic. +revision = '46ccc3cb06c4' +down_revision = '45e4c0464b06' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('statute', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('citation', sa.Unicode(), nullable=False), + sa.Column('date_created', sa.Date(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('citation') + ) + op.drop_table('statue') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('statue', + sa.Column('id', sa.INTEGER(), server_default="nextval('statue_id_seq'::regclass)", nullable=False), + sa.Column('citation', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('date_created', sa.DATE(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=u'statue_pkey') + ) + op.drop_table('statute') + ### end Alembic commands ### diff --git a/ccj/models/migrations/versions/c52485120f6_.py b/ccj/models/migrations/versions/c52485120f6_.py new file mode 100644 index 0000000..af20e7b --- /dev/null +++ b/ccj/models/migrations/versions/c52485120f6_.py @@ -0,0 +1,47 @@ +"""empty message + +Revision ID: c52485120f6 +Revises: 117bd134663c +Create Date: 2014-05-13 19:31:44.083493 + +""" + +# revision identifiers, used by Alembic. +revision = 'c52485120f6' +down_revision = '117bd134663c' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('statue', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('citation', sa.Unicode(), nullable=True), + sa.Column('date_created', sa.Date(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('charge_description', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.Unicode(), nullable=True), + sa.Column('date_created', sa.Date(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.drop_table('charge') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('charge', + sa.Column('id', sa.INTEGER(), server_default="nextval('charge_id_seq'::regclass)", nullable=False), + sa.Column('description', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('citation', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('primary_type', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('date_created', sa.DATE(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name=u'charge_pkey') + ) + op.drop_table('charge_description') + op.drop_table('statue') + ### end Alembic commands ### diff --git a/ccj/models/models.py b/ccj/models/models.py index 7673cd8..a5511c3 100644 --- a/ccj/models/models.py +++ b/ccj/models/models.py @@ -29,21 +29,21 @@ class Person(db.Model): # a hash to uniquely identify the person # if he or she came back to the jail - hash = db.Column(db.Unicode(64), unique=True) + hash = db.Column(db.Unicode(64), unique=True, nullable=False) # gender can only be M(male) or F(female) - gender = db.Column(db.Enum("M", "F")) + gender = db.Column(db.Enum("M", "F", name="pgender")) # race would be harder to parse with an # enum so we are sticking to good old # strings - race = db.Column(db.Unicode) + race = db.Column(db.Unicode(4)) # the date this person was added to the # database date_created = db.Column(db.Date) -class Charge(db.Model): +class ChargeDescription(db.Model): """ Every inmate gets a charge when they are booked. Often this changes overtime for @@ -54,12 +54,19 @@ class Charge(db.Model): id = db.Column(db.Integer, primary_key=True) # the charge will have a description - description = db.Column(db.Unicode) + description = db.Column(db.Unicode, nullable=False, unique=True) - # and a official citation - citation = db.Column(db.Unicode) + date_created = db.Column(db.Date) - primary_type = db.Column(db.Unicode) +class Statute(db.Model): + """ + A statute. + + """ + + id = db.Column(db.Integer, primary_key=True) + + citation = db.Column(db.Unicode, nullable=False, unique=True) date_created = db.Column(db.Date) @@ -135,7 +142,11 @@ class CourtRoom(db.Model): # the court room's number number = db.Column(db.Integer) - # court_building_id = + court_building_id = db.Column(db.Integer, db.ForeignKey('court_building.id')) + + court_building = db.relationship('CourtBuilding', + backref=db.backref('court_rooms', lazy='dynamic')) + date_created = db.Column(db.Date) diff --git a/ccj/rest_api.py b/ccj/rest_api.py index 067126b..642e789 100644 --- a/ccj/rest_api.py +++ b/ccj/rest_api.py @@ -87,4 +87,19 @@ def less_resource(self, db_model): self._less.create_api(db_model, url_prefix='', methods=['GET']) return self +def get_or_create(session, model, **kwargs): + """ + Django styled get or create method for + sqlalchemy. + + kwargs are the values that would UNIQUELY + identified the record. + + """ + instance = session.query(model).filter_by(**kwargs).first() + if instance: + return (False, instance) + else: + instance = model(**kwargs) + return (True, instance)