Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix to how we build participation tables #475

Merged
merged 4 commits into from
May 5, 2017
Merged

Fix to how we build participation tables #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
343a6a5
Some broken tests and work to do, but revised participation tables
Apr 26, 2017
5065a51
Fixing tests for changes to the participation tables
Apr 27, 2017
4747e41
Add back participating population
Apr 27, 2017
d73f1be
Fix tests to match test data from Rhode Island
May 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 29 additions & 18 deletions crime_data/common/cdemodels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def query(self):
qry = qry.filter(newmodels.ParticipationRate.county_id == self.county_id)

if self.year:
qry = qry.filter(newmodels.ParticipationRate.data_year == self.year)
qry = qry.filter(newmodels.ParticipationRate.year == self.year)

return qry

Expand Down Expand Up @@ -131,18 +131,18 @@ def total_agencies_for_year(self, data_year):
return self._participation_for_year(data_year).total_agencies

@property
def reporting_agencies(self):
return self.reporting_agencies_for_year(self.current_year)
def participating_agencies(self):
return self.participating_agencies_for_year(self.current_year)

def reporting_agencies_for_year(self, data_year):
return self._participation_for_year(data_year).reporting_agencies
def participating_agencies_for_year(self, data_year):
return self._participation_for_year(data_year).participating_agencies

@property
def reporting_rate(self):
return self.reporting_rate_for_year(self.current_year)
def participation_rate(self):
return self.participating_rate_for_year(self.current_year)

def reporting_rate_for_year(self, data_year):
return self._participation_for_year(data_year).reporting_rate
def participation_rate_for_year(self, data_year):
return self._participation_for_year(data_year).participation_rate

@property
def total_population(self):
Expand All @@ -154,11 +154,13 @@ def total_population_for_year(self, data_year):
return self._participation_for_year(data_year).total_population

@property
def covered_population(self):
return self.covered_population_for_year(self.current_year)
def participating_population(self):
"""Returns the population for the given year"""
return self.participating_population_for_year(self.current_year)

def covered_population_for_year(self, data_year):
return self._participation_for_year(data_year).covered_population
def participating_population_for_year(self, data_year):
"""Returns the population for a given year"""
return self._participation_for_year(data_year).participating_population

def police_officers_for_year(self, data_year):
"""Returns the number of police officers for a given year"""
Expand All @@ -185,7 +187,7 @@ def _participation_for_year(self, year):

@property
def participation_rates(self):
return CdeParticipationRate(state_id=self.state_id).query.order_by('data_year DESC').all()
return CdeParticipationRate(state_id=self.state_id).query.order_by('year DESC').all()


class CdeRefCounty(RefCounty):
Expand Down Expand Up @@ -252,13 +254,22 @@ def total_agencies_for_year(self, data_year):
return self._participation_for_year(data_year).total_agencies

@property
def reporting_agencies(self):
def participating_agencies(self):
"""Returns the number of agencies for the most recent year"""
return self.reporting_agencies_for_year(self.current_year)
return self.participating_agencies_for_year(self.current_year)

def reporting_agencies_for_year(self, data_year):
def participating_agencies_for_year(self, data_year):
"""Counts the number of agencies for that county in a year."""
return self._participation_for_year(data_year).reporting_agencies
return self._participation_for_year(data_year).participating_agencies

@property
def participating_population(self):
"""Returns the population for the given year"""
return self.participating_population_for_year(self.current_year)

def participating_population_for_year(self, data_year):
"""Returns the population for a given year"""
return self._participation_for_year(data_year).participating_population

@property
def total_agencies(self):
Expand Down
43 changes: 11 additions & 32 deletions crime_data/common/marshmallow_schemas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -865,20 +865,12 @@ class Meta:
victim_type_name = marsh_fields.String(dump_only=True)


class AgencyParticipationSchema(Schema):
class AgencyParticipationSchema(ma.ModelSchema):
class Meta:
model = newmodels.AgencyAnnualParticipation
fields = ('state_name', 'state_abbr', 'year', 'agency_ori',
'agency_name', 'reported', 'months_reported',
'reported_nibrs', 'months_reported_nibrs',
'population_group_code',
'population_group', 'agency_population',)

model = newmodels.AgencyParticipation
exclude = ('agency_id', )
ordered = True

year = marsh_fields.Integer(attribute='data_year')


class ArsonCountSchema(Schema):
"""
Expand Down Expand Up @@ -985,30 +977,16 @@ class ParticipationRateSchema(ma.ModelSchema):
"""Response format for participation record"""

class Meta:
# model = cdemodels.CdeParticipationRate
model = newmodels.ParticipationRate
ordered = True
fields = ('year', 'total_population', 'covered_population',
'total_agencies', 'reporting_agencies', 'reporting_rate',
'nibrs_reporting_agencies', 'nibrs_reporting_rate',
'nibrs_covered_population',)
exclude = ('participation_id', 'state_id', 'state_abbr', 'state_name', )

year = marsh_fields.Integer(attribute='data_year')
total_population = marsh_fields.Integer()
covered_population = marsh_fields.Integer()
total_agencies = marsh_fields.Integer()
reporting_agencies = marsh_fields.Integer()
reporting_rate = marsh_fields.Float()
nibrs_reporting_agencies = marsh_fields.Integer()
nibrs_reporting_rate = marsh_fields.Float()
nibrs_covered_population = marsh_fields.Integer()

class StateParticipationRateSchema(ParticipationRateSchema):
class Meta:
fields = ('year', 'state_name', 'total_population', 'covered_population',
'total_agencies', 'reporting_agencies', 'reporting_rate',
'nibrs_reporting_agencies', 'nibrs_reporting_rate', 'nibrs_covered_population', )

state_name = marsh_fields.String()
model = newmodels.ParticipationRate
exclude = ('participation_id', )
ordered = True


class StateDetailResponseSchema(ma.ModelSchema):
Expand All @@ -1018,17 +996,18 @@ class Meta:
model = cdemodels.CdeRefState
ordered = True
fields = ('state_id', 'name', 'postal_abbr', 'fips_code', 'current_year',
'reporting_agencies', 'covered_population', 'reporting_rate',
'participating_agencies', 'participation_rate', 'participating_population',
'total_population', 'total_agencies', 'police_officers', 'counties',
'participation', )

name = marsh_fields.String(attribute='state_name')
postal_abbr = marsh_fields.String(attribute='state_postal_abbr')
fips_code = marsh_fields.String(attribute='state_fips_code')
total_population = marsh_fields.Integer()
covered_population = marsh_fields.Integer()
participating_population = marsh_fields.Integer()
total_agencies = marsh_fields.Integer()
reporting_agencies = marsh_fields.Integer()
participating_agencies = marsh_fields.Integer()
participation_rate = marsh_fields.Float()
police_officers = marsh_fields.Integer()
current_year = marsh_fields.Integer()

Expand Down
47 changes: 30 additions & 17 deletions crime_data/common/newmodels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
from sqlalchemy import or_,and_


class AgencyAnnualParticipation(db.Model):
class AgencyParticipation(db.Model):
"""Represents agency participation for a single month."""

__tablename__ = 'cde_annual_participation'
__tablename__ = 'agency_participation'

data_year = db.Column(db.SmallInteger, nullable=False, primary_key=True)
year = db.Column(db.SmallInteger, nullable=False, primary_key=True)
state_name = db.Column(db.String)
state_abbr = db.Column(db.String)
agency_id = db.Column(db.Integer, nullable=False, primary_key=True)
Expand All @@ -36,8 +36,11 @@ class AgencyAnnualParticipation(db.Model):
population_group = db.Column(db.String)
reported = db.Column(db.SmallInteger, nullable=False)
months_reported = db.Column(db.SmallInteger, nullable=False)
reported_nibrs = db.Column(db.SmallInteger, nullable=False)
months_reported_nibrs = db.Column(db.SmallInteger, nullable=False)
nibrs_reported = db.Column(db.SmallInteger, nullable=False)
nibrs_months_reported = db.Column(db.SmallInteger, nullable=False)
covered = db.Column(db.SmallInteger)
participated = db.Column(db.SmallInteger)
nibrs_participated = db.Column(db.SmallInteger)

@classmethod
def column_is_string(cls, col_name):
Expand Down Expand Up @@ -67,21 +70,31 @@ def filtered(cls, filters, args=None):


class ParticipationRate(db.Model):
__tablename__ = 'cde_participation_rates'
__tablename__ = 'participation_rates'

data_year = db.Column(db.SmallInteger, nullable=False, primary_key=True)
total_population = db.Column(db.BigInteger)
covered_population = db.Column(db.BigInteger)
total_agencies = db.Column(db.Integer)
reporting_agencies = db.Column(db.Integer)
reporting_rate = db.Column(db.Float)
nibrs_reporting_agencies = db.Column(db.Integer)
nibrs_reporting_rate = db.Column(db.Float)
nibrs_covered_population = db.Column(db.BigInteger)
state_id = db.Column(db.Integer)
county_id = db.Column(db.Integer)
participation_id = db.Column(db.Integer, nullable=False, primary_key=True)
year = db.Column(db.SmallInteger, nullable=False)
state_id = db.Column(db.Integer,
db.ForeignKey(RefState.state_id,
deferrable=True,
initially='DEFERRED'),
nullable=True)
county_id = db.Column(db.Integer,
db.ForeignKey(RefCounty.county_id,
deferrable=True,
initially='DEFERRED'),
nullable=True)
state_name = db.Column(db.String)
county_name = db.Column(db.String)
total_agencies = db.Column(db.Integer)
participating_agencies = db.Column(db.Integer)
participation_rate = db.Column(db.Float)
nibrs_participating_agencies = db.Column(db.Integer)
nibrs_participation_rate = db.Column(db.Float)
covered_agencies = db.Column(db.Integer)
covered_rate = db.Column(db.Float)
total_population = db.Column(db.BigInteger)
participating_population = db.Column(db.BigInteger)


class CreatableModel:
Expand Down
28 changes: 4 additions & 24 deletions crime_data/resources/participation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def get(self, args, state_id=None, state_abbr=None):
self.verify_api_key(args)

state = cdemodels.CdeRefState.get(abbr=state_abbr, state_id=state_id).one()
rates = cdemodels.CdeParticipationRate(state_id=state.state_id).query.order_by('data_year DESC').all()
rates = cdemodels.CdeParticipationRate(state_id=state.state_id).query.order_by('year DESC').all()
filename = '{}_state_participation'.format(state.state_postal_abbr)
return self.render_response(rates, args, csv_filename=filename)

Expand All @@ -41,37 +41,17 @@ def get(self, args):
rates = cdemodels.CdeParticipationRate().query
rates = rates.filter(newmodels.ParticipationRate.state_id == None)
rates = rates.filter(newmodels.ParticipationRate.county_id == None)
rates = rates.order_by('data_year DESC').all()
rates = rates.order_by('year DESC').all()
filename = 'participation_rates'
return self.render_response(rates, args, csv_filename=filename)


class AgenciesParticipation(CdeResource):

schema = marshmallow_schemas.AgencyParticipationSchema(many=True,
only=('state_name', 'state_abbr', 'year', 'agency_ori',
'agency_name', 'reported',
'months_reported',
'months_reported_nibrs',
'population_group_code',
'population_group',
'agency_population',)
)

tables = newmodels.AgencyAnnualParticipation
schema = marshmallow_schemas.AgencyParticipationSchema(many=True)
tables = newmodels.AgencyParticipation
is_groupable = False

def postprocess_filters(self, filters, args):
years = [x for x in filters if x[0] == 'year']

print(filters)
if years:
y = years[0]
filters = [x for x in filters if x[0] != 'year']
filters.append(('data_year', y[1], y[2]))

return filters

@use_args(marshmallow_schemas.ArgumentsSchema)
@cache(max_age=DEFAULT_MAX_AGE, public=True)
@tuning_page
Expand Down
45 changes: 24 additions & 21 deletions crime_data/static/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@
},
"monthsReportedNibrsParam": {
"in": "query",
"name": "reported_nibrs",
"name": "nibrs_reported",
"description": "Months in a year that an agency has reported NIBRS. This makes the most sense when combined with a `year` argument.",
"required": false,
"type": "string"
Expand Down Expand Up @@ -1086,13 +1086,13 @@
"minimum": 0,
"maximum": 12
},
"reported_nibrs": {
"nibrs_reported": {
"type": "integer",
"description": "This is 1 if agency reported to NIBRS at least once in the given year",
"minimum": 0,
"maximum": 1
},
"months_reported_nibrs": {
"nibrs_months_reported": {
"type": "integer",
"description": "How many months the agency filed a NIBRS crime report in the given year",
"minimum": 0,
Expand All @@ -1104,14 +1104,13 @@
"type": "object",
"required": [
"year",
"reporting_agencies",
"participating_agencies",
"total_agencies",
"reporting_rate",
"participation_rate",
"total_population",
"nibrs_reporting_agencies",
"nibrs_reporting_rate",
"covered_population",
"nibrs_covered_population"
"participating_population",
"nibrs_participating_agencies",
"nibrs_participation_rate"
],
"prvoperties": {
"year": {
Expand All @@ -1120,17 +1119,18 @@
"description": "The year that these participation statistics were calculated for",
"minimum": 1960
},
"reporting_agencies": {
"participating_agencies": {
"format": "int32",
"type": "integer"
"type": "integer",
"description": "How many agencies either reported crime or were covered by another agency that reported"
},
"total_agencies": {
"description": "The total number of agencies for the specific year",
"format": "int32",
"type": "integer",
"minimum": 0
},
"reporting_rate": {
"participation_rate": {
"description": "A rate of which agencies have filed either an SRS or NIBRS report in the given year",
"format": "float",
"type": "number",
Expand All @@ -1146,22 +1146,25 @@
],
"minimum": 0
},
"nibrs_reporting_agencies": {
"description": "The number of agencies who filed a NIBRS report in the given year",
"participating_population": {
"description": "The sum of the population within the jurisdictions of all agencies that participated for that year",
"format": "int32",
"type": [
"integer",
"null"
],
"minimum": 0
},
"nibrs_participating_agencies": {
"description": "The number of agencies who filed a NIBRS report in the given year or were covered by another agency that filed NIBRS",
"type": "integer"
},
"nibrs_reporting_rate": {
"nibrs_participation_rate": {
"description": "A rate of how many agencies filed a NIBRS report in a given year",
"format": "float",
"type": "number",
"minimum": 0,
"maximum": 1
},
"covered_population": {
"description": "The total population within jurisdictions of agencies who filed either a SRS or NIBRS report in the given year. format: int32 type: integer minimum: 0"
},
"nibrs_covered_population": {
"description": "The total population within jurisdictions of agencies who filed either a NIBRS report in the given year. format: int32 type: integer minimum: 0"
}
}
},
Expand Down
Loading