Skip to content

Commit

Permalink
Fix handling of longitude negative sign
Browse files Browse the repository at this point in the history 
SWITRS stores longitude without the negative sign. Since all California
longitudes are negative, the longitude value is multiplied by -1 after
conversion from a string to a float.
  • Loading branch information
@agude
agude committed Sep 25, 2016
1 parent 29c2e1c commit da51b4e
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 3 deletions.
48 changes: 47 additions & 1 deletion load_into_database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,52 @@ def convert(**kwargs):
else:
return val

def negative(**kwargs):
"""Convert a value to a dtype and then multiplies by -1.
A list of nulls may optionally be provided, in which case if the input
value matches one of the strings in the list, None is returned.
Negative calls val.strip() before performing any other work.
Args:
**kwargs: Two specific keywords must be passed, a third is optional:
- val (str): A value to convert to dtype.
- dtype (callable): A callable object that returns the desired
type, if None then the val is passed through unchanged. The
returned object should allow multiplication by -1, or None will
be returned instead.
- nulls (iterable, optional): An iterable containing strings to check
against. If val if found to be equal to a string in this list,
None is returned.
Returns:
converted_val: Returns dtype(val) if val is not in nulls, otherwise
None. If dtype(val) raises a ValueError, None is returned.
"""
# Get the arguments
val = kwargs.get("val")
dtype = kwargs.get("dtype")
nulls = kwargs.get("nulls", None)

# Strip spaces
sval = val.strip()

# Return None if the val matches a string in nulls
if nulls is not None:
if sval in nulls:
return None

# Otherwise return the converted value
if dtype is not None:
try:
return -1 * dtype(sval)
except ValueError:
return None
# Note: val and not sval because this is the identity operation
else:
return val


def string_to_bool(**kwargs):
"""Convert Y/N or y/n to a True/False, or None if in a list of nulls.
Expand Down Expand Up @@ -415,7 +461,7 @@ def override_parent(self):
(72, "Primary_Ramp", DataType.TEXT, None, convert),
(73, "Secondary_Ramp", DataType.TEXT, None, convert),
(74, "Latitude", DataType.REAL, None, convert),
(75, "Longitude", DataType.REAL, None, convert),
(75, "Longitude", DataType.REAL, None, negative),
)

self.special_members = (
Expand Down
4 changes: 2 additions & 2 deletions tests/test_collisionrow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
ROWS = (
(
#Case_ID Collision_Year Process_Date Jurisdiction Collision_Date Collision_Time Officer_ID Reporting_District DOW CHP_Shift Population County_City_Location Special_Condition Beat_Type CHP_Beat_Type City_Division_LAPD CHP_Beat_Class Beat_Number Primary_Road Secondary_Road Distance Direction Intersection Weather_1 Weather_2 State_Highway_Indicator Caltrans_County Caltrans_District State_Route Route_Suffix Postmile_Prefix Postmile Location_Type Ramp_Intersection Side_Of_Highway Tow_Away Collision_Severity Killed_Victims Injured_Victims Party_Count Primary_Collision_Factor PCF_Violation_Code PCF_Violation_Category PCF_Violation PCF_Violation_Subsection Hit_And_Run Type_Of_Collision Motor_Vehicle_Involved_With Ped_Action Road_Surface Road_Condition_1 Road_Condition_2 Lighting Control_Device CHP_Road_Type Pedestrian_Collision Bicycle_Collision Motorcycle_Collision Truck_Collision Not_Private_Property Alcohol_Involved Statewide_Vehicle_Type_At_Fault CHP_Vehicle_Type_At_Fault Severe_Injury_Count Other_Visible_Injury_Count Complaint_Of_Pain_Injury_Count Pedestrian_Killed_Count Pedestrian_Injured_Count Bicyclist_Killed_Count Bicyclist_Injured_Count Motorcyclist_Killed_Count Motorcyclist_Injured_Count Primary_Ramp Secondary_Ramp Latitude Longitude
['0100010101011401155', '2001', '20010416', '0000', '20010101', '114', '1155qq', '0', '1', '4', '4', '198', '-', '0', 'A', '', '0', '073', 'DUBLIN_BL', 'SCARLETT_CT', '267000000', 'W', 'N', 'A', 'D', 'N', 'abc', '20', '280', 'A', 'A', '234000.222', 'H', '8', 'N', 'Y', '0', '999', '999', '999', 'A', 'C', '01', '23152', 'A', 'F', 'E', 'I', 'A', 'A', 'H', 'C', 'C', 'D', 'A', 'N', 'N', 'N', 'N', 'NY', 'N', 'A', '00', '0', '0', '0', '0', '0', '0', '0', '0', '0', 'NF-NB', 'EF-EB', '37.7749', '-122.4194'],
['0100010101011401155', '2001', '20010416', '0000', '20010101', '114', '1155qq', '0', '1', '4', '4', '198', '-', '0', 'A', '', '0', '073', 'DUBLIN_BL', 'SCARLETT_CT', '267000000', 'W', 'N', 'A', 'D', 'N', 'abc', '20', '280', 'A', 'A', '234000.222', 'H', '8', 'N', 'Y', '0', '999', '999', '999', 'A', 'C', '01', '23152', 'A', 'F', 'E', 'I', 'A', 'A', 'H', 'C', 'C', 'D', 'A', 'N', 'N', 'N', 'N', 'NY', 'N', 'A', '00', '0', '0', '0', '0', '0', '0', '0', '0', '0', 'NF-NB', 'EF-EB', '37.7749', '122.4194'],
['0100010101011401155', 0, '1155qq', '0', '4', '4', '198', None, '0', 'A', None, '0', '073', 'DUBLIN_BL', 'SCARLETT_CT', 267000000., 'W', False, 'A', 'D', False, 'abc', 20, 280, 'A', 'A', 234000.222, 'H', 8, 'N', True, 0, 999, 999, 999, 'A', 'C', '01', 23152, 'A', 'F', 'E', 'I', 'A', 'A', 'H', 'C', 'C', 'D', 'A', False, False, False, False, False, False, 'A', '00', 0, 0, 0, 0, 0, 0, 0, 0, 0, 'NF-NB', 'EF-EB', 37.7749, -122.4194, "2001-01-01", "01:14:00", "2001-04-16"],
),
(
#Case_ID Collision_Year Process_Date Jurisdiction Collision_Date Collision_Time Officer_ID Reporting_District DOW CHP_Shift Population County_City_Location Special_Condition Beat_Type CHP_Beat_Type City_Division_LAPD CHP_Beat_Class Beat_Number Primary_Road Secondary_Road Distance Direction Intersection Weather_1 Weather_2 State_Highway_Indicator Caltrans_County Caltrans_District State_Route Route_Suffix Postmile_Prefix Postmile Location_Type Ramp_Intersection Side_Of_Highway Tow_Away Collision_Severity Killed_Victims Injured_Victims Party_Count Primary_Collision_Factor PCF_Violation_Code PCF_Violation_Category PCF_Violation PCF_Violation_Subsection Hit_And_Run Type_Of_Collision Motor_Vehicle_Involved_With Ped_Action Road_Surface Road_Condition_1 Road_Condition_2 Lighting Control_Device CHP_Road_Type Pedestrian_Collision Bicycle_Collision Motorcycle_Collision Truck_Collision Not_Private_Property Alcohol_Involved Statewide_Vehicle_Type_At_Fault CHP_Vehicle_Type_At_Fault Severe_Injury_Count Other_Visible_Injury_Count Complaint_Of_Pain_Injury_Count Pedestrian_Killed_Count Pedestrian_Injured_Count Bicyclist_Killed_Count Bicyclist_Injured_Count Motorcyclist_Killed_Count Motorcyclist_Injured_Count Primary_Ramp Secondary_Ramp Latitude Longitude
['3337743', '2007', '20080214', '1900', '20070821', '0910', '478242', '0453', '2', '5', '0', '1949', '0', '8', '0', '-', '-', '43T1', 'HOXIE_AV', 'IMPERIAL_HWY', '400.02', 'S', 'Y', 'A', '-', 'Y', '123', '99', '80', 'B', 'B', '38', 'I', '4', 'E', 'N', '4', '0', '0', '1', 'E', 'W', '08', '22107', '', 'N', 'E', 'J', 'E', 'C', 'A', '-', 'A', 'A', '4', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'O', '99', '999', '999', '999', '999', '999', '999', '999', '999', '999', 'EF-EB', '-', '34.0522', '-118.2437'],
['3337743', '2007', '20080214', '1900', '20070821', '0910', '478242', '0453', '2', '5', '0', '1949', '0', '8', '0', '-', '-', '43T1', 'HOXIE_AV', 'IMPERIAL_HWY', '400.02', 'S', 'Y', 'A', '-', 'Y', '123', '99', '80', 'B', 'B', '38', 'I', '4', 'E', 'N', '4', '0', '0', '1', 'E', 'W', '08', '22107', '', 'N', 'E', 'J', 'E', 'C', 'A', '-', 'A', 'A', '4', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'O', '99', '999', '999', '999', '999', '999', '999', '999', '999', '999', 'EF-EB', '-', '34.0522', '118.2437'],
['3337743', 1900, '478242', '0453', '5', '0', '1949', '0', '8', '0', None, None, '43T1', 'HOXIE_AV', 'IMPERIAL_HWY', 400.02, 'S', True, 'A', None, True, '123', 99, 80, 'B', 'B', 38, 'I', 4, 'E', False, 4, 0, 0, 1, 'E', 'W', '08', 22107, None, 'N', 'E', 'J', 'E', 'C', 'A', None, 'A', 'A', '4', True, True, True, True, True, True, 'O', None, 999, 999, 999, 999, 999, 999, 999, 999, 999, 'EF-EB', None, 34.0522, -118.2437, "2007-08-21", "09:10:00", "2008-02-14"],
),
(
Expand Down

0 comments on commit da51b4e

Please sign in to comment.