From 5b30b17b548336c50391ce184c6fddd0b7df70b4 Mon Sep 17 00:00:00 2001 From: Lucas K Date: Thu, 23 Jan 2020 12:52:54 -0600 Subject: [PATCH] Added UNIQUE constraint protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added try/catch against UNIQUE Constraint/Integrity error, it’ll still raise exception if it breaks again. Also fixed #15 --- .pylintrc | 4 ++-- maps/scraper/geocoder/jobmanager.py | 2 +- maps/scraper/springdale.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.pylintrc b/.pylintrc index 2948b38..c5f9fa3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -367,13 +367,13 @@ ignore-on-opaque-inference=yes # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local +ignored-classes=optparse.Values,thread._local,_thread._local,scoped_session # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=flask_sqlalchemy +ignored-modules= # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/maps/scraper/geocoder/jobmanager.py b/maps/scraper/geocoder/jobmanager.py index 4c56c1c..e76d168 100644 --- a/maps/scraper/geocoder/jobmanager.py +++ b/maps/scraper/geocoder/jobmanager.py @@ -88,7 +88,7 @@ def create(self, addresses): data=input_data) response_json = res.json() - if response_json['statusCode'] == '400': + if int(response_json['statusCode']) >= 400: # if there's an error creating the job, raise an error! Yay raise BingAPIError(response_json['errorDetails']) diff --git a/maps/scraper/springdale.py b/maps/scraper/springdale.py index d653627..f89c578 100644 --- a/maps/scraper/springdale.py +++ b/maps/scraper/springdale.py @@ -2,6 +2,8 @@ import requests import pytz +from sqlalchemy.exc import IntegrityError + from maps import db from maps.models import Call, CallQuery from maps.scraper.geocoder import geocode_lookup @@ -69,7 +71,14 @@ def scrape_to_db(): new_calls = geocode_calls(new_calls) for call in new_calls: - db.session.merge(call) + try: + db.session.merge(call) + except IntegrityError: + existing_call = CallQuery.get_existing_springdale(call) + if existing_call: + existing_call.notes = disposition + else: + raise Exception(f'Call {call} was given unique constraint error, but it can\'t be found in the db') # Commit new calls db.session.commit()