Skip to content
Browse files

complete port of bootcamp code

  • Loading branch information...
1 parent 0ce6889 commit c0a8d3c70edb7518fe1f3b659128c9efa562ca44 @zstumgoren zstumgoren committed Feb 23, 2013
Showing with 31 additions and 15 deletions.
  1. +4 −1 projects/fdic/.gitignore
  2. +11 −0 projects/fdic/README.md
  3. +4 −5 projects/fdic/save_to_csv.py
  4. +12 −9 projects/fdic/save_to_db.py
View
5 projects/fdic/.gitignore
@@ -1,2 +1,5 @@
-*.vim
+*.csv
*.pyc
+*.txt
+*.sqlite
+*.vim
View
11 projects/fdic/README.md
@@ -0,0 +1,11 @@
+The Life of a Data Project
+
+We're scraping the FDIC failed bank list.
+
+In the process, we're showing how to:
+
+* write a reusable scraper
+* save the data to a csv
+* save the data to a db
+* do. stuff. with. data.
+* rinse and repeat.
View
9 projects/fdic/save_to_csv.py
@@ -71,19 +71,18 @@ def convertdatestring(datestring):
except:
row[6] = ''
-filename = os.path.join(PROJECT_DIR, 'fdic_output.txt')
+filename = os.path.join(PROJECT_DIR, 'fdic.txt')
# This is a Python idiom you'll see often.
# You're opening a file so that you can read data from it.
# Then, you use the csv module to help write the data to a file
# http://docs.python.org/2/library/csv.html#csv.DictReader
with open(filename, 'wb') as outputfile:
- wtr = csv.DictWriter(outputfile, delimiter='|', fieldnames=headers,
- lineterminator='\n', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
+ wtr = csv.writer(outputfile, delimiter='|', quotechar='"')
- # Add headers to output
- wtr.writeheader()
+ # Add headers tooutput
+ wtr.writerow(headers)
# Write the data
wtr.writerows(data)
View
21 projects/fdic/save_to_db.py
@@ -4,27 +4,27 @@
import os
import sqlite3
-# Import our dynamically calculated project directory
-# It's a bit of magic that makes this code work on Macs, Windows, and Linux :)
from settings import PROJECT_DIR
-# Create a SQLite database in our project directory
-db_file = os.path.join(PROJECT_DIR, 'bootcamp.sqlite')
+from scraper import scrape_data
-# Now we're ready to connect to the database
+# Construct the file path to our (soon-to-be-created) SQLite database
+db_file = os.path.join(PROJECT_DIR, 'fdic.sqlite')
+
+# Now we're ready to create our database and open a connection to it
# http://docs.python.org/2/library/sqlite3.html
conn = sqlite3.connect(db_file)
-# Once we're connected, we get a database "cursor"
-# (which let's you send SQL statements to the database)
+# Once we're connected, we need a database "cursor" so
+# we can send SQL statements to the db
cur = conn.cursor()
# Here's the SQL to create our database table
TBL_CREATE_STMT = """
CREATE TABLE IF NOT EXISTS failed_banks (
bank varchar (54) NOT NULL,
city varchar (17) NOT NULL,
- tate varchar (4) NOT NULL,
+ state varchar (4) NOT NULL,
cert_num INTEGER NOT NULL,
acq_inst VARCHAR (65) NOT NULL,
closed DATE NOT NULL,
@@ -38,9 +38,12 @@
# Commit our change
conn.commit()
+# Get results data (recall that it's a list of two elements [headers, data]
+results = scrape_data()
+data = results[1]
cur.executemany('INSERT INTO failed_banks (bank, city, state, cert_num, acq_inst, ' \
- 'closed, updated, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);', to_db)
+ 'closed, updated, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);', data)
# Commit our inserts
conn.commit()
# Close db connection

0 comments on commit c0a8d3c

Please sign in to comment.
Something went wrong with that request. Please try again.