From ac84ee907fb1d8c4957652e3df422e410e322067 Mon Sep 17 00:00:00 2001
From: Thomas Levine <occurrence@thomaslevine.com>
Date: Mon, 11 Mar 2013 00:59:20 -0400
Subject: [PATCH] schema

---
 readme.md        |  3 +++
 requirements.txt |  3 +++
 run.py           | 14 +++++---------
 schema.sql       | 33 +++++++++++++++++++++++++++++++++
 4 files changed, 44 insertions(+), 9 deletions(-)
 create mode 100644 requirements.txt
 mode change 100644 => 100755 run.py
 create mode 100644 schema.sql

diff --git a/readme.md b/readme.md
index 9d72871..7d3bd0b 100644
--- a/readme.md
+++ b/readme.md
@@ -6,6 +6,9 @@ current auroras.
     # Set your credentials
     export FLICKR_KEY=12345abcde...
 
+    # Create schema
+    sqlite3 aurora.db < schema.sql
+
     # Download
     ./run
 
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..63e4acc
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+requests
+lxml
+dumptruck
diff --git a/run.py b/run.py
old mode 100644
new mode 100755
index 6440676..34456ab
--- a/run.py
+++ b/run.py
@@ -3,6 +3,7 @@
 import datetime
 from requests import get
 from lxml.etree import fromstring
+from dumptruck import DumpTruck
 
 FLICKR_REST = 'http://api.flickr.com/services/rest/'
 GROUPS = ['40371529@N00']
@@ -54,9 +55,6 @@ def parse(text):
 
 def group(dt, group_id, verbose = False):
     'Download a group.'
-    if not os.path.isdir(group):
-        os.mkdir(group)
-
     n_pages = 1
     n_page = 1
     while n_page <= n_pages:
@@ -71,7 +69,7 @@ def group(dt, group_id, verbose = False):
         dt.insert(data, 'photo')
 
         # Continue
-        photos = fromstring(text).xpath('//photos')[0]
+        photos = fromstring(text.encode('utf-8')).xpath('//photos')[0]
         n_page = int(photos.xpath('@page')[0])
         n_pages = int(photos.xpath('@pages')[0])
         if verbose:
@@ -80,11 +78,9 @@ def group(dt, group_id, verbose = False):
 
 
 def main():
-    # Make directories.
-    dt = DumpTruck(dbname = 'aurora.db')
-    for group_id in groups:
-        group(dt, group_id)
-
+    dt = DumpTruck(dbname = 'aurora.db', adapt_and_convert = True)
+    for group_id in GROUPS:
+        group(dt, group_id, verbose = True)
 
 if __name__ == '__main__':
     main()
diff --git a/schema.sql b/schema.sql
new file mode 100644
index 0000000..ff81711
--- /dev/null
+++ b/schema.sql
@@ -0,0 +1,33 @@
+CREATE TABLE photo (
+  -- Page
+  page INTEGER,
+  within_page INTEGER,
+
+  -- Standard response
+  id TEXT NOT NULL,
+  owner TEXT NOT NULL,
+  title TEXT NOT NULL,
+  dateadded DATETIME NOT NULL,
+
+  -- Special fields
+  date_upload DATETIME NOT NULL,
+  date_taken DATETIME NOT NULL,
+  description TEXT NOT NULL,
+  url_l TEXT NOT NULL,
+  longitude REAL NOT NULL,
+  latitude REAL NOT NULL,
+  UNIQUE(page, within_page)
+);
+
+CREATE VIEW aurora AS SELECT
+  url_l AS 'url',
+  'http://www.flickr.com/photos/' || owner || '/' || id AS 'photostream_url'
+  'http://www.flickr.com/people/' || owner AS 'owner_url'
+  date_taken,
+  date_upload,
+  dateadded AS date_added,
+  title,
+  description,
+  longitude,
+  latitude
+FROM photo;