schema

readme.md

@@ -6,6 +6,9 @@ current auroras.
     # Set your credentials
     export FLICKR_KEY=12345abcde...
 
+    # Create schema
+    sqlite3 aurora.db < schema.sql
+
     # Download
     ./run
 

requirements.txt

@@ -0,0 +1,3 @@
+requests
+lxml
+dumptruck

run.py

@@ -3,6 +3,7 @@
 import datetime
 from requests import get
 from lxml.etree import fromstring
+from dumptruck import DumpTruck
 
 FLICKR_REST = 'http://api.flickr.com/services/rest/'
 GROUPS = ['40371529@N00']
@@ -54,9 +55,6 @@ def parse(text):
 
 def group(dt, group_id, verbose = False):
     'Download a group.'
-    if not os.path.isdir(group):
-        os.mkdir(group)
-
     n_pages = 1
     n_page = 1
     while n_page <= n_pages:
@@ -71,7 +69,7 @@ def group(dt, group_id, verbose = False):
         dt.insert(data, 'photo')
 
         # Continue
-        photos = fromstring(text).xpath('//photos')[0]
+        photos = fromstring(text.encode('utf-8')).xpath('//photos')[0]
         n_page = int(photos.xpath('@page')[0])
         n_pages = int(photos.xpath('@pages')[0])
         if verbose:
@@ -80,11 +78,9 @@ def group(dt, group_id, verbose = False):
 
 
 def main():
-    # Make directories.
-    dt = DumpTruck(dbname = 'aurora.db')
-    for group_id in groups:
-        group(dt, group_id)
-
+    dt = DumpTruck(dbname = 'aurora.db', adapt_and_convert = True)
+    for group_id in GROUPS:
+        group(dt, group_id, verbose = True)
 
 if __name__ == '__main__':
     main()

schema.sql

@@ -0,0 +1,33 @@
+CREATE TABLE photo (
+  -- Page
+  page INTEGER,
+  within_page INTEGER,
+
+  -- Standard response
+  id TEXT NOT NULL,
+  owner TEXT NOT NULL,
+  title TEXT NOT NULL,
+  dateadded DATETIME NOT NULL,
+
+  -- Special fields
+  date_upload DATETIME NOT NULL,
+  date_taken DATETIME NOT NULL,
+  description TEXT NOT NULL,
+  url_l TEXT NOT NULL,
+  longitude REAL NOT NULL,
+  latitude REAL NOT NULL,
+  UNIQUE(page, within_page)
+);
+
+CREATE VIEW aurora AS SELECT
+  url_l AS 'url',
+  'http://www.flickr.com/photos/' || owner || '/' || id AS 'photostream_url'
+  'http://www.flickr.com/people/' || owner AS 'owner_url'
+  date_taken,
+  date_upload,
+  dateadded AS date_added,
+  title,
+  description,
+  longitude,
+  latitude
+FROM photo;