Navigation Menu

Skip to content
This repository has been archived by the owner on Jun 21, 2021. It is now read-only.

Commit

Permalink
Move away from elixir to flask-sqlalchemy
Browse files Browse the repository at this point in the history
  • Loading branch information
yuvipanda committed Sep 3, 2012
1 parent a8dd9fe commit f20418f
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 29 deletions.
11 changes: 6 additions & 5 deletions api/app.py
Expand Up @@ -2,11 +2,11 @@
from flask import *
import json

setup_all()

app = Flask(__name__)
app.debug = True

setup_app(app)

@app.route('/issues')
def firstIssues():
return issues(0)
Expand Down Expand Up @@ -48,7 +48,8 @@ def register_device():
if regID:
if Device.query.filter_by(regID=regID).count() == 0:
device = Device(regID=regID)
session.commit()
db.session.add(device)
db.session.commit()
return ("", 200)
else:
return ("regID already registered", 200)
Expand All @@ -61,8 +62,8 @@ def deregister_device():
if regID:
if Device.query.filter_by(regID=regID).count() != 0:
device = Device.query.filter_by(regID=regID).one()
device.delete()
session.commit()
db.session.delete(device)
db.session.commit()
return ("", 200)
else:
return ("No regID specified", 400)
Expand Down
54 changes: 34 additions & 20 deletions api/db.py
@@ -1,21 +1,34 @@
import json
import time

from elixir import *
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
import settings

metadata.bind = settings.DATABASE
db = SQLAlchemy()

class Post(Entity):
using_options(tablename='posts')
def setup_script():
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = settings.DATABASE
db.init_app(app)
app.test_request_context().push()

permalink = Field(Unicode(255))
title = Field(Unicode(255))
content = Field(UnicodeText)
author = Field(Unicode(255))
author_link = Field(Unicode(255))
image_link = Field(Unicode(1024))
issue = ManyToOne('Issue')
def setup_app(app):
app.config['SQLALCHEMY_DATABASE_URI'] = settings.DATABASE
db.init_app(app)

class Post(db.Model):
__tablename__ = "posts"

id = db.Column(db.Integer, primary_key=True)
permalink = db.Column(db.String(255), index=True)
title = db.Column(db.String(1024))
content = db.Column(db.Text())
author = db.Column(db.String(255))
author_link = db.Column(db.String(255))
image_link = db.Column(db.String(1024))
issue_id = db.Column(db.Integer, db.ForeignKey('issues.id'))
issue = db.relationship('Issue', backref=db.backref('posts', lazy='dynamic'))

def serialize(self, exclude_content=False):
data = {
Expand All @@ -30,13 +43,13 @@ def serialize(self, exclude_content=False):
data['content'] = self.content
return data

class Issue(Entity):
using_options(tablename='issues')

date = Field(DateTime)
permalink = Field(Unicode(255))
posts = OneToMany('Post')
class Issue(db.Model):
__tablename__ = "issues"

id = db.Column(db.Integer, primary_key=True)
date = db.Column(db.DateTime, index=True)
permalink = db.Column(db.String(1024), index=True)
#posts = OneToMany('Post')

def serialize(self):
return {
Expand All @@ -45,7 +58,8 @@ def serialize(self):
'permalink': self.permalink
}

class Device(Entity):
using_options(tablename='devices')
class Device(db.Model):
__tablename__ = 'devices'

regID = Field(Unicode(1024))
id = db.Column(db.Integer, primary_key=True)
regID = db.Column(db.String(1024))
13 changes: 9 additions & 4 deletions api/scraper.py
Expand Up @@ -7,11 +7,15 @@
import urllib2
import re

from flask import Flask

from db import *

START_YEAR = 2005
CUR_YEAR = datetime.now().year

setup_script()

api = MWApi('http://en.wikipedia.org')

def content_for_title(title):
Expand Down Expand Up @@ -80,6 +84,7 @@ def save_issue(date_string):
if Issue.query.filter_by(date=date).count() != 0:
return "Skipping %s" % date
cur_issue = Issue(date=date, permalink="en.wikipedia.org/wiki/Wikipedia:Wikipedia_Signpost/Archives/" + date.strftime("%Y-%m-%d"))
db.session.add(cur_issue)

issue = "Wikipedia:Wikipedia Signpost/Archives/" + date.strftime("%Y-%m-%d")
doc = html.document_fromstring(content_for_title(issue))
Expand All @@ -96,13 +101,13 @@ def save_issue(date_string):
else:
print permalink
raise
Post(permalink=permalink, title=title, content=content, author=author_name, author_link=author_link, issue=cur_issue, image_link=image_link)
session.commit()
post = Post(permalink=permalink, title=title, content=content, author=author_name, author_link=author_link, issue=cur_issue, image_link=image_link)
db.session.add(post)
db.session.commit()
return "Done %s\n%s" % (date, "\n".join(articles))

if __name__ == "__main__":
setup_all()
create_all()
db.create_all()

issues = [issue for issue in get_subpages("Wikipedia_Signpost/Archives/", 4) if '-' in issue]
for issue in issues:
Expand Down

0 comments on commit f20418f

Please sign in to comment.