Permalink
Browse files

Database Migration for Scans => Appraisals

Adds a try-hard parser in case eveparse fails.
Adds babel commands to a new Makefile (because my memory sucks)
Adds database migrations via alembic
Moves populate_types script to tools directory
  • Loading branch information...
1 parent 76a71b8 commit ac92e2c4eacd7dac9aef0a9518c086b576edd3b5 @sudorandom sudorandom committed Jan 4, 2014
View
@@ -0,0 +1,9 @@
+all: compile_translations
+
+compile_translations:
+ @pybabel compile -d evepraisal/translations
+
+extract_translations:
+ @pybabel compile -d evepraisal/translations
+
+.PHONY: all extract_translations compile_translations
View
@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = evepraisal/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+sqlalchemy.url =
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
@@ -4,6 +4,7 @@
"""
import locale
import json
+import os
from flask import Flask, g, session
from flask.ext.cache import Cache
@@ -24,7 +25,8 @@
'30002053': 'Hek',
}
app.config['USER_AGENT'] = 'Evepraisal/1.0 +http://evepraisal.com/'
-app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///../data/scans.db'
+app.config['SQLALCHEMY_DATABASE_URI'] = ('sqlite:////%s/data/scans.db'
+ % os.getcwd())
app.config['CACHE_TYPE'] = 'memcached'
app.config['CACHE_KEY_PREFIX'] = 'evepraisal'
app.config['CACHE_MEMCACHED_SERVERS'] = ['127.0.0.1:11211']
@@ -44,7 +46,10 @@
cache.init_app(app)
# Late import so modules can import their dependencies properly
-from . import models, views, routes, filters # NOQA
+from . import models, views, routes, filters
+
+
+__all__ = ['models', 'views', 'routes', 'filters', 'app', 'db', 'cache']
def ignore_errors(f, *args, **kwargs):
@@ -0,0 +1 @@
+Generic single-database configuration.
@@ -0,0 +1,77 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import sys
+import os.path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from evepraisal import db, app
+target_metadata = db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL
+ and not an Engine, though an Engine is acceptable
+ here as well. By skipping the Engine creation
+ we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the
+ script output.
+
+ """
+ context.configure(url=app.config['SQLALCHEMY_DATABASE_URI'])
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+ config.set_section_option(config.config_ini_section,
+ 'sqlalchemy.url',
+ app.config['SQLALCHEMY_DATABASE_URI'])
+ engine = engine_from_config(
+ config.get_section(config.config_ini_section),
+ prefix='sqlalchemy.',
+ poolclass=pool.NullPool)
+
+ connection = engine.connect()
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata
+ )
+
+ try:
+ with context.begin_transaction():
+ context.run_migrations()
+ finally:
+ connection.close()
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
+
@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}
@@ -0,0 +1,86 @@
+"""Adds Appraisals table. Moves data from Scans table into the new one.
+
+Revision ID: 47e13f1d69fc
+Revises: None
+Create Date: 2014-01-03 19:56:07.845816
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '47e13f1d69fc'
+down_revision = None
+import json
+
+from alembic import op
+import sqlalchemy as sa
+import evepaste
+
+from evepraisal.models import Appraisals
+from evepraisal import db
+from evepraisal.parser import parse
+
+
+class Scans(db.Model):
+ __tablename__ = 'Scans'
+
+ Id = db.Column(db.Integer(), primary_key=True)
+ #: Being split up into RawInput, ParsedJson, Prices
+ Data = db.Column(db.Text())
+ #: Being kept (with an added index)
+ Created = db.Column(db.Integer())
+ #: Going away
+ SellValue = db.Column(db.Float())
+ #: Going away
+ BuyValue = db.Column(db.Float())
+ #: Being kept (with an added index)
+ Public = db.Column(db.Boolean(), default=True)
+ #: Being kept (with an added index)
+ UserId = db.Column(db.Integer(), db.ForeignKey('Users.Id'))
+
+
+def upgrade():
+ FAILED_COUNT = 0
+ MODIFIED_COUNT = 0
+ SUCCESS_COUNT = 0
+ for i, scan in enumerate(Scans.query.filter()):
+ scan_data = json.loads(scan.Data)
+ prices = [[item.get('typeID'), {'all': item.get('all'),
+ 'buy': item.get('buy'),
+ 'sell': item.get('sell')}]
+ for item in scan_data['line_items']]
+
+ try:
+ kind, result, bad_lines = parse(scan_data['raw_paste'])
+ except evepaste.Unparsable:
+ print('--[Unparsable]---------')
+ print()
+ print([scan_data['raw_paste']])
+ print('-'*20)
+ kind = 'listing'
+ result = [{'name': item['typeName'], 'quantity': item['count']}
+ for item in scan_data['line_items']]
+ bad_lines = scan_data['bad_line_items']
+ FAILED_COUNT += 1
+
+ appraisal = Appraisals(Id=scan.Id,
+ Created=scan.Created,
+ RawInput=scan_data['raw_paste'],
+ Parsed=result,
+ Kind=kind,
+ BadLines=bad_lines,
+ Prices=prices,
+ Market=int(scan_data['solar_system']),
+ Public=scan.Public,
+ UserId=scan.UserId)
+ db.session.add(appraisal)
+ SUCCESS_COUNT += 1
+
+ db.session.commit()
+ print "Sucesses", SUCCESS_COUNT
+ print "Modified", MODIFIED_COUNT
+ print "Failed", FAILED_COUNT
+
+
+def downgrade():
+ db.session.query(Appraisals).delete()
+ db.session.commit()
@@ -96,12 +96,3 @@ def relative_time(past):
return format_timedelta(delta, locale='en_US') + postfix
except Exception:
return ''
-
-
-@app.template_filter('bpc_count')
-def bpc_count(bad_lines):
- c = 0
- for line in bad_lines:
- if '(copy)' in line.lower():
- c += 1
- return c
View
@@ -3,11 +3,11 @@
from . import db
from helpers import iter_types
-from sqlalchemy.types import TypeDecorator, Unicode
+from sqlalchemy.types import TypeDecorator, VARCHAR
class JsonType(TypeDecorator):
- impl = Unicode
+ impl = VARCHAR
def process_bind_param(self, value, engine):
return json.dumps(value)
@@ -16,24 +16,6 @@ def process_result_value(self, value, engine):
return json.loads(value)
-class Scans(db.Model):
- __tablename__ = 'Scans'
-
- Id = db.Column(db.Integer(), primary_key=True)
- #: Being split up into RawInput, ParsedJson, Prices
- Data = db.Column(db.Text())
- #: Being kept (with an added index)
- Created = db.Column(db.Integer())
- #: Going away
- SellValue = db.Column(db.Float())
- #: Going away
- BuyValue = db.Column(db.Float())
- #: Being kept (with an added index)
- Public = db.Column(db.Boolean(), default=True)
- #: Being kept (with an added index)
- UserId = db.Column(db.Integer(), db.ForeignKey('Users.Id'))
-
-
class Appraisals(db.Model):
__tablename__ = 'Appraisals'
View
@@ -0,0 +1,30 @@
+import evepaste
+from models import get_type_by_name
+
+
+def parse(raw_paste):
+ try:
+ return evepaste.parse(raw_paste)
+ except evepaste.Unparsable:
+ kind, results, bad_lines = tryhard_parser(raw_paste)
+ if not results:
+ raise
+ return 'listing', results, bad_lines
+
+
+def tryhard_parser(raw_paste):
+ results = []
+ bad_lines = []
+ lines = raw_paste.split('\n')
+ for line in lines:
+ if '\t' in line:
+ parts = line.split('\t', 2)
+ name = parts[0].strip()
+ if get_type_by_name(name):
+ results.append({'name': name, 'quantity': 1})
+ else:
+ bad_lines.append(line)
+ else:
+ bad_lines.append(line)
+
+ return 'listing', results, bad_lines
Oops, something went wrong.

0 comments on commit ac92e2c

Please sign in to comment.