From 16f7bf9054b2ab2875a90aaa5861ef805954002e Mon Sep 17 00:00:00 2001 From: Maxime Date: Fri, 17 Jul 2015 00:55:36 +0000 Subject: [PATCH] Cosmetics --- .gitignore | 1 + app.db | Bin 57344 -> 59392 bytes app.py | 132 --------------------------------------- app/__init__.py | 10 ++- app/models.py | 2 +- app/templates/index.html | 13 +++- app/utils.py | 15 ++++- app/views.py | 10 ++- app/viz.py | 1 + config.py | 17 ++--- requirements.txt | 1 + 11 files changed, 55 insertions(+), 147 deletions(-) delete mode 100644 app.py diff --git a/.gitignore b/.gitignore index 7449a91d059b..57c945dd2bc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc +*.db tmp diff --git a/app.db b/app.db index 42d893593db1508ec4fe763c0977540cde107c31..16e9d6aa8fde4fa9ef0664766bf9449ae7d16be6 100644 GIT binary patch delta 1766 zcmaKsU1$_n6vywKt7c|3$*yg>iAI_1n$%RC%$=QY2}y|2M&gP!ejz%Y-JR&P*^lf; zV=O_Z2>Mbm9q2~Pfj}@z(YJsL(gd}Jp?ftPJ3PT882Ix<8 z9bNV!^tZ_UEk2KadNkx-Ul=C|X92p0E}^p|;WHF{1!4GN%gDXdD56$))2$0D6udd@ zoPyzoRs`akCI)jE>%ex~8uuPXBGmF$cneSr`iQzm9i-M%GI~huMT2M@{RAJuTORxJ z*D&0O;xO>XR_{?$I7V8z3(#Na0lMo&_+L=nd^8V026+$GE?(4t(0y`|Ku3S0$LKyu zxPg|d4aCDUOR8EbT`CwwtB^02OO4*o$;r(TnI)?8 znvOB2$f6`-Z{);@&gKxn0DXvFrOr`1;S+cYZUDc7_W_>jYeKu6o$lB-ErUd9K^IkB zR5&8OD&yeI1S5(NsB0KyJ8KPXPHx!c>% zW*SIRXWUs;eRJL>JvSIlB^>5ym9A3F@aO?13vVWNMU7fS@0*$zz{(#r{DY4aT zY{a%kGf6XRm0~)(J63h?nOq;O1JPyC_?9YUhW^?NF$ik=F}Co%jZHLYYinC}WWc?B zWq4J7T!^i%53w(xGj`TU74zdIt7wdvS$lNC-FtNa3htS!@x|t9Hr2_#%NDJ2rI=&Q zRVnw*)n1xk%TC7XLzC{3D;He;+6{_|eC$wgXv%rr+w;Hq!Y~gRi>X z=vVFY{J5K}8BWdkqGsHw8B5MTP2N;94%ZCteEafh9iN{s7+QCGVjcNb>yf3^y8i)p Cc?_ih delta 451 zcmZp;z}#?vd4e=+Dgy(9#Y6>r#?*}oOPD#Bn35Qnl9&}YHqK^bVyv0ixN~w6Tcwbx zf{}rhsfm?=v7V8Ip|Pbo^X9i~;*1H!Qm<)H2f4oc#HwvUEXWQAuWEj-#eRwL(#9Noi4@0!(PK#w`a9 aLj!AtS|wht$@;e+Z*25t+8ldtF(Uw&0*vAS diff --git a/app.py b/app.py deleted file mode 100644 index 1145b5dc4c69..000000000000 --- a/app.py +++ /dev/null @@ -1,132 +0,0 @@ -from pydruid import client -from pydruid.utils.filters import Dimension -from dateutil.parser import parse -from datetime import datetime, timedelta -from flask import Flask, render_template, request -from flask_bootstrap import Bootstrap -import json -from wtforms import Form, SelectMultipleField, SelectField, TextField -import pandas as pd -pd.set_option('display.max_colwidth', -1) - -ROW_LIMIT = 10000 -PORT = 8088 -query = client.PyDruid("http://10.181.47.80:8080", 'druid/v2') - -app = Flask(__name__) -Bootstrap(app) - -def latest_metadata(datasource): - max_time = query.time_boundary(datasource=datasource)[0]['result']['maxTime'] - max_time = parse(max_time) - intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' - intervals += max_time.isoformat() - return query.segment_metadata( - datasource=datasource, - intervals=intervals)[-1]['columns'] - -@app.route("/datasource//") -def datasource(datasource): - - metadata = latest_metadata(datasource) - grain = ['all', 'none', 'minute', 'hour', 'day'] - since_l = { - '1hour': timedelta(hours=1), - '1day': timedelta(days=1), - '7days': timedelta(days=7), - '28days': timedelta(days=28), - 'all': timedelta(days=365*100) - } - limits = [0, 5, 10, 25, 50, 100, 500] - limit = request.args.get("limit") - try: - limit = int(limit) - if limit not in limits: - limits.append(limit) - limits = sorted(limits) - except: - pass - class QueryForm(Form): - groupby = SelectMultipleField( - 'Group by', choices=[(m, m) for m in sorted(metadata.keys())]) - granularity = SelectField( - 'Granularity', choices=[(g, g) for g in grain]) - since = SelectField( - 'Since', choices=[(s, s) for s in since_l.keys()]) - limit = SelectField( - 'Limit', choices=[(s, s) for s in limits]) - flt_col_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in sorted(metadata.keys())]) - flt_op_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in ['==', 'in', '<', '>']]) - flt_eq_1 = TextField("Super") - - groupby = request.args.getlist("groupby") or [] - granularity = request.args.get("granularity") - limit = int(request.args.get("limit", ROW_LIMIT)) or ROW_LIMIT - since = request.args.get("since", "all") - from_dttm = (datetime.now() - since_l[since]).isoformat() - - # Building filters - i = 1 - filters = [] - while True: - col = request.args.get("flt_col_" + str(i)) - op = request.args.get("flt_op_" + str(i)) - eq = request.args.get("flt_eq_" + str(i)) - print (col,op,eq) - if col and op and eq: - filters.append(Dimension(col)==eq) - filters = Dimension(col)==eq - else: - break - i += 1 - print filters - - results=[] - results = query.groupby( - datasource=datasource, - granularity=granularity or 'all', - intervals=from_dttm + '/' + datetime.now().isoformat(), - dimensions=groupby, - aggregations={"count": client.doublesum("count")}, - filter=filters, - limit_spec={ - "type": "default", - "limit": limit, - "columns": [{ - "dimension" : "count", - "direction" : "descending", - },], - }, - ) - - df = query.export_pandas() - if df is not None and not df.empty: - df = df.sort(df.columns[0], ascending=False) - if granularity == 'all': - del df['timestamp'] - - table = df.to_html( - classes=["table", "table-striped", 'table-bordered'], index=False) - else: - table = None - - return render_template( - 'panoramix/datasource.html', - table=table, - datasource=datasource, - latest_metadata=json.dumps( - metadata, - sort_keys=True, - indent=2), - results=json.dumps( - results, - sort_keys=True, - indent=2), - form=QueryForm(request.args), - ) - -if __name__ == '__main__': - app.debug = True - app.run(host='0.0.0.0', port=PORT) diff --git a/app/__init__.py b/app/__init__.py index ccc0a50e31c3..8c8ef8829e96 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,6 @@ import logging from flask import Flask -from flask.ext.appbuilder import SQLA, AppBuilder +from flask.ext.appbuilder import SQLA, AppBuilder, IndexView """ Logging configuration @@ -12,8 +12,13 @@ app = Flask(__name__) app.config.from_object('config') db = SQLA(app) + +class MyIndexView(IndexView): + index_template = 'index.html' + appbuilder = AppBuilder( - app, db.session, base_template='panoramix/base.html') + app, db.session, base_template='panoramix/base.html', + indexview=MyIndexView) #appbuilder.app_name = 'Panoramix' @@ -31,4 +36,3 @@ def set_sqlite_pragma(dbapi_connection, connection_record): """ from app import views - diff --git a/app/models.py b/app/models.py index 39259323ac9c..81c51050e354 100644 --- a/app/models.py +++ b/app/models.py @@ -43,7 +43,7 @@ def latest_metadata(cls, name): print "---" * 100 print name print results - max_time = results[0]['result']['maxTime'] + max_time = results[0]['result']['minTime'] max_time = parse(max_time) intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' intervals += (max_time + timedelta(seconds=1)).isoformat() diff --git a/app/templates/index.html b/app/templates/index.html index 2970ab159201..626b725243ee 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,2 +1,13 @@ -{% extends "appbuilder/baselayout.html" %} +{% extends "appbuilder/base.html" %} +{% block content %} +
+
+

Panoramix

+

Panoramix is an interactive visualization platform built on top of Druid.io

+
+
+
+ +
+{% endblock %} diff --git a/app/utils.py b/app/utils.py index b92ebfe224ce..48e91e7709bb 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,5 +1,6 @@ import config -from datetime import timedelta +from datetime import timedelta, datetime +import parsedatetime since_l = { '1hour': timedelta(hours=1), @@ -16,3 +17,15 @@ def get_pydruid_client(): config.DRUID_BASE_ENDPOINT) +def parse_human_datetime(s): + """ + Use the parsedatetime lib to return ``datetime.datetime`` from human + generated strings + + >>> parse_human_datetime("now") <= datetime.now() + True + """ + cal = parsedatetime.Calendar() + d = cal.parse(s)[0] + return datetime( + d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec) diff --git a/app/views.py b/app/views.py index 90a3eaa80c3f..a24d3f828775 100644 --- a/app/views.py +++ b/app/views.py @@ -1,3 +1,6 @@ +from datetime import timedelta +import logging + from flask import request, redirect, flash from flask.ext.appbuilder.models.sqla.interface import SQLAInterface from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose @@ -5,7 +8,6 @@ import config from wtforms import Form, SelectMultipleField, SelectField, TextField from wtforms.fields import Field -from datetime import timedelta class OmgWtForm(Form): field_order = ( @@ -131,7 +133,11 @@ def refresh_datasources(self): ).format(**config.__dict__) datasources = json.loads(requests.get(endpoint).text) for datasource in datasources: - models.Datasource.sync_to_db(datasource) + try: + models.Datasource.sync_to_db(datasource) + except Exception as e: + logging.exception(e) + logging.error("Failed at syncing " + datasource) flash("Refreshed metadata from Druid!", 'info') return redirect("/datasourcemodelview/list/") diff --git a/app/viz.py b/app/viz.py index f1957aa6ee64..543c831b66da 100644 --- a/app/viz.py +++ b/app/viz.py @@ -200,6 +200,7 @@ class TimeSeriesBarViz(TimeSeriesViz): verbose_name = "Time Series - Bar Chart" chart_kind = "bar" + class TimeSeriesStackedBarViz(TimeSeriesViz): verbose_name = "Time Series - Stacked Bar Chart" chart_kind = "bar" diff --git a/config.py b/config.py index 333efa26c41a..65e596a920b5 100644 --- a/config.py +++ b/config.py @@ -76,14 +76,17 @@ BABEL_DEFAULT_FOLDER = 'translations' # The allowed translation for you app LANGUAGES = { - 'en': {'flag':'gb', 'name':'English'}, - 'pt': {'flag':'pt', 'name':'Portuguese'}, - 'pt_BR': {'flag':'br', 'name': 'Pt Brazil'}, - 'es': {'flag':'es', 'name':'Spanish'}, - 'de': {'flag':'de', 'name':'German'}, - 'zh': {'flag':'cn', 'name':'Chinese'}, - 'ru': {'flag':'ru', 'name':'Russian'} + 'en': {'flag':'us', 'name':'English'}, + 'fr': {'flag':'fr', 'name':'French'}, } +""" +'pt': {'flag':'pt', 'name':'Portuguese'}, +'pt_BR': {'flag':'br', 'name': 'Pt Brazil'}, +'es': {'flag':'es', 'name':'Spanish'}, +'de': {'flag':'de', 'name':'German'}, +'zh': {'flag':'cn', 'name':'Chinese'}, +'ru': {'flag':'ru', 'name':'Russian'} +""" #--------------------------------------------------- # Image and file configuration #--------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index a5717712317a..c7c6de1f0c0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pydruid +parsedatetime python-dateutil flask flask-appbuilder