Merge pull request #204 from flask-dashboard/development

Development

flask_monitoringdashboard/constants.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.0.6",
+	"version": "2.0.7",
 	"author": "Patrick Vogel, Thijs Klooster & Bogdan Petre",
 	"email": "patrickvogel@live.nl"
 }

flask_monitoringdashboard/core/profiler/outlierProfiler.py

@@ -27,7 +27,7 @@ def __init__(self, current_thread, endpoint):
         self._memory = ''
         self._stacktrace = ''
 
-        self._request = str(request.headers), str(request.environ), str(request.url)
+        self._request = str(request.headers), str(request.environ), request.url.encode('utf-8')
 
     def run(self):
         # sleep for average * ODC ms

flask_monitoringdashboard/core/profiler/stacktraceProfiler.py

@@ -18,6 +18,7 @@
 FILENAME = 'flask_monitoringdashboard/core/measurement.py'
 FILENAME_LEN = len(FILENAME)
 
+
 class StacktraceProfiler(threading.Thread):
     """
     Used for profiling the performance per line code.
@@ -102,7 +103,7 @@ def insert_lines_db(self, db_session, request_id):
             path, fun, line = key
             fn, ln = self._path_hash.get_last_fn_ln(path)
             indent = self._path_hash.get_indent(path)
-            duration = val * self._duration / self._total
+            duration = val * self._duration / self._total if self._total != 0 else 0
             add_stack_line(db_session, request_id, position=position, indent=indent, duration=duration,
                            code_line=(fn, ln, fun, line))
             position += 1

flask_monitoringdashboard/database/__init__.py

@@ -3,9 +3,11 @@
     For information about how to access the database via a session-variable, see: session_scope() 
 """
 import datetime
+import random
+import time
 from contextlib import contextmanager
 
-from sqlalchemy import Column, Integer, String, DateTime, create_engine, Float, Boolean, TEXT, ForeignKey
+from sqlalchemy import Column, Integer, String, DateTime, create_engine, Float, Boolean, TEXT, ForeignKey, exc
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship, scoped_session
 
@@ -177,9 +179,13 @@ def session_scope():
     try:
         yield session
         session.commit()
-    except Exception:
+    except exc.OperationalError:
         session.rollback()
-        raise
+        time.sleep(0.5 + random.random())
+        session.commit()
+    except Exception as e:
+        session.rollback()
+        print('No commit has been made, due to the following error: {}'.format(e))
     finally:
         session.close()
 

flask_monitoringdashboard/main.py

@@ -14,6 +14,8 @@ def create_app():
 
     dashboard.config.version = '3.1'
     dashboard.config.database_name = 'sqlite:///flask_monitoring_dashboard_v10.db'
+    # dashboard.config.database_name = 'postgresql://user:password@localhost:5432/db_name'
+    # dashboard.config.database_name = 'mysql+pymysql://user:password@localhost:3306/db_name'
     dashboard.bind(app)
 
     def f():

flask_monitoringdashboard/test/core/profiler/test_stacktrace_profiler.py

@@ -22,4 +22,4 @@ def test_after_run(self):
             ip = request.environ['REMOTE_ADDR']
             thread = StacktraceProfiler(current_thread, Endpoint(id=0, name=NAME), ip)
             thread._keeprunning = False
-            self.assertRaises(ValueError, thread.run)
+            thread.run()

migration/migrate_sqlite_mysql.py

@@ -0,0 +1,94 @@
+"""
+    Use this file for migrating your data from SQLite to MySQL. It only works for version >=2.0.0 of the Flask
+    Monitoring Dashboard. For migrating your data from version 1.x.x to 2.y.y, use the script "migrate_v1_to_v2.py".
+
+    Before running the script, make sure to change the SQLITE_URL and MYSQL_URL variables. Refer to
+    http://docs.sqlalchemy.org/en/latest/core/engines.html on how to configure them. Also, make sure MySQL Server
+    is running on your machine.
+"""
+import timeit
+
+from contextlib import contextmanager
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from flask_monitoringdashboard.database import Base, Endpoint, Request, Outlier, CodeLine, StackLine, Test, \
+    TestResult, TestEndpoint
+
+SQLITE_URL = 'sqlite:////Users/user/Flask-MonitoringDashboard/fmd.db'
+MYSQL_URL = 'mysql+pymysql://user:password@localhost:3306/database'
+ENTITIES = [Endpoint, Request, Outlier, CodeLine, StackLine, Test, TestResult, TestEndpoint]
+CHUNK_SIZE = 10000
+
+
+def create_db_session(db_url, clean=False):
+    engine = create_engine(db_url)
+    if clean:
+        Base.metadata.drop_all(engine)
+        Base.metadata.create_all(engine)
+    Base.metadata.bind = engine
+    return sessionmaker(bind=engine)
+
+
+@contextmanager
+def session_scope(db_session):
+    session = db_session()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+
+def get_row_count(db_session_old, entity_class):
+    with session_scope(db_session_old) as db_session:
+        count = db_session.query(entity_class).count()
+        db_session.expunge_all()
+    return count
+
+
+def get_old_instances(db_session_old, entity_class, start, end):
+    with session_scope(db_session_old) as db_session:
+        old_instances = db_session.query(entity_class).slice(start, end).all()
+        db_session.expunge_all()
+    return old_instances
+
+
+def migrate_batch(old_instances, db_session_new, entity_class):
+    new_instances = []
+    with session_scope(db_session_new) as db_session:
+        for (index, old_instance) in enumerate(old_instances):
+            # _sa_instance_state is a non-db value used internally by SQLAlchemy
+            del old_instance.__dict__['_sa_instance_state']
+            new_instances.append(entity_class(**old_instance.__dict__))
+        db_session.bulk_save_objects(new_instances)
+
+
+def migrate_all(db_session_old, db_session_new, entity_class, count):
+    chunks = count // CHUNK_SIZE
+    for i in range(chunks+1):
+        print("Moving batch %d of %d for entity %s..." % (i+1, chunks+1, entity_class.__name__))
+        old_instances = get_old_instances(db_session_old, entity_class, i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE)
+        migrate_batch(old_instances, db_session_new, entity_class)
+
+
+def main():
+    db_session_old = create_db_session(SQLITE_URL)
+    db_session_new = create_db_session(MYSQL_URL, clean=True)
+    t0 = timeit.default_timer()
+    t_previous = t0
+    for entity_class in ENTITIES:
+        count = get_row_count(db_session_old, entity_class)
+        migrate_all(db_session_old, db_session_new, entity_class, count)
+        t_now = timeit.default_timer()
+        print("Moving %s took %f seconds" % (entity_class.__name__, t_now - t_previous))
+        t_previous = t_now
+    print("Total time was %f seconds" % (t_previous - t0))
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,7 +1,7 @@
 flask>=0.9            # for monitoring the web-service
 sqlalchemy>=1.1.9     # for database support
 wtforms>=2.1          # for generating forms
-flask_wtf             # also for generating forms
+flask_wtf>=0.13       # also for generating forms
 plotly<3.0.0          # for generating graphs
 configparser          # for parsing the config-file
 psutil                # for logging extra CPU-info

setup.py

@@ -39,8 +39,6 @@ def get_description():
         'Programming Language :: Python',
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Framework :: Flask'],