From 5c7e18db9f975439e16db66b4165fb11ba039011 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 26 Jan 2018 13:33:17 +0100 Subject: [PATCH 01/65] Adds SFTPUser class to database wrapper. --- backend/db.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/db.py b/backend/db.py index 4f2718972..bcbaab9cf 100644 --- a/backend/db.py +++ b/backend/db.py @@ -252,6 +252,17 @@ class Meta: db_table = 'dataset_version_current' +class SFTPUser(BaseModel): + sftp_user = PrimaryKeyField(db_column='sftp_user_pk') + user = ForeignKeyField(db_column='user_pk', rel_model=User, to_field='user', related_name='sftp_user') + user_uid = IntegerField(unique=True) + user_name = CharField(null=False) + password_hash = CharField(null=False) + + class Meta: + db_table = 'sftp_user' + + def get_dataset(dataset): dataset = Dataset.select().where( Dataset.short_name == dataset).get() return dataset From 7ed4a16436055816467a3e26c72dc4a228292669 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Tue, 30 Jan 2018 14:19:34 +0100 Subject: [PATCH 02/65] Basic structure for sFTP credentials system, still needs expiration date support and backend improvements. --- backend/application.py | 74 +++++++++++++++++++ backend/route.py | 1 + .../js/controller.datasetAdminController.js | 28 ++++++- .../templates/ng-templates/dataset-admin.html | 27 +++++++ 4 files changed, 128 insertions(+), 2 deletions(-) diff --git a/backend/application.py b/backend/application.py index c2c3ae467..9d6e7dcef 100644 --- a/backend/application.py +++ b/backend/application.py @@ -6,6 +6,9 @@ import peewee import smtplib import tornado.web +import random +import string +import crypt import uuid import math @@ -477,3 +480,74 @@ def get(self, dataset): self.set_header("Content-Type", logo_entry.mimetype) self.write(logo_entry.data) self.finish() + + +class SFTPAccess(handlers.AdminHandler): + """ + Creates, or re-enables, sFTP users in the database. + """ + def get(self, dataset): + """ + Returns sFTP credentials for the current user. + """ + + password = None + username = None + expires = datetime.today() + timedelta(days=30) + # Check if an sFTP user exists for the current user + try: + data = self.current_user.sftp_user.get() + username = data.user_name + except db.SFTPUser.DoesNotExist: + # Otherwise return empty values + pass + + self.finish({'user':username, + 'expires':expires.strftime("%Y-%m-%d %H:%M"), + 'password':password}) + + def post(self, dataset): + """ + Handles generation of new credentials. This function either creates a + new set of sftp credentials for a user, or updates the old ones with a + new password and expiry date. + """ + + # Create a new password + username = "_".join(self.current_user.name.split()) + "_sftp" + password = self.generate_password() + expires = datetime.today() + timedelta(days=30) + + # Check if an sFTP user exists for the current user when the database is ready + try: + self.current_user.sftp_user.get() + # if we have a user, update it + db.SFTPUser.update(password_hash = crypt.crypt( password, username ) + ).where(db.SFTPUser.user == self.current_user.get_id()).execute() + # TODO: update expiry date to expires.timestamp() + except db.SFTPUser.DoesNotExist: + # if there is no user, insert the user in the database + db.SFTPUser.insert(user = self.current_user.get_id(), + user_uid = self.get_uid(), + user_name = username, + password_hash = crypt.crypt( password, username ) + ).execute() + # TODO: set expiry date to expires.timestamp() when the database is ready + + self.finish({'user':username, + 'expires':expires.strftime("%Y-%m-%d %H:%M"), + 'password':password}) + + def get_uid(self): + """ + Returns the uid of the current users sftp user. + """ + return self.current_user.get_id() + + def generate_password(self, size = 12): + """ + Generates a password of length 'size', comprised of random lowercase and + uppercase letters, and numbers. + """ + chars = string.ascii_letters + string.digits + return ''.join(random.SystemRandom().choice(chars) for _ in range(size)) diff --git a/backend/route.py b/backend/route.py index f550c0b92..58e930496 100644 --- a/backend/route.py +++ b/backend/route.py @@ -60,6 +60,7 @@ def __init__(self, settings): (r"/api/datasets", application.ListDatasets), (r"/api/datasets/(?P[^\/]+)", application.GetDataset), (r"/api/datasets/(?P[^\/]+)/log/(?P[^\/]+)/(?P[^\/]+)", application.LogEvent), + (r"/api/datasets/(?P[^\/]+)/sftp_access", application.SFTPAccess), (r"/api/datasets/(?P[^\/]+)/logo", application.ServeLogo), (r"/api/datasets/(?P[^\/]+)/files", application.DatasetFiles), (r"/api/datasets/(?P[^\/]+)/collection", application.Collection), diff --git a/frontend/src/js/controller.datasetAdminController.js b/frontend/src/js/controller.datasetAdminController.js index e0e8dd449..85298da52 100644 --- a/frontend/src/js/controller.datasetAdminController.js +++ b/frontend/src/js/controller.datasetAdminController.js @@ -1,10 +1,12 @@ (function() { angular.module("App") - .controller("datasetAdminController", ["$routeParams", "User", "Dataset", "DatasetUsers", - function($routeParams, User, Dataset, DatasetUsers) { + .controller("datasetAdminController", ["$routeParams", "$http", "$cookies", "User", "Dataset", "DatasetUsers", + function($routeParams, $http, $cookies, User, Dataset, DatasetUsers) { var localThis = this; localThis.revokeUser = revokeUser; localThis.approveUser = approveUser; + localThis.postSFTPForm = postSFTPForm; + localThis.sftp = {"user":"", "password":"", "expires":null}; activate(); @@ -24,6 +26,11 @@ } ); + $http.get("/api/datasets/" + $routeParams.dataset + "/sftp_access") + .then(function(data) { + return localThis.sftp = data.data; + }); + } function getUsers() { @@ -52,5 +59,22 @@ } ); } + + function postSFTPForm(valid) { + if (!valid) { + return; + } + $.ajax({ + url:"/api/datasets/" + $routeParams.dataset + "/sftp_access", + type:"POST", + data:{"_xsrf": $cookies.get("_xsrf")}, + contentType:"application/x-www-form-urlencoded", + success: function(data){ + $("#sftp-user").html(data.user); + $("#sftp-password").html(data.password); + $("#sftp-expires").html(data.expires); + } + }); + } }]); })(); diff --git a/frontend/templates/ng-templates/dataset-admin.html b/frontend/templates/ng-templates/dataset-admin.html index e33539af6..cf39bc14b 100644 --- a/frontend/templates/ng-templates/dataset-admin.html +++ b/frontend/templates/ng-templates/dataset-admin.html @@ -13,6 +13,7 @@ @@ -72,6 +73,32 @@ +
+
+
+ + + + + + + + + + + + + + + + + + +
sFTP username:{{ ctrl.sftp.user }}
sFTP password{{ ctrl.sftp.password }}
Expiry date:{{ ctrl.sftp.expires }}
+
+
+
+
From 5eed15a034a33cbda3d027127fc10f00f9afc950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=28Kusalananda=29=20K=C3=A4h=C3=A4ri?= Date: Tue, 30 Jan 2018 16:23:24 +0100 Subject: [PATCH 03/65] Add missing sftp_user.account_expires TIMESTAMP field --- sql/patch-master-db.sql | 1 + sql/swefreq.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/sql/patch-master-db.sql b/sql/patch-master-db.sql index c46cc4061..244c39542 100644 --- a/sql/patch-master-db.sql +++ b/sql/patch-master-db.sql @@ -46,6 +46,7 @@ CREATE TABLE IF NOT EXISTS sftp_user ( user_uid INTEGER NOT NULL, user_name VARCHAR(50) NOT NULL, password_hash VARCHAR(100) NOT NULL, + account_expires TIMESTAMP NOT NULL, CONSTRAINT FOREIGN KEY (user_pk) REFERENCES user(user_pk), CONSTRAINT UNIQUE (user_uid) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/sql/swefreq.sql b/sql/swefreq.sql index 4fa12802a..6b2f0363f 100644 --- a/sql/swefreq.sql +++ b/sql/swefreq.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS sftp_user ( user_uid INTEGER NOT NULL, user_name VARCHAR(50) NOT NULL, password_hash VARCHAR(100) NOT NULL, + account_expires TIMESTAMP NOT NULL, CONSTRAINT FOREIGN KEY (user_pk) REFERENCES user(user_pk), CONSTRAINT UNIQUE (user_uid) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; From 1ed8b245dae9022c9e74cedf3c44bd0565ba0afa Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Tue, 30 Jan 2018 16:46:42 +0100 Subject: [PATCH 04/65] Adds proper account expiry to sFTP system. --- backend/application.py | 11 +++++++---- backend/db.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/application.py b/backend/application.py index 9d6e7dcef..50d706b83 100644 --- a/backend/application.py +++ b/backend/application.py @@ -493,17 +493,18 @@ def get(self, dataset): password = None username = None - expires = datetime.today() + timedelta(days=30) + expires = None # Check if an sFTP user exists for the current user try: data = self.current_user.sftp_user.get() username = data.user_name + expires = data.expires.strftime("%Y-%m-%d %H:%M") except db.SFTPUser.DoesNotExist: # Otherwise return empty values pass self.finish({'user':username, - 'expires':expires.strftime("%Y-%m-%d %H:%M"), + 'expires':expires, 'password':password}) def post(self, dataset): @@ -522,7 +523,8 @@ def post(self, dataset): try: self.current_user.sftp_user.get() # if we have a user, update it - db.SFTPUser.update(password_hash = crypt.crypt( password, username ) + db.SFTPUser.update(password_hash = crypt.crypt( password, username ), + account_expires = expires ).where(db.SFTPUser.user == self.current_user.get_id()).execute() # TODO: update expiry date to expires.timestamp() except db.SFTPUser.DoesNotExist: @@ -530,7 +532,8 @@ def post(self, dataset): db.SFTPUser.insert(user = self.current_user.get_id(), user_uid = self.get_uid(), user_name = username, - password_hash = crypt.crypt( password, username ) + password_hash = crypt.crypt( password, username ), + account_expires = expires ).execute() # TODO: set expiry date to expires.timestamp() when the database is ready diff --git a/backend/db.py b/backend/db.py index bcbaab9cf..529847886 100644 --- a/backend/db.py +++ b/backend/db.py @@ -258,6 +258,7 @@ class SFTPUser(BaseModel): user_uid = IntegerField(unique=True) user_name = CharField(null=False) password_hash = CharField(null=False) + account_expires = DateTimeField(null=False) class Meta: db_table = 'sftp_user' From 18dce6a23997772897599d6c244081dc25769fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=28Kusalananda=29=20K=C3=A4h=C3=A4ri?= Date: Wed, 31 Jan 2018 08:59:24 +0100 Subject: [PATCH 05/65] Back out recent schema addition from this branch --- sql/patch-master-db.sql | 1 - sql/swefreq.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/sql/patch-master-db.sql b/sql/patch-master-db.sql index 244c39542..c46cc4061 100644 --- a/sql/patch-master-db.sql +++ b/sql/patch-master-db.sql @@ -46,7 +46,6 @@ CREATE TABLE IF NOT EXISTS sftp_user ( user_uid INTEGER NOT NULL, user_name VARCHAR(50) NOT NULL, password_hash VARCHAR(100) NOT NULL, - account_expires TIMESTAMP NOT NULL, CONSTRAINT FOREIGN KEY (user_pk) REFERENCES user(user_pk), CONSTRAINT UNIQUE (user_uid) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/sql/swefreq.sql b/sql/swefreq.sql index 6b2f0363f..4fa12802a 100644 --- a/sql/swefreq.sql +++ b/sql/swefreq.sql @@ -22,7 +22,6 @@ CREATE TABLE IF NOT EXISTS sftp_user ( user_uid INTEGER NOT NULL, user_name VARCHAR(50) NOT NULL, password_hash VARCHAR(100) NOT NULL, - account_expires TIMESTAMP NOT NULL, CONSTRAINT FOREIGN KEY (user_pk) REFERENCES user(user_pk), CONSTRAINT UNIQUE (user_uid) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; From f8445feb92d33872b05c64a75b02f399b75bcc03 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Feb 2018 13:58:38 +0100 Subject: [PATCH 06/65] Fixes most of the requested changes. --- backend/application.py | 6 +-- .../js/controller.datasetAdminController.js | 30 ++++---------- frontend/src/js/factory.sFTPAccess.js | 30 ++++++++++++++ .../templates/ng-templates/dataset-admin.html | 39 +++++++++---------- 4 files changed, 58 insertions(+), 47 deletions(-) create mode 100644 frontend/src/js/factory.sFTPAccess.js diff --git a/backend/application.py b/backend/application.py index 50d706b83..7d634aa7c 100644 --- a/backend/application.py +++ b/backend/application.py @@ -498,7 +498,7 @@ def get(self, dataset): try: data = self.current_user.sftp_user.get() username = data.user_name - expires = data.expires.strftime("%Y-%m-%d %H:%M") + expires = data.account_expires.strftime("%Y-%m-%d %H:%M") except db.SFTPUser.DoesNotExist: # Otherwise return empty values pass @@ -525,11 +525,11 @@ def post(self, dataset): # if we have a user, update it db.SFTPUser.update(password_hash = crypt.crypt( password, username ), account_expires = expires - ).where(db.SFTPUser.user == self.current_user.get_id()).execute() + ).where(db.SFTPUser.user == self.current_user).execute() # TODO: update expiry date to expires.timestamp() except db.SFTPUser.DoesNotExist: # if there is no user, insert the user in the database - db.SFTPUser.insert(user = self.current_user.get_id(), + db.SFTPUser.insert(user = self.current_user, user_uid = self.get_uid(), user_name = username, password_hash = crypt.crypt( password, username ), diff --git a/frontend/src/js/controller.datasetAdminController.js b/frontend/src/js/controller.datasetAdminController.js index 85298da52..199bbf84e 100644 --- a/frontend/src/js/controller.datasetAdminController.js +++ b/frontend/src/js/controller.datasetAdminController.js @@ -1,12 +1,12 @@ (function() { angular.module("App") - .controller("datasetAdminController", ["$routeParams", "$http", "$cookies", "User", "Dataset", "DatasetUsers", - function($routeParams, $http, $cookies, User, Dataset, DatasetUsers) { + .controller("datasetAdminController", ["$routeParams", "User", "Dataset", "DatasetUsers", "SFTPAccess", + function($routeParams, User, Dataset, DatasetUsers, SFTPAccess) { var localThis = this; localThis.revokeUser = revokeUser; localThis.approveUser = approveUser; - localThis.postSFTPForm = postSFTPForm; localThis.sftp = {"user":"", "password":"", "expires":null}; + localThis.createSFTPCredentials = function() { return SFTPAccess.createCredentials( $routeParams.dataset ); } activate(); @@ -26,10 +26,10 @@ } ); - $http.get("/api/datasets/" + $routeParams.dataset + "/sftp_access") - .then(function(data) { - return localThis.sftp = data.data; - }); + SFTPAccess.getCredentials( $routeParams.dataset ) + .then( function(data) { + localThis.sftp = data; + }); } @@ -60,21 +60,5 @@ ); } - function postSFTPForm(valid) { - if (!valid) { - return; - } - $.ajax({ - url:"/api/datasets/" + $routeParams.dataset + "/sftp_access", - type:"POST", - data:{"_xsrf": $cookies.get("_xsrf")}, - contentType:"application/x-www-form-urlencoded", - success: function(data){ - $("#sftp-user").html(data.user); - $("#sftp-password").html(data.password); - $("#sftp-expires").html(data.expires); - } - }); - } }]); })(); diff --git a/frontend/src/js/factory.sFTPAccess.js b/frontend/src/js/factory.sFTPAccess.js new file mode 100644 index 000000000..12d1980c6 --- /dev/null +++ b/frontend/src/js/factory.sFTPAccess.js @@ -0,0 +1,30 @@ +(function() { + angular.module("App") + .factory("SFTPAccess", ["$http", "$cookies", function($http, $cookies) { + return { + getCredentials: getCredentials, + createCredentials: createCredentials, + }; + + function getCredentials( dataset ) { + return $http.get("/api/datasets/" + dataset + "/sftp_access") + .then(function(data) { + return data.data; + }); + } + + function createCredentials( dataset ) { + $.ajax({ + url:"/api/datasets/" + dataset + "/sftp_access", + type:"POST", + data:{"_xsrf": $cookies.get("_xsrf")}, + contentType:"application/x-www-form-urlencoded", + success: function(data){ + $("#sftp-user").html(data.user); + $("#sftp-password").html(data.password); + $("#sftp-expires").html(data.expires); + } + }); + } + }]); +})(); diff --git a/frontend/templates/ng-templates/dataset-admin.html b/frontend/templates/ng-templates/dataset-admin.html index cf39bc14b..dcfd67f5d 100644 --- a/frontend/templates/ng-templates/dataset-admin.html +++ b/frontend/templates/ng-templates/dataset-admin.html @@ -75,27 +75,24 @@
-
- - - - - - - - - - - - - - - - - - -
sFTP username:{{ ctrl.sftp.user }}
sFTP password{{ ctrl.sftp.password }}
Expiry date:{{ ctrl.sftp.expires }}
-
+ + + + + + + + + + + + + + + + + +
sFTP username:{{ ctrl.sftp.user }}
sFTP password{{ ctrl.sftp.password }}
Expiry date:{{ ctrl.sftp.expires }}
From db4281abfc64cb50c696af8b1bb155ac635c4892 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 08:41:45 +0100 Subject: [PATCH 07/65] Changes sFTP password hash to SHA-256. --- backend/application.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/application.py b/backend/application.py index 7d634aa7c..ca6064e52 100644 --- a/backend/application.py +++ b/backend/application.py @@ -3,12 +3,12 @@ from os import path import logging from datetime import datetime, timedelta +from peewee import fn import peewee import smtplib import tornado.web import random import string -import crypt import uuid import math @@ -523,7 +523,7 @@ def post(self, dataset): try: self.current_user.sftp_user.get() # if we have a user, update it - db.SFTPUser.update(password_hash = crypt.crypt( password, username ), + db.SFTPUser.update(password_hash = fn.SHA2(password, 256), account_expires = expires ).where(db.SFTPUser.user == self.current_user).execute() # TODO: update expiry date to expires.timestamp() @@ -532,7 +532,7 @@ def post(self, dataset): db.SFTPUser.insert(user = self.current_user, user_uid = self.get_uid(), user_name = username, - password_hash = crypt.crypt( password, username ), + password_hash = fn.SHA2(password, 256), account_expires = expires ).execute() # TODO: set expiry date to expires.timestamp() when the database is ready From 136dd521571def3994f70c545206ce39484b844e Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 09:28:02 +0100 Subject: [PATCH 08/65] Changes the UID generation. --- backend/application.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/application.py b/backend/application.py index ca6064e52..e6b9f68d3 100644 --- a/backend/application.py +++ b/backend/application.py @@ -530,7 +530,7 @@ def post(self, dataset): except db.SFTPUser.DoesNotExist: # if there is no user, insert the user in the database db.SFTPUser.insert(user = self.current_user, - user_uid = self.get_uid(), + user_uid = self.get_next_free_uid(), user_name = username, password_hash = fn.SHA2(password, 256), account_expires = expires @@ -541,11 +541,21 @@ def post(self, dataset): 'expires':expires.strftime("%Y-%m-%d %H:%M"), 'password':password}) - def get_uid(self): + def get_next_free_uid(self): """ - Returns the uid of the current users sftp user. + Returns the next free uid >= 10000, and higher than the current uid's + from the sftp_user table in the database. """ - return self.current_user.get_id() + default = 10000 + next_uid = default + try: + current_max_uid = db.SFTPUser.select(fn.MAX(db.SFTPUser.user_uid)).get().user_uid + if current_max_uid: + next_uid = current_max_uid+1 + except db.SFTPUser.DoesNotExist: + pass + + return next_uid def generate_password(self, size = 12): """ From 7588605f775d5580044b0c9fdccb5acb12cd13ea Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 09:49:14 +0100 Subject: [PATCH 09/65] Removes completed TODOs --- backend/application.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/application.py b/backend/application.py index e6b9f68d3..58a731131 100644 --- a/backend/application.py +++ b/backend/application.py @@ -526,7 +526,6 @@ def post(self, dataset): db.SFTPUser.update(password_hash = fn.SHA2(password, 256), account_expires = expires ).where(db.SFTPUser.user == self.current_user).execute() - # TODO: update expiry date to expires.timestamp() except db.SFTPUser.DoesNotExist: # if there is no user, insert the user in the database db.SFTPUser.insert(user = self.current_user, @@ -535,7 +534,6 @@ def post(self, dataset): password_hash = fn.SHA2(password, 256), account_expires = expires ).execute() - # TODO: set expiry date to expires.timestamp() when the database is ready self.finish({'user':username, 'expires':expires.strftime("%Y-%m-%d %H:%M"), From 8cc8e37d169fca00bbb036f01826708a34e4e7f9 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 10:01:15 +0100 Subject: [PATCH 10/65] Moves get_next_free_uid function from application to db. --- backend/application.py | 18 +----------------- backend/db.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/backend/application.py b/backend/application.py index 58a731131..d054d6410 100644 --- a/backend/application.py +++ b/backend/application.py @@ -529,7 +529,7 @@ def post(self, dataset): except db.SFTPUser.DoesNotExist: # if there is no user, insert the user in the database db.SFTPUser.insert(user = self.current_user, - user_uid = self.get_next_free_uid(), + user_uid = db.get_next_free_uid(), user_name = username, password_hash = fn.SHA2(password, 256), account_expires = expires @@ -539,22 +539,6 @@ def post(self, dataset): 'expires':expires.strftime("%Y-%m-%d %H:%M"), 'password':password}) - def get_next_free_uid(self): - """ - Returns the next free uid >= 10000, and higher than the current uid's - from the sftp_user table in the database. - """ - default = 10000 - next_uid = default - try: - current_max_uid = db.SFTPUser.select(fn.MAX(db.SFTPUser.user_uid)).get().user_uid - if current_max_uid: - next_uid = current_max_uid+1 - except db.SFTPUser.DoesNotExist: - pass - - return next_uid - def generate_password(self, size = 12): """ Generates a password of length 'size', comprised of random lowercase and diff --git a/backend/db.py b/backend/db.py index 529847886..41b604a4c 100644 --- a/backend/db.py +++ b/backend/db.py @@ -10,6 +10,7 @@ MySQLDatabase, PrimaryKeyField, TextField, + fn, ) import settings @@ -264,6 +265,23 @@ class Meta: db_table = 'sftp_user' +def get_next_free_uid(): + """ + Returns the next free uid >= 10000, and higher than the current uid's + from the sftp_user table in the database. + """ + default = 10000 + next_uid = default + try: + current_max_uid = SFTPUser.select(fn.MAX(SFTPUser.user_uid)).get().user_uid + if current_max_uid: + next_uid = current_max_uid+1 + except SFTPUser.DoesNotExist: + pass + + return next_uid + + def get_dataset(dataset): dataset = Dataset.select().where( Dataset.short_name == dataset).get() return dataset From 6f20e0bf2b4d7fc309056722bdbdf29d59f61674 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 10:32:03 +0100 Subject: [PATCH 11/65] Updates javascript to be more angular --- .../js/controller.datasetAdminController.js | 7 ++++++- frontend/src/js/factory.sFTPAccess.js | 19 ++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/frontend/src/js/controller.datasetAdminController.js b/frontend/src/js/controller.datasetAdminController.js index 199bbf84e..c20c28e43 100644 --- a/frontend/src/js/controller.datasetAdminController.js +++ b/frontend/src/js/controller.datasetAdminController.js @@ -6,7 +6,12 @@ localThis.revokeUser = revokeUser; localThis.approveUser = approveUser; localThis.sftp = {"user":"", "password":"", "expires":null}; - localThis.createSFTPCredentials = function() { return SFTPAccess.createCredentials( $routeParams.dataset ); } + localThis.createSFTPCredentials = function() { + SFTPAccess.createCredentials( $routeParams.dataset ) + .then( function(data) { + localThis.sftp = data; + }); + } activate(); diff --git a/frontend/src/js/factory.sFTPAccess.js b/frontend/src/js/factory.sFTPAccess.js index 12d1980c6..249daad5e 100644 --- a/frontend/src/js/factory.sFTPAccess.js +++ b/frontend/src/js/factory.sFTPAccess.js @@ -14,17 +14,14 @@ } function createCredentials( dataset ) { - $.ajax({ - url:"/api/datasets/" + dataset + "/sftp_access", - type:"POST", - data:{"_xsrf": $cookies.get("_xsrf")}, - contentType:"application/x-www-form-urlencoded", - success: function(data){ - $("#sftp-user").html(data.user); - $("#sftp-password").html(data.password); - $("#sftp-expires").html(data.expires); - } - }); + return $http.post("/api/datasets/" + dataset + "/sftp_access", + $.param({"_xsrf": $cookies.get("_xsrf")}), + {headers : { + "Content-Type": "application/x-www-form-urlencoded;" + }}) + .then( function(data) { + return data.data; + }); } }]); })(); From dcf7a97a24be857361ecd6903cae32286c4285ed Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 2 Feb 2018 10:44:53 +0100 Subject: [PATCH 12/65] Adds missing semi-colon --- frontend/src/js/controller.datasetAdminController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/controller.datasetAdminController.js b/frontend/src/js/controller.datasetAdminController.js index c20c28e43..21324ce94 100644 --- a/frontend/src/js/controller.datasetAdminController.js +++ b/frontend/src/js/controller.datasetAdminController.js @@ -11,7 +11,7 @@ .then( function(data) { localThis.sftp = data; }); - } + }; activate(); From afdc553125bbafe3016ba71d3afbd9c792d019d1 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 5 Feb 2018 16:42:43 +0100 Subject: [PATCH 13/65] Updated layout of sFTP credential generation --- .../templates/ng-templates/dataset-admin.html | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/frontend/templates/ng-templates/dataset-admin.html b/frontend/templates/ng-templates/dataset-admin.html index dcfd67f5d..628828618 100644 --- a/frontend/templates/ng-templates/dataset-admin.html +++ b/frontend/templates/ng-templates/dataset-admin.html @@ -74,25 +74,36 @@
-
- - - - - - - - - - - - - - - - - -
sFTP username:{{ ctrl.sftp.user }}
sFTP password{{ ctrl.sftp.password }}
Expiry date:{{ ctrl.sftp.expires }}
+
+

sFTP Credentials

+
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+
+ +
+
+
From 048dee77d563de5f6b7e2aa82aa1a444b4253b27 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 12 Feb 2018 11:09:10 +0100 Subject: [PATCH 14/65] Empty patch master db --- sql/patch-master-db.sql | 73 ----------------------------------------- 1 file changed, 73 deletions(-) diff --git a/sql/patch-master-db.sql b/sql/patch-master-db.sql index 244c39542..a74587a7a 100644 --- a/sql/patch-master-db.sql +++ b/sql/patch-master-db.sql @@ -1,75 +1,2 @@ -- Patches a database that is using the master checkout of the -- swefreq.sql schema definition to the develop version. - --- Add some unique constraints -ALTER TABLE study ADD - CONSTRAINT UNIQUE (pi_email, title); -ALTER TABLE dataset_version ADD - CONSTRAINT UNIQUE (dataset_pk, version); -ALTER TABLE collection ADD - CONSTRAINT UNIQUE (name); -ALTER TABLE sample_set ADD - CONSTRAINT UNIQUE (dataset_pk, collection_pk); -ALTER TABLE dataset_file ADD - CONSTRAINT UNIQUE (uri); - --- Add user.identity and user.identity_type - -ALTER TABLE user - ADD COLUMN ( - identity VARCHAR(100), - identity_type ENUM ('google', 'elixir')), - ADD CONSTRAINT - UNIQUE (identity, identity_type); - --- Fill the above with existing data - -UPDATE user -SET identity_type = 'google'; - -UPDATE user -SET identity = email; - --- Set those two columns as "NOT NULL" -ALTER TABLE user -MODIFY COLUMN - identity VARCHAR(100) NOT NULL, -MODIFY COLUMN - identity_type ENUM ('google', 'elixir') - NOT NULL; - --- Add sftp_user and associated triggers - -CREATE TABLE IF NOT EXISTS sftp_user ( - sftp_user_pk INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT, - user_pk INTEGER NOT NULL, - user_uid INTEGER NOT NULL, - user_name VARCHAR(50) NOT NULL, - password_hash VARCHAR(100) NOT NULL, - account_expires TIMESTAMP NOT NULL, - CONSTRAINT FOREIGN KEY (user_pk) REFERENCES user(user_pk), - CONSTRAINT UNIQUE (user_uid) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - --- Triggers on INSERT and UPDATE of sftp_user.user_uid to make sure the --- new value is always 10000 or larger. - -DELIMITER $$ - -CREATE TRIGGER check_before_insert BEFORE INSERT ON sftp_user -FOR EACH ROW -IF NEW.user_uid < 10000 THEN - SIGNAL SQLSTATE '38001' - SET MESSAGE_TEXT = 'Check constraint failed on sftp_user.user_uid insert'; -END IF$$ - -CREATE TRIGGER check_before_update BEFORE UPDATE ON sftp_user -FOR EACH ROW -IF NEW.user_uid < 10000 THEN - SIGNAL SQLSTATE '38002' - SET MESSAGE_TEXT = 'Check constraint failed on sftp_user.user_uid update'; -END IF$$ - -DELIMITER ; - --- End of triggers From 1783875e4e2f9c2a925cf877b033dff18e1ae904 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 26 Jan 2018 15:04:11 +0100 Subject: [PATCH 15/65] Explicitly use python3 when rebuilding templates --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 91d0ea40b..60abdb5a5 100644 --- a/Makefile +++ b/Makefile @@ -44,4 +44,4 @@ static/js/app.js: $(JAVASCRIPT_FILES) static/templates/%.html: frontend/templates/%.html mkdir -p $$( dirname $@ ) 2>/dev/null || true - python scripts/compile_template.py ${COMPILE_TEMPLATE_OPTS} -b frontend/templates -s $< >$@ + python3 scripts/compile_template.py ${COMPILE_TEMPLATE_OPTS} -b frontend/templates -s $< >$@ From 276dec82857e2534f6c343aad0ac7f7d24487507 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 26 Jan 2018 16:12:14 +0100 Subject: [PATCH 16/65] Dockerfiles and compose file, WIP --- Dockerfile-backend | 13 +++++++++++++ Dockerfile-frontend-rebuilder | 18 ++++++++++++++++++ docker-compose.yml | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 Dockerfile-backend create mode 100644 Dockerfile-frontend-rebuilder create mode 100644 docker-compose.yml diff --git a/Dockerfile-backend b/Dockerfile-backend new file mode 100644 index 000000000..c2f9361ba --- /dev/null +++ b/Dockerfile-backend @@ -0,0 +1,13 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + libmysqlclient-dev + +ADD . /code +WORKDIR /code/backend + +RUN pip3 install -r requirements.txt + +CMD ["python3", "route.py", "--develop"] diff --git a/Dockerfile-frontend-rebuilder b/Dockerfile-frontend-rebuilder new file mode 100644 index 000000000..bdd473967 --- /dev/null +++ b/Dockerfile-frontend-rebuilder @@ -0,0 +1,18 @@ +FROM node:6.12 + +RUN apt-get update && \ + apt-get install -y \ + rsync \ + python3 \ + python3-pip \ + python3-pyinotify \ + inotify-tools \ + libmysqlclient-dev + +ADD . /code +WORKDIR /code + +RUN pip3 install --upgrade pip && \ + pip3 install -r backend/requirements.txt && \ + pip3 install inotify +CMD ["python3", "scripts/watch_frontend.py"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..1a861d0f3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +version: "3.2" +services: + web: + build: + context: ./ + dockerfile: Dockerfile-backend + depends_on: + - db + ports: + - 4000:4000 + restart: on-failure + volumes: + - type: bind + source: . + target: /code + db: + image: "mysql:5.7" + volumes: + - type: volume + source: mysql-data-volume + target: /var/lib/mysql/ + rebuilder: + build: + context: ./ + dockerfile: Dockerfile-frontend-rebuilder + volumes: + - type: bind + source: . + target: /code + + +volumes: + mysql-data-volume: + external: true From 9d7f319a82e094b8f25e310226ba0c7c55dd4ebb Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Sat, 27 Jan 2018 08:24:46 +0100 Subject: [PATCH 17/65] Script to watch files for changes and rerun --- scripts/watch_frontend.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 scripts/watch_frontend.py diff --git a/scripts/watch_frontend.py b/scripts/watch_frontend.py new file mode 100644 index 000000000..946bbc369 --- /dev/null +++ b/scripts/watch_frontend.py @@ -0,0 +1,31 @@ +import inotify.adapters +import re +import subprocess + +def files_to_watch(): + r = re.compile("^/code/frontend/(templates|src)") + return lambda x: r.match(x[2]) + +def events_to_watch(): + skip_events = ['IN_ISDIR', 'IN_OPEN', 'IN_ACCESS', 'IN_CLOSE_NOWRITE'] + return lambda x: all(event not in x[1] for event in skip_events) + +def comb(*args): + return lambda x: all(f(x) for f in args) + +def main(): + i = inotify.adapters.InotifyTree("/code/frontend") + grepper = comb(files_to_watch(), events_to_watch()) + + while True: + changes = list( filter(grepper, i.event_gen(timeout_s=0.5, yield_nones=False)) ) + if changes: + print("Files updated rerunning") + for c in changes: + (first, type_names, path, filename) = c + print(" PATH=[{}] FILENAME=[{}] EVENT_TYPES={}".format(path, filename, type_names)) + subprocess.call('make') + + +if __name__ == '__main__': + main() From 5fb0a705a88db033d9d0c148a443fc8c09163135 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 2 Feb 2018 14:11:51 +0100 Subject: [PATCH 18/65] Renamed scripts related to docker volume creation --- .../create_docker_db_volume_tarball.sh | 0 .../download_and_create_docker_db_volume.sh | 2 +- test/travis_before_install.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename test/docker-db/setup_database.sh => scripts/create_docker_db_volume_tarball.sh (100%) rename test/docker-db/load_database_in_volume.sh => scripts/download_and_create_docker_db_volume.sh (99%) diff --git a/test/docker-db/setup_database.sh b/scripts/create_docker_db_volume_tarball.sh similarity index 100% rename from test/docker-db/setup_database.sh rename to scripts/create_docker_db_volume_tarball.sh diff --git a/test/docker-db/load_database_in_volume.sh b/scripts/download_and_create_docker_db_volume.sh similarity index 99% rename from test/docker-db/load_database_in_volume.sh rename to scripts/download_and_create_docker_db_volume.sh index fb9bf51a0..e7370748f 100755 --- a/test/docker-db/load_database_in_volume.sh +++ b/scripts/download_and_create_docker_db_volume.sh @@ -8,7 +8,7 @@ VOLUME='mysql-data-volume' echo "Downloading data" -curl -O https://swefreq.nbis.se/static/testing/mysql-data.tar.bz2 +curl -O https://swefreq.nbis.se/static/testing/mysql-data.tar.bz2 tar xjf mysql-data.tar.bz2 echo "Creating datavolume and filling it with data" diff --git a/test/travis_before_install.sh b/test/travis_before_install.sh index 6c44a9df0..3e5190a9e 100755 --- a/test/travis_before_install.sh +++ b/test/travis_before_install.sh @@ -7,5 +7,5 @@ docker pull ubuntu:16.04 VOLUME='mysql-data-volume' MYSQL_PORT=3366 -test/docker-db/load_database_in_volume.sh +scripts/download_and_create_docker_db_volume.sh docker run -v $VOLUME:/var/lib/mysql --rm --name mysql -d -p $MYSQL_PORT:3306 mysql:5.7 From d4338150ca318e63c9315d5a52c30821bd1f468a Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 2 Feb 2018 16:31:13 +0100 Subject: [PATCH 19/65] Script to load mysql dummy data into docker container --- scripts/load_mysql_dummy_data.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 scripts/load_mysql_dummy_data.sh diff --git a/scripts/load_mysql_dummy_data.sh b/scripts/load_mysql_dummy_data.sh new file mode 100755 index 000000000..4e939f00a --- /dev/null +++ b/scripts/load_mysql_dummy_data.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +docker run -v mysql-data-volume:/var/lib/mysql --rm --name mysql -d mysql:5.7 + +docker run -i --rm --link mysql:mysql mysql:5.7 mysql -h mysql -u swefreq swefreq_test < sql/swefreq.sql +docker run -i --rm --link mysql:mysql mysql:5.7 mysql -h mysql -u swefreq swefreq_test < test/data/load_dummy_data.sql + +docker container stop mysql From 9838b84e9bc103449becfc3d14b6ba2a67c71a5f Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 2 Feb 2018 16:31:34 +0100 Subject: [PATCH 20/65] Updated development mode instructions --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e204bb52c..8a35000ac 100644 --- a/README.md +++ b/README.md @@ -55,19 +55,17 @@ The application has only been tested with python 3.5.2. It will most likely work Quick development mode ---------------------- -1. Install docker -2. Look at `test/travis_before_install.sh` to initiate the mysql docker image. +1. Install docker (and docker-compose in case it's not included in the installation) +2. Create test database + 2.1. Initiate a mysql data volume by running `./scripts/download_and_create_docker_db_volume.sh` + 2.2. Load mysql dummy data by running `./scripts/load_mysql_dummy_data.sh` 3. Copy `settings_sample.json` into `settings.json` and - Change mysqlSchema into `swefreq_test`. - - Change mysqlPort to 3366 - Update the credentials for elixir and google oauth. - Elixir/redirectUri: http://localhost:4000/elixir/login - redirectUri: http://localhost:4000/login -4. Run "Test 2. Load the swefreq schema" from `test/travis_script.sh`. -5. Run `make` in the root directory of the project. -6. Make a symbolic link from `backend/static` to `static`. -7. Run the server: +4. Make a symbolic link from `backend/static` to `static`. +5. Run the server: ```bash -$ cd backend -$ python route.py --develop +$ docker-compose up ``` From 358869d03865795b2f70213934c7428af8f0343b Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 5 Feb 2018 16:55:57 +0100 Subject: [PATCH 21/65] Working frontend rebuilder, didnt always use python3 before --- Dockerfile-frontend-rebuilder | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile-frontend-rebuilder b/Dockerfile-frontend-rebuilder index bdd473967..270f94946 100644 --- a/Dockerfile-frontend-rebuilder +++ b/Dockerfile-frontend-rebuilder @@ -1,13 +1,18 @@ -FROM node:6.12 +FROM ubuntu:16.04 RUN apt-get update && \ apt-get install -y \ + curl \ rsync \ python3 \ python3-pip \ python3-pyinotify \ inotify-tools \ - libmysqlclient-dev + libmysqlclient-dev && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 5 + +RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ + apt-get install -y nodejs ADD . /code WORKDIR /code @@ -15,4 +20,5 @@ WORKDIR /code RUN pip3 install --upgrade pip && \ pip3 install -r backend/requirements.txt && \ pip3 install inotify -CMD ["python3", "scripts/watch_frontend.py"] + +CMD ["python", "scripts/watch_frontend.py"] From 21a9aeb9fb734bf4868bd59969b027d84c6a8f7e Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 9 Feb 2018 15:51:10 +0100 Subject: [PATCH 22/65] Added instructions to specify mysql db host in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8a35000ac..04b8d704e 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Quick development mode - Update the credentials for elixir and google oauth. - Elixir/redirectUri: http://localhost:4000/elixir/login - redirectUri: http://localhost:4000/login + - Set `mysqlHost` to `db` 4. Make a symbolic link from `backend/static` to `static`. 5. Run the server: ```bash From 35c53e2283826b4f2924111534aff306a4c43e48 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 9 Feb 2018 15:51:51 +0100 Subject: [PATCH 23/65] Explicitly install node deps and call make clean and make in frontend rebuilder --- scripts/watch_frontend.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/watch_frontend.py b/scripts/watch_frontend.py index 946bbc369..7846360fc 100644 --- a/scripts/watch_frontend.py +++ b/scripts/watch_frontend.py @@ -1,6 +1,7 @@ import inotify.adapters import re import subprocess +import os def files_to_watch(): r = re.compile("^/code/frontend/(templates|src)") @@ -13,7 +14,17 @@ def events_to_watch(): def comb(*args): return lambda x: all(f(x) for f in args) +def install_node_deps(): + os.chdir('frontend') + subprocess.call(['npm','install']) + os.chdir('..') + def main(): + install_node_deps() + + subprocess.call(['make', 'clean']) + subprocess.call('make') + i = inotify.adapters.InotifyTree("/code/frontend") grepper = comb(files_to_watch(), events_to_watch()) From b945f958b4fd356d161c35de6cc27f63ee8d5a0a Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 12 Feb 2018 10:59:25 +0100 Subject: [PATCH 24/65] Use the mysql client from the docker container instead --- scripts/create_docker_db_volume_tarball.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/create_docker_db_volume_tarball.sh b/scripts/create_docker_db_volume_tarball.sh index 1183f6ded..e45853098 100755 --- a/scripts/create_docker_db_volume_tarball.sh +++ b/scripts/create_docker_db_volume_tarball.sh @@ -1,6 +1,5 @@ #!/bin/bash -MYSQL_PORT=3366 MYSQL_HOST=127.0.0.1 MYSQL_PASS=rootpw @@ -12,11 +11,11 @@ fi echo "Creating data volume and starting mysql:5.7 docker container" docker volume create swefreq-mysql-data > /dev/null -docker run -v swefreq-mysql-data:/var/lib/mysql --rm --name mysql -e MYSQL_ROOT_PASSWORD=$MYSQL_PASS -d -p $MYSQL_PORT:3306 mysql:5.7 >/dev/null +docker run -v swefreq-mysql-data:/var/lib/mysql --rm --name mysql -e MYSQL_ROOT_PASSWORD=$MYSQL_PASS -d mysql:5.7 >/dev/null # Wait for it to start -while ! mysql -uroot -p$MYSQL_PASS -h127.0.0.1 -P$MYSQL_PORT -e exit 2>/dev/null; do +while ! docker run -i --rm --link mysql:mysql mysql:5.7 mysql -h mysql -uroot -p$MYSQL_PASS -e exit 2>/dev/null; do echo -ne "\rWaiting for mysql to initialize..."; sleep 1; done @@ -24,10 +23,11 @@ echo " DONE"; echo "Creating swefreq user and swefreq_test database" -mysql -u root -P$MYSQL_PORT -h$MYSQL_HOST -p$MYSQL_PASS <<__END__ +docker run -i --rm --link mysql:mysql mysql:5.7 mysql -h mysql -uroot -p$MYSQL_PASS <<__END__ CREATE USER IF NOT EXISTS 'swefreq'; CREATE DATABASE IF NOT EXISTS swefreq_test; GRANT ALL ON swefreq_test.* TO 'swefreq'@'%'; +SET GLOBAL event_scheduler = ON; __END__ From b1524834be73b9a618adeaf35fedb360362d13b0 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 12 Feb 2018 12:21:20 +0100 Subject: [PATCH 25/65] Unbreak tests --- backend/test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/backend/test.py b/backend/test.py index 50ae56ac5..4834e81e8 100644 --- a/backend/test.py +++ b/backend/test.py @@ -8,10 +8,6 @@ class RequestTests(TestCase): HOST = 'http://localhost:4000' - def __init__(self): - super().__init__() - self._session = None - def newSession(self): if hasattr(self, '_session'): self.destroySession() @@ -285,10 +281,6 @@ def test_admin_is_admin(self): class TestUserManagement(RequestTests): - def __init__(self): - super().__init__() - self._email = None - def setUp(self): self.newSession() From a6ec0878f34f6fcfbdce3e36658c6b00c88b4217 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 12 Feb 2018 13:12:45 +0100 Subject: [PATCH 26/65] Handle more email exceptions --- backend/application.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/application.py b/backend/application.py index e7b178ddf..99f01f3e1 100644 --- a/backend/application.py +++ b/backend/application.py @@ -6,6 +6,7 @@ from peewee import fn import peewee import smtplib +import socket import tornado.web import random import string @@ -365,6 +366,8 @@ def post(self, dataset, email): server.sendmail(msg['from'], [msg['to']], msg.as_string()) except smtplib.SMTPException as e: logging.error("Email error: {}".format(e)) + except socket.gaierror as e: + logging.error("Email error: {}".format(e)) self.finish() From 212b46a2be7508349a346c921babb724e0dd1f99 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:45:20 +0100 Subject: [PATCH 27/65] Rebuild all templates in case the dataset-base.html was modified --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 60abdb5a5..835e3d810 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,6 @@ static/js/app.js: $(JAVASCRIPT_FILES) mkdir -p $$( dirname $@ ) cat $^ >$@ -static/templates/%.html: frontend/templates/%.html +static/templates/%.html: frontend/templates/%.html frontend/templates/ng-templates/dataset-base.html mkdir -p $$( dirname $@ ) 2>/dev/null || true python3 scripts/compile_template.py ${COMPILE_TEMPLATE_OPTS} -b frontend/templates -s $< >$@ From ce74d9868373f9d9acb4e6adf05cc7c34b483676 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:50:33 +0100 Subject: [PATCH 28/65] Updated schema with data_contact --- sql/patch-master-db.sql | 5 +++++ sql/swefreq.sql | 2 ++ 2 files changed, 7 insertions(+) diff --git a/sql/patch-master-db.sql b/sql/patch-master-db.sql index a74587a7a..4013fac5e 100644 --- a/sql/patch-master-db.sql +++ b/sql/patch-master-db.sql @@ -1,2 +1,7 @@ -- Patches a database that is using the master checkout of the -- swefreq.sql schema definition to the develop version. + +ALTER TABLE dataset_version + ADD COLUMN ( + data_contact_name VARCHAR(100) DEFAULT NULL, + data_contact_link VARCHAR(100) DEFAULT NULL); diff --git a/sql/swefreq.sql b/sql/swefreq.sql index 6b2f0363f..32398c3f7 100644 --- a/sql/swefreq.sql +++ b/sql/swefreq.sql @@ -89,6 +89,8 @@ CREATE TABLE IF NOT EXISTS dataset_version ( var_call_ref VARCHAR(50) DEFAULT NULL, available_from TIMESTAMP DEFAULT CURRENT_TIMESTAMP, ref_doi VARCHAR(100) DEFAULT NULL, + data_contact_name VARCHAR(100) DEFAULT NULL, + data_contact_link VARCHAR(100) DEFAULT NULL, CONSTRAINT UNIQUE (dataset_pk, version), CONSTRAINT FOREIGN KEY (dataset_pk) REFERENCES dataset(dataset_pk) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; From 151c91eded954c18988f6f32f2e70da74205bb4c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:51:00 +0100 Subject: [PATCH 29/65] Updated ORM with data_contact --- backend/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/db.py b/backend/db.py index 9fd3e7d80..e9271d0de 100644 --- a/backend/db.py +++ b/backend/db.py @@ -127,6 +127,8 @@ class DatasetVersion(BaseModel): var_call_ref = CharField(null=True) available_from = DateTimeField() ref_doi = CharField(null=True) + data_contact_name = CharField(null=True) + data_contact_link = CharField(null=True) class Meta: db_table = 'dataset_version' From e13e316c2375a900fa6125f06f3e6c2211112c47 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:51:17 +0100 Subject: [PATCH 30/65] Whitespace --- backend/db.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/db.py b/backend/db.py index e9271d0de..ae38cab8f 100644 --- a/backend/db.py +++ b/backend/db.py @@ -119,14 +119,14 @@ class Meta: class DatasetVersion(BaseModel): - dataset_version = PrimaryKeyField(db_column='dataset_version_pk') - dataset = ForeignKeyField(db_column='dataset_pk', rel_model=Dataset, to_field='dataset', related_name='versions') - version = CharField() - description = TextField() - terms = TextField() - var_call_ref = CharField(null=True) - available_from = DateTimeField() - ref_doi = CharField(null=True) + dataset_version = PrimaryKeyField(db_column='dataset_version_pk') + dataset = ForeignKeyField(db_column='dataset_pk', rel_model=Dataset, to_field='dataset', related_name='versions') + version = CharField() + description = TextField() + terms = TextField() + var_call_ref = CharField(null=True) + available_from = DateTimeField() + ref_doi = CharField(null=True) data_contact_name = CharField(null=True) data_contact_link = CharField(null=True) From 39346b15e10b16eeb39fbefcd0a3535071ce74fd Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:51:54 +0100 Subject: [PATCH 31/65] Updated dummy data with data contact --- test/data/load_dummy_data.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/data/load_dummy_data.sql b/test/data/load_dummy_data.sql index ac2b5034c..d8238201b 100644 --- a/test/data/load_dummy_data.sql +++ b/test/data/load_dummy_data.sql @@ -16,11 +16,11 @@ INSERT INTO sample_set (sample_set_pk, dataset_pk, collection_pk, sample_size, p (1000002, 1000001, 1000002, 15, 'SamplePheno2 Coll1'), (1000003, 1000002, 1000003, 20, 'SamplePheno2 Coll2'); -INSERT INTO dataset_version (dataset_version_pk, dataset_pk, version, description, terms, var_call_ref, available_from, ref_doi) - VALUES (1000001, 1000001, 'Version 1-1', 'Dataset 1-1, description', 'Dataset 1-1, terms', 'CallRef11', '2017-01-01', 'datset11DOI'), - (1000002, 1000002, 'Version 2-1', 'Dataset 2-1, description', 'Dataset 2-1, terms', 'CallRef21', '2017-02-01', 'datset21DOI'), - (1000003, 1000002, 'Version 2-2', 'Dataset 2-2, description', 'Dataset 2-2, terms', 'CallRef22', '2017-02-02', 'datset22DOI'), - (1000004, 1000002, 'InvVer 2-3', 'Dataset 2-3, description', 'Dataset 2-3, terms', 'CallRef23', '2030-02-03', 'datset23DOI'); +INSERT INTO dataset_version (dataset_version_pk, dataset_pk, version, description, terms, var_call_ref, available_from, ref_doi, data_contact_name, data_contact_link) + VALUES (1000001, 1000001, 'Version 1-1', 'Dataset 1-1, description', 'Dataset 1-1, terms', 'CallRef11', '2017-01-01', 'datset11DOI', "Gunnar Green", "gunnar.green@example.com"), + (1000002, 1000002, 'Version 2-1', 'Dataset 2-1, description', 'Dataset 2-1, terms', 'CallRef21', '2017-02-01', 'datset21DOI', NULL, NULL), + (1000003, 1000002, 'Version 2-2', 'Dataset 2-2, description', 'Dataset 2-2, terms', 'CallRef22', '2017-02-02', 'datset22DOI', "Strummer project", "https://example.com/strummer"), + (1000004, 1000002, 'InvVer 2-3', 'Dataset 2-3, description', 'Dataset 2-3, terms', 'CallRef23', '2030-02-03', 'datset23DOI', "Drummer project", "https://example.com/drummer"); INSERT INTO dataset_file(dataset_file_pk, dataset_version_pk, name, uri, bytes) VALUES (1000001, 1000001, 'File11-1', '/release/file111.txt', 100), From c1df88bf4e598fa7d3ccc6dd4252e6c2b22dfbd3 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:52:29 +0100 Subject: [PATCH 32/65] Renamed download tab into "dataset access" --- frontend/templates/ng-templates/dataset-base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/templates/ng-templates/dataset-base.html b/frontend/templates/ng-templates/dataset-base.html index 902e64803..11cdbc170 100644 --- a/frontend/templates/ng-templates/dataset-base.html +++ b/frontend/templates/ng-templates/dataset-base.html @@ -19,7 +19,7 @@

{{ ctrl.dataset.fullName }}

  • Information
  • Terms
    • -
  • Download
    • +
  • Dataset Access
  • Beacon
  • Graphical Browser
  • From f41a266ca2da2228eaf38c0e19db42aee6faf1df Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:53:12 +0100 Subject: [PATCH 33/65] Frontend function to verify if contact_link is email or not --- frontend/src/js/controller.datasetDownloadController.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/js/controller.datasetDownloadController.js b/frontend/src/js/controller.datasetDownloadController.js index 6aef55e1d..c7e86d3cc 100644 --- a/frontend/src/js/controller.datasetDownloadController.js +++ b/frontend/src/js/controller.datasetDownloadController.js @@ -10,6 +10,7 @@ localThis.createTemporaryLink = createTemporaryLink; localThis.copyLink = copyLink; localThis.canCopy = clipboard.supported; + localThis.dataContactIsEmail = dataContactIsEmail; localThis.host = $location.protocol() + "://" + $location.host(); @@ -86,5 +87,10 @@ function copyLink(link) { clipboard.copyText(link); } + + function dataContactIsEmail() { + return localThis.hasOwnProperty('dataset') && + localThis.dataset.version.dataContactLink.indexOf('@') > -1; + } }]); })(); From f4e5da71aaff95246bfa061cc4dfa123918309f9 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:54:09 +0100 Subject: [PATCH 34/65] Reword into "[access] summary files" instead of "[access] dataset" --- frontend/templates/ng-templates/dataset-download.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/templates/ng-templates/dataset-download.html b/frontend/templates/ng-templates/dataset-download.html index 7b05144df..2cc431aff 100644 --- a/frontend/templates/ng-templates/dataset-download.html +++ b/frontend/templates/ng-templates/dataset-download.html @@ -6,12 +6,12 @@
    -
    You need to login for this access
    +
    You need to login to request access to the summary files.
    -

    Request access to the dataset

    +

    Request access to summary files

    From fb72e5e22febe7be63e09dc9f664033996c63c96 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 22 Feb 2018 15:54:44 +0100 Subject: [PATCH 35/65] Notify user about summary files and full dataset --- frontend/templates/ng-templates/dataset-download.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frontend/templates/ng-templates/dataset-download.html b/frontend/templates/ng-templates/dataset-download.html index 2cc431aff..742f7b664 100644 --- a/frontend/templates/ng-templates/dataset-download.html +++ b/frontend/templates/ng-templates/dataset-download.html @@ -2,6 +2,17 @@ [% extends "ng-templates/dataset-base.html" %] [% block controller %]datasetDownloadController[% endblock %] [% block contents %] +
    +
    + +
    +
    From 8734987acca7cda449309d9ba1fe5f64d458b3ee Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Fri, 23 Feb 2018 10:07:48 +0100 Subject: [PATCH 36/65] Adds DROP IF EXISTS to triggers to silence ansible errors. --- sql/swefreq.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/swefreq.sql b/sql/swefreq.sql index 6b2f0363f..f47dc31b8 100644 --- a/sql/swefreq.sql +++ b/sql/swefreq.sql @@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS sftp_user ( DELIMITER $$ +DROP TRIGGER IF EXISTS check_before_insert $$ CREATE TRIGGER check_before_insert BEFORE INSERT ON sftp_user FOR EACH ROW IF NEW.user_uid < 10000 THEN @@ -39,6 +40,7 @@ IF NEW.user_uid < 10000 THEN SET MESSAGE_TEXT = 'Check constraint failed on sftp_user.user_uid insert'; END IF$$ +DROP TRIGGER IF EXISTS check_before_update $$ CREATE TRIGGER check_before_update BEFORE UPDATE ON sftp_user FOR EACH ROW IF NEW.user_uid < 10000 THEN From 5cd05837b93c38fe3a9a217b24346c1f23e20062 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Mon, 26 Feb 2018 09:35:09 +0100 Subject: [PATCH 37/65] Updated text about download of summary files --- frontend/templates/ng-templates/dataset-download.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/templates/ng-templates/dataset-download.html b/frontend/templates/ng-templates/dataset-download.html index 742f7b664..d3b7f7a21 100644 --- a/frontend/templates/ng-templates/dataset-download.html +++ b/frontend/templates/ng-templates/dataset-download.html @@ -5,8 +5,8 @@
    + + +
    +
    +

    sFTP Access

    +
    +
    +
    +
    + + + + + + + + + +
    Username + Password + Expiry date + +
    {{ ctrl.sftp.user }}{{ ctrl.sftp.password }}{{ ctrl.sftp.expires }}
    +
    +
    +
    From 66bfae9c3120eb5928c945e5a8a03e2e899b43db Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Tue, 27 Mar 2018 15:41:49 +0200 Subject: [PATCH 39/65] Eslint cleanup --- frontend/src/js/controller.datasetDownloadController.js | 4 ++-- frontend/src/js/factory.sFTPAccess.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/controller.datasetDownloadController.js b/frontend/src/js/controller.datasetDownloadController.js index c7e86d3cc..3336bcbdc 100644 --- a/frontend/src/js/controller.datasetDownloadController.js +++ b/frontend/src/js/controller.datasetDownloadController.js @@ -89,8 +89,8 @@ } function dataContactIsEmail() { - return localThis.hasOwnProperty('dataset') && - localThis.dataset.version.dataContactLink.indexOf('@') > -1; + return localThis.hasOwnProperty("dataset") && + localThis.dataset.version.dataContactLink.indexOf("@") > -1; } }]); })(); diff --git a/frontend/src/js/factory.sFTPAccess.js b/frontend/src/js/factory.sFTPAccess.js index 1061d2711..d8411318e 100644 --- a/frontend/src/js/factory.sFTPAccess.js +++ b/frontend/src/js/factory.sFTPAccess.js @@ -13,7 +13,7 @@ }); } - function createCredentials( dataset ) { + function createCredentials() { return $http.post("/api/users/sftp_access", $.param({"_xsrf": $cookies.get("_xsrf")}), {headers : { From 639e59c6bb9c715d0eddb3849d80d434e84ed1c5 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 29 Mar 2018 10:52:14 +0200 Subject: [PATCH 40/65] Makes sure only dataset administrators can generate sFTP credentials. --- backend/application.py | 10 ++++ .../src/js/controller.profileController.js | 19 ++++--- frontend/templates/ng-templates/profile.html | 52 ++++++++++--------- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/backend/application.py b/backend/application.py index 870c3cdfd..8c4f67420 100644 --- a/backend/application.py +++ b/backend/application.py @@ -492,6 +492,11 @@ def get(self): """ Returns sFTP credentials for the current user. """ + for dataset in db.Dataset.select(): + if self.current_user.is_admin( dataset ): + break + else: + self.finish({'user':None, 'expires':None, 'password':None}) password = None username = None @@ -515,6 +520,11 @@ def post(self): new set of sftp credentials for a user, or updates the old ones with a new password and expiry date. """ + for dataset in db.Dataset.select(): + if self.current_user.is_admin( dataset ): + break + else: + self.finish({'user':None, 'expires':None, 'password':None}) # Create a new password username = "_".join(self.current_user.name.split()) + "_sftp" diff --git a/frontend/src/js/controller.profileController.js b/frontend/src/js/controller.profileController.js index f6688a0d7..ad236f349 100644 --- a/frontend/src/js/controller.profileController.js +++ b/frontend/src/js/controller.profileController.js @@ -4,13 +4,8 @@ function(User, Countries, SFTPAccess) { var localThis = this; localThis.sftp = {"user":"", "password":"", "expires":null}; - localThis.createSFTPCredentials = function() { - SFTPAccess.createCredentials() - .then( function(data) { - localThis.sftp = data; - }); - }; - + localThis.createSFTPCredentials = createSFTPCredentials; + localThis.isAdmin = false; activate(); function activate() { @@ -26,6 +21,9 @@ User.getDatasets().then(function(data) { localThis.datasets = data; + $.each(data, function(_,dataset) { + localThis.isAdmin = localThis.isAdmin || dataset.isAdmin; + }); }); SFTPAccess.getCredentials() @@ -40,5 +38,12 @@ localThis.user.country = { name: country, id: country }; } } + + function createSFTPCredentials() { + SFTPAccess.createCredentials() + .then( function(data) { + localThis.sftp = data; + }); + }; }]); })(); diff --git a/frontend/templates/ng-templates/profile.html b/frontend/templates/ng-templates/profile.html index e0672d636..2ce947b73 100644 --- a/frontend/templates/ng-templates/profile.html +++ b/frontend/templates/ng-templates/profile.html @@ -56,9 +56,9 @@

    Dataset Access

    - + + @@ -70,28 +70,30 @@

    Dataset Access

    -
    -
    -

    sFTP Access

    -
    -
    -
    -
    -
    Dataset - Subscribed to email updates - Access + DatasetSubscribed to email updatesAccess
    {{row.shortName}}
    - - - - - - - - -
    Username - Password - Expiry date - -
    {{ ctrl.sftp.user }}{{ ctrl.sftp.password }}{{ ctrl.sftp.expires }}
    -
    +
    +
    +
    +

    sFTP Access

    +
    +
    +
    +
    + + + + + + + + + + + + + +
    UsernamePasswordExpiry date
    {{ ctrl.sftp.user }}{{ ctrl.sftp.password }}{{ ctrl.sftp.expires }}
    +
    +
    From b75f716693588f2228879e2e982fafd41df30684 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 29 Mar 2018 14:51:46 +0200 Subject: [PATCH 41/65] Adds profile link to header menu. --- frontend/src/js/controller.profileController.js | 2 +- frontend/templates/index.html | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/controller.profileController.js b/frontend/src/js/controller.profileController.js index ad236f349..c3a27cd57 100644 --- a/frontend/src/js/controller.profileController.js +++ b/frontend/src/js/controller.profileController.js @@ -44,6 +44,6 @@ .then( function(data) { localThis.sftp = data; }); - }; + } }]); })(); diff --git a/frontend/templates/index.html b/frontend/templates/index.html index 9a8418b33..27cfc34cb 100644 --- a/frontend/templates/index.html +++ b/frontend/templates/index.html @@ -49,7 +49,12 @@
    From 9b2d284ac2d614c5404a62d8d73fa6c9ea813915 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 13:54:06 +0200 Subject: [PATCH 55/65] Added test coverage through coveralls in the travis system --- .travis.yml | 1 + backend/settings.py | 6 +++++- test/travis_script.sh | 10 ++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc53665ad..e9d8926e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ before_install: - test/travis_before_install.sh install: - pip install -r backend/requirements.txt + - pip install coverage coveralls script: - test/travis_script.sh diff --git a/backend/settings.py b/backend/settings.py index 4cbd431b3..b0a62ebc4 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -1,6 +1,10 @@ import json -json_settings_fh = open("../settings.json") +try: + json_settings_fh = open("settings.json") +except FileNotFoundError: + json_settings_fh = open("../settings.json") + json_settings = json.load(json_settings_fh) json_settings_fh.close() diff --git a/test/travis_script.sh b/test/travis_script.sh index 1124cc748..f4ccef324 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -31,13 +31,14 @@ mysql -u swefreq -h 127.0.0.1 -P 3366 swefreq_test < test/data/load_dummy_data.s echo ">>> Test 3. Check that the backend starts" -cd backend +cd backend ../test/01_daemon_starts.sh +cd .. echo ">>> Test 4. the backend API" -python route.py --develop 1>http_log.txt 2>&1 & +coverage run backend/route.py --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! function exit_handler() { @@ -51,3 +52,8 @@ trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started python test.py -v + +kill -2 $BACKEND_PID +sleep 2 + +coveralls From 93e116b2f006435e358f4d7b9b05be687b6e1e5c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 15:01:39 +0200 Subject: [PATCH 56/65] Updated Dockerfile-backend to run from base dir instea --- Dockerfile-backend | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile-backend b/Dockerfile-backend index c2f9361ba..2ccc0b728 100644 --- a/Dockerfile-backend +++ b/Dockerfile-backend @@ -6,8 +6,8 @@ RUN apt-get update && apt-get install -y \ libmysqlclient-dev ADD . /code -WORKDIR /code/backend +WORKDIR /code -RUN pip3 install -r requirements.txt +RUN pip3 install -r backend/requirements.txt -CMD ["python3", "route.py", "--develop"] +CMD ["python3", "backend/route.py", "--develop"] From 9dbc07d3f2961c22b694f578c2a3b3da33a90f66 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 15:02:21 +0200 Subject: [PATCH 57/65] Run from base directory in travis test --- test/travis_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index f4ccef324..58e6dba3e 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -51,7 +51,7 @@ function exit_handler() { trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started -python test.py -v +python backend/test.py -v kill -2 $BACKEND_PID sleep 2 From 6a520e9e9418c8d1338b42bb48bc5e68d7f7b415 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:12:59 +0200 Subject: [PATCH 58/65] More coverage settings and show result in log --- .coveragerc | 5 +++++ test/travis_script.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8db5982de --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[run] +omit = /usr/local/*,/home/travis/virtualenv/* + +[report] +omit = /usr/local/*,/home/travis/virtualenv/* diff --git a/test/travis_script.sh b/test/travis_script.sh index 58e6dba3e..f6b7fe57e 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -38,7 +38,7 @@ cd .. echo ">>> Test 4. the backend API" -coverage run backend/route.py --develop 1>http_log.txt 2>&1 & +coverage run backend/route.py --port=4000 --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! function exit_handler() { @@ -57,3 +57,4 @@ kill -2 $BACKEND_PID sleep 2 coveralls +coverage report From b0b5494245faf41b1a46f5fbfab3c81a8b9e4eec Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:31:33 +0200 Subject: [PATCH 59/65] Only run the test coverage stuff if we have the ".coverage" file --- test/travis_script.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index f6b7fe57e..66d7f3104 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -56,5 +56,7 @@ python backend/test.py -v kill -2 $BACKEND_PID sleep 2 -coveralls -coverage report +if [ -f .coverage ]; then + coveralls + coverage report +fi From 902c171db6c705e92af0d19e021fd51c954a840d Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 14:40:01 +0200 Subject: [PATCH 60/65] Added a way to quit the webapp in developer mode --- backend/application.py | 7 +++++++ backend/route.py | 1 + test/travis_script.sh | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/application.py b/backend/application.py index 4df01482d..49c0f90ce 100644 --- a/backend/application.py +++ b/backend/application.py @@ -8,6 +8,7 @@ import smtplib import socket import tornado.web +import tornado import random import string import uuid @@ -41,6 +42,12 @@ def build_dataset_structure(dataset_version, user=None, dataset=None): return r +class QuitHandler(handlers.UnsafeHandler): + def get(self): + ioloop = tornado.ioloop.IOLoop.instance() + ioloop.stop() + + class GetSchema(handlers.UnsafeHandler): """ Returns the schema.org, and bioschemas.org, annotation for a given diff --git a/backend/route.py b/backend/route.py index 200b3b7a6..3b0c6de06 100644 --- a/backend/route.py +++ b/backend/route.py @@ -92,6 +92,7 @@ def __init__(self, settings): ## Adding developer login handler if settings.get('develop', False): self.declared_handlers.insert(-1, ("/developer/login", auth.DeveloperLoginHandler)) + self.declared_handlers.insert(-1, ("/developer/quit", application.QuitHandler)) # google oauth key self.oauth_key = tornado_settings["google_oauth"]["key"] diff --git a/test/travis_script.sh b/test/travis_script.sh index 66d7f3104..b83c44003 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -52,10 +52,11 @@ trap exit_handler EXIT sleep 2 # Lets wait a little bit so the server has started python backend/test.py -v - -kill -2 $BACKEND_PID sleep 2 +# Quit the app +curl localhost:4000/developer/quit + if [ -f .coverage ]; then coveralls coverage report From fe3a6808689ee9dfd7d096c7d00bb7837b591c18 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 16:07:45 +0200 Subject: [PATCH 61/65] Added travis and coverage badges to readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 04b8d704e..9d563bfd4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ SweFreq - Swedish Frequency database ==================================== +[![Travis Status][travis-badge]][travis-link] +[![Coverage Status][coveralls-badge]][coverage-link] + Running on a production system ------------------------------ @@ -70,3 +73,8 @@ Quick development mode ```bash $ docker-compose up ``` + +[travis-badge]: https://travis-ci.org/NBISweden/swefreq.svg?branch=develop +[travis-link]: https://travis-ci.org/NBISweden/swefreq +[coveralls-badge]: https://coveralls.io/repos/github/NBISweden/swefreq/badge.svg?branch=develop +[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=release From db0efb7e4349480c71550a7de3275a80413f443c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 4 Apr 2018 16:12:14 +0200 Subject: [PATCH 62/65] Updated coveralls badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d563bfd4..a9591ac71 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ SweFreq - Swedish Frequency database ==================================== [![Travis Status][travis-badge]][travis-link] -[![Coverage Status][coveralls-badge]][coverage-link] +[![Coverage Status][coveralls-badge]][coveralls-link] Running on a production system @@ -77,4 +77,4 @@ $ docker-compose up [travis-badge]: https://travis-ci.org/NBISweden/swefreq.svg?branch=develop [travis-link]: https://travis-ci.org/NBISweden/swefreq [coveralls-badge]: https://coveralls.io/repos/github/NBISweden/swefreq/badge.svg?branch=develop -[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=release +[coveralls-link]: https://coveralls.io/github/NBISweden/swefreq?branch=develop From a02753fe2cc8160dd92e7db7721e4e5a47a40f4d Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 09:44:26 +0200 Subject: [PATCH 63/65] Moved the sleeps around --- test/travis_script.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index b83c44003..b3a98b970 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -41,6 +41,8 @@ echo ">>> Test 4. the backend API" coverage run backend/route.py --port=4000 --develop 1>http_log.txt 2>&1 & BACKEND_PID=$! +sleep 2 # Lets wait a little bit so the server has started + function exit_handler() { kill -9 $BACKEND_PID @@ -50,12 +52,12 @@ function exit_handler() { trap exit_handler EXIT -sleep 2 # Lets wait a little bit so the server has started + python backend/test.py -v -sleep 2 # Quit the app curl localhost:4000/developer/quit +sleep 2 # Lets wait a little bit so the server has stopped if [ -f .coverage ]; then coveralls From 0ea9885713cf6e703c2cb9885a62f6eb7bdc01ff Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 09:54:05 +0200 Subject: [PATCH 64/65] Better exit handler --- test/travis_script.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/travis_script.sh b/test/travis_script.sh index b3a98b970..a187e9604 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -44,10 +44,14 @@ BACKEND_PID=$! sleep 2 # Lets wait a little bit so the server has started function exit_handler() { + rv=$? + # Ignore errors in the exit handler + set +e kill -9 $BACKEND_PID echo "THE HTTP LOG WAS:" cat http_log.txt + exit $rv } trap exit_handler EXIT From 209ba62db381b953c39178d925c60d0d9741d589 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 5 Apr 2018 11:12:19 +0200 Subject: [PATCH 65/65] Updated travis script --- .pylintrc | 3 ++- test/travis_script.sh | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3e4f6d1d3..5b77e41e8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -138,7 +138,8 @@ disable=print-statement, keyword-arg-before-vararg, arguments-differ, line-too-long, - import-error + import-error, + no-self-use # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/test/travis_script.sh b/test/travis_script.sh index a187e9604..34f904dc0 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -32,9 +32,7 @@ mysql -u swefreq -h 127.0.0.1 -P 3366 swefreq_test < test/data/load_dummy_data.s echo ">>> Test 3. Check that the backend starts" -cd backend -../test/01_daemon_starts.sh -cd .. +(cd backend && ../test/01_daemon_starts.sh) echo ">>> Test 4. the backend API" @@ -43,10 +41,12 @@ BACKEND_PID=$! sleep 2 # Lets wait a little bit so the server has started -function exit_handler() { +exit_handler() { rv=$? # Ignore errors in the exit handler set +e + # We want to make sure the background process has stopped, otherwise the + # travis test will stall for a long time. kill -9 $BACKEND_PID echo "THE HTTP LOG WAS:"