From 51825e6b4b2d3ed5b51c10898da9bacdb4b9f7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 8 Aug 2019 12:50:43 +0200 Subject: [PATCH 1/2] Remove old importer from system. --- scripts/importer/data_importer/old_db.py | 303 ------------ .../importer/data_importer/old_db_importer.py | 447 ------------------ scripts/importer/importer.py | 9 - 3 files changed, 759 deletions(-) delete mode 100644 scripts/importer/data_importer/old_db.py delete mode 100644 scripts/importer/data_importer/old_db_importer.py diff --git a/scripts/importer/data_importer/old_db.py b/scripts/importer/data_importer/old_db.py deleted file mode 100644 index 7decaf19d..000000000 --- a/scripts/importer/data_importer/old_db.py +++ /dev/null @@ -1,303 +0,0 @@ -from peewee import ( - BlobField, - CharField, - DateTimeField, - Field, - FloatField, - ForeignKeyField, - IntegerField, - Model, - MySQLDatabase, - PrimaryKeyField, - TextField, - fn, - ) -import settings - -mysql_database = MySQLDatabase( - settings.mysql_schema, - host=settings.mysql_host, - user=settings.mysql_user, - password=settings.mysql_passwd, - port=settings.mysql_port - ) - - -class MySQLModel(Model): - class Meta: - database = mysql_database - - -class EnumField(Field): - db_field = 'string' # The same as for CharField - - def __init__(self, values=None, *args, **kwargs): - self.values = values or [] - super().__init__(*args, **kwargs) - - def db_value(self, value): - if value not in self.values: - raise ValueError("Illegal value for '{}'".format(self.db_column)) - return value - - def python_value(self, value): - if value not in self.values: - raise ValueError("Illegal value for '{}'".format(self.db_column)) - return value - - -class User(MySQLModel): - user = PrimaryKeyField(db_column='user_pk') - name = CharField(null=True) - email = CharField(unique=True) - identity = CharField(unique=True) - identity_type = EnumField(null=False, values=['google', 'elixir']) - affiliation = CharField(null=True) - country = CharField(null=True) - - def is_admin(self, dataset): - return (DatasetAccess.select() - .where(DatasetAccess.dataset == dataset, - DatasetAccess.user == self, - DatasetAccess.is_admin) - .count()) - - def has_access(self, dataset): - return DatasetAccessCurrent.select().where( - DatasetAccessCurrent.dataset == dataset, - DatasetAccessCurrent.user == self, - ).count() - - def has_requested_access(self, dataset): - return DatasetAccessPending.select().where( - DatasetAccessPending.dataset == dataset, - DatasetAccessPending.user == self - ).count() - - class Meta: - db_table = 'user' - - -class Study(MySQLModel): - study = PrimaryKeyField(db_column='study_pk') - pi_name = CharField() - pi_email = CharField() - contact_name = CharField() - contact_email = CharField() - title = CharField() - description = TextField(null=True) - publication_date = DateTimeField() - ref_doi = CharField(null=True) - - class Meta: - db_table = 'study' - - -class Dataset(MySQLModel): - dataset = PrimaryKeyField(db_column='dataset_pk') - study = ForeignKeyField(Study, db_column='study_pk', to_field='study', related_name='datasets') - short_name = CharField() - full_name = CharField() - browser_uri = CharField(null=True) - beacon_uri = CharField(null=True) - avg_seq_depth = FloatField(null=True) - seq_type = CharField(null=True) - seq_tech = CharField(null=True) - seq_center = CharField(null=True) - dataset_size = IntegerField() - mongodb_collection = CharField(null=False) - - def has_image(self): - try: - DatasetLogo.get(DatasetLogo.dataset == self) - return True - except DatasetLogo.DoesNotExist: - return False - - class Meta: - db_table = 'dataset' - - -class DatasetVersion(MySQLModel): - dataset_version = PrimaryKeyField(db_column='dataset_version_pk') - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='versions') - version = CharField() - description = TextField() - terms = TextField() - var_call_ref = CharField(null=True) - available_from = DateTimeField() - ref_doi = CharField(null=True) - data_contact_name = CharField(null=True) - data_contact_link = CharField(null=True) - - class Meta: - db_table = 'dataset_version' - - -class Collection(MySQLModel): - collection = PrimaryKeyField(db_column = 'collection_pk') - name = CharField(null = True) - ethnicity = CharField(null = True) - - class Meta: - db_table = 'collection' - - -class SampleSet(MySQLModel): - sample_set = PrimaryKeyField(db_column='sample_set_pk') - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='sample_sets') - collection = ForeignKeyField(Collection, db_column='collection_pk', to_field='collection', related_name='sample_sets') - sample_size = IntegerField() - phenotype = CharField(null=True) - - class Meta: - db_table = 'sample_set' - - -class DatasetFile(MySQLModel): - dataset_file = PrimaryKeyField(db_column='dataset_file_pk') - dataset_version = ForeignKeyField(DatasetVersion, db_column='dataset_version_pk', to_field='dataset_version', related_name='files') - name = CharField() - uri = CharField() - bytes = IntegerField() - - class Meta: - db_table = 'dataset_file' - - -class UserAccessLog(MySQLModel): - user_access_log = PrimaryKeyField(db_column='user_access_log_pk') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='access_logs') - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='access_logs') - action = EnumField(null=True, values=['access_requested','access_granted','access_revoked','private_link']) - ts = DateTimeField() - - class Meta: - db_table = 'user_access_log' - - -class UserConsentLog(MySQLModel): - user_consent_log = PrimaryKeyField(db_column='user_consent_log_pk') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='consent_logs') - dataset_version = ForeignKeyField(DatasetVersion, db_column='dataset_version_pk', to_field='dataset_version', related_name='consent_logs') - ts = DateTimeField() - - class Meta: - db_table = 'user_consent_log' - - -class UserDownloadLog(MySQLModel): - user_download_log = PrimaryKeyField(db_column='user_download_log_pk') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='download_logs') - dataset_file = ForeignKeyField(DatasetFile, db_column='dataset_file_pk', to_field='dataset_file', related_name='download_logs') - ts = DateTimeField() - - class Meta: - db_table = 'user_download_log' - - -class DatasetAccess(MySQLModel): - dataset_access = PrimaryKeyField(db_column='dataset_access_pk') - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='access') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='access') - wants_newsletter = IntegerField(null=True) - is_admin = IntegerField(null=True) - - class Meta: - db_table = 'dataset_access' - - -class DatasetAccessCurrent(DatasetAccess): - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='access_current') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='access_current') - has_access = IntegerField() - access_requested = DateTimeField() - - class Meta: - db_table = 'dataset_access_current' - - -class DatasetAccessPending(DatasetAccess): - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='access_pending') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='access_pending') - has_access = IntegerField() - access_requested = DateTimeField() - - class Meta: - db_table = 'dataset_access_pending' - - -class DatasetLogo(MySQLModel): - dataset_logo = PrimaryKeyField(db_column='dataset_logo_pk') - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='logo') - mimetype = CharField() - data = BlobField() - - class Meta: - db_table = 'dataset_logo' - - -class Linkhash(MySQLModel): - linkhash = PrimaryKeyField(db_column='linkhash_pk') - dataset_version = ForeignKeyField(DatasetVersion, db_column='dataset_version_pk', to_field='dataset_version', related_name='link_hashes') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='link_hashes') - hash = CharField() - expires_on = DateTimeField() - - class Meta: - db_table = 'linkhash' - - -class DatasetVersionCurrent(DatasetVersion): - dataset = ForeignKeyField(Dataset, db_column='dataset_pk', to_field='dataset', related_name='current_version') - - class Meta: - db_table = 'dataset_version_current' - - -class SFTPUser(MySQLModel): - sftp_user = PrimaryKeyField(db_column='sftp_user_pk') - user = ForeignKeyField(User, db_column='user_pk', to_field='user', related_name='sftp_user') - user_uid = IntegerField(unique=True) - user_name = CharField(null=False) - password_hash = CharField(null=False) - account_expires = DateTimeField(null=False) - - class Meta: - db_table = 'sftp_user' - - -def get_next_free_uid(): - """ - Returns the next free uid >= 10000, and higher than the current uid's - from the sftp_user table in the database. - """ - default = 10000 - next_uid = default - try: - current_max_uid = SFTPUser.select(fn.MAX(SFTPUser.user_uid)).get().user_uid - if current_max_uid: - next_uid = current_max_uid+1 - except SFTPUser.DoesNotExist: - pass - - return next_uid - - -def get_admin_datasets(user): - return DatasetAccess.select().where( DatasetAccess.user == user, DatasetAccess.is_admin) - - -def get_dataset(dataset): - dataset = Dataset.select().where( Dataset.short_name == dataset).get() - return dataset - - -def build_dict_from_row(row): - d = {} - for field in row._meta.sorted_fields: #pylint: disable=protected-access - column = field.db_column - if column.endswith("_pk"): - continue - d[column] = getattr(row, column) - return d diff --git a/scripts/importer/data_importer/old_db_importer.py b/scripts/importer/data_importer/old_db_importer.py deleted file mode 100644 index a1beb077c..000000000 --- a/scripts/importer/data_importer/old_db_importer.py +++ /dev/null @@ -1,447 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import time -import logging -import db -from peewee import OperationalError, InterfaceError -from . import old_db - -from .data_importer import DataImporter - -class OldDbImporter( DataImporter ): - - def __init__(self, settings): - super().__init__(settings) - self.reference_sets = [] - self.id_map = {'collection':{}, - 'study':{}, - 'dataset':{}, - 'dataset_version':{}, - 'dataset_file':{}, - 'user':{} - } - - def _select_reference_set(self, short_name): - if len(self.reference_sets) == 1: - logging.info(("Only one reference set is available, %s, " - "will default to this set for all datasets"), - self.reference_sets[0].name) - return self.reference_sets[0].id - elif short_name.lower() in [r.name.lower() for r in self.reference_sets]: - refset = [r for r in self.reference_sets if r.name.lower() == short_name.lower()][0] - logging.info("Auto-selecting reference set '%s' based on name.", refset.name) - return refset - else: - print("Select a reference set to use with this dataset") - retval = -1 - while retval not in [r.id for r in self.reference_sets]: - for reference_set in self.reference_sets: - print(" {} : {}".format(reference_set.id, reference_set.name)) - try: - retval = int(input("Please select a reference: ")) - except ValueError: - print("Please select a number in {}".format([r.id for r in self.reference_sets])) - return retval - - def _move_collections(self): - logging.info("Moving Collections") - for collection in old_db.Collection.select(): - logging.info(" - Moving '{}'".format(collection.name)) - - try: - new_id = db.Collection.get(name = collection.name, - ethnicity = collection.ethnicity).id - except db.Collection.DoesNotExist: - if self.settings.dry_run: - continue - new_id = (db.Collection - .insert(name = collection.name, - ethnicity = collection.ethnicity) - .execute()) - - self.id_map['collection'][collection.collection] = new_id - - def _move_studies(self): - logging.info("Moving Studies") - for study in old_db.Study.select(): - logging.info(" - Moving '{}'".format(study.title)) - - try: - new_id = db.Study.get(pi_name = study.pi_name, - pi_email = study.pi_email, - contact_name = study.contact_name, - contact_email = study.contact_email, - title = study.title, - description = study.description, - publication_date = study.publication_date, - ref_doi = study.ref_doi).id - except db.Study.DoesNotExist: - if self.settings.dry_run: - continue - new_id = (db.Study - .insert(pi_name = study.pi_name, - pi_email = study.pi_email, - contact_name = study.contact_name, - contact_email = study.contact_email, - title = study.title, - description = study.description, - publication_date = study.publication_date, - ref_doi = study.ref_doi) - .execute()) - - self.id_map['study'][study.study] = new_id - - def _move_datasets(self): - logging.info("Moving Datasets") - for dataset in old_db.Dataset.select(): - logging.info(" - Moving '{}'".format(dataset.short_name)) - try: - study_ref_id = self.id_map['study'][dataset.study.study] - except KeyError: - if not self.settings.dry_run: - raise - study_ref_id = -1 - try: - # short_name is unique, so we only really need to check that. - new_id = db.Dataset.get(study = study_ref_id, - short_name = dataset.short_name).id - except db.Dataset.DoesNotExist: - if self.settings.dry_run: - continue - new_id = (db.Dataset - .insert(study = study_ref_id, - short_name = dataset.short_name, - full_name = dataset.full_name, - browser_uri = dataset.browser_uri, - beacon_uri = dataset.beacon_uri, - avg_seq_depth = dataset.avg_seq_depth, - seq_type = dataset.seq_type, - seq_tech = dataset.seq_tech, - seq_center = dataset.seq_center, - dataset_size = dataset.dataset_size) - .execute()) - - self.id_map['dataset'][dataset.dataset] = new_id - - def _move_dataset_logos(self): - logging.info("Moving Dataset Logos") - for dataset_file in old_db.DatasetLogo.select(): - try: - dataset_ref_id = self.id_map['dataset'][dataset_file.dataset.dataset] - except KeyError: - if not self.settings.dry_run: - raise - dataset_ref_id = -1 - try: - db.DatasetLogo.get(dataset = dataset_ref_id, - mimetype = dataset_file.mimetype, - data = dataset_file.data) - except db.DatasetLogo.DoesNotExist: - if self.settings.dry_run: - continue - db.DatasetLogo.insert(dataset = dataset_ref_id, - mimetype = dataset_file.mimetype, - data = dataset_file.data).execute() - - def _move_dataset_versions(self): - logging.info("Moving Dataset Versions") - for dataset_version in old_db.DatasetVersion.select(): - try: - dataset_ref_id = self.id_map['dataset'][dataset_version.dataset.dataset] - dataset = db.Dataset.get(id = dataset_ref_id) - except KeyError: - if not self.settings.dry_run: - raise - dataset_ref_id = -1 - try: - new_id = db.DatasetVersion.get(dataset = dataset_ref_id, - version = dataset_version.version, - description = dataset_version.description, - terms = dataset_version.terms, - var_call_ref = dataset_version.var_call_ref, - available_from = dataset_version.available_from, - ref_doi = dataset_version.ref_doi, - data_contact_name = dataset_version.data_contact_name, - data_contact_link = dataset_version.data_contact_link).id - except db.DatasetVersion.DoesNotExist: - target_reference_id = self._select_reference_set(dataset.short_name) - if self.settings.dry_run: - continue - new_id = (db.DatasetVersion - .insert(dataset = dataset_ref_id, - reference_set = target_reference_id, - version = dataset_version.version, - description = dataset_version.description, - terms = dataset_version.terms, - var_call_ref = dataset_version.var_call_ref, - available_from = dataset_version.available_from, - ref_doi = dataset_version.ref_doi, - data_contact_name = dataset_version.data_contact_name, - data_contact_link = dataset_version.data_contact_link, - coverage_levels = [1,5,10,15,20,25,30,50,100] - ) - .execute()) - - self.id_map['dataset_version'][dataset_version.dataset_version] = new_id - - def _move_dataset_files(self): - logging.info("Moving Dataset Files") - for dataset_file in old_db.DatasetFile.select(): - logging.info(" - Moving '{}'".format(dataset_file.name)) - try: - dataset_version_ref_id = self.id_map['dataset_version'][dataset_file.dataset_version.dataset_version] - except KeyError: - if not self.settings.dry_run: - raise - dataset_version_ref_id = -1 - try: - new_id = db.DatasetFile.get(dataset_version = dataset_version_ref_id, - name = dataset_file.name, - uri = dataset_file.uri, - file_size = dataset_file.bytes).id - except db.DatasetFile.DoesNotExist: - if self.settings.dry_run: - continue - new_id = (db.DatasetFile - .insert(dataset_version = dataset_version_ref_id, - name = dataset_file.name, - uri = dataset_file.uri, - file_size = dataset_file.bytes).execute()) - - self.id_map['dataset_file'][dataset_file.dataset_file] = new_id - - def _move_sample_sets(self): - logging.info("Moving Sample Sets") - for sample_set in old_db.SampleSet.select(): - try: - dataset_ref_id = self.id_map['dataset'][sample_set.dataset.dataset] - collection_ref_id = self.id_map['collection'][sample_set.collection.collection] - except KeyError: - if not self.settings.dry_run: - raise - dataset_ref_id = -1 - collection_ref_id = -1 - try: - db.SampleSet.get(dataset = dataset_ref_id, - collection = collection_ref_id, - sample_size = sample_set.sample_size, - phenotype = sample_set.phenotype) - except db.SampleSet.DoesNotExist: - if self.settings.dry_run: - continue - db.SampleSet.insert(dataset = dataset_ref_id, - collection = collection_ref_id, - sample_size = sample_set.sample_size, - phenotype = sample_set.phenotype).execute() - - def _move_database(self): - self._move_collections() - self._move_studies() - self._move_datasets() - self._move_dataset_logos() - self._move_dataset_versions() - self._move_sample_sets() - self._move_dataset_files() - - def _move_users(self): - logging.info("Moving Users") - for user in old_db.User.select(): - try: - new_id = (db.User - .get(name = user.name, - email = user.email, - identity = user.identity, - identity_type = user.identity_type, - affiliation = user.affiliation, - country = user.country).id) - except db.User.DoesNotExist: - if self.settings.dry_run: - continue - new_id = (db.User - .insert(name = user.name, - email = user.email, - identity = user.identity, - identity_type = user.identity_type, - affiliation = user.affiliation, - country = user.country).execute()) - - self.id_map['user'][user.user] = new_id - - def _move_sftp_users(self): - logging.info("Moving SFTP Users") - for user in old_db.SFTPUser.select(): - try: - user_ref_id = self.id_map['user'][user.user.user] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - try: - # user_uid is unique, so we rely on that - db.SFTPUser.get(user = user_ref_id, - user_uid = user.user_uid) - except db.SFTPUser.DoesNotExist: - if self.settings.dry_run: - continue - db.SFTPUser.insert(user = user_ref_id, - user_uid = user.user_uid, - user_name = user.user_name, - password_hash = user.password_hash, - account_expires = user.account_expires).execute() - - def _move_user_access_logs(self): - logging.info("Moving User Access Logs") - for log in old_db.UserAccessLog.select(): - try: - user_ref_id = self.id_map['user'][log.user.user] - dataset_ref_id = self.id_map['dataset'][log.dataset.dataset] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - dataset_ref_id = -1 - try: - db.UserAccessLog.get(user = user_ref_id, - dataset = dataset_ref_id, - action = log.action, - ts = log.ts) - except db.UserAccessLog.DoesNotExist: - if self.settings.dry_run: - continue - db.UserAccessLog.insert(user = user_ref_id, - dataset = dataset_ref_id, - action = log.action, - ts = log.ts).execute() - - def _move_user_consent_logs(self): - logging.info("Moving User Consent Logs") - for log in old_db.UserConsentLog.select(): - try: - user_ref_id = self.id_map['user'][log.user.user] - version_ref_id = self.id_map['dataset_version'][log.dataset_version.dataset_version] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - version_ref_id = -1 - try: - db.UserConsentLog.get(user = user_ref_id, - dataset_version = version_ref_id, - ts = log.ts) - except db.UserConsentLog.DoesNotExist: - if self.settings.dry_run: - continue - db.UserConsentLog.insert(user = user_ref_id, - dataset_version = version_ref_id, - ts = log.ts).execute() - - def _move_user_download_logs(self): - logging.info("Moving User Download Logs") - for log in old_db.UserDownloadLog.select(): - try: - user_ref_id = self.id_map['user'][log.user.user] - file_ref_id = self.id_map['dataset_file'][log.dataset_file.dataset_file] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - file_ref_id = -1 - try: - db.UserDownloadLog.get(user = user_ref_id, - dataset_file = file_ref_id, - ts = log.ts) - except db.UserDownloadLog.DoesNotExist: - if self.settings.dry_run: - continue - db.UserDownloadLog.insert(user = user_ref_id, - dataset_file = file_ref_id, - ts = log.ts).execute() - - def _move_dataset_access(self): - logging.info("Moving Dataset Access Records") - for access in old_db.DatasetAccess.select(): - try: - user_ref_id = self.id_map['user'][access.user.user] - dataset_ref_id = self.id_map['dataset'][access.dataset.dataset] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - dataset_ref_id = -1 - try: - db.DatasetAccess.get(dataset = dataset_ref_id, - user = user_ref_id, - wants_newsletter = access.wants_newsletter, - is_admin = access.is_admin) - except db.DatasetAccess.DoesNotExist: - if self.settings.dry_run: - continue - db.DatasetAccess.insert(dataset = dataset_ref_id, - user = user_ref_id, - wants_newsletter = access.wants_newsletter, - is_admin = access.is_admin).execute() - - def _move_linkhashes(self): - logging.info("Moving Linkhashes") - for linkhash in old_db.Linkhash.select(): - try: - user_ref_id = self.id_map['user'][linkhash.user.user] - version_ref_id = self.id_map['dataset_version'][linkhash.dataset_version.dataset_version] - except KeyError: - if not self.settings.dry_run: - raise - user_ref_id = -1 - version_ref_id = -1 - try: - db.Linkhash.get(dataset_version = version_ref_id, - user = user_ref_id, - hash = linkhash.hash, - expires_on = linkhash.expires_on) - except db.Linkhash.DoesNotExist: - if self.settings.dry_run: - continue - db.Linkhash.insert(dataset_version = version_ref_id, - user = user_ref_id, - hash = linkhash.hash, - expires_on = linkhash.expires_on).execute() - - def _move_userbase(self): - self._move_users() - self._move_sftp_users() - self._move_user_access_logs() - self._move_user_consent_logs() - self._move_user_download_logs() - self._move_dataset_access() - self._move_linkhashes() - - def prepare_data(self): - """ - Connects to the old and new databases. - """ - logging.info("Checking connection to old database") - try: - old_db.Collection.get() - except OperationalError: - logging.error("Could not connect to old database") - sys.exit(1) - logging.info("Checking connection to new database") - try: - db.ReferenceSet.get() - for reference_set in db.ReferenceSet.select(): - self.reference_sets += [reference_set] - except db.ReferenceSet.DoesNotExist: - logging.error(("Connection works, but no reference sets are available." - "use '--add_reference' to add a new reference set and" - "Then use this tool again.")) - sys.exit(1) - except (OperationalError, InterfaceError): - logging.error("Could not connect to new database") - sys.exit(1) - - def start_import(self): - start = time.time() - self._move_database() - self._move_userbase() - - logging.info("Moved data in {}".format(self._time_since(start))) diff --git a/scripts/importer/importer.py b/scripts/importer/importer.py index e489efae6..28717925b 100755 --- a/scripts/importer/importer.py +++ b/scripts/importer/importer.py @@ -12,7 +12,6 @@ """ from data_importer.reference_set_importer import ReferenceSetImporter -from data_importer.old_db_importer import OldDbImporter from data_importer.raw_data_importer import RawDataImporter if __name__ == '__main__': @@ -77,9 +76,6 @@ help="Insert new reference set.") PARSER.add_argument("--add_raw_data", action="store_true", help="Adds a Coverage and Variants to the database.") - PARSER.add_argument("--move_studies", action="store_true", - help=("Moves studies and datasets from an old database " - "to a new one.")) PARSER.add_argument("--dry_run", action="store_true", help="Do not insert anything into the database") @@ -121,11 +117,6 @@ IMPORTER.count_entries() IMPORTER.start_import() - if ARGS.move_studies: - IMPORTER = OldDbImporter(ARGS) - IMPORTER.prepare_data() - IMPORTER.start_import() - if ARGS.add_raw_data: logging.info("Adding raw data %s", "(dry run)" if ARGS.dry_run else '') IMPORTER = RawDataImporter(ARGS) From 3f261d38ffb76ddf8838fb376157d5e28fa59bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Wed, 14 Aug 2019 07:15:57 +0200 Subject: [PATCH 2/2] Remove mysql from settings. --- backend/settings.py | 7 ------- settings_sample.json | 6 ------ 2 files changed, 13 deletions(-) diff --git a/backend/settings.py b/backend/settings.py index c6543e4b5..c2e255ca0 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -34,13 +34,6 @@ psql_user = json_settings["postgresUser"] psql_pass = json_settings["postgresPass"] -# MySql settings -mysql_host = json_settings["mysqlHost"] -mysql_schema = json_settings["mysqlSchema"] -mysql_user = json_settings["mysqlUser"] -mysql_passwd = json_settings["mysqlPasswd"] -mysql_port = json_settings["mysqlPort"] - # e-mail config mail_server = json_settings["mailServer"] from_address = json_settings["fromAddress"] diff --git a/settings_sample.json b/settings_sample.json index a530f12f0..11cfbe695 100644 --- a/settings_sample.json +++ b/settings_sample.json @@ -1,12 +1,6 @@ { "cookieSecret" : "Something random for tornado to sign cookies with", - "mysqlHost" : "127.0.0.1", - "mysqlPasswd" : "password", - "mysqlSchema" : "swefreq", - "mysqlUser" : "swefreq", - "mysqlPort" : 3306, - "postgresHost" : "postgres host", "postgresPort" : 5432, "postgresUser" : "postgres",