Skip to content

Commit

Permalink
Merge 2f8005f into 4bf8bb7
Browse files Browse the repository at this point in the history
  • Loading branch information
roefem committed Aug 13, 2020
2 parents 4bf8bb7 + 2f8005f commit 87a3c57
Show file tree
Hide file tree
Showing 13 changed files with 248 additions and 48 deletions.
5 changes: 3 additions & 2 deletions atramhasis/alembic/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def run_migrations_offline():
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(url=url)
context.configure(url=url, render_as_batch=True)

with context.begin_transaction():
context.run_migrations()
Expand All @@ -82,7 +82,8 @@ def run_migrations_online():
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=metadata
target_metadata=metadata,
render_as_batch=True
)

try:
Expand Down
44 changes: 22 additions & 22 deletions atramhasis/alembic/versions/3924a7ad2f8_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ def upgrade():
sa.ForeignKeyConstraint(['language_id'], ['language.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('op.f("ix_label_labeltype_id")', 'label', ['labeltype_id'], unique=False)
op.create_index('op.f("ix_label_language_id")', 'label', ['language_id'], unique=False)
op.create_index('ix_label_labeltype_id', 'label', ['labeltype_id'], unique=False)
op.create_index('ix_label_language_id', 'label', ['language_id'], unique=False)
op.create_table('note',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('note', sa.Text(), nullable=False),
Expand All @@ -56,8 +56,8 @@ def upgrade():
sa.ForeignKeyConstraint(['notetype_id'], ['notetype.name'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('op.f("ix_note_language_id")', 'note', ['language_id'], unique=False)
op.create_index('op.f("ix_note_notetype_id")', 'note', ['notetype_id'], unique=False)
op.create_index('ix_note_language_id', 'note', ['language_id'], unique=False)
op.create_index('ix_note_notetype_id', 'note', ['notetype_id'], unique=False)
op.create_table('concept',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('type', sa.String(length=30), nullable=True),
Expand All @@ -67,8 +67,8 @@ def upgrade():
sa.ForeignKeyConstraint(['conceptscheme_id'], ['conceptscheme.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('op.f("ix_concept_concept_id")', 'concept', ['concept_id'], unique=False)
op.create_index('op.f("ix_concept_conceptscheme_id")', 'concept', ['conceptscheme_id'], unique=False)
op.create_index('ix_concept_concept_id', 'concept', ['concept_id'], unique=False)
op.create_index('ix_concept_conceptscheme_id', 'concept', ['conceptscheme_id'], unique=False)
op.create_table('collection_concept',
sa.Column('collection_id', sa.Integer(), nullable=False),
sa.Column('concept_id', sa.Integer(), nullable=False),
Expand Down Expand Up @@ -101,11 +101,11 @@ def upgrade():
sa.ForeignKeyConstraint(['conceptscheme_id'], ['conceptscheme.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('op.f("ix_visitation_concept_id")', 'visitation', ['concept_id'], unique=False)
op.create_index('op.f("ix_visitation_conceptscheme_id")', 'visitation', ['conceptscheme_id'], unique=False)
op.create_index('op.f("ix_visitation_depth")', 'visitation', ['depth'], unique=False)
op.create_index('op.f("ix_visitation_lft")', 'visitation', ['lft'], unique=False)
op.create_index('op.f("ix_visitation_rght")', 'visitation', ['rght'], unique=False)
op.create_index('ix_visitation_concept_id', 'visitation', ['concept_id'], unique=False)
op.create_index('ix_visitation_conceptscheme_id', 'visitation', ['conceptscheme_id'], unique=False)
op.create_index('ix_visitation_depth', 'visitation', ['depth'], unique=False)
op.create_index('ix_visitation_lft', 'visitation', ['lft'], unique=False)
op.create_index('ix_visitation_rght', 'visitation', ['rght'], unique=False)
op.create_table('concept_note',
sa.Column('concept_id', sa.Integer(), nullable=False),
sa.Column('note_id', sa.Integer(), nullable=False),
Expand Down Expand Up @@ -151,23 +151,23 @@ def downgrade():
op.drop_table('concept_hierarchy_concept')
op.drop_table('concept_related_concept')
op.drop_table('concept_note')
op.drop_index('op.f("ix_visitation_rght")', table_name='visitation')
op.drop_index('op.f("ix_visitation_lft")', table_name='visitation')
op.drop_index('op.f("ix_visitation_depth")', table_name='visitation')
op.drop_index('op.f("ix_visitation_conceptscheme_id")', table_name='visitation')
op.drop_index('op.f("ix_visitation_concept_id")', table_name='visitation')
op.drop_index('ix_visitation_rght', table_name='visitation')
op.drop_index('ix_visitation_lft', table_name='visitation')
op.drop_index('ix_visitation_depth', table_name='visitation')
op.drop_index('ix_visitation_conceptscheme_id', table_name='visitation')
op.drop_index('ix_visitation_concept_id', table_name='visitation')
op.drop_table('visitation')
op.drop_table('conceptscheme_label')
op.drop_table('conceptscheme_note')
op.drop_table('collection_concept')
op.drop_index('op.f("ix_concept_conceptscheme_id")', table_name='concept')
op.drop_index('op.f("ix_concept_concept_id")', table_name='concept')
op.drop_index('ix_concept_conceptscheme_id', table_name='concept')
op.drop_index('ix_concept_concept_id', table_name='concept')
op.drop_table('concept')
op.drop_index('op.f("ix_note_notetype_id")', table_name='note')
op.drop_index('op.f("ix_note_language_id")', table_name='note')
op.drop_index('ix_note_notetype_id', table_name='note')
op.drop_index('ix_note_language_id', table_name='note')
op.drop_table('note')
op.drop_index('op.f("ix_label_language_id")', table_name='label')
op.drop_index('op.f("ix_label_labeltype_id")', table_name='label')
op.drop_index('ix_label_language_id', table_name='label')
op.drop_index('ix_label_labeltype_id', table_name='label')
op.drop_table('label')
op.drop_table('notetype')
op.drop_table('conceptscheme')
Expand Down
3 changes: 2 additions & 1 deletion atramhasis/alembic/versions/3ac8aca026fd_note_markup.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ def upgrade():


def downgrade():
op.drop_column('note', 'markup')
with op.batch_alter_table("note") as batch_op:
batch_op.drop_column('markup')
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ def upgrade():


def downgrade():
op.drop_column('source', 'markup')
with op.batch_alter_table("source") as batch_op:
batch_op.drop_column('markup')
63 changes: 63 additions & 0 deletions atramhasis/alembic/versions/3e9675b35dfc_cascades.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""cascades
Revision ID: 3e9675b35dfc
Revises: cb568ec81000
Create Date: 2020-08-12 14:48:35.592316
"""

# revision identifiers, used by Alembic.
revision = '3e9675b35dfc'
down_revision = 'cb568ec81000'

from alembic import op

is_sqlite = op.get_bind().dialect.name == 'sqlite'
convention = ({"fk": "FK_%(table_name)s_%(referred_table_name)s"} if is_sqlite else
{"fk": "%(table_name)s_%(column_0_name)s_fkey"})


def upgrade():

args = [
('conceptscheme_source', 'conceptscheme', 'conceptscheme_id', 'id'),
('conceptscheme_source', 'source', 'source_id', 'id'),
('concept_source', 'concept', 'concept_id', 'id'),
('concept_source', 'source', 'source_id', 'id'),
('conceptscheme_language', 'conceptscheme', 'conceptscheme_id', 'id'),
('match', 'concept', 'concept_id', 'id'),
('label', 'language', 'language_id', 'id'),
('note', 'language', 'language_id', 'id'),
('collection_concept', 'concept', 'concept_id', 'id'),
('collection_concept', 'concept', 'collection_id', 'id'),
('conceptscheme_note', 'conceptscheme', 'conceptscheme_id', 'id'),
('conceptscheme_label', 'conceptscheme', 'conceptscheme_id', 'id'),
('visitation', 'concept', 'concept_id', 'id'),
('visitation', 'conceptscheme', 'conceptscheme_id', 'id'),
('concept_note', 'concept', 'concept_id', 'id'),
('concept_note', 'note', 'note_id', 'id'),
('concept_related_concept', 'concept', 'concept_id_from', 'id'),
('concept_related_concept', 'concept', 'concept_id_to', 'id'),
('concept_hierarchy_concept', 'concept', 'concept_id_narrower', 'id'),
('concept_hierarchy_concept', 'concept', 'concept_id_broader', 'id'),
('concept_hierarchy_collection', 'concept', 'concept_id_broader', 'id'),
('concept_hierarchy_collection', 'concept', 'collection_id_narrower', 'id'),
('concept_label', 'concept', 'concept_id', 'id'),
('concept_label', 'label', 'label_id', 'id'),
]

for source_table, referent_table, local_col, remote_col in args:
with op.batch_alter_table(source_table, naming_convention=convention) as batch_op:
constraint_name = convention['fk'] % {
'table_name': source_table,
'referred_table_name': referent_table,
'column_0_name': local_col
}
batch_op.drop_constraint(constraint_name)
batch_op.create_foreign_key(
constraint_name, referent_table,
[local_col], [remote_col], ondelete='cascade')


def downgrade():
pass
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ def upgrade():


def downgrade():
op.drop_column('concept', 'infer_concept_relations')
with op.batch_alter_table("concept") as batch_op:
batch_op.drop_column('infer_concept_relations')
2 changes: 1 addition & 1 deletion atramhasis/data/datamanagers.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def get_most_popular_concepts_for_conceptscheme(self, conceptscheme_id, max=5, p
ConceptVisitLog.concept_id,
func.count(ConceptVisitLog.concept_id).label('count')
).filter(
and_(ConceptVisitLog.conceptscheme_id == conceptscheme_id,
and_(ConceptVisitLog.conceptscheme_id == str(conceptscheme_id),
ConceptVisitLog.visited_at >= start_date)
).group_by(
ConceptVisitLog.concept_id
Expand Down
95 changes: 95 additions & 0 deletions atramhasis/scripts/delete_scheme.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
import argparse
import logging

from pyramid.paster import get_appsettings
from pyramid.paster import setup_logging
from pytz import timezone
from sqlalchemy import engine_from_config

try:
from builtins import input
except ImportError:
input = raw_input

timezone_brussels = timezone('Europe/Brussels')
log = logging.getLogger(__name__)


def delete_scheme(settings, scheme_id):
engine = engine_from_config(settings, 'sqlalchemy.')
with engine.connect() as con:
concept_ids = con.execute(
'select id from concept where conceptscheme_id={}'.format(scheme_id)
)
for row in concept_ids:
concept_id = row[0]
delete_concept(concept_id, con)

con.execute('delete from note where note.id in'
'(select note_id from conceptscheme_note '
'where conceptscheme_id={})'.format(scheme_id))
con.execute('delete from source where source.id in'
'(select source_id from conceptscheme_source '
'where conceptscheme_id={})'.format(scheme_id))
con.execute('delete from label where label.id in'
'(select label_id from conceptscheme_label '
'where conceptscheme_id={})'.format(scheme_id))
con.execute('delete from conceptscheme where id = {}'.format(scheme_id))


def delete_concept(concept_id, con):
con.execute('delete from note where note.id in'
'(select note_id from concept_note where concept_id={})'
.format(concept_id))
con.execute('delete from source where source.id in'
'(select source_id from concept_source '
'where concept_id={})'.format(concept_id))
con.execute('delete from label where label.id in'
'(select label_id from concept_label '
'where concept_id={})'.format(concept_id))
delete_child_concepts(concept_id, con)
con.execute('delete from concept where id = {}'.format(concept_id))


def delete_child_concepts(concept_id, con):
select_children = ('select collection_id_narrower as child '
'from concept_hierarchy_collection '
'where concept_id_broader = {0} '
'union '
'select concept_id_narrower as child '
'from concept_hierarchy_concept '
'where concept_id_broader = {0};'.format(concept_id))
children = con.execute(select_children)
for row in children:
child_id = row[0]
delete_concept(child_id, con)


def main():
parser = argparse.ArgumentParser(
description="Delete a conceptscheme. ",
usage="remove_schema development.ini --id=1")
parser.add_argument('settings_file',
help="<The location of the settings file>#<app-name>")
parser.add_argument("--id", type=int, required=True,
help="the conceptscheme id")
parser.add_argument("--no-input", action='store_true',
help="Don't stop script for user input")
args, _ = parser.parse_known_args()

config_uri = args.settings_file
setup_logging(config_uri)
settings = get_appsettings(config_uri)
id = args.id
print(
"The conceptscheme with id {} will be deleted".format(id)
)
if not args.no_input:
input("Press [Enter] to continue.")

delete_scheme(settings, id)


if __name__ == '__main__': # pragma: no cover
main()
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ def run(self):
dump_rdf = atramhasis.scripts.dump_rdf:main
generate_ldf_config = atramhasis.scripts.generate_ldf_config:main
sitemap_generator = atramhasis.scripts.sitemap_generator:main
delete_scheme = atramhasis.scripts.delete_scheme:main
[pyramid.scaffold]
atramhasis_scaffold=atramhasis.scaffolds:AtramhasisTemplate
atramhasis_demo=atramhasis.scaffolds:AtramhasisDemoTemplate
Expand Down
24 changes: 8 additions & 16 deletions tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,23 +59,15 @@ def setup_db(guarantee_empty=False):

def _reset_db():
engine = engine_from_config(SETTINGS, prefix='sqlalchemy.')
if engine.name == 'sqlite':
# Can't alembic downgrade sqlite because it can't do
# ALTER TABLE X DROP COLUMN
try:
os.remove(engine.url.database)
except OSError:
pass
elif engine.name == 'postgresql':
try:
engine.execute("DELETE FROM concept_note")
engine.execute("DELETE FROM note")
engine.execute("DELETE FROM concept_label")
engine.execute("DELETE FROM label")
command.downgrade(ALEMBIC_CONFIG, 'base')
except ProgrammingError:
"""The tables may not exist if it's first time."""
try:
engine.execute("DELETE FROM concept_note")
engine.execute("DELETE FROM note")
engine.execute("DELETE FROM concept_label")
engine.execute("DELETE FROM label")
command.downgrade(ALEMBIC_CONFIG, 'base')
except (ProgrammingError, OperationalError):
"""The tables may not exist if it's first time."""
command.downgrade(ALEMBIC_CONFIG, 'base')
engine.dispose()


Expand Down
2 changes: 1 addition & 1 deletion tests/conf_test.ini
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jinja2.filters =
label_sort = atramhasis.utils.label_sort
available_languages = en nl it
sqlalchemy.url = sqlite:///test.db
# sqlalchemy.url = postgresql://postgres:postgres@localhost:5432/atramhasis_test
;sqlalchemy.url = postgresql://postgres:postgres@localhost:5432/atramhasis_test


atramhasis.session_factory.secret = testing123
Expand Down
46 changes: 46 additions & 0 deletions tests/test_script_delete_scheme.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import os

from paste.deploy import appconfig
from skosprovider_sqlalchemy.models import Collection
from skosprovider_sqlalchemy.models import Concept
from skosprovider_sqlalchemy.models import ConceptScheme
from skosprovider_sqlalchemy.models import Label
from skosprovider_sqlalchemy.models import LabelType
from skosprovider_sqlalchemy.models import Language
from skosprovider_sqlalchemy.models import Note
from skosprovider_sqlalchemy.models import NoteType
from skosprovider_sqlalchemy.models import Source
from skosprovider_sqlalchemy.models import Visitation

from atramhasis.scripts import delete_scheme
from tests import DbTest
from tests import db_session
from tests import fill_db
from tests import setup_db

TEST_DIR = os.path.dirname(__file__)
settings = appconfig('config:' + os.path.join(TEST_DIR, 'conf_test.ini'))


def setUpModule():
setup_db()
fill_db()


class DeleteSchemeTest(DbTest):

def test_delete(self):
with db_session() as session:
for id in range(1, 10):
delete_scheme.delete_scheme(settings, id)
assert len(session.query(ConceptScheme).all()) == 0
assert len(session.query(Concept).all()) == 0
assert len(session.query(Collection).all()) == 0
assert len(session.query(Note).all()) == 0
assert len(session.query(Source).all()) == 0
assert len(session.query(Visitation).all()) == 0
assert len(session.query(Label).all()) == 0
assert len(session.query(LabelType).all()) != 0
assert len(session.query(NoteType).all()) != 0
assert len(session.query(Language).all()) != 0

0 comments on commit 87a3c57

Please sign in to comment.