From 481b6e1aed0e0b7cb22ead7149c30879db0b39c7 Mon Sep 17 00:00:00 2001 From: Mikko Nieminen Date: Mon, 12 Dec 2022 12:38:29 +0100 Subject: [PATCH] add syncstudytables management command (#1509) --- CHANGELOG.rst | 1 + docs_manual/source/admin_commands.rst | 5 + .../management/commands/syncstudytables.py | 131 ++++++++++++++++++ samplesheets/plugins.py | 3 + samplesheets/tests/test_commands.py | 110 ++++++++++++++- 5 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 samplesheets/management/commands/syncstudytables.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6363c6bf..2e6a0e54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,7 @@ Added - **Samplesheets** - Mac keyboard shortcut support for multi-cell copying (#1531) - Study render table caching (#1509) + - ``syncstudytables`` management command (#1509) Changed ------- diff --git a/docs_manual/source/admin_commands.rst b/docs_manual/source/admin_commands.rst index 70127d30..c70875dd 100644 --- a/docs_manual/source/admin_commands.rst +++ b/docs_manual/source/admin_commands.rst @@ -56,3 +56,8 @@ operations regarding sample sheets, landing zones, iRODS data and ontologies. Find orphans in iRODS project collections. ``syncnames`` Synchronize alternative names for sample sheet material search. +``syncstudytables`` + Build study render tables in cache for all study tables. These will be + automatically built when accessing sample sheets if existing cache is not + up-to-date, but this can be used to e.g. regenerate the cache if something + has been changed in study table rendering. diff --git a/samplesheets/management/commands/syncstudytables.py b/samplesheets/management/commands/syncstudytables.py new file mode 100644 index 00000000..bd29f1cb --- /dev/null +++ b/samplesheets/management/commands/syncstudytables.py @@ -0,0 +1,131 @@ +"""Syncstudytables management command""" + +import sys + +from django.core.management.base import BaseCommand + +# Projectroles dependency +from projectroles.management.logging import ManagementCommandLogger +from projectroles.models import Project, SODAR_CONSTANTS +from projectroles.plugins import get_backend_api + +from samplesheets.models import Investigation +from samplesheets.rendering import ( + SampleSheetTableBuilder, + STUDY_TABLE_CACHE_ITEM, +) + + +logger = ManagementCommandLogger(__name__) +table_builder = SampleSheetTableBuilder() + + +# SODAR constants +PROJECT_TYPE_PROJECT = SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'] +# Local constants +APP_NAME = 'samplesheets' + + +class Command(BaseCommand): + help = 'Syncs study render tables in sodarcache for optimized rendering' + + @classmethod + def _get_log_project(cls, project): + """Return logging-friendly project title""" + return '"{}" ({})'.format(project.title, project.sodar_uuid) + + @classmethod + def _get_log_study(cls, study): + """Return logging-friendly project title""" + return '"{}" ({})'.format(study.get_title(), study.sodar_uuid) + + def add_arguments(self, parser): + parser.add_argument( + '-p', + '--project', + metavar='UUID', + type=str, + help='Limit sync to a project', + ) + + def handle(self, *args, **options): + cache_backend = get_backend_api('sodar_cache') + if not cache_backend: + logger.error('Sodarcache not enabled, exiting') + sys.exit(1) + + q_kwargs = {'type': PROJECT_TYPE_PROJECT} + if options.get('project'): + q_kwargs['sodar_uuid'] = options['project'] + projects = Project.objects.filter(**q_kwargs).order_by('full_title') + if not projects: + logger.info( + 'No project{} found'.format( + 's' if not options.get('project') else '' + ) + ) + return + if options.get('project'): + project = projects.first() + logger.info( + 'Limiting sync to project {}'.format( + self._get_log_project(project) + ) + ) + + for project in projects: + study_count = 0 + try: + investigation = Investigation.objects.get( + project=project, active=True + ) + except Investigation.DoesNotExist: + logger.debug( + 'No investigation found, skipping for project {}'.format( + self._get_log_project(project) + ) + ) + continue + logger.debug( + 'Building study render tables for project {}..'.format( + self._get_log_project(project) + ) + ) + for study in investigation.studies.all(): + try: + study_tables = table_builder.build_study_tables( + study, use_config=True + ) + except Exception as ex: + logger.error( + 'Error building tables for study {}: {}'.format( + self._get_log_study(study), ex + ) + ) + continue + item_name = STUDY_TABLE_CACHE_ITEM.format( + study=study.sodar_uuid + ) + try: + cache_backend.set_cache_item( + app_name=APP_NAME, + name=item_name, + data=study_tables, + project=project, + ) + logger.info('Set cache item "{}"'.format(item_name)) + study_count += 1 + except Exception as ex: + logger.error( + 'Failed to set cache item "{}": {}'.format( + item_name, ex + ) + ) + logger.info( + 'Built {} study table{} for project {}'.format( + study_count, + 's' if study_count != 1 else '', + self._get_log_project(project), + ) + ) + logger.info('Study table cache sync done') diff --git a/samplesheets/plugins.py b/samplesheets/plugins.py index 1dba7505..91fb7197 100644 --- a/samplesheets/plugins.py +++ b/samplesheets/plugins.py @@ -625,6 +625,9 @@ def update_cache(self, name=None, project=None, user=None): :raise: Exception if required backends (sodar_cache and omics_irods) are not found. """ + # NOTE: This will not sync cached study render tables, they are created + # synchronously upon access if not up-to-date. To sync the cache + # for all study tables, use the syncstudytables command. cache_backend = get_backend_api('sodar_cache') irods_backend = get_backend_api('omics_irods') if not cache_backend or not irods_backend: diff --git a/samplesheets/tests/test_commands.py b/samplesheets/tests/test_commands.py index 1f82be0d..2a4ba126 100644 --- a/samplesheets/tests/test_commands.py +++ b/samplesheets/tests/test_commands.py @@ -1,14 +1,21 @@ """Tests for management commands in the samplesheets app""" +import uuid + from django.core.management import call_command from test_plus.test import TestCase # Projectroles dependency from projectroles.models import Role, SODAR_CONSTANTS +from projectroles.plugins import get_backend_api from projectroles.tests.test_models import ProjectMixin, RoleAssignmentMixin +# Sodarcache dependency +from sodarcache.models import JSONCacheItem + from samplesheets.models import GenericMaterial +from samplesheets.rendering import STUDY_TABLE_CACHE_ITEM from samplesheets.tests.test_io import ( SampleSheetIOMixin, SHEET_DIR, @@ -18,6 +25,7 @@ # Local constants +APP_NAME = 'samplesheets' SHEET_PATH = SHEET_DIR + 'i_small.zip' SHEET_PATH_INSERTED = SHEET_DIR_SPECIAL + 'i_small_insert.zip' SHEET_PATH_ALT = SHEET_DIR + 'i_small2.zip' @@ -32,7 +40,6 @@ class TestSyncnamesCommand( def setUp(self): # Make owner user self.user_owner = self.make_user('owner') - # Init project, role and assignment self.project = self._make_project( 'TestProject', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None @@ -40,23 +47,116 @@ def setUp(self): self.role_owner = Role.objects.get_or_create( name=SODAR_CONSTANTS['PROJECT_ROLE_OWNER'] )[0] - self.assignment_owner = self._make_assignment( + self.owner_as = self._make_assignment( self.project, self.user_owner, self.role_owner ) - # Import investigation self.investigation = self.import_isa_from_file(SHEET_PATH, self.project) self.study = self.investigation.studies.first() - # Clear alt names from imported materials for m in GenericMaterial.objects.all(): m.alt_names = ALT_NAMES_INVALID m.save() def test_command(self): - """Test syncnames command""" + """Test syncnames""" for m in GenericMaterial.objects.all(): self.assertEqual(m.alt_names, ALT_NAMES_INVALID) call_command('syncnames') for m in GenericMaterial.objects.all(): self.assertEqual(m.alt_names, get_alt_names(m.name)) + + +class TestSyncstudytablesCommand( + ProjectMixin, RoleAssignmentMixin, SampleSheetIOMixin, TestCase +): + """Tests for the syncstudytables command""" + + def setUp(self): + # Init user and role + self.user_owner = self.make_user('owner') + self.role_owner = Role.objects.get_or_create( + name=SODAR_CONSTANTS['PROJECT_ROLE_OWNER'] + )[0] + + # Init project + self.project = self._make_project( + 'TestProject', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None + ) + self.owner_as = self._make_assignment( + self.project, self.user_owner, self.role_owner + ) + self.investigation = self.import_isa_from_file(SHEET_PATH, self.project) + self.study = self.investigation.studies.first() + + # Init second project + self.project2 = self._make_project( + 'TestProject2', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None + ) + self.owner_as2 = self._make_assignment( + self.project2, self.user_owner, self.role_owner + ) + self.investigation2 = self.import_isa_from_file( + SHEET_PATH_ALT, self.project2 + ) + self.study2 = self.investigation2.studies.first() + + # Init helpers + self.cache_name = STUDY_TABLE_CACHE_ITEM.format( + study=self.study.sodar_uuid + ) + self.cache_name2 = STUDY_TABLE_CACHE_ITEM.format( + study=self.study2.sodar_uuid + ) + self.cache_args = [APP_NAME, self.cache_name, self.project] + self.cache_args2 = [APP_NAME, self.cache_name2, self.project2] + self.cache_backend = get_backend_api('sodar_cache') + + def test_sync_all(self): + """Test syncstudytables for all projects""" + self.assertEqual(JSONCacheItem.objects.count(), 0) + call_command('syncstudytables') + self.assertEqual(JSONCacheItem.objects.count(), 2) + cache_item = self.cache_backend.get_cache_item(*self.cache_args) + self.assertIsInstance(cache_item, JSONCacheItem) + self.assertNotEqual(cache_item.data, {}) + cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2) + self.assertIsInstance(cache_item2, JSONCacheItem) + self.assertNotEqual(cache_item2.data, {}) + + def test_sync_all_existing(self): + """Test syncstudytables for all projects with existing items""" + self.cache_backend.set_cache_item( + APP_NAME, self.cache_name, {}, project=self.project + ) + self.cache_backend.set_cache_item( + APP_NAME, self.cache_name2, {}, project=self.project2 + ) + + self.assertEqual(JSONCacheItem.objects.count(), 2) + call_command('syncstudytables') + self.assertEqual(JSONCacheItem.objects.count(), 2) + cache_item = self.cache_backend.get_cache_item(*self.cache_args) + self.assertIsInstance(cache_item, JSONCacheItem) + self.assertNotEqual(cache_item.data, {}) + cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2) + self.assertIsInstance(cache_item2, JSONCacheItem) + self.assertNotEqual(cache_item2.data, {}) + + def test_sync_limit(self): + """Test syncstudytables limiting for single project""" + self.assertEqual(JSONCacheItem.objects.count(), 0) + call_command('syncstudytables', project=str(self.project.sodar_uuid)) + self.assertEqual(JSONCacheItem.objects.count(), 1) + cache_item = self.cache_backend.get_cache_item(*self.cache_args) + self.assertIsInstance(cache_item, JSONCacheItem) + self.assertNotEqual(cache_item.data, {}) + cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2) + self.assertIsNone(cache_item2) + + def test_sync_limit_invalid_project(self): + """Test syncstudytables limiting for non-existent project""" + self.assertEqual(JSONCacheItem.objects.count(), 0) + invalid_uuid = uuid.uuid4() + call_command('syncstudytables', project=str(invalid_uuid)) + self.assertEqual(JSONCacheItem.objects.count(), 0)