Add sync_isilon_allocations management command

coldfront/config/plugins/isilon.py

@@ -9,6 +9,7 @@
 ISILON_NFS_ROOT_CLIENTS = ENV.str('ISILON_NFS_ROOT_CLIENTS', '')
 ISILON_NFS_FASSE_CLIENTS = ENV.str('ISILON_NFS_FASSE_CLIENTS', '')
 ISILON_NFS_CANNON_CLIENTS = ENV.str('ISILON_NFS_CANNON_CLIENTS', '')
+ISILON_PATH_IGNORE = ENV.list('ISILON_PATH_IGNORE', default=[])
 LOGGING['handlers']['isilon'] = {
     'class': 'logging.handlers.TimedRotatingFileHandler',
     'filename': 'logs/isilon.log',

coldfront/plugins/fasrc/utils.py

@@ -204,9 +204,8 @@ def pull_quota_data(self):
         ----------
         volumes : List of volume names to collect. Optional, default None.
         """
-        logger = logging.getLogger('coldfront.import_quotas')
         query = ATTAllocationQuery()
-        query.produce_query_statement('isilon', volumes=self.volumes)
+        # query.produce_query_statement('isilon', volumes=self.volumes)
         query.produce_query_statement('quota', volumes=self.volumes)
         query.produce_query_statement('volume', volumes=self.volumes)
         query.produce_query_statement('tapeallocation')
@@ -216,7 +215,6 @@ def pull_quota_data(self):
 
 
 def matched_dict_processing(allocation, data_dicts, paired_allocs, log_message):
-    logger = logging.getLogger('coldfront.import_quotas')
     if len(data_dicts) == 1:
         logger.debug(log_message)
         paired_allocs[allocation] = data_dicts[0]
@@ -228,7 +226,6 @@ def matched_dict_processing(allocation, data_dicts, paired_allocs, log_message):
 
 def pair_allocations_data(project, quota_dicts):
     """pair allocations with usage dicts"""
-    logger = logging.getLogger('coldfront.import_quotas')
     allocs = project.allocation_set.filter(
         status__name__in=['Active','Pending Deactivation'],
         resources__resource_type__name='Storage'
@@ -259,7 +256,6 @@ def pair_allocations_data(project, quota_dicts):
 def push_quota_data(result_file):
     """update group quota & usage values in Coldfront from a JSON of quota data.
     """
-    logger = logging.getLogger('coldfront.import_quotas')
     errored_allocations = {}
     missing_allocations = []
     result_json = read_json(result_file)
@@ -332,7 +328,6 @@ def match_entries_with_projects(result_json):
 
 
 def pull_push_quota_data(volumes=None):
-    logger = logging.getLogger('coldfront.import_quotas')
     att_data = QuotaDataPuller(volumes=volumes).pull('ATTQuery')
     resp_json_by_lab = {entry['lab']:[] for entry in att_data}
     for entry in att_data:

coldfront/plugins/isilon/management/commands/sync_isilon_allocations.py

@@ -0,0 +1,47 @@
+import logging
+
+from django.core.management.base import BaseCommand
+
+from coldfront.core.resource.models import Resource
+from coldfront.plugins.isilon.utils import print_log_error, sync_isilon_resource_allocations
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Sync Isilon/PowerScale directory smartquotas into ColdFront allocations
+    """
+    help = 'Sync Isilon/PowerScale directory smartquotas into ColdFront allocations'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-r', '--resource', help='Sync only the named isilon/powerscale Storage resource'
+        )
+
+    def handle(self, *args, **options):
+        isilon_resources = Resource.objects.filter(
+            resourceattribute__value__in=('isilon', 'powerscale'),
+            is_available=True,
+        )
+        if options.get('resource'):
+            isilon_resources = isilon_resources.filter(name=options['resource'])
+
+        if not isilon_resources.exists():
+            logger.warning('No active isilon/powerscale resources found matching %s', options.get('resource'))
+            return
+
+        all_missing_projects = []
+        for resource in isilon_resources:
+            try:
+                report = sync_isilon_resource_allocations(resource)
+            except Exception as e:
+                print_log_error(e, f'Could not sync allocations for resource {resource.name}')
+                continue
+            all_missing_projects.extend(report['missing_projects'])
+            logger.info('isilon allocation sync report for %s: %s', resource.name, report)
+
+        if all_missing_projects:
+            logger.warning(
+                'sync_isilon_allocations: no matching ColdFront Project found for groups: %s',
+                sorted(set(all_missing_projects)),
+            )

coldfront/plugins/isilon/tasks.py

@@ -4,3 +4,11 @@ def pull_isilon_quotas():
     """Pull Isilon quotas
     """
     management.call_command('pull_isilon_quotas')
+
+def sync_isilon_allocations(resource_name=None):
+    """Sync Isilon/PowerScale directory smartquotas into ColdFront allocations
+    """
+    if resource_name:
+        management.call_command('sync_isilon_allocations', resource=resource_name)
+    else:
+        management.call_command('sync_isilon_allocations')