DM-32830: Add first version of PanDA authentication tools.

bin.src/panda_auth

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import sys
+import os
+
+from lsst.utils import doImport
+
+if __name__ == '__main__':
+    # Check for env vars before importing code because the LSST click
+    # code hides that the problem is missing environment variables required
+    # during imports of third-party modules.
+    for key in ["PANDA_AUTH", "PANDA_VERIFY_HOST", "PANDA_AUTH_VO", "PANDA_URL_SSL", "PANDA_URL",
+                "IDDS_CONFIG"]:
+        if key not in os.environ:
+            raise OSError(f"Missing environment variable {key}")
+
+    panda_auth_mod = doImport("lsst.ctrl.bps.wms.panda.cli.panda_auth")
+    sys.exit(panda_auth_mod.main())

config/bps_idf.yaml

@@ -3,6 +3,9 @@ iddsServer: "https://aipanda015.cern.ch:443/idds"
 placeholderParams: ['qgraphNodeId', 'qgraphId']
 defaultPreCmdOpts: "--long-log --log-level=VERBOSE --log-file payload-log.json"
 
+# Limit the number of jobs in a single PanDA task
+maxJobsPerTask: 70000
+
 #IDF PanDA specific settings:
 computeSite: LSST
 payload:

doc/changes/DM-32830.feature.rst

@@ -0,0 +1,3 @@
+* New panda_auth command for handling PanDA authentication token.
+  Includes status, reset, and clean capabilities.
+* Added early check of PanDA authentication token in submission process.

doc/changes/DM-32830.misc.rst

@@ -0,0 +1,3 @@
+* Changed printing of submit directory early.
+* Changed PanDA plugin to only print the numeric id when outputing the request/run id.
+* Set maximum number of jobs in a PanDA task (maxJobsPerTask) to 70000 in config/bps_idf.yaml.

python/lsst/ctrl/bps/drivers.py

@@ -157,6 +157,7 @@ def _init_submission_driver(config_file, **kwargs):
             reason = exc.strerror
         raise type(exc)(f"cannot create submit directory '{submit_path}': {reason}") from None
     config[".bps_defined.submitPath"] = str(submit_path)
+    print(f"Submit dir: {submit_path}")
 
     # save copy of configs (orig and expanded config)
     shutil.copy2(config_file, submit_path)
@@ -288,7 +289,6 @@ def prepare_driver(config_file, **kwargs):
         wms_workflow = prepare(generic_workflow_config, generic_workflow, submit_path)
 
     wms_workflow_config = generic_workflow_config
-    print(f"Submit dir: {wms_workflow.submit_path}")
     return wms_workflow_config, wms_workflow
 
 

python/lsst/ctrl/bps/wms/panda/cli/cmd/__init__.py

@@ -0,0 +1,24 @@
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+__all__ = ["clean", "reset", "status"]
+
+from .panda_auth_commands import clean, reset, status

python/lsst/ctrl/bps/wms/panda/cli/cmd/panda_auth_commands.py

@@ -0,0 +1,65 @@
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""Subcommand definitions for the PanDA auth commands.
+"""
+
+__all__ = [
+    "status",
+    "reset",
+    "clean",
+]
+
+
+import click
+
+from lsst.daf.butler.cli.utils import MWCommand
+from ...panda_auth_drivers import (
+    panda_auth_status_driver,
+    panda_auth_reset_driver,
+    panda_auth_clean_driver
+)
+
+
+class PandaAuthCommand(MWCommand):
+    """Command subclass with panda-auth-command specific overrides."""
+
+    extra_epilog = "See 'panda_auth --help' for more options."
+
+
+@click.command(cls=PandaAuthCommand)
+def status(*args, **kwargs):
+    """Print informatino about auth token.
+    """
+    panda_auth_status_driver(*args, **kwargs)
+
+
+@click.command(cls=PandaAuthCommand)
+def reset(*args, **kwargs):
+    """Get new auth token.
+    """
+    panda_auth_reset_driver(*args, **kwargs)
+
+
+@click.command(cls=PandaAuthCommand)
+def clean(*args, **kwargs):
+    """Clean up token and token cache files.
+    """
+    panda_auth_clean_driver(*args, **kwargs)

python/lsst/ctrl/bps/wms/panda/cli/panda_auth.py

@@ -0,0 +1,51 @@
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import click
+
+from lsst.daf.butler.cli.butler import LoaderCLI
+from lsst.daf.butler.cli.opt import (
+    log_level_option,
+    long_log_option,
+    log_file_option,
+    log_tty_option,
+    log_label_option,
+)
+
+
+class PandaAuthCli(LoaderCLI):
+
+    localCmdPkg = "lsst.ctrl.bps.wms.panda.cli.cmd"
+
+
+@click.command(cls=PandaAuthCli,
+               context_settings=dict(help_option_names=["-h", "--help"]),
+               epilog=None)
+@log_level_option(default=["INFO"])
+@long_log_option()
+@log_file_option()
+@log_tty_option()
+@log_label_option()
+def cli(log_level, long_log, log_file, log_tty, log_label):
+    pass
+
+
+def main():
+    return cli()

python/lsst/ctrl/bps/wms/panda/panda_auth_drivers.py

@@ -0,0 +1,69 @@
+# This file is part of ctrl_bps.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Driver functions for each panda_auth subcommand.
+
+Driver functions ensure that ensure all setup work is done before running
+the subcommand method.
+"""
+
+
+__all__ = [
+    "panda_auth_clean_driver",
+    "panda_auth_reset_driver",
+    "panda_auth_status_driver",
+]
+
+
+import logging
+from datetime import datetime
+
+from .panda_auth_utils import panda_auth_clean, panda_auth_status, panda_auth_update
+
+_LOG = logging.getLogger(__name__)
+
+
+def panda_auth_clean_driver():
+    """Clean up token and token cache files.
+    """
+    panda_auth_clean()
+
+
+def panda_auth_reset_driver():
+    """Get new auth token.
+    """
+    panda_auth_update(None, True)
+
+
+def panda_auth_status_driver():
+    """Gather information about a token if it exists.
+    """
+    status = panda_auth_status()
+    if status:
+        print(f"{'Filename:':15} {status['filename']}")
+        print(f"{'Valid starting:':15} {datetime.fromtimestamp(status['iat'])}")
+        print(f"{'Expires:':15} {datetime.fromtimestamp(status['exp'])}")
+        print(f"{'Name:':15} {status['name']}")
+        print(f"{'Email:':15} {status['email']}")
+        print(f"{'Groups:':15} {','.join(status['groups'])}")
+        print(f"{'Organization:':15} {status['organisation_name']}")
+    else:
+        print("No token found")