Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enumerate variables needed by the HTCondor workflow #376

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Enumerate variables needed by the HTCondor workflow #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0ad219
data: update datafind server environment variable
duncanmmacleod Aug 25, 2023
3b9156a
batch: don't pass getenv=true to condor
duncanmmacleod Aug 25, 2023
bf72398
batch: add CONDA_EXE to GETENV_VARIABLES
duncanmmacleod Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 38 additions & 5 deletions gwsumm/batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,30 @@
__author__ = 'Duncan Macleod <duncan.macleod@ligo.org>'
__credits__ = 'Alex Urban <alexander.urban@ligo.org>'

# environment variables to pass to data-processing jobs
GETENV_VARIABLES = [
# -- IGWN resources
# address of dqsegdb server
"DEFAULT_SEGMENT_SERVER",
# address of gwdatafind server
"GWDATAFIND_SERVER",
# address of NDS2 server
"NDSSERVER",
# -- software stuff
# Conda (to generate package list for HTML)
"CONDA_EXE",
# GWpy customisations
"GWPY*",
# -- Auth handling
# Kerberos
"KRB5*",
# SciTokens
"BEARER_TOKEN*",
"SCITOKEN*",
# X.509
"X509*",
]

PROG = ('python -m gwsumm.batch' if sys.argv[0].endswith('.py')
else os.path.basename(sys.argv[0]))

Expand All @@ -67,7 +91,6 @@ def __init__(self, universe, tag='gw_summary',
logdir, '%s-%s.err' % (tag, self.logtag)))
self.set_stdout_file(os.path.join(
logdir, '%s-%s.out' % (tag, self.logtag)))
cmds.setdefault('getenv', 'True')
for key, val in cmds.items():
if hasattr(self, 'set_%s' % key.lower()):
getattr(self, 'set_%s' % key.lower())(val)
Expand Down Expand Up @@ -524,15 +547,25 @@ def main(args=None):
globalconfig = ','.join(args.global_config)

jobs = []
job_kw = {
"subdir": outdir,
"logdir": logdir,
**condorcmds,
}
if not args.skip_html_wrapper:
htmljob = GWSummaryJob(
'local', subdir=outdir, logdir=logdir,
tag='%s_local' % args.file_tag, **condorcmds)
'local',
tag='%s_local' % args.file_tag,
**job_kw,
)
jobs.append(htmljob)
if not args.html_wrapper_only:
datajob = GWSummaryJob(
universe, subdir=outdir, logdir=logdir,
tag=args.file_tag, **condorcmds)
universe,
tag=args.file_tag,
getenv=" ".join(GETENV_VARIABLES),
**job_kw,
)
jobs.append(datajob)

# add common command-line options
Expand Down
2 changes: 1 addition & 1 deletion gwsumm/data/timeseries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ def _get_timeseries_dict(channels, segments, config=None,
if nds is None and cache is not None:
nds = False
elif nds is None:
nds = 'LIGO_DATAFIND_SERVER' not in os.environ
nds = 'GWDATAFIND_SERVER' not in os.environ

# read new data
query &= (abs(new) > 0)
Expand Down
Loading