From f6fbb3d8292f4c5e3a51c1a352920bf2692e4e2f Mon Sep 17 00:00:00 2001 From: Shreyas Bhat Date: Tue, 28 Jun 2022 14:38:09 +0000 Subject: [PATCH] Fix templates so they work with new StrictUndefined setting for rendering In some cases, we want template rendering to fail if a variable is not defined. In other cases, we want to check for definition, and if it's not defined, proceed without executing a certain part of the template. Specifying the latter as {% if key is defined %} rather than {% if key %} allows us to do that. --- templates/dag/dag.dag.condor.sub | 4 ++-- templates/dag/dagman_wrapper.sh | 2 +- templates/dataset_dag/dagbegin.cmd | 16 ++++++++-------- templates/dataset_dag/dagend.cmd | 16 ++++++++-------- templates/simple/simple.cmd | 20 ++++++++++---------- templates/simple/simple.sh | 4 ++-- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/templates/dag/dag.dag.condor.sub b/templates/dag/dag.dag.condor.sub index af69651d..bd470cbc 100644 --- a/templates/dag/dag.dag.condor.sub +++ b/templates/dag/dag.dag.condor.sub @@ -28,7 +28,7 @@ environment = _CONDOR_SCHEDD_ADDRESS_FILE=/var/lib/condor/spool/.schedd_address x509userproxy = {{proxy}} {{lines|join("\n+")}} -{% if subgroup %} +{% if subgroup is defined %} +AccountingGroup = "group_{{group}}.{{subgroup}}.{{user}}" {% else %} +AccountingGroup = "group_{{group}}.{{user}}" @@ -37,7 +37,7 @@ x509userproxy = {{proxy}} +Jobsub_Group="{{group}}" +JobsubJobId="$(CLUSTER).$(PROCESS)@{{schedd}}" -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} use_oauth_services = {{group}}_{{role}} {% else %} use_oauth_services = {{group}} diff --git a/templates/dag/dagman_wrapper.sh b/templates/dag/dagman_wrapper.sh index db29d1c6..51105446 100755 --- a/templates/dag/dagman_wrapper.sh +++ b/templates/dag/dagman_wrapper.sh @@ -3,7 +3,7 @@ # condor wants to copy in the condor_dagman executable, but we # want it to run the local one, so we give it this one... -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} export BEARER_TOKEN_FILE=$_CONDOR_CREDS/{{group}}_{{role}}.use {% else %} export BEARER_TOKEN_FILE=$_CONDOR_CREDS/{{group}}.use diff --git a/templates/dataset_dag/dagbegin.cmd b/templates/dataset_dag/dagbegin.cmd index ae093457..0779588e 100644 --- a/templates/dataset_dag/dagbegin.cmd +++ b/templates/dataset_dag/dagbegin.cmd @@ -1,5 +1,5 @@ # generated by jobsub_lite -# {%if debug%}debug{%endif%} +# {%if debug is defined%}debug{%endif%} universe = vanilla executable = sambegin.sh arguments = @@ -19,10 +19,10 @@ transfer_error = True transfer_executable= True when_to_transfer_output = ON_EXIT_OR_EVICT transfer_output_files = .empty_file -{%if cpu %}request_cpus = {{cpu}}{%endif%} -{%if memory %}request_memory = {{memory}}{%endif%} -{%if disk %}request_disk = {{disk}}KB{%endif%} -{%if OS %}+DesiredOS={{OS}}{%endif%} +{%if cpu is defined %}request_cpus = {{cpu}}{%endif%} +{%if memory is defined %}request_memory = {{memory}}{%endif%} +{%if disk is defined %}request_disk = {{disk}}KB{%endif%} +{%if OS is defined %}+DesiredOS={{OS}}{%endif%} +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" @@ -35,19 +35,19 @@ notify_user = {{email_to}} +Jobsub_Group="{{group}}" +JobsubJobId="$(CLUSTER).$(PROCESS)@{{schedd}}" +Drain = False -{%if blacklist %} +{%if blacklist is defined %} +Blacklist_Sites = "{{blacklist}}" {% endif %} +GeneratedBy ="{{version}} {{schedd}}" {{resource_provides_quoted|join("\n+DESIRED_")}} {{lines|join("\n+")}} -requirements = target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))) {%if append_condor_requirements %} && {{append_condor_requirements}} {%endif%} +requirements = target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))) {%if append_condor_requirements is defined %} && {{append_condor_requirements}} {%endif%} {% if no_singularity is false %} +SingularityImage="{{singularity_image}}" {% endif %} -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} use_oauth_services = {{group}}_{{role}} {% else %} use_oauth_services = {{group}} diff --git a/templates/dataset_dag/dagend.cmd b/templates/dataset_dag/dagend.cmd index 3b0ccd50..218f5273 100644 --- a/templates/dataset_dag/dagend.cmd +++ b/templates/dataset_dag/dagend.cmd @@ -1,5 +1,5 @@ # generated by jobsub_lite -# {%if debug%}debug{%endif%} +# {%if debug is defined %}debug{%endif%} universe = vanilla executable = samend.sh arguments = @@ -20,10 +20,10 @@ transfer_error = True transfer_executable= True transfer_output_files=.empty_file when_to_transfer_output = ON_EXIT_OR_EVICT -{%if cpu %}request_cpus = {{cpu}}{%endif%} -{%if memory %}request_memory = {{memory}}{%endif%} -{%if disk %}request_disk = {{disk}}KB{%endif%} -{%if OS %}+DesiredOS={{OS}}{%endif%} +{%if cpu is defined %}request_cpus = {{cpu}}{%endif%} +{%if memory is defined %}request_memory = {{memory}}{%endif%} +{%if disk is defined %}request_disk = {{disk}}KB{%endif%} +{%if OS is defined %}+DesiredOS={{OS}}{%endif%} +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" @@ -36,19 +36,19 @@ notify_user = {{email_to}} +Jobsub_Group="{{group}}" +JobsubJobId="$(CLUSTER).$(PROCESS)@{{schedd}}" +Drain = False -{%if blacklist %} +{%if blacklist is defined%} +Blacklist_Sites = "{{blacklist}}" {% endif %} +GeneratedBy ="{{version}} {{schedd}}" {{resource_provides_quoted|join("\n+DESIRED_")}} {{lines|join("\n+")}} -requirements = target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))) {%if append_condor_requirements %} && {{append_condor_requirements}} {%endif%} +requirements = target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))) {%if append_condor_requirements is defined%} && {{append_condor_requirements}} {%endif%} {% if no_singularity is false %} +SingularityImage="{{singularity_image}}" {% endif %} -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} use_oauth_services = {{group}}_{{role}} {% else %} use_oauth_services = {{group}} diff --git a/templates/simple/simple.cmd b/templates/simple/simple.cmd index c9be80f7..cacc426b 100644 --- a/templates/simple/simple.cmd +++ b/templates/simple/simple.cmd @@ -1,6 +1,6 @@ # generated by jobsub_lite -# {%if debug%}debug{%endif%} +# {%if debug is defined %}debug{%endif%} universe = vanilla executable = {{script_name|default('simple.sh')}} arguments = {{exe_arguments|join(" ")}} @@ -18,10 +18,10 @@ transfer_executable= True transfer_input_files = {{executable|basename}} when_to_transfer_output = ON_EXIT_OR_EVICT transfer_output_files = .empty_file -{%if cpu %}request_cpus = {{cpu}}{%endif%} -{%if memory %}request_memory = {{memory}}{%endif%} -{%if disk %}request_disk = {{disk}}KB{%endif%} -{%if OS %}+DesiredOS={{OS}}{%endif%} +{%if cpu is defined %}request_cpus = {{cpu}}{%endif%} +{%if memory is defined %}request_memory = {{memory}}{%endif%} +{%if disk is defined %}request_disk = {{disk}}KB{%endif%} +{%if OS is defined %}+DesiredOS={{OS}}{%endif%} +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" @@ -31,7 +31,7 @@ transfer_output_files = .empty_file +JOB_EXPECTED_MAX_LIFETIME = {{expected_lifetime}} notify_user = {{email_to}} -{% if subgroup %} +{% if subgroup is defined %} +AccountingGroup = "group_{{group}}.{{subgroup}}.{{user}}" {% else %} +AccountingGroup = "group_{{group}}.{{user}}" @@ -41,16 +41,16 @@ notify_user = {{email_to}} +JobsubJobId="$(CLUSTER).$(PROCESS)@{{schedd}}" +Drain = False -{% if site and site != 'LOCAL' %} +{% if site is defined and site != 'LOCAL' %} +DESIRED_SITES = "{{site}}" {% endif %} -{%if blacklist %} +{%if blacklist is defined %} +Blacklist_Sites = "{{blacklist}}" {% endif %} +GeneratedBy ="{{version}} {{schedd}}" {{resource_provides_quoted|join("\n+DESIRED_")}} {{lines|join("\n+")}} -requirements = {%if overwrite_requirements %}{{overwrite_requirements}}{%else%}target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))){%endif%}{%if append_condor_requirements %} && {{append_condor_requirements}} {%endif%} +requirements = {%if overwrite_requirements is defined %}{{overwrite_requirements}}{%else%}target.machine =!= MachineAttrMachine1 && target.machine =!= MachineAttrMachine2 && (isUndefined(DesiredOS) || stringListsIntersect(toUpper(DesiredOS),IFOS_installed)) && (stringListsIntersect(toUpper(target.HAS_usage_model), toUpper(my.DESIRED_usage_model))){%endif%}{%if append_condor_requirements is defined %} && {{append_condor_requirements}} {%endif%} {% if no_singularity is false %} +SingularityImage="{{singularity_image}}" @@ -63,7 +63,7 @@ requirements = {%if overwrite_requirements %}{{overwrite_requirements}}{%else%} # # # -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} use_oauth_services = {{group}}_{{role}} {% else %} use_oauth_services = {{group}} diff --git a/templates/simple/simple.sh b/templates/simple/simple.sh index b27bdadf..fd1c54ec 100644 --- a/templates/simple/simple.sh +++ b/templates/simple/simple.sh @@ -7,7 +7,7 @@ umask 002 -{% if role and role != 'Analysis' %} +{% if role is defined and role != 'Analysis' %} export BEARER_TOKEN_FILE=$PWD/.condor_creds/{{group}}_{{role}}.use {% else %} export BEARER_TOKEN_FILE=$PWD/.condor_creds/{{group}}.use @@ -320,7 +320,7 @@ ${JSB_TMP}/ifdh.sh log poms_data=$poms_data echo `date` $JOBSUBJOBID BEGIN EXECUTION $JOBSUB_EXE_SCRIPT {{exe_arguments|join(" ")}} >&2 echo `date` $JOBSUBJOBID BEGIN EXECUTION $JOBSUB_EXE_SCRIPT {{exe_arguments|join(" ")}} -{%if timeout%} timeout {{timeout}} {%endif%} $JOBSUB_EXE_SCRIPT {{exe_arguments|join(" ")}} +{%if timeout is defined %} timeout {{timeout}} {%endif%} $JOBSUB_EXE_SCRIPT {{exe_arguments|join(" ")}} JOB_RET_STATUS=$? # copy out -d directories