diff --git a/bin/jobsub_submit b/bin/jobsub_submit index fd246001..db095371 100755 --- a/bin/jobsub_submit +++ b/bin/jobsub_submit @@ -201,7 +201,6 @@ def main(): d1 = os.path.join(PREFIX, "templates", "simple") d2 = os.path.join(PREFIX, "templates", "dag") parse_dagnabbit(d1, varg, submitdir, schedd_name) - varg["N"] = 1 render_files(d2, varg, submitdir, dlist=[d2, submitdir]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) @@ -210,8 +209,13 @@ def main(): do_dataset_defaults(varg) d1 = os.path.join(PREFIX, "templates", "dataset_dag") d2 = f"{PREFIX}/templates/simple" + # so we render the simple area (d2) with -N 1 because + # we are making a loop of 1..N in th dataset_dag area + # otherwise we get N submissions of N jobs -> N^2 jobs... + saveN = varg["N"] + varg["N"] = "1" render_files(d2, varg, submitdir, dlist=[d1, d2]) - varg["N"] = 1 + varg["N"] = saveN render_files(d1, varg, submitdir, dlist=[d1, d2, submitdir]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) @@ -220,7 +224,6 @@ def main(): d1 = os.path.join(PREFIX, "templates", "maxconcurrent_dag") d2 = os.path.join(PREFIX, "templates", "simple") render_files(d2, varg, submitdir, dlist=[d1, d2]) - varg["N"] = 1 render_files(d1, varg, submitdir, dlist=[d1, d2, varg["dest"]]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) diff --git a/lib/dagnabbit.py b/lib/dagnabbit.py index e7ec6627..1793a980 100644 --- a/lib/dagnabbit.py +++ b/lib/dagnabbit.py @@ -140,6 +140,8 @@ def parse_dagnabbit( ) as csf: csf.write(jinja_env.get_template("simple.sh").render(**thesevalues)) of.write(f"JOB {name} {name}.cmd\n") + of.write(f'VARS {name} JOBSUBJOBSECTION="{count}" nodename="$(JOB)"') + if in_serial: if last_serial: of.write(f"PARENT {last_serial} CHILD {name}\n") diff --git a/lib/get_parser.py b/lib/get_parser.py index 071d9598..09d3be71 100644 --- a/lib/get_parser.py +++ b/lib/get_parser.py @@ -68,6 +68,7 @@ def get_parser() -> argparse.ArgumentParser: parser.add_argument( "--dataset-definition", "--dataset_definition", + "--dataset", help="SAM dataset definition used in a Directed Acyclic Graph (DAG)", ) parser.add_argument("--debug", type=int, default=0, help="Turn on debugging") diff --git a/templates/dataset_dag/dagbegin.cmd b/templates/dataset_dag/dagbegin.cmd index 2b8fd733..6cfa3c70 100644 --- a/templates/dataset_dag/dagbegin.cmd +++ b/templates/dataset_dag/dagbegin.cmd @@ -19,10 +19,7 @@ transfer_error = True transfer_executable= True when_to_transfer_output = ON_EXIT_OR_EVICT transfer_output_files = .empty_file -{%if cpu is defined and cpu %}request_cpus = {{cpu}}{%endif%} -{%if memory is defined and memory %}request_memory = {{memory}}{%endif%} -{%if disk is defined and disk %}request_disk = {{disk}}KB{%endif%} -{%if OS is defined and OS %}+DesiredOS={{OS}}{%endif%} +request_memory = 100mb +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" diff --git a/templates/dataset_dag/dagend.cmd b/templates/dataset_dag/dagend.cmd index 1195df38..b1b60036 100644 --- a/templates/dataset_dag/dagend.cmd +++ b/templates/dataset_dag/dagend.cmd @@ -20,10 +20,7 @@ transfer_error = True transfer_executable= True transfer_output_files=.empty_file when_to_transfer_output = ON_EXIT_OR_EVICT -{%if cpu is defined and cpu %}request_cpus = {{cpu}}{%endif%} -{%if memory is defined and memory %}request_memory = {{memory}}{%endif%} -{%if disk is defined and disk %}request_disk = {{disk}}KB{%endif%} -{%if OS is defined and OS %}+DesiredOS={{OS}}{%endif%} +request_memory = 100mb +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" diff --git a/templates/dataset_dag/dataset.dag b/templates/dataset_dag/dataset.dag index 49243cd1..8a75809f 100644 --- a/templates/dataset_dag/dataset.dag +++ b/templates/dataset_dag/dataset.dag @@ -3,6 +3,8 @@ JOB SAM_START dagbegin.cmd {%for i in range(N) %} JOB WORKER_{{i}} simple.cmd +VARS WORKER_{{i}} JOBSUBJOBSECTION="{{i}}" nodename="$(JOB)" + PARENT SAM_START CHILD WORKER_{{i}} PARENT WORKER_{{i}} CHILD SAM_END {%endfor%} diff --git a/templates/simple/simple.cmd b/templates/simple/simple.cmd index 42feaa32..34c9b6ef 100644 --- a/templates/simple/simple.cmd +++ b/templates/simple/simple.cmd @@ -8,7 +8,12 @@ arguments = {{exe_arguments|join(" ")}} output = {{filebase}}.out error = {{filebase}}.err log = {{filebase}}.log -environment = CLUSTER=$(Cluster);PROCESS=$(Process);CONDOR_TMP={{outdir}};BEARER_TOKEN_FILE=.condor_creds/{{group}}.use;CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};{{environment|join(';')}} + +{%if not (( dag is defined and dag ) or (dataset_dag is defined and dataset_dag)) %} +JOBSUBJOBSECTION=$(Process) +{%endif%} + +environment = CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};BEARER_TOKEN_FILE=.condor_creds/{{group}}.use;CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};{{environment|join(';')}} rank = Mips / 2 + Memory job_lease_duration = 3600 notification = Never diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..85778846 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers= + unit + integration diff --git a/tests/test_condor_unit.py b/tests/test_condor_unit.py index e3d366d5..91294ccf 100644 --- a/tests/test_condor_unit.py +++ b/tests/test_condor_unit.py @@ -115,6 +115,7 @@ class TestCondorUnit: # lib/condor.py routines... + @pytest.mark.unit def test_get_schedd_1(self): """make sure we get our test schedd back with test_vargs""" schedd = condor.get_schedd(TestUnit.test_vargs) @@ -122,18 +123,21 @@ def test_get_schedd_1(self): print("schedd name: {0}".format(schedd["Name"])) assert schedd["Name"] == TestUnit.test_schedd + @pytest.mark.unit def test_load_submit_file_1(self, get_submit_file): """make sure load_submit_file result has bits of the submit file""" res = condor.load_submit_file(get_submit_file) assert str(res[0]).find("universe = vanilla") >= 0 assert str(res[0]).find("executable = /bin/true") >= 0 + @pytest.mark.unit def test_submit_1(self, get_submit_file, needs_credentials): """actually submit a job with condor_submit""" res = condor.submit(get_submit_file, TestUnit.test_vargs, TestUnit.test_schedd) print("got: ", res) assert res + @pytest.mark.unit def test_submit_dag_1(self, get_dag_file, needs_credentials): """actually submit a dag with condor_submit_dag""" # XXX fix me diff --git a/tests/test_creds_unit.py b/tests/test_creds_unit.py index 30754a2b..5a6d37df 100644 --- a/tests/test_creds_unit.py +++ b/tests/test_creds_unit.py @@ -40,6 +40,7 @@ class TestCredUnit: # lib/creds.py routines... + @pytest.mark.unit def test_get_creds_1(self): """get credentials, make sure the credentials files returned exist""" diff --git a/tests/test_dagnabbit_unit.py b/tests/test_dagnabbit_unit.py index 2481540a..34da9e05 100644 --- a/tests/test_dagnabbit_unit.py +++ b/tests/test_dagnabbit_unit.py @@ -1,5 +1,6 @@ import os import sys +import pytest # # we assume everwhere our current directory is in the package @@ -26,6 +27,7 @@ class TestDagnabbitUnit: # lib/dagnabbit.py tests # + @pytest.mark.unit def test_parse_dagnabbit_dagTest(self): """test dagnabbit parser on old jobsub dagTest example""" self.do_one_dagnabbit( diff --git a/tests/test_fake_ifdh_unit.py b/tests/test_fake_ifdh_unit.py index 35f89ede..1992e090 100644 --- a/tests/test_fake_ifdh_unit.py +++ b/tests/test_fake_ifdh_unit.py @@ -17,6 +17,7 @@ import fake_ifdh +@pytest.mark.unit def test_getTmp(): if os.environ.get("TMPDIR", None): del os.environ["TMPDIR"] @@ -24,23 +25,27 @@ def test_getTmp(): assert res == "/tmp" +@pytest.mark.unit def test_getTmp_override(): os.environ["TMPDIR"] = "/var/tmp" res = fake_ifdh.getTmp() assert res == "/var/tmp" +@pytest.mark.unit def test_getExp_GROUP(): os.environ["GROUP"] = "samdev" res = fake_ifdh.getExp() assert res == "samdev" +@pytest.mark.unit def test_getRole(): res = fake_ifdh.getRole() assert res == fake_ifdh.DEFAULT_ROLE +@pytest.mark.unit def test_getRole_override(): override_role = "Hamburgler" res = fake_ifdh.getRole(override_role) @@ -59,27 +64,32 @@ def fermilab_token(clear_token): return fake_ifdh.getToken("Analysis") +@pytest.mark.unit def test_checkToken_fail(): tokenfile = "/dev/null" with pytest.raises(ValueError): res = fake_ifdh.checkToken(tokenfile) +@pytest.mark.unit def test_checkToken_success(fermilab_token): res = fake_ifdh.checkToken(fermilab_token) assert res +@pytest.mark.unit def test_getToken_good(clear_token, fermilab_token): assert os.path.exists(fermilab_token) +@pytest.mark.unit def test_getToken_fail(clear_token): with pytest.raises(PermissionError): os.environ["GROUP"] = "bozo" fake_ifdh.getToken("Analysis") +@pytest.mark.unit def test_getProxy_good(clear_token): os.environ["GROUP"] = "fermilab" @@ -87,6 +97,7 @@ def test_getProxy_good(clear_token): assert os.path.exists(proxy) +@pytest.mark.unit def test_getProxy_fail(clear_token): try: os.environ["GROUP"] = "bozo" @@ -97,6 +108,7 @@ def test_getProxy_fail(clear_token): assert False +@pytest.mark.unit def test_cp(): dest = __file__ + ".copy" fake_ifdh.cp(__file__, dest) diff --git a/tests/test_get_parser_unit.py b/tests/test_get_parser_unit.py index d49f778d..afd744f9 100644 --- a/tests/test_get_parser_unit.py +++ b/tests/test_get_parser_unit.py @@ -171,6 +171,7 @@ class TestGetParserUnit: # lib/get_parser.py routines... + @pytest.mark.unit def test_get_parser_small(self): """ Try a few common arguments on a get_parser() generated parser @@ -185,6 +186,7 @@ def test_get_parser_small(self): assert "SAM_EXPERIMENT" in res.environment assert res.group == TestUnit.test_group + @pytest.mark.unit def test_check_all_test_args(self, find_all_arguments, all_test_args): # make sure we have a test argument for all the arguments in # the source, and that we find all the arguments in the source @@ -202,6 +204,7 @@ def test_check_all_test_args(self, find_all_arguments, all_test_args): arg = arg.lstrip("-") assert arg in allargs + @pytest.mark.unit def test_get_parser_all(self, find_all_arguments, all_test_args): """ Validate an all arguments list diff --git a/tests/test_jobsub_submit_unit.py b/tests/test_jobsub_submit_unit.py index eb111b99..bb680ede 100644 --- a/tests/test_jobsub_submit_unit.py +++ b/tests/test_jobsub_submit_unit.py @@ -30,6 +30,7 @@ class TestJobsubSubmitUnit: # jobsub_submit functions + @pytest.mark.unit def test_get_basefiles_1(self): """test the get_basefiles routine on our source directory, we should be in it""" @@ -37,6 +38,7 @@ def test_get_basefiles_1(self): fl = jobsub_submit.get_basefiles(dlist) assert os.path.basename(__file__) in fl + @pytest.mark.unit def test_render_files_1(self): """test render files on the dataset_dag directory""" srcdir = os.path.dirname(os.path.dirname(__file__)) + "/templates/dataset_dag" @@ -47,6 +49,7 @@ def test_render_files_1(self): jobsub_submit.render_files(srcdir, args, dest) assert os.path.exists("%s/dagbegin.cmd" % dest) + @pytest.mark.unit def test_render_files_undefined_vars(self, tmp_path): """Test rendering files when a template variable is undefined. Should raise jinja2.exceptions.UndefinedError @@ -58,11 +61,13 @@ def test_render_files_undefined_vars(self, tmp_path): with pytest.raises(exceptions.UndefinedError, match="is undefined"): jobsub_submit.render_files(srcdir, args, dest) + @pytest.mark.unit def test_cleanup_1(self): # cleanup doesn't actually do anything right now... jobsub_submit.cleanup("") assert True + @pytest.mark.unit def test_do_dataset_defaults_1(self): """make sure do_dataset_defaults sets arguments its supposed to""" varg = TestUnit.test_vargs.copy() diff --git a/tests/test_packages_unit.py b/tests/test_packages_unit.py index 2d3c894f..cb0ddfba 100644 --- a/tests/test_packages_unit.py +++ b/tests/test_packages_unit.py @@ -1,5 +1,6 @@ import os import sys +import pytest # # we assume everwhere our current directory is in the package @@ -22,6 +23,7 @@ class TestPackagesUnit: # lib/packages.py routines + @pytest.mark.unit def test_pkg_find_1(self): """make sure we can find the poms_client ups package""" sp1 = sys.path.copy() @@ -32,6 +34,7 @@ def test_pkg_find_1(self): assert os.path.exists(os.environ["POMS_CLIENT_DIR"]) __import__("poms_client") + @pytest.mark.unit def test_pkg_orig_env_1(self): """make sure orig_env puts the environment back""" packages.pkg_find("poms_client", "-g poms41") diff --git a/tests/test_submit_wait_int.py b/tests/test_submit_wait_int.py index c3686897..71df2789 100644 --- a/tests/test_submit_wait_int.py +++ b/tests/test_submit_wait_int.py @@ -131,18 +131,21 @@ def run_launch(cmd): def lookaround_launch(extra): """Simple submit of our lookaround script""" assert run_launch( - f"jobsub_submit -e SAM_EXPERIMENT {extra} --resource-provides=usage_model=OPPORTUNISTIC,DEDICATED,OFFSITE file://`pwd`/job_scripts/lookaround.sh" + f"jobsub_submit --debug=1 -e SAM_EXPERIMENT {extra} --resource-provides=usage_model=OPPORTUNISTIC,DEDICATED,OFFSITE file://`pwd`/job_scripts/lookaround.sh" ) +@pytest.mark.integration def test_launch_lookaround_samdev(samdev): lookaround_launch("--devserver") +@pytest.mark.integration def test_launch_lookaround_dune(dune): lookaround_launch("--devserver") +@pytest.mark.integration def test_launch_lookaround_dune_gp(dune_gp): lookaround_launch("") @@ -162,10 +165,12 @@ def dagnabbit_launch(extra, which=""): os.chdir(os.path.dirname(__file__)) +@pytest.mark.integration def test_launch_dagnabbit_simple(samdev): dagnabbit_launch("--devserver", "") +@pytest.mark.integration def test_launch_dagnabbit_dropbox(samdev): dagnabbit_launch("--devserver", "Dropbox") @@ -174,6 +179,7 @@ def fife_launch(extra): assert run_launch( """ jobsub_submit \ + --debug=1 \ -e EXPERIMENT \ -e IFDH_DEBUG \ -e IFDH_FORCE \ @@ -193,7 +199,7 @@ def fife_launch(extra): --disk=100MB \ --memory=500MB \ %(extra)s \ - --dataset=gen_cfg \ + --dataset-definition=gen_cfg \ file://///grid/fermiapp/products/common/db/../prd/fife_utils/v3_3_2/NULL/libexec/fife_wrap \ --find_setups \ --setup-unquote 'hypotcode%%20v1_1' \ @@ -226,22 +232,27 @@ def fife_launch(extra): ) +@pytest.mark.integration def test_samdev_fife_launch(samdev): fife_launch("--devserver") +@pytest.mark.integration def test_dune_fife_launch(dune): fife_launch("--devserver") +@pytest.mark.integration def test_nova_fife_launch(nova): fife_launch("--devserver") +@pytest.mark.integration def test_dune_gp_fife_launch(dune_gp): fife_launch("") +@pytest.mark.integration def test_wait_for_jobs(): """Not really a test, but we have to wait for jobs to complete...""" count = 1 @@ -278,6 +289,7 @@ def test_wait_for_jobs(): assert True +@pytest.mark.integration def test_fetch_output(): for jid in joblist: if jid.find("dunesched") > 0: @@ -291,6 +303,7 @@ def test_fetch_output(): os.system("jobsub_transfer_data %s" % jid) +@pytest.mark.integration def test_check_job_output(): for outdir in outdirs: fl = glob.glob("%s/*.log" % outdir) diff --git a/tests/test_tarfiles_unit.py b/tests/test_tarfiles_unit.py index 15bcc315..3c67189e 100644 --- a/tests/test_tarfiles_unit.py +++ b/tests/test_tarfiles_unit.py @@ -1,6 +1,7 @@ import os import sys import time +import pytest # # we assume everwhere our current directory is in the package @@ -27,22 +28,26 @@ class TestTarfilesUnit: # lib/tarfiles.py routines... + @pytest.mark.unit def test_tar_up_1(self): """make sure tar up makes a tarfile""" tarfile = tarfiles.tar_up(os.path.dirname(__file__), None) assert os.path.exists(tarfile) os.unlink(tarfile) + @pytest.mark.unit def test_slurp_file_1(self): """make sure tar slurp_file makes a digest""" digest, tf = tarfiles.slurp_file(__file__) assert len(digest) == 64 + @pytest.mark.unit def test_dcache_persistent_path_1(self): """make sure persistent path gives /pnfs/ path digest""" path = tarfiles.dcache_persistent_path(TestUnit.test_group, __file__) assert path[:6] == "/pnfs/" + @pytest.mark.unit def test_tarfile_publisher_1(self, needs_credentials): """test the tarfile publisher object""" proxy, token = needs_credentials @@ -70,6 +75,7 @@ def test_tarfile_publisher_1(self, needs_credentials): assert location is not None + @pytest.mark.unit def test_do_tarballs_1(self, needs_credentials): """test that the do_tarballs method does a dropbox:path processing""" diff --git a/tests/test_utils_unit.py b/tests/test_utils_unit.py index cda6877c..d24576ed 100644 --- a/tests/test_utils_unit.py +++ b/tests/test_utils_unit.py @@ -1,5 +1,6 @@ import os import sys +import pytest # # we assume everwhere our current directory is in the package @@ -28,17 +29,20 @@ class TestUtilsUnit: # utils.py routines... # + @pytest.mark.unit def test_fixquote_1(self): """check the fixquote routine""" assert utils.fixquote("test1") == "test1" assert utils.fixquote("test2=test3") == 'test2="test3"' assert utils.fixquote("test2=test3=test4") == 'test2="test3=test4"' + @pytest.mark.unit def test_grep_n_1(self): """check the grep_n routine on us""" assert utils.grep_n(r"class (\w*):", 1, __file__) == "TestUtilsUnit" assert utils.grep_n(r"import (\w*)", 1, __file__) == "os" + @pytest.mark.unit def test_fix_unit_1(self): """test fixing units on '64gb' memory""" args = TestUnit.test_vargs.copy() @@ -46,12 +50,14 @@ def test_fix_unit_1(self): utils.fix_unit(args, "memory", memtable, -1, "b", -2) assert args["memory"] == 64 * 1024 + @pytest.mark.unit def test_get_principal_1(self): """make sure get_principal returns a string starting with $USER""" # blatantly assumes you have a valid principal... res = utils.get_principal() assert res.split("@")[0] == os.environ["USER"] + @pytest.mark.unit def test_set_extras_1(self, needs_credentials): """call set_extras_n_fix_units, verify one thing from environment and one unit conversion...""" @@ -61,6 +67,7 @@ def test_set_extras_1(self, needs_credentials): assert args["user"] == os.environ["USER"] assert args["memory"] == 64 * 1024 + @pytest.mark.unit def test_get_client_dn_valid_proxy_provided( self, needs_credentials, clear_x509_user_proxy ): @@ -70,6 +77,7 @@ def test_get_client_dn_valid_proxy_provided( client_dn = utils.get_client_dn(proxy=_proxy) assert os.environ["USER"] in client_dn + @pytest.mark.unit def test_get_client_dn_env_plus_proxy_provided(self, needs_credentials): """Call get_client_dn with proxy specified, env set. Should grab proxy from passed-in arg""" @@ -82,6 +90,7 @@ def test_get_client_dn_env_plus_proxy_provided(self, needs_credentials): assert os.environ["USER"] in client_dn os.environ["X509_USER_PROXY"] = old_x509_user_proxy_value + @pytest.mark.unit def test_get_client_dn_no_proxy_provided(self, needs_credentials): """Call get_client_dn with no proxy specified. Should grab proxy from env""" @@ -89,6 +98,7 @@ def test_get_client_dn_no_proxy_provided(self, needs_credentials): client_dn = utils.get_client_dn() assert os.environ["USER"] in client_dn + @pytest.mark.unit def test_get_client_dn_no_proxy_provided_no_env( self, needs_credentials, clear_x509_user_proxy ): @@ -99,6 +109,7 @@ def test_get_client_dn_no_proxy_provided_no_env( client_dn = utils.get_client_dn() assert os.environ["USER"] in client_dn + @pytest.mark.unit def test_get_client_dn_bad_proxy(self): """If we give a bad proxy file, or there's some other problem, we should get "" as the return value"""