diff --git a/bin/jobsub_submit b/bin/jobsub_submit index db095371..fd246001 100755 --- a/bin/jobsub_submit +++ b/bin/jobsub_submit @@ -201,6 +201,7 @@ def main(): d1 = os.path.join(PREFIX, "templates", "simple") d2 = os.path.join(PREFIX, "templates", "dag") parse_dagnabbit(d1, varg, submitdir, schedd_name) + varg["N"] = 1 render_files(d2, varg, submitdir, dlist=[d2, submitdir]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) @@ -209,13 +210,8 @@ def main(): do_dataset_defaults(varg) d1 = os.path.join(PREFIX, "templates", "dataset_dag") d2 = f"{PREFIX}/templates/simple" - # so we render the simple area (d2) with -N 1 because - # we are making a loop of 1..N in th dataset_dag area - # otherwise we get N submissions of N jobs -> N^2 jobs... - saveN = varg["N"] - varg["N"] = "1" render_files(d2, varg, submitdir, dlist=[d1, d2]) - varg["N"] = saveN + varg["N"] = 1 render_files(d1, varg, submitdir, dlist=[d1, d2, submitdir]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) @@ -224,6 +220,7 @@ def main(): d1 = os.path.join(PREFIX, "templates", "maxconcurrent_dag") d2 = os.path.join(PREFIX, "templates", "simple") render_files(d2, varg, submitdir, dlist=[d1, d2]) + varg["N"] = 1 render_files(d1, varg, submitdir, dlist=[d1, d2, varg["dest"]]) if not varg.get("no_submit", False): os.chdir(varg["submitdir"]) diff --git a/lib/dagnabbit.py b/lib/dagnabbit.py index 1793a980..e7ec6627 100644 --- a/lib/dagnabbit.py +++ b/lib/dagnabbit.py @@ -140,8 +140,6 @@ def parse_dagnabbit( ) as csf: csf.write(jinja_env.get_template("simple.sh").render(**thesevalues)) of.write(f"JOB {name} {name}.cmd\n") - of.write(f'VARS {name} JOBSUBJOBSECTION="{count}" nodename="$(JOB)"') - if in_serial: if last_serial: of.write(f"PARENT {last_serial} CHILD {name}\n") diff --git a/lib/get_parser.py b/lib/get_parser.py index 09d3be71..071d9598 100644 --- a/lib/get_parser.py +++ b/lib/get_parser.py @@ -68,7 +68,6 @@ def get_parser() -> argparse.ArgumentParser: parser.add_argument( "--dataset-definition", "--dataset_definition", - "--dataset", help="SAM dataset definition used in a Directed Acyclic Graph (DAG)", ) parser.add_argument("--debug", type=int, default=0, help="Turn on debugging") diff --git a/templates/dataset_dag/dagbegin.cmd b/templates/dataset_dag/dagbegin.cmd index 6cfa3c70..2b8fd733 100644 --- a/templates/dataset_dag/dagbegin.cmd +++ b/templates/dataset_dag/dagbegin.cmd @@ -19,7 +19,10 @@ transfer_error = True transfer_executable= True when_to_transfer_output = ON_EXIT_OR_EVICT transfer_output_files = .empty_file -request_memory = 100mb +{%if cpu is defined and cpu %}request_cpus = {{cpu}}{%endif%} +{%if memory is defined and memory %}request_memory = {{memory}}{%endif%} +{%if disk is defined and disk %}request_disk = {{disk}}KB{%endif%} +{%if OS is defined and OS %}+DesiredOS={{OS}}{%endif%} +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" diff --git a/templates/dataset_dag/dagend.cmd b/templates/dataset_dag/dagend.cmd index b1b60036..1195df38 100644 --- a/templates/dataset_dag/dagend.cmd +++ b/templates/dataset_dag/dagend.cmd @@ -20,7 +20,10 @@ transfer_error = True transfer_executable= True transfer_output_files=.empty_file when_to_transfer_output = ON_EXIT_OR_EVICT -request_memory = 100mb +{%if cpu is defined and cpu %}request_cpus = {{cpu}}{%endif%} +{%if memory is defined and memory %}request_memory = {{memory}}{%endif%} +{%if disk is defined and disk %}request_disk = {{disk}}KB{%endif%} +{%if OS is defined and OS %}+DesiredOS={{OS}}{%endif%} +JobsubClientDN="{{clientdn}}" +JobsubClientIpAddress="{{ipaddr}}" +Owner="{{user}}" diff --git a/templates/dataset_dag/dataset.dag b/templates/dataset_dag/dataset.dag index 8a75809f..49243cd1 100644 --- a/templates/dataset_dag/dataset.dag +++ b/templates/dataset_dag/dataset.dag @@ -3,8 +3,6 @@ JOB SAM_START dagbegin.cmd {%for i in range(N) %} JOB WORKER_{{i}} simple.cmd -VARS WORKER_{{i}} JOBSUBJOBSECTION="{{i}}" nodename="$(JOB)" - PARENT SAM_START CHILD WORKER_{{i}} PARENT WORKER_{{i}} CHILD SAM_END {%endfor%} diff --git a/templates/simple/simple.cmd b/templates/simple/simple.cmd index 34c9b6ef..42feaa32 100644 --- a/templates/simple/simple.cmd +++ b/templates/simple/simple.cmd @@ -8,12 +8,7 @@ arguments = {{exe_arguments|join(" ")}} output = {{filebase}}.out error = {{filebase}}.err log = {{filebase}}.log - -{%if not (( dag is defined and dag ) or (dataset_dag is defined and dataset_dag)) %} -JOBSUBJOBSECTION=$(Process) -{%endif%} - -environment = CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};BEARER_TOKEN_FILE=.condor_creds/{{group}}.use;CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};{{environment|join(';')}} +environment = CLUSTER=$(Cluster);PROCESS=$(Process);CONDOR_TMP={{outdir}};BEARER_TOKEN_FILE=.condor_creds/{{group}}.use;CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};{{environment|join(';')}} rank = Mips / 2 + Memory job_lease_duration = 3600 notification = Never diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index 85778846..00000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -markers= - unit - integration diff --git a/tests/test_condor_unit.py b/tests/test_condor_unit.py index 91294ccf..e3d366d5 100644 --- a/tests/test_condor_unit.py +++ b/tests/test_condor_unit.py @@ -115,7 +115,6 @@ class TestCondorUnit: # lib/condor.py routines... - @pytest.mark.unit def test_get_schedd_1(self): """make sure we get our test schedd back with test_vargs""" schedd = condor.get_schedd(TestUnit.test_vargs) @@ -123,21 +122,18 @@ def test_get_schedd_1(self): print("schedd name: {0}".format(schedd["Name"])) assert schedd["Name"] == TestUnit.test_schedd - @pytest.mark.unit def test_load_submit_file_1(self, get_submit_file): """make sure load_submit_file result has bits of the submit file""" res = condor.load_submit_file(get_submit_file) assert str(res[0]).find("universe = vanilla") >= 0 assert str(res[0]).find("executable = /bin/true") >= 0 - @pytest.mark.unit def test_submit_1(self, get_submit_file, needs_credentials): """actually submit a job with condor_submit""" res = condor.submit(get_submit_file, TestUnit.test_vargs, TestUnit.test_schedd) print("got: ", res) assert res - @pytest.mark.unit def test_submit_dag_1(self, get_dag_file, needs_credentials): """actually submit a dag with condor_submit_dag""" # XXX fix me diff --git a/tests/test_creds_unit.py b/tests/test_creds_unit.py index 5a6d37df..30754a2b 100644 --- a/tests/test_creds_unit.py +++ b/tests/test_creds_unit.py @@ -40,7 +40,6 @@ class TestCredUnit: # lib/creds.py routines... - @pytest.mark.unit def test_get_creds_1(self): """get credentials, make sure the credentials files returned exist""" diff --git a/tests/test_dagnabbit_unit.py b/tests/test_dagnabbit_unit.py index 34da9e05..2481540a 100644 --- a/tests/test_dagnabbit_unit.py +++ b/tests/test_dagnabbit_unit.py @@ -1,6 +1,5 @@ import os import sys -import pytest # # we assume everwhere our current directory is in the package @@ -27,7 +26,6 @@ class TestDagnabbitUnit: # lib/dagnabbit.py tests # - @pytest.mark.unit def test_parse_dagnabbit_dagTest(self): """test dagnabbit parser on old jobsub dagTest example""" self.do_one_dagnabbit( diff --git a/tests/test_fake_ifdh_unit.py b/tests/test_fake_ifdh_unit.py index 1992e090..35f89ede 100644 --- a/tests/test_fake_ifdh_unit.py +++ b/tests/test_fake_ifdh_unit.py @@ -17,7 +17,6 @@ import fake_ifdh -@pytest.mark.unit def test_getTmp(): if os.environ.get("TMPDIR", None): del os.environ["TMPDIR"] @@ -25,27 +24,23 @@ def test_getTmp(): assert res == "/tmp" -@pytest.mark.unit def test_getTmp_override(): os.environ["TMPDIR"] = "/var/tmp" res = fake_ifdh.getTmp() assert res == "/var/tmp" -@pytest.mark.unit def test_getExp_GROUP(): os.environ["GROUP"] = "samdev" res = fake_ifdh.getExp() assert res == "samdev" -@pytest.mark.unit def test_getRole(): res = fake_ifdh.getRole() assert res == fake_ifdh.DEFAULT_ROLE -@pytest.mark.unit def test_getRole_override(): override_role = "Hamburgler" res = fake_ifdh.getRole(override_role) @@ -64,32 +59,27 @@ def fermilab_token(clear_token): return fake_ifdh.getToken("Analysis") -@pytest.mark.unit def test_checkToken_fail(): tokenfile = "/dev/null" with pytest.raises(ValueError): res = fake_ifdh.checkToken(tokenfile) -@pytest.mark.unit def test_checkToken_success(fermilab_token): res = fake_ifdh.checkToken(fermilab_token) assert res -@pytest.mark.unit def test_getToken_good(clear_token, fermilab_token): assert os.path.exists(fermilab_token) -@pytest.mark.unit def test_getToken_fail(clear_token): with pytest.raises(PermissionError): os.environ["GROUP"] = "bozo" fake_ifdh.getToken("Analysis") -@pytest.mark.unit def test_getProxy_good(clear_token): os.environ["GROUP"] = "fermilab" @@ -97,7 +87,6 @@ def test_getProxy_good(clear_token): assert os.path.exists(proxy) -@pytest.mark.unit def test_getProxy_fail(clear_token): try: os.environ["GROUP"] = "bozo" @@ -108,7 +97,6 @@ def test_getProxy_fail(clear_token): assert False -@pytest.mark.unit def test_cp(): dest = __file__ + ".copy" fake_ifdh.cp(__file__, dest) diff --git a/tests/test_get_parser_unit.py b/tests/test_get_parser_unit.py index afd744f9..d49f778d 100644 --- a/tests/test_get_parser_unit.py +++ b/tests/test_get_parser_unit.py @@ -171,7 +171,6 @@ class TestGetParserUnit: # lib/get_parser.py routines... - @pytest.mark.unit def test_get_parser_small(self): """ Try a few common arguments on a get_parser() generated parser @@ -186,7 +185,6 @@ def test_get_parser_small(self): assert "SAM_EXPERIMENT" in res.environment assert res.group == TestUnit.test_group - @pytest.mark.unit def test_check_all_test_args(self, find_all_arguments, all_test_args): # make sure we have a test argument for all the arguments in # the source, and that we find all the arguments in the source @@ -204,7 +202,6 @@ def test_check_all_test_args(self, find_all_arguments, all_test_args): arg = arg.lstrip("-") assert arg in allargs - @pytest.mark.unit def test_get_parser_all(self, find_all_arguments, all_test_args): """ Validate an all arguments list diff --git a/tests/test_jobsub_submit_unit.py b/tests/test_jobsub_submit_unit.py index bb680ede..eb111b99 100644 --- a/tests/test_jobsub_submit_unit.py +++ b/tests/test_jobsub_submit_unit.py @@ -30,7 +30,6 @@ class TestJobsubSubmitUnit: # jobsub_submit functions - @pytest.mark.unit def test_get_basefiles_1(self): """test the get_basefiles routine on our source directory, we should be in it""" @@ -38,7 +37,6 @@ def test_get_basefiles_1(self): fl = jobsub_submit.get_basefiles(dlist) assert os.path.basename(__file__) in fl - @pytest.mark.unit def test_render_files_1(self): """test render files on the dataset_dag directory""" srcdir = os.path.dirname(os.path.dirname(__file__)) + "/templates/dataset_dag" @@ -49,7 +47,6 @@ def test_render_files_1(self): jobsub_submit.render_files(srcdir, args, dest) assert os.path.exists("%s/dagbegin.cmd" % dest) - @pytest.mark.unit def test_render_files_undefined_vars(self, tmp_path): """Test rendering files when a template variable is undefined. Should raise jinja2.exceptions.UndefinedError @@ -61,13 +58,11 @@ def test_render_files_undefined_vars(self, tmp_path): with pytest.raises(exceptions.UndefinedError, match="is undefined"): jobsub_submit.render_files(srcdir, args, dest) - @pytest.mark.unit def test_cleanup_1(self): # cleanup doesn't actually do anything right now... jobsub_submit.cleanup("") assert True - @pytest.mark.unit def test_do_dataset_defaults_1(self): """make sure do_dataset_defaults sets arguments its supposed to""" varg = TestUnit.test_vargs.copy() diff --git a/tests/test_packages_unit.py b/tests/test_packages_unit.py index cb0ddfba..2d3c894f 100644 --- a/tests/test_packages_unit.py +++ b/tests/test_packages_unit.py @@ -1,6 +1,5 @@ import os import sys -import pytest # # we assume everwhere our current directory is in the package @@ -23,7 +22,6 @@ class TestPackagesUnit: # lib/packages.py routines - @pytest.mark.unit def test_pkg_find_1(self): """make sure we can find the poms_client ups package""" sp1 = sys.path.copy() @@ -34,7 +32,6 @@ def test_pkg_find_1(self): assert os.path.exists(os.environ["POMS_CLIENT_DIR"]) __import__("poms_client") - @pytest.mark.unit def test_pkg_orig_env_1(self): """make sure orig_env puts the environment back""" packages.pkg_find("poms_client", "-g poms41") diff --git a/tests/test_submit_wait_int.py b/tests/test_submit_wait_int.py index 71df2789..c3686897 100644 --- a/tests/test_submit_wait_int.py +++ b/tests/test_submit_wait_int.py @@ -131,21 +131,18 @@ def run_launch(cmd): def lookaround_launch(extra): """Simple submit of our lookaround script""" assert run_launch( - f"jobsub_submit --debug=1 -e SAM_EXPERIMENT {extra} --resource-provides=usage_model=OPPORTUNISTIC,DEDICATED,OFFSITE file://`pwd`/job_scripts/lookaround.sh" + f"jobsub_submit -e SAM_EXPERIMENT {extra} --resource-provides=usage_model=OPPORTUNISTIC,DEDICATED,OFFSITE file://`pwd`/job_scripts/lookaround.sh" ) -@pytest.mark.integration def test_launch_lookaround_samdev(samdev): lookaround_launch("--devserver") -@pytest.mark.integration def test_launch_lookaround_dune(dune): lookaround_launch("--devserver") -@pytest.mark.integration def test_launch_lookaround_dune_gp(dune_gp): lookaround_launch("") @@ -165,12 +162,10 @@ def dagnabbit_launch(extra, which=""): os.chdir(os.path.dirname(__file__)) -@pytest.mark.integration def test_launch_dagnabbit_simple(samdev): dagnabbit_launch("--devserver", "") -@pytest.mark.integration def test_launch_dagnabbit_dropbox(samdev): dagnabbit_launch("--devserver", "Dropbox") @@ -179,7 +174,6 @@ def fife_launch(extra): assert run_launch( """ jobsub_submit \ - --debug=1 \ -e EXPERIMENT \ -e IFDH_DEBUG \ -e IFDH_FORCE \ @@ -199,7 +193,7 @@ def fife_launch(extra): --disk=100MB \ --memory=500MB \ %(extra)s \ - --dataset-definition=gen_cfg \ + --dataset=gen_cfg \ file://///grid/fermiapp/products/common/db/../prd/fife_utils/v3_3_2/NULL/libexec/fife_wrap \ --find_setups \ --setup-unquote 'hypotcode%%20v1_1' \ @@ -232,27 +226,22 @@ def fife_launch(extra): ) -@pytest.mark.integration def test_samdev_fife_launch(samdev): fife_launch("--devserver") -@pytest.mark.integration def test_dune_fife_launch(dune): fife_launch("--devserver") -@pytest.mark.integration def test_nova_fife_launch(nova): fife_launch("--devserver") -@pytest.mark.integration def test_dune_gp_fife_launch(dune_gp): fife_launch("") -@pytest.mark.integration def test_wait_for_jobs(): """Not really a test, but we have to wait for jobs to complete...""" count = 1 @@ -289,7 +278,6 @@ def test_wait_for_jobs(): assert True -@pytest.mark.integration def test_fetch_output(): for jid in joblist: if jid.find("dunesched") > 0: @@ -303,7 +291,6 @@ def test_fetch_output(): os.system("jobsub_transfer_data %s" % jid) -@pytest.mark.integration def test_check_job_output(): for outdir in outdirs: fl = glob.glob("%s/*.log" % outdir) diff --git a/tests/test_tarfiles_unit.py b/tests/test_tarfiles_unit.py index 3c67189e..15bcc315 100644 --- a/tests/test_tarfiles_unit.py +++ b/tests/test_tarfiles_unit.py @@ -1,7 +1,6 @@ import os import sys import time -import pytest # # we assume everwhere our current directory is in the package @@ -28,26 +27,22 @@ class TestTarfilesUnit: # lib/tarfiles.py routines... - @pytest.mark.unit def test_tar_up_1(self): """make sure tar up makes a tarfile""" tarfile = tarfiles.tar_up(os.path.dirname(__file__), None) assert os.path.exists(tarfile) os.unlink(tarfile) - @pytest.mark.unit def test_slurp_file_1(self): """make sure tar slurp_file makes a digest""" digest, tf = tarfiles.slurp_file(__file__) assert len(digest) == 64 - @pytest.mark.unit def test_dcache_persistent_path_1(self): """make sure persistent path gives /pnfs/ path digest""" path = tarfiles.dcache_persistent_path(TestUnit.test_group, __file__) assert path[:6] == "/pnfs/" - @pytest.mark.unit def test_tarfile_publisher_1(self, needs_credentials): """test the tarfile publisher object""" proxy, token = needs_credentials @@ -75,7 +70,6 @@ def test_tarfile_publisher_1(self, needs_credentials): assert location is not None - @pytest.mark.unit def test_do_tarballs_1(self, needs_credentials): """test that the do_tarballs method does a dropbox:path processing""" diff --git a/tests/test_utils_unit.py b/tests/test_utils_unit.py index d24576ed..cda6877c 100644 --- a/tests/test_utils_unit.py +++ b/tests/test_utils_unit.py @@ -1,6 +1,5 @@ import os import sys -import pytest # # we assume everwhere our current directory is in the package @@ -29,20 +28,17 @@ class TestUtilsUnit: # utils.py routines... # - @pytest.mark.unit def test_fixquote_1(self): """check the fixquote routine""" assert utils.fixquote("test1") == "test1" assert utils.fixquote("test2=test3") == 'test2="test3"' assert utils.fixquote("test2=test3=test4") == 'test2="test3=test4"' - @pytest.mark.unit def test_grep_n_1(self): """check the grep_n routine on us""" assert utils.grep_n(r"class (\w*):", 1, __file__) == "TestUtilsUnit" assert utils.grep_n(r"import (\w*)", 1, __file__) == "os" - @pytest.mark.unit def test_fix_unit_1(self): """test fixing units on '64gb' memory""" args = TestUnit.test_vargs.copy() @@ -50,14 +46,12 @@ def test_fix_unit_1(self): utils.fix_unit(args, "memory", memtable, -1, "b", -2) assert args["memory"] == 64 * 1024 - @pytest.mark.unit def test_get_principal_1(self): """make sure get_principal returns a string starting with $USER""" # blatantly assumes you have a valid principal... res = utils.get_principal() assert res.split("@")[0] == os.environ["USER"] - @pytest.mark.unit def test_set_extras_1(self, needs_credentials): """call set_extras_n_fix_units, verify one thing from environment and one unit conversion...""" @@ -67,7 +61,6 @@ def test_set_extras_1(self, needs_credentials): assert args["user"] == os.environ["USER"] assert args["memory"] == 64 * 1024 - @pytest.mark.unit def test_get_client_dn_valid_proxy_provided( self, needs_credentials, clear_x509_user_proxy ): @@ -77,7 +70,6 @@ def test_get_client_dn_valid_proxy_provided( client_dn = utils.get_client_dn(proxy=_proxy) assert os.environ["USER"] in client_dn - @pytest.mark.unit def test_get_client_dn_env_plus_proxy_provided(self, needs_credentials): """Call get_client_dn with proxy specified, env set. Should grab proxy from passed-in arg""" @@ -90,7 +82,6 @@ def test_get_client_dn_env_plus_proxy_provided(self, needs_credentials): assert os.environ["USER"] in client_dn os.environ["X509_USER_PROXY"] = old_x509_user_proxy_value - @pytest.mark.unit def test_get_client_dn_no_proxy_provided(self, needs_credentials): """Call get_client_dn with no proxy specified. Should grab proxy from env""" @@ -98,7 +89,6 @@ def test_get_client_dn_no_proxy_provided(self, needs_credentials): client_dn = utils.get_client_dn() assert os.environ["USER"] in client_dn - @pytest.mark.unit def test_get_client_dn_no_proxy_provided_no_env( self, needs_credentials, clear_x509_user_proxy ): @@ -109,7 +99,6 @@ def test_get_client_dn_no_proxy_provided_no_env( client_dn = utils.get_client_dn() assert os.environ["USER"] in client_dn - @pytest.mark.unit def test_get_client_dn_bad_proxy(self): """If we give a bad proxy file, or there's some other problem, we should get "" as the return value"""