From 8de0d448066c225376f723a73b449cb3a4f63bff Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Sat, 31 Mar 2018 08:35:46 -0700 Subject: [PATCH 1/4] wreck: don't ignore lwj.environ in wreckrun/submit wreck execution first reads a global lwj.environ table and uses that as the default environment for all jobs, overridden by any environment variables encoded in the per-job `environ` table. Currently, flux-wreckrun and flux-submit do not check for lwj.environ and instead export the entire (filtered) current environment to each job, so the lwj.environ is largely going ignored. Fix this by first reading a "default environment" from lwj.environ in wreckrun/submit, and only push variables that are not already the same as the default env to the job. This should cut down on KVS size for large numbers of jobs with largely the same, but slightly different, environments. (If all environment tables are the same, then this change does nothing since all environ entries in KVS will be squashed by CAS) --- src/bindings/lua/wreck.lua | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/bindings/lua/wreck.lua b/src/bindings/lua/wreck.lua index 819342241aa5..3df85f883d08 100644 --- a/src/bindings/lua/wreck.lua +++ b/src/bindings/lua/wreck.lua @@ -131,7 +131,8 @@ function wreck:usage() end end -local function get_filtered_env () + +function wreck.get_filtered_env () local env = posix.getenv() env.HOSTNAME = nil env.ENVIRONMENT = nil @@ -142,6 +143,19 @@ local function get_filtered_env () return (env) end +local function get_job_env (arg) + local f = arg.flux + local env = wreck.get_filtered_env () + local default_env = {} + if f then default_env = f:kvs_get ("lwj.environ") or {} end + for k,v in pairs (env) do + -- If same key=value is already in default env no need to + -- export it again, remove: + if default_env[k] == env[k] then env[k] = nil end + end + return (env) +end + local function job_kvspath (f, id) assert (id, "Required argument id missing!") @@ -332,7 +346,7 @@ function wreck:jobreq () ntasks = self.ntasks, ncores = self.ncores, cmdline = self.cmdline, - environ = get_filtered_env (), + environ = get_job_env { flux = self.flux }, cwd = posix.getcwd (), walltime =self.walltime or 0, From aa90d71dddc28b3d3522d3d0f6ab4c43dc565504 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Sat, 31 Mar 2018 08:48:57 -0700 Subject: [PATCH 2/4] wreck: Add -S, --skip-env option to wreckrun/submit Add an option -S, --skip-env to submit and wreckrun to skip the export of current environment to the job. This can be used to speed up submit/wreckrun when the global lwj.environ is sufficient, since that default table doesn't need to be fetched from the KVS before submitting the job to the instance. --- src/bindings/lua/wreck.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bindings/lua/wreck.lua b/src/bindings/lua/wreck.lua index 3df85f883d08..944f3648498e 100644 --- a/src/bindings/lua/wreck.lua +++ b/src/bindings/lua/wreck.lua @@ -52,6 +52,7 @@ local default_opts = { ['error'] = { char = "E", arg = "FILENAME" }, ['input'] = { char = "i", arg = "HOW" }, ['label-io'] = { char = "l", }, + ['skip-env'] = { char = "S", }, ['options'] = { char = 'o', arg = "OPTIONS.." }, } @@ -119,6 +120,7 @@ function wreck:usage() -i /dev/null:* closes all stdin, etc.) -E, --error=FILENAME Send stderr to a different location than stdout. -l, --labelio Prefix lines of output with task id + -S, --skip-env Skip export of environment to job ]]) for _,v in pairs (self.extra_options) do local optstr = v.name .. (v.arg and "="..v.arg or "") @@ -346,7 +348,7 @@ function wreck:jobreq () ntasks = self.ntasks, ncores = self.ncores, cmdline = self.cmdline, - environ = get_job_env { flux = self.flux }, + environ = self.opts.S and {} or get_job_env { flux = self.flux }, cwd = posix.getcwd (), walltime =self.walltime or 0, From c3ead226b09a8bfc7dece335df83bef904f46d73 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Sat, 31 Mar 2018 08:54:57 -0700 Subject: [PATCH 3/4] wreck: add flux wreck environment manipulation commands Add flux-wreck setenv, getenv, unsetenv commands to set, get, and manipulate the global wreck environment under lwj.environ. --- src/cmd/flux-wreck | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/cmd/flux-wreck b/src/cmd/flux-wreck index 55a2f6f851a8..25fc790ac885 100755 --- a/src/cmd/flux-wreck +++ b/src/cmd/flux-wreck @@ -416,6 +416,66 @@ prog:SubCommand { end } +prog:SubCommand { + name = "setenv", + usage = "[VAR=VALUE|all]", + description = "Export environment to all jobs", + handler = function (self, arg) + local f = assert (require 'flux'.new ()) + local getenv = wreck.get_filtered_env + local env = f:kvs_get ("lwj.environ") or {} + for _,expr in ipairs (arg) do + if expr == "all" then + for k,v in pairs(getenv()) do + env[k] = v + end + else + local var,val = expr:match("(.*)=(.*)") + env[var] = val + end + end + f:kvs_put ("lwj.environ", env) + f:kvs_commit () + end +} + +prog:SubCommand { + name = "unsetenv", + usage = "VAR [VAR...]", + description = "Export environment to all jobs", + handler = function (self, arg) + local f = assert (require 'flux'.new ()) + local env = f:kvs_get ("lwj.environ") or {} + for _,var in ipairs (arg) do + env[var] = nil + end + f:kvs_put ("lwj.environ", env) + f:kvs_commit () + end +} + +prog:SubCommand { + name = "getenv", + usage = "VAR [VAR...]", + description = "Export environment to all jobs", + handler = function (self, arg) + local f = assert (require 'flux'.new ()) + local env = f:kvs_get ("lwj.environ") or {} + if #arg == 0 then + for k,v in pairs (env) do + print (k.."="..v) + end + end + for _,k in ipairs (arg) do + local v = env[k] or "" + print (k.."="..v) + end + end +} + + + + -- return keys in dir as a table sorted by number local function sorted_keys (dir) local results = {} From 42b982e6c1fc373a1e6d74ae99544b562355d65e Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Sat, 31 Mar 2018 09:57:22 -0700 Subject: [PATCH 4/4] t/t2000-wreck-env.t: add tests for wreck global environment Add tests for wreck global environment lwj.environ, and corresponding commands to manipulate and use that environemnt. Note: the test is split from t2000-wreck.t but still prefixed with t2000 for easier test devlepment, parallel tests, etc. Since this is short-lived code, it is probably ok for now. --- t/Makefile.am | 2 ++ t/t2000-wreck-env.t | 51 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100755 t/t2000-wreck-env.t diff --git a/t/Makefile.am b/t/Makefile.am index b0bfc9af5dc4..f03e83472613 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -62,6 +62,7 @@ TESTS = \ t1105-proxy.t \ t1999-wreck-rcalc.t \ t2000-wreck.t \ + t2000-wreck-env.t \ t2001-jsc.t \ t2002-pmi.t \ t2003-recurse.t \ @@ -141,6 +142,7 @@ check_SCRIPTS = \ t1105-proxy.t \ t1999-wreck-rcalc.t \ t2000-wreck.t \ + t2000-wreck-env.t \ t2001-jsc.t \ t2002-pmi.t \ t2003-recurse.t \ diff --git a/t/t2000-wreck-env.t b/t/t2000-wreck-env.t new file mode 100755 index 000000000000..ee9407eea6bb --- /dev/null +++ b/t/t2000-wreck-env.t @@ -0,0 +1,51 @@ +#!/bin/sh +# + +test_description='Test basic wreck functionality + +Test basic functionality of wreckrun facility. +' + +. `dirname $0`/sharness.sh +SIZE=${FLUX_TEST_SIZE:-4} +test_under_flux ${SIZE} wreck + +# Return the previous jobid +last_job_id() { + flux wreck last-jobid +} +# Return previous job path in kvs +last_job_path() { + flux wreck last-jobid -p +} + +test_expect_success 'flux-wreck: setenv/getenv works' ' + flux wreck setenv FOO=BAR && + flux wreck getenv FOO +' +test_expect_success 'flux-wreck: unsetenv works' ' + flux wreck unsetenv FOO && + test "$(flux wreck getenv FOO)" = "FOO=" +' +test_expect_success 'flux-wreck: setenv all' ' + flux wreck setenv all && + flux env /usr/bin/env | sort | grep -ve FLUX_URI -e HOSTNAME -e ENVIRONMENT > env.expected && + flux wreck getenv | sort > env.output && + test_cmp env.expected env.output +' +test_expect_success 'wreck: global lwj.environ exported to jobs' ' + flux wreck setenv FOO=bar && + test "$(flux wreckrun -n1 printenv FOO)" = "bar" +' +test_expect_success 'wreck: wreckrun exports environment vars not in global env' ' + BAR=baz flux wreckrun -n1 printenv BAR > printenv.out && + test "$(cat printenv.out)" = "baz" +' +test_expect_success 'wreck: wreckrun --skip-env works' ' + ( export BAR=baz && + test_must_fail flux wreckrun --skip-env -n1 printenv BAR > printenv2.out + ) && + test "$(cat printenv2.out)" = "" +' + +test_done