Skip to content

Introduce -tune command line option to set env vars and mca params from ... #474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 1, 2015
Merged

Introduce -tune command line option to set env vars and mca params from ... #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions opal/mca/base/mca_base_cmd_line.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,18 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
}
}

{
opal_cmd_line_init_t entry =
{"mca_base_envar_file_prefix", '\0', "tune", NULL, 1,
NULL, OPAL_CMD_LINE_TYPE_STRING,
"Application profile options file list"
};
ret = opal_cmd_line_make_opt_mca(cmd, entry);
if (OPAL_SUCCESS != ret) {
return ret;
}
}

return ret;
}

Expand Down
5 changes: 5 additions & 0 deletions opal/mca/base/mca_base_parse_paramfile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ int mca_base_parse_paramfile(const char *paramfile, opal_list_t *list)
return opal_util_keyval_parse(paramfile, save_value);
}

int mca_base_internal_env_store(void)
{
return opal_util_keyval_save_internal_envars(save_value);
}

static void save_value(const char *name, const char *value)
{
mca_base_var_file_value_t *fv;
Expand Down
151 changes: 104 additions & 47 deletions opal/mca/base/mca_base_var.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,18 @@ static char *cwd = NULL;
bool mca_base_var_initialized = false;
static char * force_agg_path = NULL;
static char *mca_base_var_files = NULL;
static char *mca_base_envar_files = NULL;
static char **mca_base_var_file_list = NULL;
static char *mca_base_var_override_file = NULL;
static char *mca_base_var_file_prefix = NULL;
static char *mca_base_envar_file_prefix = NULL;
static char *mca_base_param_file_path = NULL;
static char *mca_base_env_list = NULL;
static char *mca_base_env_list_sep = ";";
static char *mca_base_env_list_internal = NULL;
static bool mca_base_var_suppress_override_warning = false;
static opal_list_t mca_base_var_file_values;
static opal_list_t mca_base_envar_file_values;
static opal_list_t mca_base_var_override_values;

static int mca_base_var_count = 0;
Expand Down Expand Up @@ -123,8 +127,8 @@ static const char *info_lvl_strings[] = {
/*
* local functions
*/
static int fixup_files(char **file_list, char * path, bool rel_path_search);
static int read_files (char *file_list, opal_list_t *file_values);
static int fixup_files(char **file_list, char * path, bool rel_path_search, char sep);
static int read_files (char *file_list, opal_list_t *file_values, char sep);
static int mca_base_var_cache_files (bool rel_path_search);
static int var_set_initial (mca_base_var_t *var);
static int var_get (int vari, mca_base_var_t **var_out, bool original);
Expand Down Expand Up @@ -239,6 +243,7 @@ int mca_base_var_init(void)
/* Init the file param value list */

OBJ_CONSTRUCT(&mca_base_var_file_values, opal_list_t);
OBJ_CONSTRUCT(&mca_base_envar_file_values, opal_list_t);
OBJ_CONSTRUCT(&mca_base_var_override_values, opal_list_t);
OBJ_CONSTRUCT(&mca_base_var_index_hash, opal_hash_table_t);

Expand Down Expand Up @@ -285,33 +290,27 @@ int mca_base_var_init(void)
free(name);
}
}

/* Register internal MCA variable mca_base_env_list_internal. It can be set only during
* parsing of amca conf file and contains SHELL env variables specified via -x there.
* Its format is the same as for mca_base_env_list.
*/
(void)mca_base_var_register ("opal", "mca", "base", "env_list_internal",
"Store SHELL env variables from amca conf file",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_internal);
}

return OPAL_SUCCESS;
}

int mca_base_var_process_env_list(char ***argv)
static void process_env_list(char *env_list, char ***argv, char sep)
{
int i;
char** tokens;
char* ptr;
char* param, *value;
char sep;

if (NULL == mca_base_env_list) {
return OPAL_SUCCESS;
}
sep = ';';
if (NULL != mca_base_env_list_sep) {
if (1 == strlen(mca_base_env_list_sep)) {
sep = mca_base_env_list_sep[0];
} else {
opal_show_help("help-mca-var.txt", "incorrect-env-list-sep",
true, mca_base_env_list_sep);
return OPAL_SUCCESS;
}
}
tokens = opal_argv_split(mca_base_env_list, (int)sep);
tokens = opal_argv_split(env_list, (int)sep);
if (NULL != tokens) {
for (i = 0; NULL != tokens[i]; i++) {
if (NULL == (ptr = strchr(tokens[i], '='))) {
Expand All @@ -329,7 +328,7 @@ int mca_base_var_process_env_list(char ***argv)
}
} else {
opal_show_help("help-mca-var.txt", "incorrect-env-list-param",
true, tokens[i], mca_base_env_list);
true, tokens[i], env_list);
}
} else {
param = strdup(tokens[i]);
Expand All @@ -342,9 +341,59 @@ int mca_base_var_process_env_list(char ***argv)
}
opal_argv_free(tokens);
}
}

int mca_base_var_process_env_list(char ***argv)
{
char sep;
sep = ';';
if (NULL != mca_base_env_list_sep) {
if (1 == strlen(mca_base_env_list_sep)) {
sep = mca_base_env_list_sep[0];
} else {
opal_show_help("help-mca-var.txt", "incorrect-env-list-sep",
true, mca_base_env_list_sep);
return OPAL_SUCCESS;
}
}
if (NULL != mca_base_env_list) {
process_env_list(mca_base_env_list, argv, sep);
}

return OPAL_SUCCESS;
}

int mca_base_var_process_env_list_from_file(char ***argv)
{
if (NULL != mca_base_env_list_internal) {
process_env_list(mca_base_env_list_internal, argv, ';');
}
return OPAL_SUCCESS;
}

static void resolve_relative_paths(char **file_prefix, char *file_path, bool rel_path_search, char **files, char sep)
{
char *tmp_str;
/*
* Resolve all relative paths.
* the file list returned will contain only absolute paths
*/
if( OPAL_SUCCESS != fixup_files(file_prefix, file_path, rel_path_search, sep) ) {
#if 0
/* JJH We need to die! */
abort();
#else
;
#endif
}
else {
/* Prepend the files to the search list */
asprintf(&tmp_str, "%s%c%s", *file_prefix, sep, *files);
free (*files);
*files = tmp_str;
}
}

static int mca_base_var_cache_files(bool rel_path_search)
{
char *tmp;
Expand Down Expand Up @@ -381,6 +430,7 @@ static int mca_base_var_cache_files(bool rel_path_search)
if (OPAL_SUCCESS != ret) {
return ret;
}
mca_base_envar_files = strdup(mca_base_var_files);

(void) mca_base_var_register_synonym (ret, "opal", "mca", NULL, "param_files",
MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
Expand Down Expand Up @@ -430,6 +480,15 @@ static int mca_base_var_cache_files(bool rel_path_search)
return ret;
}

mca_base_envar_file_prefix = NULL;
ret = mca_base_var_register ("opal", "mca", "base", "envar_file_prefix",
"Aggregate MCA parameter file set for env variables",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY, &mca_base_envar_file_prefix);
if (0 > ret) {
return ret;
}

ret = asprintf(&mca_base_param_file_path, "%s" OPAL_PATH_SEP "amca-param-sets%c%s",
opal_install_dirs.opaldatadir, OPAL_ENV_SEP, cwd);
if (0 > ret) {
Expand Down Expand Up @@ -467,32 +526,17 @@ static int mca_base_var_cache_files(bool rel_path_search)
}

if (NULL != mca_base_var_file_prefix) {
char *tmp_str;

/*
* Resolve all relative paths.
* the file list returned will contain only absolute paths
*/
if( OPAL_SUCCESS != fixup_files(&mca_base_var_file_prefix, mca_base_param_file_path, rel_path_search) ) {
#if 0
/* JJH We need to die! */
abort();
#else
;
#endif
}
else {
/* Prepend the files to the search list */
asprintf(&tmp_str, "%s%c%s", mca_base_var_file_prefix, OPAL_ENV_SEP, mca_base_var_files);
free (mca_base_var_files);
mca_base_var_files = tmp_str;
}
resolve_relative_paths(&mca_base_var_file_prefix, mca_base_param_file_path, rel_path_search, &mca_base_var_files, OPAL_ENV_SEP);
}
read_files (mca_base_var_files, &mca_base_var_file_values, OPAL_ENV_SEP);

read_files (mca_base_var_files, &mca_base_var_file_values);
if (NULL != mca_base_envar_file_prefix) {
resolve_relative_paths(&mca_base_envar_file_prefix, mca_base_param_file_path, rel_path_search, &mca_base_envar_files, ',');
}
read_files (mca_base_envar_files, &mca_base_envar_file_values, ',');

if (0 == access(mca_base_var_override_file, F_OK)) {
read_files (mca_base_var_override_file, &mca_base_var_override_values);
read_files (mca_base_var_override_file, &mca_base_var_override_values, OPAL_ENV_SEP);
}

return OPAL_SUCCESS;
Expand Down Expand Up @@ -1064,6 +1108,12 @@ int mca_base_var_finalize(void)
}
OBJ_DESTRUCT(&mca_base_var_file_values);

while (NULL !=
(item = opal_list_remove_first(&mca_base_envar_file_values))) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&mca_base_envar_file_values);

while (NULL !=
(item = opal_list_remove_first(&mca_base_var_override_values))) {
OBJ_RELEASE(item);
Expand Down Expand Up @@ -1095,7 +1145,7 @@ int mca_base_var_finalize(void)


/*************************************************************************/
static int fixup_files(char **file_list, char * path, bool rel_path_search) {
static int fixup_files(char **file_list, char * path, bool rel_path_search, char sep) {
int exit_status = OPAL_SUCCESS;
char **files = NULL;
char **search_path = NULL;
Expand All @@ -1105,7 +1155,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search) {
int count, i, argc = 0;

search_path = opal_argv_split(path, OPAL_ENV_SEP);
files = opal_argv_split(*file_list, OPAL_ENV_SEP);
files = opal_argv_split(*file_list, sep);
count = opal_argv_count(files);

/* Read in reverse order, so we can preserve the original ordering */
Expand Down Expand Up @@ -1167,7 +1217,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search) {
tmp_file = NULL;

free(*file_list);
*file_list = opal_argv_join(argv, OPAL_ENV_SEP);
*file_list = opal_argv_join(argv, sep);

cleanup:
if( NULL != files ) {
Expand All @@ -1189,21 +1239,23 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search) {
return exit_status;
}

static int read_files(char *file_list, opal_list_t *file_values)
static int read_files(char *file_list, opal_list_t *file_values, char sep)
{
int i, count;

/* Iterate through all the files passed in -- read them in reverse
order so that we preserve unix/shell path-like semantics (i.e.,
the entries farthest to the left get precedence) */

mca_base_var_file_list = opal_argv_split(file_list, OPAL_ENV_SEP);
mca_base_var_file_list = opal_argv_split(file_list, sep);
count = opal_argv_count(mca_base_var_file_list);

for (i = count - 1; i >= 0; --i) {
mca_base_parse_paramfile(mca_base_var_file_list[i], file_values);
}

mca_base_internal_env_store();

return OPAL_SUCCESS;
}

Expand Down Expand Up @@ -1723,6 +1775,11 @@ static int var_set_initial (mca_base_var_t *var)
return ret;
}

ret = var_set_from_file (var, &mca_base_envar_file_values);
if (OPAL_ERR_NOT_FOUND != ret) {
return ret;
}

ret = var_set_from_file (var, &mca_base_var_file_values);
if (OPAL_ERR_NOT_FOUND != ret) {
return ret;
Expand Down
1 change: 1 addition & 0 deletions opal/mca/base/mca_base_var.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,7 @@ OPAL_DECLSPEC int mca_base_var_dump(int vari, char ***out, mca_base_var_dump_typ
* their assigned value, to the provided argv
*/
OPAL_DECLSPEC int mca_base_var_process_env_list(char ***argv);
OPAL_DECLSPEC int mca_base_var_process_env_list_from_file(char ***argv);

END_C_DECLS

Expand Down
7 changes: 7 additions & 0 deletions opal/mca/base/mca_base_vari.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,13 @@ OPAL_DECLSPEC int mca_base_var_generate_full_name4 (const char *project, const c
const char *component, const char *variable,
char **full_name);

/**
* \internal
*
* Call save_value callback for generated internal mca parameter storing env variables
*/
OPAL_DECLSPEC int mca_base_internal_env_store(void);

/**
* \internal
*
Expand Down
3 changes: 3 additions & 0 deletions opal/util/keyval/keyval_lex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ enum {
OPAL_UTIL_KEYVAL_PARSE_EQUAL,
OPAL_UTIL_KEYVAL_PARSE_SINGLE_WORD,
OPAL_UTIL_KEYVAL_PARSE_VALUE,
OPAL_UTIL_KEYVAL_PARSE_MCAVAR,
OPAL_UTIL_KEYVAL_PARSE_ENVVAR,
OPAL_UTIL_KEYVAL_PARSE_ENVEQL,

OPAL_UTIL_KEYVAL_PARSE_MAX
};
Expand Down
10 changes: 10 additions & 0 deletions opal/util/keyval/keyval_lex.l
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ CHAR [A-Za-z0-9_\-\.]

%x VALUE
%x comment
%x MCA_VALUE

%%

Expand All @@ -78,6 +79,15 @@ CHAR [A-Za-z0-9_\-\.]
<VALUE>{WHITE}*\n { BEGIN(INITIAL); return OPAL_UTIL_KEYVAL_PARSE_NEWLINE; }
<VALUE>[^\n]*[^\t \n]/[\t ]* { return OPAL_UTIL_KEYVAL_PARSE_VALUE; }

<MCA_VALUE>[^\n \t]+ { BEGIN(INITIAL); return OPAL_UTIL_KEYVAL_PARSE_VALUE; }
<MCA_VALUE>"'"[^\n]*"'"{WHITE}+ { BEGIN(INITIAL); return OPAL_UTIL_KEYVAL_PARSE_VALUE; }
<MCA_VALUE>"\""[^\n]*"\""{WHITE}+ { BEGIN(INITIAL); return OPAL_UTIL_KEYVAL_PARSE_VALUE; }
<MCA_VALUE>{WHITE}*\n { BEGIN(INITIAL); BEGIN(INITIAL); return OPAL_UTIL_KEYVAL_PARSE_NEWLINE; }

"-"?"-mca"{WHITE}+{CHAR}+{WHITE}+ {BEGIN(MCA_VALUE); return OPAL_UTIL_KEYVAL_PARSE_MCAVAR; }
"-"?"-x"{WHITE}+{CHAR}+{WHITE}*"="{WHITE}* {BEGIN(MCA_VALUE); return OPAL_UTIL_KEYVAL_PARSE_ENVEQL; }
"-"?"-x"{WHITE}+{CHAR}+{WHITE}*/[^=] { return OPAL_UTIL_KEYVAL_PARSE_ENVVAR; }

. { return OPAL_UTIL_KEYVAL_PARSE_ERROR; }

%%
Expand Down
Loading