From d6780db16e43086cc4f5220f2c2a50f77f16bfbf Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Thu, 21 Feb 2019 14:32:08 +0200 Subject: [PATCH 1/9] DEV-1 (Pluggable Preprocessor): Prepare code layout * add/document new "-p" (preprocessor) cmdline option * hook the new "-p" option into the parsing code * re-organize cfg file managing code into a separate file --- cfg.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ cfg.h | 28 ++++++++++++++++++++++ help_msg.h | 4 ++++ main.c | 49 +++++++------------------------------- 4 files changed, 111 insertions(+), 40 deletions(-) create mode 100644 cfg.c create mode 100644 cfg.h diff --git a/cfg.c b/cfg.c new file mode 100644 index 00000000000..507788de33a --- /dev/null +++ b/cfg.c @@ -0,0 +1,70 @@ +/* + * OpenSIPS configuration file parsing + * + * Copyright (C) 2019 OpenSIPS Solutions + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA + */ + +#include +#include + +#include "config.h" +#include "globals.h" +#include "cfg.h" + +extern FILE *yyin; +extern int yyparse(); +#ifdef DEBUG_PARSER +extern int yydebug; +#endif + +int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) +{ + FILE *cfg_stream; + + /* fill missing arguments with the default values*/ + if (!cfg_file) + cfg_file = CFG_FILE; + + if (strlen(cfg_file) == 1 && cfg_file[0] == '-') { + cfg_stream = stdin; + } else { + /* load config file or die */ + cfg_stream = fopen(cfg_file, "r"); + if (!cfg_stream) { + LM_ERR("loading config file %s: %s\n", cfg_file, + strerror(errno)); + return -1; + } + } + +#ifdef DEBUG_PARSER + /* used for parser debugging */ + yydebug = 1; +#endif + + /* parse the config file, prior to this only default values + e.g. for debugging settings will be used */ + yyin = cfg_stream; + if (yyparse() != 0 || cfg_errors) { + LM_ERR("bad config file (%d errors)\n", cfg_errors); + return -1; + } + + return 0; +} diff --git a/cfg.h b/cfg.h new file mode 100644 index 00000000000..fe1c18d6f99 --- /dev/null +++ b/cfg.h @@ -0,0 +1,28 @@ +/* + * OpenSIPS configuration file parsing + * + * Copyright (C) 2019 OpenSIPS Solutions + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA + */ + +#ifndef __OSS_CFG_H__ +#define __OSS_CFG_H__ + +int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline); + +#endif /* __OSS_CFG_H__ */ diff --git a/help_msg.h b/help_msg.h index 2268f45ba2d..71266f8d03f 100644 --- a/help_msg.h +++ b/help_msg.h @@ -94,6 +94,10 @@ Options:\n\ -t dir Chroot to \"dir\"\n\ -u uid Change uid \n\ -g gid Change gid \n\ + -p pp_cmd Preprocess the configuration file (along with any others\n\ + included) using the specified system command. The command \n\ + shall receive input via stdin and it must output the\n\ + result to stdout\n\ -P file Create a pid file\n\ -G file Create a pgid file\n" #ifdef UNIT_TESTS diff --git a/main.c b/main.c index 170ad3bebbd..f7a4fa188bb 100644 --- a/main.c +++ b/main.c @@ -100,6 +100,7 @@ #include "help_msg.h" #include "config.h" +#include "cfg.h" #include "dprint.h" #include "daemonize.h" #include "route.h" @@ -300,13 +301,6 @@ unsigned long pkg_mem_size=PKG_MEM_SIZE * 1024 * 1024; int my_argc; char **my_argv; -extern FILE* yyin; -extern int yyparse(); -#ifdef DEBUG_PARSER -extern int yydebug; -#endif - - int is_main = 1; /* flag = is this the "main" process? */ char* pid_file = 0; /* filename as asked by user */ @@ -876,9 +870,8 @@ int main(int argc, char** argv) { /* configure by default logging to syslog */ int cfg_log_stderr = 1; - FILE* cfg_stream = NULL; int c,r; - char *tmp; + char *tmp, *preproc = NULL; int tmp_len; int port; int proto; @@ -895,7 +888,7 @@ int main(int argc, char** argv) /* process pkg mem size from command line */ opterr=0; - options="f:cCm:M:b:l:n:N:rRvdDFEVhw:t:u:g:P:G:W:o:" + options="f:cCm:M:b:l:n:N:rRvdDFEVhw:t:u:g:p:P:G:W:o:" #ifdef UNIT_TESTS "T" #endif @@ -1063,6 +1056,9 @@ int main(int argc, char** argv) case 'g': /* ignoring it, parsed previously */ break; + case 'p': + preproc=optarg; + break; case 'P': pid_file=optarg; break; @@ -1095,23 +1091,6 @@ int main(int argc, char** argv) log_stderr = cfg_log_stderr; - if (!testing_framework) { - /* fill missing arguments with the default values*/ - if (cfg_file==0) cfg_file=CFG_FILE; - - if (strlen(cfg_file) == 1 && cfg_file[0] == '-') { - cfg_stream = stdin; - } else { - /* load config file or die */ - cfg_stream=fopen (cfg_file, "r"); - if (cfg_stream==0){ - LM_ERR("loading config file(%s): %s\n", cfg_file, - strerror(errno)); - goto error00; - } - } - } - /* seed the prng, try to use /dev/urandom if possible */ /* no debugging information is logged, because the standard log level prior the config file parsing is L_NOTICE */ @@ -1144,11 +1123,6 @@ int main(int argc, char** argv) goto error; } - /* used for parser debugging */ -#ifdef DEBUG_PARSER - yydebug = 1; -#endif - /* init shm mallocs * this must be here * -to allow setting shm mem size from the command line @@ -1166,14 +1140,9 @@ int main(int argc, char** argv) set_osips_state( STATE_STARTING ); - if (!testing_framework) { - /* parse the config file, prior to this only default values - e.g. for debugging settings will be used */ - yyin=cfg_stream; - if ((yyparse()!=0)||(cfg_errors)){ - LM_ERR("bad config file (%d errors)\n", cfg_errors); - goto error00; - } + if (!testing_framework && parse_opensips_cfg(cfg_file, preproc) < 0) { + LM_ERR("failed to parse config file\n"); + goto error00; } /* shm statistics, module stat groups, memory warming */ From e83904d75666eca38dbb0f16350e7531d5c76d4e Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Thu, 21 Feb 2019 19:51:31 +0200 Subject: [PATCH 2/9] DEV-1: Implement config file flattening + adnotations The main idea is to be able to feed the preprocessor with a contiguous stream of input. The consequence of doing this is that we must pre-resolve all "include_file" statements by creating one large cfg file, in a process called "flattening". Example of flattening + adnotation: ------ cfg/opensips-3.0-preproc-1.cfg ------ log_level = 4 log_stderror = yes include_file "cfg/opensips-3.0-preproc-2.cfg" loadmodule "tm.so" include_file "cfg/opensips-3.0-preproc-4.cfg" loadmodule "dialog.so" ------ cfg/opensips-3.0-preproc-2.cfg ------ listen = udp:*:5060 include_file "cfg/opensips-3.0-preproc-3.cfg" ------ cfg/opensips-3.0-preproc-3.cfg ------ mpath = "modules/" loadmodule "proto_udp.so" ------ cfg/opensips-3.0-preproc-4.cfg ------ loadmodule "usrloc.so" These four files with nested includes will be flattened into this beast: --------------------------------------------------- __OSSPP_FILEBEGIN__ "cfg/opensips-3.0-preproc-1.cfg" __OSSPP_LINE__ 1 log_level = 4 __OSSPP_LINE__ 2 log_stderror = yes __OSSPP_LINE__ 3 __OSSPP_LINE__ 4 __OSSPP_FILEBEGIN__ "cfg/opensips-3.0-preproc-2.cfg" __OSSPP_LINE__ 1 listen = udp:*:5060 __OSSPP_LINE__ 2 __OSSPP_FILEBEGIN__ "cfg/opensips-3.0-preproc-3.cfg" __OSSPP_LINE__ 1 mpath = "modules/" __OSSPP_LINE__ 2 loadmodule "proto_udp.so" __OSSPP_FILEEND__ __OSSPP_FILEEND__ __OSSPP_LINE__ 5 loadmodule "tm.so" __OSSPP_LINE__ 6 __OSSPP_FILEBEGIN__ "cfg/opensips-3.0-preproc-4.cfg" __OSSPP_LINE__ 1 loadmodule "usrloc.so" __OSSPP_FILEEND__ __OSSPP_LINE__ 7 loadmodule "dialog.so" __OSSPP_FILEEND__ --------------------------------------------------- --- cfg.c | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.c | 2 +- 2 files changed, 255 insertions(+), 1 deletion(-) diff --git a/cfg.c b/cfg.c index 507788de33a..28ebef36ddb 100644 --- a/cfg.c +++ b/cfg.c @@ -20,12 +20,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA */ +#include #include #include #include "config.h" #include "globals.h" #include "cfg.h" +#include "ut.h" extern FILE *yyin; extern int yyparse(); @@ -33,6 +35,15 @@ extern int yyparse(); extern int yydebug; #endif +str include_v1 = str_init("include_file"); +str include_v2 = str_init("import_file"); + +str cfgtok_line = str_init("__OSSPP_LINE__"); +str cfgtok_filebegin = str_init("__OSSPP_FILEBEGIN__"); +str cfgtok_fileend = str_init("__OSSPP_FILEEND__"); + +static FILE *flatten_opensips_cfg(FILE *cfg, const char *cfg_path); + int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) { FILE *cfg_stream; @@ -53,6 +64,19 @@ int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) } } + cfg_stream = flatten_opensips_cfg(cfg_stream, + cfg_stream == stdin ? "stdin" : cfg_file); + if (!cfg_stream) { + LM_ERR("failed to expand file imports for %s, oom?\n", cfg_file); + return -1; + } + + if (preproc_cmdline) { + // TODO: some dup magic, to push into pp stdin / read from its stdout + // TODO: execvp(chopped(preproc_cmdline), flattened_cfg) + // TODO: cfg_stream = fdopen(stdout_fd); + } + #ifdef DEBUG_PARSER /* used for parser debugging */ yydebug = 1; @@ -63,8 +87,238 @@ int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) yyin = cfg_stream; if (yyparse() != 0 || cfg_errors) { LM_ERR("bad config file (%d errors)\n", cfg_errors); + fclose(cfg_stream); + return -1; + } + + fclose(cfg_stream); + return 0; +} + +static int extend_cfg_buf(char **buf, int *sz, int *bytes_left) +{ + *buf = realloc(*buf, *sz + 4096); + if (!*buf) { + LM_ERR("oom\n"); return -1; } + *sz += 4096; + *bytes_left += 4096; return 0; } + +/* search for '(include|import)_file "filepath"' patterns */ +int extract_included_file(char *line, int line_len, char **out_path) +{ + str lin; + char *fstart, *p = NULL, enclose = 0; + + lin.s = line; + lin.len = line_len; + + if (line_len > include_v1.len && + !memcmp(line, include_v1.s, include_v1.len)) { + p = line + include_v1.len; + line_len -= include_v1.len; + } else if (line_len > include_v2.len && + !memcmp(line, include_v2.s, include_v2.len)) { + p = line + include_v2.len; + line_len -= include_v2.len; + } + + if (!p) + return -1; + + while (line_len > 0 && isspace(*p)) { + line_len--; + p++; + } + + if (line_len < 3) // "f" + return -1; + + if (*p != '"' && *p != '\'') + return -1; + + enclose = *p++; + line_len--; + + fstart = p; + + while (line_len > 0 && *p != enclose) { + line_len--; + p++; + } + + if (line_len == 0 || p - fstart < 2) // ""_ + return -1; + + *out_path = malloc(p - fstart); + if (!*out_path) { + LM_ERR("oom\n"); + return -1; + } + + memcpy(*out_path, fstart, p - fstart); + return 0; +} + +static int __flatten_opensips_cfg(FILE *cfg, const char *cfg_path, + char **flattened, int *sz, int *bytes_left) +{ + FILE *included_cfg; + ssize_t line_len; + char *line = NULL, *included_cfg_path; + unsigned long line_buf_sz = 0; + int cfg_path_len = strlen(cfg_path); + int line_counter = 1, printed; + + if (cfg_path_len >= 2048) { + LM_ERR("file path too large: %.*s...\n", 2048, cfg_path); + goto out_err; + } + + if (*bytes_left < cfgtok_filebegin.len + 1 + 1+cfg_path_len+1 + 1) { + if (extend_cfg_buf(flattened, sz, bytes_left) < 0) { + LM_ERR("oom\n"); + goto out_err; + } + } + + /* print "start of file" adnotation */ + sprintf(*flattened + *sz - *bytes_left, "%.*s \"%.*s\"\n", + cfgtok_filebegin.len, cfgtok_filebegin.s, cfg_path_len, cfg_path); + *bytes_left -= cfgtok_filebegin.len + 1 +1+cfg_path_len+1 + 1; + + for (;;) { + line_len = getline(&line, &line_buf_sz, cfg); + if (line_len == -1) { + if (ferror(cfg)) { + if (errno == EINTR) { + continue; + } else { + LM_ERR("failed to read from cfg file %.*s: %d (%s)\n", + cfg_path_len, cfg_path, errno, strerror(errno)); + goto out_err; + } + } + + if (!feof(cfg)) { + LM_ERR("unhandled read error in cfg file %.*s: %d (%s)\n", + cfg_path_len, cfg_path, errno, strerror(errno)); + goto out_err; + } + + line_len = strlen(line); + + break; + } else if (line_len == 0) { + continue; + } + + /* fix ending lines with a missing '\n' character ;) */ + if (feof(cfg)) { + if (line[line_len - 1] != '\n') { + if (line_buf_sz < line_len + 1) { + line = realloc(line, line_len + 1); + line_buf_sz = line_len + 1; + } + + line[line_len] = '\n'; + line_len += 1; + } + } + + /* finally... we have a line! print "line number" adnotation */ + if (*bytes_left < cfgtok_line.len + 1 + 10) { + if (extend_cfg_buf(flattened, sz, bytes_left) < 0) { + LM_ERR("oom\n"); + goto out_err; + } + } + + printed = sprintf(*flattened + *sz - *bytes_left, "%.*s %d\n", + cfgtok_line.len, cfgtok_line.s, line_counter); + line_counter++; + *bytes_left -= printed; + + /* if it's an include, skip printing the line, but do print the file */ + if (extract_included_file(line, line_len, &included_cfg_path) == 0) { + included_cfg = fopen(included_cfg_path, "r"); + if (!included_cfg) { + LM_ERR("failed to open %s: %d (%s)\n", included_cfg_path, + errno, strerror(errno)); + goto out_err; + } + + if (__flatten_opensips_cfg(included_cfg, included_cfg_path, + flattened, sz, bytes_left)) { + LM_ERR("failed to flatten cfg file (internal err), oom?\n"); + fclose(included_cfg); + goto out_err; + } + + free(included_cfg_path); + fclose(included_cfg); + } else { + if (*bytes_left < line_len) { + if (extend_cfg_buf(flattened, sz, bytes_left) < 0) { + LM_ERR("oom\n"); + goto out_err; + } + } + + printed = sprintf(*flattened + *sz - *bytes_left, "%.*s", + (int)line_len, line); + *bytes_left -= printed; + } + } + + free(line); + + if (*bytes_left < cfgtok_fileend.len + 1) { + if (extend_cfg_buf(flattened, sz, bytes_left) < 0) { + LM_ERR("oom\n"); + goto out_err; + } + } + + /* print "end of file" adnotation */ + sprintf(*flattened + *sz - *bytes_left, "%.*s\n", + cfgtok_fileend.len, cfgtok_fileend.s); + *bytes_left -= cfgtok_fileend.len + 1; + + fclose(cfg); + return 0; + +out_err: + fclose(cfg); + return -1; +} + +/* + * - flatten any recursive includes into one big resulting file + * - adnotate each line of the final file + * - close given FILE * and return a new one, corresponding to the new file + */ +static FILE *flatten_opensips_cfg(FILE *cfg, const char *cfg_path) +{ + int sz = 0, bytes_left = 0; + char *flattened = NULL; + + if (__flatten_opensips_cfg(cfg, cfg_path, &flattened, &sz, &bytes_left)) { + LM_ERR("failed to flatten cfg file (internal err), out of memory?\n"); + return NULL; + } + +#ifdef EXTRA_DEBUG + LM_NOTICE("flattened config file:\n%.*s\n", sz - bytes_left, flattened); +#endif + + cfg = fmemopen(flattened, sz, "r"); + if (!cfg) + LM_ERR("failed to obtain file for flattened cfg buffer\n"); + + return cfg; +} diff --git a/main.c b/main.c index f7a4fa188bb..3f06f0cb86c 100644 --- a/main.c +++ b/main.c @@ -1141,7 +1141,7 @@ int main(int argc, char** argv) set_osips_state( STATE_STARTING ); if (!testing_framework && parse_opensips_cfg(cfg_file, preproc) < 0) { - LM_ERR("failed to parse config file\n"); + LM_ERR("failed to parse config file %s\n", cfg_file); goto error00; } From 3050f7c57f779538980f0e1a4d5e09dac778e3e6 Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Thu, 21 Feb 2019 23:25:26 +0200 Subject: [PATCH 3/9] DEV-1: Hook the newly added logic into lexer/parser --- cfg.lex | 12 ++++++++++-- cfg.y | 26 ++++++++++++++++++++++++- cfg.c => cfg_pp.c | 48 +++++++++++++++++++++++++++++++++++++++++++++-- cfg.h => cfg_pp.h | 12 ++++++++---- main.c | 2 +- 5 files changed, 90 insertions(+), 10 deletions(-) rename cfg.c => cfg_pp.c (89%) rename cfg.h => cfg_pp.h (80%) diff --git a/cfg.lex b/cfg.lex index 959d50d9c53..04695b824c1 100644 --- a/cfg.lex +++ b/cfg.lex @@ -91,7 +91,7 @@ int column=1; int startcolumn=1; int startline=1; - char *finame = 0; + const char *finame = 0; static char* addchar(struct str_buf *, char); static char* addstr(struct str_buf *, char*, int); @@ -215,6 +215,9 @@ SYNC_TOKEN "sync" ASYNC_TOKEN "async" LAUNCH_TOKEN "launch" IS_MYSELF "is_myself" +PPTOK_LINE "__OSSPP_LINE__" +PPTOK_FILEBEG "__OSSPP_FILEBEGIN__" +PPTOK_FILEEND "__OSSPP_FILEEND__" /*ACTION LVALUES*/ URIHOST "uri:host" @@ -531,6 +534,12 @@ IMPORTFILE "import_file" return LAUNCH_TOKEN;} {IS_MYSELF} { count(); yylval.strval=yytext; return IS_MYSELF;} +{PPTOK_LINE} { count(); yylval.strval=yytext; + return PPTOK_LINE;} +{PPTOK_FILEBEG} { count(); yylval.strval=yytext; + return PPTOK_FILEBEG;} +{PPTOK_FILEEND} { count(); yylval.strval=yytext; + return PPTOK_FILEEND;} {FORK} { count(); yylval.strval=yytext; return FORK; /*obsolete*/ } {DEBUG_MODE} { count(); yylval.strval=yytext; return DEBUG_MODE; } @@ -945,7 +954,6 @@ static void count(void) startcolumn=column; for (i=0; i= + CFG_MAX_INCLUDE_DEPTH) { + LM_ERR("max nested cfg files reached! (%d)\n", CFG_MAX_INCLUDE_DEPTH); + return -1; + } else { + cfg_include_stackp++; + } + + *cfg_include_stackp = cfg_file; + + finame = cfg_file; + startline = 1; + column = 1; + return 0; +} + +int cfg_pop(void) +{ + if (!cfg_include_stackp) { + LM_ERR("no more files to pop!\n"); + return -1; + } + + if (cfg_include_stackp == cfg_include_stack) { + cfg_include_stackp = NULL; + } else { + cfg_include_stackp--; + finame = *cfg_include_stackp; + column = 1; + } + + return 0; +} diff --git a/cfg.h b/cfg_pp.h similarity index 80% rename from cfg.h rename to cfg_pp.h index fe1c18d6f99..c0462016183 100644 --- a/cfg.h +++ b/cfg_pp.h @@ -1,5 +1,5 @@ /* - * OpenSIPS configuration file parsing + * OpenSIPS configuration file pre-processing * * Copyright (C) 2019 OpenSIPS Solutions * @@ -20,9 +20,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,USA */ -#ifndef __OSS_CFG_H__ -#define __OSS_CFG_H__ +#ifndef __OSS_CFG_PP_H__ +#define __OSS_CFG_PP_H__ + +#define CFG_MAX_INCLUDE_DEPTH 20 int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline); +int cfg_push(const char *cfg_file); +int cfg_pop(void); -#endif /* __OSS_CFG_H__ */ +#endif /* __OSS_CFG_PP_H__ */ diff --git a/main.c b/main.c index 3f06f0cb86c..161e45747ce 100644 --- a/main.c +++ b/main.c @@ -100,7 +100,7 @@ #include "help_msg.h" #include "config.h" -#include "cfg.h" +#include "cfg_pp.h" #include "dprint.h" #include "daemonize.h" #include "route.h" From 9f251028a8d92076abb8d9929c28f9fa9daa3770 Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Thu, 21 Feb 2019 23:26:27 +0200 Subject: [PATCH 4/9] DEV-1: Clear deprecated "stacked Yacc buffers" logic --- cfg.lex | 257 +------------------------------------------------------- 1 file changed, 2 insertions(+), 255 deletions(-) diff --git a/cfg.lex b/cfg.lex index 04695b824c1..3f334737567 100644 --- a/cfg.lex +++ b/cfg.lex @@ -81,7 +81,6 @@ int left; }; - static int comment_nest=0; static int state=0; static struct str_buf s_buf; @@ -97,27 +96,6 @@ static char* addstr(struct str_buf *, char*, int); static void count(); -#define MAX_INCLUDE_DEPTH 10 -#define MAX_INCLUDE_FNAME 128 - - static struct oss_yy_state { - YY_BUFFER_STATE state; - int line; - int column; - int startcolumn; - int startline; - char *finame; - } include_stack[MAX_INCLUDE_DEPTH]; - static int include_stack_ptr = 0; - - static int oss_push_yy_state(char *fin, int mode); - static int oss_pop_yy_state(void); - - static struct oss_yy_fname { - char *fname; - struct oss_yy_fname *next; - } *oss_yy_fname_list = 0; - /* hack to solve the duplicate declaration of 'isatty' function */ #if YY_FLEX_MAJOR_VERSION <= 2 && YY_FLEX_MINOR_VERSION <= 5 && YY_FLEX_SUBMINOR_VERSION < 36 #define YY_NO_UNISTD_H @@ -132,7 +110,6 @@ /* start conditions */ %x STRING1 STRING2 COMMENT COMMENT_LN SCRIPTVARS -%x INCLF IMPTF /* action keywords */ FORWARD forward @@ -473,9 +450,6 @@ IMPORTFILE "import_file" {FOR} { count(); yylval.strval=yytext; return FOR; } {IN} { count(); yylval.strval=yytext; return IN; } -{INCLUDEFILE} { count(); BEGIN(INCLF); } -{IMPORTFILE} { count(); BEGIN(IMPTF); } - {SET_ADV_ADDRESS} { count(); yylval.strval=yytext; return SET_ADV_ADDRESS; } {SET_ADV_PORT} { count(); yylval.strval=yytext; @@ -849,32 +823,6 @@ IMPORTFILE "import_file" memset(&s_buf, 0, sizeof(s_buf)); return ID; } -[ \t]* /* eat the whitespace */ -[^ \t\n]+ { /* get the include file name */ - memset(&s_buf, 0, sizeof(s_buf)); - addstr(&s_buf, yytext, yyleng); - if(oss_push_yy_state(s_buf.s, 0)<0) - { - LM_CRIT("error at %s line %d\n", (finame)?finame:"cfg", line); - exit(-1); - } - memset(&s_buf, 0, sizeof(s_buf)); - BEGIN(INITIAL); -} - -[ \t]* /* eat the whitespace */ -[^ \t\n]+ { /* get the import file name */ - memset(&s_buf, 0, sizeof(s_buf)); - addstr(&s_buf, yytext, yyleng); - if(oss_push_yy_state(s_buf.s, 1)<0) - { - LM_CRIT("error at %s line %d\n", (finame)?finame:"cfg", line); - exit(-1); - } - memset(&s_buf, 0, sizeof(s_buf)); - BEGIN(INITIAL); -} - <> { switch(state){ case STRING_S: @@ -899,8 +847,8 @@ IMPORTFILE "import_file" " unclosed variable\n"); break; } - if(oss_pop_yy_state()<0) - return 0; + + return 0; } %% @@ -965,204 +913,3 @@ static void count(void) } int yywrap(void) { return 1; } - - -static int oss_push_yy_state(char *fin, int mode) -{ - struct oss_yy_fname *fn = NULL; - FILE *fp = NULL; - char *x = NULL; - char *newf = NULL; - char fbuf[MAX_INCLUDE_FNAME]; - int i, j, l; - char *tmpfiname = 0; - - if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) - { - LM_CRIT("too many includes\n"); - return -1; - } - l = strlen(fin); - if(l>=MAX_INCLUDE_FNAME) - { - LM_CRIT("included file name too long: %s\n", fin); - return -1; - } - if(fin[0]!='"' || fin[l-1]!='"') - { - LM_CRIT("included file name must be between quotes: %s\n", fin); - return -1; - } - j = 0; - for(i=1; ifname, newf)==0) - { - if(newf!=fbuf) - pkg_free(newf); - newf = fbuf; - break; - } - fn = fn->next; - } - if(fn==0) - { - fn = (struct oss_yy_fname*)pkg_malloc(sizeof(struct oss_yy_fname)); - if(fn==0) - { - if(newf!=fbuf) - pkg_free(newf); - LM_CRIT("no more pkg\n"); - return -1; - } - if(newf==fbuf) - { - fn->fname = (char*)pkg_malloc(strlen(fbuf)+1); - if(fn->fname==0) - { - pkg_free(fn); - LM_CRIT("no more pkg!\n"); - return -1; - } - strcpy(fn->fname, fbuf); - } else { - fn->fname = newf; - } - fn->next = oss_yy_fname_list; - oss_yy_fname_list = fn; - } - - finame = fn->fname; - - yy_switch_to_buffer( yy_create_buffer(yyin, YY_BUF_SIZE ) ); - - return 0; - -} - -static int oss_pop_yy_state(void) -{ - include_stack_ptr--; - if (include_stack_ptr<0 ) - return -1; - - yy_delete_buffer( YY_CURRENT_BUFFER ); - yy_switch_to_buffer(include_stack[include_stack_ptr].state); - line=include_stack[include_stack_ptr].line; - column=include_stack[include_stack_ptr].column; - startline=include_stack[include_stack_ptr].startline; - startcolumn=include_stack[include_stack_ptr].startcolumn; - finame = include_stack[include_stack_ptr].finame; - return 0; -} From 5405515f1ba3e0e2b283eab912493b653d43dc9d Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Thu, 21 Feb 2019 23:27:22 +0200 Subject: [PATCH 5/9] DEV-1: Display a file inclusion backtrace on syntax errors --- cfg.y | 1 + cfg_pp.c | 10 ++++++++++ cfg_pp.h | 1 + 3 files changed, 12 insertions(+) diff --git a/cfg.y b/cfg.y index a63370c458f..c9565407891 100644 --- a/cfg.y +++ b/cfg.y @@ -2771,6 +2771,7 @@ static inline void ALLOW_UNUSED warn(char* s) static void yyerror(char* s) { + cfg_dump_backtrace(L_CRIT); LM_CRIT("parse error in config file %s, line %d, column %d-%d: %s\n", get_cfg_file_name, line, startcolumn, column, s); cfg_errors++; diff --git a/cfg_pp.c b/cfg_pp.c index c2965fd2450..2223f416cbc 100644 --- a/cfg_pp.c +++ b/cfg_pp.c @@ -366,3 +366,13 @@ int cfg_pop(void) return 0; } + +void cfg_dump_backtrace(int loglevel) +{ + const char **it; + int frame = 0; + + LM_GEN1(loglevel, "IncludeStack (last included file at the bottom)\n"); + for (it = cfg_include_stack; it <= cfg_include_stackp; it++) + LM_GEN1(loglevel, "%2d. %s\n", frame++, *it); +} diff --git a/cfg_pp.h b/cfg_pp.h index c0462016183..be20a9ec4b3 100644 --- a/cfg_pp.h +++ b/cfg_pp.h @@ -28,5 +28,6 @@ int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline); int cfg_push(const char *cfg_file); int cfg_pop(void); +void cfg_dump_backtrace(int loglevel); #endif /* __OSS_CFG_PP_H__ */ From 46491c764cdb9b68269ca298d2098c484a07b9fe Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Fri, 22 Feb 2019 10:31:37 +0200 Subject: [PATCH 6/9] DEV-1: Various fixes * avoid unnecessary strdup operation * do not close file streams multiple times * fix buffer size for final config file * remove unused lexer tokens --- cfg.lex | 4 ---- cfg_pp.c | 24 +++++------------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/cfg.lex b/cfg.lex index 3f334737567..c041b065a17 100644 --- a/cfg.lex +++ b/cfg.lex @@ -379,10 +379,6 @@ COM_END "\*/" EAT_ABLE [\ \t\b\r] WHITESPACE [ \t\r\n] -/* include files */ -INCLUDEFILE "include_file" -IMPORTFILE "import_file" - %% diff --git a/cfg_pp.c b/cfg_pp.c index 2223f416cbc..07b6f064fd8 100644 --- a/cfg_pp.c +++ b/cfg_pp.c @@ -115,11 +115,7 @@ static int extend_cfg_buf(char **buf, int *sz, int *bytes_left) /* search for '(include|import)_file "filepath"' patterns */ int extract_included_file(char *line, int line_len, char **out_path) { - str lin; - char *fstart, *p = NULL, enclose = 0; - - lin.s = line; - lin.len = line_len; + char *p = NULL, enclose = 0; if (line_len > include_v1.len && !memcmp(line, include_v1.s, include_v1.len)) { @@ -148,23 +144,17 @@ int extract_included_file(char *line, int line_len, char **out_path) enclose = *p++; line_len--; - fstart = p; + *out_path = p; while (line_len > 0 && *p != enclose) { line_len--; p++; } - if (line_len == 0 || p - fstart < 2) // ""_ - return -1; - - *out_path = malloc(p - fstart); - if (!*out_path) { - LM_ERR("oom\n"); + if (line_len == 0 || p - *out_path < 2) // ""_ return -1; - } - memcpy(*out_path, fstart, p - fstart); + *p = '\0'; return 0; } @@ -259,12 +249,8 @@ static int __flatten_opensips_cfg(FILE *cfg, const char *cfg_path, if (__flatten_opensips_cfg(included_cfg, included_cfg_path, flattened, sz, bytes_left)) { LM_ERR("failed to flatten cfg file (internal err), oom?\n"); - fclose(included_cfg); goto out_err; } - - free(included_cfg_path); - fclose(included_cfg); } else { if (*bytes_left < line_len) { if (extend_cfg_buf(flattened, sz, bytes_left) < 0) { @@ -320,7 +306,7 @@ static FILE *flatten_opensips_cfg(FILE *cfg, const char *cfg_path) LM_NOTICE("flattened config file:\n%.*s\n", sz - bytes_left, flattened); #endif - cfg = fmemopen(flattened, sz, "r"); + cfg = fmemopen(flattened, sz - bytes_left, "r"); if (!cfg) LM_ERR("failed to obtain file for flattened cfg buffer\n"); From 55e874ff3ddea54b54ecfe8561836f9331871102 Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Fri, 22 Feb 2019 10:45:05 +0200 Subject: [PATCH 7/9] DEV-1: Avoid printing the include trace multiple times --- cfg_pp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cfg_pp.c b/cfg_pp.c index 07b6f064fd8..7d507209c5c 100644 --- a/cfg_pp.c +++ b/cfg_pp.c @@ -355,9 +355,14 @@ int cfg_pop(void) void cfg_dump_backtrace(int loglevel) { + static int called_before; const char **it; int frame = 0; + if (called_before) + return; + + called_before = 1; LM_GEN1(loglevel, "IncludeStack (last included file at the bottom)\n"); for (it = cfg_include_stack; it <= cfg_include_stackp; it++) LM_GEN1(loglevel, "%2d. %s\n", frame++, *it); From 134424deabb8fecaa4863a952d8149d8d4db675a Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Fri, 22 Feb 2019 12:07:41 +0200 Subject: [PATCH 8/9] DEV-1: Digest the pp tokens within lexer, not parser! Using the lexer to consume the preprocessor tokens makes life so much easier, as otherwise the complexity of the scripting language grammer would skyrocket. Example problem which is avoided by this: if ( condition ) { $var(x) = 5; } Although the coding style is horrible, this is still valid script syntax, but, all of a sudden, the grammar must be extended in order to support random tokens on any possible line of this "if" statement. Nope, not going there. Just use the lexer to eat the tokens and be done with the problem! --- cfg.lex | 44 +++++++++++++++++++++++++++++++++++++------- cfg.y | 26 +------------------------- cfg_pp.c | 22 +++++++++++++++------- cfg_pp.h | 4 +++- 4 files changed, 56 insertions(+), 40 deletions(-) diff --git a/cfg.lex b/cfg.lex index c041b065a17..d3b35f92651 100644 --- a/cfg.lex +++ b/cfg.lex @@ -59,12 +59,14 @@ %{ #include "cfg.tab.h" + #include "cfg_pp.h" #include "dprint.h" #include "globals.h" #include "mem/mem.h" #include #include #include "ip_addr.h" + #include "ut.h" /* states */ @@ -83,6 +85,7 @@ static int comment_nest=0; static int state=0; + static str st; static struct str_buf s_buf; int line=1; int np=0; @@ -90,7 +93,7 @@ int column=1; int startcolumn=1; int startline=1; - const char *finame = 0; + char *finame = 0; static char* addchar(struct str_buf *, char); static char* addstr(struct str_buf *, char*, int); @@ -110,6 +113,7 @@ /* start conditions */ %x STRING1 STRING2 COMMENT COMMENT_LN SCRIPTVARS +%x PPTOK_LINE PPTOK_FILEBEG PPTOK_FILEEND /* action keywords */ FORWARD forward @@ -378,6 +382,7 @@ COM_END "\*/" EAT_ABLE [\ \t\b\r] WHITESPACE [ \t\r\n] +SPACE [ ] %% @@ -446,6 +451,10 @@ WHITESPACE [ \t\r\n] {FOR} { count(); yylval.strval=yytext; return FOR; } {IN} { count(); yylval.strval=yytext; return IN; } +{PPTOK_LINE} { count(); BEGIN(PPTOK_LINE); } +{PPTOK_FILEBEG} { count(); BEGIN(PPTOK_FILEBEG); } +{PPTOK_FILEEND} { count(); BEGIN(PPTOK_FILEEND); } + {SET_ADV_ADDRESS} { count(); yylval.strval=yytext; return SET_ADV_ADDRESS; } {SET_ADV_PORT} { count(); yylval.strval=yytext; @@ -504,12 +513,6 @@ WHITESPACE [ \t\r\n] return LAUNCH_TOKEN;} {IS_MYSELF} { count(); yylval.strval=yytext; return IS_MYSELF;} -{PPTOK_LINE} { count(); yylval.strval=yytext; - return PPTOK_LINE;} -{PPTOK_FILEBEG} { count(); yylval.strval=yytext; - return PPTOK_FILEBEG;} -{PPTOK_FILEEND} { count(); yylval.strval=yytext; - return PPTOK_FILEEND;} {FORK} { count(); yylval.strval=yytext; return FORK; /*obsolete*/ } {DEBUG_MODE} { count(); yylval.strval=yytext; return DEBUG_MODE; } @@ -818,6 +821,33 @@ WHITESPACE [ \t\r\n] yylval.strval=s_buf.s; memset(&s_buf, 0, sizeof(s_buf)); return ID; } +{SPACE}[^\n]+ { /* grab the line number */ + st.s = yytext + 1; + st.len = yyleng - 1; + if (str2int(&st, (unsigned int *)&line) != 0) { + LM_CRIT("bad line number integer: '%.*s'\n", st.len, st.s); + exit(-1); + } + BEGIN(INITIAL); +} + +{SPACE}{QUOTES}.*{QUOTES} { /* grab the file path */ + st.s = yytext + 2; + st.len = yyleng - 3; + if (cfg_push(&st) != 0) { + LM_ERR("max nested includes reached!\n"); + exit(-1); + } + BEGIN(INITIAL); +} + + { + if (cfg_pop() != 0) { + LM_ERR("internal error during cfg_pop()\n"); + exit(-1); + } + BEGIN(INITIAL); +} <> { switch(state){ diff --git a/cfg.y b/cfg.y index c9565407891..31818eea934 100644 --- a/cfg.y +++ b/cfg.y @@ -401,9 +401,6 @@ static struct multi_str *tmp_mod; %token AUTO_SCALING_PROFILE %token AUTO_SCALING_CYCLE %token TIMER_WORKERS -%token PPTOK_LINE -%token PPTOK_FILEBEG -%token PPTOK_FILEEND @@ -517,8 +514,7 @@ statements: statements statement {} | statements error { yyerror(""); YYABORT;} ; -statement: preproc_stm - | assign_stm +statement: assign_stm | module_stm | {rt=REQUEST_ROUTE;} route_stm | {rt=FAILURE_ROUTE;} failure_route_stm @@ -727,26 +723,6 @@ auto_scale_profile_def: } ; -preproc_stm: PPTOK_LINE NUMBER { - line = $2; - } - | PPTOK_LINE error { - yyerror("BUG: PPTOK_LINE followed by non-number!"); - } - | PPTOK_FILEBEG STRING { - if (cfg_push($2) != 0) { - yyerror("max nested includes reached!\n"); - } - } - | PPTOK_FILEBEG error { - yyerror("BUG: PPTOK_FILEBEG followed by non-string!"); - } - | PPTOK_FILEEND { - if (cfg_pop() != 0) { - yyerror("internal error during cfg_pop()\n"); - } - } - assign_stm: DEBUG EQUAL snumber { yyerror("\'debug\' is deprecated, use \'log_level\' instead\n");} | FORK EQUAL NUMBER diff --git a/cfg_pp.c b/cfg_pp.c index 7d507209c5c..a0efe0c39b1 100644 --- a/cfg_pp.c +++ b/cfg_pp.c @@ -29,7 +29,7 @@ #include "cfg_pp.h" #include "ut.h" -extern const char *finame; +extern char *finame; extern int startline; extern int column; @@ -313,9 +313,9 @@ static FILE *flatten_opensips_cfg(FILE *cfg, const char *cfg_path) return cfg; } -const char *cfg_include_stack[CFG_MAX_INCLUDE_DEPTH]; -const char **cfg_include_stackp; -int cfg_push(const char *cfg_file) +static char *cfg_include_stack[CFG_MAX_INCLUDE_DEPTH]; +static char **cfg_include_stackp; +int cfg_push(const str *cfg_file) { if (!cfg_include_stackp) { cfg_include_stackp = cfg_include_stack; @@ -327,9 +327,15 @@ int cfg_push(const char *cfg_file) cfg_include_stackp++; } - *cfg_include_stackp = cfg_file; + *cfg_include_stackp = malloc(cfg_file->len + 1); + if (!*cfg_include_stackp) { + LM_ERR("oom\n"); + return -1; + } + memcpy(*cfg_include_stackp, cfg_file->s, cfg_file->len); + (*cfg_include_stackp)[cfg_file->len] = '\0'; - finame = cfg_file; + finame = *cfg_include_stackp; startline = 1; column = 1; return 0; @@ -342,6 +348,8 @@ int cfg_pop(void) return -1; } + free(*cfg_include_stackp); + if (cfg_include_stackp == cfg_include_stack) { cfg_include_stackp = NULL; } else { @@ -356,7 +364,7 @@ int cfg_pop(void) void cfg_dump_backtrace(int loglevel) { static int called_before; - const char **it; + char **it; int frame = 0; if (called_before) diff --git a/cfg_pp.h b/cfg_pp.h index be20a9ec4b3..686f1d6cef1 100644 --- a/cfg_pp.h +++ b/cfg_pp.h @@ -23,10 +23,12 @@ #ifndef __OSS_CFG_PP_H__ #define __OSS_CFG_PP_H__ +#include "str.h" + #define CFG_MAX_INCLUDE_DEPTH 20 int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline); -int cfg_push(const char *cfg_file); +int cfg_push(const str *cfg_file); int cfg_pop(void); void cfg_dump_backtrace(int loglevel); From 3162d504593ecf7989567d469a582a4e210ec89d Mon Sep 17 00:00:00 2001 From: Liviu Chircu Date: Fri, 22 Feb 2019 16:17:00 +0200 Subject: [PATCH 9/9] DEV-1: Exec the preprocessor cmdline; Feed output to Yacc --- cfg_pp.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/cfg_pp.c b/cfg_pp.c index a0efe0c39b1..c93546e4298 100644 --- a/cfg_pp.c +++ b/cfg_pp.c @@ -47,6 +47,7 @@ str cfgtok_filebegin = str_init("__OSSPP_FILEBEGIN__"); str cfgtok_fileend = str_init("__OSSPP_FILEEND__"); static FILE *flatten_opensips_cfg(FILE *cfg, const char *cfg_path); +static FILE *exec_preprocessor(FILE *flat_cfg, const char *preproc_cmdline); int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) { @@ -76,9 +77,11 @@ int parse_opensips_cfg(const char *cfg_file, const char *preproc_cmdline) } if (preproc_cmdline) { - // TODO: some dup magic, to push into pp stdin / read from its stdout - // TODO: execvp(chopped(preproc_cmdline), flattened_cfg) - // TODO: cfg_stream = fdopen(stdout_fd); + cfg_stream = exec_preprocessor(cfg_stream, preproc_cmdline); + if (!cfg_stream) { + LM_ERR("failed to exec preprocessor cmd: '%s'\n", preproc_cmdline); + return -1; + } } #ifdef DEBUG_PARSER @@ -367,7 +370,7 @@ void cfg_dump_backtrace(int loglevel) char **it; int frame = 0; - if (called_before) + if (called_before || !cfg_include_stackp) return; called_before = 1; @@ -375,3 +378,104 @@ void cfg_dump_backtrace(int loglevel) for (it = cfg_include_stack; it <= cfg_include_stackp; it++) LM_GEN1(loglevel, "%2d. %s\n", frame++, *it); } + +static FILE *exec_preprocessor(FILE *flat_cfg, const char *preproc_cmdline) +{ + FILE *final_cfg; + int parent_w[2], parent_r[2]; + char chunk[1024]; + ssize_t left, written; + size_t bytes; + char *p, *tok, *cmd, **argv = NULL, *pp_binary = NULL; + int argv_len = 0; + + if (strlen(preproc_cmdline) == 0) { + LM_ERR("preprocessor command (-p) is an empty string!\n"); + goto out_err; + } + + if (pipe(parent_w) != 0 || pipe(parent_r) != 0) { + LM_ERR("failed to create pipe: %d (%s)\n", errno, strerror(errno)); + goto out_err; + } + + /* fork a data-hungry preprocessor beast! (a.k.a. some tiny sed) */ + if (fork() == 0) { + close(parent_w[1]); + if (dup2(parent_w[0], STDIN_FILENO) < 0) { + LM_ERR("dup2 failed with: %d (%s)\n", errno, strerror(errno)); + exit(-1); + } + close(parent_w[0]); + + close(parent_r[0]); + if (dup2(parent_r[1], STDOUT_FILENO) < 0) { + LM_ERR("dup2 failed with: %d (%s)\n", errno, strerror(errno)); + exit(-1); + } + close(parent_w[1]); + + for (cmd = strdup(preproc_cmdline); ; cmd = NULL) { + tok = strtok(cmd, " \t\r\n"); + if (!tok) + break; + + if (!pp_binary) + pp_binary = tok; + + argv = realloc(argv, (argv_len + 1) * sizeof *argv); + argv[argv_len++] = tok; + } + + argv = realloc(argv, (argv_len + 1) * sizeof *argv); + argv[argv_len++] = NULL; + + execvp(pp_binary, argv); + LM_ERR("failed to exec preprocessor '%s': %d (%s)\n", + preproc_cmdline, errno, strerror(errno)); + exit(-1); + } + + close(parent_w[0]); + close(parent_r[1]); + + /* push the cfg file into the new process' stdin */ + do { + bytes = fread(chunk, 1, 1024, flat_cfg); + if (ferror(flat_cfg)) { + LM_ERR("failed to read from flat cfg: %d (%s)\n", + errno, strerror(errno)); + goto out_err_pipes; + } + + if (bytes == 0) + continue; + + left = bytes; + p = chunk; + do { + written = write(parent_w[1], p, left); + left -= written; + p += written; + } while (left > 0); + + } while (!feof(flat_cfg)); + + fclose(flat_cfg); + close(parent_w[1]); + + /* and we're done, let's see what the process barfed up! */ + final_cfg = fdopen(parent_r[0], "r"); + if (!final_cfg) + LM_ERR("failed to open final cfg file: %d (%s)\n", + errno, strerror(errno)); + + return final_cfg; + +out_err_pipes: + close(parent_w[1]); + close(parent_r[0]); +out_err: + fclose(flat_cfg); + return NULL; +}