[formats] add a "-i" flag to make it easier to install formats

NEWS

@@ -16,6 +16,7 @@ lnav v0.7.1:
        filters should be almost instantaneous.
      * The filter-in, filter-out, and highlight commands now support
        tab-completion of text in the current view.
+     * Add a '-i' flag that installs format files in: ~/.lnav/formats/installed
 
 lnav v0.7.0:
      Features:

docs/source/formats.rst

@@ -157,3 +157,31 @@ with the following contents::
             ]
         }
     }
+
+Installing Formats
+------------------
+
+File formats are loaded from subdirectories in :file:`/etc/lnav/formats` and
+:file:`~/.lnav/formats/`.  You can manually create these subdirectories and
+copy the format files into there.  Or, you can pass the '-i' option to **lnav**
+to automatically install formats from the command-line.  For example::
+
+    $ lnav -i myformat.json
+    info: installed: /home/example/.lnav/formats/installed/myformat_log.json
+
+Formats installed using this method will be placed in the :file:`installed`
+subdirectory and named based on the first format name found in the file.
+
+Format files can also be made executable by adding a shebang (#!) line to the
+top of the file, like so::
+
+    #! /usr/bin/env lnav -i
+    {
+        "myformat_log" : ...
+    }
+
+Executing the format file should then install it automatically::
+
+    $ chmod ugo+rx myformat.json
+    $ ./myformat.json
+    info: installed: /home/example/.lnav/formats/installed/myformat_log.json

lnav.1

@@ -49,15 +49,22 @@ Print help and exit
 \fB\-H\fR
 Display the internal help text.
 .TP
+\fB\-I\fR path
+Add the given configuration directory to the search path.
+.TP
+\fB\-i\fR
+Install the given format files in the $HOME/.lnav/formats/installed directory
+and exit.
+.TP
+\fB\-C\fR
+Check the configuration and exit.
+.TP
 \fB\-d\fR file
 Write debug messages to the given file.
 .TP
 \fB\-V\fR
 Print version information.
 .TP
-\fB\-s\fR
-Load the most recent syslog messages file.
-.TP
 \fB\-a\fR
 Load all of the most recent log file types.
 .TP

src/CMakeLists.txt

@@ -93,6 +93,11 @@ set(diag_STAT_SRCS
     time_T.hh
     top_status_source.hh
 
+    yajl/api/yajl_common.h
+    yajl/api/yajl_gen.h
+    yajl/api/yajl_parse.h
+    yajl/api/yajl_tree.h
+
     ../../lbuild/src/config.h
     )
 

src/lnav.cc

@@ -2842,11 +2842,11 @@ static void usage(void)
         "  -h         Print this message, then exit.\n"
         "  -H         Display the internal help text.\n"
         "  -I path    An additional configuration directory.\n"
+        "  -i         Install the given format files and exit.\n"
         "  -C         Check configuration and then exit.\n"
         "  -d file    Write debug messages to the given file.\n"
         "  -V         Print version information.\n"
         "\n"
-        "  -s         Load the most recent syslog messages file.\n"
         "  -a         Load all of the most recent log file types.\n"
         "  -r         Load older rotated log files as well.\n"
         "  -t         Prepend timestamps to the lines of data being read in\n"
@@ -3930,6 +3930,16 @@ int sql_progress(const struct log_cursor &lc)
     return 0;
 }
 
+static void print_errors(vector<string> error_list)
+{
+    for (std::vector<std::string>::iterator iter = error_list.begin();
+         iter != error_list.end();
+         ++iter) {
+        fprintf(stderr, "%s%s", iter->c_str(),
+                (*iter)[iter->size() - 1] == '\n' ? "" : "\n");
+    }
+}
+
 int main(int argc, char *argv[])
 {
     std::vector<std::string> loader_errors;
@@ -3950,7 +3960,7 @@ int main(int argc, char *argv[])
     sql_install_logger();
 
     lnav_data.ld_debug_log_name = "/dev/null";
-    while ((c = getopt(argc, argv, "hHarsCc:I:f:d:nqtw:VW")) != -1) {
+    while ((c = getopt(argc, argv, "hHarsCc:I:if:d:nqtw:VW")) != -1) {
         switch (c) {
         case 'h':
             usage();
@@ -4000,6 +4010,10 @@ int main(int argc, char *argv[])
             lnav_data.ld_config_paths.push_back(optarg);
             break;
 
+            case 'i':
+                lnav_data.ld_flags |= LNF_INSTALL;
+                break;
+
         case 'd':
             lnav_data.ld_debug_log_name = optarg;
             break;
@@ -4055,14 +4069,55 @@ int main(int argc, char *argv[])
 
     lnav_log_file = fopen(lnav_data.ld_debug_log_name, "a");
 
+    if (lnav_data.ld_flags & LNF_INSTALL) {
+        string installed_path = dotlnav_path("formats/installed/");
+
+        if (argc == 0) {
+            fprintf(stderr, "error: expecting file format paths\n");
+            return EXIT_FAILURE;
+        }
+
+        for (lpc = 0; lpc < argc; lpc++) {
+            vector<string> format_list = load_format_file(argv[lpc], loader_errors);
+
+            if (!loader_errors.empty()) {
+                print_errors(loader_errors);
+                return EXIT_FAILURE;
+            }
+            if (format_list.empty()) {
+                fprintf(stderr, "error: format file is empty: %s\n", argv[lpc]);
+                return EXIT_FAILURE;
+            }
+
+            string dst_name = format_list[0] + ".json";
+            string dst_path = installed_path + dst_name;
+            auto_fd in_fd, out_fd;
+
+            if ((in_fd = open(argv[lpc], O_RDONLY)) == -1) {
+                perror("unable to open file to install");
+            }
+            else if ((out_fd = open(dst_path.c_str(),
+                    O_WRONLY | O_CREAT, 0644)) == -1) {
+                fprintf(stderr, "error: unable to open destination: %s -- %s\n",
+                        dst_path.c_str(), strerror(errno));
+            }
+            else {
+                char buffer[2048];
+                ssize_t rc;
+
+                while ((rc = read(in_fd, buffer, sizeof(buffer))) > 0) {
+                    write(out_fd, buffer, rc);
+                }
+
+                fprintf(stderr, "info: installed: %s\n", dst_path.c_str());
+            }
+        }
+        return EXIT_SUCCESS;
+    }
+
     load_formats(lnav_data.ld_config_paths, loader_errors);
     if (!loader_errors.empty()) {
-        for (std::vector<std::string>::iterator iter = loader_errors.begin();
-             iter != loader_errors.end();
-             ++iter) {
-            fprintf(stderr, "%s%s", iter->c_str(),
-                (*iter)[iter->size() - 1] == '\n' ? "" : "\n");
-        }
+        print_errors(loader_errors);
         return EXIT_FAILURE;
     }
 

src/lnav.hh

@@ -79,6 +79,7 @@ enum {
     LNB_QUIET,
     LNB_ROTATED,
     LNB_CHECK_CONFIG,
+    LNB_INSTALL,
 };
 
 /** Flags set on the lnav command-line. */
@@ -92,6 +93,7 @@ typedef enum {
     LNF_HEADLESS  = (1L << LNB_HEADLESS),
     LNF_QUIET     = (1L << LNB_QUIET),
     LNF_CHECK_CONFIG = (1L << LNB_CHECK_CONFIG),
+    LNF_INSTALL   = (1L << LNB_INSTALL),
 
     LNF__ALL      = (LNF_SYSLOG|LNF_HELP)
 } lnav_flags_t;

src/lnav_config.cc

@@ -89,7 +89,12 @@ void ensure_dotlnav(void)
     if (!path.empty()) {
         mkdir(path.c_str(), 0755);
     }
-
+
+    path = dotlnav_path("formats/installed");
+    if (!path.empty()) {
+        mkdir(path.c_str(), 0755);
+    }
+
     path = dotlnav_path("crash");
     if (!path.empty()) {
         mkdir(path.c_str(), 0755);

src/log_format_loader.cc

@@ -56,6 +56,7 @@ static map<string, external_log_format *> LOG_FORMATS;
 static external_log_format *ensure_format(yajlpp_parse_context *ypc)
 {
     const string &name = ypc->get_path_fragment(0);
+    vector<string> *formats = (vector<string> *)ypc->ypc_userdata;
     external_log_format *retval;
 
     retval = LOG_FORMATS[name];
@@ -64,6 +65,10 @@ static external_log_format *ensure_format(yajlpp_parse_context *ypc)
     }
     retval->elf_source_path.insert(ypc->ypc_source.substr(0, ypc->ypc_source.rfind('/')));
 
+    if (find(formats->begin(), formats->end(), name) == formats->end()) {
+        formats->push_back(name);
+    }
+
     return retval;
 }
 
@@ -415,59 +420,81 @@ static void write_sample_file(void)
     }
 }
 
+std::vector<string> load_format_file(const string &filename, std::vector<string> &errors)
+{
+    std::vector<string> retval;
+    auto_fd fd;
+
+    log_info("loading formats from file: %s", filename.c_str());
+    yajlpp_parse_context ypc(filename, format_handlers);
+    ypc.ypc_userdata = &retval;
+    if ((fd = open(filename.c_str(), O_RDONLY)) == -1) {
+        char errmsg[1024];
+
+        snprintf(errmsg, sizeof(errmsg),
+                "error: unable to open format file -- %s",
+                filename.c_str());
+        errors.push_back(errmsg);
+    }
+    else {
+        yajl_handle handle;
+        char buffer[2048];
+        off_t offset = 0;
+        int rc = -1;
+
+        handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc);
+        yajl_config(handle, yajl_allow_comments, 1);
+        while (true) {
+            rc = read(fd, buffer, sizeof(buffer));
+            if (rc == 0) {
+                break;
+            }
+            else if (rc == -1) {
+                errors.push_back(filename +
+                        ":unable to read file -- " +
+                        string(strerror(errno)));
+                break;
+            }
+            if (offset == 0 && (rc > 2) &&
+                    (buffer[0] == '#') && (buffer[1] == '!')) {
+                // Turn it into a JavaScript comment.
+                buffer[0] = buffer[1] = '/';
+            }
+            if (yajl_parse(handle, (const unsigned char *)buffer, rc) != yajl_status_ok) {
+                errors.push_back(filename +
+                        ": invalid json -- " +
+                        string((char *)yajl_get_error(handle, 1, (unsigned char *)buffer, rc)));
+                break;
+            }
+            offset += rc;
+        }
+        if (rc == 0) {
+            if (yajl_complete_parse(handle) != yajl_status_ok) {
+                errors.push_back(filename +
+                        ": invalid json -- " +
+                        string((char *)yajl_get_error(handle, 0, NULL, 0)));
+            }
+        }
+        yajl_free(handle);
+    }
+
+    return retval;
+}
+
 static void load_from_path(const string &path, std::vector<string> &errors)
 {
     string format_path = path + "/formats/*/*.json";
     static_root_mem<glob_t, globfree> gl;
-    yajl_handle handle;
 
     log_info("loading formats from path: %s", format_path.c_str());
     if (glob(format_path.c_str(), 0, NULL, gl.inout()) == 0) {
         for (int lpc = 0; lpc < (int)gl->gl_pathc; lpc++) {
             string filename(gl->gl_pathv[lpc]);
-            auto_fd fd;
+            vector<string> format_list;
 
-            log_info("loading formats from file: %s", filename.c_str());
-            yajlpp_parse_context ypc(filename, format_handlers);
-            if ((fd = open(gl->gl_pathv[lpc], O_RDONLY)) == -1) {
-                char errmsg[1024];
-
-                snprintf(errmsg, sizeof(errmsg),
-                         "error: unable to open format file -- %s",
-                         gl->gl_pathv[lpc]);
-                perror(errmsg);
-            }
-            else {
-                char buffer[2048];
-                int rc = -1;
-
-                handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc);
-                while (true) {
-                    rc = read(fd, buffer, sizeof(buffer));
-                    if (rc == 0) {
-                        break;
-                    }
-                    else if (rc == -1) {
-                        errors.push_back(filename +
-                                         ":unable to read file -- " +
-                                         string(strerror(errno)));
-                        break;
-                    }
-                    if (yajl_parse(handle, (const unsigned char *)buffer, rc) != yajl_status_ok) {
-                        errors.push_back(filename +
-                            ": invalid json -- " +
-                            string((char *)yajl_get_error(handle, 1, (unsigned char *)buffer, rc)));
-                        break;
-                    }
-                }
-                if (rc == 0) {
-                    if (yajl_complete_parse(handle) != yajl_status_ok) {
-                        errors.push_back(filename +
-                            ": invalid json -- " +
-                            string((char *)yajl_get_error(handle, 0, NULL, 0)));
-                    }
-                }
-                yajl_free(handle);
+            format_list = load_format_file(filename, errors);
+            if (format_list.empty()) {
+                log_warning("Empty format file: %s", filename.c_str());
             }
         }
     }
@@ -484,6 +511,8 @@ void load_formats(const std::vector<std::string> &extra_paths,
     write_sample_file();
 
     handle = yajl_alloc(&ypc_builtin.ypc_callbacks, NULL, &ypc_builtin);
+    ypc_builtin.ypc_userdata = &retval;
+    yajl_config(handle, yajl_allow_comments, 1);
     if (yajl_parse(handle,
                    (const unsigned char *)default_log_formats_json,
                    strlen(default_log_formats_json)) != yajl_status_ok) {

src/log_format_loader.hh

@@ -35,6 +35,9 @@
 #include <vector>
 #include <string>
 
+std::vector<std::string> load_format_file(
+        const std::string &filename, std::vector<std::string> &errors);
+
 void load_formats(const std::vector<std::string> &extra_paths,
                   std::vector<std::string> &errors);