Skip to content
Permalink
Browse files

Added the ability to parse server cache status and a new panel to dis…

…play metrics.

Closes #919
  • Loading branch information
allinurl committed Nov 22, 2019
1 parent d7484e4 commit cd1698492e826323ed71c2507221c9443b5916bf
Showing with 126 additions and 15 deletions.
  1. +3 −0 config/goaccess.conf
  2. +10 −0 goaccess.1
  3. +5 −0 src/commons.c
  4. +3 −2 src/commons.h
  5. +1 −0 src/csv.c
  6. +1 −0 src/gholder.c
  7. +7 −3 src/goaccess.c
  8. +1 −0 src/json.c
  9. +7 −0 src/labels.h
  10. +4 −0 src/output.c
  11. +64 −2 src/parser.c
  12. +2 −0 src/parser.h
  13. +2 −2 src/settings.c
  14. +4 −1 src/sort.c
  15. +8 −3 src/ui.c
  16. +2 −1 src/ui.h
  17. +2 −1 src/util.c
@@ -460,6 +460,7 @@ double-decode false
#enable-panel KEYPHRASES
#enable-panel STATUS_CODES
#enable-panel REMOTE_USER
#enable-panel CACHE_STATUS
#enable-panel GEO_LOCATION

# Hide a referer but still count it. Wild cards are allowed. i.e., *.bing.com
@@ -508,6 +509,7 @@ ignore-panel REFERRERS
ignore-panel KEYPHRASES
#ignore-panel STATUS_CODES
#ignore-panel REMOTE_USER
#ignore-panel CACHE_STATUS
#ignore-panel GEO_LOCATION

# Ignore referers from being counted.
@@ -575,6 +577,7 @@ real-os true
#sort-panel KEYPHRASES,BY_HITS,ASC
#sort-panel STATUS_CODES,BY_HITS,ASC
#sort-panel REMOTE_USER,BY_HITS,ASC
#sort-panel CACHE_STATUS,BY_HITS,ASC
#sort-panel GEO_LOCATION,BY_HITS,ASC

# Consider the following extensions as static files
@@ -102,6 +102,14 @@ authentication. If the document is not password protected, this part will be
"-" just like the previous one. This panel is not enabled unless
.I %e
is given within the log-format variable.
.IP "Cache Status"
If you are using caching on your server, you may be at the point where you
want to know if your request is being cached and served from the cache. This
panel shows the cache status of the object the server served. This panel is not
enabled unless
.I %C
is given within the log-format variable. The status can be either
`MISS`, `BYPASS`, `EXPIRED`, `STALE`, `UPDATING`, `REVALIDATED` or `HIT`

.P
.I NOTE:
@@ -824,6 +832,8 @@ The canonical Server Name of the server serving the request (Virtual Host).
.IP %e
This is the userid of the person requesting the document as determined by HTTP
authentication.
.IP %C
The cache status of the object the server served.
.IP %h
host (the client IP address, either IPv4 or IPv6)
.IP %r
@@ -216,6 +216,7 @@ get_module_enum (const char *str)
{"KEYPHRASES" , KEYPHRASES} ,
{"STATUS_CODES" , STATUS_CODES} ,
{"REMOTE_USER" , REMOTE_USER} ,
{"CACHE_STATUS" , CACHE_STATUS} ,
#ifdef HAVE_GEOLOCATION
{"GEO_LOCATION" , GEO_LOCATION} ,
#endif
@@ -425,6 +426,10 @@ verify_panels (void)
if (str_inarray ("REMOTE_USER", conf.ignore_panels, ignore_panel_idx) < 0)
remove_module (REMOTE_USER);
}
if (!strstr (conf.log_format, "%C") && ignore_panel_idx < TOTAL_MODULES) {
if (str_inarray ("CACHE_STATUS", conf.ignore_panels, ignore_panel_idx) < 0)
remove_module (CACHE_STATUS);
}
}

/* Build an array of available modules (ignores listed panels).
@@ -51,9 +51,9 @@ struct tm *now_tm;

/* total number of modules */
#ifdef HAVE_GEOLOCATION
#define TOTAL_MODULES 15
#define TOTAL_MODULES 16
#else
#define TOTAL_MODULES 14
#define TOTAL_MODULES 15
#endif

/* maximum number of items within a panel */
@@ -105,6 +105,7 @@ typedef enum MODULES
KEYPHRASES,
STATUS_CODES,
REMOTE_USER,
CACHE_STATUS,
#ifdef HAVE_GEOLOCATION
GEO_LOCATION,
#endif
@@ -81,6 +81,7 @@ static GPanel paneling[] = {
{KEYPHRASES , print_csv_data} ,
{STATUS_CODES , print_csv_data} ,
{REMOTE_USER , print_csv_data} ,
{CACHE_STATUS , print_csv_data} ,
#ifdef HAVE_GEOLOCATION
{GEO_LOCATION , print_csv_data} ,
#endif
@@ -85,6 +85,7 @@ static GPanel paneling[] = {
{KEYPHRASES , add_data_to_holder, NULL} ,
{STATUS_CODES , add_root_to_holder, NULL} ,
{REMOTE_USER , add_data_to_holder, NULL} ,
{CACHE_STATUS , add_data_to_holder, NULL} ,
#ifdef HAVE_GEOLOCATION
{GEO_LOCATION , add_root_to_holder, NULL} ,
#endif
@@ -1029,6 +1029,10 @@ get_keys (void)
set_module_to (&gscroll, GEO_LOCATION);
break;
#endif
case 38: /* Shift + 5 */
/* reset expanded module */
set_module_to (&gscroll, CACHE_STATUS);
break;
case 9: /* TAB */
/* reset expanded module */
collapse_current_module ();
@@ -1344,8 +1348,8 @@ handle_signal_action (int sig_number)
conf.stop_processing = 1;

if (!conf.output_stdout) {
cleanup(EXIT_SUCCESS);
exit(EXIT_SUCCESS);
cleanup (EXIT_SUCCESS);
exit (EXIT_SUCCESS);
}

break;
@@ -1543,7 +1547,7 @@ main (int argc, char **argv)

/* clean */
clean:
cleanup(ret);
cleanup (ret);

return ret ? EXIT_FAILURE : EXIT_SUCCESS;
}
@@ -94,6 +94,7 @@ static GPanel paneling[] = {
{KEYPHRASES , print_json_data , NULL } ,
{STATUS_CODES , print_json_data , print_json_sub_items } ,
{REMOTE_USER , print_json_data , NULL } ,
{CACHE_STATUS , print_json_data , NULL } ,
#ifdef HAVE_GEOLOCATION
{GEO_LOCATION , print_json_data , print_json_sub_items } ,
#endif
@@ -138,6 +138,13 @@
#define REMOTE_USER_LABEL \
N_("Remote User")

#define CACHE_STATUS_HEAD \
N_("The cache status of the object served")
#define CACHE_STATUS_DESC \
N_("Data sorted by hits [, avgts, cumts, maxts]")
#define CACHE_STATUS_LABEL \
N_("Cache Status")

#define NOT_FOUND_HEAD \
N_("Not Found URLs (404s)")
#define NOT_FOUND_DESC \
@@ -129,6 +129,10 @@ static GHTML htmldef[] = {
{CHART_VBAR, hits_visitors_plot, 0, 0, NULL, NULL},
{CHART_VBAR, hits_bw_plot, 0, 0, NULL, NULL},
}},
{CACHE_STATUS, 1, print_metrics, {
{CHART_VBAR, hits_visitors_plot, 0, 0, NULL, NULL},
{CHART_VBAR, hits_bw_plot, 0, 0, NULL, NULL},
}},
#ifdef HAVE_GEOLOCATION
{GEO_LOCATION, 1, print_metrics, {
{CHART_VBAR, hits_visitors_plot, 0, 1, NULL, NULL},
@@ -85,6 +85,7 @@ static int gen_keyphrase_key (GKeyData * kdata, GLogItem * logitem);
static int gen_os_key (GKeyData * kdata, GLogItem * logitem);
static int gen_vhost_key (GKeyData * kdata, GLogItem * logitem);
static int gen_remote_user_key (GKeyData * kdata, GLogItem * logitem);
static int gen_cache_status_key (GKeyData * kdata, GLogItem * logitem);
static int gen_referer_key (GKeyData * kdata, GLogItem * logitem);
static int gen_ref_site_key (GKeyData * kdata, GLogItem * logitem);
static int gen_request_key (GKeyData * kdata, GLogItem * logitem);
@@ -310,6 +311,19 @@ static GParse paneling[] = {
NULL,
NULL,
NULL,
}, {
CACHE_STATUS,
gen_cache_status_key,
insert_data,
NULL,
insert_hit,
insert_visitor,
insert_bw,
insert_cumts,
insert_maxts,
NULL,
NULL,
NULL,
},
};
/* *INDENT-ON* */
@@ -455,6 +469,7 @@ init_log_item (GLog * glog)
logitem->uniq_key = NULL;
logitem->vhost = NULL;
logitem->userid = NULL;
logitem->cache_status = NULL;

memset (logitem->site, 0, sizeof (logitem->site));

@@ -507,6 +522,8 @@ free_glog (GLogItem * logitem)
free (logitem->uniq_key);
if (logitem->userid != NULL)
free (logitem->userid);
if (logitem->cache_status != NULL)
free (logitem->cache_status);
if (logitem->vhost != NULL)
free (logitem->vhost);

@@ -527,7 +544,7 @@ decode_hex (char *url, char *out)
if (*c != '%' || !isxdigit (c[1]) || !isxdigit (c[2])) {
*ptr++ = *c;
} else {
*ptr++ = (char)((B16210 (c[1]) * 16) + (B16210 (c[2])));
*ptr++ = (char) ((B16210 (c[1]) * 16) + (B16210 (c[2])));
c += 2;
}
}
@@ -765,6 +782,24 @@ contains_usecs (void)
conf.serve_usecs = 1; /* flag */
}

static int
is_cache_hit (const char *tkn)
{
const char *statuses[] = {
"MISS",
"BYPASS",
"EXPIRED",
"STALE",
"UPDATING",
"REVALIDATED",
"HIT",
};

if (str_inarray (tkn, statuses, CACHE_STATUS_LEN) != -1)
return 1;
return 0;
}

/* Determine if the given token is a valid HTTP protocol.
*
* If not valid, 1 is returned.
@@ -1076,6 +1111,16 @@ parse_specifier (GLogItem * logitem, char **str, const char *p, const char *end)
return spec_err (logitem, SPEC_TOKN_NUL, *p, NULL);
logitem->userid = tkn;
break;
/* cache status */
case 'C':
if (logitem->cache_status)
return spec_err (logitem, SPEC_TOKN_SET, *p, NULL);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL)
return spec_err (logitem, SPEC_TOKN_NUL, *p, NULL);
if (is_cache_hit (tkn))
logitem->cache_status = tkn;
break;
/* remote hostname (IP only) */
case 'h':
if (logitem->host)
@@ -2134,6 +2179,22 @@ gen_remote_user_key (GKeyData * kdata, GLogItem * logitem)
return 0;
}

/* A wrapper to generate a unique key for the cache status panel.
*
* On error, 1 is returned.
* On success, the generated cache status key is assigned to our key data
* structure. */
static int
gen_cache_status_key (GKeyData * kdata, GLogItem * logitem)
{
if (!logitem->cache_status)
return 1;

get_kdata (kdata, logitem->cache_status, logitem->cache_status);

return 0;
}

/* A wrapper to generate a unique key for the hosts panel.
*
* On error, 1 is returned.
@@ -2616,7 +2677,8 @@ fgetline (FILE * fp)
if (!fgets (buf, sizeof (buf), fp)) {
if (conf.process_and_exit && errno == EAGAIN) {
nanosleep ((const struct timespec[]) { {
0, 100000000L}}, NULL);
0, 100000000L}
}, NULL);
continue;
} else
break;
@@ -39,6 +39,7 @@
#define LINE_LEN 23
#define ERROR_LEN 255
#define REF_SITE_LEN 511 /* maximum length of a referring site */
#define CACHE_STATUS_LEN 7

#define SPEC_TOKN_SET 0x1
#define SPEC_TOKN_NUL 0x2
@@ -71,6 +72,7 @@ typedef struct GLogItem_
char *uniq_key;
char *vhost;
char *userid;
char *cache_status;

char site[REF_SITE_LEN + 1];

@@ -350,7 +350,7 @@ size_t
get_selected_format_idx (void)
{
if (conf.log_format == NULL)
return (size_t)-1;
return (size_t) -1;
if (strcmp (conf.log_format, logs.common) == 0)
return COMMON;
else if (strcmp (conf.log_format, logs.vcommon) == 0)
@@ -372,7 +372,7 @@ get_selected_format_idx (void)
else if (strcmp (conf.log_format, logs.awss3) == 0)
return AWSS3;
else
return (size_t)-1;
return (size_t) -1;
}

/* Determine the selected log format from the config file or command line
@@ -74,6 +74,8 @@ const int sort_choices[][SORT_MAX_OPTS] = {
{SORT_BY_HITS, SORT_BY_VISITORS, SORT_BY_DATA, SORT_BY_BW, SORT_BY_AVGTS, SORT_BY_CUMTS, SORT_BY_MAXTS, -1},
/* REMOTE_USER */
{SORT_BY_HITS, SORT_BY_VISITORS, SORT_BY_DATA, SORT_BY_BW, SORT_BY_AVGTS, SORT_BY_CUMTS, SORT_BY_MAXTS, -1},
/* CACHE_STATUS */
{SORT_BY_HITS, SORT_BY_VISITORS, SORT_BY_DATA, SORT_BY_BW, SORT_BY_AVGTS, SORT_BY_CUMTS, SORT_BY_MAXTS, -1},
#ifdef HAVE_GEOLOCATION
/* GEO_LOCATION */
{SORT_BY_HITS, SORT_BY_VISITORS, SORT_BY_DATA, SORT_BY_BW, SORT_BY_AVGTS, SORT_BY_CUMTS, SORT_BY_MAXTS, -1},
@@ -112,6 +114,7 @@ GSort module_sort[TOTAL_MODULES] = {
{KEYPHRASES , SORT_BY_HITS , SORT_DESC } ,
{STATUS_CODES , SORT_BY_HITS , SORT_DESC } ,
{REMOTE_USER , SORT_BY_HITS , SORT_DESC } ,
{CACHE_STATUS , SORT_BY_HITS , SORT_DESC } ,
#ifdef HAVE_GEOLOCATION
{GEO_LOCATION , SORT_BY_HITS , SORT_DESC } ,
#endif
@@ -122,7 +125,7 @@ GSort module_sort[TOTAL_MODULES] = {
int
strcmp_asc (const void *a, const void *b)
{
return strcmp (*((char *const *) a), *((char * const*) b));
return strcmp (*((char *const *) a), *((char *const *) b));
}

/* Sort 'data' metric ascending */

0 comments on commit cd16984

Please sign in to comment.
You can’t perform that action at this time.