diff --git a/metadata_exporter.c b/metadata_exporter.c index 539403d..6749dd2 100644 --- a/metadata_exporter.c +++ b/metadata_exporter.c @@ -262,7 +262,6 @@ static struct json_object *create_fake_restart_obj() return obj; } -#if 0 static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id, uint8_t event_param, char *event_value_str, uint64_t tstamp) { @@ -412,7 +411,6 @@ static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id, return obj; } -#endif static ssize_t send_netlink_json(uint8_t *snd_buf, struct json_object *parsed_obj, int32_t sock_fd, @@ -592,17 +590,17 @@ static void test_netlink(uint32_t packets) while(1) { gettimeofday(&tv, NULL); - /*if (i == 0) + if (i == 0) obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,", i+1); else - obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1);*/ + obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1); - /*if (i < 4) + if (i < 4) obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_L3_UP, "1,2,1", i+1); else - obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec);*/ + obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec); - obj_to_send = create_fake_restart_obj(); + //obj_to_send = create_fake_restart_obj(); if (!obj_to_send) continue; diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 5e1fe4a..417a0bd 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -71,7 +71,7 @@ static void md_sqlite_copy_db(struct md_writer_sqlite *mws, uint8_t from_timeout (mws->session_id_file && !mws->session_id)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Can't export DB. # node_id %d " - "# valid_timestamp %u # session_id_file %s # session_id %d\n", + "# valid_timestamp %u # session_id_file %s # session_id %lu\n", mws->node_id, mws->valid_timestamp, mws->session_id_file ? mws->session_id_file : "EMPTY", @@ -163,7 +163,7 @@ static uint8_t md_sqlite_update_nodeid_db(struct md_writer_sqlite *mws, const ch } static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws, - const char *sql_str, uint64_t tstamp_offset) + const char *sql_str, uint64_t orig_boot_time, uint64_t real_boot_time) { int32_t retval; sqlite3_stmt *update_timestamp; @@ -174,12 +174,17 @@ static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws, return RETVAL_FAILURE; } - if ((retval = sqlite3_bind_int64(update_timestamp, 1, tstamp_offset))) { + if ((retval = sqlite3_bind_int64(update_timestamp, 1, orig_boot_time))) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); return RETVAL_FAILURE; } - if ((retval = sqlite3_bind_int64(update_timestamp, 2, FIRST_VALID_TIMESTAMP))) { + if ((retval = sqlite3_bind_int64(update_timestamp, 2, real_boot_time))) { + META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); + return RETVAL_FAILURE; + } + + if ((retval = sqlite3_bind_int64(update_timestamp, 3, real_boot_time))) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); return RETVAL_FAILURE; } @@ -291,11 +296,32 @@ static sqlite3* md_sqlite_configure_db(struct md_writer_sqlite *mws, const char return db_handle; } +static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot_time) +{ + struct timeval tv; + uint64_t uptime; + + //read uptime + if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) { + return RETVAL_FAILURE; + } + + gettimeofday(&tv, NULL); + + *boot_time = tv.tv_sec - uptime; + + if (*boot_time < mws->orig_boot_time) { + return RETVAL_FAILURE; + } + + return RETVAL_SUCCESS; +} + static int md_sqlite_configure(struct md_writer_sqlite *mws, const char *db_filename, uint32_t node_id, uint32_t db_interval, uint32_t db_events, const char *meta_prefix, const char *gps_prefix, const char *monitor_prefix, const char *usage_prefix, - const char *system_prefix) + const char *system_prefix, const char *ntp_fix_file) { sqlite3 *db_handle = md_sqlite_configure_db(mws, db_filename); const char *dump_events, *dump_updates, *dump_gps, *dump_monitor, @@ -439,6 +465,11 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, mws->system_prefix_len = strlen(system_prefix); } + if (ntp_fix_file) { + memset(mws->ntp_fix_file, 0, sizeof(mws->ntp_fix_file)); + memcpy(mws->ntp_fix_file, ntp_fix_file, strlen(ntp_fix_file)); + } + if (mws->node_id && (md_sqlite_update_nodeid_db(mws, UPDATE_EVENT_ID) || md_sqlite_update_nodeid_db(mws, UPDATE_UPDATES_ID) || md_sqlite_update_nodeid_db(mws, UPDATE_SYSTEM_ID))) { @@ -454,7 +485,7 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, system_helpers_read_uint64_from_file(mws->last_conn_tstamp_path, &(mws->dump_tstamp)); - return RETVAL_SUCCESS; + return md_sqlite_read_boot_time(mws, &(mws->orig_boot_time)); } void md_sqlite_usage() @@ -474,6 +505,7 @@ void md_sqlite_usage() fprintf(stderr, " \"api_version\":\tbackend API version (default: 1)\n"); fprintf(stderr, " \"last_conn_tstamp_path\":\toptional path to file where we read/store timestamp of last conn dump\n"); fprintf(stderr, " \"output_format\":\tJSON/SQL (default SQL)\n"); + fprintf(stderr, " \"ntp_fix_file\":\tFile to check for NTP fix\n"); fprintf(stderr, "}\n"); } @@ -483,7 +515,7 @@ int32_t md_sqlite_init(void *ptr, json_object* config) uint32_t node_id = 0, interval = DEFAULT_TIMEOUT, num_events = EVENT_LIMIT; const char *db_filename = NULL, *meta_prefix = NULL, *gps_prefix = NULL, *monitor_prefix = NULL, *usage_prefix = NULL, - *output_format = NULL, *system_prefix = NULL; + *output_format = NULL, *system_prefix = NULL, *ntp_fix_file = NULL; json_object* subconfig; if (json_object_object_get_ex(config, "sqlite", &subconfig)) { @@ -515,7 +547,9 @@ int32_t md_sqlite_init(void *ptr, json_object* config) else if (!strcmp(key, "last_conn_tstamp_path")) mws->last_conn_tstamp_path = strdup(json_object_get_string(val)); else if (!strcmp(key, "output_format")) - output_format = json_object_get_string(val); + output_format = json_object_get_string(val); + else if (!strcmp(key, "ntp_fix_file")) + ntp_fix_file = json_object_get_string(val); } } @@ -528,7 +562,8 @@ int32_t md_sqlite_init(void *ptr, json_object* config) (gps_prefix && strlen(gps_prefix) > 117) || (monitor_prefix && strlen(monitor_prefix) > 117) || (usage_prefix && strlen(usage_prefix) > 117) || - (system_prefix && strlen(system_prefix) > 117)) { + (system_prefix && strlen(system_prefix) > 117) || + (ntp_fix_file && strlen(ntp_fix_file) > 127)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "SQLite temp file prefix too long\n"); return RETVAL_FAILURE; } @@ -563,31 +598,31 @@ int32_t md_sqlite_init(void *ptr, json_object* config) return md_sqlite_configure(mws, db_filename, node_id, interval, num_events, meta_prefix, gps_prefix, monitor_prefix, usage_prefix, - system_prefix); + system_prefix, ntp_fix_file); } static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) { struct timeval tv; - uint64_t real_boot_time, uptime; + uint64_t real_boot_time; + gettimeofday(&tv, NULL); - //We have yet to get proper timestamp, so do not export any events - if (tv.tv_sec < FIRST_VALID_TIMESTAMP) + if (mws->ntp_fix_file[0] && access(mws->ntp_fix_file, F_OK)) return RETVAL_FAILURE; - //read uptime - if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) + if (md_sqlite_read_boot_time(mws, &real_boot_time)) { return RETVAL_FAILURE; + } - real_boot_time = tv.tv_sec - uptime; + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %" PRIu64 " orig boot %" PRIu64 "\n", real_boot_time, mws->orig_boot_time); if (md_sqlite_update_timestamp_db(mws, UPDATE_EVENT_TSTAMP, - real_boot_time) || + mws->orig_boot_time, real_boot_time) || md_sqlite_update_timestamp_db(mws, UPDATE_UPDATES_TSTAMP, - real_boot_time) || + mws->orig_boot_time, real_boot_time) || md_sqlite_update_timestamp_db(mws, UPDATE_SYSTEM_TSTAMP, - real_boot_time)) { + mws->orig_boot_time, real_boot_time)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Could not update tstamp in database\n"); return RETVAL_FAILURE; } @@ -680,7 +715,7 @@ static void md_sqlite_handle(struct md_writer *writer, struct md_event *event) //We have received an indication that a valid timestamp is present, so //check and update - if (!mws->valid_timestamp && event->tstamp > FIRST_VALID_TIMESTAMP) { + if (!mws->valid_timestamp) { if (md_sqlite_check_valid_tstamp(mws)) { printf("Invalid timestamp\n"); return; diff --git a/metadata_writer_sqlite.h b/metadata_writer_sqlite.h index eabe0b5..d7b670e 100644 --- a/metadata_writer_sqlite.h +++ b/metadata_writer_sqlite.h @@ -36,14 +36,6 @@ #define MAX_PATH_LEN 128 #define FAKE_UPDATE_LIMIT 120 -//This the first valid timestamp of an event and the value is not randomly -//chosen, it is the time when this piece of code was written. And -//since time is never supposed to move backwards ... Note that this check -//assumes that all nodes will have some offset time lower than this, and -//then ntp (or something else) will set a correct time. A good starting -//point is epoch -#define FIRST_VALID_TIMESTAMP 1455740094 - #define CREATE_SQL "CREATE TABLE IF NOT EXISTS NetworkEvent(" \ "NodeId INTEGER NOT NULL," \ "SessionId INTEGER NOT NULL," \ @@ -206,15 +198,15 @@ "WHERE NodeId=0" #define UPDATE_EVENT_TSTAMP "UPDATE NetworkEvent SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_UPDATES_TSTAMP "UPDATE NetworkUpdates SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_SYSTEM_TSTAMP "UPDATE RebootEvent SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_EVENT_SESSION_ID "UPDATE NetworkEvent SET "\ @@ -325,6 +317,22 @@ struct backend_timeout_handle; struct md_writer_sqlite { MD_WRITER; + uint64_t dump_tstamp; + uint64_t last_msg_tstamp; + uint64_t last_gps_insert; + uint64_t orig_boot_time; + + //TODO: Consider moving this to the generic writer struct if need be + //These values keep track of the unique session id (and multiplier), which + //are normally assumed to be the boot counter (+ multiplier) + uint64_t session_id; + uint64_t session_id_multip; + + float gps_speed; + + size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len, + usage_prefix_len, system_prefix_len; + sqlite3 *db_handle; sqlite3_stmt *insert_event, *insert_update; @@ -342,6 +350,8 @@ struct md_writer_sqlite { char *session_id_file; char *node_id_file; const char *last_conn_tstamp_path; + struct backend_timeout_handle *timeout_handle; + struct timeval first_fake_update; uint32_t node_id; uint32_t db_interval; @@ -356,25 +366,9 @@ struct md_writer_sqlite { uint8_t file_failed; uint8_t do_fake_updates; uint8_t valid_timestamp; - struct timeval first_fake_update; - uint64_t dump_tstamp; - uint64_t last_msg_tstamp; - uint64_t last_gps_insert; - - //TODO: Consider moving this to the generic writer struct if need be - //These values keep track of the unique session id (and multiplier), which - //are normally assumed to be the boot counter (+ multiplier) - uint64_t session_id; - uint64_t session_id_multip; - - float gps_speed; - - struct backend_timeout_handle *timeout_handle; char meta_prefix[128], gps_prefix[128], monitor_prefix[128], - usage_prefix[128], system_prefix[128]; - size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len, - usage_prefix_len, system_prefix_len; + usage_prefix[128], system_prefix[128], ntp_fix_file[128]; uint8_t api_version; uint8_t delete_conn_update;