From 752d742787ffa303d3af0c5eb1772cc1c3c8d541 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Fri, 6 Apr 2018 14:17:01 +0200 Subject: [PATCH 01/10] sqlite: Improve alignement of struct Move uint64_t/pointers higher up in the struct, to improve alignment on 64 bit machines. --- metadata_writer_sqlite.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/metadata_writer_sqlite.h b/metadata_writer_sqlite.h index eabe0b5..5778eb4 100644 --- a/metadata_writer_sqlite.h +++ b/metadata_writer_sqlite.h @@ -325,6 +325,21 @@ struct backend_timeout_handle; struct md_writer_sqlite { MD_WRITER; + uint64_t dump_tstamp; + uint64_t last_msg_tstamp; + uint64_t last_gps_insert; + + //TODO: Consider moving this to the generic writer struct if need be + //These values keep track of the unique session id (and multiplier), which + //are normally assumed to be the boot counter (+ multiplier) + uint64_t session_id; + uint64_t session_id_multip; + + float gps_speed; + + size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len, + usage_prefix_len, system_prefix_len; + sqlite3 *db_handle; sqlite3_stmt *insert_event, *insert_update; @@ -342,6 +357,8 @@ struct md_writer_sqlite { char *session_id_file; char *node_id_file; const char *last_conn_tstamp_path; + struct backend_timeout_handle *timeout_handle; + struct timeval first_fake_update; uint32_t node_id; uint32_t db_interval; @@ -356,25 +373,9 @@ struct md_writer_sqlite { uint8_t file_failed; uint8_t do_fake_updates; uint8_t valid_timestamp; - struct timeval first_fake_update; - - uint64_t dump_tstamp; - uint64_t last_msg_tstamp; - uint64_t last_gps_insert; - - //TODO: Consider moving this to the generic writer struct if need be - //These values keep track of the unique session id (and multiplier), which - //are normally assumed to be the boot counter (+ multiplier) - uint64_t session_id; - uint64_t session_id_multip; - - float gps_speed; - struct backend_timeout_handle *timeout_handle; char meta_prefix[128], gps_prefix[128], monitor_prefix[128], usage_prefix[128], system_prefix[128]; - size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len, - usage_prefix_len, system_prefix_len; uint8_t api_version; uint8_t delete_conn_update; From a7409f315bd9274f23447cd41c1f897e75f37459 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Fri, 6 Apr 2018 15:38:39 +0200 Subject: [PATCH 02/10] sqlite: Improve boot time handling Until now we have assumed that clock will always either be correct on boot or be set to epoch. This assumption no longer holds, some of the devices we use have an RTC that seems very hard to disabled. Instead of spending time disabling the RTC, we can instead remove the assumption and this commit is the first step. We store events in the database also before we have a valid timestamp, and then update the timestamp once we have a valid fix. Since we assume that system time starts from zero (this code is not run if we have assumed that timestamp is already valid), what is stored in the database is offset (in time) of the event from the system boot time. When we receive a valid fix, we then calculate the real boot time (time_now - uptime) and then add the real boot time to the offsets we already have. When we remove the assumption about zero or valid, the computation becomes a bit more complicated. The values stored in the database before we have a valid fix is no longer necessarily an offset from zero, but instead a valid timestamp. We therefore need to calculate the boot time relative to system clock when data exporter starts. When we get a valid timestamp, we need to find the difference between the previously mentioned real boot time and the original boot time. This is the offset that will be added to all the timestamps in the database (if any). We still have one assumption, and this that the inital time at boot will always be less than the correct time and that one second is one second. We have some devices where time seems random (hello 2084, 2102 and 2076), but those devices we can force to epoch. If we get a device with forcing fails, we need to make some more adjustmenets to this code. However, I consider that pre-mature optimization and having a very negative outlook on thins (and it is Friday afternoon ...). --- metadata_exporter.c | 8 +++----- metadata_writer_sqlite.c | 30 ++++++++++++++++++++++++------ metadata_writer_sqlite.h | 1 + 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/metadata_exporter.c b/metadata_exporter.c index 539403d..4593cc1 100644 --- a/metadata_exporter.c +++ b/metadata_exporter.c @@ -262,7 +262,6 @@ static struct json_object *create_fake_restart_obj() return obj; } -#if 0 static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id, uint8_t event_param, char *event_value_str, uint64_t tstamp) { @@ -412,7 +411,6 @@ static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id, return obj; } -#endif static ssize_t send_netlink_json(uint8_t *snd_buf, struct json_object *parsed_obj, int32_t sock_fd, @@ -595,14 +593,14 @@ static void test_netlink(uint32_t packets) /*if (i == 0) obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,", i+1); else - obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1);*/ + obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1); - /*if (i < 4) + if (i < 4) obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_L3_UP, "1,2,1", i+1); else obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec);*/ - obj_to_send = create_fake_restart_obj(); + //obj_to_send = create_fake_restart_obj(); if (!obj_to_send) continue; diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 5e1fe4a..059781e 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -71,7 +71,7 @@ static void md_sqlite_copy_db(struct md_writer_sqlite *mws, uint8_t from_timeout (mws->session_id_file && !mws->session_id)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Can't export DB. # node_id %d " - "# valid_timestamp %u # session_id_file %s # session_id %d\n", + "# valid_timestamp %u # session_id_file %s # session_id %lu\n", mws->node_id, mws->valid_timestamp, mws->session_id_file ? mws->session_id_file : "EMPTY", @@ -291,6 +291,21 @@ static sqlite3* md_sqlite_configure_db(struct md_writer_sqlite *mws, const char return db_handle; } +static int md_sqlite_read_boot_time(uint64_t *boot_time) +{ + struct timeval tv; + uint64_t uptime; + gettimeofday(&tv, NULL); + + //read uptime + if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) { + return RETVAL_FAILURE; + } + + *boot_time = tv.tv_sec - uptime; + return RETVAL_SUCCESS; +} + static int md_sqlite_configure(struct md_writer_sqlite *mws, const char *db_filename, uint32_t node_id, uint32_t db_interval, uint32_t db_events, const char *meta_prefix, const char *gps_prefix, @@ -454,7 +469,7 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, system_helpers_read_uint64_from_file(mws->last_conn_tstamp_path, &(mws->dump_tstamp)); - return RETVAL_SUCCESS; + return md_sqlite_read_boot_time(&(mws->orig_boot_time)); } void md_sqlite_usage() @@ -569,18 +584,21 @@ int32_t md_sqlite_init(void *ptr, json_object* config) static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) { struct timeval tv; - uint64_t real_boot_time, uptime; + uint64_t real_boot_time, boot_diff; + gettimeofday(&tv, NULL); //We have yet to get proper timestamp, so do not export any events if (tv.tv_sec < FIRST_VALID_TIMESTAMP) return RETVAL_FAILURE; - //read uptime - if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) + if (md_sqlite_read_boot_time(&real_boot_time)) { return RETVAL_FAILURE; + } + + boot_diff = real_boot_time - mws->orig_boot_time; - real_boot_time = tv.tv_sec - uptime; + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %lu orig boot %lu tdiff %lu\n", real_boot_time, mws->orig_boot_time, boot_diff); if (md_sqlite_update_timestamp_db(mws, UPDATE_EVENT_TSTAMP, real_boot_time) || diff --git a/metadata_writer_sqlite.h b/metadata_writer_sqlite.h index 5778eb4..7c7a663 100644 --- a/metadata_writer_sqlite.h +++ b/metadata_writer_sqlite.h @@ -328,6 +328,7 @@ struct md_writer_sqlite { uint64_t dump_tstamp; uint64_t last_msg_tstamp; uint64_t last_gps_insert; + uint64_t orig_boot_time; //TODO: Consider moving this to the generic writer struct if need be //These values keep track of the unique session id (and multiplier), which From 84b2f7387fc7d0412e016646d59cda1ef0d279ba Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 13:30:08 +0200 Subject: [PATCH 03/10] sqlite: Update timestamp update Second step of removing epoch assumption. Since timestamps are no longer guaranteed to be offset from 0, we need to remove the original boot time and then add the new boot time. I.e., the first calculation will give us offset from boot time, and the second calculation will give us the correct time of the event. --- metadata_exporter.c | 4 ++-- metadata_writer_sqlite.c | 23 +++++++++++++---------- metadata_writer_sqlite.h | 6 +++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/metadata_exporter.c b/metadata_exporter.c index 4593cc1..6749dd2 100644 --- a/metadata_exporter.c +++ b/metadata_exporter.c @@ -590,7 +590,7 @@ static void test_netlink(uint32_t packets) while(1) { gettimeofday(&tv, NULL); - /*if (i == 0) + if (i == 0) obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,", i+1); else obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1); @@ -598,7 +598,7 @@ static void test_netlink(uint32_t packets) if (i < 4) obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_L3_UP, "1,2,1", i+1); else - obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec);*/ + obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec); //obj_to_send = create_fake_restart_obj(); diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 059781e..7e31213 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -163,7 +163,7 @@ static uint8_t md_sqlite_update_nodeid_db(struct md_writer_sqlite *mws, const ch } static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws, - const char *sql_str, uint64_t tstamp_offset) + const char *sql_str, uint64_t orig_boot_time, uint64_t real_boot_time) { int32_t retval; sqlite3_stmt *update_timestamp; @@ -174,12 +174,17 @@ static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws, return RETVAL_FAILURE; } - if ((retval = sqlite3_bind_int64(update_timestamp, 1, tstamp_offset))) { + if ((retval = sqlite3_bind_int64(update_timestamp, 1, orig_boot_time))) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); return RETVAL_FAILURE; } - if ((retval = sqlite3_bind_int64(update_timestamp, 2, FIRST_VALID_TIMESTAMP))) { + if ((retval = sqlite3_bind_int64(update_timestamp, 2, real_boot_time))) { + META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); + return RETVAL_FAILURE; + } + + if ((retval = sqlite3_bind_int64(update_timestamp, 3, real_boot_time))) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval)); return RETVAL_FAILURE; } @@ -584,7 +589,7 @@ int32_t md_sqlite_init(void *ptr, json_object* config) static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) { struct timeval tv; - uint64_t real_boot_time, boot_diff; + uint64_t real_boot_time; gettimeofday(&tv, NULL); @@ -596,16 +601,14 @@ static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) return RETVAL_FAILURE; } - boot_diff = real_boot_time - mws->orig_boot_time; - - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %lu orig boot %lu tdiff %lu\n", real_boot_time, mws->orig_boot_time, boot_diff); + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %lu orig boot %lu\n", real_boot_time, mws->orig_boot_time); if (md_sqlite_update_timestamp_db(mws, UPDATE_EVENT_TSTAMP, - real_boot_time) || + mws->orig_boot_time, real_boot_time) || md_sqlite_update_timestamp_db(mws, UPDATE_UPDATES_TSTAMP, - real_boot_time) || + mws->orig_boot_time, real_boot_time) || md_sqlite_update_timestamp_db(mws, UPDATE_SYSTEM_TSTAMP, - real_boot_time)) { + mws->orig_boot_time, real_boot_time)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Could not update tstamp in database\n"); return RETVAL_FAILURE; } diff --git a/metadata_writer_sqlite.h b/metadata_writer_sqlite.h index 7c7a663..8d10dfc 100644 --- a/metadata_writer_sqlite.h +++ b/metadata_writer_sqlite.h @@ -206,15 +206,15 @@ "WHERE NodeId=0" #define UPDATE_EVENT_TSTAMP "UPDATE NetworkEvent SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_UPDATES_TSTAMP "UPDATE NetworkUpdates SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_SYSTEM_TSTAMP "UPDATE RebootEvent SET " \ - "Timestamp = Timestamp + ? "\ + "Timestamp = (Timestamp - ?) + ? "\ "WHERE Timestamp < ?" #define UPDATE_EVENT_SESSION_ID "UPDATE NetworkEvent SET "\ From bb2766f8eaced82532fcdabbc5bfe143a7ebd0b3 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 14:09:18 +0200 Subject: [PATCH 04/10] sqlite: Replace timestamp with NTP file Replace the "good" timestamp we have used as a limit before with a file that will be written when NTP fix has been obtained. We assume that all devices run NTP (all of our does), and a file seems like the easiest "detect fix" solution that works across devices. --- metadata_writer_sqlite.c | 24 ++++++++++++++++-------- metadata_writer_sqlite.h | 10 +--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 7e31213..f58d376 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -315,7 +315,7 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, const char *db_filename, uint32_t node_id, uint32_t db_interval, uint32_t db_events, const char *meta_prefix, const char *gps_prefix, const char *monitor_prefix, const char *usage_prefix, - const char *system_prefix) + const char *system_prefix, const char *ntp_fix_file) { sqlite3 *db_handle = md_sqlite_configure_db(mws, db_filename); const char *dump_events, *dump_updates, *dump_gps, *dump_monitor, @@ -459,6 +459,11 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, mws->system_prefix_len = strlen(system_prefix); } + if (ntp_fix_file) { + memset(mws->ntp_fix_file, 0, sizeof(mws->ntp_fix_file)); + memcpy(mws->ntp_fix_file, ntp_fix_file, strlen(ntp_fix_file)); + } + if (mws->node_id && (md_sqlite_update_nodeid_db(mws, UPDATE_EVENT_ID) || md_sqlite_update_nodeid_db(mws, UPDATE_UPDATES_ID) || md_sqlite_update_nodeid_db(mws, UPDATE_SYSTEM_ID))) { @@ -494,6 +499,7 @@ void md_sqlite_usage() fprintf(stderr, " \"api_version\":\tbackend API version (default: 1)\n"); fprintf(stderr, " \"last_conn_tstamp_path\":\toptional path to file where we read/store timestamp of last conn dump\n"); fprintf(stderr, " \"output_format\":\tJSON/SQL (default SQL)\n"); + fprintf(stderr, " \"ntp_fix_file\":\tFile to check for NTP fix\n"); fprintf(stderr, "}\n"); } @@ -503,7 +509,7 @@ int32_t md_sqlite_init(void *ptr, json_object* config) uint32_t node_id = 0, interval = DEFAULT_TIMEOUT, num_events = EVENT_LIMIT; const char *db_filename = NULL, *meta_prefix = NULL, *gps_prefix = NULL, *monitor_prefix = NULL, *usage_prefix = NULL, - *output_format = NULL, *system_prefix = NULL; + *output_format = NULL, *system_prefix = NULL, *ntp_fix_file = NULL; json_object* subconfig; if (json_object_object_get_ex(config, "sqlite", &subconfig)) { @@ -535,7 +541,9 @@ int32_t md_sqlite_init(void *ptr, json_object* config) else if (!strcmp(key, "last_conn_tstamp_path")) mws->last_conn_tstamp_path = strdup(json_object_get_string(val)); else if (!strcmp(key, "output_format")) - output_format = json_object_get_string(val); + output_format = json_object_get_string(val); + else if (!strcmp(key, "ntp_fix_file")) + ntp_fix_file = json_object_get_string(val); } } @@ -548,7 +556,8 @@ int32_t md_sqlite_init(void *ptr, json_object* config) (gps_prefix && strlen(gps_prefix) > 117) || (monitor_prefix && strlen(monitor_prefix) > 117) || (usage_prefix && strlen(usage_prefix) > 117) || - (system_prefix && strlen(system_prefix) > 117)) { + (system_prefix && strlen(system_prefix) > 117) || + (ntp_fix_file && strlen(ntp_fix_file) > 127)) { META_PRINT_SYSLOG(mws->parent, LOG_ERR, "SQLite temp file prefix too long\n"); return RETVAL_FAILURE; } @@ -583,7 +592,7 @@ int32_t md_sqlite_init(void *ptr, json_object* config) return md_sqlite_configure(mws, db_filename, node_id, interval, num_events, meta_prefix, gps_prefix, monitor_prefix, usage_prefix, - system_prefix); + system_prefix, ntp_fix_file); } static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) @@ -593,8 +602,7 @@ static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) gettimeofday(&tv, NULL); - //We have yet to get proper timestamp, so do not export any events - if (tv.tv_sec < FIRST_VALID_TIMESTAMP) + if (mws->ntp_fix_file[0] && access(mws->ntp_fix_file, F_OK)) return RETVAL_FAILURE; if (md_sqlite_read_boot_time(&real_boot_time)) { @@ -701,7 +709,7 @@ static void md_sqlite_handle(struct md_writer *writer, struct md_event *event) //We have received an indication that a valid timestamp is present, so //check and update - if (!mws->valid_timestamp && event->tstamp > FIRST_VALID_TIMESTAMP) { + if (!mws->valid_timestamp) { if (md_sqlite_check_valid_tstamp(mws)) { printf("Invalid timestamp\n"); return; diff --git a/metadata_writer_sqlite.h b/metadata_writer_sqlite.h index 8d10dfc..d7b670e 100644 --- a/metadata_writer_sqlite.h +++ b/metadata_writer_sqlite.h @@ -36,14 +36,6 @@ #define MAX_PATH_LEN 128 #define FAKE_UPDATE_LIMIT 120 -//This the first valid timestamp of an event and the value is not randomly -//chosen, it is the time when this piece of code was written. And -//since time is never supposed to move backwards ... Note that this check -//assumes that all nodes will have some offset time lower than this, and -//then ntp (or something else) will set a correct time. A good starting -//point is epoch -#define FIRST_VALID_TIMESTAMP 1455740094 - #define CREATE_SQL "CREATE TABLE IF NOT EXISTS NetworkEvent(" \ "NodeId INTEGER NOT NULL," \ "SessionId INTEGER NOT NULL," \ @@ -376,7 +368,7 @@ struct md_writer_sqlite { uint8_t valid_timestamp; char meta_prefix[128], gps_prefix[128], monitor_prefix[128], - usage_prefix[128], system_prefix[128]; + usage_prefix[128], system_prefix[128], ntp_fix_file[128]; uint8_t api_version; uint8_t delete_conn_update; From 98a36c34c8b0c3e7f0e53824a5c6309db6502265 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 15:04:50 +0200 Subject: [PATCH 05/10] sqlite: Debug for uptime Add debug calculation for uptime. --- metadata_writer_sqlite.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index f58d376..046fabc 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -296,7 +296,7 @@ static sqlite3* md_sqlite_configure_db(struct md_writer_sqlite *mws, const char return db_handle; } -static int md_sqlite_read_boot_time(uint64_t *boot_time) +static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot_time) { struct timeval tv; uint64_t uptime; @@ -308,6 +308,7 @@ static int md_sqlite_read_boot_time(uint64_t *boot_time) } *boot_time = tv.tv_sec - uptime; + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%lu %lu %lu\n", *boot_time, tv.tv_sec, uptime); return RETVAL_SUCCESS; } @@ -479,7 +480,7 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws, system_helpers_read_uint64_from_file(mws->last_conn_tstamp_path, &(mws->dump_tstamp)); - return md_sqlite_read_boot_time(&(mws->orig_boot_time)); + return md_sqlite_read_boot_time(mws, &(mws->orig_boot_time)); } void md_sqlite_usage() @@ -605,7 +606,7 @@ static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) if (mws->ntp_fix_file[0] && access(mws->ntp_fix_file, F_OK)) return RETVAL_FAILURE; - if (md_sqlite_read_boot_time(&real_boot_time)) { + if (md_sqlite_read_boot_time(mws, &real_boot_time)) { return RETVAL_FAILURE; } From 9c07ba58f07b6034fa65998b89d42c23a0c108ae Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 15:09:54 +0200 Subject: [PATCH 06/10] sqlite: Fix formatting Use correct type for uint64. --- metadata_writer_sqlite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 046fabc..11ec76c 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -308,7 +308,7 @@ static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot } *boot_time = tv.tv_sec - uptime; - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%lu %lu %lu\n", *boot_time, tv.tv_sec, uptime); + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 "%" PRIu64 "%" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); return RETVAL_SUCCESS; } @@ -610,7 +610,7 @@ static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws) return RETVAL_FAILURE; } - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %lu orig boot %lu\n", real_boot_time, mws->orig_boot_time); + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %" PRIu64 " orig boot %" PRIu64 "\n", real_boot_time, mws->orig_boot_time); if (md_sqlite_update_timestamp_db(mws, UPDATE_EVENT_TSTAMP, mws->orig_boot_time, real_boot_time) || From b874ab9e2c5550e89377fc42991b14d8901d6865 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 15:12:57 +0200 Subject: [PATCH 07/10] sqlite: More formatting improvements --- metadata_writer_sqlite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 11ec76c..780e688 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -308,7 +308,7 @@ static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot } *boot_time = tv.tv_sec - uptime; - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 "%" PRIu64 "%" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 " %ld %" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); return RETVAL_SUCCESS; } From 0bda3f1c86cdc0410e096f346a7c7db4d54e13f3 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 15:36:24 +0200 Subject: [PATCH 08/10] sqlite: Remove debug output --- metadata_writer_sqlite.c | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 780e688..1bfecdf 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -308,7 +308,6 @@ static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot } *boot_time = tv.tv_sec - uptime; - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 " %ld %" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); return RETVAL_SUCCESS; } From 108f06d92d7763491e2eaebd17a5801d99c8b25c Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 15:47:41 +0200 Subject: [PATCH 09/10] sqlite: Read time after uptime We should read time after we have read uptime. The reason is that if we for example read time at sec.999, then second will have wrapped when we read uptime. While we might risk that the same happens with the new order of operation, it is easier to work with a larger timestamp than a large uptime. For example, with a larger timestamp, we risk having one second as offset to the original boot time. With a small timestamp, we risk having -1 second as offset and that is slightly harder to deal with. The boot time is meant as an estimation anyway, so 100% accurate value is not needed. --- metadata_writer_sqlite.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 1bfecdf..424049e 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -300,14 +300,16 @@ static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot { struct timeval tv; uint64_t uptime; - gettimeofday(&tv, NULL); //read uptime if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) { return RETVAL_FAILURE; } + gettimeofday(&tv, NULL); + *boot_time = tv.tv_sec - uptime; + META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 " %ld %" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); return RETVAL_SUCCESS; } From 5fcfc910742a4b4b03ebfa5d0f71cf4817d220e5 Mon Sep 17 00:00:00 2001 From: Kristian Evensen Date: Mon, 9 Apr 2018 20:48:08 +0200 Subject: [PATCH 10/10] sqlite: Remove debug output, finish epoch ass. Remove some debug out, as the last commit of removing the epoch assumption. There is no need to worry about the time. The issue I tried to solve, was that for example the real boot time could be one second less than the original boot time. This can happen if for example the uptime and timeofday is read at different seconds. One second is not important for us, there is a higher latency in when we discover that a network is down for example. Our goal with the DB calculation is to shift the time offset to that of the boot time. This is done by doing (timestamp - orig_boot_time) + real_boot_time, for all timestamps less than real_boot_time. If real_boot_time is less than orig_boot_time, then the statement has no effect. This is ok, as we input absolute time in the database. The code is meant to handle large differences in time, for example from epoch or some default rtc value. And that works fine. --- metadata_writer_sqlite.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metadata_writer_sqlite.c b/metadata_writer_sqlite.c index 424049e..417a0bd 100644 --- a/metadata_writer_sqlite.c +++ b/metadata_writer_sqlite.c @@ -309,7 +309,11 @@ static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot gettimeofday(&tv, NULL); *boot_time = tv.tv_sec - uptime; - META_PRINT_SYSLOG(mws->parent, LOG_INFO, "%" PRIu64 " %ld %" PRIu64 "\n", *boot_time, tv.tv_sec, uptime); + + if (*boot_time < mws->orig_boot_time) { + return RETVAL_FAILURE; + } + return RETVAL_SUCCESS; }