Skip to content

Remove epoch assumption #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Apr 9, 2018
Merged

Remove epoch assumption #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
752d742
sqlite: Improve alignement of struct
kristrev Apr 6, 2018
a7409f3
sqlite: Improve boot time handling
kristrev Apr 6, 2018
84b2f73
sqlite: Update timestamp update
kristrev Apr 9, 2018
bb2766f
sqlite: Replace timestamp with NTP file
kristrev Apr 9, 2018
98a36c3
sqlite: Debug for uptime
kristrev Apr 9, 2018
9c07ba5
sqlite: Fix formatting
kristrev Apr 9, 2018
b874ab9
sqlite: More formatting improvements
kristrev Apr 9, 2018
0bda3f1
sqlite: Remove debug output
kristrev Apr 9, 2018
108f06d
sqlite: Read time after uptime
kristrev Apr 9, 2018
5fcfc91
sqlite: Remove debug output, finish epoch ass.
kristrev Apr 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions metadata_exporter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,6 @@ static struct json_object *create_fake_restart_obj()
return obj;
}

#if 0
static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id,
uint8_t event_param, char *event_value_str, uint64_t tstamp)
{
Expand Down Expand Up @@ -412,7 +411,6 @@ static struct json_object *create_fake_conn_obj(uint64_t l3_id, uint64_t l4_id,

return obj;
}
#endif

static ssize_t send_netlink_json(uint8_t *snd_buf,
struct json_object *parsed_obj, int32_t sock_fd,
Expand Down Expand Up @@ -592,17 +590,17 @@ static void test_netlink(uint32_t packets)
while(1) {
gettimeofday(&tv, NULL);

/*if (i == 0)
if (i == 0)
obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,", i+1);
else
obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1);*/
obj_to_send = create_fake_conn_obj(0, 0, CONN_EVENT_META_UPDATE, "1,2,1,4", i+1);

/*if (i < 4)
if (i < 4)
obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_L3_UP, "1,2,1", i+1);
else
obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec);*/
obj_to_send = create_fake_conn_obj(1, 2, CONN_EVENT_DATA_USAGE_UPDATE, "1,2,1,4", tv.tv_sec);

obj_to_send = create_fake_restart_obj();
//obj_to_send = create_fake_restart_obj();

if (!obj_to_send)
continue;
Expand Down
75 changes: 55 additions & 20 deletions metadata_writer_sqlite.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ static void md_sqlite_copy_db(struct md_writer_sqlite *mws, uint8_t from_timeout
(mws->session_id_file && !mws->session_id))
{
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Can't export DB. # node_id %d "
"# valid_timestamp %u # session_id_file %s # session_id %d\n",
"# valid_timestamp %u # session_id_file %s # session_id %lu\n",
mws->node_id,
mws->valid_timestamp,
mws->session_id_file ? mws->session_id_file : "EMPTY",
Expand Down Expand Up @@ -163,7 +163,7 @@ static uint8_t md_sqlite_update_nodeid_db(struct md_writer_sqlite *mws, const ch
}

static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws,
const char *sql_str, uint64_t tstamp_offset)
const char *sql_str, uint64_t orig_boot_time, uint64_t real_boot_time)
{
int32_t retval;
sqlite3_stmt *update_timestamp;
Expand All @@ -174,12 +174,17 @@ static uint8_t md_sqlite_update_timestamp_db(struct md_writer_sqlite *mws,
return RETVAL_FAILURE;
}

if ((retval = sqlite3_bind_int64(update_timestamp, 1, tstamp_offset))) {
if ((retval = sqlite3_bind_int64(update_timestamp, 1, orig_boot_time))) {
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval));
return RETVAL_FAILURE;
}

if ((retval = sqlite3_bind_int64(update_timestamp, 2, FIRST_VALID_TIMESTAMP))) {
if ((retval = sqlite3_bind_int64(update_timestamp, 2, real_boot_time))) {
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval));
return RETVAL_FAILURE;
}

if ((retval = sqlite3_bind_int64(update_timestamp, 3, real_boot_time))) {
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Bind failed %s\n", sqlite3_errstr(retval));
return RETVAL_FAILURE;
}
Expand Down Expand Up @@ -291,11 +296,32 @@ static sqlite3* md_sqlite_configure_db(struct md_writer_sqlite *mws, const char
return db_handle;
}

static int md_sqlite_read_boot_time(struct md_writer_sqlite *mws, uint64_t *boot_time)
{
struct timeval tv;
uint64_t uptime;

//read uptime
if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime)) {
return RETVAL_FAILURE;
}

gettimeofday(&tv, NULL);

*boot_time = tv.tv_sec - uptime;

if (*boot_time < mws->orig_boot_time) {
return RETVAL_FAILURE;
}

return RETVAL_SUCCESS;
}

static int md_sqlite_configure(struct md_writer_sqlite *mws,
const char *db_filename, uint32_t node_id, uint32_t db_interval,
uint32_t db_events, const char *meta_prefix, const char *gps_prefix,
const char *monitor_prefix, const char *usage_prefix,
const char *system_prefix)
const char *system_prefix, const char *ntp_fix_file)
{
sqlite3 *db_handle = md_sqlite_configure_db(mws, db_filename);
const char *dump_events, *dump_updates, *dump_gps, *dump_monitor,
Expand Down Expand Up @@ -439,6 +465,11 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws,
mws->system_prefix_len = strlen(system_prefix);
}

if (ntp_fix_file) {
memset(mws->ntp_fix_file, 0, sizeof(mws->ntp_fix_file));
memcpy(mws->ntp_fix_file, ntp_fix_file, strlen(ntp_fix_file));
}

if (mws->node_id && (md_sqlite_update_nodeid_db(mws, UPDATE_EVENT_ID) ||
md_sqlite_update_nodeid_db(mws, UPDATE_UPDATES_ID) ||
md_sqlite_update_nodeid_db(mws, UPDATE_SYSTEM_ID))) {
Expand All @@ -454,7 +485,7 @@ static int md_sqlite_configure(struct md_writer_sqlite *mws,
system_helpers_read_uint64_from_file(mws->last_conn_tstamp_path,
&(mws->dump_tstamp));

return RETVAL_SUCCESS;
return md_sqlite_read_boot_time(mws, &(mws->orig_boot_time));
}

void md_sqlite_usage()
Expand All @@ -474,6 +505,7 @@ void md_sqlite_usage()
fprintf(stderr, " \"api_version\":\tbackend API version (default: 1)\n");
fprintf(stderr, " \"last_conn_tstamp_path\":\toptional path to file where we read/store timestamp of last conn dump\n");
fprintf(stderr, " \"output_format\":\tJSON/SQL (default SQL)\n");
fprintf(stderr, " \"ntp_fix_file\":\tFile to check for NTP fix\n");
fprintf(stderr, "}\n");
}

Expand All @@ -483,7 +515,7 @@ int32_t md_sqlite_init(void *ptr, json_object* config)
uint32_t node_id = 0, interval = DEFAULT_TIMEOUT, num_events = EVENT_LIMIT;
const char *db_filename = NULL, *meta_prefix = NULL, *gps_prefix = NULL,
*monitor_prefix = NULL, *usage_prefix = NULL,
*output_format = NULL, *system_prefix = NULL;
*output_format = NULL, *system_prefix = NULL, *ntp_fix_file = NULL;

json_object* subconfig;
if (json_object_object_get_ex(config, "sqlite", &subconfig)) {
Expand Down Expand Up @@ -515,7 +547,9 @@ int32_t md_sqlite_init(void *ptr, json_object* config)
else if (!strcmp(key, "last_conn_tstamp_path"))
mws->last_conn_tstamp_path = strdup(json_object_get_string(val));
else if (!strcmp(key, "output_format"))
output_format = json_object_get_string(val);
output_format = json_object_get_string(val);
else if (!strcmp(key, "ntp_fix_file"))
ntp_fix_file = json_object_get_string(val);
}
}

Expand All @@ -528,7 +562,8 @@ int32_t md_sqlite_init(void *ptr, json_object* config)
(gps_prefix && strlen(gps_prefix) > 117) ||
(monitor_prefix && strlen(monitor_prefix) > 117) ||
(usage_prefix && strlen(usage_prefix) > 117) ||
(system_prefix && strlen(system_prefix) > 117)) {
(system_prefix && strlen(system_prefix) > 117) ||
(ntp_fix_file && strlen(ntp_fix_file) > 127)) {
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "SQLite temp file prefix too long\n");
return RETVAL_FAILURE;
}
Expand Down Expand Up @@ -563,31 +598,31 @@ int32_t md_sqlite_init(void *ptr, json_object* config)

return md_sqlite_configure(mws, db_filename, node_id, interval,
num_events, meta_prefix, gps_prefix, monitor_prefix, usage_prefix,
system_prefix);
system_prefix, ntp_fix_file);
}

static uint8_t md_sqlite_check_valid_tstamp(struct md_writer_sqlite *mws)
{
struct timeval tv;
uint64_t real_boot_time, uptime;
uint64_t real_boot_time;

gettimeofday(&tv, NULL);

//We have yet to get proper timestamp, so do not export any events
if (tv.tv_sec < FIRST_VALID_TIMESTAMP)
if (mws->ntp_fix_file[0] && access(mws->ntp_fix_file, F_OK))
return RETVAL_FAILURE;

//read uptime
if (system_helpers_read_uint64_from_file("/proc/uptime", &uptime))
if (md_sqlite_read_boot_time(mws, &real_boot_time)) {
return RETVAL_FAILURE;
}

real_boot_time = tv.tv_sec - uptime;
META_PRINT_SYSLOG(mws->parent, LOG_INFO, "Real boot %" PRIu64 " orig boot %" PRIu64 "\n", real_boot_time, mws->orig_boot_time);

if (md_sqlite_update_timestamp_db(mws, UPDATE_EVENT_TSTAMP,
real_boot_time) ||
mws->orig_boot_time, real_boot_time) ||
md_sqlite_update_timestamp_db(mws, UPDATE_UPDATES_TSTAMP,
real_boot_time) ||
mws->orig_boot_time, real_boot_time) ||
md_sqlite_update_timestamp_db(mws, UPDATE_SYSTEM_TSTAMP,
real_boot_time)) {
mws->orig_boot_time, real_boot_time)) {
META_PRINT_SYSLOG(mws->parent, LOG_ERR, "Could not update tstamp in database\n");
return RETVAL_FAILURE;
}
Expand Down Expand Up @@ -680,7 +715,7 @@ static void md_sqlite_handle(struct md_writer *writer, struct md_event *event)

//We have received an indication that a valid timestamp is present, so
//check and update
if (!mws->valid_timestamp && event->tstamp > FIRST_VALID_TIMESTAMP) {
if (!mws->valid_timestamp) {
if (md_sqlite_check_valid_tstamp(mws)) {
printf("Invalid timestamp\n");
return;
Expand Down
50 changes: 22 additions & 28 deletions metadata_writer_sqlite.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,6 @@
#define MAX_PATH_LEN 128
#define FAKE_UPDATE_LIMIT 120

//This the first valid timestamp of an event and the value is not randomly
//chosen, it is the time when this piece of code was written. And
//since time is never supposed to move backwards ... Note that this check
//assumes that all nodes will have some offset time lower than this, and
//then ntp (or something else) will set a correct time. A good starting
//point is epoch
#define FIRST_VALID_TIMESTAMP 1455740094

#define CREATE_SQL "CREATE TABLE IF NOT EXISTS NetworkEvent(" \
"NodeId INTEGER NOT NULL," \
"SessionId INTEGER NOT NULL," \
Expand Down Expand Up @@ -206,15 +198,15 @@
"WHERE NodeId=0"

#define UPDATE_EVENT_TSTAMP "UPDATE NetworkEvent SET " \
"Timestamp = Timestamp + ? "\
"Timestamp = (Timestamp - ?) + ? "\
"WHERE Timestamp < ?"

#define UPDATE_UPDATES_TSTAMP "UPDATE NetworkUpdates SET " \
"Timestamp = Timestamp + ? "\
"Timestamp = (Timestamp - ?) + ? "\
"WHERE Timestamp < ?"

#define UPDATE_SYSTEM_TSTAMP "UPDATE RebootEvent SET " \
"Timestamp = Timestamp + ? "\
"Timestamp = (Timestamp - ?) + ? "\
"WHERE Timestamp < ?"

#define UPDATE_EVENT_SESSION_ID "UPDATE NetworkEvent SET "\
Expand Down Expand Up @@ -325,6 +317,22 @@ struct backend_timeout_handle;
struct md_writer_sqlite {
MD_WRITER;

uint64_t dump_tstamp;
uint64_t last_msg_tstamp;
uint64_t last_gps_insert;
uint64_t orig_boot_time;

//TODO: Consider moving this to the generic writer struct if need be
//These values keep track of the unique session id (and multiplier), which
//are normally assumed to be the boot counter (+ multiplier)
uint64_t session_id;
uint64_t session_id_multip;

float gps_speed;

size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len,
usage_prefix_len, system_prefix_len;

sqlite3 *db_handle;

sqlite3_stmt *insert_event, *insert_update;
Expand All @@ -342,6 +350,8 @@ struct md_writer_sqlite {
char *session_id_file;
char *node_id_file;
const char *last_conn_tstamp_path;
struct backend_timeout_handle *timeout_handle;
struct timeval first_fake_update;

uint32_t node_id;
uint32_t db_interval;
Expand All @@ -356,25 +366,9 @@ struct md_writer_sqlite {
uint8_t file_failed;
uint8_t do_fake_updates;
uint8_t valid_timestamp;
struct timeval first_fake_update;

uint64_t dump_tstamp;
uint64_t last_msg_tstamp;
uint64_t last_gps_insert;

//TODO: Consider moving this to the generic writer struct if need be
//These values keep track of the unique session id (and multiplier), which
//are normally assumed to be the boot counter (+ multiplier)
uint64_t session_id;
uint64_t session_id_multip;

float gps_speed;

struct backend_timeout_handle *timeout_handle;
char meta_prefix[128], gps_prefix[128], monitor_prefix[128],
usage_prefix[128], system_prefix[128];
size_t meta_prefix_len, gps_prefix_len, monitor_prefix_len,
usage_prefix_len, system_prefix_len;
usage_prefix[128], system_prefix[128], ntp_fix_file[128];

uint8_t api_version;
uint8_t delete_conn_update;
Expand Down