Replace use of filesystem's cache directory with SQLite via KIOEventStore

KeenClient.xcodeproj/project.pbxproj

@@ -53,13 +53,17 @@
 		017EE13F14E30C96000F3868 /* KeenClientTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 017EE13E14E30C96000F3868 /* KeenClientTests.m */; };
 		01BA1B0514E895BC00CF9F84 /* KeenConstants.h in Headers */ = {isa = PBXBuildFile; fileRef = 01BA1B0314E895BC00CF9F84 /* KeenConstants.h */; };
 		01BA1B0614E895BC00CF9F84 /* KeenConstants.m in Sources */ = {isa = PBXBuildFile; fileRef = 01BA1B0414E895BC00CF9F84 /* KeenConstants.m */; };
-		CA6410D818E37E7C00E53E3C /* KIOEventStore.h in Headers */ = {isa = PBXBuildFile; fileRef = CA6410D618E37E7C00E53E3C /* KIOEventStore.h */; };
+		CA6410D818E37E7C00E53E3C /* KIOEventStore.h in Headers */ = {isa = PBXBuildFile; fileRef = CA6410D618E37E7C00E53E3C /* KIOEventStore.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		CA6410D918E37E7C00E53E3C /* KIOEventStore.m in Sources */ = {isa = PBXBuildFile; fileRef = CA6410D718E37E7C00E53E3C /* KIOEventStore.m */; };
 		CA6410E218E39F3A00E53E3C /* KIOEventStoreTests.m in Sources */ = {isa = PBXBuildFile; fileRef = CA6410E118E39F3A00E53E3C /* KIOEventStoreTests.m */; };
 		CA795C4E18E32BAB00EBDFC4 /* libsqlite3.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = CA795C4D18E32BAB00EBDFC4 /* libsqlite3.dylib */; };
 		CA795C4F18E32BB800EBDFC4 /* libsqlite3.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = CA795C4D18E32BAB00EBDFC4 /* libsqlite3.dylib */; };
 		CA795C5018E32BBF00EBDFC4 /* libsqlite3.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = CA795C4D18E32BAB00EBDFC4 /* libsqlite3.dylib */; };
 		CA795C5118E32BC600EBDFC4 /* libsqlite3.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = CA795C4D18E32BAB00EBDFC4 /* libsqlite3.dylib */; };
+		DEFA50181947A66200F78E13 /* KIOEventStore.m in Sources */ = {isa = PBXBuildFile; fileRef = CA6410D718E37E7C00E53E3C /* KIOEventStore.m */; };
+		DEFA50191947A67E00F78E13 /* KIOEventStore.m in Sources */ = {isa = PBXBuildFile; fileRef = CA6410D718E37E7C00E53E3C /* KIOEventStore.m */; };
+		DEFA501A1947A70800F78E13 /* KIOEventStore.h in Headers */ = {isa = PBXBuildFile; fileRef = CA6410D618E37E7C00E53E3C /* KIOEventStore.h */; settings = {ATTRIBUTES = (Public, ); }; };
+		DEFA501B1947A72600F78E13 /* KIOEventStore.h in Headers */ = {isa = PBXBuildFile; fileRef = CA6410D618E37E7C00E53E3C /* KIOEventStore.h */; settings = {ATTRIBUTES = (Public, ); }; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -313,6 +317,7 @@
 			files = (
 				0111012614EF709A009794A5 /* KeenClient.h in Headers */,
 				010EB07F1676CF34002B07B6 /* KeenProperties.h in Headers */,
+				DEFA501B1947A72600F78E13 /* KIOEventStore.h in Headers */,
 				0111012D14EF709A009794A5 /* KeenConstants.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -323,6 +328,7 @@
 			files = (
 				0111013E14EF70AB009794A5 /* KeenClient.h in Headers */,
 				010EB0801676CF41002B07B6 /* KeenProperties.h in Headers */,
+				DEFA501A1947A70800F78E13 /* KIOEventStore.h in Headers */,
 				0111014514EF70AB009794A5 /* KeenConstants.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -486,7 +492,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "export TMP_DIR=/tmp/keenios\n\nrm -rf ${TMP_DIR}\nrm -rf ${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\nrm -rf ${BUILT_PRODUCTS_DIR}/KeenClient.zip\n\nlipo -create \"${BUILT_PRODUCTS_DIR}/../${CONFIGURATION}-iphonesimulator/libKeenClient-Simulator.a\" \"${BUILT_PRODUCTS_DIR}/../${CONFIGURATION}-iphoneos/libKeenClient-Device.a\" -output \"${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\"\n\nmkdir -p ${TMP_DIR}\n\ncp -r \"${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\" \"${BUILT_PRODUCTS_DIR}/usr/local/include/KeenClient.h\" \"${BUILT_PRODUCTS_DIR}/usr/local/include/KeenProperties.h\" ${TMP_DIR}/.\n\ncd ${TMP_DIR}\nzip -r \"${BUILT_PRODUCTS_DIR}/KeenClient.zip\" *";
+			shellScript = "export TMP_DIR=/tmp/keenios\n\nrm -rf ${TMP_DIR}\nrm -rf ${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\nrm -rf ${BUILT_PRODUCTS_DIR}/KeenClient.zip\n\nlipo -create \"${BUILT_PRODUCTS_DIR}/../${CONFIGURATION}-iphonesimulator/libKeenClient-Simulator.a\" \"${BUILT_PRODUCTS_DIR}/../${CONFIGURATION}-iphoneos/libKeenClient-Device.a\" -output \"${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\"\n\nmkdir -p ${TMP_DIR}\n\ncp -r \"${BUILT_PRODUCTS_DIR}/libKeenClient-Aggregate.a\" \"${BUILT_PRODUCTS_DIR}/usr/local/include/KeenClient.h\" \"${BUILT_PRODUCTS_DIR}/usr/local/include/KeenProperties.h\" \"${BUILT_PRODUCTS_DIR}/usr/local/include/KIOEventStore.h\" ${TMP_DIR}/.\n\ncd ${TMP_DIR}\nzip -r \"${BUILT_PRODUCTS_DIR}/KeenClient.zip\" *";
 		};
 		017EE12C14E30C96000F3868 /* ShellScript */ = {
 			isa = PBXShellScriptBuildPhase;
@@ -508,6 +514,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				DEFA50181947A66200F78E13 /* KIOEventStore.m in Sources */,
 				0111011E14EF709A009794A5 /* KeenClient.m in Sources */,
 				010EB08C1676D1D6002B07B6 /* KeenProperties.m in Sources */,
 				0111011F14EF709A009794A5 /* KeenConstants.m in Sources */,
@@ -518,6 +525,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				DEFA50191947A67E00F78E13 /* KIOEventStore.m in Sources */,
 				0111013614EF70AB009794A5 /* KeenClient.m in Sources */,
 				010EB08D1676D1EE002B07B6 /* KeenProperties.m in Sources */,
 				0111013714EF70AB009794A5 /* KeenConstants.m in Sources */,

KeenClient/KIOEventStore.h

@@ -14,8 +14,6 @@
 // The project ID for this store.
 @property (nonatomic, strong) NSString *projectId;
 
-- (id)initWithProjectId:(NSString *)pid;
-
  /**
   Reset any pending events so they can be resent.
   */
@@ -30,13 +28,13 @@
  /**
   Add an event to the store.
   */
-- (BOOL)addEvent: (NSString *)eventData;
+- (BOOL)addEvent: (NSData *)eventData collection: (NSString *)coll;
 
  /**
-  Get a list of events that are ready to send to Keen. Events that are
-  returned have been flagged as pending in the underlying store.
+  Get a dictionary of events keyed by id that are ready to send to Keen. Events
+  that are returned have been flagged as pending in the underlying store.
   */
-- (NSMutableArray *)getEvents;
+- (NSMutableDictionary *)getEvents;
 
  /**
   Get a count of pending events.
@@ -52,4 +50,20 @@
   Purge pending events that were returned from a previous call to getEvents.
   */
 - (void)purgePendingEvents;
+
+ /**
+  Delete an event from the store
+  */
+- (void)deleteEvent: (NSNumber *)eventId;
+
+/**
+ Delete all events from the store
+ */
+- (void)deleteAllEvents;
+
+
+/**
+ Delete events starting at an offset. Helps to keep the "queue" bounded.
+ */
+- (void)deleteEventsFromOffset: (NSNumber *)offset;
 @end

KeenClient/KIOEventStore.m

@@ -24,20 +24,16 @@ @implementation KIOEventStore {
     sqlite3_stmt *make_pending_stmt;
     sqlite3_stmt *reset_pending_stmt;
     sqlite3_stmt *purge_stmt;
+    sqlite3_stmt *delete_stmt;
+    sqlite3_stmt *delete_all_stmt;
+    sqlite3_stmt *age_out_stmt;
 }
 
 - (instancetype)init {
-    NSAssert(NO, @"init not allowed, use initWithProjectId");
-    [self release];
-    return nil;
-}
-
-- (instancetype)initWithProjectId:(NSString *)pid {
     self = [super init];
 
     if(self) {
         dbIsOpen = NO;
-        self.projectId = pid;
         // First, let's open the database.
         if ([self openDB]) {
             // Then try and create the table.
@@ -49,14 +45,14 @@ - (instancetype)initWithProjectId:(NSString *)pid {
             // Now we'll init prepared statements for all the things we might do.
 
             // This statement inserts events into the table.
-            char *insert_sql = "INSERT INTO events (projectId, eventData, pending) VALUES (?, ?, 0)";
+            char *insert_sql = "INSERT INTO events (projectId, collection, eventData, pending) VALUES (?, ?, ?, 0)";
             if (sqlite3_prepare_v2(keen_dbname, insert_sql, -1, &insert_stmt, NULL) != SQLITE_OK) {
                 [self handleSQLiteFailure:@"prepare insert statement"];
                 [self closeDB];
             }
 
             // This statement finds non-pending events in the table.
-            char *find_sql = "SELECT id, eventData FROM events WHERE pending=0 AND projectId=?";
+            char *find_sql = "SELECT id, collection, eventData FROM events WHERE pending=0 AND projectId=?";
             if(sqlite3_prepare_v2(keen_dbname, find_sql, -1, &find_stmt, NULL) != SQLITE_OK) {
                 [self handleSQLiteFailure:@"prepare find statement"];
                 [self closeDB];
@@ -95,12 +91,30 @@ - (instancetype)initWithProjectId:(NSString *)pid {
             if(sqlite3_prepare_v2(keen_dbname, purge_sql, -1, &purge_stmt, NULL) != SQLITE_OK) {
                 [self closeDB];
             }
+
+            // This statement deletes a specific event.
+            char *delete_sql = "DELETE FROM events WHERE id=?";
+            if(sqlite3_prepare_v2(keen_dbname, delete_sql, -1, &delete_stmt, NULL) != SQLITE_OK) {
+                [self closeDB];
+            }
+
+            // This statement deletes all events.
+            char *delete_all_sql = "DELETE FROM events";
+            if(sqlite3_prepare_v2(keen_dbname, delete_all_sql, -1, &delete_all_stmt, NULL) != SQLITE_OK) {
+                [self closeDB];
+            }
+
+            // This statement deletes old events at a given offset.
+            char *age_out_sql = "DELETE FROM events WHERE id <= (SELECT id FROM events ORDER BY id DESC LIMIT 1 OFFSET ?)";
+            if(sqlite3_prepare_v2(keen_dbname, age_out_sql, -1, &age_out_stmt, NULL) != SQLITE_OK) {
+                [self closeDB];
+            }
         }
     }
     return self;
 }
 
-- (BOOL)addEvent:(NSString *)eventData {
+- (BOOL)addEvent:(NSData *)eventData collection: (NSString *)coll {
     BOOL wasAdded = NO;
 
     if (!dbIsOpen) {
@@ -113,7 +127,12 @@ - (BOOL)addEvent:(NSString *)eventData {
         [self closeDB];
     }
 
-    if (sqlite3_bind_blob(insert_stmt, 2, [eventData UTF8String], -1, SQLITE_STATIC) != SQLITE_OK) {
+    if (sqlite3_bind_text(insert_stmt, 2, [coll UTF8String], -1, SQLITE_STATIC) != SQLITE_OK) {
+        [self handleSQLiteFailure:@"bind coll to add event statement"];
+        [self closeDB];
+    }
+
+    if (sqlite3_bind_blob(insert_stmt, 3, [eventData bytes], (int) [eventData length], SQLITE_TRANSIENT) != SQLITE_OK) {
         [self handleSQLiteFailure:@"bind insert statement"];
         [self closeDB];
     }
@@ -133,10 +152,10 @@ - (BOOL)addEvent:(NSString *)eventData {
     return wasAdded;
 }
 
-- (NSMutableArray *)getEvents{
+- (NSMutableDictionary *)getEvents{
 
-    // Create an array to hold the contents of our select.
-    NSMutableArray *events = [NSMutableArray array];
+    // Create a dictionary to hold the contents of our select.
+    NSMutableDictionary *events = [NSMutableDictionary dictionary];
 
     if (!dbIsOpen) {
         KCLog(@"DB is closed, skipping getEvents");
@@ -148,16 +167,19 @@ - (NSMutableArray *)getEvents{
         [self handleSQLiteFailure:@"bind pid to find statement"];
     }
 
-    // This statement has no bindings, so can just step it immediately.
     while (sqlite3_step(find_stmt) == SQLITE_ROW) {
         // Fetch data out the statement
-        int eventId = sqlite3_column_int(find_stmt, 0);
-        const void *dataPtr = sqlite3_column_blob(find_stmt, 1);
-        int dataSize = sqlite3_column_bytes(find_stmt, 1);
+        long long eventId = sqlite3_column_int64(find_stmt, 0);
+
+        NSString *coll = [NSString stringWithUTF8String:(char *)sqlite3_column_text(find_stmt, 1)];
+
+        const void *dataPtr = sqlite3_column_blob(find_stmt, 2);
+        int dataSize = sqlite3_column_bytes(find_stmt, 2);
+
+        NSData *data = [[NSData alloc] initWithBytes:dataPtr length:dataSize];
 
         // Bind and mark the event pending.
-        if(sqlite3_bind_int(make_pending_stmt, 1, eventId) != SQLITE_OK) {
-            // XXX What to do here?
+        if(sqlite3_bind_int64(make_pending_stmt, 1, eventId) != SQLITE_OK) {
             [self handleSQLiteFailure:@"bind int for make pending"];
         }
         if (sqlite3_step(make_pending_stmt) != SQLITE_DONE) {
@@ -168,10 +190,13 @@ - (NSMutableArray *)getEvents{
         sqlite3_reset(make_pending_stmt);
         sqlite3_clear_bindings(make_pending_stmt);
 
-        // Add the event to the array.
-        // XXX What frees this?
-        NSData *data = [[[NSData alloc] initWithBytes:dataPtr length:dataSize] autorelease];
-        [events addObject:data];
+        if ([events objectForKey:coll] == nil) {
+            // We don't have an entry in the dictionary yet for this collection
+            // so create one.
+            [events setObject:[NSMutableDictionary dictionary] forKey:coll];
+        }
+
+        [[events objectForKey:coll] setObject:data forKey:[NSNumber numberWithUnsignedLongLong:eventId]];
     }
 
     // Reset things
@@ -254,6 +279,52 @@ - (NSUInteger)getTotalEventCount {
     return eventCount;
 }
 
+- (void)deleteEvent: (NSNumber *)eventId {
+
+    if (!dbIsOpen) {
+        KCLog(@"DB is closed, skipping deleteEvent");
+        return;
+    }
+    if (sqlite3_bind_int64(delete_stmt, 1, [eventId unsignedLongLongValue]) != SQLITE_OK) {
+        [self handleSQLiteFailure:@"bind eventid to delete statement"];
+    }
+    if (sqlite3_step(delete_stmt) != SQLITE_DONE) {
+        [self handleSQLiteFailure:@"delete event"];
+    };
+    sqlite3_reset(delete_stmt);
+    sqlite3_clear_bindings(delete_stmt);
+}
+
+- (void)deleteAllEvents {
+
+    if (!dbIsOpen) {
+        KCLog(@"DB is closed, skipping deleteEvent");
+        return;
+    }
+    if (sqlite3_step(delete_all_stmt) != SQLITE_DONE) {
+        [self handleSQLiteFailure:@"delete all events"];
+    };
+    sqlite3_reset(delete_all_stmt);
+    sqlite3_clear_bindings(delete_all_stmt);
+}
+
+- (void)deleteEventsFromOffset: (NSNumber *)offset {
+
+    if (!dbIsOpen) {
+        KCLog(@"DB is closed, skipping deleteEvent");
+        return;
+    }
+    if (sqlite3_bind_int64(age_out_stmt, 1, [offset unsignedLongLongValue]) != SQLITE_OK) {
+        [self handleSQLiteFailure:@"bind offset to ageOut statement"];
+    }
+    if (sqlite3_step(age_out_stmt) != SQLITE_DONE) {
+        [self handleSQLiteFailure:@"delete all events"];
+    };
+    sqlite3_reset(age_out_stmt);
+    sqlite3_clear_bindings(age_out_stmt);
+}
+
+
 - (void)purgePendingEvents {
 
     if (!dbIsOpen) {
@@ -294,7 +365,7 @@ - (BOOL)createTable {
     }
 
     char *err;
-    NSString *sql = [NSString stringWithFormat:@"CREATE TABLE IF NOT EXISTS 'events' (ID INTEGER PRIMARY KEY AUTOINCREMENT, projectId TEXT, eventData BLOB, pending INTEGER);"];
+    NSString *sql = [NSString stringWithFormat:@"CREATE TABLE IF NOT EXISTS 'events' (ID INTEGER PRIMARY KEY AUTOINCREMENT, collection TEXT, projectId TEXT, eventData BLOB, pending INTEGER, dateCreated TIMESTAMP DEFAULT CURRENT_TIMESTAMP);"];
     if (sqlite3_exec(keen_dbname, [sql UTF8String], NULL, NULL, &err) != SQLITE_OK) {
         KCLog(@"Failed to create table: %@", [NSString stringWithCString:err encoding:NSUTF8StringEncoding]);
         sqlite3_free(err); // Free that error message
@@ -320,6 +391,9 @@ - (void)closeDB {
     sqlite3_finalize(make_pending_stmt);
     sqlite3_finalize(reset_pending_stmt);
     sqlite3_finalize(purge_stmt);
+    sqlite3_finalize(delete_stmt);
+    sqlite3_finalize(delete_all_stmt);
+    sqlite3_finalize(age_out_stmt);
 
     // Free our DB. This is safe on null pointers.
     sqlite3_close(keen_dbname);

KeenClient/KeenClient.h

@@ -8,6 +8,7 @@
 
 #import <Foundation/Foundation.h>
 #import <CoreLocation/CoreLocation.h>
+#import "KIOEventStore.h"
 #import "KeenProperties.h"
 
 // defines a type for the block we'll use with our global properties
@@ -150,6 +151,14 @@ typedef NSDictionary* (^KeenGlobalPropertiesBlock)(NSString *eventCollection);
  */
 + (Boolean)isLoggingEnabled;
 
+
+/**
+ Call this to indiscriminately delete all events queued for sending.
+ */
++ (void)clearAllEvents;
+
++ (KIOEventStore *)getEventStore;
+
 /**
  Call this if your code needs to use more than one Keen project.  By convention, if you
  call this, you're responsible for releasing the returned instance once you're finished with it.
@@ -228,6 +237,11 @@ typedef NSDictionary* (^KeenGlobalPropertiesBlock)(NSString *eventCollection);
  */
 - (void)refreshCurrentLocation;
 
+/**
+ * Import fs-based data into the SQLite database.
+ */
+- (void)importFileData;
+
 // defines the KCLog macro
 #define KEEN_LOGGING_ENABLED [[KeenClient sharedClient] loggingEnabled]
 #define KCLog(message, ...)if([KeenClient isLoggingEnabled]) NSLog(message, ##__VA_ARGS__)