fix: add limit of events data length when query from db (#304)

* fix: add limit of events data length when query from db

* style: code format

* fix: limit event data size logic

* style: code format

* fix: crash because events.count is 0 when set ASAEnabled true

* style: code format

* fix(database): count of events logic

* fix: fail to build raw events

* style: code format

* fix: limit event data size logic

* Revert "fix(database): count of events logic"

This reverts commit 20b8853.

* fix(database): count of events logic

* fix: vacuum 操作改为 3 天/次

* feat(compress): separate encryptEnabled into encryptEnabled and compressEnabled, compressEnabled is true in default

* style: code format

* fix(compress): compressEnabled is false in default

* fix: 在 release 环境下，将不再由于 debugEnabled 为 true 而频繁发送事件

* style: code format

---------

Co-authored-by: GIOSDK <sdk-integration@growingio.com>

Example/GrowingAnalyticsTests/GrowingAnalyticsStartTests/GrowingAnalyticsStartTests.m

@@ -125,6 +125,7 @@ - (void)testDefaultConfiguration_Autotracker {
     XCTAssertEqual(configuration.idMappingEnabled, NO);
     XCTAssertEqualObjects(configuration.urlScheme, nil);
     XCTAssertEqual(configuration.encryptEnabled, NO);
+    XCTAssertEqual(configuration.compressEnabled, NO);
     XCTAssertEqual(configuration.impressionScale, 0);
     XCTAssertEqualObjects(configuration.dataSourceId, nil);
     XCTAssertEqualObjects(configuration.networkConfig, nil);
@@ -144,6 +145,7 @@ - (void)testSetConfiguration_Autotracker {
     config.idMappingEnabled = YES;
     config.urlScheme = @"growing.autotracker";
     config.encryptEnabled = YES;
+    config.compressEnabled = YES;
     config.impressionScale = 0.5;
     config.dataSourceId = @"12345";
     GrowingNetworkConfig *networkConfig = [GrowingNetworkConfig config];
@@ -165,6 +167,7 @@ - (void)testSetConfiguration_Autotracker {
     XCTAssertEqual(configuration.idMappingEnabled, YES);
     XCTAssertEqualObjects(configuration.urlScheme, @"growing.autotracker");
     XCTAssertEqual(configuration.encryptEnabled, YES);
+    XCTAssertEqual(configuration.compressEnabled, YES);
     XCTAssertEqual(configuration.impressionScale, 0.5);
     XCTAssertEqualObjects(configuration.dataSourceId, @"12345");
     XCTAssertNotNil(configuration.networkConfig);
@@ -188,6 +191,7 @@ - (void)testDefaultConfiguration_Tracker {
     XCTAssertEqual(configuration.idMappingEnabled, NO);
     XCTAssertEqualObjects(configuration.urlScheme, nil);
     XCTAssertEqual(configuration.encryptEnabled, NO);
+    XCTAssertEqual(configuration.compressEnabled, NO);
     XCTAssertEqualObjects(configuration.dataSourceId, nil);
     XCTAssertEqualObjects(configuration.networkConfig, nil);
 }
@@ -206,6 +210,7 @@ - (void)testSetConfiguration_Tracker {
     config.idMappingEnabled = YES;
     config.urlScheme = @"growing.tracker";
     config.encryptEnabled = YES;
+    config.compressEnabled = YES;
     config.dataSourceId = @"12345";
     GrowingNetworkConfig *networkConfig = [GrowingNetworkConfig config];
     networkConfig.requestTimeout = 0.3f;
@@ -225,6 +230,7 @@ - (void)testSetConfiguration_Tracker {
     XCTAssertEqual(configuration.idMappingEnabled, YES);
     XCTAssertEqualObjects(configuration.urlScheme, @"growing.tracker");
     XCTAssertEqual(configuration.encryptEnabled, YES);
+    XCTAssertEqual(configuration.compressEnabled, YES);
     XCTAssertEqualObjects(configuration.dataSourceId, @"12345");
     XCTAssertNotNil(configuration.networkConfig);
     XCTAssertEqual(configuration.networkConfig.requestTimeout, 0.3f);

Example/GrowingAnalyticsTests/GrowingAnalyticsStartTests/GrowingAnalyticsStartTests2.m

@@ -61,6 +61,7 @@ - (void)testDefaultConfiguration_Autotracker {
     XCTAssertEqual(configuration.idMappingEnabled, NO);
     XCTAssertEqualObjects(configuration.urlScheme, nil);
     XCTAssertEqual(configuration.encryptEnabled, NO);
+    XCTAssertEqual(configuration.compressEnabled, NO);
     XCTAssertEqual(configuration.impressionScale, 0);
     XCTAssertEqualObjects(configuration.dataSourceId, nil);
 }
@@ -79,6 +80,7 @@ - (void)testSetConfiguration_Autotracker {
     config.idMappingEnabled = YES;
     config.urlScheme = @"growing.autotracker";
     config.encryptEnabled = YES;
+    config.compressEnabled = YES;
     config.impressionScale = 0.5;
     config.dataSourceId = @"12345";
     [GrowingRealAutotracker trackerWithConfiguration:config launchOptions:nil];
@@ -97,6 +99,7 @@ - (void)testSetConfiguration_Autotracker {
     XCTAssertEqual(configuration.idMappingEnabled, YES);
     XCTAssertEqualObjects(configuration.urlScheme, @"growing.autotracker");
     XCTAssertEqual(configuration.encryptEnabled, YES);
+    XCTAssertEqual(configuration.compressEnabled, YES);
     XCTAssertEqual(configuration.impressionScale, 0.5);
     XCTAssertEqualObjects(configuration.dataSourceId, @"12345");
 }
@@ -118,6 +121,7 @@ - (void)testDefaultConfiguration_Tracker {
     XCTAssertEqual(configuration.idMappingEnabled, NO);
     XCTAssertEqualObjects(configuration.urlScheme, nil);
     XCTAssertEqual(configuration.encryptEnabled, NO);
+    XCTAssertEqual(configuration.compressEnabled, NO);
     XCTAssertEqualObjects(configuration.dataSourceId, nil);
 }
 
@@ -135,6 +139,7 @@ - (void)testSetConfiguration_Tracker {
     config.idMappingEnabled = YES;
     config.urlScheme = @"growing.tracker";
     config.encryptEnabled = YES;
+    config.compressEnabled = YES;
     config.dataSourceId = @"12345";
     [GrowingRealTracker trackerWithConfiguration:config launchOptions:nil];
 
@@ -151,6 +156,7 @@ - (void)testSetConfiguration_Tracker {
     XCTAssertEqual(configuration.idMappingEnabled, YES);
     XCTAssertEqualObjects(configuration.urlScheme, @"growing.tracker");
     XCTAssertEqual(configuration.encryptEnabled, YES);
+    XCTAssertEqual(configuration.compressEnabled, YES);
     XCTAssertEqualObjects(configuration.dataSourceId, @"12345");
 }
 

Example/GrowingAnalyticsTests/ModulesTests/ProtobufTests/ProtobufDatabaseTest.m

@@ -95,7 +95,7 @@ - (void)testDatabaseInstanceMethods {
     // insert events
     XCTAssertEqual([database insertEvents:events], YES);
 
-    NSArray *array2 = [database getEventsByCount:insertCount policy:GrowingEventSendPolicyInstant];
+    NSArray *array2 = [database getEventsByCount:insertCount limitSize:1000000 policy:GrowingEventSendPolicyInstant];
     XCTAssertEqual(array2.count, insertCount);
 
     // delete events
@@ -157,6 +157,7 @@ - (void)testDatabaseEventIO {
     NSInteger insertCount = 1;
     NSArray *array = [database
         getEventsByCount:5
+                      limitSize:1000000
                   policy:GrowingEventSendPolicyInstant | GrowingEventSendPolicyMobileData |
                          GrowingEventSendPolicyWiFi];  // 避免多线程情况下，刚好还有其他事件产生入库，这里数值设定大一点
     XCTAssertGreaterThanOrEqual(array.count, insertCount);

Example/GrowingAnalyticsTests/ServicesTests/DatabaseTests/JSONDatabaseTest.m

@@ -90,7 +90,7 @@ - (void)testDatabaseInstanceMethods {
     // insert events
     XCTAssertEqual([database insertEvents:events], YES);
 
-    NSArray *array2 = [database getEventsByCount:insertCount policy:GrowingEventSendPolicyInstant];
+    NSArray *array2 = [database getEventsByCount:insertCount limitSize:1000000 policy:GrowingEventSendPolicyInstant];
     XCTAssertEqual(array2.count, insertCount);
 
     // delete events
@@ -151,6 +151,7 @@ - (void)testDatabaseEventIO {
     NSInteger insertCount = 1;
     NSArray *array = [database
         getEventsByCount:5
+                      limitSize:1000000
                   policy:GrowingEventSendPolicyInstant | GrowingEventSendPolicyMobileData |
                          GrowingEventSendPolicyWiFi];  // 避免多线程情况下，刚好还有其他事件产生入库，这里数值设定大一点
     XCTAssertGreaterThanOrEqual(array.count, insertCount);

Example/GrowingAnalyticsTests/TrackerCoreTests/DatabaseTests/DatabaseTest.m

@@ -82,7 +82,7 @@ - (void)testDatabase {
     {
         XCTAssertTrue([self.database clearAllItems]);
         XCTAssertEqual(self.database.countOfEvents, 0);
-        NSArray *events2 = [self.database getEventsByCount:1 policy:GrowingEventSendPolicyInstant];
+        NSArray *events2 = [self.database getEventsByCount:1 limitSize:1000000 policy:GrowingEventSendPolicyInstant];
         XCTAssertEqual(events2.count, 0);
     }
 
@@ -96,7 +96,7 @@ - (void)testDatabase {
                                                    sdkVersion:self.event.sdkVersion];
         XCTAssertNoThrow([self.database setEvent:event forKey:uuid]);
         XCTAssertEqual(self.database.countOfEvents, 1);
-        NSArray *events = [self.database getEventsByCount:1 policy:GrowingEventSendPolicyInstant];
+        NSArray *events = [self.database getEventsByCount:1 limitSize:1000000 policy:GrowingEventSendPolicyInstant];
         XCTAssertEqual(events.count, 1);
     }
 

Example/Podfile.lock

@@ -71,16 +71,16 @@ PODS:
     - GrowingAPM/Core
   - GrowingAPM/UIMonitor (1.0.1):
     - GrowingAPM/Core
-  - GrowingToolsKit (2.0.1):
-    - GrowingToolsKit/Default (= 2.0.1)
-  - GrowingToolsKit/APMCore (2.0.1):
+  - GrowingToolsKit (2.0.2):
+    - GrowingToolsKit/Default (= 2.0.2)
+  - GrowingToolsKit/APMCore (2.0.2):
     - GrowingAPM/Core
     - GrowingToolsKit/Core
-  - GrowingToolsKit/Core (2.0.1)
-  - GrowingToolsKit/CrashMonitor (2.0.1):
+  - GrowingToolsKit/Core (2.0.2)
+  - GrowingToolsKit/CrashMonitor (2.0.2):
     - GrowingAPM/CrashMonitor
     - GrowingToolsKit/APMCore
-  - GrowingToolsKit/Default (2.0.1):
+  - GrowingToolsKit/Default (2.0.2):
     - GrowingToolsKit/Core
     - GrowingToolsKit/CrashMonitor
     - GrowingToolsKit/EventsList
@@ -90,20 +90,20 @@ PODS:
     - GrowingToolsKit/SDKInfo
     - GrowingToolsKit/Settings
     - GrowingToolsKit/XPathTrack
-  - GrowingToolsKit/EventsList (2.0.1):
+  - GrowingToolsKit/EventsList (2.0.2):
     - GrowingToolsKit/Core
-  - GrowingToolsKit/LaunchTime (2.0.1):
+  - GrowingToolsKit/LaunchTime (2.0.2):
     - GrowingAPM/UIMonitor
     - GrowingToolsKit/APMCore
-  - GrowingToolsKit/NetFlow (2.0.1):
+  - GrowingToolsKit/NetFlow (2.0.2):
     - GrowingToolsKit/Core
-  - GrowingToolsKit/Realtime (2.0.1):
+  - GrowingToolsKit/Realtime (2.0.2):
     - GrowingToolsKit/Core
-  - GrowingToolsKit/SDKInfo (2.0.1):
+  - GrowingToolsKit/SDKInfo (2.0.2):
     - GrowingToolsKit/Core
-  - GrowingToolsKit/Settings (2.0.1):
+  - GrowingToolsKit/Settings (2.0.2):
     - GrowingToolsKit/Core
-  - GrowingToolsKit/XPathTrack (2.0.1):
+  - GrowingToolsKit/XPathTrack (2.0.2):
     - GrowingToolsKit/Core
   - GrowingUtils/AutotrackerCore (1.2.3):
     - GrowingUtils/TrackerCore
@@ -124,12 +124,12 @@ PODS:
   - OHHTTPStubs/NSURLSession (9.1.0):
     - OHHTTPStubs/Core
   - OHHTTPStubs/OHPathHelpers (9.1.0)
-  - Protobuf (3.26.0)
+  - Protobuf (3.26.1)
   - SDCycleScrollView (1.82):
     - SDWebImage (>= 5.0.0)
-  - SDWebImage (5.19.0):
-    - SDWebImage/Core (= 5.19.0)
-  - SDWebImage/Core (5.19.0)
+  - SDWebImage (5.19.1):
+    - SDWebImage/Core (= 5.19.1)
+  - SDWebImage/Core (5.19.1)
 
 DEPENDENCIES:
   - GrowingAnalytics/ABTesting (from `../`)
@@ -160,15 +160,15 @@ EXTERNAL SOURCES:
     :path: "../"
 
 SPEC CHECKSUMS:
-  GrowingAnalytics: c24033146e037d32387883bf57f0be44f42add13
+  GrowingAnalytics: 8dd586c3ab1aa9ecbb9a628b4325aef869679b26
   GrowingAPM: 3c4de0384935b654e6798b95606f47883a99418b
-  GrowingToolsKit: c1f7753484ca3e18dedb2fe083317fb19c1fecb4
+  GrowingToolsKit: 53160d19690da0b78e04a9242abde7af86442922
   GrowingUtils: 68aee2c96849bf9b674740503da30c2da468e79d
   KIF: 7660c626b0f2d4562533590960db70a36d640558
   OHHTTPStubs: 90eac6d8f2c18317baeca36698523dc67c513831
-  Protobuf: 5685c66a07eaad9d18ce5ab618e9ac01fd04b5aa
+  Protobuf: a53f5173a603075b3522a5c50be63a67a5f3353a
   SDCycleScrollView: a0d74c3384caa72bdfc81470bdbc8c14b3e1fbcf
-  SDWebImage: 981fd7e860af070920f249fd092420006014c3eb
+  SDWebImage: 40b0b4053e36c660a764958bff99eed16610acbb
 
 PODFILE CHECKSUM: 51bab161e69d216b5eff74379bef32902e83107b
 

GrowingTrackerCore/Database/GrowingEventDatabase.h

@@ -34,7 +34,9 @@
 
 - (void)setEvent:(id<GrowingEventPersistenceProtocol>)event forKey:(NSString *)key;
 
-- (NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count policy:(NSUInteger)mask;
+- (NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count
+                                                         limitSize:(NSUInteger)limitSize
+                                                            policy:(NSUInteger)mask;
 
 - (NSData *)buildRawEventsFromEvents:(NSArray<id<GrowingEventPersistenceProtocol>> *)events;
 

GrowingTrackerCore/Database/GrowingEventDatabase.m

@@ -183,8 +183,10 @@ - (void)setEvent:(id<GrowingEventPersistenceProtocol>)event forKey:(NSString *)k
     }
 }
 
-- (NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count policy:(NSUInteger)mask {
-    NSArray *events = [self.db getEventsByCount:count policy:mask];
+- (NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count
+                                                         limitSize:(NSUInteger)limitSize
+                                                            policy:(NSUInteger)mask {
+    NSArray *events = [self.db getEventsByCount:count limitSize:limitSize policy:mask];
     if (!events) {
         [self handleDatabaseError:[self.db lastError]];
     }

GrowingTrackerCore/Event/GrowingEventManager.m

@@ -299,17 +299,15 @@ - (void)sendEventsOfChannel_unsafe:(GrowingEventChannel *)channel {
     }
     events = removeV3AutotrackEvents.copy;
 
-    if (events.count == 0) {
-        return;
-    }
-
     for (NSObject<GrowingEventInterceptor> *obj in self.allInterceptor) {
         if ([obj respondsToSelector:@selector(growingEventManagerEventsWillSend:channel:)]) {
             events = [obj growingEventManagerEventsWillSend:events channel:channel];
         }
     }
 
-    channel.isUploading = YES;
+    if (events.count == 0) {
+        return;
+    }
 
 #ifdef DEBUG
     [self prettyLogForEvents:events withChannel:channel];
@@ -341,20 +339,25 @@ - (void)sendEventsOfChannel_unsafe:(GrowingEventChannel *)channel {
         }
 
         rawEvents = [dbClass buildRawEventsFromJsonObjects:jsonObjects];
+    } else {
+        // 该channel的持久化数据格式与配置相同
+        rawEvents = [channel.db buildRawEventsFromEvents:events];
     }
 
     if (!rawEvents) {
-        // 该channel的持久化数据格式与配置相同
-        rawEvents = [channel.db buildRawEventsFromEvents:events];
+        GIOLogError(@"-sendEventsOfChannel_unsafe: error : fail to build raw events");
+        return;
     }
 
-    NSObject<GrowingRequestProtocol> *eventRequest = [[GrowingEventRequest alloc] initWithEvents:rawEvents];
     id<GrowingEventNetworkService> service =
         [[GrowingServiceManager sharedInstance] createService:@protocol(GrowingEventNetworkService)];
     if (!service) {
         GIOLogError(@"-sendEventsOfChannel_unsafe: error : no network service support");
         return;
     }
+
+    channel.isUploading = YES;
+    NSObject<GrowingRequestProtocol> *eventRequest = [[GrowingEventRequest alloc] initWithEvents:rawEvents];
     [service sendRequest:eventRequest
               completion:^(NSHTTPURLResponse *_Nonnull httpResponse, NSData *_Nonnull data, NSError *_Nonnull error) {
                   if (error) {
@@ -411,9 +414,15 @@ - (void)writeToDatabaseWithEvent:(GrowingBaseEvent *)event {
                                                                                                uuid:uuidString];
     [eventChannel.db setEvent:waitForPersist forKey:uuidString];
 
-    BOOL debugEnabled = GrowingConfigurationManager.sharedInstance.trackConfiguration.debugEnabled;
-    if (GrowingEventSendPolicyInstant & event.sendPolicy || debugEnabled) {  // send event instantly
+    if (GrowingEventSendPolicyInstant & event.sendPolicy) {
         [self sendEventsInstantWithChannel:eventChannel];
+#if defined(DEBUG) && DEBUG
+    } else {
+        BOOL debugEnabled = GrowingConfigurationManager.sharedInstance.trackConfiguration.debugEnabled;
+        if (debugEnabled) {
+            [self sendEventsInstantWithChannel:eventChannel];
+        }
+#endif
     }
 }
 
@@ -431,7 +440,7 @@ - (void)removeEvents_unsafe:(NSArray<__kindof id<GrowingEventPersistenceProtocol
 
 - (NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsToBeUploadUnsafe:(GrowingEventChannel *)channel
                                                                      policy:(NSUInteger)mask {
-    return [channel.db getEventsByCount:kGrowingMaxBatchSize policy:mask];
+    return [channel.db getEventsByCount:kGrowingMaxBatchSize limitSize:2 * kGrowingUnit_MB policy:mask];
 }
 
 #pragma mark Event Log

GrowingTrackerCore/GrowingTrackConfiguration.m

@@ -61,6 +61,7 @@ - (instancetype)initWithAccountId:(NSString *)accountId {
         _idMappingEnabled = NO;
         _urlScheme = nil;
         _encryptEnabled = NO;
+        _compressEnabled = NO;
         _networkConfig = nil;
         _useProtobuf = YES;
 
@@ -105,6 +106,7 @@ - (id)copyWithZone:(NSZone *)zone {
     configuration->_idMappingEnabled = _idMappingEnabled;
     configuration->_urlScheme = _urlScheme;
     configuration->_encryptEnabled = _encryptEnabled;
+    configuration->_compressEnabled = _compressEnabled;
     configuration->_networkConfig = [_networkConfig copy];
     configuration->_useProtobuf = _useProtobuf;
 

GrowingTrackerCore/Public/GrowingEventDatabaseService.h

@@ -67,9 +67,12 @@ extern NSString *const GrowingEventDatabaseErrorDomain;
 
 /// 获取事件
 /// @param count 数量
+/// @param limitSize 大小限制
 /// @param mask 允许的发送协议（数组）
 /// @return 事件对象数组，可为空；若返回值为nil，表示读取错误
-- (nullable NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count policy:(NSUInteger)mask;
+- (nullable NSArray<id<GrowingEventPersistenceProtocol>> *)getEventsByCount:(NSUInteger)count
+                                                                  limitSize:(NSUInteger)limitSize
+                                                                     policy:(NSUInteger)mask;
 
 /// 写入事件数据
 /// @param event 事件数据

GrowingTrackerCore/Public/GrowingTrackConfiguration.h

@@ -40,6 +40,7 @@ FOUNDATION_EXPORT NSString *const kGrowingDefaultDataCollectionServerHost;
 @property (nonatomic, assign) BOOL idMappingEnabled;
 @property (nonatomic, copy) NSString *urlScheme;
 @property (nonatomic, assign) BOOL encryptEnabled;
+@property (nonatomic, assign) BOOL compressEnabled;
 @property (nonatomic, copy) GrowingNetworkConfig *networkConfig;
 @property (nonatomic, assign) BOOL useProtobuf;