From c26aff31c706f4fa296dd83ae1f6c6b21ce49bab Mon Sep 17 00:00:00 2001 From: Pete Markowsky Date: Mon, 20 Sep 2021 17:49:29 -0400 Subject: [PATCH 1/2] Initial commit of santametricservice. The santametricservice is an XPC helper service to write metrics. It consists of Formatters and Writers. This initial commit only has support for the rawJSON format and writing to a file. This is a new daemon to be included. Docs and packaging will be updated in future PRs. --- Source/santametricservice/BUILD | 54 ++++ Source/santametricservice/Formats/BUILD | 45 ++++ .../Formats/SNTMetricFormat.h | 19 ++ .../Formats/SNTMetricRawJsonFormat.h | 21 ++ .../Formats/SNTMetricRawJsonFormat.m | 98 ++++++++ .../Formats/SNTMetricRawJsonFormatTest.m | 155 ++++++++++++ .../Formats/testdata/json/test.json | 111 +++++++++ Source/santametricservice/Info.plist | 26 ++ Source/santametricservice/SNTMetricService.h | 20 ++ Source/santametricservice/SNTMetricService.m | 121 +++++++++ .../santametricservice/SNTMetricServiceTest.m | 232 ++++++++++++++++++ Source/santametricservice/Writers/BUILD | 35 +++ .../Writers/SNTMetricFileWriter.h | 17 ++ .../Writers/SNTMetricFileWriter.m | 77 ++++++ .../Writers/SNTMetricFileWriterTest.m | 101 ++++++++ .../Writers/SNTMetricWriter.h | 23 ++ Source/santametricservice/main.m | 35 +++ docs/deployment/configuration.md | 2 +- 18 files changed, 1191 insertions(+), 1 deletion(-) create mode 100644 Source/santametricservice/BUILD create mode 100644 Source/santametricservice/Formats/BUILD create mode 100644 Source/santametricservice/Formats/SNTMetricFormat.h create mode 100644 Source/santametricservice/Formats/SNTMetricRawJsonFormat.h create mode 100644 Source/santametricservice/Formats/SNTMetricRawJsonFormat.m create mode 100644 Source/santametricservice/Formats/SNTMetricRawJsonFormatTest.m create mode 100644 Source/santametricservice/Formats/testdata/json/test.json create mode 100644 Source/santametricservice/Info.plist create mode 100644 Source/santametricservice/SNTMetricService.h create mode 100644 Source/santametricservice/SNTMetricService.m create mode 100644 Source/santametricservice/SNTMetricServiceTest.m create mode 100644 Source/santametricservice/Writers/BUILD create mode 100644 Source/santametricservice/Writers/SNTMetricFileWriter.h create mode 100644 Source/santametricservice/Writers/SNTMetricFileWriter.m create mode 100644 Source/santametricservice/Writers/SNTMetricFileWriterTest.m create mode 100644 Source/santametricservice/Writers/SNTMetricWriter.h create mode 100644 Source/santametricservice/main.m diff --git a/Source/santametricservice/BUILD b/Source/santametricservice/BUILD new file mode 100644 index 000000000..991d1ff37 --- /dev/null +++ b/Source/santametricservice/BUILD @@ -0,0 +1,54 @@ +load("@build_bazel_rules_apple//apple:macos.bzl", "macos_command_line_application") +load("//:helper.bzl", "santa_unit_test") + +package(default_visibility = ["//:santa_package_group"]) + +licenses(["notice"]) # Apache 2.0 + +objc_library( + name = "SNTMetricServiceLib", + srcs = [ + "SNTMetricService.h", + "SNTMetricService.m", + "main.m", + ], + deps = [ + "//Source/common:SNTXPCMetricServiceInterface", + "//Source/common:SNTLogging", + "//Source/common:SNTConfigurator", + "//Source/common:SNTMetricSet", + "//Source/santametricservice/Formats:SNTMetricRawJsonFormat", + "//Source/santametricservice/Writers:SNTMetricFileWriter", + "@MOLCodesignChecker", + "@MOLXPCConnection", + ], +) + +santa_unit_test( + name="SNTMetricServiceTest", + srcs = ["SNTMetricServiceTest.m"], + deps = [ + ":SNTMetricServiceLib", + "@OCMock", + ], +) + +test_suite( + name="unit_tests", + tests = [ + ":SNTMetricServiceTest", + "//Source/santametricservice/Formats:SNTMetricRawJsonFormatTest", + "//Source/santametricservice/Writers:SNTMetricFileWriterTest", + ], +) + + +macos_command_line_application( + name = "santametricservice", + bundle_id = "com.google.santa.metricservice", + infoplists = ["Info.plist"], + minimum_os_version = "10.15", + version = "//:version", + visibility = ["//:santa_package_group"], + deps = [":SNTMetricServiceLib"], +) diff --git a/Source/santametricservice/Formats/BUILD b/Source/santametricservice/Formats/BUILD new file mode 100644 index 000000000..b24f90ccf --- /dev/null +++ b/Source/santametricservice/Formats/BUILD @@ -0,0 +1,45 @@ +load("@build_bazel_rules_apple//apple:macos.bzl", "macos_command_line_application") +load("//:helper.bzl", "santa_unit_test") + +package(default_visibility = ["//:santa_package_group"]) + +licenses(["notice"]) # Apache 2.0 + +objc_library( + name = "SNTMetricFormat", + hdrs = ["SNTMetricFormat.h"], +) + +objc_library( + name = "SNTMetricRawJsonFormat", + srcs = [ + "SNTMetricRawJsonFormat.h", + "SNTMetricRawJsonFormat.m", + "SNTMetricFormat.h", + + ], + deps = [ + ":SNTMetricFormat", + "//Source/common:SNTLogging", + ], +) + +santa_unit_test( + name = "SNTMetricRawJsonFormatTest", + srcs = [ + "SNTMetricRawJsonFormatTest.m", + ], + structured_resources = glob(["testdata/**"]), + deps = [ + ":SNTMetricRawJsonFormat", + "//Source/common:SNTMetricSet", + ], +) + + +test_suite( + name = "format_tests", + tests = [ + ":SNTMetricRawJsonFormatTest", + ], +) diff --git a/Source/santametricservice/Formats/SNTMetricFormat.h b/Source/santametricservice/Formats/SNTMetricFormat.h new file mode 100644 index 000000000..9ce6a06b7 --- /dev/null +++ b/Source/santametricservice/Formats/SNTMetricFormat.h @@ -0,0 +1,19 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import + +@protocol SNTMetricFormat +- (NSArray *)convert:(NSDictionary *)metrics error:(NSError **)err; +@end diff --git a/Source/santametricservice/Formats/SNTMetricRawJsonFormat.h b/Source/santametricservice/Formats/SNTMetricRawJsonFormat.h new file mode 100644 index 000000000..913662ca5 --- /dev/null +++ b/Source/santametricservice/Formats/SNTMetricRawJsonFormat.h @@ -0,0 +1,21 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import + +#import "Source/santametricservice/Formats/SNTMetricFormat.h" + +@interface SNTMetricRawJsonFormat : NSObject +- (NSArray *) convert:(NSDictionary *)metrics error:(NSError **)err; +@end diff --git a/Source/santametricservice/Formats/SNTMetricRawJsonFormat.m b/Source/santametricservice/Formats/SNTMetricRawJsonFormat.m new file mode 100644 index 000000000..528bdbee6 --- /dev/null +++ b/Source/santametricservice/Formats/SNTMetricRawJsonFormat.m @@ -0,0 +1,98 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +#import "Source/common/SNTLogging.h" + +#import "Source/santametricservice/Formats/SNTMetricRawJsonFormat.h" + +@implementation SNTMetricRawJsonFormat { + NSDateFormatter *dateFormatter; +} + +- (instancetype)init +{ + self = [super init]; + if (self) { + dateFormatter = [[NSDateFormatter alloc] init]; + [dateFormatter setDateFormat:@"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"]; + } + return self; +} + +- (NSArray *)normalizeArray:(NSArray *)arr +{ + NSMutableArray *normalized = [NSMutableArray arrayWithArray: arr]; + + for (int i = 0; i < [arr count];i++) { + if ([arr[i] isKindOfClass: [NSArray class]]) { + normalized[i] = [self normalizeArray: (NSArray *)arr[i]]; + } else if ([arr[i] isKindOfClass: [NSDictionary class]]) { + normalized[i] = [self normalize: (NSDictionary *)arr[i]]; + } + } + + return normalized; +} + +/** + * Normalizes the metrics dictionary for exporting to JSON + **/ +- (NSDictionary *)normalize: (NSDictionary *)metrics +{ + // Convert NSDate's to RFC3339 in strings as NSDate's cannot be serialized + // to JSON. + NSMutableDictionary *normalizedMetrics = [NSMutableDictionary dictionaryWithDictionary:metrics]; + + for (NSString *key in metrics) { + const id object = [metrics objectForKey: key]; + if ([object isKindOfClass: [NSDate class]]) { + normalizedMetrics[key] = [self->dateFormatter stringFromDate: (NSDate *)object]; + } else if ([object isKindOfClass: [NSDictionary class]]) { + normalizedMetrics[key] = [self normalize: metrics[key]]; + } else if ([object isKindOfClass: [NSArray class]]) { + normalizedMetrics[key] = [self normalizeArray: (NSArray *)object]; + } + } + + return (NSDictionary *)normalizedMetrics; +} + +/* + * Convert normalies and converts the metrics dictionary to a single JSON + * object. + * + * @param metrics an NSDictionary exported by the SNTMetricSet + * @param error a pointer to an NSError to allow errors to bubble up. + * + * Returns an NSArray containing one entry of all metrics serialized to JSON or + * nil on error. + */ +- (NSArray *) convert:(NSDictionary *)metrics error:(NSError **)err +{ + NSDictionary *normalizedMetrics = [self normalize: metrics]; + + if (![NSJSONSerialization isValidJSONObject: normalizedMetrics]) { + LOGE(@"unable to convert metrics to JSON: invalid metrics"); + return nil; + } + + NSData *json = [NSJSONSerialization dataWithJSONObject: normalizedMetrics + options: NSJSONWritingPrettyPrinted + error: err]; + if (json == nil && *err != nil) { + return nil; + } + + return @[ json ]; +} +@end \ No newline at end of file diff --git a/Source/santametricservice/Formats/SNTMetricRawJsonFormatTest.m b/Source/santametricservice/Formats/SNTMetricRawJsonFormatTest.m new file mode 100644 index 000000000..f1fba7482 --- /dev/null +++ b/Source/santametricservice/Formats/SNTMetricRawJsonFormatTest.m @@ -0,0 +1,155 @@ +#import + +#import "Source/common/SNTMetricSet.h" +#import "Source/santametricservice/Formats/SNTMetricRawJsonFormat.h" + +NSDictionary *validMetricsDict = nil; + +@interface SNTMetricRawJsonFormatTest : XCTestCase +@end + +@implementation SNTMetricRawJsonFormatTest + +- (void)initializeValidMetricsDict +{ + NSDateFormatter *formatter = NSDateFormatter.new; + [formatter setDateFormat:@"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"]; + NSDate *fixedDate = [formatter dateFromString:@"2021-09-16T21:07:34.826Z"]; + + validMetricsDict = @{ + @"root_labels" : @{@"hostname" : @"testHost", @"username" : @"testUser"}, + @"metrics" : @{ + @"/build/label" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantString], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @"20210809.0.1" + } ] + } + }, + @"/santa/events" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeCounter], + @"fields" : @{ + @"rule_type" : @[ + @{ + @"value" : @"binary", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @1, + }, + @{ + @"value" : @"certificate", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @2, + }, + ], + }, + }, + @"/santa/rules" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"rule_type" : @[ + @{ + @"value" : @"binary", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @1 + }, + @{ + @"value" : @"certificate", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @3 + } + ] + }, + }, + @"/santa/using_endpoint_security_framework" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantBool], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithBool:YES] + } ] + } + }, + @"/proc/birth_timestamp" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithLong:1250999830800] + } ] + }, + }, + @"/proc/memory/virtual_size" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithInt:987654321] + } ] + } + }, + @"/proc/memory/resident_size" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithInt:123456789] + } ] + }, + }, + } + }; +} + +- (void)setUp +{ + [self initializeValidMetricsDict]; +} + +- (void)testMetricsConversionToJSON +{ + SNTMetricRawJsonFormat *formatter = [[SNTMetricRawJsonFormat alloc] init]; + NSError *err = nil; + NSArray *output = [formatter convert:validMetricsDict error: &err]; + + XCTAssertEqual(1, [output count]); + XCTAssertNotNil(output[0]); + XCTAssertNil(err); + + NSDictionary *jsonDict = [NSJSONSerialization JSONObjectWithData:output[0] + options:NSJSONReadingAllowFragments + error:&err]; + XCTAssertNotNil(jsonDict); + + NSString *path = [[NSBundle bundleForClass:[self class]] resourcePath]; + path = [path stringByAppendingPathComponent:@"testdata/json/test.json"]; + + NSFileManager *filemgr = [NSFileManager defaultManager]; + NSData *goldenFileData = [filemgr contentsAtPath: path]; + + XCTAssertNotNil(goldenFileData, @"unable to open / read golden file"); + + NSDictionary *expectedJsonDict = [NSJSONSerialization JSONObjectWithData:goldenFileData + options:NSJSONReadingAllowFragments + error: &err]; + + XCTAssertNotNil(expectedJsonDict); + XCTAssertEqualObjects(expectedJsonDict, jsonDict, @"generated JSON does not match golden file."); +} + +@end \ No newline at end of file diff --git a/Source/santametricservice/Formats/testdata/json/test.json b/Source/santametricservice/Formats/testdata/json/test.json new file mode 100644 index 000000000..36ddbbe45 --- /dev/null +++ b/Source/santametricservice/Formats/testdata/json/test.json @@ -0,0 +1,111 @@ +{ + "metrics" : { + "/santa/rules" : { + "type" : 7, + "fields" : { + "rule_type" : [ + { + "value" : "binary", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 1 + }, + { + "value" : "certificate", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 3 + } + ] + } + }, + "/santa/events" : { + "type" : 9, + "fields" : { + "rule_type" : [ + { + "value" : "binary", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 1 + }, + { + "value" : "certificate", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 2 + } + ] + } + }, + "/proc/memory/resident_size" : { + "type" : 7, + "fields" : { + "" : [ + { + "value" : "", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 123456789 + } + ] + } + }, + "/build/label" : { + "type" : 2, + "fields" : { + "" : [ + { + "value" : "", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : "20210809.0.1" + } + ] + } + }, + "/proc/birth_timestamp" : { + "type" : 3, + "fields" : { + "" : [ + { + "value" : "", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 1250999830800 + } + ] + } + }, + "/proc/memory/virtual_size" : { + "type" : 7, + "fields" : { + "" : [ + { + "value" : "", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : 987654321 + } + ] + } + }, + "/santa/using_endpoint_security_framework" : { + "type" : 1, + "fields" : { + "" : [ + { + "value" : "", + "created" : "2021-09-16T21:07:34.826Z", + "last_updated" : "2021-09-16T21:07:34.826Z", + "data" : true + } + ] + } + } + }, + "root_labels" : { + "hostname" : "testHost", + "username" : "testUser" + } +} diff --git a/Source/santametricservice/Info.plist b/Source/santametricservice/Info.plist new file mode 100644 index 000000000..52f9026d2 --- /dev/null +++ b/Source/santametricservice/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + santametricservice + CFBundleExecutable + santametricservice + CFBundleIdentifier + com.google.santa.metricservice + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + santametricservice + CFBundleShortVersionString + ${SANTA_VERSION} + CFBundleSignature + ???? + CFBundleVersion + ${SANTA_VERSION} + NSHumanReadableCopyright + Google LLC. + + diff --git a/Source/santametricservice/SNTMetricService.h b/Source/santametricservice/SNTMetricService.h new file mode 100644 index 000000000..a8830842c --- /dev/null +++ b/Source/santametricservice/SNTMetricService.h @@ -0,0 +1,20 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import + +#import "Source/common/SNTXPCMetricServiceInterface.h" + +@interface SNTMetricService : NSObject +@end diff --git a/Source/santametricservice/SNTMetricService.m b/Source/santametricservice/SNTMetricService.m new file mode 100644 index 000000000..b8c38b0b6 --- /dev/null +++ b/Source/santametricservice/SNTMetricService.m @@ -0,0 +1,121 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#include +#include + +#import "Source/common/SNTConfigurator.h" +#import "Source/common/SNTLogging.h" + +#import "Source/santametricservice/Formats/SNTMetricRawJsonFormat.h" +#import "Source/santametricservice/Writers/SNTMetricFileWriter.h" +#import "SNTMetricService.h" + +@interface SNTMetricService () +@property MOLXPCConnection *notifierConnection; +@property MOLXPCConnection *listener; +@property(nonatomic) dispatch_queue_t queue; +@end + +@implementation SNTMetricService { + @private + SNTMetricRawJsonFormat* rawJsonFormatter; + NSDictionary *metricWriters; +} + +- (instancetype)init { + self = [super init]; + + rawJsonFormatter = [[SNTMetricRawJsonFormat alloc] init]; + metricWriters = @{@"file": [[SNTMetricFileWriter alloc] init]}; + + _queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0); + return self; +} + +/** + * Helper function to format NSError's for logging error messages. + */ +- (NSString *)messageFromError:(NSError *)error +{ + NSString *message = [error localizedDescription]; + NSString *details = [error localizedFailureReason] ? + [error localizedFailureReason] : @""; + + return [NSString stringWithFormat:@"%@ %@", message, details]; +} + + +/** + * Converts the exported Metrics dicitionary to the appropriate monitoring + * format. + * + * @param metrics NSDictionary containing the exported metrics + * @param format SNTMetricFormatType the exported metrics format + * @return An array of metrics formatted according to the specified format or + * nil on error; + */ +- (NSArray *) convertMetrics:(NSDictionary *)metrics + toFormat:(SNTMetricFormatType)format + error:(NSError **)err +{ + switch (format) { + case SNTMetricFormatTypeRawJSON: + return [self->rawJsonFormatter convert: metrics error:err]; + default: + return nil; + } +} + +/** + * Exports the metrics for a configured monitoring system, if santa is + * configured to do so. + * + * @param metrics The NSDictionary from a MetricSet export call. + */ +- (void)exportForMonitoring:(NSDictionary *)metrics { + SNTConfigurator *config = [SNTConfigurator configurator]; + + if (![config exportMetrics]) { + return; + } + + if (metrics == nil) { + LOGE(@"nil metrics dictionary sent for export"); + return; + } + + NSError *err; + NSArray *formattedMetrics = [self convertMetrics:metrics + toFormat:config.metricFormat + error: &err]; + + if (err != nil) { + LOGE(@"unable to format metrics as %@", [self messageFromError: err]); + return; + } + + const id writer = metricWriters[config.metricURL.scheme]; + + if (writer) { + BOOL ok = [writer write:formattedMetrics toURL: config.metricURL error:&err]; + + if (!ok) { + if (err != nil) { + LOGE(@"unable to write metrics: %@", [self messageFromError: err]); + } + } + } +} +@end diff --git a/Source/santametricservice/SNTMetricServiceTest.m b/Source/santametricservice/SNTMetricServiceTest.m new file mode 100644 index 000000000..b224e513c --- /dev/null +++ b/Source/santametricservice/SNTMetricServiceTest.m @@ -0,0 +1,232 @@ +#include +#import + +#import "Source/common/SNTCommonEnums.h" +#import "Source/common/SNTConfigurator.h" +#import "Source/common/SNTMetricSet.h" + +#import + +#import "Source/santametricservice/SNTMetricService.h" + +NSDictionary *validMetricsDict = nil; + +@interface SNTMetricServiceTest : XCTestCase +@property id mockConfigurator; +@property NSString *tempDir; +@property NSURL *jsonURL; +@end + +@implementation SNTMetricServiceTest + +- (void)initializeValidMetricsDict +{ + NSDateFormatter *formatter = NSDateFormatter.new; + [formatter setDateFormat:@"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"]; + NSDate *fixedDate = [formatter dateFromString:@"2021-09-16T21:07:34.826Z"]; + + validMetricsDict = @{ + @"root_labels" : @{@"hostname" : @"testHost", @"username" : @"testUser"}, + @"metrics" : @{ + @"/build/label" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantString], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @"20210809.0.1" + } ] + } + }, + @"/santa/events" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeCounter], + @"fields" : @{ + @"rule_type" : @[ + @{ + @"value" : @"binary", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @1, + }, + @{ + @"value" : @"certificate", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @2, + }, + ], + }, + }, + @"/santa/rules" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"rule_type" : @[ + @{ + @"value" : @"binary", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @1 + }, + @{ + @"value" : @"certificate", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : @3 + } + ] + }, + }, + @"/santa/using_endpoint_security_framework" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantBool], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithBool:YES] + } ] + } + }, + @"/proc/birth_timestamp" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeConstantInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithLong:1250999830800] + } ] + }, + }, + @"/proc/memory/virtual_size" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithInt:987654321] + } ] + } + }, + @"/proc/memory/resident_size" : @{ + @"type" : [NSNumber numberWithInt:(int)SNTMetricTypeGaugeInt64], + @"fields" : @{ + @"" : @[ @{ + @"value" : @"", + @"created" : fixedDate, + @"last_updated" : fixedDate, + @"data" : [NSNumber numberWithInt:123456789] + } ] + }, + }, + } + }; +} + +- (void)setUp +{ + [self initializeValidMetricsDict]; + //create the configurator + self.mockConfigurator = OCMClassMock([SNTConfigurator class]); + OCMStub([self.mockConfigurator configurator]).andReturn(self.mockConfigurator); + + // create a temp dir + char template[] = "/tmp/sntmetricsservicetestdata.XXXXXXX"; + char *tempPath = mkdtemp(template); + + if (tempPath == NULL) { + NSLog(@"Unable to make temp directory"); + exit(1); + } + + self.tempDir = [[NSFileManager defaultManager] + stringWithFileSystemRepresentation: tempPath + length: strlen(tempPath)]; + self.jsonURL = [NSURL URLWithString: [NSString pathWithComponents: @[@"file://", self.tempDir, @"test.json"]]]; + +} + +- (void)tearDown +{ + [self.mockConfigurator stopMocking]; + + //delete the temp dir + [[NSFileManager defaultManager] removeItemAtPath: self.tempDir + error:NULL]; +} + +- (NSDate *) createNSDateFromDateString:(NSString *)dateString +{ + NSDateFormatter *formatter = [[NSDateFormatter alloc] init]; + + [formatter setTimeZone:[NSTimeZone timeZoneWithName:@"UTC"]]; + [formatter setDateFormat:@"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"]; + + return [formatter dateFromString: dateString]; +} + +- (NSDictionary *)convertJSONDateStringsToNSDateWithJson: (NSDictionary *)jsonData +{ + NSMutableDictionary *jsonDict = [jsonData mutableCopy]; + + for (NSString *metricName in jsonDict[@"metrics"]) { + NSMutableDictionary *metric = jsonDict[@"metrics"][metricName]; + + for (NSString *field in metric[@"fields"]) { + NSMutableArray *values = metric[@"fields"][field]; + + for (int i = 0; i < [values count]; i++) { + values[i][@"created"] = [self createNSDateFromDateString: values[i][@"created"]]; + values[i][@"last_updated"] = [self createNSDateFromDateString: values[i][@"last_updated"]]; + } + } + } + + return jsonDict; +} + +- (void)testDefaultConfigOptionsDoNotExport +{ + SNTMetricService *ms = [[SNTMetricService alloc] init]; + //OCMStub([self.mockConfigurator exportMetrics]).andReturn(NO); + + [ms exportForMonitoring: validMetricsDict]; + + // Check the temp dir + NSArray* items = [[NSFileManager defaultManager] contentsOfDirectoryAtPath: self.tempDir + error:NULL]; + XCTAssertEqual(0, [items count]); +} + +- (void)testWritingRawJSONFile +{ + OCMStub([self.mockConfigurator exportMetrics]).andReturn(YES); + OCMStub([self.mockConfigurator metricFormat]).andReturn(SNTMetricFormatTypeRawJSON); + OCMStub([self.mockConfigurator metricURL]).andReturn(self.jsonURL); + + + SNTMetricService *ms = [[SNTMetricService alloc] init]; + [ms exportForMonitoring: validMetricsDict]; + + // Ensure that this has written 1 file that is well formed. + NSArray* items = [[NSFileManager defaultManager] contentsOfDirectoryAtPath: self.tempDir + error:NULL]; + XCTAssertEqual(1, [items count], @"failed to create JSON metrics file"); + + NSData *jsonData = [NSData dataWithContentsOfFile:self.jsonURL.path + options:NSDataReadingUncached + error:nil]; + + NSDictionary *parsedJSONData = [NSJSONSerialization JSONObjectWithData:jsonData + options:NSJSONReadingMutableContainers + error:nil]; + + // Convert JSON's date strings back into dates. + [self convertJSONDateStringsToNSDateWithJson: parsedJSONData]; + + + XCTAssertEqualObjects(validMetricsDict, parsedJSONData, @"invalid json created"); +} +@end diff --git a/Source/santametricservice/Writers/BUILD b/Source/santametricservice/Writers/BUILD new file mode 100644 index 000000000..b5445fd0e --- /dev/null +++ b/Source/santametricservice/Writers/BUILD @@ -0,0 +1,35 @@ +load("@build_bazel_rules_apple//apple:macos.bzl", "macos_command_line_application") +load("//:helper.bzl", "santa_unit_test") + +package(default_visibility = ["//:santa_package_group"]) + +licenses(["notice"]) # Apache 2.0 + +objc_library( + name = "SNTMetricWriter", + hdrs = ["SNTMetricWriter.h"], +) + +objc_library( + name = "SNTMetricFileWriter", + srcs = [ + "SNTMetricFileWriter.h", + "SNTMetricFileWriter.m", + "SNTMetricWriter.h", + + ], + deps = [ + ":SNTMetricWriter", + "//Source/common:SNTLogging", + ], +) + +santa_unit_test( + name = "SNTMetricFileWriterTest", + srcs = [ + "SNTMetricFileWriterTest.m", + ], + deps = [ + ":SNTMetricFileWriter", + ], +) diff --git a/Source/santametricservice/Writers/SNTMetricFileWriter.h b/Source/santametricservice/Writers/SNTMetricFileWriter.h new file mode 100644 index 000000000..49e158d45 --- /dev/null +++ b/Source/santametricservice/Writers/SNTMetricFileWriter.h @@ -0,0 +1,17 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +#import "Source/santametricservice/Writers/SNTMetricWriter.h" + +@interface SNTMetricFileWriter : NSObject +@end \ No newline at end of file diff --git a/Source/santametricservice/Writers/SNTMetricFileWriter.m b/Source/santametricservice/Writers/SNTMetricFileWriter.m new file mode 100644 index 000000000..44a47fc50 --- /dev/null +++ b/Source/santametricservice/Writers/SNTMetricFileWriter.m @@ -0,0 +1,77 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import "Source/common/SNTLogging.h" +#import "Source/santametricservice/Writers/SNTMetricFileWriter.h" + +@implementation SNTMetricFileWriter + +/* + * Open a file for appending. + */ +- (NSFileHandle *)fileHandleForAppendingAtPath:(NSString *)path createMode:(mode_t)mode { + int fd; + if (!path) { + return nil; + } + + fd = open([path fileSystemRepresentation], O_WRONLY | O_APPEND | O_TRUNC | O_CREAT, mode); + if (fd < 0) { + return nil; + } + return [[NSFileHandle alloc] initWithFileDescriptor:fd closeOnDealloc:YES]; +} + +/** + * Write serialzied metrics to the file one JSON object per line. + **/ +- (BOOL)write:(NSArray *)metrics toURL:(NSURL *)url error: (NSError **)error { + // open the file and write it. + @autoreleasepool { + if (![url isFileURL]) { + LOGE(@"url supplied to SNTMetricFileOutput is not a file url, given %@", url.absoluteString); + return NO; + } + + NSFileHandle *file = [self fileHandleForAppendingAtPath:url.path createMode:0600]; + const char newline[1] = {'\n'}; + + if (file == nil) { + LOGE(@"Unable to open file %@ to write metrics", url.path); + return NO; + } + + NSMutableData *lineData; + + for (int i = 0; i < [metrics count]; i++) { + lineData = [NSMutableData dataWithData: metrics[i]]; + + [lineData appendBytes: newline length: 1]; + + if (@available(macos 10.15, *)) { + [file writeData:lineData error:error]; + + if (*error != nil) { + return NO; + } + } else { + [file writeData:lineData]; + } + } + } + + return YES; +} + +@end diff --git a/Source/santametricservice/Writers/SNTMetricFileWriterTest.m b/Source/santametricservice/Writers/SNTMetricFileWriterTest.m new file mode 100644 index 000000000..ea78bea1d --- /dev/null +++ b/Source/santametricservice/Writers/SNTMetricFileWriterTest.m @@ -0,0 +1,101 @@ +#import + +#import "Source/santametricservice/Writers/SNTMetricFileWriter.h" + +@interface SNTMetricFileWriterTest : XCTestCase +@property NSString *tempDir; +@end + +@implementation SNTMetricFileWriterTest + +- (void)setUp +{ + // create a temp dir + char template[] = "/tmp/sntmetricfileoutputtest.XXXXXXX"; + char *tempPath = mkdtemp(template); + + if (tempPath == NULL) { + NSLog(@"Unable to make temp directory"); + exit(1); + } + + self.tempDir = [[NSFileManager defaultManager] + stringWithFileSystemRepresentation: tempPath + length: strlen(tempPath)]; +} + +- (void)tearDown +{ + //delete the temp dir + [[NSFileManager defaultManager] removeItemAtPath: self.tempDir + error:NULL]; +} + +- (void)testWritingToNonFileURLFails +{ + NSString *testURL = @"http://www.google.com"; + + SNTMetricFileWriter *fileWriter = [[SNTMetricFileWriter alloc] init]; + + NSError *err; + + NSData *firstLine = [@"AAAAAAAA" dataUsingEncoding:NSUTF8StringEncoding]; + + NSArray *input = @[firstLine]; + + BOOL result = [fileWriter write: input toURL:[NSURL URLWithString: testURL] error:&err]; + XCTAssertFalse(result); +} + +- (void)testWritingDataToFileWorks +{ + NSString *testFile = [NSString pathWithComponents: @[@"file://", self.tempDir, @"test.data"]]; + NSURL *url = [NSURL URLWithString: testFile]; + + + SNTMetricFileWriter *fileWriter = [[SNTMetricFileWriter alloc] init]; + + NSError *err; + + NSData *firstLine = [@"AAAAAAAA" dataUsingEncoding:NSUTF8StringEncoding]; + NSData *secondLine = [@"BBBBBBBB" dataUsingEncoding:NSUTF8StringEncoding]; + + NSArray *input = @[firstLine]; + + BOOL success = [fileWriter write: input toURL: url error:&err]; + + if (!success) { + NSLog(@"error: %@\n", err); + } + + XCTAssertEqual(YES, success); + XCTAssertNil(err); + const char newline[1] = {'\n'}; + + // Read file ensure it only contains the first line followed by a Newline + NSData *testFileContents = [NSData dataWithContentsOfFile: url.path]; + NSMutableData *expected = [NSMutableData dataWithData:firstLine]; + + [expected appendBytes: newline length: 1]; + + XCTAssertEqualObjects(expected, testFileContents); + + [expected appendData: secondLine]; + [expected appendBytes: newline length: 1]; + + // Test that calling a second time overwrites the file and that multiple rows + // are separated by a newline + input = @[firstLine, secondLine]; + + success = [fileWriter write:input toURL:url error:&err]; + XCTAssertEqual(YES, success); + XCTAssertNil(err); + + if (!success) { + NSLog(@"error: %@\n", err); + } + + testFileContents = [NSData dataWithContentsOfFile: url.path]; + XCTAssertEqualObjects(expected, testFileContents); +} +@end \ No newline at end of file diff --git a/Source/santametricservice/Writers/SNTMetricWriter.h b/Source/santametricservice/Writers/SNTMetricWriter.h new file mode 100644 index 000000000..2d47263b6 --- /dev/null +++ b/Source/santametricservice/Writers/SNTMetricWriter.h @@ -0,0 +1,23 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import + +/** + * An SNTMetricWriter outputs a serialized SNTMetricSet to the external + * monitoring system. + * */ +@protocol SNTMetricWriter +- (BOOL) write:(NSArray *)data toURL:(NSURL *)url error:(NSError **) error; +@end \ No newline at end of file diff --git a/Source/santametricservice/main.m b/Source/santametricservice/main.m new file mode 100644 index 000000000..6f0ad6c33 --- /dev/null +++ b/Source/santametricservice/main.m @@ -0,0 +1,35 @@ +/// Copyright 2021 Google Inc. All rights reserved. +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import + +#import + +#import "Source/common/SNTLogging.h" +#import "Source/common/SNTXPCMetricServiceInterface.h" +#import "Source/santametricservice/SNTMetricService.h" + +int main(int argc, const char *argv[]) { + @autoreleasepool { + NSDictionary *infoDict = [[NSBundle mainBundle] infoDictionary]; + LOGI(@"Started, version %@", infoDict[@"CFBundleVersion"]); + MOLXPCConnection *c = + [[MOLXPCConnection alloc] initServerWithName:[SNTXPCMetricServiceInterface serviceID]]; + c.privilegedInterface = c.unprivilegedInterface = + [SNTXPCMetricServiceInterface metricServiceInterface]; + c.exportedObject = [[SNTMetricService alloc] init]; + [c resume]; + [[NSRunLoop mainRunLoop] run]; + } +} diff --git a/docs/deployment/configuration.md b/docs/deployment/configuration.md index 9b3754309..52bf8f517 100644 --- a/docs/deployment/configuration.md +++ b/docs/deployment/configuration.md @@ -47,7 +47,7 @@ Additionally, there are options that can be controlled by both. | EventLogType | String | Defines how event logs are stored. Options are 1) syslog: Sent to ASL or ULS (if built with the 10.12 SDK or later). 2) filelog: Sent to a file on disk. Use EventLogPath to specify a path. Defaults to filelog | | EventLogPath | String | If EventLogType is set to filelog, EventLogPath will provide the path to save logs. Defaults to /var/db/santa/santa.log. If you change this value ensure you also update com.google.santa.newsyslog.conf with the new path. | | EnableMachineIDDecoration | Bool | If YES, this appends the MachineID to the end of each log line. Defaults to NO. | -| MetricFormat | String | Format to export metrics as, supported formats are "rawjson" for a single JSON blob and "json" for one metric per line. Defaults to "". | +| MetricFormat | Integer | Format to export metrics as 0 = None, 1 = Raw JSON blob, 2 = JSON one metric per line. Defaults to 0. | | MetricURL | String | URL describing where monitoring metrics should be exported. | *overridable by the sync server: run `santactl status` to check the current From 1221437fe32fc7333d29d1f24b5146b7903a2c0f Mon Sep 17 00:00:00 2001 From: Pete Markowsky Date: Wed, 22 Sep 2021 10:11:05 -0400 Subject: [PATCH 2/2] Fixed up formatting. --- Source/santametricservice/Formats/BUILD | 8 +++----- Source/santametricservice/Writers/BUILD | 15 +++++++-------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Source/santametricservice/Formats/BUILD b/Source/santametricservice/Formats/BUILD index b24f90ccf..6cd69d32f 100644 --- a/Source/santametricservice/Formats/BUILD +++ b/Source/santametricservice/Formats/BUILD @@ -15,8 +15,7 @@ objc_library( srcs = [ "SNTMetricRawJsonFormat.h", "SNTMetricRawJsonFormat.m", - "SNTMetricFormat.h", - + "SNTMetricFormat.h", ], deps = [ ":SNTMetricFormat", @@ -31,12 +30,11 @@ santa_unit_test( ], structured_resources = glob(["testdata/**"]), deps = [ - ":SNTMetricRawJsonFormat", - "//Source/common:SNTMetricSet", + ":SNTMetricRawJsonFormat", + "//Source/common:SNTMetricSet", ], ) - test_suite( name = "format_tests", tests = [ diff --git a/Source/santametricservice/Writers/BUILD b/Source/santametricservice/Writers/BUILD index b5445fd0e..557cedc02 100644 --- a/Source/santametricservice/Writers/BUILD +++ b/Source/santametricservice/Writers/BUILD @@ -13,23 +13,22 @@ objc_library( objc_library( name = "SNTMetricFileWriter", srcs = [ - "SNTMetricFileWriter.h", - "SNTMetricFileWriter.m", - "SNTMetricWriter.h", - + "SNTMetricFileWriter.h", + "SNTMetricFileWriter.m", + "SNTMetricWriter.h", ], deps = [ - ":SNTMetricWriter", - "//Source/common:SNTLogging", + ":SNTMetricWriter", + "//Source/common:SNTLogging", ], ) santa_unit_test( name = "SNTMetricFileWriterTest", srcs = [ - "SNTMetricFileWriterTest.m", + "SNTMetricFileWriterTest.m", ], deps = [ - ":SNTMetricFileWriter", + ":SNTMetricFileWriter", ], )