Permalink
Browse files

Adding support for better test harness

This uses a branch of the Autobahn test suite that will return test
result information.
  • Loading branch information...
1 parent aa22978 commit ddfdad81d1b76fc9bc394c9be40709724e40f816 @mikelikespie mikelikespie committed Jul 29, 2012
View
113 SRWebSocketTests/SRTAutobahnTests.m
@@ -58,6 +58,22 @@ - (id)initWithBaseURL:(NSURL *)url agent:(NSString *)agent;
@end
+@interface TestInfoOperation : SRTWebSocketOperation <SRWebSocketDelegate>
+
+@property (nonatomic) NSDictionary *info;
+
+- (id)initWithBaseURL:(NSURL *)url caseNumber:(NSInteger)caseNumber;
+
+@end
+
+@interface TestResultsOperation : SRTWebSocketOperation <SRWebSocketDelegate>
+
+@property (nonatomic) NSDictionary *info;
+
+- (id)initWithBaseURL:(NSURL *)url caseNumber:(NSInteger)caseNumber agent:(NSString *)agent;
+
+@end
+
@implementation SRTAutobahnTests {
SRWebSocket *_curWebSocket;
NSInteger _testCount;
@@ -66,12 +82,23 @@ @implementation SRTAutobahnTests {
NSString *_testURLString;
NSURL *_prefixURL;
NSString *_agent;
+ NSString *_description;
+}
+
+- (id)initWithInvocation:(NSInvocation *)anInvocation description:(NSString *)description;
+{
+ self = [self initWithInvocation:anInvocation];
+ if (self) {
+ _description = description;
+ }
+ return self;
}
- (id)initWithInvocation:(NSInvocation *)anInvocation;
{
self = [super initWithInvocation:anInvocation];
if (self) {
+ [self raiseAfterFailure];
_testURLString = [[NSProcessInfo processInfo].environment objectForKey:@"SR_TEST_URL"];
_prefixURL = [NSURL URLWithString:_testURLString];
_agent = [NSBundle bundleForClass:[self class]].bundleIdentifier;
@@ -105,13 +132,14 @@ - (BOOL) isEmpty;
return NO;
}
-- (void) performTest:(SenTestRun *) aRun
+- (void) performTest:(SenTestCaseRun *) aRun
{
if (self.invocation) {
[super performTest:aRun];
return;
}
- for (int i = 0; i < aRun.test.testCaseCount; i++) {
+ [aRun start];
+ for (int i = 1; i <= aRun.test.testCaseCount; i++) {
SEL sel = @selector(performTestWithNumber:);
NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:[[self class] instanceMethodSignatureForSelector:sel]];
@@ -120,12 +148,21 @@ - (void) performTest:(SenTestRun *) aRun
[invocation setArgument:&i atIndex:2];
- SenTestCase *testCase = [[[self class] alloc] initWithInvocation:invocation];
+ NSString *description = [self caseDescriptionForCaseNumber:i];
+
+ SenTestCase *testCase = [[[self class] alloc] initWithInvocation:invocation description:description];
SenTestCaseRun *run = [[SenTestCaseRun alloc] initWithTest:testCase];
[testCase performTest:run];
+
+ for (NSException *e in run.exceptions) {
+ [aRun addException:e];
+ }
+
+// [aRun ]
}
+ [aRun stop];
[self updateReports];
}
@@ -137,10 +174,25 @@ - (NSInteger)testNum;
return i;
}
+- (NSString *)caseDescriptionForCaseNumber:(NSInteger)caseNumber;
+{
+ TestInfoOperation *testInfoOperation = [[TestInfoOperation alloc] initWithBaseURL:_prefixURL caseNumber:caseNumber];
+
+ [testInfoOperation start];
+
+ [self runCurrentRunLoopUntilTestPasses:^BOOL{
+ return testInfoOperation.isFinished;
+ } timeout:60 * 60];
+
+ STAssertNil(testInfoOperation.error, @"Updating the report should not have errored");
+
+ return [NSString stringWithFormat:@"%@ - %@", [testInfoOperation.info objectForKey:@"id"], [testInfoOperation.info objectForKey:@"description"]];
+}
+
- (NSString *)description;
{
- if (self.invocation) {
- return [NSString stringWithFormat:@"Autobahn test %d", self.testNum];
+ if (_description) {
+ return _description;
} else {
return @"Autobahn Test Harness";
}
@@ -157,13 +209,21 @@ - (void)performTestWithNumber:(NSInteger)testNumber;
testQueue.maxConcurrentOperationCount = 1;
-
-
TestOperation *testOp = [[TestOperation alloc] initWithBaseURL:_prefixURL testNumber:testNumber agent:_agent];
[testQueue addOperation:testOp];
+ TestResultsOperation *resultOp = [[TestResultsOperation alloc] initWithBaseURL:_prefixURL caseNumber:testNumber agent:_agent];
+ [resultOp addDependency:testOp];
+ [testQueue addOperation:resultOp];
+
testQueue.suspended = NO;
+
+ [self runCurrentRunLoopUntilTestPasses:^BOOL{
+ return resultOp.isFinished;
+ } timeout:60 * 60];
+ STAssertTrue(!testOp.error, @"Test operation should not have failed");
+ STAssertEqualObjects(@"OK", [resultOp.info objectForKey:@"behavior"], @"Test behavior should be OK");
}
- (void)updateReports;
@@ -177,7 +237,6 @@ - (void)updateReports;
} timeout:60 * 60];
STAssertNil(updateReportOperation.error, @"Updating the report should not have errored");
-
}
@end
@@ -246,3 +305,41 @@ - (void)start;
@end
+@implementation TestInfoOperation
+
+@synthesize info = _info;
+
+- (id)initWithBaseURL:(NSURL *)url caseNumber:(NSInteger)caseNumber;
+{
+ NSString *path = [[url URLByAppendingPathComponent:@"getCaseInfo"] absoluteString];
+ path = [path stringByAppendingFormat:@"?case=%d", caseNumber];
+
+ return [super initWithURL:[NSURL URLWithString:path]];
+}
+
+- (void)webSocket:(SRWebSocket *)webSocket didReceiveMessage:(NSString *)message;
+{
+ self.info = [NSJSONSerialization JSONObjectWithData:[message dataUsingEncoding:NSUTF8StringEncoding] options:0 error:NULL];
+}
+
+@end
+
+
+@implementation TestResultsOperation
+
+@synthesize info = _info;
+
+- (id)initWithBaseURL:(NSURL *)url caseNumber:(NSInteger)caseNumber agent:(NSString *)agent;
+{
+ NSString *path = [[url URLByAppendingPathComponent:@"getCaseStatus"] absoluteString];
+ path = [path stringByAppendingFormat:@"?case=%d&agent=%@", caseNumber, agent];
+
+ return [super initWithURL:[NSURL URLWithString:path]];
+}
+
+- (void)webSocket:(SRWebSocket *)webSocket didReceiveMessage:(NSString *)message;
+{
+ self.info = [NSJSONSerialization JSONObjectWithData:[message dataUsingEncoding:NSUTF8StringEncoding] options:0 error:NULL];
+}
+
+@end
View
89 SocketRocket.xcodeproj/xcshareddata/xcschemes/SocketRocketTests.xcscheme
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+ LastUpgradeVersion = "0440"
+ version = "1.3">
+ <BuildAction
+ parallelizeBuildables = "YES"
+ buildImplicitDependencies = "YES">
+ <BuildActionEntries>
+ <BuildActionEntry
+ buildForTesting = "YES"
+ buildForRunning = "YES"
+ buildForProfiling = "NO"
+ buildForArchiving = "NO"
+ buildForAnalyzing = "NO">
+ <BuildableReference
+ BuildableIdentifier = "primary"
+ BlueprintIdentifier = "F6BDA801145900D200FE3253"
+ BuildableName = "SRWebSocketTests.octest"
+ BlueprintName = "SRWebSocketTests"
+ ReferencedContainer = "container:SocketRocket.xcodeproj">
+ </BuildableReference>
+ </BuildActionEntry>
+ </BuildActionEntries>
+ </BuildAction>
+ <TestAction
+ selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+ selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+ shouldUseLaunchSchemeArgsEnv = "NO"
+ buildConfiguration = "Debug">
+ <Testables>
+ <TestableReference
+ skipped = "NO">
+ <BuildableReference
+ BuildableIdentifier = "primary"
+ BlueprintIdentifier = "F6BDA801145900D200FE3253"
+ BuildableName = "SRWebSocketTests.octest"
+ BlueprintName = "SRWebSocketTests"
+ ReferencedContainer = "container:SocketRocket.xcodeproj">
+ </BuildableReference>
+ </TestableReference>
+ </Testables>
+ </TestAction>
+ <LaunchAction
+ selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+ selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+ launchStyle = "0"
+ useCustomWorkingDirectory = "NO"
+ buildConfiguration = "Debug"
+ ignoresPersistentStateOnLaunch = "NO"
+ debugDocumentVersioning = "YES"
+ allowLocationSimulation = "YES">
+ <AdditionalOptions>
+ <AdditionalOption
+ key = "MallocScribble"
+ value = ""
+ isEnabled = "YES">
+ </AdditionalOption>
+ <AdditionalOption
+ key = "OBJC_PRINT_EXCEPTIONS"
+ value = "YES"
+ isEnabled = "YES">
+ </AdditionalOption>
+ <AdditionalOption
+ key = "MallocGuardEdges"
+ value = ""
+ isEnabled = "YES">
+ </AdditionalOption>
+ <AdditionalOption
+ key = "NSZombieEnabled"
+ value = "YES"
+ isEnabled = "YES">
+ </AdditionalOption>
+ </AdditionalOptions>
+ </LaunchAction>
+ <ProfileAction
+ shouldUseLaunchSchemeArgsEnv = "YES"
+ savedToolIdentifier = ""
+ useCustomWorkingDirectory = "NO"
+ buildConfiguration = "Release"
+ debugDocumentVersioning = "YES">
+ </ProfileAction>
+ <AnalyzeAction
+ buildConfiguration = "Debug">
+ </AnalyzeAction>
+ <ArchiveAction
+ buildConfiguration = "Release"
+ revealArchiveInOrganizer = "YES">
+ </ArchiveAction>
+</Scheme>
View
3 TestSupport/run_test.sh
@@ -21,5 +21,8 @@ extra_opts="VALID_ARCHS=i386 ARCH=i386"
SHARED_ARGS="-arch i386 -configuration $CONFIGURATION -sdk iphonesimulator"
xcodebuild -scheme SocketRocketTests $SHARED_ARGS TEST_AFTER_BUILD=YES clean build $extra_opts
+RESULT=$?
kill $CHILD_PID
+
+exit $RESULT

0 comments on commit ddfdad8

Please sign in to comment.