Skip to content

Commit

Permalink
Stream integration test: split a 100k sample of the twitter public st…
Browse files Browse the repository at this point in the history
…ream into 1k chunks and feed them to the parser one by one. Fix several EOF-related bugs that were uncovered.
  • Loading branch information
stig committed Dec 8, 2010
1 parent 23fc2a2 commit 81858a6
Show file tree
Hide file tree
Showing 105 changed files with 309 additions and 23 deletions.
19 changes: 17 additions & 2 deletions Classes/SBJsonTokeniser.m
Expand Up @@ -277,7 +277,7 @@ - (void)skipWhitespace {
}

- (sbjson_token_t)match:(const char *)utf8 ofLength:(NSUInteger)len andReturn:(sbjson_token_t)tok {
if (buf.length - offset < len)
if (buf.length - offset - 1 < len)
return sbjson_token_eof;

if (strncmp([self bytes], utf8, len)) {
Expand All @@ -297,6 +297,10 @@ - (int)decodeHexQuad:(const char *)hexQuad {
for (int i = 0; i < 4; i++) {
ret *= 16;
switch (c = hexQuad[i]) {
case '\0':
return -2;
break;

case '0' ... '9':
ret += c - '0';
break;
Expand All @@ -320,6 +324,7 @@ - (int)decodeHexQuad:(const char *)hexQuad {

- (int)parseUnicodeEscape:(const char *)bytes index:(NSUInteger *)index {
int hi = [self decodeHexQuad:bytes + *index];
if (hi == -2) return -2; // EOF
if (hi < 0) {
self.error = @"Missing hex quad";
return -1;
Expand Down Expand Up @@ -360,9 +365,17 @@ - (sbjson_token_t)matchString {

for (;;) {
switch (*c++) {
case '\0':
return sbjson_token_eof;
break;

case '\\':
ret = sbjson_token_string_encoded;
switch (*c++) {
case '\0':
return sbjson_token_eof;
break;

case 'b':
case 't':
case 'n':
Expand All @@ -378,7 +391,9 @@ - (sbjson_token_t)matchString {
case 'u': {
NSUInteger i = 0;
int ch = [self parseUnicodeEscape:c index:&i];
if (ch < 0)
if (ch == -2)
return sbjson_token_eof;
if (ch == -1)
return sbjson_token_error;
c += i;
break;
Expand Down
12 changes: 6 additions & 6 deletions JSON.xcodeproj/project.pbxproj
Expand Up @@ -44,8 +44,8 @@
BC74B6ED0FC9FAD0000BD3DD /* MaxDepthTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC74B6EB0FC9FAD0000BD3DD /* MaxDepthTest.m */; };
BC74B7AE0FCA8B89000BD3DD /* ProxyTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC74B7AD0FCA8B89000BD3DD /* ProxyTest.m */; };
BC74B7AF0FCA8B89000BD3DD /* ProxyTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC74B7AD0FCA8B89000BD3DD /* ProxyTest.m */; };
BC81A51B12ADA115005C034C /* multi.json in Resources */ = {isa = PBXBuildFile; fileRef = BC81A51A12ADA115005C034C /* multi.json */; };
BC81A51C12ADA115005C034C /* multi.json in Resources */ = {isa = PBXBuildFile; fileRef = BC81A51A12ADA115005C034C /* multi.json */; };
BC81A69612AF019F005C034C /* StreamParserIntegrationTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC81A69512AF019F005C034C /* StreamParserIntegrationTest.m */; };
BC81A69712AF019F005C034C /* StreamParserIntegrationTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC81A69512AF019F005C034C /* StreamParserIntegrationTest.m */; };
BC8F72A71235331400678720 /* WriterTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC8F72A61235331400678720 /* WriterTest.m */; };
BC8F72A81235331400678720 /* WriterTest.m in Sources */ = {isa = PBXBuildFile; fileRef = BC8F72A61235331400678720 /* WriterTest.m */; };
BC9602B712A452CF0024BDA2 /* SBJsonStreamParserAdapter.h in Headers */ = {isa = PBXBuildFile; fileRef = BC9602B512A452CF0024BDA2 /* SBJsonStreamParserAdapter.h */; settings = {ATTRIBUTES = (Private, ); }; };
Expand Down Expand Up @@ -307,7 +307,7 @@
BC651A81125A571C00280F5C /* SBProxyForJson.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SBProxyForJson.h; sourceTree = "<group>"; };
BC74B6EB0FC9FAD0000BD3DD /* MaxDepthTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MaxDepthTest.m; sourceTree = "<group>"; };
BC74B7AD0FCA8B89000BD3DD /* ProxyTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ProxyTest.m; sourceTree = "<group>"; };
BC81A51A12ADA115005C034C /* multi.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; name = multi.json; path = Tests/Stream/multi.json; sourceTree = SOURCE_ROOT; };
BC81A69512AF019F005C034C /* StreamParserIntegrationTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = StreamParserIntegrationTest.m; path = Tests/StreamParserIntegrationTest.m; sourceTree = SOURCE_ROOT; };
BC8F72A61235331400678720 /* WriterTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = WriterTest.m; sourceTree = "<group>"; };
BC9602B512A452CF0024BDA2 /* SBJsonStreamParserAdapter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SBJsonStreamParserAdapter.h; sourceTree = "<group>"; };
BC9602B612A452CF0024BDA2 /* SBJsonStreamParserAdapter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SBJsonStreamParserAdapter.m; sourceTree = "<group>"; };
Expand Down Expand Up @@ -517,6 +517,7 @@
53D229960C96124500276605 /* Tests */ = {
isa = PBXGroup;
children = (
BC81A69512AF019F005C034C /* StreamParserIntegrationTest.m */,
BCE5E2B4129C80A000C156C0 /* StreamParserDelegate.h */,
BCE5E2B5129C80A000C156C0 /* StreamParserDelegate.m */,
BCE5E2B6129C80A000C156C0 /* StreamParserTest.m */,
Expand All @@ -536,7 +537,6 @@
BCB559FF1221D09300ACE34F /* Test Data */ = {
isa = PBXGroup;
children = (
BC81A51A12ADA115005C034C /* multi.json */,
BCB55A001221D09400ACE34F /* array.json */,
BCB55A011221D09400ACE34F /* array.json.pretty */,
BCB55A021221D09400ACE34F /* array.json.terse */,
Expand Down Expand Up @@ -934,7 +934,6 @@
BCB55AB61221D09400ACE34F /* string.json in Resources */,
BCB55AB71221D09400ACE34F /* string.json.pretty in Resources */,
BCB55AB81221D09400ACE34F /* string.json.terse in Resources */,
BC81A51C12ADA115005C034C /* multi.json in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1032,7 +1031,6 @@
BCB55B111221D09400ACE34F /* string.json in Resources */,
BCB55B121221D09400ACE34F /* string.json.pretty in Resources */,
BCB55B131221D09400ACE34F /* string.json.terse in Resources */,
BC81A51B12ADA115005C034C /* multi.json in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1182,6 +1180,7 @@
BCE5E2F1129C825700C156C0 /* StreamParserTest.m in Sources */,
BC9602FF12A4564E0024BDA2 /* StreamParserAdapterDelegate.m in Sources */,
BC96030B12A45E3D0024BDA2 /* StreamParserAdapterTest.m in Sources */,
BC81A69712AF019F005C034C /* StreamParserIntegrationTest.m in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1215,6 +1214,7 @@
BCE5E2F3129C826A00C156C0 /* StreamParserTest.m in Sources */,
BC9602FE12A4564E0024BDA2 /* StreamParserAdapterDelegate.m in Sources */,
BC96030A12A45E3D0024BDA2 /* StreamParserAdapterTest.m in Sources */,
BC81A69612AF019F005C034C /* StreamParserIntegrationTest.m in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
4 changes: 0 additions & 4 deletions Tests/Stream/multi.json

This file was deleted.

1 change: 1 addition & 0 deletions Tests/Stream/xaa
@@ -0,0 +1 @@
{"coordinates":null,"text":":) Our secret weapon is no alternative. Golda Meir","entities":{"user_mentions":[],"urls":[],"hashtags":[]},"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"place":null,"contributors":null,"retweet_count":0,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"source":"\u003Ca href=\"http:\/\/www.socialoomph.com\" rel=\"nofollow\"\u003ESocialOomph\u003C\/a\u003E","retweeted":false,"geo":null,"created_at":"Wed Dec 08 07:43:21 +0000 2010","truncated":false,"id_str":"12411897663979520","user":{"follow_request_sent":null,"following":null,"verified":false,"profile_sidebar_border_color":"C0DEED","show_all_inline_media":false,"geo_enabled":false,"description":"Young and Energetic. I just sold my first house and will be posting Tweets about my new career as a house flipper","listed_count":49,"profile_background_tile":false,"profile_background_image_url":"http:\/\/a3.twimg.com\/profile_background_images\/38409971\/problemsolver.jpg","profile_background_color":"C0DEED"
2 changes: 2 additions & 0 deletions Tests/Stream/xab
@@ -0,0 +1,2 @@
,"location":"Ontario Canada","profile_use_background_image":true,"profile_text_color":"4d1c12","followers_count":7991,"statuses_count":41594,"time_zone":"Quito","created_at":"Sun Sep 20 20:32:38 +0000 2009","friends_count":8789,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/426248904\/homes_normal.jpg","notifications":null,"profile_link_color":"055673","url":"http:\/\/www.the-problemsolver.com","name":"We Flip","id_str":"75869159","favourites_count":0,"screen_name":"house_flipper","id":75869159,"contributors_enabled":false,"lang":"en","utc_offset":-18000,"profile_sidebar_fill_color":"c1a38c"},"id":12411897663979520,"in_reply_to_user_id":null,"favorited":false}
{"coordinates":null,"text":"RT @therealdjrell click (retweet) if you still up","entities":{"hashtags":[],"urls":[],"user_mentions":[{"indices":[3,17],"name":"Dj Rell OF BBC","screen_name":"therealdjrell","id_str":"180669193","id":180669193}]},"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"place":null,"co
1 change: 1 addition & 0 deletions Tests/Stream/xac
@@ -0,0 +1 @@
ntributors":null,"retweet_count":null,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"source":"\u003Ca href=\"http:\/\/www.htc.com\" rel=\"nofollow\"\u003E HTC Peep\u003C\/a\u003E","retweeted":false,"geo":null,"created_at":"Wed Dec 08 07:43:21 +0000 2010","truncated":false,"id_str":"12411897680764928","user":{"following":null,"listed_count":4,"profile_background_tile":true,"profile_background_image_url":"http:\/\/a1.twimg.com\/profile_background_images\/179020444\/Picnik_collage.jpg","profile_sidebar_border_color":"3d3739","profile_use_background_image":true,"description":"BABY KNOWS WHAT SHE WANTS & ALWAYS GETS IT. :) ArizonaStateUni (DowntownPHX) '14 * 18 YrOld * Chicago is Home * FOLLOW ME & I'LL FOLLOW YOU! :) #TEAMLAKERS","statuses_count":2206,"time_zone":"Arizona","friends_count":115,"profile_background_color":"0f0604","location":"N 33\u00b027' 0'' \/ W 112\u00b04' 0''","profile_text_color":"66585a","followers_count":120,"contributors_enabled":false,"created_at":"Tue Apr 07 01:55:15 +0000
2 changes: 2 additions & 0 deletions Tests/Stream/xad
@@ -0,0 +1,2 @@
2009","protected":false,"profile_image_url":"http:\/\/a2.twimg.com\/profile_images\/1182977830\/FB1_normal.jpg","follow_request_sent":null,"verified":false,"notifications":null,"profile_link_color":"cf1919","url":"http:\/\/www.facebook.com\/profile.php?id=610064376","name":"Shayla Brown","id_str":"29351068","favourites_count":16,"screen_name":"BabyistheDream4","id":29351068,"show_all_inline_media":false,"lang":"en","geo_enabled":false,"utc_offset":-25200,"profile_sidebar_fill_color":"ffffff"},"id":12411897680764928,"in_reply_to_user_id":null,"favorited":false}
{"coordinates":null,"text":"THT..boseenn..(\u02c7_\u02c7'!l)\u200b\u200e ..","entities":{"urls":[],"hashtags":[],"user_mentions":[]},"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"place":null,"contributors":null,"retweet_count":0,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"source":"\u003Ca href=\"http:\/\/blackberry.com\/twitter\" rel=\"nofollow\"\u003ETwitter for BlackBerry\u00ae\u003C\/a\u003E","retweeted":false,"ge
1 change: 1 addition & 0 deletions Tests/Stream/xae
@@ -0,0 +1 @@
o":null,"created_at":"Wed Dec 08 07:43:21 +0000 2010","truncated":false,"id_str":"12411897668177920","user":{"follow_request_sent":null,"following":null,"verified":false,"profile_sidebar_border_color":"eb1a44","show_all_inline_media":false,"geo_enabled":false,"description":"\u2661 Simple and nice Lady \u2661..\u263a ","listed_count":3,"profile_background_tile":false,"profile_background_image_url":"http:\/\/s.twimg.com\/a\/1290538325\/images\/themes\/theme1\/bg.png","profile_background_color":"e01660","location":"\u00dcT: -6.171394,106.868656","profile_use_background_image":true,"profile_text_color":"333333","followers_count":205,"statuses_count":6045,"time_zone":"Pacific Time (US & Canada)","created_at":"Thu Oct 29 06:35:04 +0000 2009","friends_count":232,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1145910676\/Picture_202584_normal.jpg","notifications":null,"profile_link_color":"CC3366","url":"http:\/\/null","name":"dila Zitria yuzar","id_str":"86008365","favourites_count":17

0 comments on commit 81858a6

Please sign in to comment.