Skip to content
Browse files

Changed how single/multi line comments are parsed. Partially addresses

  • Loading branch information...
1 parent 1fad580 commit 95429b0e60d257eb305e57b58bb028f202d6df19 @tomaz committed
Showing with 21 additions and 7 deletions.
  1. +14 −6 Parsing/GBTokenizer.m
  2. +7 −1 Testing/GBTokenizerTesting.m
View
20 Parsing/GBTokenizer.m
@@ -16,12 +16,14 @@
@interface GBTokenizer ()
- (BOOL)consumeComments;
-- (NSString *)commentValueFromString:(NSString *)value;
+- (NSString *)commentValueFromString:(NSString *)value isMultiline:(BOOL)multiline;
- (NSArray *)allTokensFromTokenizer:(PKTokenizer *)tokenizer;
@property (retain) NSString *filename;
@property (retain) NSString *input;
@property (retain) NSArray *tokens;
@property (assign) NSUInteger tokenIndex;
+@property (assign) BOOL isLastCommentMultiline;
+@property (assign) BOOL isPreviousCommentMultiline;
@property (retain) NSMutableString *lastCommentBuilder;
@property (retain) NSMutableString *previousCommentBuilder;
@property (retain) GBSourceInfo *lastCommentSourceInfo;
@@ -195,6 +197,8 @@ - (BOOL)consumeComments {
[self.previousCommentBuilder setString:self.lastCommentBuilder];
startingPreviousToken = startingLastToken;
[self.lastCommentBuilder setString:@""];
+ self.isPreviousCommentMultiline = self.isLastCommentMultiline;
+ self.isLastCommentMultiline = NO;
startingLastToken = token;
}
previousSingleLineEndOffset = [token offset] + [[token stringValue] length];
@@ -207,6 +211,8 @@ - (BOOL)consumeComments {
[self.previousCommentBuilder setString:self.lastCommentBuilder];
startingPreviousToken = startingLastToken;
[self.lastCommentBuilder setString:@""];
+ self.isPreviousCommentMultiline = self.isLastCommentMultiline;
+ self.isLastCommentMultiline = YES;
startingLastToken = token;
}
@@ -236,7 +242,7 @@ - (BOOL)consumeComments {
return YES;
}
-- (NSString *)commentValueFromString:(NSString *)value {
+- (NSString *)commentValueFromString:(NSString *)value isMultiline:(BOOL)multiline {
if ([value length] == 0) return nil;
NSArray *lines = [value componentsSeparatedByCharactersInSet:[NSCharacterSet newlineCharacterSet]];
NSMutableArray *comments = [NSMutableArray arrayWithCapacity:[lines count]];
@@ -251,7 +257,7 @@ - (NSString *)commentValueFromString:(NSString *)value {
}
[comments addObject:line];
}];
-
+
// If all lines start with a *, ignore the prefix. Note that we ignore first line as it can only contain /** and text! We also ignore last line as if it only contains */
NSString *prefixRegex = @"(?m:^\\s*\\*[ ]*)";
__block BOOL stripPrefix = ([comments count] > 1);
@@ -261,7 +267,7 @@ - (NSString *)commentValueFromString:(NSString *)value {
if (idx == [comments count]-1 && [stripped length] == 0) {
return;
}
- if (idx > 0 && ![stripped isMatchedByRegex:prefixRegex]) {
+ if ((!multiline || idx > 0) && ![stripped isMatchedByRegex:prefixRegex]) {
stripPrefix = NO;
*stop = YES;
}
@@ -293,13 +299,13 @@ - (void)resetComments {
- (GBComment *)lastComment {
if ([self.lastCommentBuilder length] == 0) return nil;
- NSString *value = [self commentValueFromString:self.lastCommentBuilder];
+ NSString *value = [self commentValueFromString:self.lastCommentBuilder isMultiline:self.isLastCommentMultiline];
return [GBComment commentWithStringValue:value sourceInfo:self.lastCommentSourceInfo];
}
- (GBComment *)previousComment {
if ([self.previousCommentBuilder length] == 0) return nil;
- NSString *value = [self commentValueFromString:self.previousCommentBuilder];
+ NSString *value = [self commentValueFromString:self.previousCommentBuilder isMultiline:self.isPreviousCommentMultiline];
return [GBComment commentWithStringValue:value sourceInfo:self.previousCommentSourceInfo];
}
@@ -331,6 +337,8 @@ - (NSArray *)allTokensFromTokenizer:(PKTokenizer *)tokenizer {
@synthesize previousComment;
@synthesize previousCommentBuilder;
@synthesize previousCommentSourceInfo;
+@synthesize isLastCommentMultiline;
+@synthesize isPreviousCommentMultiline;
@synthesize singleLineCommentRegex;
@synthesize multiLineCommentRegex;
@synthesize commentDelimiterRegex;
View
8 Testing/GBTokenizerTesting.m
@@ -315,7 +315,7 @@ - (void)testLastCommentString_shouldRemoveSuffixLine {
assertThat([tokenizer.lastComment stringValue], is(@"line"));
}
-- (void)testLastCommentString_shouldRemoveCommonPrefix {
+- (void)testLastCommentString_shouldRemoveCommonPrefixInMultilineComments {
GBTokenizer *tokenizer1 = [GBTokenizer tokenizerWithSource:[PKTokenizer tokenizerWithString:@"/** first\n * second */ ONE"] filename:@"file"];
GBTokenizer *tokenizer2 = [GBTokenizer tokenizerWithSource:[PKTokenizer tokenizerWithString:@"/** \n * first\n * second */ ONE"] filename:@"file"];
GBTokenizer *tokenizer3 = [GBTokenizer tokenizerWithSource:[PKTokenizer tokenizerWithString:@"/** \n * first\n * second\n */ ONE"] filename:@"file"];
@@ -325,6 +325,12 @@ - (void)testLastCommentString_shouldRemoveCommonPrefix {
assertThat([tokenizer3.lastComment stringValue], is(@"\nfirst\nsecond\n"));
}
+- (void)testLastCommentString_shouldKeepCommonPrefixInSingleLineComments {
+ GBTokenizer *tokenizer = [GBTokenizer tokenizerWithSource:[PKTokenizer tokenizerWithString:@"/// halo\n/// * first\n/// * second"] filename:@"file"];
+ // verify
+ assertThat([tokenizer.lastComment stringValue], is(@"halo\n* first\n* second"));
+}
+
- (void)testLastCommentString_shouldKeepExampleTabs {
// setup & execute
GBTokenizer *tokenizer = [GBTokenizer tokenizerWithSource:[PKTokenizer tokenizerWithString:@"/** line1\n\n\texample1\n\texample2\n\nline2 */\n ONE"] filename:@"file"];

0 comments on commit 95429b0

Please sign in to comment.
Something went wrong with that request. Please try again.