Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 504 lines (427 sloc) 20.017 kB
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
1 //
2 // GBTokenizer.m
3 // appledoc
4 //
5 // Created by Tomaz Kragelj on 25.7.10.
6 // Copyright (C) 2010, Gentle Bytes. All rights reserved.
7 //
8
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
9 #import "RegexKitLite.h"
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
10 #import "PKToken+GBToken.h"
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
11 #import "GBApplicationSettingsProvider.h"
49bc255 @tomaz Implemented debugDescription method for returning detailed object des…
authored
12 #import "GBSourceInfo.h"
13 #import "GBComment.h"
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
14 #import "GBTokenizer.h"
15
16 @interface GBTokenizer ()
17
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
18 - (BOOL)consumeComments;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
19 - (NSString *)commentValueFromString:(NSString *)value isMultiline:(BOOL)multiline;
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
20 - (NSString *)lineByPreprocessingHeaderDocDirectives:(NSString *)line;
21 - (NSArray *)linesByReorderingHeaderDocDirectives:(NSArray *)lines;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
22 - (NSArray *)allTokensFromTokenizer:(PKTokenizer *)tokenizer;
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
23 - (NSUInteger)offsetOfLineContainingOffset:(NSUInteger)offset;
24 - (NSInteger)indentationAtOffset:(NSUInteger)offset;
63ba09d @sptramer [ISSUE-439] First pass on converting appledoc from GC->ARC.
sptramer authored
25 @property (strong) NSString *filename;
26 @property (strong) NSString *input;
27 @property (strong) NSArray *tokens;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
28 @property (assign) NSUInteger tokenIndex;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
29 @property (assign) BOOL isLastCommentMultiline;
30 @property (assign) BOOL isPreviousCommentMultiline;
63ba09d @sptramer [ISSUE-439] First pass on converting appledoc from GC->ARC.
sptramer authored
31 @property (strong) NSMutableString *lastCommentBuilder;
32 @property (strong) NSMutableString *previousCommentBuilder;
33 @property (strong) PKToken *lastCommentToken;
34 @property (strong) PKToken *previousCommentToken;
35 @property (strong) NSString *singleLineCommentAfterRegex;
36 @property (strong) NSString *singleLineCommentRegex;
37 @property (strong) NSString *multiLineCommentRegex;
38 @property (strong) NSString *commentDelimiterRegex;
39 @property (strong) GBApplicationSettingsProvider *settings;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
40
41 @end
42
43 #pragma mark -
44
45 @implementation GBTokenizer
46
47 #pragma mark Initialization & disposal
48
2796c5a @tomaz Added filename information to GBTokenizer to make it accessible for a…
authored
49 + (id)tokenizerWithSource:(PKTokenizer *)tokenizer filename:(NSString *)filename {
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
50 return [self tokenizerWithSource:tokenizer filename:filename settings:nil];
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
51 }
52
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
53 + (id)tokenizerWithSource:(PKTokenizer *)tokenizer filename:(NSString *)filename settings:(id)settings {
63ba09d @sptramer [ISSUE-439] First pass on converting appledoc from GC->ARC.
sptramer authored
54 return [[self alloc] initWithSourceTokenizer:tokenizer filename:filename settings:settings];
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
55 }
56
a136f4d @tomaz Fixed compiler warnings after upgrading to Xcode 4.3. Closes #186.
authored
57 - (id)initWithSourceTokenizer:(PKTokenizer *)tokenizer filename:(NSString *)aFilename settings:(id)theSettings {
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
58 NSParameterAssert(tokenizer != nil);
a136f4d @tomaz Fixed compiler warnings after upgrading to Xcode 4.3. Closes #186.
authored
59 NSParameterAssert(aFilename != nil);
60 NSParameterAssert([aFilename length] > 0);
be8fe38 @tomaz Added line numbers when logging matched objects and comments.
authored
61 GBLogDebug(@"Initializing tokenizer...");
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
62 self = [super init];
63 if (self) {
a136f4d @tomaz Fixed compiler warnings after upgrading to Xcode 4.3. Closes #186.
authored
64 self.settings = theSettings;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
65 self.singleLineCommentAfterRegex = @"(?m-s:\\s*///<(.*)$)";
66 self.singleLineCommentRegex = @"(?m-s:\\s*///([^<].*)$)";
05b2e46 @tomaz Added basic support for headerdoc comments. Closes #95.
authored
67 self.multiLineCommentRegex = @"(?s:/\\*[*!](.*)\\*/)";
d7ece46 @tomaz Fixed HTML and XML handling within example blocks. Closes #54.
authored
68 self.commentDelimiterRegex = @"^[!@#$%^&*()_=+`~,<.>/?;:'\"-]{3,}$";
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
69 self.tokenIndex = 0;
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
70 self.lastCommentBuilder = [NSMutableString string];
71 self.previousCommentBuilder = [NSMutableString string];
a136f4d @tomaz Fixed compiler warnings after upgrading to Xcode 4.3. Closes #186.
authored
72 self.filename = aFilename;
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
73 self.input = tokenizer.string;
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
74 self.tokens = [self allTokensFromTokenizer:tokenizer];
75 [self consumeComments];
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
76 }
77 return self;
78 }
79
80 #pragma mark Tokenizing handling
81
82 - (PKToken *)lookahead:(NSUInteger)offset {
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
83 NSUInteger delta = 0;
84 NSUInteger counter = 0;
85 while (counter <= offset) {
86 NSUInteger index = self.tokenIndex + delta;
87 if (index >= [self.tokens count]) return [PKToken EOFToken];
88 if ([[self.tokens objectAtIndex:index] isComment]) {
89 delta++;
90 continue;
91 }
92 delta++;
93 counter++;
94 }
95 return [self.tokens objectAtIndex:self.tokenIndex + delta - 1];
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
96 }
97
3d2d2a8 @groue Added support for termination macros for method declaration and prope…
groue authored
98 - (void)lookaheadTo:(NSString *)end usingBlock:(void (^)(PKToken *token, BOOL *stop))block {
99 NSUInteger tokenCount = [self.tokens count];
100 BOOL quit = NO;
101 for (NSUInteger index = self.tokenIndex; index < tokenCount; ++index) {
102 PKToken *token = [self.tokens objectAtIndex:index];
103 if ([token isComment]) {
104 index++;
105 continue;
106 }
107 if ([token matches:end]) {
108 break;
109 }
110 block(token, &quit);
111 if (quit) break;
112 }
113 }
114
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
115 - (PKToken *)currentToken {
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
116 if ([self eof]) return [PKToken EOFToken];
117 return [self.tokens objectAtIndex:self.tokenIndex];
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
118 }
119
d1c6d80 Fixes issue #38
Daniel Höpfl authored
120 - (GBComment *)postfixCommentFrom:(PKToken *)startToken
121 {
122 NSString *postfixValue = nil;
123 BOOL isMultiline = NO;
124 if (self.tokenIndex > 0) {
125 NSUInteger pos = self.tokenIndex;
126 PKToken *token = nil;
127 do {
128 if (pos < self.tokens.count) {
129 token = [self.tokens objectAtIndex:pos];
130
131 NSArray *postfixLines = [[token stringValue] componentsMatchedByRegex:self.singleLineCommentAfterRegex capture:1];
132 if ([postfixLines count] > 0) {
133 NSString *value = [NSString string];
134 for (NSString *match in postfixLines) value = [value stringByAppendingString:match];
135
136 if (postfixValue) {
137 postfixValue = [@"\n" stringByAppendingString:postfixValue];
138 postfixValue = [value stringByAppendingString:postfixValue];
139 isMultiline = YES;
140 }
141 else postfixValue = value;
142 }
143 }
144 --pos;
145 } while (pos && startToken != token);
146 }
147 if (!postfixValue) return nil;
148
149 postfixValue = [self commentValueFromString:postfixValue isMultiline:isMultiline];
150 GBSourceInfo *sourceInfo = [self sourceInfoForToken:startToken];
151
152 return [GBComment commentWithStringValue:postfixValue sourceInfo:sourceInfo];
153 }
154
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
155 - (void)consume:(NSUInteger)count {
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
156 if (count == 0) return;
157 while (count > 0 && ![self eof]) {
158 self.tokenIndex++;
c190eaf @tomaz Implemented methods comments registration.
authored
159 [self consumeComments];
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
160 count--;
161 }
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
162 }
163
8bebbe8 @tomaz Implemented class declaration parsing and renamed GBTokenizer consume…
authored
164 - (void)consumeTo:(NSString *)end usingBlock:(void (^)(PKToken *token, BOOL *consume, BOOL *stop))block {
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
165 [self consumeFrom:nil to:end usingBlock:block];
166 }
167
dad75ed @tomaz Implemented simple block properties and method arguments parsing.
authored
168 - (void)consumeFrom:(NSString *)start to:(NSString *)end usingBlock:(void (^)(PKToken *token, BOOL *consume, BOOL *stop))block {
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
169 // Skip starting token.
170 if (start) {
171 if (![[self currentToken] matches:start]) return;
172 [self consume:1];
173 }
174
175 // Report all tokens until EOF or ending token is found.
dad75ed @tomaz Implemented simple block properties and method arguments parsing.
authored
176 NSUInteger level = 1;
72ab97c @tomaz Refactored GBTokenizer block consuming method to allow quitting at an…
authored
177 BOOL quit = NO;
dad75ed @tomaz Implemented simple block properties and method arguments parsing.
authored
178 while (![self eof]) {
179 // Handle multiple hierarchy.
180 if (start && [[self currentToken] matches:start]) level++;
181 if ([[self currentToken] matches:end]) {
182 if (!start) break;
183 if (--level == 0) break;
184 }
185
186 // Report the token.
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
187 BOOL consume = YES;
46f88c5 @0xced Fix call to nil block
0xced authored
188 if (block) block([self currentToken], &consume, &quit);
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
189 if (consume) [self consume:1];
72ab97c @tomaz Refactored GBTokenizer block consuming method to allow quitting at an…
authored
190 if (quit) break;
28a51ce @tomaz Implemented class parsing for class name, superclass name, adopted pr…
authored
191 }
192
193 // Skip ending token if found.
194 if ([[self currentToken] matches:end]) [self consume:1];
195 }
196
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
197 - (BOOL)eof {
198 return (self.tokenIndex >= [self.tokens count]);
199 }
200
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
201 #pragma mark Token information handling
202
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
203 - (GBSourceInfo *)sourceInfoForCurrentToken {
204 return [self sourceInfoForToken:[self currentToken]];
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
205 }
206
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
207 - (GBSourceInfo *)sourceInfoForToken:(PKToken *)token {
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
208 NSParameterAssert(token != nil);
7d6a323 @tomaz Copied all the changes from the old generating branch.
authored
209 NSUInteger lines = [self.input numberOfLinesInRange:NSMakeRange(0, [token offset])];
210 return [GBSourceInfo infoWithFilename:self.filename lineNumber:lines];
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
211 }
212
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
213 #pragma mark Comments handling
214
215 - (BOOL)consumeComments {
0bb1c7d @tomaz Enabled line wrapping and updated comments to fit better.
authored
216 // This method checks if current token is a comment and consumes all comments until non-comment token is detected or EOF reached. The result of the method is that current index is positioned on the first non-comment token. If current token is not comment, the method doesn't do anything, but simply returns NO to indicate it didn't find a comment and therefore it didn't move current token. This is also where we do initial comments handling such as removing starting and ending chars etc.
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
217 if ([self eof]) return NO;
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
218
716231f @Coeur Fixing 'unused' warning
Coeur authored
219 //PKToken *startingPreviousToken = nil;
220 //PKToken *startingLastToken = nil;
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
221 NSUInteger previousSingleLineEndOffset = NSNotFound;
222 NSInteger previousSingleLineIndentation = -1;
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
223 while (![self eof] && [[self currentToken] isComment]) {
224 PKToken *token = [self currentToken];
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
225 NSString *value = nil;
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
226
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
227 // Match single line comments. Note that we can simplify the code with assumption that there's only one single line comment per match. If regex finds more (should never happen though), we simply combine them together. Then we check if the comment is a continuation of previous single liner by testing the string offset and indentation. If so we group the values together, otherwise we create a new single line comment. Finally we remember current comment offset to allow grouping of next single line comment.
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
228 NSArray *singleLiners = [[token stringValue] componentsMatchedByRegex:self.singleLineCommentRegex capture:1];
229 if ([singleLiners count] > 0) {
230 value = [NSString string];
231 for (NSString *match in singleLiners) value = [value stringByAppendingString:match];
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
232 NSInteger tokenIndentation = [self indentationAtOffset:[token offset]];
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
233 BOOL isContinuingPreviousSingleLiner = ([token offset] == previousSingleLineEndOffset + 1);
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
234 if (!isContinuingPreviousSingleLiner && previousSingleLineIndentation > 0 && tokenIndentation == previousSingleLineIndentation) {
235 isContinuingPreviousSingleLiner = ([token offset] == previousSingleLineEndOffset + previousSingleLineIndentation + 1);
236 }
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
237 if (isContinuingPreviousSingleLiner) {
238 [self.lastCommentBuilder appendString:@"\n"];
239 } else {
240 [self.previousCommentBuilder setString:self.lastCommentBuilder];
716231f @Coeur Fixing 'unused' warning
Coeur authored
241 //startingPreviousToken = startingLastToken;
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
242 [self.lastCommentBuilder setString:@""];
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
243 self.isPreviousCommentMultiline = self.isLastCommentMultiline;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
244 self.previousCommentToken = self.lastCommentToken;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
245 self.isLastCommentMultiline = NO;
716231f @Coeur Fixing 'unused' warning
Coeur authored
246 //startingLastToken = token;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
247 self.lastCommentToken = token;
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
248 }
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
249 previousSingleLineEndOffset = [token offset] + [[token stringValue] length];
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
250 previousSingleLineIndentation = tokenIndentation;
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
251 }
d1c6d80 Fixes issue #38
Daniel Höpfl authored
252
c772576 @tomaz Implemented last stand-alone comment parsing.
authored
253 // Match multiple line comments and only process last (in reality we should only have one comment in each mutliline comment token, but let's handle any strange cases graceosly).
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
254 else {
255 NSArray *multiLiners = [[token stringValue] componentsMatchedByRegex:self.multiLineCommentRegex capture:1];
256 value = [multiLiners lastObject];
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
257 [self.previousCommentBuilder setString:self.lastCommentBuilder];
716231f @Coeur Fixing 'unused' warning
Coeur authored
258 //startingPreviousToken = startingLastToken;
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
259 [self.lastCommentBuilder setString:@""];
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
260 self.isPreviousCommentMultiline = self.isLastCommentMultiline;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
261 self.previousCommentToken = self.lastCommentToken;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
262 self.isLastCommentMultiline = YES;
716231f @Coeur Fixing 'unused' warning
Coeur authored
263 //startingLastToken = token;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
264 self.lastCommentToken = token;
265 }
266
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
267 // Append string value to current comment and proceed with next token.
dbf7903 @lhunath Fixed a crash when trying to append a nil value.
lhunath authored
268 if (value)
269 [self.lastCommentBuilder appendString:value];
d1c6d80 Fixes issue #38
Daniel Höpfl authored
270
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
271 self.tokenIndex++;
9a499e4 @tomaz Fixed comment source info handling when two consequtive comments are …
authored
272 }
d1c6d80 Fixes issue #38
Daniel Höpfl authored
273
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
274 // If last comment contains @name, we should assign it to previous one and reset current! This should ideally be handled by higher level component, but it's simplest to do it here. Note that we don't deal with source info here, we'll do immediately after this as long as we properly setup tokens.
275 if (self.settings && [self.lastCommentBuilder isMatchedByRegex:self.settings.commentComponents.methodGroupRegex]) {
276 self.previousCommentBuilder = [self.lastCommentBuilder mutableCopy];
277 [self.lastCommentBuilder setString:@""];
716231f @Coeur Fixing 'unused' warning
Coeur authored
278 //startingPreviousToken = startingLastToken;
279 //startingLastToken = nil;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
280 self.previousCommentToken = self.lastCommentToken;
281 self.lastCommentToken = nil;
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
282 }
d1c6d80 Fixes issue #38
Daniel Höpfl authored
283
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
284 return YES;
285 }
286
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
287 - (NSString *)commentValueFromString:(NSString *)value isMultiline:(BOOL)multiline {
c772576 @tomaz Implemented last stand-alone comment parsing.
authored
288 if ([value length] == 0) return nil;
289 NSArray *lines = [value componentsSeparatedByCharactersInSet:[NSCharacterSet newlineCharacterSet]];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
290 NSMutableArray *strippedLines = [NSMutableArray arrayWithCapacity:[lines count]];
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
291
7c06615 @tomaz Refactored tokenizer to use regex for parsing comment delimiters.
authored
292 // First pass: removes delimiters. We simply detect 3+ delimiter chars in any combination. If removing delimiter yields empty line, discard it.
4bb10f2 @tomaz Updated comments parsing to use regular expressions instead of NSStri…
authored
293 [lines enumerateObjectsUsingBlock:^(NSString *line, NSUInteger idx, BOOL *stop) {
294 NSString *stripped = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
7c06615 @tomaz Refactored tokenizer to use regex for parsing comment delimiters.
authored
295 NSString *delimited = [stripped stringByReplacingOccurrencesOfRegex:self.commentDelimiterRegex withString:@""];
296 if ([stripped length] > [delimited length]) {
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
297 if ([delimited length] > 0) [strippedLines addObject:delimited];
7c06615 @tomaz Refactored tokenizer to use regex for parsing comment delimiters.
authored
298 return;
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
299 }
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
300 [strippedLines addObject:line];
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
301 }];
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
302
4bb10f2 @tomaz Updated comments parsing to use regular expressions instead of NSStri…
authored
303 // If all lines start with a *, ignore the prefix. Note that we ignore first line as it can only contain /** and text! We also ignore last line as if it only contains */
3b97103 @tomaz Fixed comment delimiters detection regex and cleaned up some code.
authored
304 NSString *prefixRegex = @"(?m:^\\s*\\*[ ]*)";
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
305 __block BOOL stripPrefix = ([strippedLines count] > 1);
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
306 if (stripPrefix) {
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
307 [strippedLines enumerateObjectsUsingBlock:^(NSString *line, NSUInteger idx, BOOL *stop) {
4bb10f2 @tomaz Updated comments parsing to use regular expressions instead of NSStri…
authored
308 NSString *stripped = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
309 if (idx == [strippedLines count]-1 && [stripped length] == 0) {
4bb10f2 @tomaz Updated comments parsing to use regular expressions instead of NSStri…
authored
310 return;
311 }
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
312 if ((!multiline || idx > 0) && ![stripped isMatchedByRegex:prefixRegex]) {
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
313 stripPrefix = NO;
314 *stop = YES;
315 }
316 }];
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
317 }
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
318
319 // Preprocess header doc directives.
320 NSArray *preprocessedLines = [self linesByReorderingHeaderDocDirectives:strippedLines];
0da48e1 Improved support for HeaderDocs
Felix Lamouroux authored
321
d7ece46 @tomaz Fixed HTML and XML handling within example blocks. Closes #54.
authored
322 // Finally remove common line prefix and a single prefix space (but leave multiple spaces to properly handle space prefixed example blocks!) and compose all objects into final comment.
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
323 NSCharacterSet *spacesSet = [NSCharacterSet characterSetWithCharactersInString:@" "];
d7ece46 @tomaz Fixed HTML and XML handling within example blocks. Closes #54.
authored
324 NSString *spacesPrefixRegex = @"^ {2,}";
325 NSString *tabPrefixRegex = @"^\t";
c772576 @tomaz Implemented last stand-alone comment parsing.
authored
326 NSMutableString *result = [NSMutableString stringWithCapacity:[value length]];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
327 [preprocessedLines enumerateObjectsUsingBlock:^(NSString *line, NSUInteger idx, BOOL *stop) {
d7ece46 @tomaz Fixed HTML and XML handling within example blocks. Closes #54.
authored
328 if (stripPrefix) line = [line stringByReplacingOccurrencesOfRegex:prefixRegex withString:@""];
329 if (![line isMatchedByRegex:spacesPrefixRegex] && ![line isMatchedByRegex:tabPrefixRegex]) line = [line stringByTrimmingCharactersInSet:spacesSet];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
330 line = [self lineByPreprocessingHeaderDocDirectives:line];
4bb10f2 @tomaz Updated comments parsing to use regular expressions instead of NSStri…
authored
331 [result appendString:line];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
332 if (idx < [strippedLines count] - 1) [result appendString:@"\n"];
1aa458c @tomaz Implemented comment prefixes and suffixes handling.
authored
333 }];
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
334
23666fb @tomaz Fixed comments reporting when empty comment is detected.
authored
335 // If the result is empty string, return nil, otherwise return the comment string.
336 if ([result length] == 0) return nil;
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
337 return result;
338 }
339
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
340 - (NSString *)lineByPreprocessingHeaderDocDirectives:(NSString *)line {
341 if (!self.settings.preprocessHeaderDoc) return line;
342
343 // Remove the entire line when it contains @method or property or class.
3f0d4c7 @bulusoy Ability to parse @discussion @abstract @brief @details
bulusoy authored
344 //line = [line stringByReplacingOccurrencesOfRegex:@"(?m:@(protocol|method|property|class).*$)" withString:@""];
d863afd @jerryhjones Also preprocess doxygen tags
jerryhjones authored
345
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
346 // Remove unsupported headerDoc words.
3f0d4c7 @bulusoy Ability to parse @discussion @abstract @brief @details
bulusoy authored
347 //line = [line stringByReplacingOccurrencesOfRegex:@"(?m:^\\s*@(discussion|abstract))\\s?" withString:@"\n"];
d863afd @jerryhjones Also preprocess doxygen tags
jerryhjones authored
348
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
349 // Replace methodgroup with name.
350 line = [line stringByReplacingOccurrencesOfRegex:@"(?:@(methodgroup|group))" withString:@"@name"];
3de683a @jerryhjones Improved regular expressions for handling @brief and @details
jerryhjones authored
351
352 // Remove unsupported Doxygen words. This should ease the pain of migrating large amount of comments using doxygen markup.
353 // Comments like the following are cleaned up, and made ready for the markup appledoc expects
354
355 /**
356 @brief Brief Comment
357 @details Detailed Comment.
358 */
359
360 // Becomes....
361
362 /**
363 Brief Comment
364
365 Detailed Comment.
366 */
367
368
3f0d4c7 @bulusoy Ability to parse @discussion @abstract @brief @details
bulusoy authored
369 line = [line stringByReplacingOccurrencesOfRegex:@"(?m:^\\s*@updated).*$?" withString:@"\n"];
370
3de683a @jerryhjones Improved regular expressions for handling @brief and @details
jerryhjones authored
371 // Removes any occurance of @brief and it's surrounding whitespace
3f0d4c7 @bulusoy Ability to parse @discussion @abstract @brief @details
bulusoy authored
372 //line = [line stringByReplacingOccurrencesOfRegex:@"\\s*@brief\\s*" withString:@""];
3de683a @jerryhjones Improved regular expressions for handling @brief and @details
jerryhjones authored
373
374 // Replaces any occurance of @details and it's surrounding whitespace with a newline
3f0d4c7 @bulusoy Ability to parse @discussion @abstract @brief @details
bulusoy authored
375 //line = [line stringByReplacingOccurrencesOfRegex:@"^\\s*@details\\s*" withString:@"\n"];
3de683a @jerryhjones Improved regular expressions for handling @brief and @details
jerryhjones authored
376
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
377 return line;
378 }
379
380 - (NSArray *)linesByReorderingHeaderDocDirectives:(NSArray *)lines {
d6db7d8 proper way to test for MacOSX Lion
Pascal Drouilly authored
381 #if __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
382 if (!self.settings.preprocessHeaderDoc) return lines;
383
384 // Make sure that @param and @return is placed at the end (after abstract etc.)
385 NSMutableArray *reorderedParams = [NSMutableArray array];
386 NSMutableArray *reorderedNonParams = [NSMutableArray array];
387 NSRegularExpression *directiveExpression = [NSRegularExpression regularExpressionWithPattern:@"^\\s*@(param|result|return)" options:NSRegularExpressionDotMatchesLineSeparators error:nil];
388 NSRegularExpression *lineExpression = [NSRegularExpression regularExpressionWithPattern:@"^\\s*@[a-z]" options:NSRegularExpressionDotMatchesLineSeparators error:nil];
389
390 BOOL isParamBlock = NO;
391 for (NSString *line in lines) {
392 if ([directiveExpression numberOfMatchesInString:line options:0 range:NSMakeRange(0, [line length])] > 0) {
393 isParamBlock = YES;
394 } else if ([lineExpression numberOfMatchesInString:line options:0 range:NSMakeRange(0, [line length])] > 0) {
395 isParamBlock = NO;
396 }
397
398 if (isParamBlock) {
399 [reorderedParams addObject:line];
400 } else {
401 [reorderedNonParams addObject:line];
402 }
403 }
404
405 [reorderedNonParams addObjectsFromArray:reorderedParams];
406 return reorderedNonParams;
89da786 Conditionally include code in method linesByReorderingHeaderDocDirect…
Pascal Drouilly authored
407 #else
408 return lines;
409 #endif
2e063c0 @tomaz Cleaned up HeaderDoc parsing code and made it optional. References #9…
authored
410 }
411
4262bb8 @tomaz Fixed comments parsing for non-trivial sources. Closes #43.
authored
412 - (void)resetComments {
1bf7946 @tomaz Fixed undocumented methods and properties parsing. Closes #57.
authored
413 GBLogDebug(@"Resetting comments...");
414 [self.lastCommentBuilder setString:@""];
415 [self.previousCommentBuilder setString:@""];
4262bb8 @tomaz Fixed comments parsing for non-trivial sources. Closes #43.
authored
416 }
417
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
418 - (GBComment *)lastComment {
6d5bed5 @tomaz Refactored log messages to yield better information.
authored
419 if ([self.lastCommentBuilder length] == 0) return nil;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
420 NSString *value = [self commentValueFromString:self.lastCommentBuilder isMultiline:self.isLastCommentMultiline];
d1c6d80 Fixes issue #38
Daniel Höpfl authored
421 GBSourceInfo *sourceInfo = [self sourceInfoForToken:self.lastCommentToken];
422 return [GBComment commentWithStringValue:value sourceInfo:sourceInfo];
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
423 }
424
425 - (GBComment *)previousComment {
6d5bed5 @tomaz Refactored log messages to yield better information.
authored
426 if ([self.previousCommentBuilder length] == 0) return nil;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
427 NSString *value = [self commentValueFromString:self.previousCommentBuilder isMultiline:self.isPreviousCommentMultiline];
d1c6d80 Fixes issue #38
Daniel Höpfl authored
428 GBSourceInfo *sourceInfo = [self sourceInfoForToken:self.previousCommentToken];
429 return [GBComment commentWithStringValue:value sourceInfo:sourceInfo];
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
430 }
431
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
432 #pragma mark Helper methods
433
434 - (NSArray *)allTokensFromTokenizer:(PKTokenizer *)tokenizer {
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
435 // Return all appledoc comments too, but ignore ordinary C comments!
436 BOOL reportsComments = tokenizer.commentState.reportsCommentTokens;
437 tokenizer.commentState.reportsCommentTokens = YES;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
438 NSMutableArray *result = [NSMutableArray array];
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
439 PKToken *token;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
440 while ((token = [tokenizer nextToken]) != [PKToken EOFToken]) {
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
441 if ([token isComment] && ![token isAppledocComment]) continue;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
442 [result addObject:token];
443 }
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
444 tokenizer.commentState.reportsCommentTokens = reportsComments;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
445 return result;
446 }
447
2eb41ab @idpaterson Allows single-line comments spanning multiple lines to be grouped if …
idpaterson authored
448 - (NSUInteger)offsetOfLineContainingOffset:(NSUInteger)offset {
449 // This method returns the offset of the first character in the line
450 // containing the character at the specific offset.
451 NSRange newlineRange = [self.input rangeOfCharacterFromSet:[NSCharacterSet newlineCharacterSet]
452 options:NSBackwardsSearch
453 range:NSMakeRange(0, offset)];
454 if (newlineRange.location != NSNotFound) {
455 return newlineRange.location + 1;
456 }
457 // First line
458 return 0;
459 }
460
461 - (NSInteger)indentationAtOffset:(NSUInteger)offset {
462 // This method returns the number of tab or space characters preceding the
463 // offset if and only if it is only preceded by such indentation characters,
464 // otherwise returns -1.
465 NSUInteger lineOffset = [self offsetOfLineContainingOffset:offset];
466 NSRange lineToOffsetRange = NSMakeRange(lineOffset, offset - lineOffset);
467
468 // Short-circuit logic if offset is at the start of the line
469 if (lineToOffsetRange.length == 0) {
470 return 0;
471 }
472
473 NSCharacterSet * nonWhitespace = [[NSCharacterSet whitespaceCharacterSet] invertedSet];
474 NSRange nonWhitespaceRange = [self.input rangeOfCharacterFromSet:nonWhitespace
475 options:0
476 range:lineToOffsetRange];
477 // Line contains only whitespace preceding the offset: indentation
478 if (nonWhitespaceRange.location == NSNotFound) {
479 return lineToOffsetRange.length;
480 }
481 return -1;
482 }
483
484
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
485 #pragma mark Properties
486
2796c5a @tomaz Added filename information to GBTokenizer to make it accessible for a…
authored
487 @synthesize filename;
6cfd48f @tomaz Refactored file data creation by moving the method to GBTokenizer.
authored
488 @synthesize input;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
489 @synthesize tokens;
490 @synthesize tokenIndex;
47e2c01 @tomaz Implemented basic comments parsing and tests.
authored
491 @synthesize lastComment;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
492 @synthesize lastCommentToken;
c772576 @tomaz Implemented last stand-alone comment parsing.
authored
493 @synthesize previousComment;
9047fcf @tomaz Refactored comment handling with GBModelBase by removing registerComm…
authored
494 @synthesize previousCommentBuilder;
d1c6d80 Fixes issue #38
Daniel Höpfl authored
495 @synthesize previousCommentToken;
95429b0 @tomaz Changed how single/multi line comments are parsed. Partially addresse…
authored
496 @synthesize isLastCommentMultiline;
497 @synthesize isPreviousCommentMultiline;
10fc1a2 @tomaz Replaced comments stripping with regular expressions.
authored
498 @synthesize singleLineCommentRegex;
499 @synthesize multiLineCommentRegex;
7c06615 @tomaz Refactored tokenizer to use regex for parsing comment delimiters.
authored
500 @synthesize commentDelimiterRegex;
c68cb86 @tomaz Implemented proper handling of @name sections in more complex circums…
authored
501 @synthesize settings;
84598f7 @tomaz Implemented unit testing target and basic class parsing tests.
authored
502
503 @end
Something went wrong with that request. Please try again.