Skip to content

JDK-8305100 [REDO] Clean up JavadocTokenizer #13265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
JimLaskey wants to merge 8 commits into from
Closed

JDK-8305100 [REDO] Clean up JavadocTokenizer #13265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ba776a4
Clean up JavadocTokenizer
Mar 28, 2023
28ddb83
Merge branch 'master' into 8305100
Mar 28, 2023
cda5a5a
Clean up JavadocTokenizer
Mar 28, 2023
c3e9b00
Skip whitespace before @deprecated
Mar 29, 2023
83e68b3
Clean up end of comment detection
Mar 29, 2023
c0860de
Handle line terminators.
Mar 29, 2023
0fd05c9
Tweaks to get tests running
Mar 30, 2023
a9d5ba0
Correct position for end of line
Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 102 additions & 86 deletions src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,15 +307,6 @@ protected boolean acceptOneOfThenPut(char ch1, char ch2) {
return false;
}

/**
* Test if the current character is a line terminator.
*
* @return true if current character is a line terminator.
*/
private boolean isEOLN() {
return isOneOf('\n', '\r');
}

/**
* Skip and process a line terminator sequence.
*/
Expand Down Expand Up @@ -1094,7 +1085,7 @@ public Token readToken() {
if (scannerDebug) {
System.out.println("nextToken(" + pos
+ "," + endPos + ")=|" +
new String(getRawCharacters(pos, endPos))
getRawString(pos, endPos)
+ "|");
}
}
Expand Down Expand Up @@ -1146,13 +1137,11 @@ protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style)
if (scannerDebug) {
System.out.println("processComment(" + pos
+ "," + endPos + "," + style + ")=|"
+ new String(getRawCharacters(pos, endPos))
+ getRawString(pos, endPos)
+ "|");
}

char[] buf = getRawCharacters(pos, endPos);

return new BasicComment(style, fac, buf, pos);
return new BasicComment(style,this, pos, endPos);
}

/**
Expand All @@ -1167,8 +1156,8 @@ protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style)
protected void processWhiteSpace(int pos, int endPos) {
if (scannerDebug) {
System.out.println("processWhitespace(" + pos
+ "," + endPos + ")=|" +
new String(getRawCharacters(pos, endPos))
+ "," + endPos + ")=|"
+ getRawString(pos, endPos)
+ "|");
}
}
Expand All @@ -1182,8 +1171,8 @@ protected void processWhiteSpace(int pos, int endPos) {
protected void processLineTerminator(int pos, int endPos) {
if (scannerDebug) {
System.out.println("processTerminator(" + pos
+ "," + endPos + ")=|" +
new String(getRawCharacters(pos, endPos))
+ "," + endPos + ")=|"
+ getRawString(pos, endPos)
+ "|");
}
}
Expand All @@ -1206,9 +1195,6 @@ public Position.LineMap getLineMap() {
protected static class BasicComment extends PositionTrackingReader implements Comment {
/**
* Style of comment
* LINE starting with //
* BLOCK starting with /*
* JAVADOC starting with /**
*/
CommentStyle cs;

Expand All @@ -1225,13 +1211,13 @@ protected static class BasicComment extends PositionTrackingReader implements Co
/**
* Constructor.
*
* @param cs comment style
* @param sf Scan factory.
* @param array Array containing contents of source.
* @param offset Position offset in original source buffer.
* @param cs comment style
* @param reader existing reader
* @param pos start of meaningful content in buffer.
* @param endPos end of meaningful content in buffer.
*/
protected BasicComment(CommentStyle cs, ScannerFactory sf, char[] array, int offset) {
super(sf, array, offset);
protected BasicComment(CommentStyle cs, UnicodeReader reader, int pos, int endPos) {
super(reader, pos, endPos);
this.cs = cs;
}

Expand All @@ -1247,8 +1233,7 @@ public String getText() {
/**
* Return buffer position in original buffer mapped from buffer position in comment.
*
* @param pos buffer position in comment.
*
* @param pos buffer position in comment.
* @return buffer position in original buffer.
*/
public int getSourcePos(int pos) {
Expand All @@ -1257,11 +1242,8 @@ public int getSourcePos(int pos) {

/**
* Return style of comment.
* LINE starting with //
* BLOCK starting with /*
* JAVADOC starting with /**
*
* @return
* @return style of comment.
*/
public CommentStyle getStyle() {
return cs;
Expand All @@ -1273,76 +1255,110 @@ public CommentStyle getStyle() {
* @return true if comment contains @deprecated.
*/
public boolean isDeprecated() {
if (!scanned && cs == CommentStyle.JAVADOC) {
if (!scanned) {
scanDocComment();
}

return deprecatedFlag;
}

/**
* Scan JAVADOC comment for details.
* Remove closing star(s) slash from comment.
*
* @param line line reader
*
* @return new line reader if detected otherwise original line reader.
*/
protected void scanDocComment() {
try {
boolean deprecatedPrefix = false;
accept("/**");
UnicodeReader trimEndOfComment(UnicodeReader line) {
int pos = line.position();
boolean allWhitespace = true;

forEachLine:
while (isAvailable()) {
// Skip optional WhiteSpace at beginning of line
skipWhitespace();
while (line.isAvailable()) {
int endPos = line.position();

// Skip optional consecutive Stars
while (accept('*')) {
if (is('/')) {
return;
}
}
if (line.skip('*') != 0 && line.is('/')) {
return line.lineReader(allWhitespace ? endPos : pos, endPos);
} else {
allWhitespace = allWhitespace && line.isWhitespace();
line.next();
}
}

// Skip optional WhiteSpace after Stars
skipWhitespace();
line.reset(pos);

// At beginning of line in the JavaDoc sense.
deprecatedPrefix = deprecatedFlag || accept("@deprecated");
return line;
}

if (deprecatedPrefix && isAvailable()) {
if (Character.isWhitespace(get())) {
deprecatedFlag = true;
} else if (accept('*')) {
if (is('/')) {
deprecatedFlag = true;
return;
}
}
}
/**
* Trim the first part of the JavaDoc comment.
*
* @param line line reader
*
* @return modified line reader
*/
UnicodeReader trimJavadocComment(UnicodeReader line) {
line = trimEndOfComment(line);
int pos = line.position();
line.skipWhitespace();

// Skip rest of line
while (isAvailable()) {
switch (get()) {
case '*':
next();
if (!line.isAvailable()) {
return line;
}

if (is('/')) {
return;
}
if (line.skip('*') == 0) {
line.reset(pos);
}

break;
case '\r': // (Spec 3.4)
case '\n': // (Spec 3.4)
accept('\r');
accept('\n');
continue forEachLine;
return line;
}

default:
next();
break;
}
} // rest of line
} // forEachLine
return;
} finally {
/**
* Put the line into the buffer.
*
* @param line line reader
*/
protected void putLine(UnicodeReader line) {
// ignore, overridden in subclass
}

/**
* Scan document comment for content.
*/
protected void scanDocComment() {
if (!scanned) {
deprecatedFlag = false;
scanned = true;

if (!accept("/**")) {
return;
}

skip('*');
skipWhitespace();

if (isEOLN()) {
accept('\r');
accept('\n');
}

while (isAvailable()) {
UnicodeReader line = lineReader();
line = trimJavadocComment(line);

// If standalone @deprecated tag
int pos = line.position();
line.skipWhitespace();

if (line.accept("@deprecated") &&
(!line.isAvailable() ||
line.isWhitespace() ||
line.isEOLN() ||
line.get() == EOI)) {
deprecatedFlag = true;
}
Comment on lines +1351 to +1357
Copy link
Contributor

@jonathan-gibbons jonathan-gibbons Mar 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As mentioned recently elsewhere, this is generally insufficient, albeit no worse than before.
Generally, we should move away from this code, and eliminate deprecated Flag entirely. Deprecation is (or should be) indicated by the @Deprecated annotation, with associated documentation given by a plain/regular @deprecated tag, as read by DocCommentParser or equivalent.


line.reset(pos);
putLine(line);
}
}
}
}
Expand Down
Loading