Permalink
Browse files

Issue #3729: Reevaluate tokens in IllegalTokenText

  • Loading branch information...
Vladlis authored and rnveach committed Feb 16, 2017
1 parent e60bf18 commit cd88872f9db17c838641bcd32ad7f334893827c0
@@ -23,8 +23,8 @@
import com.puppycrawl.tools.checkstyle.api.AbstractCheck;
import com.puppycrawl.tools.checkstyle.api.DetailAST;
import com.puppycrawl.tools.checkstyle.api.TokenTypes;
import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
import com.puppycrawl.tools.checkstyle.utils.TokenUtils;
/**
* <p>
@@ -82,7 +82,16 @@
@Override
public int[] getAcceptableTokens() {
return TokenUtils.getAllTokenIds();
return new int[] {
TokenTypes.NUM_DOUBLE,
TokenTypes.NUM_FLOAT,
TokenTypes.NUM_INT,
TokenTypes.NUM_LONG,
TokenTypes.IDENT,
TokenTypes.COMMENT_CONTENT,
TokenTypes.STRING_LITERAL,
TokenTypes.CHAR_LITERAL,
};
}
@Override
@@ -102,6 +102,14 @@ public static int getIntFromField(Field field, Object object) {
}
}
/**
* Get total number of TokenTypes.
* @return total number of TokenTypes.
*/
public static int getTokenTypesTotalNumber() {
return TOKEN_IDS.length;
}
/**
* Get all token IDs that are available in TokenTypes.
* @return array of token IDs
@@ -23,12 +23,16 @@
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import com.puppycrawl.tools.checkstyle.BaseCheckTestSupport;
import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
import com.puppycrawl.tools.checkstyle.api.TokenTypes;
import com.puppycrawl.tools.checkstyle.utils.TokenUtils;
public class IllegalTokenTextCheckTest
extends BaseCheckTestSupport {
@@ -122,4 +126,29 @@ public void testCommentToken()
verify(checkConfig, getPath("InputIllegalTokens.java"), expected);
}
@Test
public void testAcceptableTokensMakeSense() {
final int expectedTokenTypesTotalNumber = 169;
Assert.assertEquals("Total number of TokenTypes has changed, acceptable tokens in"
+ " IllegalTokenTextCheck need to be reconsidered.",
expectedTokenTypesTotalNumber, TokenUtils.getTokenTypesTotalNumber());
final IllegalTokenTextCheck check = new IllegalTokenTextCheck();
final int[] allowedTokens = check.getAcceptableTokens();
final List<Integer> tokenTypesWithMutableText = Arrays.asList(
TokenTypes.NUM_DOUBLE,
TokenTypes.NUM_FLOAT,
TokenTypes.NUM_INT,
TokenTypes.NUM_LONG,
TokenTypes.IDENT,
TokenTypes.COMMENT_CONTENT,
TokenTypes.STRING_LITERAL,
TokenTypes.CHAR_LITERAL
);
for (int tokenType : allowedTokens) {
Assert.assertTrue(TokenUtils.getTokenName(tokenType) + " should not be allowed"
+ " in this check as its text is a constant (IllegalTokenCheck should be used for"
+ " such cases).", tokenTypesWithMutableText.contains(tokenType));
}
}
}
@@ -119,43 +119,9 @@
Collectors.toSet()));
// we have no need to block specific token text
CHECKSTYLE_TOKENS_IN_CONFIG_TO_IGNORE.put("IllegalTokenText",
Stream.of("LITERAL_SUPER", "LITERAL_ASSERT", "ENUM_CONSTANT_DEF",
"TYPE_PARAMETERS", "TYPE_UPPER_BOUNDS", "NUM_DOUBLE", "LITERAL_SWITCH",
"ANNOTATIONS", "LITERAL_SHORT", "LITERAL_PROTECTED", "FOR_CONDITION",
"FOR_INIT", "LITERAL_LONG", "MINUS", "OBJBLOCK", "LITERAL_NULL",
"ANNOTATION", "LITERAL_TRUE", "COMMENT_CONTENT", "LITERAL_CHAR",
"PARAMETER_DEF", "POST_DEC", "ANNOTATION_FIELD_DEF", "BLOCK_COMMENT_END",
"TYPE", "LITERAL_INT", "BSR", "ENUM", "LABELED_STAT",
"ANNOTATION_MEMBER_VALUE_PAIR", "TYPECAST", "LITERAL_SYNCHRONIZED",
"PLUS_ASSIGN", "DOT", "LPAREN", "LITERAL_IF", "LITERAL_CATCH", "BAND",
"INTERFACE_DEF", "LOR", "BNOT", "METHOD_CALL", "AT", "ELLIPSIS",
"ARRAY_INIT", "FOR_EACH_CLAUSE", "LITERAL_THROWS", "CHAR_LITERAL",
"CASE_GROUP", "POST_INC", "SEMI", "LITERAL_FINALLY", "ASSIGN",
"RESOURCE_SPECIFICATION", "STATIC_IMPORT", "GENERIC_START", "IMPORT", "SL",
"VARIABLE_DEF", "LITERAL_DOUBLE", "RCURLY", "RESOURCE", "SR", "COMMA",
"BAND_ASSIGN", "METHOD_DEF", "LITERAL_VOID", "NUM_LONG",
"LITERAL_TRANSIENT", "LITERAL_THIS", "LCURLY", "MINUS_ASSIGN",
"TYPE_LOWER_BOUNDS", "TYPE_ARGUMENT", "LITERAL_CLASS", "INSTANCE_INIT",
"DIV", "LITERAL_VOLATILE", "STAR", "UNARY_MINUS", "FOR_ITERATOR",
"NOT_EQUAL", "LE", "LITERAL_INTERFACE", "LITERAL_FLOAT",
"LITERAL_INSTANCEOF", "BOR_ASSIGN", "LT", "SL_ASSIGN", "ELIST",
"ANNOTATION_ARRAY_INIT", "MODIFIERS", "LITERAL_BREAK", "EXTENDS_CLAUSE",
"TYPE_PARAMETER", "LITERAL_DEFAULT", "STATIC_INIT", "BSR_ASSIGN",
"TYPE_EXTENSION_AND", "BOR", "LITERAL_PRIVATE", "LITERAL_THROW",
"LITERAL_BYTE", "BXOR", "WILDCARD_TYPE", "FINAL", "PARAMETERS", "RPAREN",
"SR_ASSIGN", "UNARY_PLUS", "EMPTY_STAT", "LITERAL_STATIC",
"LITERAL_CONTINUE", "STAR_ASSIGN", "LAMBDA", "RBRACK", "BXOR_ASSIGN",
"CTOR_CALL", "LITERAL_FALSE", "DO_WHILE", "LITERAL_PUBLIC",
"LITERAL_WHILE", "PLUS", "INC", "CTOR_DEF", "GENERIC_END", "DIV_ASSIGN",
"SLIST", "LNOT", "LAND", "LITERAL_ELSE", "ABSTRACT", "STRICTFP",
"QUESTION", "LITERAL_NEW", "LITERAL_RETURN", "SINGLE_LINE_COMMENT",
"INDEX_OP", "EXPR", "BLOCK_COMMENT_BEGIN", "PACKAGE_DEF",
"IMPLEMENTS_CLAUSE", "NUM_FLOAT", "LITERAL_DO", "EOF", "GE", "RESOURCES",
"MOD", "DEC", "EQUAL", "LITERAL_BOOLEAN", "CLASS_DEF", "COLON",
"LITERAL_TRY", "ENUM_DEF", "GT", "NUM_INT", "ANNOTATION_DEF",
"LITERAL_NATIVE", "METHOD_REF", "TYPE_ARGUMENTS", "DOUBLE_COLON", "IDENT",
"MOD_ASSIGN", "LITERAL_FOR", "SUPER_CTOR_CALL", "STRING_LITERAL",
"ARRAY_DECLARATOR", "LITERAL_CASE").collect(Collectors.toSet()));
Stream.of("NUM_DOUBLE", "NUM_FLOAT", "NUM_INT", "NUM_LONG", "IDENT",
"COMMENT_CONTENT", "STRING_LITERAL", "CHAR_LITERAL")
.collect(Collectors.toSet()));
// we do not use this check as it is deprecated
CHECKSTYLE_TOKENS_IN_CONFIG_TO_IGNORE.put("WriteTag",
Stream.of("ENUM_CONSTANT_DEF", "METHOD_DEF", "CTOR_DEF", "ANNOTATION_FIELD_DEF")
@@ -234,37 +200,8 @@
.collect(Collectors.toSet()));
GOOGLE_TOKENS_IN_CONFIG_TO_IGNORE.put("IllegalTokenText", Stream.of(
// all other java tokens and text are allowed
"LITERAL_SUPER", "LITERAL_ASSERT", "ENUM_CONSTANT_DEF", "TYPE_PARAMETERS",
"TYPE_UPPER_BOUNDS", "NUM_DOUBLE", "LITERAL_SWITCH", "ANNOTATIONS",
"LITERAL_SHORT", "LITERAL_PROTECTED", "FOR_CONDITION", "FOR_INIT", "LITERAL_LONG",
"MINUS", "OBJBLOCK", "LITERAL_NULL", "ANNOTATION", "LITERAL_TRUE",
"COMMENT_CONTENT", "LITERAL_CHAR", "PARAMETER_DEF", "POST_DEC",
"ANNOTATION_FIELD_DEF", "BLOCK_COMMENT_END", "TYPE", "LITERAL_INT", "BSR", "ENUM",
"LABELED_STAT", "ANNOTATION_MEMBER_VALUE_PAIR", "TYPECAST", "LITERAL_SYNCHRONIZED",
"PLUS_ASSIGN", "DOT", "LPAREN", "LITERAL_IF", "LITERAL_CATCH", "BAND",
"INTERFACE_DEF", "LOR", "BNOT", "METHOD_CALL", "AT", "ELLIPSIS", "ARRAY_INIT",
"FOR_EACH_CLAUSE", "LITERAL_THROWS", "CASE_GROUP", "POST_INC", "SEMI",
"LITERAL_FINALLY", "ASSIGN", "RESOURCE_SPECIFICATION", "STATIC_IMPORT",
"GENERIC_START", "IMPORT", "SL", "VARIABLE_DEF", "LITERAL_DOUBLE", "RCURLY",
"RESOURCE", "SR", "COMMA", "BAND_ASSIGN", "METHOD_DEF", "LITERAL_VOID",
"NUM_LONG", "LITERAL_TRANSIENT", "LITERAL_THIS", "LCURLY", "MINUS_ASSIGN",
"TYPE_LOWER_BOUNDS", "TYPE_ARGUMENT", "LITERAL_CLASS", "INSTANCE_INIT", "DIV",
"LITERAL_VOLATILE", "STAR", "UNARY_MINUS", "FOR_ITERATOR", "NOT_EQUAL", "LE",
"LITERAL_INTERFACE", "LITERAL_FLOAT", "LITERAL_INSTANCEOF", "BOR_ASSIGN", "LT",
"SL_ASSIGN", "ELIST", "ANNOTATION_ARRAY_INIT", "MODIFIERS", "LITERAL_BREAK",
"EXTENDS_CLAUSE", "TYPE_PARAMETER", "LITERAL_DEFAULT", "STATIC_INIT", "BSR_ASSIGN",
"TYPE_EXTENSION_AND", "BOR", "LITERAL_PRIVATE", "LITERAL_THROW", "LITERAL_BYTE",
"BXOR", "WILDCARD_TYPE", "FINAL", "PARAMETERS", "RPAREN", "SR_ASSIGN",
"UNARY_PLUS", "EMPTY_STAT", "LITERAL_STATIC", "LITERAL_CONTINUE", "STAR_ASSIGN",
"LAMBDA", "RBRACK", "BXOR_ASSIGN", "CTOR_CALL", "LITERAL_FALSE", "DO_WHILE",
"LITERAL_PUBLIC", "LITERAL_WHILE", "PLUS", "INC", "CTOR_DEF", "GENERIC_END",
"DIV_ASSIGN", "SLIST", "LNOT", "LAND", "LITERAL_ELSE", "ABSTRACT", "STRICTFP",
"QUESTION", "LITERAL_NEW", "LITERAL_RETURN", "SINGLE_LINE_COMMENT", "INDEX_OP",
"EXPR", "BLOCK_COMMENT_BEGIN", "PACKAGE_DEF", "IMPLEMENTS_CLAUSE", "NUM_FLOAT",
"LITERAL_DO", "EOF", "GE", "RESOURCES", "MOD", "DEC", "EQUAL", "LITERAL_BOOLEAN",
"CLASS_DEF", "COLON", "LITERAL_TRY", "ENUM_DEF", "GT", "NUM_INT", "ANNOTATION_DEF",
"LITERAL_NATIVE", "METHOD_REF", "TYPE_ARGUMENTS", "DOUBLE_COLON", "IDENT",
"MOD_ASSIGN", "LITERAL_FOR", "SUPER_CTOR_CALL", "ARRAY_DECLARATOR", "LITERAL_CASE")
"NUM_DOUBLE", "NUM_FLOAT", "NUM_INT", "NUM_LONG", "IDENT",
"COMMENT_CONTENT", "STRING_LITERAL", "CHAR_LITERAL")
.collect(Collectors.toSet()));
GOOGLE_TOKENS_IN_CONFIG_TO_IGNORE.put("OperatorWrap", Stream.of(
// specifically allowed via '4.5.1 Where to break' because the following are
@@ -1654,7 +1654,8 @@ class SomeClass
<section name="IllegalTokenText">
<subsection name="Description">
<p>
Checks for illegal token text.
Checks specified tokens text for matching an illegal pattern from
<code>format</code> property. By default no tokens are specified.
</p>
</subsection>
@@ -1688,8 +1689,15 @@ class SomeClass
<tr>
<td>tokens</td>
<td>tokens to check</td>
<td>subset of tokens <a
href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html">TokenTypes</a>.
<td>subset of tokens
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_DOUBLE">NUM_DOUBLE</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_FLOAT">NUM_FLOAT</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_INT">NUM_INT</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_LONG">NUM_LONG</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#IDENT">IDENT</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#COMMENT_CONTENT">COMMENT_CONTENT</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#STRING_LITERAL">STRING_LITERAL</a>,
<a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#CHAR_LITERAL">CHAR_LITERAL</a>.
</td>
<td>empty</td>
</tr>

0 comments on commit cd88872

Please sign in to comment.