A generic code scanner and tokenizer
Any improvements or changes trickle down from ImGui_GM's version of brief, check that repo if there is a newer version of brief. Odds are this repo will not be updated as frequently.
- Use
Scannerto scan lines of code into a flat list of tokens - Use
Processor.getto nest tokens and apply other modifiers
test.cpp
#include <string>
// hello!
void test(std::string awoo) {
int a = 32;
if (a == 32) {
/*
this is a multi-line comment
*/
a_function<float>(a->member, 0xFF, 0b0000_1000, "Hello World!");
}
}tokens.json
[
{
"Type": "PreprocessorDirective",
"Literal": "#define",
"Line": 1
},
{
"Type": "Identifier",
"Literal": "HEWWO",
"Line": 1
},
{
"Type": "Comment",
"Literal": "// hello!\r\n",
"Line": 4
},
{
"Type": "Keyword",
"Literal": "void",
"Line": 4
},
{
"Type": "FunctionDef",
"Literal": "test",
"Line": 4,
"Children": [
{
"Type": "Identifier",
"Literal": "std",
"Line": 4
},
{
"Type": "ScopeResolver",
"Literal": "::",
"Line": 4
},
{
"Type": "Identifier",
"Literal": "string",
"Line": 4
},
{
"Type": "Identifier",
"Literal": "awoo",
"Line": 4
}
]
},
{
"Type": "BracePair",
"Literal": "{",
"Line": 4,
"Children": [
{
"Type": "Keyword",
"Literal": "int",
"Line": 5
},
{
"Type": "Identifier",
"Literal": "a",
"Line": 5
},
{
"Type": "Assign",
"Literal": "=",
"Line": 5
},
{
"Type": "Number",
"Literal": 32,
"Line": 5
},
{
"Type": "Semicolon",
"Literal": ";",
"Line": 5
},
{
"Type": "Keyword",
"Literal": "if",
"Line": 6
},
{
"Type": "ParenthesesPair",
"Literal": "(",
"Line": 6,
"Children": [
{
"Type": "Identifier",
"Literal": "a",
"Line": 6
},
{
"Type": "CompareEqual",
"Literal": "==",
"Line": 6
},
{
"Type": "Number",
"Literal": 32,
"Line": 6
}
]
},
{
"Type": "BracePair",
"Literal": "{",
"Line": 6,
"Children": [
{
"Type": "CommentMultiline",
"Literal": "/*\r\n this is a multi-line comment\r\n */",
"Line": 9
},
{
"Type": "Identifier",
"Literal": "a_function",
"Line": 10
},
{
"Type": "LessThan",
"Literal": "<",
"Line": 10
},
{
"Type": "Keyword",
"Literal": "float",
"Line": 10
},
{
"Type": "GreaterThan",
"Literal": ">",
"Line": 10
},
{
"Type": "ParenthesesPair",
"Literal": "(",
"Line": 10,
"Children": [
{
"Type": "Identifier",
"Literal": "a",
"Line": 10
},
{
"Type": "PointerAccess",
"Literal": "->",
"Line": 10
},
{
"Type": "Identifier",
"Literal": "member",
"Line": 10
},
{
"Type": "Comma",
"Literal": ",",
"Line": 10
},
{
"Type": "NumberHex",
"Literal": "0xFF",
"Line": 10
},
{
"Type": "Comma",
"Literal": ",",
"Line": 10
},
{
"Type": "NumberBinary",
"Literal": "0b0000_1000",
"Line": 10
},
{
"Type": "Comma",
"Literal": ",",
"Line": 10
},
{
"Type": "String",
"Literal": "Hello World!",
"Line": 10
}
]
},
{
"Type": "Semicolon",
"Literal": ";",
"Line": 10
}
]
}
]
},
{
"Type": "EndFile",
"Literal": "}",
"Line": 12
}
]