Skip to content

Allow underscores in integer literals #1627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slozier merged 3 commits into from
Dec 24, 2022
Merged

Allow underscores in integer literals #1627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
32c3236
Allow underscores in integer literals
slozier Oct 13, 2022
660a93f
Re-enale test_int
slozier Dec 23, 2022
1997372
Fix fromhex
slozier Dec 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 42 additions & 36 deletions Src/IronPython/Compiler/Tokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -735,11 +735,16 @@ private Token ReadNumber(int start) {
}
isPrefix0 = true;

while (NextChar('0')) { } // skip leading zeroes
// skip leading zeroes
while (true) {
NextChar('_');
if (!NextChar('0')) break;
}
}

bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand All @@ -755,7 +760,7 @@ private Token ReadNumber(int start) {
MarkTokenEnd();

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenString(), 10));
return new ConstantValueToken(ParseInteger(GetTokenSpan(), 10));

case 'j':
case 'J':
Expand Down Expand Up @@ -784,7 +789,7 @@ private Token ReadNumber(int start) {
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenString(), 10));
return new ConstantValueToken(ParseInteger(GetTokenSpan(), 10));
}
isFirstChar = false;
}
Expand All @@ -795,8 +800,9 @@ private Token ReadBinaryNumber() {
int iVal = 0;
bool useBigInt = false;
BigInteger bigInt = BigInteger.Zero;
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();
switch (ch) {
case '0':
Expand All @@ -812,7 +818,7 @@ private Token ReadBinaryNumber() {
bigInt = (BigInteger)iVal;
}

if (bits >= 32) {
if (useBigInt) {
bigInt = (bigInt << 1) | (ch - '0');
} else {
iVal = iVal << 1 | (ch - '0');
Expand All @@ -822,22 +828,21 @@ private Token ReadBinaryNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

return new ConstantValueToken(useBigInt ? bigInt : (object)iVal);
}
first = false;
isFirstChar = false;
}
}

private Token ReadOctalNumber() {
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand All @@ -855,23 +860,24 @@ private Token ReadOctalNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 8));
var span = GetTokenSpan().Slice(2);
if (!span.IsEmpty && span[0] == '_') span = span.Slice(1);
return new ConstantValueToken(ParseInteger(span, 8));
}
first = false;
isFirstChar = false;
}
}

private Token ReadHexNumber() {
bool first = true;
bool isFirstChar = true;
while (true) {
NextChar('_');
int ch = NextChar();

switch (ch) {
Expand Down Expand Up @@ -903,17 +909,17 @@ private Token ReadHexNumber() {
BufferBack();
MarkTokenEnd();

if (first) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.InvalidToken, ErrorCodes.SyntaxError);
if (isFirstChar) {
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.InvalidToken, ErrorCodes.SyntaxError);
}

// TODO: parse in place
return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 16));
var span = GetTokenSpan().Slice(2);
if (!span.IsEmpty && span[0] == '_') span = span.Slice(1);
return new ConstantValueToken(ParseInteger(span, 16));
}
first = false;
isFirstChar = false;
}
}

Expand Down Expand Up @@ -1431,10 +1437,8 @@ private void SetIndent(int spaces, StringBuilder chars) {
current = DoDedent(spaces, current);

if (spaces != current) {
ReportSyntaxError(
new SourceSpan(new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1),
BufferTokenEnd),
Resources.IndentationMismatch, ErrorCodes.IndentationError);
var errorStart = new SourceLocation(_tokenEndIndex, IndexToLocation(_tokenEndIndex).Line, IndexToLocation(_tokenEndIndex).Column - 1);
ReportSyntaxError(new SourceSpan(errorStart, BufferTokenEnd), Resources.IndentationMismatch, ErrorCodes.IndentationError);
}
}
}
Expand All @@ -1448,12 +1452,11 @@ private int DoDedent(int spaces, int current) {
return current;
}

private object ParseInteger(string s, int radix) {
try {
return LiteralParser.ParseInteger(s, radix);
} catch (ArgumentException e) {
ReportSyntaxError(BufferTokenSpan, e.Message, ErrorCodes.SyntaxError);
private object ParseInteger(ReadOnlySpan<char> s, int radix) {
if (LiteralParser.TryParseIntegerSign(s, radix, out object result)) {
return result;
}
ReportSyntaxError(BufferTokenSpan, "invalid token", ErrorCodes.SyntaxError);
return ScriptingRuntimeHelpers.Int32ToObject(0);
}

Expand Down Expand Up @@ -1673,6 +1676,9 @@ private string GetTokenSubstring(int offset, int length) {
return new String(_buffer, _start + offset, length);
}

private ReadOnlySpan<char> GetTokenSpan()
=> _buffer.AsSpan(_start, _tokenEnd - _start);

[Conditional("DEBUG")]
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
private void CheckInvariants() {
Expand Down
Loading