-
Notifications
You must be signed in to change notification settings - Fork 1.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Initial implementation of raw string literals #1304
Changes from 8 commits
a4ff50d
a31dda2
d3be382
ece0b7f
4d56ea6
7a32eec
5745c55
d5f69af
75bcbcf
ac735be
ebdf8f7
743aed6
d509aa5
d51377d
855fe32
17bc3cf
9ac7418
43ab9a6
750b034
266359a
8437c5c
51b2af9
b5791ad
1914495
4acf2ae
c8dcc8b
46ed305
cb0039b
9cf8448
54c46c1
78da5b9
3c2e90d
70709cd
6f45efc
00401d8
65facf5
0c91724
f8e8054
7cc8cbb
a45fd15
24d3149
ec4477b
1346f92
bb63820
3a8d488
aa6e246
6a77fea
eae97d5
2aba1f6
4e238e8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM | ||
// Exceptions. See /LICENSE for license information. | ||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
|
||
#ifndef CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ | ||
#define CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ | ||
|
||
// This macro is expanded immediately before each action specified below. | ||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// | ||
// Advances the current token position by yyleng columns without changing | ||
// the line number, and takes us out of the after-whitespace / after-operand | ||
// state. | ||
#define YY_USER_ACTION \ | ||
context.current_token_position.columns(yyleng); \ | ||
if (YY_START == AFTER_WHITESPACE || YY_START == AFTER_OPERAND) { \ | ||
BEGIN(INITIAL); \ | ||
} | ||
|
||
#define SIMPLE_TOKEN(name) \ | ||
Carbon::Parser::make_##name(context.current_token_position); | ||
|
||
#define ARG_TOKEN(name, arg) \ | ||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Carbon::Parser::make_##name(arg, context.current_token_position); | ||
|
||
#endif // CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,73 @@ | ||||||
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM | ||||||
// Exceptions. See /LICENSE for license information. | ||||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
|
||||||
#include "explorer/syntax/lex_scan_helper.h" | ||||||
|
||||||
#include "common/string_helpers.h" | ||||||
#include "explorer/syntax/lex_helper.h" | ||||||
#include "llvm/Support/FormatVariadic.h" | ||||||
|
||||||
namespace Carbon { | ||||||
|
||||||
auto StringLexHelper::Advance() -> bool { | ||||||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
int c = ReadChar(yyscanner_, context_); | ||||||
if (c <= 0) { | ||||||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
is_eof_ = true; | ||||||
return false; | ||||||
} | ||||||
str_.push_back(c); | ||||||
return true; | ||||||
} | ||||||
|
||||||
auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int { | ||||||
const int c = YyinputWrapper(yyscanner); | ||||||
if (c <= 0) { | ||||||
context.RecordSyntaxError("Unexpected end of file"); | ||||||
} | ||||||
return c; | ||||||
} | ||||||
|
||||||
auto ReadHashTags(Carbon::StringLexHelper& scan_helper, | ||||||
const size_t hashtag_num) -> bool { | ||||||
for (size_t i = 0; i < hashtag_num; ++i) { | ||||||
if (!scan_helper.Advance() || scan_helper.last_char() != '#') { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is backwards from what I expect:
Suggested change
That way only There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||||||
return false; | ||||||
} | ||||||
} | ||||||
return true; | ||||||
} | ||||||
|
||||||
auto ProcessSingleLineString(llvm::StringRef str, | ||||||
Carbon::ParseAndLexContext& context, | ||||||
const size_t hashtag_num) | ||||||
-> Carbon::Parser::symbol_type { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Save a copy of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Instead of copying There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Copying a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Updated with copy of |
||||||
std::string hashtags(hashtag_num, '#'); | ||||||
CARBON_CHECK(str.consume_front(hashtags + "\"") && | ||||||
str.consume_back("\"" + hashtags)); | ||||||
|
||||||
std::optional<std::string> unescaped = | ||||||
Carbon::UnescapeStringLiteral(str, hashtag_num); | ||||||
if (unescaped == std::nullopt) { | ||||||
return context.RecordSyntaxError( | ||||||
llvm::formatv("Invalid escaping in string: {0}", str)); | ||||||
} | ||||||
return ARG_TOKEN(string_literal, *unescaped); | ||||||
} | ||||||
|
||||||
auto ProcessMultiLineString(llvm::StringRef str, | ||||||
Carbon::ParseAndLexContext& context, | ||||||
const size_t hashtag_num) | ||||||
-> Carbon::Parser::symbol_type { | ||||||
std::string hashtags(hashtag_num, '#'); | ||||||
CARBON_CHECK(str.consume_front(hashtags) && str.consume_back(hashtags)); | ||||||
Carbon::ErrorOr<std::string> block_string = | ||||||
Carbon::ParseBlockStringLiteral(str, hashtag_num); | ||||||
if (!block_string.ok()) { | ||||||
return context.RecordSyntaxError(llvm::formatv( | ||||||
"Invalid block string: {0}", block_string.error().message())); | ||||||
} | ||||||
return ARG_TOKEN(string_literal, *block_string); | ||||||
} | ||||||
|
||||||
} // namespace Carbon |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM | ||
// Exceptions. See /LICENSE for license information. | ||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
|
||
#ifndef CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ | ||
#define CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ | ||
|
||
#include <string> | ||
|
||
#include "explorer/syntax/parse_and_lex_context.h" | ||
#include "explorer/syntax/parser.h" | ||
|
||
extern auto YyinputWrapper(yyscan_t yyscanner) -> int; | ||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
namespace Carbon { | ||
|
||
class StringLexHelper { | ||
josh11b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
public: | ||
StringLexHelper(const char* text, yyscan_t yyscanner, | ||
Carbon::ParseAndLexContext& context) | ||
: str_(text), yyscanner_(yyscanner), context_(context), is_eof_(false) {} | ||
// Advances yyscanner by one char. Sets is_eof to true and returns false on | ||
// EOF. | ||
auto Advance() -> bool; | ||
// Returns last scanned char. | ||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
auto last_char() -> int { return str_.back(); }; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there a reason this is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The original |
||
// Returns scanned string. | ||
SlaterLatiao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
auto str() -> const std::string& { return str_; }; | ||
|
||
auto is_eof() -> bool { return is_eof_; }; | ||
|
||
private: | ||
std::string str_; | ||
yyscan_t yyscanner_; | ||
Carbon::ParseAndLexContext& context_; | ||
// Skips reading next char. | ||
bool is_eof_; | ||
}; | ||
|
||
// Reads and returns a single character. Reports an error on EOF. | ||
auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How many places is this function called from? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is only called by |
||
|
||
// Tries to Read [hashtag_num] hashtags. Returns true on success. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd generally write Should document how much There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Edited as suggested and added comment on how many characters are read. |
||
auto ReadHashTags(Carbon::StringLexHelper& scan_helper, size_t hashtag_num) | ||
-> bool; | ||
|
||
// Removes quotes and escapes a single line string. Reports an error on | ||
// invalid escaping. | ||
auto ProcessSingleLineString(llvm::StringRef str, | ||
Carbon::ParseAndLexContext& context, | ||
size_t hashtag_num) -> Carbon::Parser::symbol_type; | ||
auto ProcessMultiLineString(llvm::StringRef str, | ||
Carbon::ParseAndLexContext& context, | ||
size_t hashtag_num) -> Carbon::Parser::symbol_type; | ||
|
||
} // namespace Carbon | ||
|
||
#endif // CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Use
int
instead ofsize_t
. Avoid unsigned types unless they are needed: https://google.github.io/styleguide/cppguide.html#Integer_TypesThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Replaced
size_t
withint
.