Skip to content

Commit cfac41c

Browse files
foonathanfoonathan
committed
Rename tokenizer stuff to include cx prefix
1 parent 6b55a13 commit cfac41c

16 files changed

+121
-120
lines changed

‎src/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ set(libclang_source
9595
libclang/preprocessor.hpp
9696
libclang/raii_wrapper.hpp
9797
libclang/template_parser.cpp
98-
libclang/tokenizer.cpp
99-
libclang/tokenizer.hpp
98+
libclang/cxtokenizer.cpp
99+
libclang/cxtokenizer.hpp
100100
libclang/type_parser.cpp
101101
libclang/variable_parser.cpp)
102102

‎src/libclang/class_parser.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,8 @@ namespace
7171
auto access = convert_access(cur);
7272
auto is_virtual = clang_isVirtualBase(cur) != 0u;
7373

74-
detail::tokenizer tokenizer(context.tu, context.file, cur);
75-
detail::token_stream stream(tokenizer, cur);
74+
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
75+
detail::cxtoken_stream stream(tokenizer, cur);
7676

7777
// [<attribute>] [virtual] [<access>] <name>
7878
// can't use spelling to get the name
@@ -108,8 +108,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_class(const detail::parse_context&
108108
clang_getCursorLexicalParent(cur)))
109109
{
110110
// out-of-line definition
111-
detail::tokenizer tokenizer(context.tu, context.file, cur);
112-
detail::token_stream stream(tokenizer, cur);
111+
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
112+
detail::cxtoken_stream stream(tokenizer, cur);
113113

114114
std::string name = detail::get_cursor_name(cur).c_str();
115115
auto pos = name.find('<');

src/libclang/tokenizer.cpp renamed to src/libclang/cxtokenizer.cpp

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// This file is subject to the license terms in the LICENSE file
33
// found in the top-level directory of this distribution.
44

5-
#include "tokenizer.hpp"
5+
#include "cxtokenizer.hpp"
66

77
#include <cctype>
88

@@ -11,7 +11,7 @@
1111

1212
using namespace cppast;
1313

14-
detail::token::token(const CXTranslationUnit& tu_unit, const CXToken& token)
14+
detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token)
1515
: value_(clang_getTokenSpelling(tu_unit, token)), kind_(clang_getTokenKind(token))
1616
{
1717
}
@@ -237,7 +237,8 @@ namespace
237237
}
238238
}
239239

240-
detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur)
240+
detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
241+
const CXCursor& cur)
241242
{
242243
auto extent = get_extent(tu, file, cur, unmunch_);
243244

@@ -247,7 +248,7 @@ detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, co
247248
tokens_.emplace_back(tu, tokenizer[i]);
248249
}
249250

250-
void detail::skip(detail::token_stream& stream, const char* str)
251+
void detail::skip(detail::cxtoken_stream& stream, const char* str)
251252
{
252253
if (*str)
253254
{
@@ -263,7 +264,7 @@ void detail::skip(detail::token_stream& stream, const char* str)
263264

264265
namespace
265266
{
266-
bool starts_with(const char*& str, const detail::token& t)
267+
bool starts_with(const char*& str, const detail::cxtoken& t)
267268
{
268269
if (std::strncmp(str, t.c_str(), t.value().length()) != 0)
269270
return false;
@@ -274,7 +275,7 @@ namespace
274275
}
275276
}
276277

277-
bool detail::skip_if(detail::token_stream& stream, const char* str, bool multi_token)
278+
bool detail::skip_if(detail::cxtoken_stream& stream, const char* str, bool multi_token)
278279
{
279280
if (!*str)
280281
return true;
@@ -298,7 +299,7 @@ namespace
298299
{
299300
// whether or not the current angle bracket can be a comparison
300301
// note: this is a heuristic I hope works often enough
301-
bool is_comparison(CXTokenKind last_kind, const detail::token& cur, CXTokenKind next_kind)
302+
bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKind next_kind)
302303
{
303304
if (cur == "<")
304305
return last_kind == CXToken_Literal;
@@ -308,7 +309,7 @@ namespace
308309
}
309310
}
310311

311-
detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
312+
detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream)
312313
{
313314
auto template_bracket = false;
314315
auto open_bracket = stream.peek().c_str();
@@ -359,15 +360,15 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
359360
return stream.cur();
360361
}
361362

362-
void detail::skip_brackets(detail::token_stream& stream)
363+
void detail::skip_brackets(detail::cxtoken_stream& stream)
363364
{
364365
auto closing = find_closing_bracket(stream);
365366
stream.set_cur(std::next(closing));
366367
}
367368

368369
namespace
369370
{
370-
bool skip_attribute_impl(detail::token_stream& stream)
371+
bool skip_attribute_impl(detail::cxtoken_stream& stream)
371372
{
372373
if (skip_if(stream, "[") && stream.peek() == "[")
373374
{
@@ -401,7 +402,7 @@ namespace
401402
}
402403
}
403404

404-
bool detail::skip_attribute(detail::token_stream& stream)
405+
bool detail::skip_attribute(detail::cxtoken_stream& stream)
405406
{
406407
auto any = false;
407408
while (skip_attribute_impl(stream))
@@ -432,7 +433,7 @@ namespace
432433
}
433434
}
434435

435-
cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
436+
cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
436437
{
437438
cpp_token_string::builder builder;
438439

@@ -448,7 +449,7 @@ cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
448449
return builder.finish();
449450
}
450451

451-
bool detail::append_scope(detail::token_stream& stream, std::string& scope)
452+
bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope)
452453
{
453454
// add identifiers and "::" to current scope name,
454455
// clear if there is any other token in between, or mismatched combination

src/libclang/tokenizer.hpp renamed to src/libclang/cxtokenizer.hpp

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// This file is subject to the license terms in the LICENSE file
33
// found in the top-level directory of this distribution.
44

5-
#ifndef CPPAST_TOKENIZER_HPP_INCLUDED
6-
#define CPPAST_TOKENIZER_HPP_INCLUDED
5+
#ifndef CPPAST_CXTOKENIZER_HPP_INCLUDED
6+
#define CPPAST_CXTOKENIZER_HPP_INCLUDED
77

88
#include <string>
99
#include <vector>
@@ -16,10 +16,10 @@ namespace cppast
1616
{
1717
namespace detail
1818
{
19-
class token
19+
class cxtoken
2020
{
2121
public:
22-
explicit token(const CXTranslationUnit& tu_unit, const CXToken& token);
22+
explicit cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token);
2323

2424
const cxstring& value() const noexcept
2525
{
@@ -41,40 +41,40 @@ namespace cppast
4141
CXTokenKind kind_;
4242
};
4343

44-
inline bool operator==(const token& tok, const char* str) noexcept
44+
inline bool operator==(const cxtoken& tok, const char* str) noexcept
4545
{
4646
return tok.value() == str;
4747
}
4848

49-
inline bool operator==(const char* str, const token& tok) noexcept
49+
inline bool operator==(const char* str, const cxtoken& tok) noexcept
5050
{
5151
return str == tok.value();
5252
}
5353

54-
inline bool operator!=(const token& tok, const char* str) noexcept
54+
inline bool operator!=(const cxtoken& tok, const char* str) noexcept
5555
{
5656
return !(tok == str);
5757
}
5858

59-
inline bool operator!=(const char* str, const token& tok) noexcept
59+
inline bool operator!=(const char* str, const cxtoken& tok) noexcept
6060
{
6161
return !(str == tok);
6262
}
6363

64-
using token_iterator = std::vector<token>::const_iterator;
64+
using cxtoken_iterator = std::vector<cxtoken>::const_iterator;
6565

66-
class tokenizer
66+
class cxtokenizer
6767
{
6868
public:
69-
explicit tokenizer(const CXTranslationUnit& tu, const CXFile& file,
70-
const CXCursor& cur);
69+
explicit cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
70+
const CXCursor& cur);
7171

72-
token_iterator begin() const noexcept
72+
cxtoken_iterator begin() const noexcept
7373
{
7474
return tokens_.begin();
7575
}
7676

77-
token_iterator end() const noexcept
77+
cxtoken_iterator end() const noexcept
7878
{
7979
return tokens_.end();
8080
}
@@ -88,14 +88,14 @@ namespace cppast
8888
}
8989

9090
private:
91-
std::vector<token> tokens_;
92-
bool unmunch_;
91+
std::vector<cxtoken> tokens_;
92+
bool unmunch_;
9393
};
9494

95-
class token_stream
95+
class cxtoken_stream
9696
{
9797
public:
98-
explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur)
98+
explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
9999
: cursor_(cur),
100100
begin_(tokenizer.begin()),
101101
cur_(begin_),
@@ -104,7 +104,7 @@ namespace cppast
104104
{
105105
}
106106

107-
const token& peek() const noexcept
107+
const cxtoken& peek() const noexcept
108108
{
109109
if (done())
110110
return *std::prev(end_);
@@ -123,7 +123,7 @@ namespace cppast
123123
--cur_;
124124
}
125125

126-
const token& get() noexcept
126+
const cxtoken& get() noexcept
127127
{
128128
auto& result = peek();
129129
bump();
@@ -140,22 +140,22 @@ namespace cppast
140140
return cursor_;
141141
}
142142

143-
token_iterator begin() const noexcept
143+
cxtoken_iterator begin() const noexcept
144144
{
145145
return begin_;
146146
}
147147

148-
token_iterator cur() const noexcept
148+
cxtoken_iterator cur() const noexcept
149149
{
150150
return cur_;
151151
}
152152

153-
token_iterator end() const noexcept
153+
cxtoken_iterator end() const noexcept
154154
{
155155
return end_;
156156
}
157157

158-
void set_cur(token_iterator iter) noexcept
158+
void set_cur(cxtoken_iterator iter) noexcept
159159
{
160160
cur_ = iter;
161161
}
@@ -166,41 +166,41 @@ namespace cppast
166166
}
167167

168168
private:
169-
CXCursor cursor_;
170-
token_iterator begin_, cur_, end_;
171-
bool unmunch_;
169+
CXCursor cursor_;
170+
cxtoken_iterator begin_, cur_, end_;
171+
bool unmunch_;
172172
};
173173

174174
// skips the next token
175175
// asserts that it has the given string
176-
void skip(token_stream& stream, const char* str);
176+
void skip(cxtoken_stream& stream, const char* str);
177177

178178
// skips the next token if it has the given string
179179
// if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
180-
bool skip_if(token_stream& stream, const char* str, bool multi_token = false);
180+
bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false);
181181

182182
// returns the location of the closing bracket
183183
// the current token must be (,[,{ or <
184184
// note: < might not work in the arguments of a template specialization
185-
token_iterator find_closing_bracket(token_stream stream);
185+
cxtoken_iterator find_closing_bracket(cxtoken_stream stream);
186186

187187
// skips brackets
188188
// the current token must be (,[,{ or <
189189
// note: < might not work in the arguments of a template specialization
190-
void skip_brackets(token_stream& stream);
190+
void skip_brackets(cxtoken_stream& stream);
191191

192192
// skips an attribute
193-
bool skip_attribute(token_stream& stream);
193+
bool skip_attribute(cxtoken_stream& stream);
194194

195195
// converts a token range to a string
196-
cpp_token_string to_string(token_stream& stream, token_iterator end);
196+
cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end);
197197

198198
// appends token to scope, if it is still valid
199199
// else clears it
200200
// note: does not consume the token if it is not valid,
201201
// returns false in that case
202-
bool append_scope(token_stream& stream, std::string& scope);
202+
bool append_scope(cxtoken_stream& stream, std::string& scope);
203203
}
204204
} // namespace cppast::detail
205205

206-
#endif // CPPAST_TOKENIZER_HPP_INCLUDED
206+
#endif // CPPAST_CXTOKENIZER_HPP_INCLUDED

‎src/libclang/debug_helper.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#include <cstdio>
88
#include <mutex>
99

10-
#include "tokenizer.hpp"
10+
#include "cxtokenizer.hpp"
1111

1212
using namespace cppast;
1313

@@ -50,7 +50,7 @@ void detail::print_tokens(const CXTranslationUnit& tu, const CXFile& file,
5050
const CXCursor& cur) noexcept
5151
{
5252
std::lock_guard<std::mutex> lock(mtx);
53-
detail::tokenizer tokenizer(tu, file, cur);
53+
detail::cxtokenizer tokenizer(tu, file, cur);
5454
for (auto& token : tokenizer)
5555
std::fprintf(stderr, "%s ", token.c_str());
5656
std::fputs("\n", stderr);

‎src/libclang/enum_parser.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ namespace
2020
DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur,
2121
"unexpected child cursor of enum");
2222

23-
detail::tokenizer tokenizer(context.tu, context.file, cur);
24-
detail::token_stream stream(tokenizer, cur);
23+
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
24+
detail::cxtoken_stream stream(tokenizer, cur);
2525

2626
// <identifier> [<attribute>],
2727
// or: <identifier> [<attribute>] = <expression>,
@@ -47,9 +47,9 @@ namespace
4747
cpp_enum::builder make_enum_builder(const detail::parse_context& context, const CXCursor& cur,
4848
type_safe::optional<cpp_entity_ref>& semantic_parent)
4949
{
50-
auto name = detail::get_cursor_name(cur);
51-
detail::tokenizer tokenizer(context.tu, context.file, cur);
52-
detail::token_stream stream(tokenizer, cur);
50+
auto name = detail::get_cursor_name(cur);
51+
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
52+
detail::cxtoken_stream stream(tokenizer, cur);
5353

5454
// [<attribute>] enum [class] [<attribute>] name [: type] {
5555
detail::skip_attribute(stream);

0 commit comments

Comments
 (0)