Skip to content

Commit dd190a1

Browse files
MoserMichaelMoserMichael
committedMay 1, 2020
add identifier parser + prevent collisions of identifiers with tokens
1 parent 1dea971 commit dd190a1

File tree

3 files changed

+301
-11
lines changed

3 files changed

+301
-11
lines changed
 

‎inc/parse.h

+198-9
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <cstring>
1111
#include "parse_text.h"
1212
#include "parsedef.h"
13+
#include "tokencollisionhelper.h"
1314
#include "dhelper.h"
1415
#include "vhelper.h"
1516
#include "analyse.h"
@@ -79,13 +80,28 @@ struct ParserBase {
7980
return false;
8081
}
8182

83+
template<typename ParserBase>
84+
static void init_collision_checker(ParserBase &base) {
85+
ERROR("shouldn't get here");
86+
}
8287
#endif
8388

8489
template<typename Stream>
8590
static void dumpJson(Stream &out, const AstType *ast) {
8691
ERROR("Not implemented\n");
8792
}
8893

94+
void init_collission_checker() {
95+
TokenCollisionChecker *ptr = new TokenCollisionChecker();
96+
colission_checker_.reset( ptr );
97+
}
98+
99+
100+
101+
std::unique_ptr<TokenCollisionChecker> colission_checker_;
102+
103+
104+
//must not be a polymorphic type, is accessed from static stuff only. don't have virtual functions here.
89105
//virtual ~ParserBase() {}
90106
};
91107

@@ -150,18 +166,52 @@ struct CharParser : ParserBase {
150166
static void dumpJson(Stream &out, const AstType *) {
151167
}
152168

169+
template<typename ParserBase>
170+
static void init_collision_checker(ParserBase &base) {
171+
}
153172

154173
private:
155174
Text_stream &text_;
156175
Text_stream::Next_char_value ch_;
157176
};
158177

178+
179+
180+
// PTopLevelParser - top level parser, initialises the collision checker;
181+
// The collision checker is used to check that variables that have the string value of a token are not counted as a variable;
182+
// the parse method initialises the collision checker and attaches it to the base parser, and after using it it is deleted.
183+
//
184+
template<typename Type>
185+
struct PTopLevelParser : ParserBase {
186+
187+
using AstType = typename Type::AstType;
188+
using ThisClass = PTopLevelParser<Type>;
189+
190+
template<typename ParserBase>
191+
static Parse_result parse(ParserBase &base) {
192+
193+
PTopLevelParser<Type>::init_collision_checker(base);
194+
//Type::init_collision_checker(base);
195+
return Type::parse(base);
196+
}
197+
198+
template<typename ParserBase>
199+
static void init_collision_checker(ParserBase &base) {
200+
201+
base.init_collission_checker();
202+
if (base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
203+
Type::init_collision_checker(base);
204+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
205+
}
206+
}
207+
};
208+
159209
//
160210
// PEof - parse is ok if at end of input
161211
//
162212

163213
template<typename Type>
164-
struct PRequireEof : ParserBase {
214+
struct PRequireEof : PTopLevelParser<Type> {
165215

166216
static inline const RuleId RULE_ID = Type::RULE_ID;
167217

@@ -170,12 +220,12 @@ struct PRequireEof : ParserBase {
170220
template<typename ParserBase>
171221
static Parse_result parse(ParserBase &base) {
172222

173-
Parse_result res = Type::parse(base);
223+
Parse_result res = PTopLevelParser<Type>::parse(base);
174224
if (!res.success_) {
175225
return res;
176226
}
177227

178-
Char_value nchar = ParserBase::current_char(base);
228+
typename ParserBase::Char_value nchar = ParserBase::current_char(base);
179229
while (nchar.first && isspace(nchar.second)) {
180230
ParserBase::next_char(base);
181231
nchar = ParserBase::current_char(base);
@@ -326,6 +376,13 @@ struct PTok : ParserBase {
326376
}
327377

328378

379+
template<typename ParserBase>
380+
static void init_collision_checker(ParserBase &base) {
381+
std::string sval;
382+
gather_token_value<Cs...>(sval);
383+
base.colission_checker_->insert(sval.c_str(), sval.size() );
384+
}
385+
329386
private:
330387
template<typename ParserBase, Char_t ch, Char_t ...Css>
331388
static inline bool parse_helper( ParserBase &text ) {
@@ -354,6 +411,17 @@ struct PTok : ParserBase {
354411
return true;
355412
}
356413

414+
template<Char_t ch, Char_t ...Css>
415+
static inline bool gather_token_value(std::string &strval) {
416+
417+
strval += ch;
418+
419+
if constexpr (sizeof...(Css) > 0) {
420+
return dump_helper<Css...>( strval );
421+
}
422+
423+
return true;
424+
}
357425

358426

359427
};
@@ -372,10 +440,13 @@ enum class Char_checker_result {
372440
typedef Char_checker_result (PTokVar_cb_t) (Char_t current_char, bool iseof, Char_t *matched_so_far);
373441

374442

375-
template<RuleId ruleId, PTokVar_cb_t checker, bool canAcceptEmptyInput = false>
443+
const int PTokVarCanAcceptEmptyInput = 1;
444+
const int PTokVarCheckTokenClash = 2;
445+
446+
template<RuleId ruleId, PTokVar_cb_t checker, int TokVarFlags = 0>
376447
struct PTokVar : ParserBase {
377448

378-
using ThisClass = PTokVar<ruleId, checker, canAcceptEmptyInput>;
449+
using ThisClass = PTokVar<ruleId, checker, TokVarFlags>;
379450

380451
static inline const RuleId RULE_ID = ruleId;
381452

@@ -421,7 +492,11 @@ struct PTokVar : ParserBase {
421492
AstEntryBase *ret = ast.release();
422493
ret->start_ = token_start_pos;
423494
ret->end_ = end_pos;
424-
return Parse_result{true, token_start_pos, end_pos, std::unique_ptr<AstEntryBase>(ret) };
495+
496+
if (has_collision(base, ast.get()->entry_)) {
497+
return Parse_result{false, token_start_pos, token_start_pos};
498+
}
499+
return Parse_result{true, token_start_pos, end_pos, std::unique_ptr<AstEntryBase>(ret) };
425500

426501
}
427502

@@ -432,7 +507,11 @@ struct PTokVar : ParserBase {
432507
AstEntryBase *ret = ast.release();
433508
ret->start_ = token_start_pos;
434509
ret->end_ = end_pos;
435-
return Parse_result{true, token_start_pos, end_pos, std::unique_ptr<AstEntryBase>(ret) };
510+
511+
if (has_collision(base, ast.get()->entry_)) {
512+
return Parse_result{false, token_start_pos, token_start_pos};
513+
}
514+
return Parse_result{true, token_start_pos, end_pos, std::unique_ptr<AstEntryBase>(ret) };
436515
}
437516

438517
}
@@ -460,7 +539,7 @@ struct PTokVar : ParserBase {
460539

461540
template<typename HelperType>
462541
static bool can_accept_empty_input(HelperType *) {
463-
return canAcceptEmptyInput;
542+
return TokVarFlags & PTokVarCanAcceptEmptyInput;
464543
}
465544

466545
#endif
@@ -475,6 +554,24 @@ struct PTokVar : ParserBase {
475554
Json<Stream>::jsonEndTag(out, true);
476555
}
477556

557+
template<typename ParserBase>
558+
static void init_collision_checker(ParserBase &base) {
559+
}
560+
561+
inline static bool has_collision(ParserBase &base, std::string &entry) {
562+
563+
if constexpr ((TokVarFlags & PTokVarCheckTokenClash) != 0) {
564+
if (base.colission_checker_ != nullptr) {
565+
566+
const Char_t *tok = (const Char_t *) entry.c_str();
567+
int len = entry.size();
568+
569+
return base.colission_checker_->has_token(tok, len);
570+
}
571+
}
572+
return false;
573+
}
574+
478575
};
479576

480577
inline Char_checker_result pparse_is_digit(Char_t current_char, bool iseof, Char_t *matched_so_far) {
@@ -504,7 +601,29 @@ inline Char_checker_result parse_print_char(Char_t current_char, bool iseof, Cha
504601
// PTokPrintChar - single printable character
505602
//
506603
template<RuleId ruleId>
507-
struct PTokChar : PTokVar<ruleId,parse_print_char> {
604+
struct PTokChar : PTokVar<ruleId,parse_print_char, PTokVarCheckTokenClash> {
605+
};
606+
607+
inline Char_checker_result pparse_identifier(Char_t current_char, bool iseof, Char_t *matched_so_far) {
608+
609+
ssize_t slen = strlen(matched_so_far);
610+
if (iseof) {
611+
if (slen >0) {
612+
return Char_checker_result::acceptNow;
613+
}
614+
}
615+
616+
if (slen == 0) {
617+
return isalpha(current_char) ? Char_checker_result::proceed : Char_checker_result::error;
618+
}
619+
return isalnum(current_char) ? Char_checker_result::proceed : Char_checker_result::acceptUnget;
620+
}
621+
622+
//
623+
// PTokIdentifier
624+
//
625+
template<RuleId ruleId>
626+
struct PTokIdentifier : PTokVar<ruleId, pparse_identifier> {
508627
};
509628

510629

@@ -579,6 +698,16 @@ struct PAny : ParserBase {
579698
Json<Stream>::jsonEndTag(out, true);
580699
}
581700

701+
template<typename ParserBase>
702+
static void init_collision_checker(ParserBase &base) {
703+
704+
if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
705+
init_collision_checker_helper<ParserBase, Types...>(base);
706+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
707+
}
708+
}
709+
710+
582711

583712
private:
584713
template<size_t FieldIndex, typename ParserBase, typename PType, typename ...PTypes>
@@ -674,6 +803,16 @@ struct PAny : ParserBase {
674803
return true;
675804
}
676805

806+
template<typename ParserBase, typename PType, typename ...PTypes>
807+
static void init_collision_checker_helper(ParserBase &base) {
808+
809+
PType::init_collision_checker(base);
810+
811+
if constexpr (sizeof...(PTypes) > 0) {
812+
return init_collision_checker_helper<ParserBase, PTypes...>( base );
813+
}
814+
}
815+
677816
};
678817

679818
//
@@ -762,6 +901,17 @@ struct POpt : ParserBase {
762901
Json<Stream>::jsonEndTag(out, true);
763902
}
764903

904+
template<typename ParserBase>
905+
static void init_collision_checker(ParserBase &base) {
906+
if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
907+
PType::init_collision_checker(base);
908+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
909+
}
910+
}
911+
912+
913+
914+
private:
765915

766916
};
767917

@@ -841,6 +991,16 @@ struct PSeq : ParserBase {
841991
Json<Stream>::jsonEndTag(out, true);
842992
}
843993

994+
template<typename ParserBase>
995+
static void init_collision_checker(ParserBase &base) {
996+
if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
997+
init_collision_checker_helper<ParserBase, Types...>(base);
998+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
999+
}
1000+
}
1001+
1002+
1003+
8441004
private:
8451005

8461006
template<size_t FieldIndex, typename Stream, typename PType, typename ...PTypes>
@@ -922,6 +1082,16 @@ struct PSeq : ParserBase {
9221082
}
9231083
#endif
9241084

1085+
template<typename ParserBase, typename PType, typename ...PTypes>
1086+
static void init_collision_checker_helper(ParserBase &base) {
1087+
1088+
PType::init_collision_checker(base);
1089+
1090+
if constexpr (sizeof...(PTypes) > 0) {
1091+
return init_collision_checker_helper<ParserBase, PTypes...>( base );
1092+
}
1093+
}
1094+
9251095

9261096
};
9271097

@@ -970,6 +1140,15 @@ struct PRepeat : ParserBase {
9701140
Json<Stream>::jsonEndTag(out, true);
9711141
}
9721142

1143+
template<typename ParserBase>
1144+
static void init_collision_checker(ParserBase &base) {
1145+
1146+
if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
1147+
Type::init_collision_checker(base);
1148+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
1149+
}
1150+
}
1151+
9731152

9741153
private:
9751154
template<typename ParserBase>
@@ -1156,6 +1335,16 @@ struct PWithAndLookaheadImpl : ParserBase {
11561335
Type::dumpJson(out, ast);
11571336
}
11581337

1338+
template<typename ParserBase>
1339+
static void init_collision_checker(ParserBase &base) {
1340+
1341+
if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) {
1342+
Type::init_collision_checker(base);
1343+
LookaheadType::init_collision_checker(base);
1344+
base.colission_checker_->remove_type_info(&typeid(ThisClass));
1345+
}
1346+
}
1347+
11591348

11601349
};
11611350

0 commit comments

Comments
 (0)
Failed to load comments.