Skip to content
This repository
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 84 lines (72 sloc) 1.705 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
#ifndef INCLUDE_TOKEN_H
#define INCLUDE_TOKEN_H

// token.h
// Revision 24-may-2012

#include <string>
#include <istream>
#include <vector>

enum TokenType
{
TokenTEOF,
TokenTComment,
TokenTWhiteSpace,
TokenTInteger,
TokenTQuoted,
TokenTIdentifier,
TokenTOperator,
TokenTUnknown
};

class Token
{
public:
Token ();
Token (TokenType tt);
Token (TokenType tt, const std::string &file);
Token (const std::string &ss, unsigned int linenum, const std::string &file);
Token (TokenType type, const std::string &ss, unsigned int linenum, const std::string &file);
Token (TokenType type, const std::string &ss, const Token &base);
Token (bool value, const Token &base);
Token (int value, const Token &base);

bool empty () const;
int getinteger() const;
std::string str() const;
std::string identifier() const;
std::string pirliteralstring() const;
std::string describe() const;
unsigned int linenum() const;
std::string file() const;
bool isspace() const;
bool isidentifier() const;
bool isinteger() const;
bool issinglequoted() const;
bool isliteralstring() const;
bool isop(const std::string &name) const;
bool isop(char name) const;
bool iskeyword(const std::string &name) const;
private:
TokenType ttype;
std::string s;
unsigned int ln;
std::string filename;
};

std::string unquote (const std::string &s);

class Tokenizer
{
public:
Tokenizer (std::istream &is_a, const char *filename= 0);
Token get ();
void unget (const Token & t);
private:
char getchar();
void ungetchar(char c);
Token getquoted();
Token getheredoc();
Token getany();

std::istream &is;
std::string name;
unsigned int ln;
char unc;
std::vector<Token> untoc;
};


#endif

// End of token.h
Something went wrong with that request. Please try again.