Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

file 85 lines (74 sloc) 1.752 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
#ifndef INCLUDE_TOKEN_H
#define INCLUDE_TOKEN_H

// token.h
// Revision 10-jul-2010

#include <string>
#include <istream>
#include <vector>

enum TokenType
{
TokenTEOF,
TokenTComment,
TokenTWhiteSpace,
TokenTInteger,
TokenTSingleQuoted,
TokenTQuoted,
TokenTIdentifier,
TokenTOperator,
TokenTUnknown
};

class Token
{
public:
Token ();
Token (TokenType tt);
Token (TokenType tt, const std::string &file);
Token (const std::string &ss, unsigned int linenum, const std::string &file);
Token (TokenType type, const std::string &ss, unsigned int linenum, const std::string &file);
Token (TokenType type, const std::string &ss, const Token &base);
Token (bool value, const Token &base);
Token (int value, const Token &base);

bool empty () const;
int getinteger() const;
std::string str() const;
std::string identifier() const;
std::string pirliteralstring() const;
std::string describe() const;
unsigned int linenum() const;
std::string file() const;
bool isspace() const;
bool isidentifier() const;
bool isinteger() const;
bool issinglequoted() const;
bool isliteralstring() const;
bool isop(const std::string &name) const;
bool isop(char name) const;
bool iskeyword(const std::string &name) const;
private:
TokenType ttype;
std::string s;
unsigned int ln;
std::string filename;
};

std::string unquote (const std::string &s);

class Tokenizer
{
public:
Tokenizer (std::istream &is_a, const char *filename= 0);
Token get ();
void unget (const Token & t);
private:
char getchar();
void ungetchar(char c);
Token getquoted();
Token getsinglequoted();
Token getheredoc();
Token getany();

std::istream &is;
std::string name;
unsigned int ln;
char unc;
std::vector<Token> untoc;
};


#endif

// End of token.h
Something went wrong with that request. Please try again.