Skip to content
This repository
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 166 lines (139 sloc) 3.544 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
#include <algorithm>
#include <ctype.h>
#include <string.h>

#include "json_tokenizer.hpp"
#include "unit_test.hpp"
#include "string_utils.hpp"

namespace json {

Token get_token(const char*& i1, const char* i2)
{
while(i1 != i2 && util::c_isspace(*i1) || *i1 == '#' || *i1 == '/' && i1+1 != i2 && (*(i1 + 1) == '/' || *(i1 + 1) == '*')) {
if(*i1 == '/' && *(i1 + 1) == '*') {
const char* begin = i1;
i1 += 2;

int nesting = 1;
while(i1 != i2) {
if(i1+1 != i2) {
if(*i1 == '/' && *(i1+1) == '*') {
++nesting;
} else if(*i1 == '*' && *(i1+1) == '/') {
if(--nesting == 0) {
++i1;
break;
}
}
}

++i1;
}

if(i1 == i2) {
TokenizerError error = { "Unexpected end of file while parsing string", begin };
throw error;
}

++i1;
} else if(*i1 == '#' || *i1 == '/') {
//ignore comments.
i1 = std::find(i1, i2, '\n');
} else {
++i1;
}
}

if(i1 == i2) {
Token result;
result.type = Token::NUM_TYPES;
result.begin = result.end = NULL;
return result;
}

if(strchr("{}[]:,", *i1)) {
Token result;
result.begin = i1;
result.end = i1+1;
switch(*i1) {
case '{': result.type = Token::TYPE_LCURLY; break;
case '}': result.type = Token::TYPE_RCURLY; break;
case '[': result.type = Token::TYPE_LSQUARE; break;
case ']': result.type = Token::TYPE_RSQUARE; break;
case ':': result.type = Token::TYPE_COLON; break;
case ',': result.type = Token::TYPE_COMMA; break;
}

i1 = result.end;
return result;
}

if(*i1 == '"' || *i1 == '\'' || *i1 == '~') {
const char quote_type = *i1;
Token result;
result.translate = quote_type == '~';
result.type = Token::TYPE_STRING;
result.begin = ++i1;
while(i1 != i2) {
if(*i1 == quote_type) {
break;
} else if(*i1 == '\\') {
++i1;
if(i1 == i2) {
break;
}
}

++i1;
}

if(i1 == i2) {
TokenizerError error = { "Unexpected end of file while parsing string", result.begin };
throw error;
}

result.end = i1;
++i1;
return result;
} else if(util::c_isalpha(*i1) || *i1 == '_') {
Token result;
result.begin = i1;
while(i1 != i2 && (util::c_isalnum(*i1) || *i1 == '_')) {
++i1;
}

result.end = i1;

if(result.end - result.begin == 4 && !memcmp("true", result.begin, 4)) {
result.type = Token::TYPE_TRUE_VALUE;
} else if(result.end - result.begin == 5 && !memcmp("false", result.begin, 5)) {
result.type = Token::TYPE_FALSE_VALUE;
} else if(result.end - result.begin == 4 && !memcmp("null", result.begin, 4)) {
result.type = Token::TYPE_NULL_VALUE;
} else {
result.type = Token::TYPE_IDENTIFIER;
}

return result;
}

if(*i1 == '-' || *i1 == '.' || util::c_isdigit(*i1)) {
bool seen_decimal = false;
Token result;
result.type = Token::TYPE_NUMBER;
result.begin = i1;
while(i1 != i2) {
if(*i1 == '.') {
if(seen_decimal) {
TokenizerError error = { "Two decimal points found in number", i1 };
throw error;

}

seen_decimal = true;
} else if(*i1 == '-') {
if(i1 != result.begin) {
TokenizerError error = { "- found in illegal position in number", i1 };
throw error;
}
} else if(!util::c_isdigit(*i1)) {
break;
}

++i1;
}

result.end = i1;
return result;
}

TokenizerError error = { "Unexpected character found", i1 };
throw error;
}

Token get_token_full(const char*& i1, const char* i2) {
Token res = get_token(i1, i2);
if(res.type == Token::TYPE_STRING) {
res.begin--;
res.end++;
}

return res;
}

}
Something went wrong with that request. Please try again.