Add comments

bia/tokenizer/tests/tokenizer.cpp

@@ -16,11 +16,10 @@ using namespace bia::string;
 class mock_receiver : public token::receiver
 {
 public:
-	void receive(bia::util::not_null<const token::token*> first,
-	             bia::util::not_null<const token::token*> last) override
+	void receive(bia::util::span<const token::token> tokens) override
 	{
-		for (auto i = first.get(), c = last.get(); i != c; ++i) {
-			tokens.push_back(*i);
+		for (const auto& i : tokens) {
+			this->tokens.push_back(i);
 		}
 	}
 

bia/tokenizer/tests/tokens.cpp

@@ -6,6 +6,7 @@
 #include <tokenizer/token/parse/any_of.hpp>
 #include <tokenizer/token/parse/number.hpp>
 #include <tokenizer/token/parse/string.hpp>
+#include <tokenizer/token/parse/whitespace_eater.hpp>
 #include <util/finally.hpp>
 #include <utility>
 
@@ -81,3 +82,18 @@ TEST_CASE("string", "[tokenizer]")
 	parse(R"("hi\aasd\bqwe\vasd\tasd\fasdw\bqwe\nwqd\rdasd\\awd\"ad\"")",
 	      "hi\aasd\bqwe\vasd\tasd\fasdw\bqwe\nwqd\rdasd\\awd\"ad\"");
 }
+
+TEST_CASE("whitespaces", "[tokenizer]")
+{
+	const auto consumed = [](const char* str, bool expect_error) -> std::streamoff {
+		const auto param = create_parameter(str);
+
+		REQUIRE(eat_whitespaces(*param).valid == expect_error);
+
+		return expect_error ? -1 : static_cast<std::streamoff>(param->input.tellg());
+	};
+
+	REQUIRE(consumed(" \t\f\v", false) == 4);
+	consumed("asd \r\f", true);
+	REQUIRE(consumed(" /* kartoffel */asd", false) == 16);
+}

bia/tokenizer/tokenizer/token/parse/whitespace_eater.hpp

@@ -22,32 +22,108 @@ namespace parse {
 template<bool RequireCmd = false>
 inline exception::syntax_details eat_whitespaces(parameter& parameter)
 {
+	enum class s
+	{
+		whitespace,
+		comment_start,
+		comment_single,
+		comment_single_ending,
+		comment_multi,
+		comment_multi_ending
+	};
+
+	auto state = s::whitespace;
 	auto eaten = false;
 	auto cmd   = false;
 
 	while (true) {
 		const auto pos = parameter.input.tellg();
+		const auto cp  = parameter.encoder.read(parameter.input);
+
+		if (cp == string::encoding::encoder::eof) {
+			eaten = true;
+			cmd   = true;
+
+			goto gt_return;
+		}
 
-		switch (parameter.encoder.read(parameter.input)) {
-		case '\r':
-		case '\n': cmd = true;
-		case ' ':
-		case '\t':
-		case '\v':
-		case '\f': eaten = true; break;
-		case string::encoding::encoder::eof: cmd = true; eaten = true;
-		default: {
-			parameter.input.seekg(pos);
-
-			if (!eaten) {
-				return { pos, "expected whitespace" };
-			} else if (RequireCmd && !cmd) {
-				return { pos, "expected cmd whitespace, like a line feed" };
+		switch (state) {
+		case s::whitespace: {
+			switch (cp) {
+			case '/': state = s::comment_start; break;
+			case '\r':
+			case '\n': cmd = true;
+			case ' ':
+			case '\t':
+			case '\v':
+			case '\f': eaten = true; break;
+			default: goto gt_return;
+			}
+
+			break;
+		}
+		case s::comment_start: {
+			switch (cp) {
+			case '/': state = s::comment_single; break;
+			case '*': state = s::comment_multi; break;
+			default: goto gt_return;
 			}
 
-			return {};
+			break;
 		}
+		case s::comment_single: {
+			if (cp == '\r' || cp == '\n') {
+				cmd   = true;
+				state = s::comment_single_ending;
+			}
+
+			break;
 		}
+		case s::comment_single_ending: {
+			switch (cp) {
+			case '\r':
+			case '\n': break;
+			default: {
+				parameter.input.seekg(pos);
+
+				state = s::whitespace;
+
+				break;
+			}
+			}
+
+			break;
+		}
+		case s::comment_multi: {
+			if (cp == '*') {
+				state = s::comment_multi_ending;
+			}
+
+			break;
+		}
+		case s::comment_multi_ending: {
+			if (cp == '/') {
+				state = s::whitespace;
+			} else if (cp != '*') {
+				state = s::comment_multi;
+			}
+
+			break;
+		}
+		}
+
+		continue;
+
+	gt_return:;
+		parameter.input.seekg(pos);
+
+		if (!eaten) {
+			return { pos, "expected whitespace" };
+		} else if (RequireCmd && !cmd) {
+			return { pos, "expected cmd whitespace, like a line feed" };
+		}
+
+		return {};
 	}
 }