Skip to content

Commit

Permalink
Support shader preprocessor concatenation symbol
Browse files Browse the repository at this point in the history
  • Loading branch information
JohanAR committed Mar 12, 2023
1 parent 550a779 commit 8f12d5d
Show file tree
Hide file tree
Showing 4 changed files with 327 additions and 10 deletions.
65 changes: 55 additions & 10 deletions servers/rendering/shader_preprocessor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -420,11 +420,11 @@ void ShaderPreprocessor::process_define(Tokenizer *p_tokenizer) {
return;
}

Vector<String> args;
if (p_tokenizer->peek() == '(') {
// Macro has arguments.
p_tokenizer->get_token();

Vector<String> args;
while (true) {
String name = p_tokenizer->get_identifier();
if (name.is_empty()) {
Expand All @@ -442,17 +442,24 @@ void ShaderPreprocessor::process_define(Tokenizer *p_tokenizer) {
return;
}
}
}

Define *define = memnew(Define);
String body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
if (body.begins_with("##")) {
set_error(RTR("'##' must not appear at beginning of macro expansion."), line);
return;
}
if (body.ends_with("##")) {
set_error(RTR("'##' must not appear at end of macro expansion."), line);
return;
}

Define *define = memnew(Define);
if (!args.is_empty()) {
define->arguments = args;
define->body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
state->defines[label] = define;
} else {
// Simple substitution macro.
Define *define = memnew(Define);
define->body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
state->defines[label] = define;
}
define->body = body;
state->defines[label] = define;
}

void ShaderPreprocessor::process_elif(Tokenizer *p_tokenizer) {
Expand Down Expand Up @@ -1074,8 +1081,12 @@ bool ShaderPreprocessor::expand_macros_once(const String &p_line, int p_line_num
}
}

result = result.substr(0, index) + " " + body + " " + result.substr(args_end + 1, result.length());
concatenate_macro_body(body);

result = result.substr(0, index) + body + result.substr(args_end + 1, result.length());
} else {
concatenate_macro_body(body);

result = result.substr(0, index) + body + result.substr(index + key.length(), result.length() - (index + key.length()));
}

Expand Down Expand Up @@ -1115,6 +1126,40 @@ bool ShaderPreprocessor::find_match(const String &p_string, const String &p_valu
return false;
}

void ShaderPreprocessor::concatenate_macro_body(String &r_body) {
int index_start = r_body.find("##");
while (index_start > -1) {
int index_end = index_start + 2; // First character after ##
// The macro was checked during creation so this should never happen
ERR_FAIL_INDEX(index_end, r_body.size());

// If there more than two # in a row, then it's not a concatenation.
bool is_concat = true;
while (index_end <= r_body.length() && r_body[index_end] == '#') {
index_end++;
is_concat = false;
}
if (!is_concat) {
index_start = r_body.find("##", index_end);
continue;
}

// Skip whitespace after ##
while (index_end < r_body.length() && is_char_space(r_body[index_end])) {
index_end++;
}

// Skip whitespace before ##
while (index_start >= 1 && is_char_space(r_body[index_start - 1])) {
index_start--;
}

r_body = r_body.substr(0, index_start) + r_body.substr(index_end, r_body.length() - index_end);

index_start = r_body.find("##", index_start);
}
}

String ShaderPreprocessor::next_directive(Tokenizer *p_tokenizer, const Vector<String> &p_directives) {
const int line = p_tokenizer->get_line();
int nesting = 0;
Expand Down
1 change: 1 addition & 0 deletions servers/rendering/shader_preprocessor.h
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ class ShaderPreprocessor {
Error expand_macros(const String &p_string, int p_line, String &r_result);
bool expand_macros_once(const String &p_line, int p_line_number, const RBMap<String, Define *>::Element *p_define_pair, String &r_expanded);
bool find_match(const String &p_string, const String &p_value, int &r_index, int &r_index_start);
void concatenate_macro_body(String &r_body);

String next_directive(Tokenizer *p_tokenizer, const Vector<String> &p_directives);
void add_to_output(const String &p_str);
Expand Down
270 changes: 270 additions & 0 deletions tests/servers/rendering/test_shader_preprocessor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
/**************************************************************************/
/* test_shader_preprocessor.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/

#ifndef TEST_SHADER_PREPROCESSOR_H
#define TEST_SHADER_PREPROCESSOR_H

#include "servers/rendering/shader_preprocessor.h"

#include "tests/test_macros.h"

namespace TestShaderPreprocessor {

void erase_all_empty(Vector<String> &p_vec) {
int idx = p_vec.find(" ");
while (idx >= 0) {
p_vec.remove_at(idx);
idx = p_vec.find(" ");
}
}

// The preprocessor changes indentation and leaves some empty lines where the
// defines where. Compact multiples of the same whitespace into a single
// before comparing with expecations, since these ought to be equivalent for
// shader code.
String compact_spaces(String &p_str) {
String compacted = p_str.strip_edges();
compacted = String(" ").join(compacted.split(" ", false));
Vector<String> lines = compacted.split("\n", false);
erase_all_empty(lines);
compacted = String("\n").join(lines);
return compacted;
}

#define CHECK_SHADER_EQ(a, b) CHECK_EQ(compact_spaces(a), compact_spaces(b))
#define CHECK_SHADER_NE(a, b) CHECK_NE(compact_spaces(a), compact_spaces(b))

TEST_CASE("[ShaderPreprocessor] Simple defines") {
String code(
"#define X 1.0 // comment\n"
"#define Y mix\n"
"#define Z X\n"
"\n"
"#define func0 \\\n"
" vec3 my_fun(vec3 arg) {\\\n"
" return pow(arg, 2.2);\\\n"
" }\n"
"\n"
"func0\n"
"\n"
"fragment() {\n"
" ALBEDO = vec3(X);\n"
" float x = Y(0., Z, X);\n"
" #undef X\n"
" float X = x;\n"
" x = -Z;\n"
"}\n");
String expected(
"vec3 my_fun(vec3 arg) { return pow(arg, 2.2); }\n"
"\n"
"fragment() {\n"
" ALBEDO = vec3(1.0);\n"
" float x = mix(0., 1.0, 1.0);\n"
" float X = x;\n"
" x = -X;\n"
"}\n");
String result;

ShaderPreprocessor preprocessor;
CHECK_EQ(preprocessor.preprocess(code, String("file.gdshader"), result), Error::OK);

CHECK_SHADER_EQ(result, expected);
}

TEST_CASE("[ShaderPreprocessor] Complex defines") {
String code(
"const float X = 2.0;\n"
"#define A(X) X*2.\n"
"#define X 1.0\n"
"#define Y Z(X, W)\n"
"#define Z max\n"
"#define C(X, Y) Z(A(Y), B(X))\n"
"#define W -X\n"
"#define B(X) X*3.\n"
"\n"
"fragment() {\n"
" float x = Y;\n"
" float y = C(5., 7.0);\n"
"}\n");
String expected(
"const float X = 2.0;\n"
"fragment() {\n"
" float x = max(1.0, -1.0);\n"
" float y = max(7.0*2., 5.*3.);\n"
"}\n");
String result;

ShaderPreprocessor preprocessor;
CHECK_EQ(preprocessor.preprocess(code, String("file.gdshader"), result), Error::OK);

CHECK_SHADER_EQ(result, expected);
}

TEST_CASE("[ShaderPreprocessor] Concatenation") {
String code(
"fragment() {\n"
" #define X 1 // this is fine ##\n"
" #define y 2\n"
" #define z 3##.## 1## 4 ## 59\n"
" #define Z(y) X ## y\n"
" #define Z2(y) y##X\n"
" #define W(y) X, y\n"
" #define A(x) fl## oat a = 1##x ##.3 ## x\n"
" #define C(x, y) x##.##y\n"
" float Z(y) = 1.2;\n"
" float Z(z) = 2.3;\n"
" float Z2(y) = z;\n"
" float Z2(z) = 2.3;\n"
" int b = max(W(3));\n"
" A(9);\n"
" Xy = C(X, y);\n"
"}\n");
String expected(
"fragment() {\n"
" float Xy = 1.2;\n"
" float Xz = 2.3;\n"
" float yX = 3.1459;\n"
" float zX = 2.3;\n"
" int b = max(1, 3);\n"
" float a = 19.39;\n"
" Xy = 1.2;\n"
"}\n");
String result;

ShaderPreprocessor preprocessor;
CHECK_EQ(preprocessor.preprocess(code, String("file.gdshader"), result), Error::OK);

CHECK_SHADER_EQ(result, expected);
}

TEST_CASE("[ShaderPreprocessor] Nested concatenation") {
// Concatenation ## should not expand adjacent tokens if they are macros,
// but this is currently not implemented in Godot's shader preprocessor.
// To force expanding, an extra macro should be required (B in this case).

String code(
"fragment() {\n"
" vec2 X = vec2(0);\n"
" #define X 1\n"
" #define y 2\n"
" #define B(x, y) C(x, y)\n"
" #define C(x, y) x##.##y\n"
" C(X, y) = B(X, y);\n"
"}\n");
String expected(
"fragment() {\n"
" vec2 X = vec2(0);\n"
" X.y = 1.2;\n"
"}\n");
String result;

ShaderPreprocessor preprocessor;
CHECK_EQ(preprocessor.preprocess(code, String("file.gdshader"), result), Error::OK);

// TODO: Reverse the check when/if this is changed
CHECK_SHADER_NE(result, expected);
}

TEST_CASE("[ShaderPreprocessor] Concatenation sorting network") {
String code(
"fragment() {\n"
" #define ARR(X) test##X\n"
" #define ACMP(a, b) ARR(a) > ARR(b)\n"
" #define ASWAP(a, b) tmp = ARR(b); ARR(b) = ARR(a); ARR(a) = tmp;\n"
" #define ACSWAP(a, b) if(ACMP(a, b)) { ASWAP(a, b) }\n"
" float test0 = 1.2;\n"
" float test1 = 0.34;\n"
" float test3 = 0.8;\n"
" float test4 = 2.9;\n"
" float tmp;\n"
" ACSWAP(0,2)\n"
" ACSWAP(1,3)\n"
" ACSWAP(0,1)\n"
" ACSWAP(2,3)\n"
" ACSWAP(1,2)\n"
"}\n");
String expected(
"fragment() {\n"
" float test0 = 1.2;\n"
" float test1 = 0.34;\n"
" float test3 = 0.8;\n"
" float test4 = 2.9;\n"
" float tmp;\n"
" if(test0 > test2) { tmp = test2; test2 = test0; test0 = tmp; }\n"
" if(test1 > test3) { tmp = test3; test3 = test1; test1 = tmp; }\n"
" if(test0 > test1) { tmp = test1; test1 = test0; test0 = tmp; }\n"
" if(test2 > test3) { tmp = test3; test3 = test2; test2 = tmp; }\n"
" if(test1 > test2) { tmp = test2; test2 = test1; test1 = tmp; }\n"
"}\n");
String result;

ShaderPreprocessor preprocessor;
CHECK_EQ(preprocessor.preprocess(code, String("file.gdshader"), result), Error::OK);

CHECK_SHADER_EQ(result, expected);
}

TEST_CASE("[ShaderPreprocessor] Undefined behaviour") {
// None of these are valid concatenation, nor valid shader code.
// Don't care about results, just make sure there's no crash.
const String filename("somefile.gdshader");
String result;
ShaderPreprocessor preprocessor;

preprocessor.preprocess("#define X ###\nX\n", filename, result);
preprocessor.preprocess("#define X ####\nX\n", filename, result);
preprocessor.preprocess("#define X #####\nX\n", filename, result);
preprocessor.preprocess("#define X 1 ### 2\nX\n", filename, result);
preprocessor.preprocess("#define X 1 #### 2\nX\n", filename, result);
preprocessor.preprocess("#define X 1 ##### 2\nX\n", filename, result);
preprocessor.preprocess("#define X ### 2\nX\n", filename, result);
preprocessor.preprocess("#define X #### 2\nX\n", filename, result);
preprocessor.preprocess("#define X ##### 2\nX\n", filename, result);
preprocessor.preprocess("#define X 1 ###\nX\n", filename, result);
preprocessor.preprocess("#define X 1 ####\nX\n", filename, result);
preprocessor.preprocess("#define X 1 #####\nX\n", filename, result);
}

TEST_CASE("[ShaderPreprocessor] Invalid concatenations") {
const String filename("somefile.gdshader");
String result;
ShaderPreprocessor preprocessor;

CHECK_NE(preprocessor.preprocess("#define X ##", filename, result), Error::OK);
CHECK_NE(preprocessor.preprocess("#define X 1 ##", filename, result), Error::OK);
CHECK_NE(preprocessor.preprocess("#define X ## 1", filename, result), Error::OK);
CHECK_NE(preprocessor.preprocess("#define X(y) ## ", filename, result), Error::OK);
CHECK_NE(preprocessor.preprocess("#define X(y) y ## ", filename, result), Error::OK);
CHECK_NE(preprocessor.preprocess("#define X(y) ## y", filename, result), Error::OK);
}

} // namespace TestShaderPreprocessor

#endif // TEST_SHADER_PREPROCESSOR_H
1 change: 1 addition & 0 deletions tests/test_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
#include "tests/scene/test_theme.h"
#include "tests/scene/test_viewport.h"
#include "tests/scene/test_visual_shader.h"
#include "tests/servers/rendering/test_shader_preprocessor.h"
#include "tests/servers/test_text_server.h"
#include "tests/test_validate_testing.h"

Expand Down

0 comments on commit 8f12d5d

Please sign in to comment.