DEV: Rework shader generation in Java (#173)

Shader generation is now written in Java, rather than Python, reducing the extent to which Python is a dependency and making testing of shader generation simpler as it can be done in JUnit.

Fixed an issue where do ... while loops were generated in WebGL 1.0 shaders.

Fixed an infinite loop issue in the generator, where if no available transformation could make progress the transformations would be applied forever.

Added a workaround for a memory exhaustion issue in shader translator.

Added tests for shader family generation.

Changed generation top-level script so that - is used instead of _ to separate arguments, in line with the way glsl-reduce works.

Changed the order of arguments to glsl-generate so that the references are specified before the donors. Removed the need for the user to explicitly make the output directory for shaders.

Replaced --avoid-long-loops with an opposite option, --allow-long-loops, so that long loops are disabled by default.

Replaced --multi-pass with an opposite --single-pass option, so that multi-pass is default.

Made --max-bytes default to 500000, which has been found to work well.

Updated glsl-fuzz walkthrough to reflect these changes.

ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java

@@ -118,13 +118,19 @@ static ShadingLanguageVersion fromVersionString(String versionString) {
     throw new RuntimeException("Unknown version string " + versionString);
   }
 
+  static boolean isWebGlCompatible(String versionString) {
+    return allWebGlSlVersions().stream().map(item -> item.getVersionString())
+        .anyMatch(item -> item.equals(versionString));
+  }
+
   static ShadingLanguageVersion webGlFromVersionString(String versionString) {
-    for (ShadingLanguageVersion shadingLanguageVersion : allWebGlSlVersions()) {
-      if (shadingLanguageVersion.getVersionString().equals(versionString)) {
-        return shadingLanguageVersion;
-      }
+    if (!isWebGlCompatible(versionString)) {
+      throw new RuntimeException("Unknown WebGL shading language version string " + versionString);
     }
-    throw new RuntimeException("Unknown WebGL shading language version string " + versionString);
+    return allWebGlSlVersions()
+        .stream()
+        .filter(item -> item.getVersionString().equals(versionString))
+        .findAny().get();
   }
 
   String getVersionString();

ast/src/main/java/com/graphicsfuzz/common/glslversion/WebGlSl.java

@@ -35,4 +35,10 @@ public boolean isWebGl() {
     return true;
   }
 
+  @Override
+  public boolean supportedDoStmt() {
+    // do ... while loops are not supported in WebGL 1.0
+    return false;
+  }
+
 }

ast/src/main/java/com/graphicsfuzz/common/util/ShaderTranslatorShadingLanguageVersionSupport.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 The GraphicsFuzz Project Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.graphicsfuzz.common.util;
+
+import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion;
+import java.util.Arrays;
+
+public final class ShaderTranslatorShadingLanguageVersionSupport {
+
+  private ShaderTranslatorShadingLanguageVersionSupport() {
+    // Utility class
+  }
+
+  public static boolean isVersionSupported(ShadingLanguageVersion shadingLanguageVersion) {
+    // Shader translator does have a -s=e31 option, but it is still marked as "in development".
+    return Arrays.asList(ShadingLanguageVersion.WEBGL_SL,
+        ShadingLanguageVersion.WEBGL2_SL,
+        ShadingLanguageVersion.ESSL_100,
+        ShadingLanguageVersion.ESSL_300).contains(shadingLanguageVersion);
+  }
+
+  public static String getShaderTranslatorArgument(ShadingLanguageVersion shadingLanguageVersion) {
+    if (!isVersionSupported(shadingLanguageVersion)) {
+      throw new IllegalArgumentException("Shader translator does not support the given shading"
+          + " language version.");
+    }
+    if (shadingLanguageVersion == ShadingLanguageVersion.WEBGL_SL) {
+      return "-s=w";
+    } else if (shadingLanguageVersion == ShadingLanguageVersion.WEBGL2_SL) {
+      return "-s=w2";
+    } else if (shadingLanguageVersion == ShadingLanguageVersion.ESSL_100) {
+      return "-s=e2";
+    }
+    assert shadingLanguageVersion == ShadingLanguageVersion.ESSL_300;
+    return "-s=e3";
+  }
+
+}

ast/src/test/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersionTest.java

@@ -17,8 +17,12 @@
 package com.graphicsfuzz.common.glslversion;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
+import com.graphicsfuzz.common.util.ShaderTranslatorShadingLanguageVersionSupport;
+import com.graphicsfuzz.util.ExecHelper;
 import com.graphicsfuzz.util.ExecHelper.RedirectType;
 import com.graphicsfuzz.util.ExecResult;
 import com.graphicsfuzz.util.ToolHelper;
@@ -105,48 +109,58 @@ private void checkDoStmtSupport(ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final boolean expectedInvalid = !shadingLanguageVersion.supportedDoStmt();
     final String program = programWithDoStmt(shadingLanguageVersion).toString();
-    checkValidity(expectedInvalid, program);
+    checkValidity(expectedInvalid, program, shadingLanguageVersion);
   }
 
   private void checkSwitchStmtSupport(ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final boolean expectedInvalid = !shadingLanguageVersion.supportedSwitchStmt();
     final String program = programWithSwitchStmt(shadingLanguageVersion).toString();
-    checkValidity(expectedInvalid, program);
+    checkValidity(expectedInvalid, program, shadingLanguageVersion);
   }
 
   private void checkUnsignedSupport(ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final boolean expectedInvalid = !shadingLanguageVersion.supportedUnsigned();
     final String program = programWithUnsigned(shadingLanguageVersion).toString();
-    checkValidity(expectedInvalid, program);
+    checkValidity(expectedInvalid, program, shadingLanguageVersion);
   }
 
   private void checkInitializersOfConstMustBeConst(ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final boolean expectedInvalid = shadingLanguageVersion.initializersOfConstMustBeConst();
     final String program = constInitializedWithNonConst(shadingLanguageVersion).toString();
-    checkValidity(expectedInvalid, program);
+    checkValidity(expectedInvalid, program, shadingLanguageVersion);
   }
 
   private void checkGlobalVariableInitializersMustBeConst(ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final boolean expectedInvalid = shadingLanguageVersion.globalVariableInitializersMustBeConst();
     final String program = globalWithNonConstInitializer(shadingLanguageVersion).toString();
-    checkValidity(expectedInvalid, program);
+    checkValidity(expectedInvalid, program, shadingLanguageVersion);
   }
 
-  private void checkValidity(boolean expectedInvalid, String program)
+  private void checkValidity(boolean expectedInvalid, String program,
+                             ShadingLanguageVersion shadingLanguageVersion)
       throws IOException, InterruptedException {
     final File shader = temporaryFolder.newFile("temp.frag");
     FileUtils.writeStringToFile(shader,
         program,
         StandardCharsets.UTF_8);
-    final ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, shader);
+    final boolean glslangValidatorSaysOk = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER,
+        shader).res == 0;
+    final boolean shaderTranslatorSaysOk =
+        !ShaderTranslatorShadingLanguageVersionSupport.isVersionSupported(shadingLanguageVersion)
+          || ToolHelper.runShaderTranslatorOnShader(RedirectType.TO_BUFFER, shader,
+              ShaderTranslatorShadingLanguageVersionSupport
+                  .getShaderTranslatorArgument(shadingLanguageVersion)).res == 0;
     if (expectedInvalid) {
-      assertNotEquals(0, result.res);
+      // If the shader is supposed to be invalid, at least one of the tools should say so
+      // (they sometimes differ in their strictness)
+      assertFalse(shaderTranslatorSaysOk && glslangValidatorSaysOk);
     } else {
-      assertEquals(0, result.res);
+      // If it is supposed to be valid, both tools should say so.
+      assertTrue(shaderTranslatorSaysOk && glslangValidatorSaysOk);
     }
     shader.delete();
   }

common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java

@@ -21,6 +21,7 @@
 import com.google.gson.JsonObject;
 import com.graphicsfuzz.alphanumcomparator.AlphanumComparator;
 import com.graphicsfuzz.common.ast.TranslationUnit;
+import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion;
 import com.graphicsfuzz.common.tool.PrettyPrinterVisitor;
 import com.graphicsfuzz.common.transformreduce.GlslShaderJob;
 import com.graphicsfuzz.common.transformreduce.ShaderJob;
@@ -129,27 +130,35 @@ public boolean areImagesOfShaderResultsSimilar(
 
   }
 
-  @SuppressWarnings("RedundantIfStatement")
   public boolean areShadersValid(
       File shaderJobFile,
       boolean throwExceptionOnInvalid)
       throws IOException, InterruptedException {
-    assertIsShaderJobFile(shaderJobFile);
-
-    String shaderJobFileNoExtension = FilenameUtils.removeExtension(shaderJobFile.toString());
-
-    final File fragmentShaderFile = new File(shaderJobFileNoExtension + ".frag");
-    final File vertexShaderFile = new File(shaderJobFileNoExtension + ".vert");
-    if (!isFile(fragmentShaderFile) && !isFile(vertexShaderFile)) {
-      throw new IllegalStateException("No frag or vert shader found for " + shaderJobFile);
-    }
-    if (isFile(fragmentShaderFile)
-        && !shaderIsValid(fragmentShaderFile, throwExceptionOnInvalid)) {
-      return false;
+    for (ShaderKind shaderKind : ShaderKind.values()) {
+      //noinspection deprecation: OK from within this class.
+      final File shaderFile = getUnderlyingShaderFile(shaderJobFile, shaderKind);
+      if (shaderFile.isFile() && !shaderIsValid(shaderFile, throwExceptionOnInvalid)) {
+        return false;
+      }
     }
-    if (isFile(vertexShaderFile)
-        && !shaderIsValid(vertexShaderFile, throwExceptionOnInvalid)) {
-      return false;
+    return true;
+  }
+
+  public boolean areShadersValidShaderTranslator(
+      File shaderJobFile,
+      boolean throwExceptionOnInvalid)
+      throws IOException, InterruptedException {
+    for (ShaderKind shaderKind : ShaderKind.values()) {
+      //noinspection deprecation: OK from within this class.
+      final File shaderFile = getUnderlyingShaderFile(shaderJobFile, shaderKind);
+      if (shaderFile.isFile()) {
+        final ShadingLanguageVersion shadingLanguageVersion = ShadingLanguageVersion
+            .getGlslVersionFromFirstTwoLines(getFirstTwoLinesOfShader(shaderJobFile, shaderKind));
+        if (!shaderIsValidShaderTranslator(shaderFile, shadingLanguageVersion,
+            throwExceptionOnInvalid)) {
+          return false;
+        }
+      }
     }
     return true;
   }
@@ -401,6 +410,10 @@ public void mkdir(File directory) throws IOException {
     Files.createDirectories(directory.toPath());
   }
 
+  public void forceMkdir(File outputDir) throws IOException {
+    FileUtils.forceMkdir(outputDir);
+  }
+
   public void moveFile(File srcFile, File destFile, boolean replaceExisting) throws IOException {
     if (replaceExisting) {
       Files.move(srcFile.toPath(), destFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
@@ -1041,14 +1054,51 @@ private void setPrimitives(ImageJob imageJob, File primitivesFile) throws IOExce
     }
   }
 
-  @SuppressWarnings("BooleanMethodIsAlwaysInverted")
   private boolean shaderIsValid(
       File shaderFile,
       boolean throwExceptionOnValidationError)
       throws IOException, InterruptedException {
-    ExecResult res = ToolHelper.runValidatorOnShader(ExecHelper.RedirectType.TO_LOG, shaderFile);
+    return checkValidationResult(ToolHelper.runValidatorOnShader(ExecHelper.RedirectType.TO_BUFFER,
+        shaderFile), shaderFile.getName(), throwExceptionOnValidationError);
+  }
+
+  private boolean shaderIsValidShaderTranslator(
+      File shaderFile,
+      ShadingLanguageVersion shadingLanguageVersion,
+      boolean throwExceptionOnValidationError)
+      throws IOException, InterruptedException {
+    if (!ShaderTranslatorShadingLanguageVersionSupport.isVersionSupported(shadingLanguageVersion)) {
+      // Shader translator does not support this shading language version, so just say that the
+      // shader is valid.
+      return true;
+    }
+    final ExecResult shaderTranslatorResult = ToolHelper.runShaderTranslatorOnShader(
+        ExecHelper.RedirectType.TO_BUFFER,
+        shaderFile,
+        ShaderTranslatorShadingLanguageVersionSupport
+            .getShaderTranslatorArgument(shadingLanguageVersion));
+    if (isMemoryExhaustedError(shaderTranslatorResult)) {
+      return true;
+    }
+    return checkValidationResult(
+        shaderTranslatorResult,
+        shaderFile.getName(),
+        throwExceptionOnValidationError);
+  }
+
+  // TODO(171): This is a workaround for an issue where shader_translator reports memory exhaustion.
+  // If the issue in shader_translator can be fixed, we should get rid of this check.
+  private boolean isMemoryExhaustedError(ExecResult shaderTranslatorResult) {
+    return shaderTranslatorResult.res != 0
+        && shaderTranslatorResult.stdout.toString().contains("memory exhausted");
+  }
+
+  private boolean checkValidationResult(ExecResult res,
+                                        String filename, boolean throwExceptionOnValidationError) {
     if (res.res != 0) {
-      LOGGER.warn("Shader {} failed to validate.", shaderFile.getName());
+      LOGGER.warn("Shader {} failed to validate.  Validator stdout: " + res.stdout + ".  "
+              + "Validator stderr: " + res.stderr,
+          filename);
       if (throwExceptionOnValidationError) {
         throw new RuntimeException("Validation failed.");
       }
@@ -1057,6 +1107,7 @@ private boolean shaderIsValid(
     return true;
   }
 
+
   /**
    * This method is quite complicated compared to others in this class,
    * so it is static in case we want to test it via a mocked ShaderJobFileOperations fileOps.

docs/glsl-fuzz-walkthrough.md

@@ -126,23 +126,19 @@ We can create some shader families from our provided sample shader jobs as follo
 # Copy the sample shaders into the current directory:
 cp -r graphicsfuzz-1.0/shaders/samples samples
 
-# Create a work directory to store our generated shader families.
-# The directory structure will allow the server
-# to find the shaders later.
-mkdir -p work/shaderfamilies
-
 # Generate several shader families from the set of sample shaders.
+# Placing the generated shaders under work/shaderfamilies will allow the server to find the shaders later.
 # Synopsis:
-# glsl-generate [options] donors references num_variants glsl_version prefix output_folder
+# glsl-generate [options] references donors num_variants glsl_version prefix output_folder
 
 # Generate some GLSL version 300 es shaders.
-glsl-generate --max_bytes 500000 --multi_pass samples/donors samples/300es 10 "300 es" family_300es work/shaderfamilies
+glsl-generate samples/300es samples/donors 10 "300 es" family_300es work/shaderfamilies
 
 # Generate some GLSL version 100 shaders.
-glsl-generate --max_bytes 500000 --multi_pass samples/donors samples/100 10 "100" family_100 work/shaderfamilies
+glsl-generate samples/100 samples/donors 10 "100" family_100 work/shaderfamilies
 
 # Generate some "Vulkan-compatible" GLSL version 300 es shaders that can be translated to SPIR-V for Vulkan testing.
-glsl-generate --max_bytes 500000 --multi_pass --generate_uniform_bindings --max_uniforms 10 samples/donors samples/310es 10 "310 es" family_vulkan work/shaderfamilies
+glsl-generate --generate-uniform-bindings --max-uniforms 10 samples/310es samples/donors 10 "310 es" family_vulkan work/shaderfamilies
 
 # The lines above will take approx. 1-2 minutes each, and will generate a shader family for every
 # shader in samples/300es or samples/100:

generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java

@@ -52,7 +52,7 @@ private static Namespace parse(String[] args) throws ArgumentParserException {
           .help("Path of folder of donor shaders.")
           .type(File.class);
 
-    parser.addArgument("output_dir")
+    parser.addArgument("output-dir")
           .help("Output directory.")
           .type(File.class);
 
@@ -64,18 +64,18 @@ private static Namespace parse(String[] args) throws ArgumentParserException {
           .help("Worker name.")
           .type(String.class);
 
-    parser.addArgument("glsl_version")
+    parser.addArgument("glsl-version")
           .help("Version of GLSL to target.")
           .type(String.class);
 
     // Optional arguments
     Generate.addGeneratorCommonArguments(parser);
 
-    parser.addArgument("--ignore_crash_strings")
+    parser.addArgument("--ignore-crash-strings")
         .help("File containing crash strings to ignore, one per line.")
         .type(File.class);
 
-    parser.addArgument("--only_variants")
+    parser.addArgument("--only-variants")
         .help("Only run variant shaders (so sacrifice finding wrong images in favour of crashes.")
         .type(Boolean.class)
         .action(Arguments.storeTrue());

generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java

@@ -80,10 +80,9 @@ public void run() {
     final GeneratorArguments generatorArguments =
         new GeneratorArguments(
             shadingLanguageVersion,
-            generator.nextInt(Integer.MAX_VALUE),
             ns.get("small"),
-            ns.get("avoid_long_loops"),
-            ns.get("multi_pass"),
+            ns.get("allow_long_loops"),
+            ns.get("single_pass"),
             ns.get("aggressively_complicate_control_flow"),
             ns.get("replace_float_literals"),
             donorsDir,
@@ -111,7 +110,7 @@ public void run() {
             generatorArguments.getReplaceFloatLiterals(),
             generatorArguments.getMaxUniforms(),
             generatorArguments.getGenerateUniformBindings());
-        Generate.generateVariant(variantShaderJob, generatorArguments);
+        Generate.generateVariant(variantShaderJob, generatorArguments, ns.getInt("seed"));
         try {
           queue.put(new ImmutablePair<>(referenceShaderJob, variantShaderJob));
         } catch (InterruptedException exception) {