Handle string templates

#303 PiperOrigin-RevId: 602189523
google · Jan 29, 2024 · 6eb82fb · 6eb82fb
1 parent 62d9e65
commit 6eb82fb
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 19 deletions.
diff --git a/java/com/google/turbine/parse/StreamLexer.java b/java/com/google/turbine/parse/StreamLexer.java
@@ -415,17 +415,24 @@ public Token next() {
             }
             readFrom();
             StringBuilder sb = new StringBuilder();
+            Token stringToken = Token.STRING_LITERAL;
             STRING:
             while (true) {
               switch (ch) {
                 case '\\':
                   eat();
-                  sb.append(escape());
+                  if (ch == '{') {
+                    eat();
+                    stringTemplate(sb);
+                    stringToken = Token.STRING_TEMPLATE;
+                  } else {
+                    sb.append(escape());
+                  }
                   continue STRING;
                 case '"':
                   saveValue(sb.toString());
                   eat();
-                  return Token.STRING_LITERAL;
+                  return stringToken;
                 case '\n':
                   throw error(ErrorKind.UNTERMINATED_STRING);
                 case ASCII_SUB:
@@ -450,6 +457,29 @@ public Token next() {
     }
   }
 
+  // String templates aren't compile-time constants, so they don't affect the API. Advance through
+  // the entire template, dropping any contained \{ ... }, and tokenize it as a single string
+  // literal.
+  private void stringTemplate(StringBuilder sb) {
+    sb.append("{}");
+    int depth = 1;
+    while (depth > 0) {
+      Token next = next();
+      switch (next) {
+        case LBRACE:
+          depth++;
+          break;
+        case RBRACE:
+          depth--;
+          break;
+        case EOF:
+          return;
+        default:
+          break;
+      }
+    }
+  }
+
   private Token textBlock() {
     OUTER:
     while (true) {
@@ -478,6 +508,7 @@ private Token textBlock() {
     }
     readFrom();
     StringBuilder sb = new StringBuilder();
+    Token stringToken = Token.STRING_LITERAL;
     while (true) {
       switch (ch) {
         case '"':
@@ -496,17 +527,23 @@ private Token textBlock() {
           value = stripIndent(value);
           value = translateEscapes(value);
           saveValue(value);
-          return Token.STRING_LITERAL;
+          return stringToken;
         case '\\':
           // Escapes are handled later (after stripping indentation), but we need to ensure
           // that \" escapes don't count towards the closing delimiter of the text block.
-          sb.appendCodePoint(ch);
           eat();
-          if (ch == ASCII_SUB && reader.done()) {
-            return Token.EOF;
+          if (ch == '{') {
+            eat();
+            stringTemplate(sb);
+            stringToken = Token.STRING_TEMPLATE;
+          } else {
+            sb.append('\\');
+            if (ch == ASCII_SUB && reader.done()) {
+              return Token.EOF;
+            }
+            sb.appendCodePoint(ch);
+            eat();
           }
-          sb.appendCodePoint(ch);
-          eat();
           continue;
         case ASCII_SUB:
           if (reader.done()) {

diff --git a/java/com/google/turbine/parse/Token.java b/java/com/google/turbine/parse/Token.java
@@ -39,6 +39,7 @@ public enum Token {
   DOUBLE_LITERAL("<double literal>"),
   CHAR_LITERAL("<char literal>"),
   STRING_LITERAL("<string literal>"),
+  STRING_TEMPLATE("<string template>"),
   AT("@"),
   EQ("=="),
   ASSIGN("="),

diff --git a/javatests/com/google/turbine/lower/IntegrationTestSupport.java b/javatests/com/google/turbine/lower/IntegrationTestSupport.java
@@ -131,6 +131,7 @@ public static Map<String, byte[]> canonicalize(Map<String, byte[]> in) {
       makeEnumsFinal(all, n);
       sortAttributes(n);
       undeprecate(n);
+      removePreviewVersion(n);
     }
 
     return toByteCode(classes);
@@ -176,6 +177,11 @@ private static void undeprecate(ClassNode n) {
         .forEach(f -> f.access &= ~Opcodes.ACC_DEPRECATED);
   }
 
+  // Mask out preview bits from version number
+  private static void removePreviewVersion(ClassNode n) {
+    n.version &= 0xffff;
+  }
+
   private static boolean isDeprecated(List<AnnotationNode> visibleAnnotations) {
     return visibleAnnotations != null
         && visibleAnnotations.stream().anyMatch(a -> a.desc.equals("Ljava/lang/Deprecated;"));

diff --git a/javatests/com/google/turbine/lower/LowerIntegrationTest.java b/javatests/com/google/turbine/lower/LowerIntegrationTest.java
@@ -17,9 +17,9 @@
 package com.google.turbine.lower;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
 import static com.google.turbine.testing.TestResources.getResource;
 import static java.util.stream.Collectors.toList;
-import static org.junit.Assume.assumeTrue;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -62,7 +62,11 @@ public class LowerIntegrationTest {
           "sealed_nested.test", 17,
           "textblock.test", 15,
           "textblock2.test", 15,
-          "B306423115.test", 15);
+          "B306423115.test", 15,
+          "string_template.test", 21);
+
+  private static final ImmutableSet<String> SOURCE_VERSION_PREVIEW =
+      ImmutableSet.of("string_template.test");
 
   @Parameters(name = "{index}: {0}")
   public static Iterable<Object[]> parameters() {
@@ -304,6 +308,7 @@ public static Iterable<Object[]> parameters() {
       "strictfp.test",
       "string.test",
       "string_const.test",
+      "string_template.test",
       "superabstract.test",
       "supplierfunction.test",
       "tbound.test",
@@ -356,6 +361,7 @@ public static Iterable<Object[]> parameters() {
     };
     ImmutableSet<String> cases = ImmutableSet.copyOf(testCases);
     assertThat(cases).containsAtLeastElementsIn(SOURCE_VERSION.keySet());
+    assertThat(cases).containsAtLeastElementsIn(SOURCE_VERSION_PREVIEW);
     List<Object[]> tests = cases.stream().map(x -> new Object[] {x}).collect(toList());
     String testShardIndex = System.getenv("TEST_SHARD_INDEX");
     String testTotalShards = System.getenv("TEST_TOTAL_SHARDS");
@@ -403,15 +409,19 @@ public void test() throws Exception {
       classpathJar = ImmutableList.of(lib);
     }
 
-    int version = SOURCE_VERSION.getOrDefault(test, 8);
-    assumeTrue(version <= Runtime.version().feature());
-    ImmutableList<String> javacopts =
-        ImmutableList.of(
-            "-source",
-            String.valueOf(version),
-            "-target",
-            String.valueOf(version),
-            "-Xpkginfo:always");
+    int actualVersion = Runtime.version().feature();
+    int requiredVersion = SOURCE_VERSION.getOrDefault(test, 8);
+    assume().that(actualVersion).isAtLeast(requiredVersion);
+    ImmutableList.Builder<String> javacoptsBuilder = ImmutableList.builder();
+    if (SOURCE_VERSION_PREVIEW.contains(test)) {
+      requiredVersion = actualVersion;
+      javacoptsBuilder.add("--enable-preview");
+    }
+    javacoptsBuilder.add(
+        "-source", String.valueOf(requiredVersion), "-target", String.valueOf(requiredVersion));
+    javacoptsBuilder.add("-Xpkginfo:always");
+
+    ImmutableList<String> javacopts = javacoptsBuilder.build();
 
     Map<String, byte[]> expected =
         IntegrationTestSupport.runJavac(input.sources, classpathJar, javacopts);

diff --git a/javatests/com/google/turbine/lower/LowerTest.java b/javatests/com/google/turbine/lower/LowerTest.java
@@ -17,6 +17,7 @@
 package com.google.turbine.lower;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
 import static com.google.turbine.testing.TestClassPaths.TURBINE_BOOTCLASSPATH;
 import static com.google.turbine.testing.TestResources.getResource;
 import static java.util.Objects.requireNonNull;
@@ -56,6 +57,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -751,6 +753,42 @@ public FieldVisitor visitField(
     assertThat(fields).containsExactly("y");
   }
 
+  // Ensure we don't emit bogus ConstantValues for string templates with a missing processor
+  @Test
+  public void stringTemplate() throws Exception {
+    assume().that(Runtime.version().feature()).isAtLeast(21);
+    BindingResult bound =
+        Binder.bind(
+            ImmutableList.of(
+                Parser.parse(
+                    "class Test {\n" //
+                        + "  public static final String X = \"hello \\{ \"world\" }\";\n"
+                        + "}")),
+            ClassPathBinder.bindClasspath(ImmutableList.of()),
+            TURBINE_BOOTCLASSPATH,
+            /* moduleVersion= */ Optional.empty());
+    ImmutableMap<String, byte[]> lowered =
+        Lower.lowerAll(
+                Lower.LowerOptions.createDefault(),
+                bound.units(),
+                bound.modules(),
+                bound.classPathEnv())
+            .bytes();
+    Map<String, Object> fields = new HashMap<>();
+    new ClassReader(lowered.get("Test"))
+        .accept(
+            new ClassVisitor(Opcodes.ASM9) {
+              @Override
+              public FieldVisitor visitField(
+                  int access, String name, String descriptor, String signature, Object value) {
+                fields.put(name, value);
+                return null;
+              }
+            },
+            0);
+    assertThat(fields).containsExactly("X", null);
+  }
+
   static String lines(String... lines) {
     return Joiner.on(System.lineSeparator()).join(lines);
   }

diff --git a/javatests/com/google/turbine/lower/testdata/string_template.test b/javatests/com/google/turbine/lower/testdata/string_template.test
@@ -0,0 +1,49 @@
+=== StringTemplates.java ===
+public class StringTemplates {
+    interface Example {
+      Object foo();
+      boolean test(String string);
+    }
+    void test(Example example, Example example0, Example example1, Example example2){
+        var m = STR."template \{example}xxx";
+        var nested = STR."template \{example.foo()+ STR."templateInner\{example}"}xxx }";
+        var nestNested = STR."template \{example0.
+                    foo() +
+                    STR."templateInner\{example1.test(STR."\{example2
+                        }")}"}xxx }";
+    }
+}
+=== Foo.java ===
+class Foo {
+    public static final int X = 42;
+    public static final String A = STR."\{X} = \{X}";
+    public static final String B = STR."";
+    public static final String C = STR."\{X}";
+    public static final String D = STR."\{X}\{X}";
+    public static final String E = STR."\{X}\{X}\{X}";
+    public static final String F = STR."     \{X}";
+    public static final String G = STR."\{X}    ";
+    public static final String H = STR."\{X} one long incredibly unbroken sentence moving from "+"topic to topic so that no-one had a chance to interrupt";
+    public static final String I = STR."\{X}  \uD83D\uDCA9  ";
+}
+=== Multiline.java ===
+import static java.lang.StringTemplate.STR;
+
+public class Multiline {
+    static String planet = "world";
+
+    static String s1 = STR."hello, \{planet}";
+
+    static String s2 = STR."""
+            hello, \{planet}
+            """;
+
+    static String s3 = STR."""
+            hello, \{
+                STR."""
+                recursion, \{
+                }
+                """
+            }
+            """;
+}
diff --git a/javatests/com/google/turbine/parse/LexerTest.java b/javatests/com/google/turbine/parse/LexerTest.java
@@ -367,6 +367,7 @@ public static List<String> lex(String input) {
           break;
         case CHAR_LITERAL:
         case STRING_LITERAL:
+        case STRING_TEMPLATE:
           tokenString =
               String.format(
                   "%s(%s)",
@@ -423,4 +424,38 @@ public void textBlockEOF() {
     assertThat(lexer.next()).isEqualTo(Token.EOF);
     assertThat(lexer.stringValue()).isEqualTo("\\");
   }
+
+  @Test
+  public void stringTemplate() {
+    assertThat(lex("STR.\"\\{X}\""))
+        .containsExactly("IDENT(STR)", "DOT", "STRING_TEMPLATE({})", "EOF");
+  }
+
+  @Test
+  public void stringTemplateNested() {
+    assertThat(lex("STR.\"template \\{example.foo()+ STR.\"templateInner\\{example}\"}xxx }\""))
+        .containsExactly("IDENT(STR)", "DOT", "STRING_TEMPLATE(template {}xxx })", "EOF");
+  }
+
+  @Test
+  public void stringTemplateNestedBraces() {
+    assertThat(lex("STR.\"\\{ new Object() {} }\" + \"\""))
+        .containsExactly(
+            "IDENT(STR)", "DOT", "STRING_TEMPLATE({})", "PLUS", "STRING_LITERAL()", "EOF");
+  }
+
+  @Test
+  public void stringTemplateBraces() {
+    assertThat(lex("\"foo \\{'{'}\"")).containsExactly("STRING_TEMPLATE(foo {})", "EOF");
+    assertThat(lex("\"foo \\{\"}\"}\"")).containsExactly("STRING_TEMPLATE(foo {})", "EOF");
+    assertThat(lex("\"foo \\{new Bar[]{}}\"")).containsExactly("STRING_TEMPLATE(foo {})", "EOF");
+    assertThat(lex("\"foo \\{\"bar \\{'}'}\"}\""))
+        .containsExactly("STRING_TEMPLATE(foo {})", "EOF");
+  }
+
+  @Test
+  public void textBlockStringTemplate() {
+    assertThat(lex("STR.\"\"\"\n\\{X}\"\"\""))
+        .containsExactly("IDENT(STR)", "DOT", "STRING_TEMPLATE({})", "EOF");
+  }
 }