Implement annotation parsing in JavaParsers.

This causes the java parser to get somewhat more complex, but
allows for macros and compiler plugins to correctly inspect
java-defined classes for annotations.

Moved a few utility methods into {Parsers,Scanners}Common as well,
so I can reuse them in the java ones.

This involves a change to the t4788 tests, as SAnnotation now
correctly has RetentionPolicy.SOURCE, and is therefore ignored
by the check in BCodeHelpers#BCAnnotGen.shouldEmitAnnotation;
also, ASM Textifier emits an // invisible marker now that
CAnnotation has RetentionPolicy.CLASS. I'm not sure what the
chances that anyone is relying upon this behavior are, but
since it does the Right Thing for separate compilation runs,
they probably should not be too disappointed.

The new tests are run-tests not just pos-tests at the suggestion
of retronym, to ensure that everything works the same at runtime
as it does at compile time. As a matter of fact, it doesn't: the
compile-time universe maintains ordering of annotation values in
the tree, while the runtime universe uses java reflection to get
at the values and therefore cannot know of their order. The test
contains one exception for that.

Review by densh, retronym

Fixes scala/bug#8928

Co-authored-by: Jason Zaugg <jzaugg@gmail.com>
Co-authored-by: Dale Wijnand <dale.wijnand@gmail.com>

src/compiler/scala/tools/nsc/ast/parser/Parsers.scala

@@ -95,6 +95,23 @@ trait ParsersCommon extends ScannersCommon { self =>
      */
     @inline final def makeParens(body: => List[Tree]): Parens =
       Parens(inParens(if (in.token == RPAREN) Nil else body))
+
+    /** {{{ part { `sep` part } }}}, or if sepFirst is true, {{{ { `sep` part } }}}. */
+    final def tokenSeparated[T](separator: Token, sepFirst: Boolean, part: => T): List[T] = {
+      val ts = new ListBuffer[T]
+      if (!sepFirst)
+        ts += part
+
+      while (in.token == separator) {
+        in.nextToken()
+        ts += part
+      }
+      ts.toList
+    }
+
+    /** {{{ tokenSeparated }}}, with the separator fixed to commas. */
+    @inline final def commaSeparated[T](part: => T): List[T] =
+      tokenSeparated(COMMA, sepFirst = false, part)
   }
 }
 
@@ -791,20 +808,6 @@ self =>
         errorTypeTree
       }
     }
-
-    /** {{{ part { `sep` part } }}},or if sepFirst is true, {{{ { `sep` part } }}}. */
-    final def tokenSeparated[T](separator: Token, sepFirst: Boolean, part: => T): List[T] = {
-      val ts = new ListBuffer[T]
-      if (!sepFirst)
-        ts += part
-
-      while (in.token == separator) {
-        in.nextToken()
-        ts += part
-      }
-      ts.toList
-    }
-    @inline final def commaSeparated[T](part: => T): List[T] = tokenSeparated(COMMA, sepFirst = false, part)
     @inline final def caseSeparated[T](part: => T): List[T] = tokenSeparated(CASE, sepFirst = true, part)
     def readAnnots(part: => Tree): List[Tree] = tokenSeparated(AT, sepFirst = true, part)
 

src/compiler/scala/tools/nsc/ast/parser/Scanners.scala

@@ -40,6 +40,9 @@ trait ScannersCommon {
   }
 
   trait ScannerCommon extends CommonTokenData {
+    /** Consume and discard the next token. */
+    def nextToken(): Unit
+
     // things to fill in, in addition to buf, decodeUni which come from CharArrayReader
     def error(off: Offset, msg: String): Unit
     def incompleteInputError(off: Offset, msg: String): Unit

src/compiler/scala/tools/nsc/javac/JavaParsers.scala

@@ -242,11 +242,21 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
 
     // -------------------- specific parsing routines ------------------
 
-    def qualId(): RefTree = {
-      var t: RefTree = atPos(in.currentPos) { Ident(ident()) }
-      while (in.token == DOT) {
+    def qualId(orClassLiteral: Boolean = false): Tree = {
+      var t: Tree = atPos(in.currentPos) { Ident(ident()) }
+      var done = false
+      while (!done && in.token == DOT) {
         in.nextToken()
-        t = atPos(in.currentPos) { Select(t, ident()) }
+        t = atPos(in.currentPos) {
+          if (orClassLiteral && in.token == CLASS) {
+            in.nextToken()
+            done = true
+            val tpeArg = convertToTypeId(t)
+            TypeApply(Select(gen.mkAttributedRef(definitions.PredefModule), nme.classOf), tpeArg :: Nil)
+          } else {
+            Select(t, ident())
+          }
+        }
       }
       t
     }
@@ -275,7 +285,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
       }
 
     def typ(): Tree = {
-      annotations()
+      annotations() // TODO: fix scala/bug#9883 (JSR 308)
       optArrayBrackets {
         if (in.token == FINAL) in.nextToken()
         if (in.token == IDENTIFIER) {
@@ -334,20 +344,73 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
     }
 
     def annotations(): List[Tree] = {
-      //var annots = new ListBuffer[Tree]
+      val annots = new ListBuffer[Tree]
       while (in.token == AT) {
         in.nextToken()
-        annotation()
+        val annot = annotation()
+        if (annot.nonEmpty) annots += annot
       }
-      List() // don't pass on annotations for now
+      annots.toList
     }
 
-    /** Annotation ::= TypeName [`(` AnnotationArgument {`,` AnnotationArgument} `)`]
+    /** Annotation ::= TypeName [`(` [AnnotationArgument {`,` AnnotationArgument}] `)`]
      */
-    def annotation() {
-      qualId()
-      if (in.token == LPAREN) { skipAhead(); accept(RPAREN) }
-      else if (in.token == LBRACE) { skipAhead(); accept(RBRACE) }
+    def annotation(): Tree = {
+      def annArg(): Tree = {
+        def annVal(): Tree = {
+          tryLiteral() match {
+            case Some(lit) => atPos(in.currentPos)(Literal(lit))
+            case _ if in.token == AT =>
+              in.nextToken()
+              annotation()
+            case _ if in.token == LBRACE =>
+              atPos(in.pos) {
+                val elts = inBracesOrNil(commaSeparated(annVal()))
+                if (elts.exists(_.isEmpty)) EmptyTree
+                else Apply(ArrayModule_overloadedApply, elts: _*)
+              }
+            case _ if in.token == IDENTIFIER =>
+              qualId(orClassLiteral = true)
+          }
+        }
+
+        if (in.token == IDENTIFIER) {
+          qualId(orClassLiteral = true) match {
+            case name: Ident if in.token == EQUALS =>
+              in.nextToken()
+              /* name = value */
+              val value = annVal()
+              if (value.isEmpty) EmptyTree else gen.mkNamedArg(name, value)
+            case rhs =>
+              /* implicit `value` arg with constant value */
+              gen.mkNamedArg(nme.value, rhs)
+          }
+        } else {
+          /* implicit `value` arg */
+          val value = annVal()
+          if (value.isEmpty) EmptyTree else gen.mkNamedArg(nme.value, value)
+        }
+      }
+
+      atPos(in.pos) {
+        val id = convertToTypeId(qualId())
+        if (in.token == LPAREN) {
+          val saved = new JavaTokenData {}.copyFrom(in) // prep to bail if non-literals/identifiers
+          accept(LPAREN)
+          val args =
+            if (in.token == RPAREN) Nil
+            else commaSeparated(atPos(in.pos)(annArg()))
+          if (in.token == RPAREN) {
+            accept(RPAREN)
+            New(id, args :: Nil)
+          } else {
+            in.copyFrom(saved)
+            skipAhead()
+            accept(RPAREN)
+            EmptyTree
+          }
+        } else New(id, ListOfNil)
+      }
     }
 
     def modifiers(inInterface: Boolean): Modifiers = {
@@ -361,7 +424,8 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
         in.token match {
           case AT if (in.lookaheadToken != INTERFACE) =>
             in.nextToken()
-            annotation()
+            val annot = annotation()
+            if (annot.nonEmpty) annots :+= annot
           case PUBLIC =>
             isPackageAccess = false
             in.nextToken()
@@ -419,10 +483,10 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
 
     def typeParam(): TypeDef =
       atPos(in.currentPos) {
-        annotations()
+        val anns = annotations()
         val name = identForType()
         val hi = if (in.token == EXTENDS) { in.nextToken() ; bound() } else EmptyTree
-        TypeDef(Modifiers(Flags.JAVA | Flags.DEFERRED | Flags.PARAM), name, Nil, TypeBoundsTree(EmptyTree, hi))
+        TypeDef(Modifiers(Flags.JAVA | Flags.DEFERRED | Flags.PARAM, tpnme.EMPTY, anns), name, Nil, TypeBoundsTree(EmptyTree, hi))
       }
 
     def bound(): Tree =
@@ -446,15 +510,15 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
 
     def formalParam(): ValDef = {
       if (in.token == FINAL) in.nextToken()
-      annotations()
+      val anns = annotations()
       var t = typ()
       if (in.token == DOTDOTDOT) {
         in.nextToken()
         t = atPos(t.pos) {
           AppliedTypeTree(scalaDot(tpnme.JAVA_REPEATED_PARAM_CLASS_NAME), List(t))
         }
       }
-     varDecl(in.currentPos, Modifiers(Flags.JAVA | Flags.PARAM), t, ident().toTermName)
+     varDecl(in.currentPos, Modifiers(Flags.JAVA | Flags.PARAM, typeNames.EMPTY, anns), t, ident().toTermName)
     }
 
     def optThrows() {
@@ -841,7 +905,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
     }
 
     def enumConst(enumType: Tree): (ValDef, Boolean) = {
-      annotations()
+      val anns = annotations()
       var hasClassBody = false
       val res = atPos(in.currentPos) {
         val name = ident()
@@ -856,7 +920,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
           skipAhead()
           accept(RBRACE)
         }
-        ValDef(Modifiers(Flags.JAVA_ENUM | Flags.STABLE | Flags.JAVA | Flags.STATIC), name.toTermName, enumType, blankExpr)
+        ValDef(Modifiers(Flags.JAVA_ENUM | Flags.STABLE | Flags.JAVA | Flags.STATIC, typeNames.EMPTY, anns), name.toTermName, enumType, blankExpr)
       }
       (res, hasClassBody)
     }
@@ -894,10 +958,10 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
       var pos = in.currentPos
       val pkg: RefTree =
         if (in.token == AT || in.token == PACKAGE) {
-          annotations()
+          annotations() // TODO: put these somewhere?
           pos = in.currentPos
           accept(PACKAGE)
-          val pkg = qualId()
+          val pkg = qualId().asInstanceOf[RefTree]
           accept(SEMI)
           pkg
         } else {

src/compiler/scala/tools/nsc/javac/JavaScanners.scala

@@ -52,12 +52,13 @@ trait JavaScanners extends ast.parser.ScannersCommon {
     /** the base of a number */
     var base: Int = 0
 
-    def copyFrom(td: JavaTokenData) = {
+    def copyFrom(td: JavaTokenData): this.type = {
       this.token = td.token
       this.pos = td.pos
       this.lastPos = td.lastPos
       this.name = td.name
       this.base = td.base
+      this
     }
   }
 

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -851,8 +851,15 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
 
         case tpnme.RuntimeAnnotationATTR =>
           val numAnnots = u2
+          val annots = new ListBuffer[AnnotationInfo]
           for (n <- 0 until numAnnots; annot <- parseAnnotation(u2))
-            sym.addAnnotation(annot)
+            annots += annot
+          /* `sym.withAnnotations(annots)`, like `sym.addAnnotation(annot)`, prepends,
+           * so if we parsed in classfile order we would wind up with the annotations
+           * in reverse order in `sym.annotations`. Instead we just read them out the
+           * other way around, for now. TODO: sym.addAnnotation add to the end?
+           */
+          sym.setAnnotations(sym.annotations ::: annots.toList)
 
         // TODO 1: parse runtime visible annotations on parameters
         // case tpnme.RuntimeParamAnnotationATTR

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -3887,6 +3887,11 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
         case Typed(t, _) =>
           tree2ConstArg(t, pt)
 
+        case tree if unit.isJava && pt.typeSymbol == ArrayClass =>
+          /* If we get here, we have a Java array annotation argument which was passed
+           * as a single value, and needs to be wrapped. */
+          trees2ConstArg(tree :: Nil, pt.typeArgs.head)
+
         case tree =>
           tryConst(tree, pt)
       }

src/interactive/scala/tools/nsc/interactive/Global.scala

@@ -1369,7 +1369,7 @@ class Global(settings: Settings, _reporter: Reporter, projectName: String = "")
     val symbols =
       Set(UnitClass, BooleanClass, ByteClass,
           ShortClass, IntClass, LongClass, FloatClass,
-          DoubleClass, NilModule, ListClass) ++ TupleClass.seq
+          DoubleClass, NilModule, ListClass, PredefModule) ++ TupleClass.seq ++ ArrayModule_overloadedApply.alternatives
     symbols.foreach(_.initialize)
   }
 

src/partest-extras/scala/tools/partest/BytecodeTest.scala

@@ -124,6 +124,10 @@ abstract class BytecodeTest {
   }
 
 // loading
+  protected def getField(classNode: ClassNode, name: String): FieldNode =
+    classNode.fields.asScala.find(_.name == name) getOrElse
+      sys.error(s"Didn't find field '$name' in class '${classNode.name}'")
+
   protected def getMethod(classNode: ClassNode, name: String): MethodNode =
     classNode.methods.asScala.find(_.name == name) getOrElse
       sys.error(s"Didn't find method '$name' in class '${classNode.name}'")

test/files/presentation/parse-invariants/src/a/A.java

@@ -1,5 +1,10 @@
 package syntax;
 
+@NoArgs
+@Empty()
+@Simple(n = 1, c = '2', f = 6.7f, d = 8.9, s = "t", z = A.class, e = P.Pluto, a = @C.I("t"))
+@Arrays({ @Array({0, 1, C._2}), @Array(3) })
+@Deprecated
 class A {
   transient volatile int x;
   strictfp void test() {
@@ -11,6 +16,7 @@ synchronized void syncMethod() {}
 
   void thrower() throws Throwable {}
 
+  @Deprecated void deprecated() {}
 }
 
 strictfp class B {}

test/files/run/t4788-separate-compilation.check

@@ -1,5 +1,9 @@
 Some(@Ljava/lang/Deprecated;())
 None
-Some(@LSAnnotation;())
-Some(@LCAnnotation;())
+None
+Some(@LCAnnotation;() // invisible)
 Some(@LRAnnotation;())
+Some(@LAnnWithArgs_0$Ann;(value="literal") // invisible)
+Some(@LAnnWithArgs_0$Ann;(value="muk") // invisible)
+Some(@LAnnWithArgs_0$Ann;(value="mukja") // invisible)
+Some(@LAnnWithArgs_0$Ann;(value="mukja") // invisible)

test/files/run/t4788-separate-compilation/AnnWithArgs_0.java

@@ -0,0 +1,19 @@
+public class AnnWithArgs_0 {
+  public static @interface Ann {
+    String value();
+  }
+
+  @Ann("literal")
+  public static int x1 = 1;
+
+  @Ann(AnnWithArgs_0.strS)
+  public static int x2 = 2;
+
+  @Ann("muk" + "ja")
+  public static int x3 = 3;
+
+  @Ann(AnnWithArgs_0.strS + "ja")
+  public static int x4 = 4;
+
+  public static final String strS = "muk";
+}

test/files/run/t4788-separate-compilation/Test_2.scala

@@ -18,6 +18,19 @@ object Test extends BytecodeTest {
       .map(_.trim)
   }
 
+  def annotationsForField(className: String, fieldName: String): Option[String] = {
+    val field = getField(loadClassNode(className, skipDebugInfo = false), fieldName)
+    val textifier = new Textifier
+    field.accept(new TraceClassVisitor(null, textifier, null))
+
+    val fieldString = stringFromWriter(w => textifier.print(w))
+    fieldString
+        .split('\n')
+        .filterNot(_.contains("@Lscala/reflect/ScalaSignature"))
+        .find(_.contains("@L"))
+        .map(_.trim)
+  }
+
   def show {
     // It seems like @java.lang.Deprecated shows up in both the
     // Deprecated attribute and RuntimeVisibleAnnotation attribute,
@@ -31,5 +44,10 @@ object Test extends BytecodeTest {
     println(annotationsForClass("S"))
     println(annotationsForClass("C"))
     println(annotationsForClass("R"))
+
+    println(annotationsForField("AnnWithArgs_0", "x1"))
+    println(annotationsForField("AnnWithArgs_0", "x2"))
+    println(annotationsForField("AnnWithArgs_0", "x3"))
+    println(annotationsForField("AnnWithArgs_0", "x4"))
   }
 }