From 75e6bb696cc434480a9ba076939f22c516e437ba Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Mon, 5 Jun 2023 16:33:32 +0900 Subject: [PATCH 01/25] Export diagnostics (including unused warnings) to SemanticDB --- compiler/src/dotty/tools/dotc/Compiler.scala | 3 +- .../dotty/tools/dotc/reporting/Reporter.scala | 9 +- .../tools/dotc/semanticdb/DiagnosticOps.scala | 20 +++ .../dotc/semanticdb/ExtractSemanticDB.scala | 146 +++++++++++------- .../dotty/tools/dotc/semanticdb/Scala3.scala | 2 + .../dotty/tools/dotc/semanticdb/Tools.scala | 20 +++ tests/semanticdb/metac.expect | 10 ++ 7 files changed, 151 insertions(+), 59 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index a6118732d4ae..7ddfea3d09a8 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -38,7 +38,7 @@ class Compiler { List(new CheckUnused.PostTyper) :: // Check for unused elements List(new YCheckPositions) :: // YCheck positions List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks - List(new semanticdb.ExtractSemanticDB) :: // Extract info into .semanticdb files + List(new semanticdb.ExtractSemanticDB.PostTyper) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks @@ -51,6 +51,7 @@ class Compiler { List(new Inlining) :: // Inline and execute macros List(new PostInlining) :: // Add mirror support for inlined code List(new CheckUnused.PostInlining) :: // Check for unused elements + List(new semanticdb.ExtractSemanticDB.PostInlining) :: // Extract info into .semanticdb files List(new Staging) :: // Check staging levels and heal staged types List(new Splicing) :: // Replace level 1 splices with holes List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures diff --git a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala index f5aadac27296..57a8d279aaa5 100644 --- a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala +++ b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala @@ -109,9 +109,14 @@ abstract class Reporter extends interfaces.ReporterResult { private var errors: List[Error] = Nil + private var warnings: List[Warning] = Nil + /** All errors reported by this reporter (ignoring outer reporters) */ def allErrors: List[Error] = errors + /** All errors reported by this reporter (ignoring outer reporters) */ + def allWarnings: List[Warning] = warnings + /** Were sticky errors reported? Overridden in StoreReporter. */ def hasStickyErrors: Boolean = false @@ -157,7 +162,9 @@ abstract class Reporter extends interfaces.ReporterResult { markReported(d) withMode(Mode.Printing)(doReport(d)) d match { - case _: Warning => _warningCount += 1 + case w: Warning => + warnings = w :: warnings + _warningCount += 1 case e: Error => errors = e :: errors _errorCount += 1 diff --git a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala new file mode 100644 index 000000000000..53beeffe541e --- /dev/null +++ b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala @@ -0,0 +1,20 @@ +package dotty.tools.dotc.semanticdb + +import dotty.tools.dotc.reporting.Diagnostic +import dotty.tools.dotc.{semanticdb => s} +import dotty.tools.dotc.interfaces.Diagnostic.{ERROR, INFO, WARNING} +import dotty.tools.dotc.core.Contexts.Context + +object DiagnosticOps: + extension (d: Diagnostic) + def toSemanticDiagnostic(using Context): s.Diagnostic = + val severity = d.level match + case ERROR => s.Diagnostic.Severity.ERROR + case WARNING => s.Diagnostic.Severity.WARNING + case INFO => s.Diagnostic.Severity.INFORMATION + case _ => s.Diagnostic.Severity.INFORMATION + s.Diagnostic( + range = Scala3.range(d.pos.span, d.pos.source), + severity = severity, + message = d.msg.message + ) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 91614aaccad2..3bc30758ff64 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -25,16 +25,17 @@ import scala.PartialFunction.condOpt import dotty.tools.dotc.{semanticdb => s} import dotty.tools.io.{AbstractFile, JarArchive} +import dotty.tools.dotc.util.Property +import dotty.tools.dotc.semanticdb.DiagnosticOps.* + /** Extract symbol references and uses to semanticdb files. * See https://scalameta.org/docs/semanticdb/specification.html#symbol-1 * for a description of the format. - * TODO: Also extract type information */ -class ExtractSemanticDB extends Phase: - import Scala3.{_, given} +class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: String, _key: Property.Key[TextDocument]) extends Phase: - override val phaseName: String = ExtractSemanticDB.name + override def phaseName: String = ExtractSemanticDB.phaseNamePrefix + suffix override val description: String = ExtractSemanticDB.description @@ -48,16 +49,94 @@ class ExtractSemanticDB extends Phase: override def run(using Context): Unit = val unit = ctx.compilationUnit - val extractor = Extractor() - extractor.extract(unit.tpdTree) - ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) + if (phaseMode == ExtractSemanticDB.PhaseMode.PostTyper) + val extractor = ExtractSemanticDB.Extractor() + extractor.extract(unit.tpdTree) + unit.tpdTree.putAttachment(_key, extractor.toTextDocument(unit.source)) + else + unit.tpdTree.getAttachment(_key) match + case None => ??? + case Some(doc) => + val warnings = ctx.reporter.allWarnings.collect { + case w if w.pos.source == ctx.source => w.toSemanticDiagnostic + } + ExtractSemanticDB.write(unit.source, doc.copy(diagnostics = warnings)) +end ExtractSemanticDB + +object ExtractSemanticDB: + import java.nio.file.Path + import java.nio.file.Files + import java.nio.file.Paths + + val phaseNamePrefix: String = "extractSemanticDB" + val description: String = "extract info into .semanticdb files" + + enum PhaseMode: + case PostTyper + case PostInlining + + /** + * The key used to retrieve the "unused entity" analysis metadata, + * from the compilation `Context` + */ + private val _key = Property.StickyKey[TextDocument] + + class PostTyper extends ExtractSemanticDB(PhaseMode.PostTyper, "PostTyper", _key) + + class PostInlining extends ExtractSemanticDB(PhaseMode.PostInlining, "PostInlining", _key) + + private def semanticdbTarget(using Context): Option[Path] = + Option(ctx.settings.semanticdbTarget.value) + .filterNot(_.isEmpty) + .map(Paths.get(_)) + + private def outputDirectory(using Context): AbstractFile = ctx.settings.outputDir.value + + private def absolutePath(path: Path): Path = path.toAbsolutePath.normalize + + private def write( + source: SourceFile, + doc: TextDocument + )(using Context): Unit = + val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value) + val outpath = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) + .resolve("META-INF") + .resolve("semanticdb") + .resolve(relPath) + .resolveSibling(source.name + ".semanticdb") + Files.createDirectories(outpath.getParent()) + val docs = TextDocuments(List(doc)) + val out = Files.newOutputStream(outpath) + try + val stream = internal.SemanticdbOutputStream.newInstance(out) + docs.writeTo(stream) + stream.flush() + finally + out.close() + end write + /** Extractor of symbol occurrences from trees */ - class Extractor extends TreeTraverser: + private class Extractor extends TreeTraverser: + import Scala3.{_, given} given s.SemanticSymbolBuilder = s.SemanticSymbolBuilder() val synth = SyntheticsExtractor() given converter: s.TypeOps = s.TypeOps() + + def toTextDocument(source: SourceFile)(using Context): TextDocument = + val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value) + TextDocument( + schema = Schema.SEMANTICDB4, + language = Language.SCALA, + uri = Tools.mkURIstring(Paths.get(relPath)), + text = "", + md5 = internal.MD5.compute(String(source.content)), + symbols = symbolInfos.toList, + occurrences = occurrences.toList, + synthetics = synthetics.toList, + ) + /** The bodies of synthetic locals */ private val localBodies = mutable.HashMap[Symbol, Tree]() @@ -468,52 +547,5 @@ class ExtractSemanticDB extends Phase: registerSymbol(vparam.symbol, symkinds) traverse(vparam.tpt) tparams.foreach(tp => traverse(tp.rhs)) - - -object ExtractSemanticDB: - import java.nio.file.Path - import java.nio.file.Files - import java.nio.file.Paths - - val name: String = "extractSemanticDB" - val description: String = "extract info into .semanticdb files" - - private def semanticdbTarget(using Context): Option[Path] = - Option(ctx.settings.semanticdbTarget.value) - .filterNot(_.isEmpty) - .map(Paths.get(_)) - - private def outputDirectory(using Context): AbstractFile = ctx.settings.outputDir.value - - def write( - source: SourceFile, - occurrences: List[SymbolOccurrence], - symbolInfos: List[SymbolInformation], - synthetics: List[Synthetic], - )(using Context): Unit = - def absolutePath(path: Path): Path = path.toAbsolutePath.normalize - val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value) - val outpath = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) - .resolve("META-INF") - .resolve("semanticdb") - .resolve(relPath) - .resolveSibling(source.name + ".semanticdb") - Files.createDirectories(outpath.getParent()) - val doc: TextDocument = TextDocument( - schema = Schema.SEMANTICDB4, - language = Language.SCALA, - uri = Tools.mkURIstring(Paths.get(relPath)), - text = "", - md5 = internal.MD5.compute(String(source.content)), - symbols = symbolInfos, - occurrences = occurrences, - synthetics = synthetics, - ) - val docs = TextDocuments(List(doc)) - val out = Files.newOutputStream(outpath) - try - val stream = internal.SemanticdbOutputStream.newInstance(out) - docs.writeTo(stream) - stream.flush() - finally - out.close() + end Extractor +end ExtractSemanticDB diff --git a/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala b/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala index e157b52fe260..cd14820c6a1d 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala @@ -484,6 +484,8 @@ object Scala3: given Ordering[SymbolInformation] = Ordering.by[SymbolInformation, String](_.symbol)(IdentifierOrdering()) + given Ordering[Diagnostic] = (x, y) => compareRange(x.range, y.range) + given Ordering[Synthetic] = (x, y) => compareRange(x.range, y.range) /** diff --git a/compiler/src/dotty/tools/dotc/semanticdb/Tools.scala b/compiler/src/dotty/tools/dotc/semanticdb/Tools.scala index d37973237a9f..6c6e69f12578 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/Tools.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/Tools.scala @@ -69,6 +69,8 @@ object Tools: sb.append("Language => ").append(languageString(doc.language)).nl sb.append("Symbols => ").append(doc.symbols.length).append(" entries").nl sb.append("Occurrences => ").append(doc.occurrences.length).append(" entries").nl + if doc.diagnostics.nonEmpty then + sb.append("Diagnostics => ").append(doc.diagnostics.length).append(" entries").nl if doc.synthetics.nonEmpty then sb.append("Synthetics => ").append(doc.synthetics.length).append(" entries").nl sb.nl @@ -78,6 +80,10 @@ object Tools: sb.append("Occurrences:").nl doc.occurrences.sorted.foreach(processOccurrence) sb.nl + if doc.diagnostics.nonEmpty then + sb.append("Diagnostics:").nl + doc.diagnostics.sorted.foreach(d => processDiag(d)) + sb.nl if doc.synthetics.nonEmpty then sb.append("Synthetics:").nl doc.synthetics.sorted.foreach(s => processSynth(s, synthPrinter)) @@ -108,6 +114,20 @@ object Tools: private def processSynth(synth: Synthetic, printer: SyntheticPrinter)(using sb: StringBuilder): Unit = sb.append(printer.pprint(synth)).nl + private def processDiag(d: Diagnostic)(using sb: StringBuilder): Unit = + d.range match + case Some(range) => processRange(sb, range) + case _ => sb.append("[):") + sb.append(" ") + d.severity match + case Diagnostic.Severity.ERROR => sb.append("[error]") + case Diagnostic.Severity.WARNING => sb.append("[warning]") + case Diagnostic.Severity.INFORMATION => sb.append("[info]") + case _ => sb.append("[unknown]") + sb.append(" ") + sb.append(d.message) + sb.nl + private def processOccurrence(occ: SymbolOccurrence)(using sb: StringBuilder, sourceFile: SourceFile): Unit = occ.range match case Some(range) => diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 0ec8a8e5d84c..f6bf86e88b7a 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -923,6 +923,7 @@ Text => empty Language => Scala Symbols => 181 entries Occurrences => 148 entries +Diagnostics => 1 entries Synthetics => 6 entries Symbols: @@ -1258,6 +1259,9 @@ Occurrences: [68:9..68:16): Neptune <- _empty_/Enums.Planet.Neptune. [68:25..68:31): Planet -> _empty_/Enums.Planet# +Diagnostics: +[48:13..48:13): [warning] `:` after symbolic operator is deprecated; use backticks around operator instead + Synthetics: [52:9..52:13):Refl => *.unapply[Option[B]] [52:31..52:50):identity[Option[B]] => *[Function1[A, Option[B]]] @@ -1537,6 +1541,7 @@ Text => empty Language => Scala Symbols => 29 entries Occurrences => 65 entries +Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -1637,6 +1642,11 @@ Occurrences: [26:57..26:58): A -> a/b/Givens.foo().(A) [26:59..26:64): empty -> a/b/Givens.Monoid#empty(). +Diagnostics: +[24:53..24:63): [warning] An inline given alias with a function value as right-hand side can significantly increase +generated code size. You should either drop the `inline` or rewrite the given with an +explicit `apply` method. + Synthetics: [12:17..12:25):sayHello => *[Int] [13:19..13:29):sayGoodbye => *[Int] From c5a80c48b5561b42fcc62d2aef2d24737994e473 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Mon, 5 Jun 2023 17:00:47 +0900 Subject: [PATCH 02/25] Add -Wunused:all to the compiler options to see Semanticdb export unused warnings --- .../dotc/semanticdb/SemanticdbTests.scala | 3 +- tests/semanticdb/metac.expect | 75 ++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala index a85cc9ad80f9..de8c2c11f9c2 100644 --- a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala +++ b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala @@ -142,7 +142,8 @@ class SemanticdbTests: "-sourceroot", expectSrc.toString, "-classpath", target.toString, "-Xignore-scala2-macros", - "-usejavacp" + "-usejavacp", + "-Wunused:all" ) ++ inputFiles().map(_.toString) val exit = Main.process(args) assertFalse(s"dotc errors: ${exit.errorCount}", exit.hasErrors) diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index f6bf86e88b7a..34eb70231150 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -8,6 +8,7 @@ Text => empty Language => Scala Symbols => 9 entries Occurrences => 18 entries +Diagnostics => 2 entries Symbols: example/Access# => class Access extends Object { self: Access => +8 decls } @@ -40,6 +41,10 @@ Occurrences: [9:6..9:8): m7 <- example/Access#m7(). [9:11..9:14): ??? -> scala/Predef.`???`(). +Diagnostics: +[3:14..3:16): [warning] unused private member +[4:20..4:22): [warning] unused private member + expect/Advanced.scala --------------------- @@ -50,6 +55,7 @@ Text => empty Language => Scala Symbols => 61 entries Occurrences => 138 entries +Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -255,6 +261,9 @@ Occurrences: [53:37..53:38): x -> advanced/HKClass#foo().(x) [53:39..53:47): toString -> scala/Tuple2#toString(). +Diagnostics: +[40:12..40:15): [warning] unused local definition + Synthetics: [27:12..27:16):s.s1 => reflectiveSelectable(*) [29:12..29:16):s.s2 => reflectiveSelectable(*) @@ -270,6 +279,7 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 52 entries +Diagnostics => 2 entries Symbols: annot/Alias. => final object Alias extends Object { self: Alias.type => +2 decls } @@ -350,6 +360,10 @@ Occurrences: [39:11..39:26): ClassAnnotation -> com/javacp/annot/ClassAnnotation# [39:28..39:33): param -> scala/annotation/meta/param# +Diagnostics: +[7:67..7:68): [warning] unused explicit parameter +[21:33..21:34): [warning] unused explicit parameter + expect/Anonymous.scala ---------------------- @@ -360,6 +374,7 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 47 entries +Diagnostics => 1 entries Synthetics => 2 entries Symbols: @@ -436,6 +451,9 @@ Occurrences: [23:33..23:39): String -> scala/Predef.String# [23:42..23:45): ??? -> scala/Predef.`???`(). +Diagnostics: +[14:8..14:9): [warning] unused local definition + Synthetics: [10:2..10:9):locally => *[Unit] [13:2..13:9):locally => *[Unit] @@ -474,6 +492,7 @@ Text => empty Language => Scala Symbols => 108 entries Occurrences => 114 entries +Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -702,6 +721,12 @@ Occurrences: [53:4..53:9): local -> local4 [53:10..53:11): + -> scala/Int#`+`(+4). +Diagnostics: +[18:9..18:10): [warning] unused explicit parameter +[20:27..20:28): [warning] unused explicit parameter +[22:27..22:28): [warning] unused explicit parameter +[24:10..24:11): [warning] unused explicit parameter + Synthetics: [51:16..51:27):List(1).map => *[Int] [51:16..51:20):List => *.apply[Int] @@ -763,6 +788,7 @@ Text => empty Language => Scala Symbols => 30 entries Occurrences => 46 entries +Diagnostics => 3 entries Symbols: endmarkers/Container# => class Container extends Object { self: Container => +5 decls } @@ -844,6 +870,11 @@ Occurrences: [64:14..64:20): String -> scala/Predef.String# [67:4..67:14): endmarkers -> endmarkers/ +Diagnostics: +[38:8..38:16): [warning] unused local definition +[42:8..42:16): [warning] unused local definition +[46:8..46:16): [warning] unused local definition + expect/EndMarkers2.scala ------------------------ @@ -923,7 +954,7 @@ Text => empty Language => Scala Symbols => 181 entries Occurrences => 148 entries -Diagnostics => 1 entries +Diagnostics => 2 entries Synthetics => 6 entries Symbols: @@ -1260,6 +1291,7 @@ Occurrences: [68:25..68:31): Planet -> _empty_/Enums.Planet# Diagnostics: +[30:12..30:17): [warning] unused explicit parameter [48:13..48:13): [warning] `:` after symbolic operator is deprecated; use backticks around operator instead Synthetics: @@ -1314,6 +1346,7 @@ Text => empty Language => Scala Symbols => 5 entries Occurrences => 23 entries +Diagnostics => 1 entries Symbols: example/Example. => final object Example extends Object { self: Example.type => +3 decls } @@ -1347,6 +1380,9 @@ Occurrences: [9:24..9:32): classTag -> scala/reflect/package.classTag(). [9:33..9:36): Int -> scala/Int# +Diagnostics: +[2:24..2:30): [warning] unused import + expect/Extension.scala ---------------------- @@ -1761,6 +1797,7 @@ Text => empty Language => Scala Symbols => 2 entries Occurrences => 16 entries +Diagnostics => 1 entries Symbols: _empty_/Imports$package. => final package object _empty_ extends Object { self: _empty_.type => +2 decls } @@ -1784,6 +1821,9 @@ Occurrences: [3:25..3:28): Int -> scala/Int# [3:30..3:33): Int -> scala/Int# +Diagnostics: +[0:26..0:34): [warning] unused import + expect/InstrumentTyper.scala ---------------------------- @@ -3508,6 +3548,7 @@ Text => empty Language => Scala Symbols => 22 entries Occurrences => 44 entries +Diagnostics => 3 entries Synthetics => 11 entries Symbols: @@ -3523,7 +3564,7 @@ example/ValPattern#app(). => method app (): Unit example/ValPattern#left. => val method left Int example/ValPattern#leftVar(). => var method leftVar Int example/ValPattern#number1. => val method number1 Int -example/ValPattern#number1Var(). => var method number1Var Int +example/ValPattern#number1Var(). => val method number1Var Int example/ValPattern#q1. => val method q1 Nothing example/ValPattern#right. => val method right Int example/ValPattern#rightVar(). => var method rightVar Int @@ -3532,7 +3573,7 @@ local1 => val local right: Int local2 => val local number1: Int local3 => var local leftVar: Int local4 => var local rightVar: Int -local5 => var local number1Var: Int +local5 => val local number1Var: Int Occurrences: [0:8..0:15): example <- example/ @@ -3580,6 +3621,11 @@ Occurrences: [39:10..39:17): leftVar -> local3 [40:10..40:18): rightVar -> local4 +Diagnostics: +[30:11..30:18): [warning] unset local variable +[30:20..30:28): [warning] unset local variable +[31:15..31:25): [warning] unset local variable + Synthetics: [5:6..5:10):Some => *.unapply[Int] [6:4..6:8):Some => *.apply[Int] @@ -3603,6 +3649,7 @@ Text => empty Language => Scala Symbols => 42 entries Occurrences => 128 entries +Diagnostics => 1 entries Symbols: example/ValUsages. => final object ValUsages extends Object { self: ValUsages.type => +2 decls } @@ -3778,6 +3825,9 @@ Occurrences: [49:2..49:3): v -> example/ValUsages.v. [49:3..49:18): .explicitSetter -> example/Vals#`explicitSetter_=`(). +Diagnostics: +[2:20..2:21): [warning] unused explicit parameter + expect/Vararg.scala ------------------- @@ -3819,6 +3869,7 @@ Text => empty Language => Scala Symbols => 8 entries Occurrences => 18 entries +Diagnostics => 1 entries Symbols: _empty_/Test_depmatch. => final object Test_depmatch extends Object { self: Test_depmatch.type => +4 decls } @@ -3850,6 +3901,9 @@ Occurrences: [6:19..6:20): U -> local0 [6:24..6:27): ??? -> scala/Predef.`???`(). +Diagnostics: +[6:8..6:9): [warning] unused local definition + expect/example-dir/FileInDir.scala ---------------------------------- @@ -4125,6 +4179,7 @@ Text => empty Language => Scala Symbols => 7 entries Occurrences => 8 entries +Diagnostics => 1 entries Symbols: i9727/Test# => class Test extends Object { self: Test => +2 decls } @@ -4145,6 +4200,9 @@ Occurrences: [4:4..4:5): b <- i9727/i9727$package.b. [4:12..4:16): Test -> i9727/Test# +Diagnostics: +[2:11..2:12): [warning] unused explicit parameter + expect/i9782.scala ------------------ @@ -4529,6 +4587,7 @@ Text => empty Language => Scala Symbols => 50 entries Occurrences => 73 entries +Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -4658,6 +4717,12 @@ Occurrences: [25:27..25:28): t <- local1 [25:33..25:36): ??? -> scala/Predef.`???`(). +Diagnostics: +[9:30..9:31): [warning] unused explicit parameter +[9:36..9:37): [warning] unused explicit parameter +[9:42..9:43): [warning] unused explicit parameter +[21:11..21:12): [warning] unused explicit parameter + Synthetics: [23:6..23:10):List => *.unapplySeq[Nothing] [24:19..24:23):List => *.unapplySeq[Nothing] @@ -4672,6 +4737,7 @@ Text => empty Language => Scala Symbols => 143 entries Occurrences => 228 entries +Diagnostics => 1 entries Synthetics => 1 entries Symbols: @@ -5049,6 +5115,9 @@ Occurrences: [119:32..119:38): Option -> scala/Option# [119:39..119:42): Int -> scala/Int# +Diagnostics: +[5:13..5:14): [warning] unused explicit parameter + Synthetics: [68:20..68:24):@ann => *[Int] From e1d0ec90c75fb4c9c0e8116357afd3f5f61f7d2f Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Mon, 5 Jun 2023 17:41:50 +0900 Subject: [PATCH 03/25] Add some comments --- compiler/src/dotty/tools/dotc/Compiler.scala | 4 ++-- .../dotty/tools/dotc/reporting/Reporter.scala | 2 +- .../dotc/semanticdb/ExtractSemanticDB.scala | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index 7ddfea3d09a8..4040b5432067 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -38,7 +38,7 @@ class Compiler { List(new CheckUnused.PostTyper) :: // Check for unused elements List(new YCheckPositions) :: // YCheck positions List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks - List(new semanticdb.ExtractSemanticDB.PostTyper) :: // Extract info into .semanticdb files + List(new semanticdb.ExtractSemanticDB.PostTyper) :: // Extract info and attach to the tree of the unit file List(new PostTyper) :: // Additional checks and cleanups after type checking List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks @@ -51,7 +51,7 @@ class Compiler { List(new Inlining) :: // Inline and execute macros List(new PostInlining) :: // Add mirror support for inlined code List(new CheckUnused.PostInlining) :: // Check for unused elements - List(new semanticdb.ExtractSemanticDB.PostInlining) :: // Extract info into .semanticdb files + List(new semanticdb.ExtractSemanticDB.PostInlining) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file List(new Staging) :: // Check staging levels and heal staged types List(new Splicing) :: // Replace level 1 splices with holes List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures diff --git a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala index 57a8d279aaa5..da89768750d2 100644 --- a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala +++ b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala @@ -114,7 +114,7 @@ abstract class Reporter extends interfaces.ReporterResult { /** All errors reported by this reporter (ignoring outer reporters) */ def allErrors: List[Error] = errors - /** All errors reported by this reporter (ignoring outer reporters) */ + /** All warnings reported by this reporter (ignoring outer reporters) */ def allWarnings: List[Warning] = warnings /** Were sticky errors reported? Overridden in StoreReporter. */ diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 3bc30758ff64..61c2dc2bb560 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -32,10 +32,23 @@ import dotty.tools.dotc.semanticdb.DiagnosticOps.* /** Extract symbol references and uses to semanticdb files. * See https://scalameta.org/docs/semanticdb/specification.html#symbol-1 * for a description of the format. + * + * Here, we define two phases for "ExtractSemanticDB", "PostTyper" and "PostInlining". + * + * The "PostTyper" phase extracts SemanticDB information such as symbol + * definitions, symbol occurrences, type information, and synthetics. + * This phase does not write the information to a .semanticdb file; + * instead, it attaches the SemanticDB information to the top-level tree. + * + * The "PostInlining" phase extracts diagnostics from "ctx.reporter" and + * attaches them to the SemanticDB information extracted in the "PostTyper" phase. + * Afterwards, it writes the SemanticDB to a ".semanticdb" file. + * We need to run this phase after the "CheckUnused.PostInlining" phase + * so that we can extract the warnings generated by "-Wunused". */ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: String, _key: Property.Key[TextDocument]) extends Phase: - override def phaseName: String = ExtractSemanticDB.phaseNamePrefix + suffix + override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + suffix override val description: String = ExtractSemanticDB.description @@ -55,7 +68,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: unit.tpdTree.putAttachment(_key, extractor.toTextDocument(unit.source)) else unit.tpdTree.getAttachment(_key) match - case None => ??? + case None => case Some(doc) => val warnings = ctx.reporter.allWarnings.collect { case w if w.pos.source == ctx.source => w.toSemanticDiagnostic From f17fb5ffe47e5e54622d6b24a7391ced18430afe Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Tue, 27 Jun 2023 20:38:17 +0900 Subject: [PATCH 04/25] ExtractSemanticDB.PostInlining now update the .semanticdb file on disk --- .../dotc/semanticdb/ExtractSemanticDB.scala | 96 +++++++++++-------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 61c2dc2bb560..02af34c5687b 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -27,6 +27,7 @@ import dotty.tools.dotc.{semanticdb => s} import dotty.tools.io.{AbstractFile, JarArchive} import dotty.tools.dotc.util.Property import dotty.tools.dotc.semanticdb.DiagnosticOps.* +import scala.util.{Using, Failure, Success} /** Extract symbol references and uses to semanticdb files. @@ -36,17 +37,15 @@ import dotty.tools.dotc.semanticdb.DiagnosticOps.* * Here, we define two phases for "ExtractSemanticDB", "PostTyper" and "PostInlining". * * The "PostTyper" phase extracts SemanticDB information such as symbol - * definitions, symbol occurrences, type information, and synthetics. - * This phase does not write the information to a .semanticdb file; - * instead, it attaches the SemanticDB information to the top-level tree. + * definitions, symbol occurrences, type information, and synthetics + * and write .semanticdb file. * * The "PostInlining" phase extracts diagnostics from "ctx.reporter" and * attaches them to the SemanticDB information extracted in the "PostTyper" phase. - * Afterwards, it writes the SemanticDB to a ".semanticdb" file. * We need to run this phase after the "CheckUnused.PostInlining" phase * so that we can extract the warnings generated by "-Wunused". */ -class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: String, _key: Property.Key[TextDocument]) extends Phase: +class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: String) extends Phase: override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + suffix @@ -65,15 +64,13 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: if (phaseMode == ExtractSemanticDB.PhaseMode.PostTyper) val extractor = ExtractSemanticDB.Extractor() extractor.extract(unit.tpdTree) - unit.tpdTree.putAttachment(_key, extractor.toTextDocument(unit.source)) + ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) else - unit.tpdTree.getAttachment(_key) match - case None => - case Some(doc) => - val warnings = ctx.reporter.allWarnings.collect { - case w if w.pos.source == ctx.source => w.toSemanticDiagnostic - } - ExtractSemanticDB.write(unit.source, doc.copy(diagnostics = warnings)) + val warnings = ctx.reporter.allWarnings.collect { + case w if w.pos.source == ctx.source => w.toSemanticDiagnostic + } + if (warnings.nonEmpty) + ExtractSemanticDB.appendDiagnostics(unit.source, warnings) end ExtractSemanticDB object ExtractSemanticDB: @@ -88,15 +85,9 @@ object ExtractSemanticDB: case PostTyper case PostInlining - /** - * The key used to retrieve the "unused entity" analysis metadata, - * from the compilation `Context` - */ - private val _key = Property.StickyKey[TextDocument] + class PostTyper extends ExtractSemanticDB(PhaseMode.PostTyper, "PostTyper") - class PostTyper extends ExtractSemanticDB(PhaseMode.PostTyper, "PostTyper", _key) - - class PostInlining extends ExtractSemanticDB(PhaseMode.PostInlining, "PostInlining", _key) + class PostInlining extends ExtractSemanticDB(PhaseMode.PostInlining, "PostInlining") private def semanticdbTarget(using Context): Option[Path] = Option(ctx.settings.semanticdbTarget.value) @@ -109,15 +100,22 @@ object ExtractSemanticDB: private def write( source: SourceFile, - doc: TextDocument + occurrences: List[SymbolOccurrence], + symbolInfos: List[SymbolInformation], + synthetics: List[Synthetic], )(using Context): Unit = - val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value) - val outpath = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) - .resolve("META-INF") - .resolve("semanticdb") - .resolve(relPath) - .resolveSibling(source.name + ".semanticdb") + val outpath = semanticdbPath(source) Files.createDirectories(outpath.getParent()) + val doc: TextDocument = TextDocument( + schema = Schema.SEMANTICDB4, + language = Language.SCALA, + uri = Tools.mkURIstring(Paths.get(relPath(source))), + text = "", + md5 = internal.MD5.compute(String(source.content)), + symbols = symbolInfos, + occurrences = occurrences, + synthetics = synthetics, + ) val docs = TextDocuments(List(doc)) val out = Files.newOutputStream(outpath) try @@ -128,6 +126,34 @@ object ExtractSemanticDB: out.close() end write + private def appendDiagnostics( + source: SourceFile, + diagnostics: Seq[Diagnostic] + )(using Context): Unit = + val path = semanticdbPath(source) + Using.Manager { use => + val in = use(Files.newInputStream(path)) + val sin = internal.SemanticdbInputStream.newInstance(in) + val docs = TextDocuments.parseFrom(sin) + + val out = use(Files.newOutputStream(path)) + val sout = internal.SemanticdbOutputStream.newInstance(out) + TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) + sout.flush() + } match + case Failure(ex) => // failed somehow, should we say something? + case Success(_) => // success to update semanticdb, say nothing + end appendDiagnostics + + private def relPath(source: SourceFile)(using ctx: Context) = + SourceFile.relativePath(source, ctx.settings.sourceroot.value) + + private def semanticdbPath(source: SourceFile)(using ctx: Context) = + absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) + .resolve("META-INF") + .resolve("semanticdb") + .resolve(relPath(source)) + .resolveSibling(source.name + ".semanticdb") /** Extractor of symbol occurrences from trees */ private class Extractor extends TreeTraverser: @@ -136,20 +162,6 @@ object ExtractSemanticDB: val synth = SyntheticsExtractor() given converter: s.TypeOps = s.TypeOps() - - def toTextDocument(source: SourceFile)(using Context): TextDocument = - val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value) - TextDocument( - schema = Schema.SEMANTICDB4, - language = Language.SCALA, - uri = Tools.mkURIstring(Paths.get(relPath)), - text = "", - md5 = internal.MD5.compute(String(source.content)), - symbols = symbolInfos.toList, - occurrences = occurrences.toList, - synthetics = synthetics.toList, - ) - /** The bodies of synthetic locals */ private val localBodies = mutable.HashMap[Symbol, Tree]() From d2b54c2fa3065f64c698f6625d07be818d0a8ff3 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Wed, 19 Jul 2023 19:56:18 +0900 Subject: [PATCH 05/25] Parallelize read/write SemanticDB --- .../dotc/semanticdb/ExtractSemanticDB.scala | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 02af34c5687b..dd11fcfe75f0 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -21,7 +21,9 @@ import transform.SymUtils._ import scala.collection.mutable import scala.annotation.{ threadUnsafe => tu, tailrec } +import scala.jdk.CollectionConverters._ import scala.PartialFunction.condOpt +import typer.ImportInfo.withRootImports import dotty.tools.dotc.{semanticdb => s} import dotty.tools.io.{AbstractFile, JarArchive} @@ -59,18 +61,28 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: // Check not needed since it does not transform trees override def isCheckable: Boolean = false - override def run(using Context): Unit = - val unit = ctx.compilationUnit - if (phaseMode == ExtractSemanticDB.PhaseMode.PostTyper) - val extractor = ExtractSemanticDB.Extractor() - extractor.extract(unit.tpdTree) - ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) - else - val warnings = ctx.reporter.allWarnings.collect { - case w if w.pos.source == ctx.source => w.toSemanticDiagnostic - } - if (warnings.nonEmpty) - ExtractSemanticDB.appendDiagnostics(unit.source, warnings) + override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = { + val appendWarnings = phaseMode == ExtractSemanticDB.PhaseMode.PostInlining + val warnings = + if (appendWarnings) + ctx.reporter.allWarnings.groupBy(w => w.pos.source) + else Map.empty + + units.asJava.parallelStream().map { unit => + val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports + if (appendWarnings) + warnings.get(unit.source).foreach { ws => + ExtractSemanticDB.appendDiagnostics(unit.source, ws.map(_.toSemanticDiagnostic)) + } + else + val extractor = ExtractSemanticDB.Extractor() + extractor.extract(unit.tpdTree) + ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) + unit + }.toList().asScala.toList + } + + def run(using Context): Unit = unsupported("run") end ExtractSemanticDB object ExtractSemanticDB: From 56c5909c99b7b8489a22fea91199714971f70a31 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Wed, 19 Jul 2023 20:13:42 +0900 Subject: [PATCH 06/25] Run AppendDiagnostics phase after crossVersionChecks So that we can add deprecation warnings to SemanticDB --- compiler/src/dotty/tools/dotc/Compiler.scala | 4 +-- .../dotc/semanticdb/ExtractSemanticDB.scala | 20 ++++++------- .../semanticdb/expect/Deprecated.expect.scala | 4 +++ tests/semanticdb/expect/Deprecated.scala | 4 +++ tests/semanticdb/metac.expect | 28 +++++++++++++++++++ 5 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 tests/semanticdb/expect/Deprecated.expect.scala create mode 100644 tests/semanticdb/expect/Deprecated.scala diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index 4040b5432067..c5a1c6d264c3 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -38,7 +38,7 @@ class Compiler { List(new CheckUnused.PostTyper) :: // Check for unused elements List(new YCheckPositions) :: // YCheck positions List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks - List(new semanticdb.ExtractSemanticDB.PostTyper) :: // Extract info and attach to the tree of the unit file + List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info and attach to the tree of the unit file List(new PostTyper) :: // Additional checks and cleanups after type checking List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks @@ -51,7 +51,6 @@ class Compiler { List(new Inlining) :: // Inline and execute macros List(new PostInlining) :: // Add mirror support for inlined code List(new CheckUnused.PostInlining) :: // Check for unused elements - List(new semanticdb.ExtractSemanticDB.PostInlining) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file List(new Staging) :: // Check staging levels and heal staged types List(new Splicing) :: // Replace level 1 splices with holes List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures @@ -72,6 +71,7 @@ class Compiler { new ExpandSAMs, // Expand single abstract method closures to anonymous classes new ElimRepeated, // Rewrite vararg parameters and arguments new RefChecks) :: // Various checks mostly related to abstract members and overriding + List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file List(new init.Checker) :: // Check initialization of objects List(new ProtectedAccessors, // Add accessors for protected members new ExtensionMethods, // Expand methods of value classes with extension methods diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index dd11fcfe75f0..fdb7b05fd178 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -62,15 +62,15 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: override def isCheckable: Boolean = false override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = { - val appendWarnings = phaseMode == ExtractSemanticDB.PhaseMode.PostInlining + val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics val warnings = - if (appendWarnings) + if (appendDiagnostics) ctx.reporter.allWarnings.groupBy(w => w.pos.source) else Map.empty - units.asJava.parallelStream().map { unit => + units.asJava.parallelStream().forEach { unit => val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports - if (appendWarnings) + if (appendDiagnostics) warnings.get(unit.source).foreach { ws => ExtractSemanticDB.appendDiagnostics(unit.source, ws.map(_.toSemanticDiagnostic)) } @@ -78,8 +78,8 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: val extractor = ExtractSemanticDB.Extractor() extractor.extract(unit.tpdTree) ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) - unit - }.toList().asScala.toList + } + units } def run(using Context): Unit = unsupported("run") @@ -94,12 +94,12 @@ object ExtractSemanticDB: val description: String = "extract info into .semanticdb files" enum PhaseMode: - case PostTyper - case PostInlining + case ExtractSemanticInfo + case AppendDiagnostics - class PostTyper extends ExtractSemanticDB(PhaseMode.PostTyper, "PostTyper") + class ExtractSemanticInfo extends ExtractSemanticDB(PhaseMode.ExtractSemanticInfo, "ExtractSemanticInfo") - class PostInlining extends ExtractSemanticDB(PhaseMode.PostInlining, "PostInlining") + class AppendDiagnostics extends ExtractSemanticDB(PhaseMode.AppendDiagnostics, "AppendDiagnostics") private def semanticdbTarget(using Context): Option[Path] = Option(ctx.settings.semanticdbTarget.value) diff --git a/tests/semanticdb/expect/Deprecated.expect.scala b/tests/semanticdb/expect/Deprecated.expect.scala new file mode 100644 index 000000000000..f81dd5eacf68 --- /dev/null +++ b/tests/semanticdb/expect/Deprecated.expect.scala @@ -0,0 +1,4 @@ +object Deprecated/*<-_empty_::Deprecated.*/ { + @deprecated/*->scala::deprecated#*/ def deprecatedMethod/*<-_empty_::Deprecated.deprecatedMethod().*/ = ???/*->scala::Predef.`???`().*/ + def main/*<-_empty_::Deprecated.main().*/ = deprecatedMethod/*->_empty_::Deprecated.deprecatedMethod().*/ +} diff --git a/tests/semanticdb/expect/Deprecated.scala b/tests/semanticdb/expect/Deprecated.scala new file mode 100644 index 000000000000..cd7bb5ac61de --- /dev/null +++ b/tests/semanticdb/expect/Deprecated.scala @@ -0,0 +1,4 @@ +object Deprecated { + @deprecated def deprecatedMethod = ??? + def main = deprecatedMethod +} diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 34eb70231150..8b95a736ba44 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -731,6 +731,34 @@ Synthetics: [51:16..51:27):List(1).map => *[Int] [51:16..51:20):List => *.apply[Int] +expect/Deprecated.scala +----------------------- + +Summary: +Schema => SemanticDB v4 +Uri => Deprecated.scala +Text => empty +Language => Scala +Symbols => 3 entries +Occurrences => 6 entries +Diagnostics => 1 entries + +Symbols: +_empty_/Deprecated. => final object Deprecated extends Object { self: Deprecated.type => +3 decls } +_empty_/Deprecated.deprecatedMethod(). => @deprecated method deprecatedMethod => Nothing +_empty_/Deprecated.main(). => method main => Nothing + +Occurrences: +[0:7..0:17): Deprecated <- _empty_/Deprecated. +[1:3..1:13): deprecated -> scala/deprecated# +[1:18..1:34): deprecatedMethod <- _empty_/Deprecated.deprecatedMethod(). +[1:37..1:40): ??? -> scala/Predef.`???`(). +[2:6..2:10): main <- _empty_/Deprecated.main(). +[2:13..2:29): deprecatedMethod -> _empty_/Deprecated.deprecatedMethod(). + +Diagnostics: +[2:13..2:29): [warning] method deprecatedMethod in object Deprecated is deprecated + expect/Empty.scala ------------------ From 50e0fd2eacafb58453a60e52a4f43a53b0c8e02d Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 20 Jul 2023 16:51:08 +0900 Subject: [PATCH 07/25] [WIP] Don't parse TextDocuments in AppendDiagnostics --- .../dotc/semanticdb/ExtractSemanticDB.scala | 63 ++++++++++---- .../dotc/semanticdb/SemanticdbTests.scala | 4 + tests/semanticdb/metac.expect | 83 ------------------- 3 files changed, 51 insertions(+), 99 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index fdb7b05fd178..2453eba09cfd 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -30,6 +30,11 @@ import dotty.tools.io.{AbstractFile, JarArchive} import dotty.tools.dotc.util.Property import dotty.tools.dotc.semanticdb.DiagnosticOps.* import scala.util.{Using, Failure, Success} +import com.google.protobuf.Empty +import com.google.protobuf.UnknownFieldSet +import com.google.protobuf.UnknownFieldSet.Field +import java.io.ByteArrayOutputStream +import java.io.BufferedOutputStream /** Extract symbol references and uses to semanticdb files. @@ -63,22 +68,20 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = { val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics - val warnings = - if (appendDiagnostics) - ctx.reporter.allWarnings.groupBy(w => w.pos.source) - else Map.empty - - units.asJava.parallelStream().forEach { unit => - val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports - if (appendDiagnostics) + if (appendDiagnostics) + val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) + units.asJava.parallelStream().forEach { unit => + val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports warnings.get(unit.source).foreach { ws => ExtractSemanticDB.appendDiagnostics(unit.source, ws.map(_.toSemanticDiagnostic)) } - else + } + else + units.foreach { unit => val extractor = ExtractSemanticDB.Extractor() extractor.extract(unit.tpdTree) ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) - } + } units } @@ -145,15 +148,43 @@ object ExtractSemanticDB: val path = semanticdbPath(source) Using.Manager { use => val in = use(Files.newInputStream(path)) - val sin = internal.SemanticdbInputStream.newInstance(in) - val docs = TextDocuments.parseFrom(sin) + // val sin = internal.SemanticdbInputStream.newInstance(in) + val textDocuments = Empty.parseFrom(in) + val docsBytes = textDocuments.getUnknownFields().getField(TextDocuments.DOCUMENTS_FIELD_NUMBER).getLengthDelimitedList() + val docFields = Empty.parseFrom(docsBytes.get(0)).getUnknownFields() + if (source.file.name == "ValPattern.scala") + println(docFields) + //docMap.put(7, ) + + // val docs = TextDocuments.parseFrom(sin) + + val bos = use(new ByteArrayOutputStream()) + val sbos = internal.SemanticdbOutputStream.newInstance(bos) + val doc = TextDocument(diagnostics = diagnostics) + doc.writeTo(sbos) + sbos.flush() + val diagnosticsOnly = Empty.parseFrom(bos.toByteArray()).getUnknownFields() + + val merged = docFields.toBuilder().mergeFrom(diagnosticsOnly).build() + // println(merged) + val field = Field.newBuilder().addLengthDelimited(merged.toByteString()).build() + + val fields = textDocuments.getUnknownFields().toBuilder().mergeField(TextDocuments.DOCUMENTS_FIELD_NUMBER, field).build() + // println(fields) + val updated = textDocuments.toBuilder().setUnknownFields(fields).build() + if (source.file.name == "ValPattern.scala") + println(updated) val out = use(Files.newOutputStream(path)) - val sout = internal.SemanticdbOutputStream.newInstance(out) - TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) - sout.flush() + val bout = new BufferedOutputStream(out) + updated.writeTo(bout) + bout.flush() + // val sout = internal.SemanticdbOutputStream.newInstance(out) + // TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) } match - case Failure(ex) => // failed somehow, should we say something? + case Failure(ex) => + println(ex.getMessage()) + // failed somehow, should we say something? case Success(_) => // success to update semanticdb, say nothing end appendDiagnostics diff --git a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala index de8c2c11f9c2..6d2eec8363ad 100644 --- a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala +++ b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala @@ -87,8 +87,12 @@ class SemanticdbTests: .resolve("semanticdb") .resolve(relpath) .resolveSibling(filename + ".semanticdb") + println(semanticdbPath) val expectPath = source.resolveSibling(filename.replace(".scala", ".expect.scala")) val doc = Tools.loadTextDocument(source, relpath, semanticdbPath) + println(semanticdbPath.getFileName().toString()) + if (semanticdbPath.getFileName().toString() == "ValPattern.scala.semanticdb") + println(doc) Tools.metac(doc, rootSrc.relativize(source))(using metacSb) val obtained = trimTrailingWhitespace(SemanticdbTests.printTextDocument(doc)) collectErrorOrUpdate(expectPath, obtained) diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 8b95a736ba44..78cb41e18882 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -8,7 +8,6 @@ Text => empty Language => Scala Symbols => 9 entries Occurrences => 18 entries -Diagnostics => 2 entries Symbols: example/Access# => class Access extends Object { self: Access => +8 decls } @@ -41,10 +40,6 @@ Occurrences: [9:6..9:8): m7 <- example/Access#m7(). [9:11..9:14): ??? -> scala/Predef.`???`(). -Diagnostics: -[3:14..3:16): [warning] unused private member -[4:20..4:22): [warning] unused private member - expect/Advanced.scala --------------------- @@ -55,7 +50,6 @@ Text => empty Language => Scala Symbols => 61 entries Occurrences => 138 entries -Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -261,9 +255,6 @@ Occurrences: [53:37..53:38): x -> advanced/HKClass#foo().(x) [53:39..53:47): toString -> scala/Tuple2#toString(). -Diagnostics: -[40:12..40:15): [warning] unused local definition - Synthetics: [27:12..27:16):s.s1 => reflectiveSelectable(*) [29:12..29:16):s.s2 => reflectiveSelectable(*) @@ -279,7 +270,6 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 52 entries -Diagnostics => 2 entries Symbols: annot/Alias. => final object Alias extends Object { self: Alias.type => +2 decls } @@ -360,10 +350,6 @@ Occurrences: [39:11..39:26): ClassAnnotation -> com/javacp/annot/ClassAnnotation# [39:28..39:33): param -> scala/annotation/meta/param# -Diagnostics: -[7:67..7:68): [warning] unused explicit parameter -[21:33..21:34): [warning] unused explicit parameter - expect/Anonymous.scala ---------------------- @@ -374,7 +360,6 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 47 entries -Diagnostics => 1 entries Synthetics => 2 entries Symbols: @@ -451,9 +436,6 @@ Occurrences: [23:33..23:39): String -> scala/Predef.String# [23:42..23:45): ??? -> scala/Predef.`???`(). -Diagnostics: -[14:8..14:9): [warning] unused local definition - Synthetics: [10:2..10:9):locally => *[Unit] [13:2..13:9):locally => *[Unit] @@ -492,7 +474,6 @@ Text => empty Language => Scala Symbols => 108 entries Occurrences => 114 entries -Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -721,12 +702,6 @@ Occurrences: [53:4..53:9): local -> local4 [53:10..53:11): + -> scala/Int#`+`(+4). -Diagnostics: -[18:9..18:10): [warning] unused explicit parameter -[20:27..20:28): [warning] unused explicit parameter -[22:27..22:28): [warning] unused explicit parameter -[24:10..24:11): [warning] unused explicit parameter - Synthetics: [51:16..51:27):List(1).map => *[Int] [51:16..51:20):List => *.apply[Int] @@ -741,7 +716,6 @@ Text => empty Language => Scala Symbols => 3 entries Occurrences => 6 entries -Diagnostics => 1 entries Symbols: _empty_/Deprecated. => final object Deprecated extends Object { self: Deprecated.type => +3 decls } @@ -756,9 +730,6 @@ Occurrences: [2:6..2:10): main <- _empty_/Deprecated.main(). [2:13..2:29): deprecatedMethod -> _empty_/Deprecated.deprecatedMethod(). -Diagnostics: -[2:13..2:29): [warning] method deprecatedMethod in object Deprecated is deprecated - expect/Empty.scala ------------------ @@ -816,7 +787,6 @@ Text => empty Language => Scala Symbols => 30 entries Occurrences => 46 entries -Diagnostics => 3 entries Symbols: endmarkers/Container# => class Container extends Object { self: Container => +5 decls } @@ -898,11 +868,6 @@ Occurrences: [64:14..64:20): String -> scala/Predef.String# [67:4..67:14): endmarkers -> endmarkers/ -Diagnostics: -[38:8..38:16): [warning] unused local definition -[42:8..42:16): [warning] unused local definition -[46:8..46:16): [warning] unused local definition - expect/EndMarkers2.scala ------------------------ @@ -982,7 +947,6 @@ Text => empty Language => Scala Symbols => 181 entries Occurrences => 148 entries -Diagnostics => 2 entries Synthetics => 6 entries Symbols: @@ -1318,10 +1282,6 @@ Occurrences: [68:9..68:16): Neptune <- _empty_/Enums.Planet.Neptune. [68:25..68:31): Planet -> _empty_/Enums.Planet# -Diagnostics: -[30:12..30:17): [warning] unused explicit parameter -[48:13..48:13): [warning] `:` after symbolic operator is deprecated; use backticks around operator instead - Synthetics: [52:9..52:13):Refl => *.unapply[Option[B]] [52:31..52:50):identity[Option[B]] => *[Function1[A, Option[B]]] @@ -1374,7 +1334,6 @@ Text => empty Language => Scala Symbols => 5 entries Occurrences => 23 entries -Diagnostics => 1 entries Symbols: example/Example. => final object Example extends Object { self: Example.type => +3 decls } @@ -1408,9 +1367,6 @@ Occurrences: [9:24..9:32): classTag -> scala/reflect/package.classTag(). [9:33..9:36): Int -> scala/Int# -Diagnostics: -[2:24..2:30): [warning] unused import - expect/Extension.scala ---------------------- @@ -1605,7 +1561,6 @@ Text => empty Language => Scala Symbols => 29 entries Occurrences => 65 entries -Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -1706,11 +1661,6 @@ Occurrences: [26:57..26:58): A -> a/b/Givens.foo().(A) [26:59..26:64): empty -> a/b/Givens.Monoid#empty(). -Diagnostics: -[24:53..24:63): [warning] An inline given alias with a function value as right-hand side can significantly increase -generated code size. You should either drop the `inline` or rewrite the given with an -explicit `apply` method. - Synthetics: [12:17..12:25):sayHello => *[Int] [13:19..13:29):sayGoodbye => *[Int] @@ -1825,7 +1775,6 @@ Text => empty Language => Scala Symbols => 2 entries Occurrences => 16 entries -Diagnostics => 1 entries Symbols: _empty_/Imports$package. => final package object _empty_ extends Object { self: _empty_.type => +2 decls } @@ -1849,9 +1798,6 @@ Occurrences: [3:25..3:28): Int -> scala/Int# [3:30..3:33): Int -> scala/Int# -Diagnostics: -[0:26..0:34): [warning] unused import - expect/InstrumentTyper.scala ---------------------------- @@ -3576,7 +3522,6 @@ Text => empty Language => Scala Symbols => 22 entries Occurrences => 44 entries -Diagnostics => 3 entries Synthetics => 11 entries Symbols: @@ -3649,11 +3594,6 @@ Occurrences: [39:10..39:17): leftVar -> local3 [40:10..40:18): rightVar -> local4 -Diagnostics: -[30:11..30:18): [warning] unset local variable -[30:20..30:28): [warning] unset local variable -[31:15..31:25): [warning] unset local variable - Synthetics: [5:6..5:10):Some => *.unapply[Int] [6:4..6:8):Some => *.apply[Int] @@ -3677,7 +3617,6 @@ Text => empty Language => Scala Symbols => 42 entries Occurrences => 128 entries -Diagnostics => 1 entries Symbols: example/ValUsages. => final object ValUsages extends Object { self: ValUsages.type => +2 decls } @@ -3853,9 +3792,6 @@ Occurrences: [49:2..49:3): v -> example/ValUsages.v. [49:3..49:18): .explicitSetter -> example/Vals#`explicitSetter_=`(). -Diagnostics: -[2:20..2:21): [warning] unused explicit parameter - expect/Vararg.scala ------------------- @@ -3897,7 +3833,6 @@ Text => empty Language => Scala Symbols => 8 entries Occurrences => 18 entries -Diagnostics => 1 entries Symbols: _empty_/Test_depmatch. => final object Test_depmatch extends Object { self: Test_depmatch.type => +4 decls } @@ -3929,9 +3864,6 @@ Occurrences: [6:19..6:20): U -> local0 [6:24..6:27): ??? -> scala/Predef.`???`(). -Diagnostics: -[6:8..6:9): [warning] unused local definition - expect/example-dir/FileInDir.scala ---------------------------------- @@ -4207,7 +4139,6 @@ Text => empty Language => Scala Symbols => 7 entries Occurrences => 8 entries -Diagnostics => 1 entries Symbols: i9727/Test# => class Test extends Object { self: Test => +2 decls } @@ -4228,9 +4159,6 @@ Occurrences: [4:4..4:5): b <- i9727/i9727$package.b. [4:12..4:16): Test -> i9727/Test# -Diagnostics: -[2:11..2:12): [warning] unused explicit parameter - expect/i9782.scala ------------------ @@ -4615,7 +4543,6 @@ Text => empty Language => Scala Symbols => 50 entries Occurrences => 73 entries -Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -4745,12 +4672,6 @@ Occurrences: [25:27..25:28): t <- local1 [25:33..25:36): ??? -> scala/Predef.`???`(). -Diagnostics: -[9:30..9:31): [warning] unused explicit parameter -[9:36..9:37): [warning] unused explicit parameter -[9:42..9:43): [warning] unused explicit parameter -[21:11..21:12): [warning] unused explicit parameter - Synthetics: [23:6..23:10):List => *.unapplySeq[Nothing] [24:19..24:23):List => *.unapplySeq[Nothing] @@ -4765,7 +4686,6 @@ Text => empty Language => Scala Symbols => 143 entries Occurrences => 228 entries -Diagnostics => 1 entries Synthetics => 1 entries Symbols: @@ -5143,9 +5063,6 @@ Occurrences: [119:32..119:38): Option -> scala/Option# [119:39..119:42): Int -> scala/Int# -Diagnostics: -[5:13..5:14): [warning] unused explicit parameter - Synthetics: [68:20..68:24):@ann => *[Int] From 2babbc4e40b60fcdee9d065035dc2ef54cc0e4b8 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 20 Jul 2023 17:25:06 +0900 Subject: [PATCH 08/25] Remove unnecessary suffix --- .../dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 2453eba09cfd..4736d43af58d 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -52,9 +52,9 @@ import java.io.BufferedOutputStream * We need to run this phase after the "CheckUnused.PostInlining" phase * so that we can extract the warnings generated by "-Wunused". */ -class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode, suffix: String) extends Phase: +class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends Phase: - override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + suffix + override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + phaseMode.toString() override val description: String = ExtractSemanticDB.description @@ -100,9 +100,9 @@ object ExtractSemanticDB: case ExtractSemanticInfo case AppendDiagnostics - class ExtractSemanticInfo extends ExtractSemanticDB(PhaseMode.ExtractSemanticInfo, "ExtractSemanticInfo") + class ExtractSemanticInfo extends ExtractSemanticDB(PhaseMode.ExtractSemanticInfo) - class AppendDiagnostics extends ExtractSemanticDB(PhaseMode.AppendDiagnostics, "AppendDiagnostics") + class AppendDiagnostics extends ExtractSemanticDB(PhaseMode.AppendDiagnostics) private def semanticdbTarget(using Context): Option[Path] = Option(ctx.settings.semanticdbTarget.value) From 5186b628ed0be47afccf18ae5e7081178f6df6ee Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 20 Jul 2023 17:32:13 +0900 Subject: [PATCH 09/25] Remove unused local + Context --- .../src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 4736d43af58d..4ab10bcbd3ba 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -71,7 +71,6 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends if (appendDiagnostics) val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) units.asJava.parallelStream().forEach { unit => - val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports warnings.get(unit.source).foreach { ws => ExtractSemanticDB.appendDiagnostics(unit.source, ws.map(_.toSemanticDiagnostic)) } @@ -191,7 +190,7 @@ object ExtractSemanticDB: private def relPath(source: SourceFile)(using ctx: Context) = SourceFile.relativePath(source, ctx.settings.sourceroot.value) - private def semanticdbPath(source: SourceFile)(using ctx: Context) = + private def semanticdbPath(source: SourceFile)(using Context) = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) .resolve("META-INF") .resolve("semanticdb") From a414fae9a2918a44ce0c08424b9985367551e76b Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 20 Jul 2023 18:15:02 +0900 Subject: [PATCH 10/25] Fix presentation compiler --- .../src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 2 +- .../main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 4ab10bcbd3ba..76d94c7a5526 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -198,7 +198,7 @@ object ExtractSemanticDB: .resolveSibling(source.name + ".semanticdb") /** Extractor of symbol occurrences from trees */ - private class Extractor extends TreeTraverser: + class Extractor extends TreeTraverser: import Scala3.{_, given} given s.SemanticSymbolBuilder = s.SemanticSymbolBuilder() val synth = SyntheticsExtractor() diff --git a/presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala b/presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala index f4e17913776d..22975c2eefcb 100644 --- a/presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala +++ b/presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala @@ -32,8 +32,7 @@ class SemanticdbTextDocumentProvider( SourceFile.virtual(filePath.toString, validCode) ) val tree = driver.currentCtx.run.units.head.tpdTree - val extract = ExtractSemanticDB() - val extractor = extract.Extractor() + val extractor = ExtractSemanticDB.Extractor() extractor.traverse(tree)(using driver.currentCtx) val path = workspace .flatMap { workspacePath => From d5065ec3c517ba4e51f3b91a6541c47372b26328 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Tue, 25 Jul 2023 21:03:58 +0900 Subject: [PATCH 11/25] Parse TextDocuments instead of using `com.google.protobuf` --- .../dotc/semanticdb/ExtractSemanticDB.scala | 43 ++-------- tests/semanticdb/metac.expect | 83 +++++++++++++++++++ 2 files changed, 89 insertions(+), 37 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 76d94c7a5526..0b7496bcee1b 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -30,9 +30,6 @@ import dotty.tools.io.{AbstractFile, JarArchive} import dotty.tools.dotc.util.Property import dotty.tools.dotc.semanticdb.DiagnosticOps.* import scala.util.{Using, Failure, Success} -import com.google.protobuf.Empty -import com.google.protobuf.UnknownFieldSet -import com.google.protobuf.UnknownFieldSet.Field import java.io.ByteArrayOutputStream import java.io.BufferedOutputStream @@ -147,43 +144,15 @@ object ExtractSemanticDB: val path = semanticdbPath(source) Using.Manager { use => val in = use(Files.newInputStream(path)) - // val sin = internal.SemanticdbInputStream.newInstance(in) - val textDocuments = Empty.parseFrom(in) - val docsBytes = textDocuments.getUnknownFields().getField(TextDocuments.DOCUMENTS_FIELD_NUMBER).getLengthDelimitedList() - val docFields = Empty.parseFrom(docsBytes.get(0)).getUnknownFields() - if (source.file.name == "ValPattern.scala") - println(docFields) - //docMap.put(7, ) - - // val docs = TextDocuments.parseFrom(sin) - - val bos = use(new ByteArrayOutputStream()) - val sbos = internal.SemanticdbOutputStream.newInstance(bos) - val doc = TextDocument(diagnostics = diagnostics) - doc.writeTo(sbos) - sbos.flush() - val diagnosticsOnly = Empty.parseFrom(bos.toByteArray()).getUnknownFields() - - val merged = docFields.toBuilder().mergeFrom(diagnosticsOnly).build() - // println(merged) - val field = Field.newBuilder().addLengthDelimited(merged.toByteString()).build() - - val fields = textDocuments.getUnknownFields().toBuilder().mergeField(TextDocuments.DOCUMENTS_FIELD_NUMBER, field).build() - // println(fields) - val updated = textDocuments.toBuilder().setUnknownFields(fields).build() - if (source.file.name == "ValPattern.scala") - println(updated) + val sin = internal.SemanticdbInputStream.newInstance(in) + val docs = TextDocuments.parseFrom(sin) val out = use(Files.newOutputStream(path)) - val bout = new BufferedOutputStream(out) - updated.writeTo(bout) - bout.flush() - // val sout = internal.SemanticdbOutputStream.newInstance(out) - // TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) + val sout = internal.SemanticdbOutputStream.newInstance(out) + TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) + sout.flush() } match - case Failure(ex) => - println(ex.getMessage()) - // failed somehow, should we say something? + case Failure(ex) => // failed somehow, should we say something? case Success(_) => // success to update semanticdb, say nothing end appendDiagnostics diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 78cb41e18882..8b95a736ba44 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -8,6 +8,7 @@ Text => empty Language => Scala Symbols => 9 entries Occurrences => 18 entries +Diagnostics => 2 entries Symbols: example/Access# => class Access extends Object { self: Access => +8 decls } @@ -40,6 +41,10 @@ Occurrences: [9:6..9:8): m7 <- example/Access#m7(). [9:11..9:14): ??? -> scala/Predef.`???`(). +Diagnostics: +[3:14..3:16): [warning] unused private member +[4:20..4:22): [warning] unused private member + expect/Advanced.scala --------------------- @@ -50,6 +55,7 @@ Text => empty Language => Scala Symbols => 61 entries Occurrences => 138 entries +Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -255,6 +261,9 @@ Occurrences: [53:37..53:38): x -> advanced/HKClass#foo().(x) [53:39..53:47): toString -> scala/Tuple2#toString(). +Diagnostics: +[40:12..40:15): [warning] unused local definition + Synthetics: [27:12..27:16):s.s1 => reflectiveSelectable(*) [29:12..29:16):s.s2 => reflectiveSelectable(*) @@ -270,6 +279,7 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 52 entries +Diagnostics => 2 entries Symbols: annot/Alias. => final object Alias extends Object { self: Alias.type => +2 decls } @@ -350,6 +360,10 @@ Occurrences: [39:11..39:26): ClassAnnotation -> com/javacp/annot/ClassAnnotation# [39:28..39:33): param -> scala/annotation/meta/param# +Diagnostics: +[7:67..7:68): [warning] unused explicit parameter +[21:33..21:34): [warning] unused explicit parameter + expect/Anonymous.scala ---------------------- @@ -360,6 +374,7 @@ Text => empty Language => Scala Symbols => 23 entries Occurrences => 47 entries +Diagnostics => 1 entries Synthetics => 2 entries Symbols: @@ -436,6 +451,9 @@ Occurrences: [23:33..23:39): String -> scala/Predef.String# [23:42..23:45): ??? -> scala/Predef.`???`(). +Diagnostics: +[14:8..14:9): [warning] unused local definition + Synthetics: [10:2..10:9):locally => *[Unit] [13:2..13:9):locally => *[Unit] @@ -474,6 +492,7 @@ Text => empty Language => Scala Symbols => 108 entries Occurrences => 114 entries +Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -702,6 +721,12 @@ Occurrences: [53:4..53:9): local -> local4 [53:10..53:11): + -> scala/Int#`+`(+4). +Diagnostics: +[18:9..18:10): [warning] unused explicit parameter +[20:27..20:28): [warning] unused explicit parameter +[22:27..22:28): [warning] unused explicit parameter +[24:10..24:11): [warning] unused explicit parameter + Synthetics: [51:16..51:27):List(1).map => *[Int] [51:16..51:20):List => *.apply[Int] @@ -716,6 +741,7 @@ Text => empty Language => Scala Symbols => 3 entries Occurrences => 6 entries +Diagnostics => 1 entries Symbols: _empty_/Deprecated. => final object Deprecated extends Object { self: Deprecated.type => +3 decls } @@ -730,6 +756,9 @@ Occurrences: [2:6..2:10): main <- _empty_/Deprecated.main(). [2:13..2:29): deprecatedMethod -> _empty_/Deprecated.deprecatedMethod(). +Diagnostics: +[2:13..2:29): [warning] method deprecatedMethod in object Deprecated is deprecated + expect/Empty.scala ------------------ @@ -787,6 +816,7 @@ Text => empty Language => Scala Symbols => 30 entries Occurrences => 46 entries +Diagnostics => 3 entries Symbols: endmarkers/Container# => class Container extends Object { self: Container => +5 decls } @@ -868,6 +898,11 @@ Occurrences: [64:14..64:20): String -> scala/Predef.String# [67:4..67:14): endmarkers -> endmarkers/ +Diagnostics: +[38:8..38:16): [warning] unused local definition +[42:8..42:16): [warning] unused local definition +[46:8..46:16): [warning] unused local definition + expect/EndMarkers2.scala ------------------------ @@ -947,6 +982,7 @@ Text => empty Language => Scala Symbols => 181 entries Occurrences => 148 entries +Diagnostics => 2 entries Synthetics => 6 entries Symbols: @@ -1282,6 +1318,10 @@ Occurrences: [68:9..68:16): Neptune <- _empty_/Enums.Planet.Neptune. [68:25..68:31): Planet -> _empty_/Enums.Planet# +Diagnostics: +[30:12..30:17): [warning] unused explicit parameter +[48:13..48:13): [warning] `:` after symbolic operator is deprecated; use backticks around operator instead + Synthetics: [52:9..52:13):Refl => *.unapply[Option[B]] [52:31..52:50):identity[Option[B]] => *[Function1[A, Option[B]]] @@ -1334,6 +1374,7 @@ Text => empty Language => Scala Symbols => 5 entries Occurrences => 23 entries +Diagnostics => 1 entries Symbols: example/Example. => final object Example extends Object { self: Example.type => +3 decls } @@ -1367,6 +1408,9 @@ Occurrences: [9:24..9:32): classTag -> scala/reflect/package.classTag(). [9:33..9:36): Int -> scala/Int# +Diagnostics: +[2:24..2:30): [warning] unused import + expect/Extension.scala ---------------------- @@ -1561,6 +1605,7 @@ Text => empty Language => Scala Symbols => 29 entries Occurrences => 65 entries +Diagnostics => 1 entries Synthetics => 3 entries Symbols: @@ -1661,6 +1706,11 @@ Occurrences: [26:57..26:58): A -> a/b/Givens.foo().(A) [26:59..26:64): empty -> a/b/Givens.Monoid#empty(). +Diagnostics: +[24:53..24:63): [warning] An inline given alias with a function value as right-hand side can significantly increase +generated code size. You should either drop the `inline` or rewrite the given with an +explicit `apply` method. + Synthetics: [12:17..12:25):sayHello => *[Int] [13:19..13:29):sayGoodbye => *[Int] @@ -1775,6 +1825,7 @@ Text => empty Language => Scala Symbols => 2 entries Occurrences => 16 entries +Diagnostics => 1 entries Symbols: _empty_/Imports$package. => final package object _empty_ extends Object { self: _empty_.type => +2 decls } @@ -1798,6 +1849,9 @@ Occurrences: [3:25..3:28): Int -> scala/Int# [3:30..3:33): Int -> scala/Int# +Diagnostics: +[0:26..0:34): [warning] unused import + expect/InstrumentTyper.scala ---------------------------- @@ -3522,6 +3576,7 @@ Text => empty Language => Scala Symbols => 22 entries Occurrences => 44 entries +Diagnostics => 3 entries Synthetics => 11 entries Symbols: @@ -3594,6 +3649,11 @@ Occurrences: [39:10..39:17): leftVar -> local3 [40:10..40:18): rightVar -> local4 +Diagnostics: +[30:11..30:18): [warning] unset local variable +[30:20..30:28): [warning] unset local variable +[31:15..31:25): [warning] unset local variable + Synthetics: [5:6..5:10):Some => *.unapply[Int] [6:4..6:8):Some => *.apply[Int] @@ -3617,6 +3677,7 @@ Text => empty Language => Scala Symbols => 42 entries Occurrences => 128 entries +Diagnostics => 1 entries Symbols: example/ValUsages. => final object ValUsages extends Object { self: ValUsages.type => +2 decls } @@ -3792,6 +3853,9 @@ Occurrences: [49:2..49:3): v -> example/ValUsages.v. [49:3..49:18): .explicitSetter -> example/Vals#`explicitSetter_=`(). +Diagnostics: +[2:20..2:21): [warning] unused explicit parameter + expect/Vararg.scala ------------------- @@ -3833,6 +3897,7 @@ Text => empty Language => Scala Symbols => 8 entries Occurrences => 18 entries +Diagnostics => 1 entries Symbols: _empty_/Test_depmatch. => final object Test_depmatch extends Object { self: Test_depmatch.type => +4 decls } @@ -3864,6 +3929,9 @@ Occurrences: [6:19..6:20): U -> local0 [6:24..6:27): ??? -> scala/Predef.`???`(). +Diagnostics: +[6:8..6:9): [warning] unused local definition + expect/example-dir/FileInDir.scala ---------------------------------- @@ -4139,6 +4207,7 @@ Text => empty Language => Scala Symbols => 7 entries Occurrences => 8 entries +Diagnostics => 1 entries Symbols: i9727/Test# => class Test extends Object { self: Test => +2 decls } @@ -4159,6 +4228,9 @@ Occurrences: [4:4..4:5): b <- i9727/i9727$package.b. [4:12..4:16): Test -> i9727/Test# +Diagnostics: +[2:11..2:12): [warning] unused explicit parameter + expect/i9782.scala ------------------ @@ -4543,6 +4615,7 @@ Text => empty Language => Scala Symbols => 50 entries Occurrences => 73 entries +Diagnostics => 4 entries Synthetics => 2 entries Symbols: @@ -4672,6 +4745,12 @@ Occurrences: [25:27..25:28): t <- local1 [25:33..25:36): ??? -> scala/Predef.`???`(). +Diagnostics: +[9:30..9:31): [warning] unused explicit parameter +[9:36..9:37): [warning] unused explicit parameter +[9:42..9:43): [warning] unused explicit parameter +[21:11..21:12): [warning] unused explicit parameter + Synthetics: [23:6..23:10):List => *.unapplySeq[Nothing] [24:19..24:23):List => *.unapplySeq[Nothing] @@ -4686,6 +4765,7 @@ Text => empty Language => Scala Symbols => 143 entries Occurrences => 228 entries +Diagnostics => 1 entries Synthetics => 1 entries Symbols: @@ -5063,6 +5143,9 @@ Occurrences: [119:32..119:38): Option -> scala/Option# [119:39..119:42): Int -> scala/Int# +Diagnostics: +[5:13..5:14): [warning] unused explicit parameter + Synthetics: [68:20..68:24):@ann => *[Int] From d7258b4f64d722196232b29e572097fa983203e6 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Tue, 25 Jul 2023 21:40:18 +0900 Subject: [PATCH 12/25] Remove Context from relPath and so on --- .../dotc/semanticdb/ExtractSemanticDB.scala | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 0b7496bcee1b..8796e2abe1b9 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -64,19 +64,43 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends override def isCheckable: Boolean = false override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = { + val sourceRoot = ctx.settings.sourceroot.value val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics if (appendDiagnostics) val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) - units.asJava.parallelStream().forEach { unit => - warnings.get(unit.source).foreach { ws => - ExtractSemanticDB.appendDiagnostics(unit.source, ws.map(_.toSemanticDiagnostic)) + units.map { unit => + val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports + val outputDir = + ExtractSemanticDB.semanticdbPath( + unit.source, + ExtractSemanticDB.outputDirectory(using unitCtx), + sourceRoot + ) + val source = unit.source + (outputDir, source) + }.asJava.parallelStream().forEach { case (out, source) => + warnings.get(source).foreach { ws => + ExtractSemanticDB.appendDiagnostics(source, ws.map(_.toSemanticDiagnostic), out) } } else units.foreach { unit => + val outputDir = + ExtractSemanticDB.semanticdbPath( + unit.source, + ExtractSemanticDB.outputDirectory(using ctx.fresh.setCompilationUnit(unit).withRootImports), + sourceRoot + ) val extractor = ExtractSemanticDB.Extractor() extractor.extract(unit.tpdTree) - ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList) + ExtractSemanticDB.write( + unit.source, + extractor.occurrences.toList, + extractor.symbolInfos.toList, + extractor.synthetics.toList, + outputDir, + sourceRoot + ) } units } @@ -105,7 +129,8 @@ object ExtractSemanticDB: .filterNot(_.isEmpty) .map(Paths.get(_)) - private def outputDirectory(using Context): AbstractFile = ctx.settings.outputDir.value + private def outputDirectory(using Context): Path = + semanticdbTarget.getOrElse(ctx.settings.outputDir.value.jpath) private def absolutePath(path: Path): Path = path.toAbsolutePath.normalize @@ -114,13 +139,14 @@ object ExtractSemanticDB: occurrences: List[SymbolOccurrence], symbolInfos: List[SymbolInformation], synthetics: List[Synthetic], - )(using Context): Unit = - val outpath = semanticdbPath(source) + outpath: Path, + sourceRoot: String + ): Unit = Files.createDirectories(outpath.getParent()) val doc: TextDocument = TextDocument( schema = Schema.SEMANTICDB4, language = Language.SCALA, - uri = Tools.mkURIstring(Paths.get(relPath(source))), + uri = Tools.mkURIstring(Paths.get(relPath(source, sourceRoot))), text = "", md5 = internal.MD5.compute(String(source.content)), symbols = symbolInfos, @@ -139,15 +165,15 @@ object ExtractSemanticDB: private def appendDiagnostics( source: SourceFile, - diagnostics: Seq[Diagnostic] - )(using Context): Unit = - val path = semanticdbPath(source) + diagnostics: Seq[Diagnostic], + outpath: Path + ): Unit = Using.Manager { use => - val in = use(Files.newInputStream(path)) + val in = use(Files.newInputStream(outpath)) val sin = internal.SemanticdbInputStream.newInstance(in) val docs = TextDocuments.parseFrom(sin) - val out = use(Files.newOutputStream(path)) + val out = use(Files.newOutputStream(outpath)) val sout = internal.SemanticdbOutputStream.newInstance(out) TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout) sout.flush() @@ -156,14 +182,14 @@ object ExtractSemanticDB: case Success(_) => // success to update semanticdb, say nothing end appendDiagnostics - private def relPath(source: SourceFile)(using ctx: Context) = - SourceFile.relativePath(source, ctx.settings.sourceroot.value) + private def relPath(source: SourceFile, sourceRoot: String) = + SourceFile.relativePath(source, sourceRoot) - private def semanticdbPath(source: SourceFile)(using Context) = - absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath)) + private def semanticdbPath(source: SourceFile, base: Path, sourceRoot: String): Path = + absolutePath(base) .resolve("META-INF") .resolve("semanticdb") - .resolve(relPath(source)) + .resolve(relPath(source, sourceRoot)) .resolveSibling(source.name + ".semanticdb") /** Extractor of symbol occurrences from trees */ From 4b5d3e760189c6280dda027c4b84003fb8ba9cd3 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Tue, 25 Jul 2023 22:13:02 +0900 Subject: [PATCH 13/25] Remove from parallel --- .../dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 8796e2abe1b9..467fd9b58071 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -67,7 +67,10 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends val sourceRoot = ctx.settings.sourceroot.value val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics if (appendDiagnostics) - val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) + val warnings = ctx.reporter.allWarnings + .groupBy(w => w.pos.source) + .map((k, ws) => (k, ws.map(_.toSemanticDiagnostic))) + .toMap units.map { unit => val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports val outputDir = @@ -80,7 +83,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends (outputDir, source) }.asJava.parallelStream().forEach { case (out, source) => warnings.get(source).foreach { ws => - ExtractSemanticDB.appendDiagnostics(source, ws.map(_.toSemanticDiagnostic), out) + ExtractSemanticDB.appendDiagnostics(source, ws, out) } } else From 3008ed8a640a0ed6e6c3f62f847383e240d31b11 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Tue, 25 Jul 2023 22:13:37 +0900 Subject: [PATCH 14/25] Remove ancii color code from SemanticDB --- compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala | 4 +++- tests/semanticdb/metac.expect | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala index 53beeffe541e..792b576234b0 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala @@ -6,6 +6,7 @@ import dotty.tools.dotc.interfaces.Diagnostic.{ERROR, INFO, WARNING} import dotty.tools.dotc.core.Contexts.Context object DiagnosticOps: + private val asciiColorCodes = "\u001B\\[[;\\d]*m".r extension (d: Diagnostic) def toSemanticDiagnostic(using Context): s.Diagnostic = val severity = d.level match @@ -13,8 +14,9 @@ object DiagnosticOps: case WARNING => s.Diagnostic.Severity.WARNING case INFO => s.Diagnostic.Severity.INFORMATION case _ => s.Diagnostic.Severity.INFORMATION + val msg = asciiColorCodes.replaceAllIn(d.msg.message, m => "") s.Diagnostic( range = Scala3.range(d.pos.span, d.pos.source), severity = severity, - message = d.msg.message + message = msg ) diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 8b95a736ba44..a8306eaed915 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -757,7 +757,7 @@ Occurrences: [2:13..2:29): deprecatedMethod -> _empty_/Deprecated.deprecatedMethod(). Diagnostics: -[2:13..2:29): [warning] method deprecatedMethod in object Deprecated is deprecated +[2:13..2:29): [warning] method deprecatedMethod in object Deprecated is deprecated expect/Empty.scala ------------------ From c9de8e2b846bb81644cea1c5ae8430435f57a83f Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Wed, 26 Jul 2023 15:41:32 +0900 Subject: [PATCH 15/25] Extractor.extract uses unitCtx --- .../src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 467fd9b58071..a6d08444c1f3 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -88,14 +88,15 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends } else units.foreach { unit => + val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports val outputDir = ExtractSemanticDB.semanticdbPath( unit.source, - ExtractSemanticDB.outputDirectory(using ctx.fresh.setCompilationUnit(unit).withRootImports), + ExtractSemanticDB.outputDirectory(using unitCtx), sourceRoot ) val extractor = ExtractSemanticDB.Extractor() - extractor.extract(unit.tpdTree) + extractor.extract(unit.tpdTree)(using unitCtx) ExtractSemanticDB.write( unit.source, extractor.occurrences.toList, From 40a2a004e60bb45ae4dcac370b3d22360bd981bf Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Wed, 26 Jul 2023 15:46:57 +0900 Subject: [PATCH 16/25] Remove Context parameter from Scala3.range and toSemanticDbDiagnositcs --- compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala | 2 +- compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala index 792b576234b0..975ea4ec8537 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala @@ -8,7 +8,7 @@ import dotty.tools.dotc.core.Contexts.Context object DiagnosticOps: private val asciiColorCodes = "\u001B\\[[;\\d]*m".r extension (d: Diagnostic) - def toSemanticDiagnostic(using Context): s.Diagnostic = + def toSemanticDiagnostic: s.Diagnostic = val severity = d.level match case ERROR => s.Diagnostic.Severity.ERROR case WARNING => s.Diagnostic.Severity.WARNING diff --git a/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala b/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala index cd14820c6a1d..fd91d17b65df 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala @@ -29,7 +29,7 @@ object Scala3: private val WILDCARDTypeName = nme.WILDCARD.toTypeName - def range(span: Span, treeSource: SourceFile)(using Context): Option[Range] = + def range(span: Span, treeSource: SourceFile): Option[Range] = def lineCol(offset: Int) = (treeSource.offsetToLine(offset), treeSource.column(offset)) val (startLine, startCol) = lineCol(span.start) val (endLine, endCol) = lineCol(span.end) From 7b29f4cae65aa42bfe16634b34f45c4bbe9136e0 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 27 Jul 2023 15:30:22 +0900 Subject: [PATCH 17/25] Remove unused --- .../src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index a6d08444c1f3..e3c054de8b91 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -27,11 +27,8 @@ import typer.ImportInfo.withRootImports import dotty.tools.dotc.{semanticdb => s} import dotty.tools.io.{AbstractFile, JarArchive} -import dotty.tools.dotc.util.Property import dotty.tools.dotc.semanticdb.DiagnosticOps.* import scala.util.{Using, Failure, Success} -import java.io.ByteArrayOutputStream -import java.io.BufferedOutputStream /** Extract symbol references and uses to semanticdb files. @@ -83,7 +80,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends (outputDir, source) }.asJava.parallelStream().forEach { case (out, source) => warnings.get(source).foreach { ws => - ExtractSemanticDB.appendDiagnostics(source, ws, out) + ExtractSemanticDB.appendDiagnostics(ws, out) } } else @@ -168,7 +165,6 @@ object ExtractSemanticDB: end write private def appendDiagnostics( - source: SourceFile, diagnostics: Seq[Diagnostic], outpath: Path ): Unit = From 4f6a09263dd2081e2d5c13330308805c8060f088 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 27 Jul 2023 16:12:34 +0900 Subject: [PATCH 18/25] Revert "Remove from parallel" This reverts commit 4b5d3e760189c6280dda027c4b84003fb8ba9cd3. --- .../dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index e3c054de8b91..34ecbed31504 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -64,10 +64,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends val sourceRoot = ctx.settings.sourceroot.value val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics if (appendDiagnostics) - val warnings = ctx.reporter.allWarnings - .groupBy(w => w.pos.source) - .map((k, ws) => (k, ws.map(_.toSemanticDiagnostic))) - .toMap + val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) units.map { unit => val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports val outputDir = @@ -80,7 +77,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends (outputDir, source) }.asJava.parallelStream().forEach { case (out, source) => warnings.get(source).foreach { ws => - ExtractSemanticDB.appendDiagnostics(ws, out) + ExtractSemanticDB.appendDiagnostics(ws.map(_.toSemanticDiagnostic), out) } } else From 499c347a7b57e4422d47a7d146240980fd87c538 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 21 Sep 2023 14:26:17 +0200 Subject: [PATCH 19/25] Remove println --- .../test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala index 6d2eec8363ad..914370147b0d 100644 --- a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala +++ b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala @@ -71,7 +71,6 @@ class SemanticdbTests: def collectErrorOrUpdate(expectPath: Path, obtained: String) = if updateExpectFiles then Files.write(expectPath, obtained.getBytes(StandardCharsets.UTF_8)) - println("updated: " + expectPath) else val expected = new String(Files.readAllBytes(expectPath), StandardCharsets.UTF_8) val expectName = expectPath.getFileName @@ -87,12 +86,8 @@ class SemanticdbTests: .resolve("semanticdb") .resolve(relpath) .resolveSibling(filename + ".semanticdb") - println(semanticdbPath) val expectPath = source.resolveSibling(filename.replace(".scala", ".expect.scala")) val doc = Tools.loadTextDocument(source, relpath, semanticdbPath) - println(semanticdbPath.getFileName().toString()) - if (semanticdbPath.getFileName().toString() == "ValPattern.scala.semanticdb") - println(doc) Tools.metac(doc, rootSrc.relativize(source))(using metacSb) val obtained = trimTrailingWhitespace(SemanticdbTests.printTextDocument(doc)) collectErrorOrUpdate(expectPath, obtained) From 71a5cd0712777b9dd3930d915d5ed18a68eef92b Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Fri, 22 Sep 2023 14:27:44 +0200 Subject: [PATCH 20/25] Fix TastyBootstrapTests by adding sharable annotation to regex --- compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala index 975ea4ec8537..4bc6e1ecb026 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala @@ -4,9 +4,10 @@ import dotty.tools.dotc.reporting.Diagnostic import dotty.tools.dotc.{semanticdb => s} import dotty.tools.dotc.interfaces.Diagnostic.{ERROR, INFO, WARNING} import dotty.tools.dotc.core.Contexts.Context +import scala.annotation.internal.sharable object DiagnosticOps: - private val asciiColorCodes = "\u001B\\[[;\\d]*m".r + @sharable private val asciiColorCodes = "\u001B\\[[;\\d]*m".r extension (d: Diagnostic) def toSemanticDiagnostic: s.Diagnostic = val severity = d.level match From d54d8a26013fc2750f155592f5c0b935f0c19174 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Mon, 16 Oct 2023 21:27:46 +0900 Subject: [PATCH 21/25] Do not run toSemanticDiagnosic in parallel Message#msg involves a lot of mutability we can't avoid thread unsafty in toSemanticDiagnostic --- .../dotc/semanticdb/ExtractSemanticDB.scala | 25 +++++++++---------- tests/semanticdb/metac.expect | 6 ++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 2d76c494803e..e7a5e8b707c3 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -65,20 +65,19 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics if (appendDiagnostics) val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source) - units.map { unit => - val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports - val outputDir = - ExtractSemanticDB.semanticdbPath( - unit.source, - ExtractSemanticDB.outputDirectory(using unitCtx), - sourceRoot - ) - val source = unit.source - (outputDir, source) - }.asJava.parallelStream().forEach { case (out, source) => - warnings.get(source).foreach { ws => - ExtractSemanticDB.appendDiagnostics(ws.map(_.toSemanticDiagnostic), out) + units.flatMap { unit => + warnings.get(unit.source).map { ws => + val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports + val outputDir = + ExtractSemanticDB.semanticdbPath( + unit.source, + ExtractSemanticDB.outputDirectory(using unitCtx), + sourceRoot + ) + (outputDir, ws.map(_.toSemanticDiagnostic)) } + }.asJava.parallelStream().forEach { case (out, warnings) => + ExtractSemanticDB.appendDiagnostics(warnings, out) } else val writeSemanticdbText = ctx.settings.semanticdbText.value diff --git a/tests/semanticdb/metac.expect b/tests/semanticdb/metac.expect index 1305e84c23db..e05a645c0141 100644 --- a/tests/semanticdb/metac.expect +++ b/tests/semanticdb/metac.expect @@ -3887,9 +3887,9 @@ Occurrences: [40:10..40:18): rightVar -> local4 Diagnostics: -[30:11..30:18): [warning] unset local variable -[30:20..30:28): [warning] unset local variable -[31:15..31:25): [warning] unset local variable +[30:11..30:18): [warning] unset local variable, consider using an immutable val instead +[30:20..30:28): [warning] unset local variable, consider using an immutable val instead +[31:15..31:25): [warning] unset local variable, consider using an immutable val instead Synthetics: [5:6..5:10):Some => *.unapply[Int] From b8565a0151c90f48921b541702dd525ba07a9a78 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Mon, 16 Oct 2023 22:34:46 +0900 Subject: [PATCH 22/25] Fix wrong rebase --- .../src/dotty/tools/dotc/reporting/Reporter.scala | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala index 25315d9ce428..3be1a159c55c 100644 --- a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala +++ b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala @@ -154,14 +154,10 @@ abstract class Reporter extends interfaces.ReporterResult { val key = w.enablingOption.name addUnreported(key, 1) case _ => - // conditional warnings that are not enabled are not fatal - val d = dia match - case w: Warning if ctx.settings.XfatalWarnings.value => w.toError - case _ => dia - if !isHidden(d) then // avoid isHidden test for summarized warnings so that message is not forced - markReported(d) - withMode(Mode.Printing)(doReport(d)) - d match { + if !isHidden(dia) then // avoid isHidden test for summarized warnings so that message is not forced + markReported(dia) + withMode(Mode.Printing)(doReport(dia)) + dia match { case w: Warning => warnings = w :: warnings _warningCount += 1 From 053e644aaa20f77f3b4c3304615246681d49a3b1 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 19 Oct 2023 14:45:28 +0900 Subject: [PATCH 23/25] Restore removed comment --- compiler/src/dotty/tools/dotc/Compiler.scala | 4 ++-- .../test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index b73b447ca282..35a716e1788d 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -39,7 +39,7 @@ class Compiler { List(new CheckShadowing) :: // Check shadowing elements List(new YCheckPositions) :: // YCheck positions List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks - List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info and attach to the tree of the unit file + List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks @@ -72,7 +72,7 @@ class Compiler { new ExpandSAMs, // Expand single abstract method closures to anonymous classes new ElimRepeated, // Rewrite vararg parameters and arguments new RefChecks) :: // Various checks mostly related to abstract members and overriding - List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file + // List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file List(new init.Checker) :: // Check initialization of objects List(new ProtectedAccessors, // Add accessors for protected members new ExtensionMethods, // Expand methods of value classes with extension methods diff --git a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala index 914370147b0d..de8c2c11f9c2 100644 --- a/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala +++ b/compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala @@ -71,6 +71,7 @@ class SemanticdbTests: def collectErrorOrUpdate(expectPath: Path, obtained: String) = if updateExpectFiles then Files.write(expectPath, obtained.getBytes(StandardCharsets.UTF_8)) + println("updated: " + expectPath) else val expected = new String(Files.readAllBytes(expectPath), StandardCharsets.UTF_8) val expectName = expectPath.getFileName From 567d4863343e23250038e9432af2d391419cb6f4 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 19 Oct 2023 16:05:48 +0900 Subject: [PATCH 24/25] Fix writesToOutputDir outputDirectory.isInstanceOf[JarArchive] was always false because `outputDirectory`'s type is java.nio.file.Path which never be an insntace of `JarArchive`. --- .../tools/dotc/semanticdb/ExtractSemanticDB.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index e7a5e8b707c3..f1b4e4637eb8 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -71,7 +71,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends val outputDir = ExtractSemanticDB.semanticdbPath( unit.source, - ExtractSemanticDB.outputDirectory(using unitCtx), + ExtractSemanticDB.semanticdbOutDir(using unitCtx), sourceRoot ) (outputDir, ws.map(_.toSemanticDiagnostic)) @@ -86,7 +86,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends val outputDir = ExtractSemanticDB.semanticdbPath( unit.source, - ExtractSemanticDB.outputDirectory(using unitCtx), + ExtractSemanticDB.semanticdbOutDir(using unitCtx), sourceRoot ) val extractor = ExtractSemanticDB.Extractor() @@ -128,8 +128,13 @@ object ExtractSemanticDB: .filterNot(_.isEmpty) .map(Paths.get(_)) - private def outputDirectory(using Context): Path = - semanticdbTarget.getOrElse(ctx.settings.outputDir.value.jpath) + /** Destination for generated classfiles */ + private def outputDirectory(using Context): AbstractFile = + ctx.settings.outputDir.value + + /** Output directory for SemanticDB files */ + private def semanticdbOutDir(using Context): Path = + semanticdbTarget.getOrElse(outputDirectory.jpath) private def absolutePath(path: Path): Path = path.toAbsolutePath.normalize From 275e6fabe49cca45d9d7be29e2fe183a96a44fe7 Mon Sep 17 00:00:00 2001 From: Rikito Taniguchi Date: Thu, 19 Oct 2023 18:38:54 +0900 Subject: [PATCH 25/25] Uncomment the phase for appending warnings --- compiler/src/dotty/tools/dotc/Compiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index 35a716e1788d..9cf15a83758e 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -72,7 +72,7 @@ class Compiler { new ExpandSAMs, // Expand single abstract method closures to anonymous classes new ElimRepeated, // Rewrite vararg parameters and arguments new RefChecks) :: // Various checks mostly related to abstract members and overriding - // List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file + List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file List(new init.Checker) :: // Check initialization of objects List(new ProtectedAccessors, // Add accessors for protected members new ExtensionMethods, // Expand methods of value classes with extension methods