From 023d66f21cb01436513ae5addbc7010f07000530 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 01:35:26 +0100 Subject: [PATCH 01/39] first dirty prototype of SBT resolution caching --- sbt/sbt-api/resources/META-INF/sbt-api.xml | 1 + .../sbt/project/SbtStructureCache.scala | 38 +++++ .../sbt/project/SbtProjectResolver.scala | 145 +++++++++++++----- 3 files changed, 147 insertions(+), 37 deletions(-) create mode 100644 sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala diff --git a/sbt/sbt-api/resources/META-INF/sbt-api.xml b/sbt/sbt-api/resources/META-INF/sbt-api.xml index f27bb6194ce..6baa70db89e 100644 --- a/sbt/sbt-api/resources/META-INF/sbt-api.xml +++ b/sbt/sbt-api/resources/META-INF/sbt-api.xml @@ -1,6 +1,7 @@ + diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala new file mode 100644 index 00000000000..bb25992cc49 --- /dev/null +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -0,0 +1,38 @@ +package org.jetbrains.sbt.project + +import com.intellij.openapi.extensions.ExtensionPointName +import org.apache.commons.io.FileUtils +import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash + +import java.io.File +import java.nio.charset.StandardCharsets + +trait SbtStructureCache { + def getCachedStructureXml(key: BuildFilesHash): Option[String] + + def cacheStructureXml(key: BuildFilesHash, data: String): Unit +} +object SbtStructureCache { + def get(): Option[SbtStructureCache] = + Some(new LocalStructureCache(new File("/Users/ghik/sbtcache"))) + + private val EpName: ExtensionPointName[SbtStructureCache] = + ExtensionPointName.create("org.intellij.sbt.structureCache") + + final case class BuildFilesHash(hash: String) +} + +class LocalStructureCache(cacheDir: File) extends SbtStructureCache { + cacheDir.mkdirs() + + private def file(key: BuildFilesHash): File = + new File(cacheDir, s"${key.hash}.xml") + + def getCachedStructureXml(key: BuildFilesHash): Option[String] = + Option(file(key)) + .filter(f => f.exists() && f.isFile) + .map(f => FileUtils.readFileToString(f, StandardCharsets.UTF_8)) + + def cacheStructureXml(key: BuildFilesHash, data: String): Unit = + FileUtils.write(file(key), data, StandardCharsets.UTF_8) +} diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index f6a8bd25cd6..02de58766d8 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -1,5 +1,6 @@ package org.jetbrains.sbt.project +import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables import com.intellij.notification.NotificationType import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.externalSystem.model.project.{ProjectData => ESProjectData, _} @@ -10,8 +11,11 @@ import com.intellij.openapi.externalSystem.service.project.ExternalSystemProject import com.intellij.openapi.module.StdModuleTypes import com.intellij.openapi.project.{Project, ProjectManager} import com.intellij.openapi.roots.DependencyScope +import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.registry.RegistryManager +import com.intellij.util.SystemProperties +import com.intellij.util.io.DigestUtil import org.jetbrains.annotations.{NonNls, Nullable, TestOnly} import org.jetbrains.plugins.scala._ import org.jetbrains.plugins.scala.build._ @@ -23,6 +27,7 @@ import org.jetbrains.plugins.scala.project.external.{AndroidJdk, JdkByHome, JdkB import org.jetbrains.plugins.scala.util.ScalaNotificationGroups import org.jetbrains.sbt.SbtUtil._ import org.jetbrains.sbt.project.SbtProjectResolver._ +import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash import org.jetbrains.sbt.project.data._ import org.jetbrains.sbt.project.module.SbtModuleType import org.jetbrains.sbt.project.settings._ @@ -135,15 +140,80 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti conversionResult.get // ok to throw here, that's the way ExternalSystem likes it } + private def buildFilesHash(projectRoot: File): SbtStructureCache.BuildFilesHash = { + val digest = DigestUtil.sha1() + projectRoot.listFiles((_, f) => f.endsWith(".sbt")).foreach { f => + DigestUtil.updateContentHash(digest, f.toPath) + } + + def hashDir(dir: File): Unit = + if (dir.exists() && dir.isDirectory) { + dir.listFiles().foreach { f => + if (f.isDirectory && f.getName != "target") { + hashDir(f) + } else if(f.isFile) { + DigestUtil.updateContentHash(digest, f.toPath) + } + } + } + + hashDir(new File(projectRoot, "project")) + BuildFilesHash(DigestUtil.digestToHash(digest)) + } + + private def getDefaultCoursierCachePath(): String = + if (SystemInfo.isWindows) + WindowsEnvVariables.INSTANCE.getLocalApplicationData + "\\Coursier\\Cache\\v1" + else if (SystemInfo.isMac) + SystemProperties.getUserHome + "/Library/Caches/Coursier/v1" + else + SystemProperties.getUserHome + "/.cache/coursier/v1" + private def dumpStructure(projectRoot: File, sbtLauncher: File, sbtVersion: Version, - settings:SbtExecutionSettings, + settings: SbtExecutionSettings, @Nullable project: Project )(implicit reporter: BuildReporter): Try[(Elem, BuildMessages)] = { val useShellImport = settings.useShellForImport && shellImportSupported(sbtVersion) && project != null val options = dumpOptions(settings) + def doDumpStructureCached(structureFile: File): Try[(Elem, BuildMessages)] = + SbtStructureCache.get() match { + case None => doDumpStructure(structureFile) + case Some(cache) => + val coursierCachePath = getDefaultCoursierCachePath() + val userHome = SystemProperties.getUserHome + val projectDir = projectRoot.getAbsolutePath + val key = buildFilesHash(projectRoot) + + cache.getCachedStructureXml(key) match { + case Some(cachedXml) => + val localizedXml = cachedXml + .replace("$COURSIER_CACHE$", coursierCachePath) + .replace("$PROJECT_DIR$", projectDir) + .replace("$USER_HOME$", userHome) + + val parsedXml = Try(XML.loadString(localizedXml)) + parsedXml.map(e => (e, BuildMessages.empty)) + + case None => + val result = doDumpStructure(structureFile) + + result.foreach { + case (elem, _) => + val unlocalizedXml = elem.toString + .replace(coursierCachePath, "$COURSIER_CACHE$") + .replace(projectDir, "$PROJECT_DIR$") + .replace(userHome, "$USER_HOME$") + + cache.cacheStructureXml(key, unlocalizedXml) + } + + result + } + } + def doDumpStructure(structureFile: File): Try[(Elem, BuildMessages)] = { val structureFilePath = normalizePath(structureFile) @@ -226,10 +296,10 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti Try((elem, BuildMessages.empty)) } else if (writeStructureFile) { log.warn(s"reused structure file created: $structureFilePath") - doDumpStructure(structureFilePath) + doDumpStructureCached(structureFilePath) } else { usingTempFile("sbt-structure", Some(".xml")) { structureFile => - doDumpStructure(structureFile) + doDumpStructureCached(structureFile) } } } @@ -260,16 +330,16 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti ) private def dumpOptions(settings: SbtExecutionSettings): Seq[String] = { - Seq("download") ++ + Seq("download") ++ settings.resolveClassifiers.seq("resolveClassifiers") ++ settings.resolveJavadocs.seq("resolveJavadocs") ++ settings.resolveSbtClassifiers.seq("resolveSbtClassifiers") } /** - * Create project preview without using sbt, since sbt import can fail and users would have to do a manual edit of the project. - * Also sbt boot makes the whole process way too slow. - */ + * Create project preview without using sbt, since sbt import can fail and users would have to do a manual edit of the project. + * Also sbt boot makes the whole process way too slow. + */ private def dummyProject(projectRoot: File, settings: SbtExecutionSettings, sbtVersion: String): Node[ESProjectData] = { // TODO add default scala sdk and sbt libs (newest versions or so) @@ -308,10 +378,10 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } /** - * This implementation is the same as in sbt.Project.normalizeModuleId to avoid inconsistencies in the import process. - * Normalize a String so that it is suitable for use as a dependency management module identifier. - * This is a best effort implementation, since valid characters are not documented or consistent. - */ + * This implementation is the same as in sbt.Project.normalizeModuleId to avoid inconsistencies in the import process. + * Normalize a String so that it is suitable for use as a dependency management module identifier. + * This is a best effort implementation, since valid characters are not documented or consistent. + */ private def normalizeModuleId(s: String) = s.toLowerCase(Locale.ENGLISH) .replaceAll("""\W+""", "-") @@ -361,9 +431,9 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } /** Choose a project jdk based on information from sbt settings and IDE. - * More specific settings from sbt are preferred over IDE settings, on the assumption that the sbt project definition - * is what is more likely to be under source control. - */ + * More specific settings from sbt are preferred over IDE settings, on the assumption that the sbt project definition + * is what is more likely to be under source control. + */ private def chooseJdk(project: sbtStructure.ProjectData, defaultJdk: Option[String]): Option[SdkReference] = { // TODO put some of this logic elsewhere in resolving process? val androidSdk = project.android.map(android => AndroidJdk(android.targetVersion)) @@ -377,7 +447,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti .orElse(default) } - private def createModuleDependencies(projectToModule: Map[ProjectData,ModuleNode]): Unit = { + private def createModuleDependencies(projectToModule: Map[ProjectData, ModuleNode]): Unit = { projectToModule.foreach { case (moduleProject, moduleNode) => moduleProject.dependencies.projects.foreach { dependencyId => val dependency = @@ -392,7 +462,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } } - private def createModules(projects: Seq[sbtStructure.ProjectData], libraryNodes: Seq[LibraryNode], moduleFilesDirectory: File): Map[ProjectData,ModuleNode] = { + private def createModules(projects: Seq[sbtStructure.ProjectData], libraryNodes: Seq[LibraryNode], moduleFilesDirectory: File): Map[ProjectData, ModuleNode] = { val unmanagedSourcesAndDocsLibrary = libraryNodes.map(_.data).find(_.getExternalName == Sbt.UnmanagedSourcesAndDocsName) val nameToProjects = projects.groupBy(_.name) @@ -428,7 +498,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti dependency.setScope(DependencyScope.COMPILE) moduleNode.add(dependency) } - (project,moduleNode) + (project, moduleNode) } val projectToModuleMap = projectToModule.toMap @@ -441,7 +511,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti val repositoryModules = data.repository.map(_.modules).getOrElse(Seq.empty) val (modulesWithoutBinaries, modulesWithBinaries) = repositoryModules.partition(_.binaries.isEmpty) val otherModuleIds = projects.flatMap(_.dependencies.modules.map(_.id)).toSet -- - repositoryModules.map(_.id).toSet + repositoryModules.map(_.id).toSet val libs = modulesWithBinaries.map(createResolvedLibrary) ++ otherModuleIds.map(createUnresolvedLibrary) @@ -461,15 +531,15 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti .orElse(java.flatMap(_.home).map(JdkByHome)) val data = SbtModuleExtData( - scalaVersion = scala.map(_.version), - scalacClasspath = scala.fold(Seq.empty[File])(_.allCompilerJars), + scalaVersion = scala.map(_.version), + scalacClasspath = scala.fold(Seq.empty[File])(_.allCompilerJars), scaladocExtraClasspath = scala.fold(Seq.empty[File])(_.extraJars), - scalacOptions = scala.fold(Seq.empty[String])(_.options), - sdk = sdk, - javacOptions = java.fold(Seq.empty[String])(_.options), - packagePrefix = packagePrefix, - basePackage = basePackages.headOption, // TODO Rename basePackages to basePackage in sbt-ide-settings? - compileOrder = CompileOrder.valueOf(compileOrder) + scalacOptions = scala.fold(Seq.empty[String])(_.options), + sdk = sdk, + javacOptions = java.fold(Seq.empty[String])(_.options), + packagePrefix = packagePrefix, + basePackage = basePackages.headOption, // TODO Rename basePackages to basePackage in sbt-ide-settings? + compileOrder = CompileOrder.valueOf(compileOrder) ) new ModuleExtNode(data) } @@ -488,7 +558,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } private def createCommandData(project: sbtStructure.ProjectData) = { - project.commands.map { c => + project.commands.map { c => new SbtCommandNode(SbtCommandData(c.name, c.help)) } } @@ -588,13 +658,14 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti return extractedExcludes.distinct val managedDirectories = project.configurations - .flatMap(configuration => configuration.sources ++ configuration.resources) - .filter(_.managed) - .map(_.file) + .flatMap(configuration => configuration.sources ++ configuration.resources) + .filter(_.managed) + .map(_.file) val defaultNames = Set("main", "test") val relevantDirectories = managedDirectories.filter(file => file.exists || !defaultNames.contains(file.getName)) + def isRelevant(f: File): Boolean = !relevantDirectories.forall(_.isOutsideOf(f)) if (isRelevant(project.target)) { @@ -618,7 +689,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } val buildId = buildBaseProject - .map(_.name + Sbt.BuildModuleSuffix) + .map(_.name + Sbt.BuildModuleSuffix) .getOrElse(build.uri.toString) val buildBaseDir = buildBaseProject @@ -691,10 +762,10 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti private def validRootPathsIn(project: sbtStructure.ProjectData, scope: String) (selector: sbtStructure.ConfigurationData => Seq[sbtStructure.DirectoryData]): Seq[sbtStructure.DirectoryData] = { project.configurations - .find(_.id == scope) - .map(selector) - .getOrElse(Seq.empty) - .filterNot(_.file.isOutsideOf(project.base)) + .find(_.id == scope) + .map(selector) + .getOrElse(Seq.empty) + .filterNot(_.file.isOutsideOf(project.base)) } protected def createLibraryDependencies(dependencies: Seq[sbtStructure.ModuleDependencyData]) @@ -711,7 +782,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } private def createUnmanagedDependencies(dependencies: Seq[sbtStructure.JarDependencyData]) - (moduleData: ModuleData): Seq[LibraryDependencyNode] = { + (moduleData: ModuleData): Seq[LibraryDependencyNode] = { dependencies.groupBy(it => scopeFor(it.configurations)).toSeq.map { case (scope, dependency) => val name = scope match { case DependencyScope.COMPILE => Sbt.UnmanagedLibraryName @@ -759,7 +830,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } override def cancelTask(taskId: ExternalSystemTaskId, listener: ExternalSystemTaskNotificationListener): Boolean = - //noinspection UnitInMap + //noinspection UnitInMap activeProcessDumper .map(_.cancel()) .isDefined From b4d2e7ff2ffeae9c12d4382cc53ae836d569ed43 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 03:14:17 +0100 Subject: [PATCH 02/39] less hacky SBT structure portability --- .../sbt/project/SbtProjectResolver.scala | 45 ++---- .../sbt/project/StructureLocalizer.scala | 150 ++++++++++++++++++ 2 files changed, 163 insertions(+), 32 deletions(-) create mode 100644 sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 02de58766d8..3f7edca1018 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -1,6 +1,5 @@ package org.jetbrains.sbt.project -import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables import com.intellij.notification.NotificationType import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.externalSystem.model.project.{ProjectData => ESProjectData, _} @@ -11,10 +10,8 @@ import com.intellij.openapi.externalSystem.service.project.ExternalSystemProject import com.intellij.openapi.module.StdModuleTypes import com.intellij.openapi.project.{Project, ProjectManager} import com.intellij.openapi.roots.DependencyScope -import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.registry.RegistryManager -import com.intellij.util.SystemProperties import com.intellij.util.io.DigestUtil import org.jetbrains.annotations.{NonNls, Nullable, TestOnly} import org.jetbrains.plugins.scala._ @@ -35,7 +32,7 @@ import org.jetbrains.sbt.project.structure.SbtStructureDump.PrintProcessOutputOn import org.jetbrains.sbt.project.structure._ import org.jetbrains.sbt.resolvers.{SbtIvyResolver, SbtMavenResolver, SbtResolver} import org.jetbrains.sbt.structure.XmlSerializer._ -import org.jetbrains.sbt.structure.{BuildData, Configuration, ConfigurationData, DependencyData, DirectoryData, JavaData, ModuleDependencyData, ModuleIdentifier, ProjectData} +import org.jetbrains.sbt.structure.{BuildData, Configuration, ConfigurationData, DependencyData, DirectoryData, JavaData, ModuleDependencyData, ModuleIdentifier, ProjectData, StructureData} import org.jetbrains.sbt.{RichBoolean, Sbt, SbtBundle, SbtUtil, usingTempFile, structure => sbtStructure} import java.io.{File, FileNotFoundException} @@ -151,7 +148,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti dir.listFiles().foreach { f => if (f.isDirectory && f.getName != "target") { hashDir(f) - } else if(f.isFile) { + } else if (f.isFile) { DigestUtil.updateContentHash(digest, f.toPath) } } @@ -161,14 +158,6 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti BuildFilesHash(DigestUtil.digestToHash(digest)) } - private def getDefaultCoursierCachePath(): String = - if (SystemInfo.isWindows) - WindowsEnvVariables.INSTANCE.getLocalApplicationData + "\\Coursier\\Cache\\v1" - else if (SystemInfo.isMac) - SystemProperties.getUserHome + "/Library/Caches/Coursier/v1" - else - SystemProperties.getUserHome + "/.cache/coursier/v1" - private def dumpStructure(projectRoot: File, sbtLauncher: File, sbtVersion: Version, @@ -182,34 +171,26 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti SbtStructureCache.get() match { case None => doDumpStructure(structureFile) case Some(cache) => - val coursierCachePath = getDefaultCoursierCachePath() - val userHome = SystemProperties.getUserHome - val projectDir = projectRoot.getAbsolutePath val key = buildFilesHash(projectRoot) + val localizer = new StructureLocalizer(projectRoot) cache.getCachedStructureXml(key) match { - case Some(cachedXml) => - val localizedXml = cachedXml - .replace("$COURSIER_CACHE$", coursierCachePath) - .replace("$PROJECT_DIR$", projectDir) - .replace("$USER_HOME$", userHome) - - val parsedXml = Try(XML.loadString(localizedXml)) - parsedXml.map(e => (e, BuildMessages.empty)) + case Some(cachedXml) => Try { + val unlocalizedStructure = XML.loadString(cachedXml).deserialize[StructureData].getRight + localizer.localize(unlocalizedStructure) + .map(localizedStructure => (localizedStructure.serialize, BuildMessages.empty)) + .getOrElse(doDumpStructure(structureFile).get) + } case None => val result = doDumpStructure(structureFile) - result.foreach { case (elem, _) => - val unlocalizedXml = elem.toString - .replace(coursierCachePath, "$COURSIER_CACHE$") - .replace(projectDir, "$PROJECT_DIR$") - .replace(userHome, "$USER_HOME$") - - cache.cacheStructureXml(key, unlocalizedXml) + val localizedData = elem.deserialize[StructureData].getRight + val unlocalizedData = localizer.unlocalize(localizedData) + val unlocalizedElem = unlocalizedData.serialize + cache.cacheStructureXml(key, unlocalizedElem.toString) } - result } } diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala new file mode 100644 index 00000000000..978c927c1d4 --- /dev/null +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -0,0 +1,150 @@ +package org.jetbrains.sbt.project + +import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables +import com.intellij.openapi.util.SystemInfo +import com.intellij.util.SystemProperties +import org.jetbrains.sbt.structure._ + +import java.io.File +import java.net.URI +import scala.util.control.NoStackTrace + +object StructureLocalizer { + private final val CoursierCache = "/$COURSIER_CACHE$" + private final val ProjectDir = "/$PROJECT_DIR$" + private final val UserHome = "/$USER_HOME$" + + private lazy val userHome: String = + SystemProperties.getUserHome + + private lazy val coursierCachePath: String = + if (SystemInfo.isWindows) + WindowsEnvVariables.INSTANCE.getLocalApplicationData + "\\Coursier\\Cache\\v1" + else if (SystemInfo.isMac) + userHome + "/Library/Caches/Coursier/v1" + else + userHome + "/.cache/coursier/v1" + + private object LocalJarNotFound extends Exception with NoStackTrace + +} +class StructureLocalizer(projectDir: File) { + + import StructureLocalizer._ + + private val projectPath = projectDir.getAbsolutePath + + def localize(sd: StructureData): Option[StructureData] = + try Option(transform(sd)(localize)) catch { + case LocalJarNotFound => None + } + + def unlocalize(sd: StructureData): StructureData = + transform(sd)(unlocalize) + + private def transform(pd: ProjectData)(f: File => File): ProjectData = + ProjectData( + pd.id, + transform(pd.buildURI)(f), + pd.name, + pd.organization, + pd.version, + f(pd.base), + pd.packagePrefix, + pd.basePackages, + f(pd.target), + pd.configurations.map(cd => ConfigurationData( + cd.id, + cd.sources.map(dd => DirectoryData(f(dd.file), dd.managed)), + cd.resources.map(dd => DirectoryData(f(dd.file), dd.managed)), + cd.excludes.map(f), + f(cd.classes), + )), + pd.java.map(jd => JavaData(jd.home.map(f), jd.options)), + pd.scala.map(sd => ScalaData( + sd.organization, + sd.version, + sd.libraryJars.map(f), + sd.compilerJars.map(f), + sd.extraJars.map(f), + sd.options + )), + pd.compileOrder, + pd.android.map(ad => AndroidData( + ad.targetVersion, + ad.manifest, + f(ad.apk), + f(ad.res), + f(ad.assets), + f(ad.gen), + f(ad.libs), + ad.isLibrary, + ad.proguardConfig, + ad.apklibs.map(al => ApkLib( + al.name, f(al.base), f(al.manifest), f(al.sources), f(al.resources), f(al.libs), f(al.gen) + )), + ad.aars.map(ar => Aar(ar.name, transform(ar.project)(f))) + )), + on(pd.dependencies)(dd => DependencyData( + dd.projects.map(pdd => ProjectDependencyData( + pdd.project, pdd.buildURI.map(transform(_)(f)), pdd.configuration, + )), + dd.modules, + dd.jars.map(jdd => JarDependencyData(f(jdd.file), jdd.configurations)), + )), + pd.resolvers, + pd.play2.map(p2d => Play2Data( + p2d.playVersion, + p2d.templatesImports, + p2d.routesImports, + p2d.confDirectory.map(f), + f(p2d.sourceDirectory), + )), + pd.settings, + pd.tasks, + pd.commands, + ) + + private def on[T](v: T)(f: T => T): T = f(v) + + private def transform(data: StructureData)(f: File => File): StructureData = + StructureData( + data.sbtVersion, + builds = data.builds.map(bd => BuildData( + transform(bd.uri)(f), bd.imports, bd.classes.map(f), bd.docs.map(f), bd.sources.map(f) + )), + projects = data.projects.map(transform(_)(f)), + data.repository.map(rd => RepositoryData(rd.modules.map(md => + ModuleData(md.id, md.binaries.map(f), md.docs.map(f), md.sources.map(f)) + ))), + data.localCachePath.map(f), + ) + + private def unlocalize(f: File): File = { + var path = f.getAbsolutePath.replace('\\', '/') + path = + if (path.startsWith(coursierCachePath)) CoursierCache + path.stripPrefix(coursierCachePath) + else if (path.startsWith(projectPath)) ProjectDir + path.stripPrefix(projectPath) + else if (path.startsWith(userHome)) UserHome + path.stripPrefix(userHome) + else path + new File(path) + } + + private def localize(f: File): File = { + var path = f.getPath.replace('\\', '/') + path = + if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) + else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) + else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome) + else path + val localFile = new File(path) + if (localFile.getName.endsWith(".jar") && !localFile.exists()) { + throw LocalJarNotFound + } + localFile + } + + private def transform(uri: URI)(f: File => File): URI = + if (uri.getScheme == "file") f(new File(uri.getPath)).toURI + else uri +} From 3fbffb576a745908c2699ea92cc7605cc6d4eb2e Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 13:32:06 +0100 Subject: [PATCH 03/39] project file hashing is deterministic --- .../sbt/project/SbtProjectResolver.scala | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 3f7edca1018..0c73c75abc6 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -36,7 +36,9 @@ import org.jetbrains.sbt.structure.{BuildData, Configuration, ConfigurationData, import org.jetbrains.sbt.{RichBoolean, Sbt, SbtBundle, SbtUtil, usingTempFile, structure => sbtStructure} import java.io.{File, FileNotFoundException} +import java.nio.file.Path import java.util.{Collections, Locale, UUID} +import scala.collection.mutable.ArrayBuffer import scala.collection.{MapView, mutable} import scala.concurrent.Await import scala.concurrent.duration.Duration @@ -138,23 +140,33 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } private def buildFilesHash(projectRoot: File): SbtStructureCache.BuildFilesHash = { - val digest = DigestUtil.sha1() - projectRoot.listFiles((_, f) => f.endsWith(".sbt")).foreach { f => - DigestUtil.updateContentHash(digest, f.toPath) - } + val rootPath = projectRoot.toPath + val filesToDigest = new ArrayBuffer[Path] + + def addFile(f: File): Unit = + filesToDigest.addOne(rootPath.relativize(f.toPath)) + + projectRoot.listFiles((_, f) => f.endsWith(".sbt")).foreach(addFile) - def hashDir(dir: File): Unit = + def scanDir(dir: File): Unit = if (dir.exists() && dir.isDirectory) { dir.listFiles().foreach { f => if (f.isDirectory && f.getName != "target") { - hashDir(f) + scanDir(f) } else if (f.isFile) { - DigestUtil.updateContentHash(digest, f.toPath) + addFile(f) } } } - hashDir(new File(projectRoot, "project")) + scanDir(new File(projectRoot, "project")) + + // files must be hashed in deterministic order + filesToDigest.sortInPlace() + val digest = DigestUtil.sha1() + filesToDigest.foreach { path => + DigestUtil.updateContentHash(digest, rootPath.resolve(path)) + } BuildFilesHash(DigestUtil.digestToHash(digest)) } From 0f119c805416ffa7c68581ed51d27b8d368acd50 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 15:34:05 +0100 Subject: [PATCH 04/39] BuildFilesHash input includes filenames --- .../src/org/jetbrains/sbt/project/SbtProjectResolver.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 0c73c75abc6..020b6342c4f 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -36,6 +36,7 @@ import org.jetbrains.sbt.structure.{BuildData, Configuration, ConfigurationData, import org.jetbrains.sbt.{RichBoolean, Sbt, SbtBundle, SbtUtil, usingTempFile, structure => sbtStructure} import java.io.{File, FileNotFoundException} +import java.nio.charset.StandardCharsets import java.nio.file.Path import java.util.{Collections, Locale, UUID} import scala.collection.mutable.ArrayBuffer @@ -165,6 +166,8 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti filesToDigest.sortInPlace() val digest = DigestUtil.sha1() filesToDigest.foreach { path => + val rawFilePath = path.toFile.getPath.replace('\\', '/') + digest.update(rawFilePath.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, rootPath.resolve(path)) } BuildFilesHash(DigestUtil.digestToHash(digest)) From bdfd1fad7a698b5a2b7ae2b0e88f3a31af1e3a2c Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 15:34:21 +0100 Subject: [PATCH 05/39] simple http-based sbt structure remote caching --- .../sbt/project/SbtStructureCache.scala | 108 +++++++++++++++++- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index bb25992cc49..74f06210f2a 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -4,8 +4,17 @@ import com.intellij.openapi.extensions.ExtensionPointName import org.apache.commons.io.FileUtils import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash -import java.io.File +import java.io.{File, FileReader, IOException} +import java.net.http.HttpRequest.BodyPublishers +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest} +import java.net.{Authenticator, PasswordAuthentication, URI} import java.nio.charset.StandardCharsets +import java.time.Duration +import java.util.Properties +import scala.annotation.tailrec +import scala.concurrent.TimeoutException +import scala.util.Using trait SbtStructureCache { def getCachedStructureXml(key: BuildFilesHash): Option[String] @@ -13,8 +22,27 @@ trait SbtStructureCache { def cacheStructureXml(key: BuildFilesHash, data: String): Unit } object SbtStructureCache { - def get(): Option[SbtStructureCache] = - Some(new LocalStructureCache(new File("/Users/ghik/sbtcache"))) + object Properties { + final val LocalStructureCacheDir = "org.intellij.sbt.structure.localCacheDir" + final val RemoteStructureCacheUri = "org.intellij.sbt.structure.remoteCacheUri" + final val RemoteStructureCacheCredentialsFile = "org.intellij.sbt.structure.remoteCacheCredentialsFile" + } + + def get(): Option[SbtStructureCache] = { + val localCache = sys.props.get(Properties.LocalStructureCacheDir).map(new File(_)) + .map(new LocalStructureCache(_)) + + val remoteCache = for { + remoteCacheUri <- sys.props.get(Properties.RemoteStructureCacheUri) + .map(v => if (v.endsWith("/")) v else s"$v/") + .map(new URI(_)) + remoteCacheCredsFile <- sys.props.get(Properties.RemoteStructureCacheCredentialsFile).map(new File(_)) + } yield new RemoteStructureCache(remoteCacheUri, remoteCacheCredsFile) + + val cacheChain = localCache.toList ++ remoteCache.toList + if (cacheChain.nonEmpty) Some(new CompositeStructureCache(cacheChain)) + else None + } private val EpName: ExtensionPointName[SbtStructureCache] = ExtensionPointName.create("org.intellij.sbt.structureCache") @@ -36,3 +64,77 @@ class LocalStructureCache(cacheDir: File) extends SbtStructureCache { def cacheStructureXml(key: BuildFilesHash, data: String): Unit = FileUtils.write(file(key), data, StandardCharsets.UTF_8) } + +class RemoteStructureCache( + repositoryUri: URI, + credentialsFile: File, +) extends SbtStructureCache { + + private lazy val authenticator = new Authenticator { + val (username, password) = { + val props = new Properties + Using.resource(new FileReader(credentialsFile))(props.load) + val u = props.getProperty("username") + val p = props.getProperty("password") + require(u != null, "username not found in credentials file") + require(p != null, "password not found in credentials file") + (u, p.toCharArray) + } + + override def getPasswordAuthentication: PasswordAuthentication = + new PasswordAuthentication(username, password) + } + + private lazy val httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .authenticator(authenticator) + .build() + + private def fileRequest(key: BuildFilesHash): HttpRequest.Builder = + HttpRequest + .newBuilder(repositoryUri.resolve(s"${key.hash}.xml")) + .timeout(Duration.ofSeconds(30)) + + def getCachedStructureXml(key: BuildFilesHash): Option[String] = try { + val request = fileRequest(key).GET().build() + val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) + val resp = httpClient.send(request, bodySubscriber) + + if (resp.statusCode() == 200) + Some(resp.body()) + else + None + } catch { + case _: IOException | _: TimeoutException => None + } + + def cacheStructureXml(key: BuildFilesHash, data: String): Unit = try { + val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() + httpClient.send(request, BodyHandlers.discarding()) + } catch { + //TODO logging + case _: IOException | _: TimeoutException => + } +} + +class CompositeStructureCache(caches: List[SbtStructureCache]) extends SbtStructureCache { + def getCachedStructureXml(key: BuildFilesHash): Option[String] = { + @tailrec def loop(tried: List[SbtStructureCache], notTried: List[SbtStructureCache]): Option[String] = + notTried match { + case Nil => None + case head :: tail => + head.getCachedStructureXml(key) match { + case Some(value) => + tried.foreach(_.cacheStructureXml(key, value)) + Some(value) + case None => + loop(head :: tried, tail) + } + } + + loop(Nil, caches) + } + + def cacheStructureXml(key: BuildFilesHash, data: String): Unit = + caches.foreach(_.cacheStructureXml(key, data)) +} From a2e5a0afd2d775e17c274e11f2e4ee098add84e7 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 16:07:51 +0100 Subject: [PATCH 06/39] tmp plugin version --- pluginXml/resources/META-INF/plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index 2e7a50dd2f0..089f46d8e75 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -33,7 +33,7 @@ Support for Play Framework, Akka and Scala.js is enabled in IntelliJ IDEA Ultimate. ]]> - VERSION + 2022.3.99-sbtcaching.1 JetBrains From 398487da4fb54a95e0187e8f3168f4c8b11117fd Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 16:13:05 +0100 Subject: [PATCH 07/39] credentials file format consistent with the one used by sbt artifact publishing --- .../src/org/jetbrains/sbt/project/SbtStructureCache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index 74f06210f2a..f58eac3fc69 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -71,10 +71,10 @@ class RemoteStructureCache( ) extends SbtStructureCache { private lazy val authenticator = new Authenticator { - val (username, password) = { + val (user, password) = { val props = new Properties Using.resource(new FileReader(credentialsFile))(props.load) - val u = props.getProperty("username") + val u = props.getProperty("user") val p = props.getProperty("password") require(u != null, "username not found in credentials file") require(p != null, "password not found in credentials file") @@ -82,7 +82,7 @@ class RemoteStructureCache( } override def getPasswordAuthentication: PasswordAuthentication = - new PasswordAuthentication(username, password) + new PasswordAuthentication(user, password) } private lazy val httpClient = HttpClient.newBuilder() From 940d0c7ee140a3e3a29f4cef9ecd408e762aaac9 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 23:30:40 +0100 Subject: [PATCH 08/39] limited hashed files to .sbt, .scala and .properties files --- .../src/org/jetbrains/sbt/project/SbtProjectResolver.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 020b6342c4f..5b081e6b6bb 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -143,6 +143,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti private def buildFilesHash(projectRoot: File): SbtStructureCache.BuildFilesHash = { val rootPath = projectRoot.toPath val filesToDigest = new ArrayBuffer[Path] + val extensions = List(".sbt", ".scala", ".properties") def addFile(f: File): Unit = filesToDigest.addOne(rootPath.relativize(f.toPath)) @@ -154,7 +155,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti dir.listFiles().foreach { f => if (f.isDirectory && f.getName != "target") { scanDir(f) - } else if (f.isFile) { + } else if (f.isFile && extensions.exists(f.getName.endsWith)) { addFile(f) } } From 3c187872f751f655ec2718a7e849647a77b9978d Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Tue, 14 Mar 2023 23:31:27 +0100 Subject: [PATCH 09/39] fixed structure localization on windows --- .../jetbrains/sbt/project/StructureLocalizer.scala | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 978c927c1d4..6c01a781f70 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -14,12 +14,15 @@ object StructureLocalizer { private final val ProjectDir = "/$PROJECT_DIR$" private final val UserHome = "/$USER_HOME$" + private def unixSep(path: String): String = + path.replace('\\', '/') + private lazy val userHome: String = - SystemProperties.getUserHome + unixSep(SystemProperties.getUserHome) private lazy val coursierCachePath: String = if (SystemInfo.isWindows) - WindowsEnvVariables.INSTANCE.getLocalApplicationData + "\\Coursier\\Cache\\v1" + unixSep(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/Cache/v1" else if (SystemInfo.isMac) userHome + "/Library/Caches/Coursier/v1" else @@ -32,7 +35,8 @@ class StructureLocalizer(projectDir: File) { import StructureLocalizer._ - private val projectPath = projectDir.getAbsolutePath + private val projectPath = + unixSep(projectDir.getAbsolutePath) def localize(sd: StructureData): Option[StructureData] = try Option(transform(sd)(localize)) catch { @@ -121,7 +125,7 @@ class StructureLocalizer(projectDir: File) { ) private def unlocalize(f: File): File = { - var path = f.getAbsolutePath.replace('\\', '/') + var path = unixSep(f.getAbsolutePath) path = if (path.startsWith(coursierCachePath)) CoursierCache + path.stripPrefix(coursierCachePath) else if (path.startsWith(projectPath)) ProjectDir + path.stripPrefix(projectPath) @@ -131,7 +135,7 @@ class StructureLocalizer(projectDir: File) { } private def localize(f: File): File = { - var path = f.getPath.replace('\\', '/') + var path = unixSep(f.getPath) path = if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) From 20107a9a3e7bdd479cc6bfcb5ea80236df09e44b Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 00:10:26 +0100 Subject: [PATCH 10/39] more workarounds for windows --- pluginXml/resources/META-INF/plugin.xml | 2 +- .../sbt/project/SbtProjectResolver.scala | 4 +++- .../sbt/project/StructureLocalizer.scala | 16 ++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index 089f46d8e75..978d8f12173 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -33,7 +33,7 @@ Support for Play Framework, Akka and Scala.js is enabled in IntelliJ IDEA Ultimate. ]]> - 2022.3.99-sbtcaching.1 + 2022.3.99-sbtcaching.2 JetBrains diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 5b081e6b6bb..2a433bbd757 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -168,8 +168,10 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti val digest = DigestUtil.sha1() filesToDigest.foreach { path => val rawFilePath = path.toFile.getPath.replace('\\', '/') + val resolved = rootPath.resolve(path) + println(s"HASHING $rawFilePath ($resolved), size: ${resolved.toFile.length()}") digest.update(rawFilePath.getBytes(StandardCharsets.UTF_8)) - DigestUtil.updateContentHash(digest, rootPath.resolve(path)) + DigestUtil.updateContentHash(digest, resolved) } BuildFilesHash(DigestUtil.digestToHash(digest)) } diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 6c01a781f70..dbf2257a92f 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -14,15 +14,15 @@ object StructureLocalizer { private final val ProjectDir = "/$PROJECT_DIR$" private final val UserHome = "/$USER_HOME$" - private def unixSep(path: String): String = + private def unix(path: String): String = path.replace('\\', '/') private lazy val userHome: String = - unixSep(SystemProperties.getUserHome) + unix(SystemProperties.getUserHome) private lazy val coursierCachePath: String = if (SystemInfo.isWindows) - unixSep(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/Cache/v1" + unix(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/Cache/v1" else if (SystemInfo.isMac) userHome + "/Library/Caches/Coursier/v1" else @@ -36,7 +36,7 @@ class StructureLocalizer(projectDir: File) { import StructureLocalizer._ private val projectPath = - unixSep(projectDir.getAbsolutePath) + unix(projectDir.getAbsolutePath) def localize(sd: StructureData): Option[StructureData] = try Option(transform(sd)(localize)) catch { @@ -125,7 +125,7 @@ class StructureLocalizer(projectDir: File) { ) private def unlocalize(f: File): File = { - var path = unixSep(f.getAbsolutePath) + var path = unix(f.getAbsolutePath) path = if (path.startsWith(coursierCachePath)) CoursierCache + path.stripPrefix(coursierCachePath) else if (path.startsWith(projectPath)) ProjectDir + path.stripPrefix(projectPath) @@ -135,7 +135,11 @@ class StructureLocalizer(projectDir: File) { } private def localize(f: File): File = { - var path = unixSep(f.getPath) + // The 'C:' that we're stripping appears when we create an Unix-style absolute path File on Windows + // and call '.getAbsolutePath` on it, e.g. `new File("/foo").getAbsolutePath` yields `C:\foo` + // We can't avoid that because `.getAbsolutePath` is used by XML serializer of `StructureData` + // in `sbt-structure`. + var path = unix(f.getPath.stripPrefix("C:")) path = if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) From 18ef0367358cd938ca155e4a831ccd99bb6ee8ab Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 00:21:48 +0100 Subject: [PATCH 11/39] cosmetic --- .../src/org/jetbrains/sbt/project/SbtProjectResolver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 2a433bbd757..786703da95a 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -169,7 +169,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti filesToDigest.foreach { path => val rawFilePath = path.toFile.getPath.replace('\\', '/') val resolved = rootPath.resolve(path) - println(s"HASHING $rawFilePath ($resolved), size: ${resolved.toFile.length()}") + log.warn(s"HASHING $rawFilePath ($resolved), size: ${resolved.toFile.length()}") digest.update(rawFilePath.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, resolved) } From e3fe4df31b4de204fdbb05b3392982ca895613d3 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 00:56:12 +0100 Subject: [PATCH 12/39] logging --- .../sbt/project/SbtStructureCache.scala | 37 ++++++++++++++----- .../sbt/project/SbtProjectResolver.scala | 21 +++++++---- .../sbt/project/StructureLocalizer.scala | 5 ++- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index f58eac3fc69..12ea6e1d3b9 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -1,5 +1,6 @@ package org.jetbrains.sbt.project +import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.extensions.ExtensionPointName import org.apache.commons.io.FileUtils import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash @@ -51,7 +52,7 @@ object SbtStructureCache { } class LocalStructureCache(cacheDir: File) extends SbtStructureCache { - cacheDir.mkdirs() + private val log = Logger.getInstance(classOf[RemoteStructureCache]) private def file(key: BuildFilesHash): File = new File(cacheDir, s"${key.hash}.xml") @@ -59,16 +60,25 @@ class LocalStructureCache(cacheDir: File) extends SbtStructureCache { def getCachedStructureXml(key: BuildFilesHash): Option[String] = Option(file(key)) .filter(f => f.exists() && f.isFile) - .map(f => FileUtils.readFileToString(f, StandardCharsets.UTF_8)) + .map { f => + log.info(s"reading $key from local cache file") + FileUtils.readFileToString(f, StandardCharsets.UTF_8) + } - def cacheStructureXml(key: BuildFilesHash, data: String): Unit = + def cacheStructureXml(key: BuildFilesHash, data: String): Unit = { + if (!cacheDir.exists()) { + cacheDir.mkdirs() + } + log.info(s"writing $key to local cache file") FileUtils.write(file(key), data, StandardCharsets.UTF_8) + } } class RemoteStructureCache( repositoryUri: URI, credentialsFile: File, ) extends SbtStructureCache { + private val log = Logger.getInstance(classOf[RemoteStructureCache]) private lazy val authenticator = new Authenticator { val (user, password) = { @@ -98,22 +108,31 @@ class RemoteStructureCache( def getCachedStructureXml(key: BuildFilesHash): Option[String] = try { val request = fileRequest(key).GET().build() val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) + log.info(s"GET ${request.uri()}") val resp = httpClient.send(request, bodySubscriber) - if (resp.statusCode() == 200) + if (resp.statusCode() == 200) { Some(resp.body()) - else + } else { + log.warn(s"failed to read $key from remote cache, received HTTP status ${resp.statusCode()}") None + } } catch { - case _: IOException | _: TimeoutException => None + case e@(_: IOException | _: TimeoutException) => + log.warn(s"failed to read $key from remote cache", e) + None } def cacheStructureXml(key: BuildFilesHash, data: String): Unit = try { val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() - httpClient.send(request, BodyHandlers.discarding()) + log.info(s"PUT ${request.uri()}") + val resp = httpClient.send(request, BodyHandlers.discarding()) + if (resp.statusCode() >= 300) { + log.warn(s"failed to save $key in remote cache, received HTTP status ${resp.statusCode()}") + } } catch { - //TODO logging - case _: IOException | _: TimeoutException => + case e@(_: IOException | _: TimeoutException) => + log.warn(s"failed to save $key in remote cache", e) } } diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 786703da95a..6fa1c4ad3ed 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -173,7 +173,9 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti digest.update(rawFilePath.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, resolved) } - BuildFilesHash(DigestUtil.digestToHash(digest)) + val hash = DigestUtil.digestToHash(digest) + log.warn(s"FINAL HASH IS $hash") + BuildFilesHash(hash) } private def dumpStructure(projectRoot: File, @@ -189,18 +191,23 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti SbtStructureCache.get() match { case None => doDumpStructure(structureFile) case Some(cache) => + log.info("sbt structure cache detected") val key = buildFilesHash(projectRoot) val localizer = new StructureLocalizer(projectRoot) + log.info(s"looking for cached structure under key ${key.hash}") cache.getCachedStructureXml(key) match { - case Some(cachedXml) => Try { - val unlocalizedStructure = XML.loadString(cachedXml).deserialize[StructureData].getRight - localizer.localize(unlocalizedStructure) - .map(localizedStructure => (localizedStructure.serialize, BuildMessages.empty)) - .getOrElse(doDumpStructure(structureFile).get) - } + case Some(cachedXml) => + log.info("found cached structure file") + Try { + val unlocalizedStructure = XML.loadString(cachedXml).deserialize[StructureData].getRight + localizer.localize(unlocalizedStructure) + .map(localizedStructure => (localizedStructure.serialize, BuildMessages.empty)) + .getOrElse(doDumpStructure(structureFile).get) + } case None => + log.info("no cached structure file found") val result = doDumpStructure(structureFile) result.foreach { case (elem, _) => diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index dbf2257a92f..c5aa5b6bb1b 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -1,6 +1,7 @@ package org.jetbrains.sbt.project import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables +import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.util.SystemInfo import com.intellij.util.SystemProperties import org.jetbrains.sbt.structure._ @@ -32,9 +33,10 @@ object StructureLocalizer { } class StructureLocalizer(projectDir: File) { - import StructureLocalizer._ + private val log = Logger.getInstance(classOf[StructureLocalizer]) + private val projectPath = unix(projectDir.getAbsolutePath) @@ -147,6 +149,7 @@ class StructureLocalizer(projectDir: File) { else path val localFile = new File(path) if (localFile.getName.endsWith(".jar") && !localFile.exists()) { + log.warn(s"file $localFile not found, forcing full sbt refresh") throw LocalJarNotFound } localFile From 967c31b4f9306248b38ccf5eb897fabb8ec66990 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 01:06:20 +0100 Subject: [PATCH 13/39] making sure hash is the same on all platforms --- .../org/jetbrains/sbt/project/SbtProjectResolver.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 6fa1c4ad3ed..fce020c5b3e 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -142,11 +142,11 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti private def buildFilesHash(projectRoot: File): SbtStructureCache.BuildFilesHash = { val rootPath = projectRoot.toPath - val filesToDigest = new ArrayBuffer[Path] + val filesToDigest = new ArrayBuffer[String] val extensions = List(".sbt", ".scala", ".properties") def addFile(f: File): Unit = - filesToDigest.addOne(rootPath.relativize(f.toPath)) + filesToDigest.addOne(rootPath.relativize(f.toPath).toFile.getPath.replace('\\', '/')) projectRoot.listFiles((_, f) => f.endsWith(".sbt")).foreach(addFile) @@ -167,10 +167,9 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti filesToDigest.sortInPlace() val digest = DigestUtil.sha1() filesToDigest.foreach { path => - val rawFilePath = path.toFile.getPath.replace('\\', '/') val resolved = rootPath.resolve(path) - log.warn(s"HASHING $rawFilePath ($resolved), size: ${resolved.toFile.length()}") - digest.update(rawFilePath.getBytes(StandardCharsets.UTF_8)) + log.warn(s"HASHING $path ($resolved), size: ${resolved.toFile.length()}") + digest.update(path.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, resolved) } val hash = DigestUtil.digestToHash(digest) From 0af8e9c67296002e0d01bd1891ecfa432398aba4 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 01:18:39 +0100 Subject: [PATCH 14/39] marking locally nonexistent files in sbt structure --- .../org/jetbrains/sbt/project/StructureLocalizer.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index c5aa5b6bb1b..cb7f7fa8258 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -14,6 +14,7 @@ object StructureLocalizer { private final val CoursierCache = "/$COURSIER_CACHE$" private final val ProjectDir = "/$PROJECT_DIR$" private final val UserHome = "/$USER_HOME$" + private final val NotFoundMarker = "/$NOT_FOUND$" private def unix(path: String): String = path.replace('\\', '/') @@ -33,6 +34,7 @@ object StructureLocalizer { } class StructureLocalizer(projectDir: File) { + import StructureLocalizer._ private val log = Logger.getInstance(classOf[StructureLocalizer]) @@ -133,6 +135,9 @@ class StructureLocalizer(projectDir: File) { else if (path.startsWith(projectPath)) ProjectDir + path.stripPrefix(projectPath) else if (path.startsWith(userHome)) UserHome + path.stripPrefix(userHome) else path + if (f.getName.endsWith(".jar") && !f.exists()) { + path = path + NotFoundMarker + } new File(path) } @@ -147,8 +152,10 @@ class StructureLocalizer(projectDir: File) { else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome) else path + val mustExist = !path.endsWith(NotFoundMarker) + path = path.stripSuffix(NotFoundMarker) val localFile = new File(path) - if (localFile.getName.endsWith(".jar") && !localFile.exists()) { + if (mustExist && localFile.getName.endsWith(".jar") && !localFile.exists()) { log.warn(s"file $localFile not found, forcing full sbt refresh") throw LocalJarNotFound } From 37bdfe2abdc6767d222e3d618e29dd85247abdd6 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 02:14:19 +0100 Subject: [PATCH 15/39] more fixes and workarounds for local-os-specific XML serialization of file paths --- .../jetbrains/sbt/project/StructureLocalizer.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index cb7f7fa8258..2ab7ae8bd92 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -24,7 +24,7 @@ object StructureLocalizer { private lazy val coursierCachePath: String = if (SystemInfo.isWindows) - unix(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/Cache/v1" + unix(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/cache/v1" else if (SystemInfo.isMac) userHome + "/Library/Caches/Coursier/v1" else @@ -143,10 +143,12 @@ class StructureLocalizer(projectDir: File) { private def localize(f: File): File = { // The 'C:' that we're stripping appears when we create an Unix-style absolute path File on Windows - // and call '.getAbsolutePath` on it, e.g. `new File("/foo").getAbsolutePath` yields `C:\foo` - // We can't avoid that because `.getAbsolutePath` is used by XML serializer of `StructureData` - // in `sbt-structure`. - var path = unix(f.getPath.stripPrefix("C:")) + // and call '.getCanonicalPath` on it, e.g. `new File("/foo").getCanonicalPath` yields `C:\foo` + // We can't avoid that because `.getCanonicalPath` is used by XML serializer & deserializer of + // `StructureData` in `sbt-structure` (unless we rewrite that serialization entirely). + val workingDir = new File(".").getCanonicalPath + var path = unix(f.getPath.stripPrefix(workingDir).stripPrefix("C:")) + println(s"PATH IS ${f.getPath} -> $path") path = if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) From 27c1e7cd767814168520b6076709e3ece878d6cc Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 02:25:38 +0100 Subject: [PATCH 16/39] more horrible workarounds for path problems --- .../src/org/jetbrains/sbt/project/StructureLocalizer.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 2ab7ae8bd92..1179491c967 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -147,8 +147,7 @@ class StructureLocalizer(projectDir: File) { // We can't avoid that because `.getCanonicalPath` is used by XML serializer & deserializer of // `StructureData` in `sbt-structure` (unless we rewrite that serialization entirely). val workingDir = new File(".").getCanonicalPath - var path = unix(f.getPath.stripPrefix(workingDir).stripPrefix("C:")) - println(s"PATH IS ${f.getPath} -> $path") + var path = unix(f.getPath.stripPrefix(workingDir).stripPrefix("/C:").stripPrefix("C:")) path = if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) From 9b9235fc8629f40ce22fd7758007701db0f6c53b Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 02:53:23 +0100 Subject: [PATCH 17/39] including some sbt settings into build definition hash --- .../sbt/project/SbtStructureCache.scala | 24 +++++++++---------- .../sbt/project/SbtProjectResolver.scala | 18 +++++++++----- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index 12ea6e1d3b9..ff199abc1d2 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -3,7 +3,7 @@ package org.jetbrains.sbt.project import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.extensions.ExtensionPointName import org.apache.commons.io.FileUtils -import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash +import org.jetbrains.sbt.project.SbtStructureCache.BuildDefinitionHash import java.io.{File, FileReader, IOException} import java.net.http.HttpRequest.BodyPublishers @@ -18,9 +18,9 @@ import scala.concurrent.TimeoutException import scala.util.Using trait SbtStructureCache { - def getCachedStructureXml(key: BuildFilesHash): Option[String] + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] - def cacheStructureXml(key: BuildFilesHash, data: String): Unit + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit } object SbtStructureCache { object Properties { @@ -48,16 +48,16 @@ object SbtStructureCache { private val EpName: ExtensionPointName[SbtStructureCache] = ExtensionPointName.create("org.intellij.sbt.structureCache") - final case class BuildFilesHash(hash: String) + final case class BuildDefinitionHash(hash: String) } class LocalStructureCache(cacheDir: File) extends SbtStructureCache { private val log = Logger.getInstance(classOf[RemoteStructureCache]) - private def file(key: BuildFilesHash): File = + private def file(key: BuildDefinitionHash): File = new File(cacheDir, s"${key.hash}.xml") - def getCachedStructureXml(key: BuildFilesHash): Option[String] = + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = Option(file(key)) .filter(f => f.exists() && f.isFile) .map { f => @@ -65,7 +65,7 @@ class LocalStructureCache(cacheDir: File) extends SbtStructureCache { FileUtils.readFileToString(f, StandardCharsets.UTF_8) } - def cacheStructureXml(key: BuildFilesHash, data: String): Unit = { + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = { if (!cacheDir.exists()) { cacheDir.mkdirs() } @@ -100,12 +100,12 @@ class RemoteStructureCache( .authenticator(authenticator) .build() - private def fileRequest(key: BuildFilesHash): HttpRequest.Builder = + private def fileRequest(key: BuildDefinitionHash): HttpRequest.Builder = HttpRequest .newBuilder(repositoryUri.resolve(s"${key.hash}.xml")) .timeout(Duration.ofSeconds(30)) - def getCachedStructureXml(key: BuildFilesHash): Option[String] = try { + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = try { val request = fileRequest(key).GET().build() val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) log.info(s"GET ${request.uri()}") @@ -123,7 +123,7 @@ class RemoteStructureCache( None } - def cacheStructureXml(key: BuildFilesHash, data: String): Unit = try { + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = try { val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() log.info(s"PUT ${request.uri()}") val resp = httpClient.send(request, BodyHandlers.discarding()) @@ -137,7 +137,7 @@ class RemoteStructureCache( } class CompositeStructureCache(caches: List[SbtStructureCache]) extends SbtStructureCache { - def getCachedStructureXml(key: BuildFilesHash): Option[String] = { + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = { @tailrec def loop(tried: List[SbtStructureCache], notTried: List[SbtStructureCache]): Option[String] = notTried match { case Nil => None @@ -154,6 +154,6 @@ class CompositeStructureCache(caches: List[SbtStructureCache]) extends SbtStruct loop(Nil, caches) } - def cacheStructureXml(key: BuildFilesHash, data: String): Unit = + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = caches.foreach(_.cacheStructureXml(key, data)) } diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index fce020c5b3e..9a1d7d82d4c 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -24,7 +24,7 @@ import org.jetbrains.plugins.scala.project.external.{AndroidJdk, JdkByHome, JdkB import org.jetbrains.plugins.scala.util.ScalaNotificationGroups import org.jetbrains.sbt.SbtUtil._ import org.jetbrains.sbt.project.SbtProjectResolver._ -import org.jetbrains.sbt.project.SbtStructureCache.BuildFilesHash +import org.jetbrains.sbt.project.SbtStructureCache.BuildDefinitionHash import org.jetbrains.sbt.project.data._ import org.jetbrains.sbt.project.module.SbtModuleType import org.jetbrains.sbt.project.settings._ @@ -37,7 +37,6 @@ import org.jetbrains.sbt.{RichBoolean, Sbt, SbtBundle, SbtUtil, usingTempFile, s import java.io.{File, FileNotFoundException} import java.nio.charset.StandardCharsets -import java.nio.file.Path import java.util.{Collections, Locale, UUID} import scala.collection.mutable.ArrayBuffer import scala.collection.{MapView, mutable} @@ -140,7 +139,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti conversionResult.get // ok to throw here, that's the way ExternalSystem likes it } - private def buildFilesHash(projectRoot: File): SbtStructureCache.BuildFilesHash = { + private def buildDefinitionHash(projectRoot: File, settings: SbtExecutionSettings): BuildDefinitionHash = { val rootPath = projectRoot.toPath val filesToDigest = new ArrayBuffer[String] val extensions = List(".sbt", ".scala", ".properties") @@ -163,9 +162,16 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti scanDir(new File(projectRoot, "project")) + def byte(b: Boolean): Byte = if(b) 1 else 0 + + val digest = DigestUtil.sha1() + digest.update(byte(settings.resolveClassifiers)) + digest.update(byte(settings.resolveSbtClassifiers)) + digest.update(byte(settings.resolveJavadocs)) + digest.update(byte(settings.preferScala2)) + // files must be hashed in deterministic order filesToDigest.sortInPlace() - val digest = DigestUtil.sha1() filesToDigest.foreach { path => val resolved = rootPath.resolve(path) log.warn(s"HASHING $path ($resolved), size: ${resolved.toFile.length()}") @@ -174,7 +180,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti } val hash = DigestUtil.digestToHash(digest) log.warn(s"FINAL HASH IS $hash") - BuildFilesHash(hash) + BuildDefinitionHash(hash) } private def dumpStructure(projectRoot: File, @@ -191,7 +197,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti case None => doDumpStructure(structureFile) case Some(cache) => log.info("sbt structure cache detected") - val key = buildFilesHash(projectRoot) + val key = buildDefinitionHash(projectRoot, settings) val localizer = new StructureLocalizer(projectRoot) log.info(s"looking for cached structure under key ${key.hash}") From aa69029344bcc19fad322d3bfd24cdc4fdefe789 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 10:28:20 +0100 Subject: [PATCH 18/39] removed debug logs --- .../src/org/jetbrains/sbt/project/SbtProjectResolver.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 9a1d7d82d4c..dbe4cddb1fa 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -174,12 +174,10 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti filesToDigest.sortInPlace() filesToDigest.foreach { path => val resolved = rootPath.resolve(path) - log.warn(s"HASHING $path ($resolved), size: ${resolved.toFile.length()}") digest.update(path.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, resolved) } val hash = DigestUtil.digestToHash(digest) - log.warn(s"FINAL HASH IS $hash") BuildDefinitionHash(hash) } From 5f4f1d4a5f63cb65e740c1da115ca48babb023e6 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 10:31:36 +0100 Subject: [PATCH 19/39] version --- pluginXml/resources/META-INF/plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index 978d8f12173..089f46d8e75 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -33,7 +33,7 @@ Support for Play Framework, Akka and Scala.js is enabled in IntelliJ IDEA Ultimate. ]]> - 2022.3.99-sbtcaching.2 + 2022.3.99-sbtcaching.1 JetBrains From df752a840fc916b8c96543d2aa8f4a612ee58e61 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 12:23:50 +0100 Subject: [PATCH 20/39] more and more hacks --- .../sbt/project/StructureLocalizer.scala | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 1179491c967..d5794147e9d 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -11,10 +11,10 @@ import java.net.URI import scala.util.control.NoStackTrace object StructureLocalizer { - private final val CoursierCache = "/$COURSIER_CACHE$" - private final val ProjectDir = "/$PROJECT_DIR$" - private final val UserHome = "/$USER_HOME$" - private final val NotFoundMarker = "/$NOT_FOUND$" + private final val CoursierCache = "$COURSIER_CACHE$" + private final val ProjectDir = "$PROJECT_DIR$" + private final val UserHome = "$USER_HOME$" + private final val NotFoundMarker = "$NOT_FOUND$" private def unix(path: String): String = path.replace('\\', '/') @@ -131,12 +131,12 @@ class StructureLocalizer(projectDir: File) { private def unlocalize(f: File): File = { var path = unix(f.getAbsolutePath) path = - if (path.startsWith(coursierCachePath)) CoursierCache + path.stripPrefix(coursierCachePath) - else if (path.startsWith(projectPath)) ProjectDir + path.stripPrefix(projectPath) - else if (path.startsWith(userHome)) UserHome + path.stripPrefix(userHome) + if (path.startsWith(coursierCachePath)) "/" + CoursierCache + path.stripPrefix(coursierCachePath) + else if (path.startsWith(projectPath)) "/" + ProjectDir + path.stripPrefix(projectPath) + else if (path.startsWith(userHome)) "/" + UserHome + path.stripPrefix(userHome) else path if (f.getName.endsWith(".jar") && !f.exists()) { - path = path + NotFoundMarker + path = path + "/" + NotFoundMarker } new File(path) } @@ -147,14 +147,16 @@ class StructureLocalizer(projectDir: File) { // We can't avoid that because `.getCanonicalPath` is used by XML serializer & deserializer of // `StructureData` in `sbt-structure` (unless we rewrite that serialization entirely). val workingDir = new File(".").getCanonicalPath - var path = unix(f.getPath.stripPrefix(workingDir).stripPrefix("/C:").stripPrefix("C:")) + var path = f.getPath.stripPrefix(workingDir) + path = path.stripPrefix("/").stripPrefix("C:") + path = unix(path).stripPrefix("/") path = if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome) else path - val mustExist = !path.endsWith(NotFoundMarker) - path = path.stripSuffix(NotFoundMarker) + val mustExist = !path.endsWith("/" + NotFoundMarker) + path = path.stripSuffix("/" + NotFoundMarker) val localFile = new File(path) if (mustExist && localFile.getName.endsWith(".jar") && !localFile.exists()) { log.warn(s"file $localFile not found, forcing full sbt refresh") From 37c0ef4696ca3e8dfcefea5eaf03a9218ca68fc8 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 12:41:14 +0100 Subject: [PATCH 21/39] logging improvements --- .../jetbrains/sbt/project/SbtStructureCache.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index ff199abc1d2..03d000dc431 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -48,14 +48,16 @@ object SbtStructureCache { private val EpName: ExtensionPointName[SbtStructureCache] = ExtensionPointName.create("org.intellij.sbt.structureCache") - final case class BuildDefinitionHash(hash: String) + final case class BuildDefinitionHash(hash: String) { + override def toString: String = hash + } } class LocalStructureCache(cacheDir: File) extends SbtStructureCache { - private val log = Logger.getInstance(classOf[RemoteStructureCache]) + private val log = Logger.getInstance(getClass) private def file(key: BuildDefinitionHash): File = - new File(cacheDir, s"${key.hash}.xml") + new File(cacheDir, s"$key.xml") def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = Option(file(key)) @@ -78,7 +80,7 @@ class RemoteStructureCache( repositoryUri: URI, credentialsFile: File, ) extends SbtStructureCache { - private val log = Logger.getInstance(classOf[RemoteStructureCache]) + private val log = Logger.getInstance(getClass) private lazy val authenticator = new Authenticator { val (user, password) = { @@ -102,7 +104,7 @@ class RemoteStructureCache( private def fileRequest(key: BuildDefinitionHash): HttpRequest.Builder = HttpRequest - .newBuilder(repositoryUri.resolve(s"${key.hash}.xml")) + .newBuilder(repositoryUri.resolve(s"$key.xml")) .timeout(Duration.ofSeconds(30)) def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = try { From ef239bc4a10ee125face711910d615b4d469bac7 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 12:43:52 +0100 Subject: [PATCH 22/39] logging improvements --- .../org/jetbrains/sbt/project/SbtStructureCache.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index 03d000dc431..957fd723372 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -113,11 +113,12 @@ class RemoteStructureCache( log.info(s"GET ${request.uri()}") val resp = httpClient.send(request, bodySubscriber) - if (resp.statusCode() == 200) { - Some(resp.body()) - } else { - log.warn(s"failed to read $key from remote cache, received HTTP status ${resp.statusCode()}") - None + resp.statusCode() match { + case 200 => Some(resp.body()) + case 404 => None + case other => + log.warn(s"failed to read $key from remote cache, received HTTP status $other") + None } } catch { case e@(_: IOException | _: TimeoutException) => From 6fcb5e6946baa6e0f7d78ee85e96b74e019ce7c5 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 13:31:10 +0100 Subject: [PATCH 23/39] improved logging of build def hashing --- .../org/jetbrains/sbt/project/SbtProjectResolver.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index dbe4cddb1fa..774330c8d92 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -162,7 +162,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti scanDir(new File(projectRoot, "project")) - def byte(b: Boolean): Byte = if(b) 1 else 0 + def byte(b: Boolean): Byte = if (b) 1 else 0 val digest = DigestUtil.sha1() digest.update(byte(settings.resolveClassifiers)) @@ -170,10 +170,18 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti digest.update(byte(settings.resolveJavadocs)) digest.update(byte(settings.preferScala2)) + val settingsStr = + settings.resolveClassifiers.seq("resolveClassifiers") ++ + settings.resolveSbtClassifiers.seq("resolveSbtClassifiers") ++ + settings.resolveJavadocs.seq("resolveJavadocs") ++ + settings.preferScala2.seq("preferScala2") + log.info(s"Hashing SBT settings: ${settingsStr.mkString(",")}") + // files must be hashed in deterministic order filesToDigest.sortInPlace() filesToDigest.foreach { path => val resolved = rootPath.resolve(path) + log.info(s"Hashing file $path of size ${resolved.toFile.length()}") digest.update(path.getBytes(StandardCharsets.UTF_8)) DigestUtil.updateContentHash(digest, resolved) } From ffa2c7d4dbbde47812b02c25930b5a301e41a707 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 13:35:32 +0100 Subject: [PATCH 24/39] improved logging of build def hashing --- .../src/org/jetbrains/sbt/project/SbtProjectResolver.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 774330c8d92..008c7479cdf 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -186,6 +186,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti DigestUtil.updateContentHash(digest, resolved) } val hash = DigestUtil.digestToHash(digest) + log.info(s"Build definition hash is $hash") BuildDefinitionHash(hash) } From d67ace1019bcb661093783c6b620e1a9fd612652 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Wed, 15 Mar 2023 13:36:08 +0100 Subject: [PATCH 25/39] cosmetic --- .../org/jetbrains/sbt/project/SbtProjectResolver.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 008c7479cdf..a4cce1af1c2 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -165,10 +165,6 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti def byte(b: Boolean): Byte = if (b) 1 else 0 val digest = DigestUtil.sha1() - digest.update(byte(settings.resolveClassifiers)) - digest.update(byte(settings.resolveSbtClassifiers)) - digest.update(byte(settings.resolveJavadocs)) - digest.update(byte(settings.preferScala2)) val settingsStr = settings.resolveClassifiers.seq("resolveClassifiers") ++ @@ -177,6 +173,11 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti settings.preferScala2.seq("preferScala2") log.info(s"Hashing SBT settings: ${settingsStr.mkString(",")}") + digest.update(byte(settings.resolveClassifiers)) + digest.update(byte(settings.resolveSbtClassifiers)) + digest.update(byte(settings.resolveJavadocs)) + digest.update(byte(settings.preferScala2)) + // files must be hashed in deterministic order filesToDigest.sortInPlace() filesToDigest.foreach { path => From 8c1ef3d9d074dce28053689850f34808758b4d2a Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Fri, 17 Mar 2023 14:36:51 +0100 Subject: [PATCH 26/39] jar paths in unlocalized structure XML are resolver-agnostic --- .../sbt/project/StructureLocalizer.scala | 154 +++++++++++------- 1 file changed, 96 insertions(+), 58 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index d5794147e9d..9103e6ec446 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -11,26 +11,34 @@ import java.net.URI import scala.util.control.NoStackTrace object StructureLocalizer { + private final val AnyResolverCache = "$ANY_RESOLVER_CACHE$" private final val CoursierCache = "$COURSIER_CACHE$" private final val ProjectDir = "$PROJECT_DIR$" private final val UserHome = "$USER_HOME$" - private final val NotFoundMarker = "$NOT_FOUND$" + + implicit class stringOps(private val str: String) extends AnyVal { + def ensureSuffix(suffix: String): String = + if (str.endsWith(suffix)) str else str + suffix + + def ensurePrefix(prefix: String): String = + if (str.startsWith(prefix)) str else prefix + str + } private def unix(path: String): String = path.replace('\\', '/') private lazy val userHome: String = - unix(SystemProperties.getUserHome) + unix(SystemProperties.getUserHome).ensureSuffix("/") private lazy val coursierCachePath: String = if (SystemInfo.isWindows) - unix(WindowsEnvVariables.INSTANCE.getLocalApplicationData) + "/Coursier/cache/v1" + unix(WindowsEnvVariables.INSTANCE.getLocalApplicationData).ensureSuffix("/") + "Coursier/cache/v1/" else if (SystemInfo.isMac) - userHome + "/Library/Caches/Coursier/v1" + userHome + "Library/Caches/Coursier/v1/" else - userHome + "/.cache/coursier/v1" + userHome + ".cache/coursier/v1/" - private object LocalJarNotFound extends Exception with NoStackTrace + private case class LocalFileNotFound(file: File) extends Exception with NoStackTrace } class StructureLocalizer(projectDir: File) { @@ -40,108 +48,135 @@ class StructureLocalizer(projectDir: File) { private val log = Logger.getInstance(classOf[StructureLocalizer]) private val projectPath = - unix(projectDir.getAbsolutePath) + unix(projectDir.getAbsolutePath).ensureSuffix("/") def localize(sd: StructureData): Option[StructureData] = try Option(transform(sd)(localize)) catch { - case LocalJarNotFound => None + case LocalFileNotFound(file) => + log.warn(s"file $file not found, forcing full sbt refresh") + None } def unlocalize(sd: StructureData): StructureData = transform(sd)(unlocalize) - private def transform(pd: ProjectData)(f: File => File): ProjectData = + private def transform(data: StructureData)(f: (File, List[String]) => File): StructureData = { + val allLocalCachePaths = data.projects.toList.flatMap(localResolverCaches) + StructureData( + data.sbtVersion, + builds = data.builds.map(bd => BuildData( + transform(bd.uri)(f(_, Nil)), + bd.imports, + bd.classes.map(f(_, Nil)), + bd.docs.map(f(_, Nil)), + bd.sources.map(f(_, Nil)), + )), + projects = data.projects.map(transform(_)(f)), + data.repository.map(rd => RepositoryData(rd.modules.map(md => ModuleData( + md.id, + md.binaries.map(f(_, allLocalCachePaths)), + md.docs.map(f(_, allLocalCachePaths)), + md.sources.map(f(_, allLocalCachePaths)) + )))), + data.localCachePath.map(f(_, Nil)), + ) + } + + private def transform(pd: ProjectData)(f: (File, List[String]) => File): ProjectData = { + val localCachePaths = localResolverCaches(pd) ProjectData( pd.id, - transform(pd.buildURI)(f), + transform(pd.buildURI)(f(_, localCachePaths)), pd.name, pd.organization, pd.version, - f(pd.base), + f(pd.base, localCachePaths), pd.packagePrefix, pd.basePackages, - f(pd.target), + f(pd.target, localCachePaths), pd.configurations.map(cd => ConfigurationData( cd.id, - cd.sources.map(dd => DirectoryData(f(dd.file), dd.managed)), - cd.resources.map(dd => DirectoryData(f(dd.file), dd.managed)), - cd.excludes.map(f), - f(cd.classes), + cd.sources.map(dd => DirectoryData(f(dd.file, localCachePaths), dd.managed)), + cd.resources.map(dd => DirectoryData(f(dd.file, localCachePaths), dd.managed)), + cd.excludes.map(f(_, localCachePaths)), + f(cd.classes, localCachePaths), )), - pd.java.map(jd => JavaData(jd.home.map(f), jd.options)), + pd.java.map(jd => JavaData(jd.home.map(f(_, localCachePaths)), jd.options)), pd.scala.map(sd => ScalaData( sd.organization, sd.version, - sd.libraryJars.map(f), - sd.compilerJars.map(f), - sd.extraJars.map(f), + sd.libraryJars.map(f(_, localCachePaths)), + sd.compilerJars.map(f(_, localCachePaths)), + sd.extraJars.map(f(_, localCachePaths)), sd.options )), pd.compileOrder, pd.android.map(ad => AndroidData( ad.targetVersion, ad.manifest, - f(ad.apk), - f(ad.res), - f(ad.assets), - f(ad.gen), - f(ad.libs), + f(ad.apk, localCachePaths), + f(ad.res, localCachePaths), + f(ad.assets, localCachePaths), + f(ad.gen, localCachePaths), + f(ad.libs, localCachePaths), ad.isLibrary, ad.proguardConfig, ad.apklibs.map(al => ApkLib( - al.name, f(al.base), f(al.manifest), f(al.sources), f(al.resources), f(al.libs), f(al.gen) + al.name, + f(al.base, localCachePaths), + f(al.manifest, localCachePaths), + f(al.sources, localCachePaths), + f(al.resources, localCachePaths), + f(al.libs, localCachePaths), + f(al.gen, localCachePaths), )), ad.aars.map(ar => Aar(ar.name, transform(ar.project)(f))) )), on(pd.dependencies)(dd => DependencyData( dd.projects.map(pdd => ProjectDependencyData( - pdd.project, pdd.buildURI.map(transform(_)(f)), pdd.configuration, + pdd.project, pdd.buildURI.map(transform(_)(f(_, localCachePaths))), pdd.configuration, )), dd.modules, - dd.jars.map(jdd => JarDependencyData(f(jdd.file), jdd.configurations)), + dd.jars.map(jdd => JarDependencyData(f(jdd.file, localCachePaths), jdd.configurations)), )), pd.resolvers, pd.play2.map(p2d => Play2Data( p2d.playVersion, p2d.templatesImports, p2d.routesImports, - p2d.confDirectory.map(f), - f(p2d.sourceDirectory), + p2d.confDirectory.map(f(_, localCachePaths)), + f(p2d.sourceDirectory, localCachePaths), )), pd.settings, pd.tasks, pd.commands, ) + } private def on[T](v: T)(f: T => T): T = f(v) - private def transform(data: StructureData)(f: File => File): StructureData = - StructureData( - data.sbtVersion, - builds = data.builds.map(bd => BuildData( - transform(bd.uri)(f), bd.imports, bd.classes.map(f), bd.docs.map(f), bd.sources.map(f) - )), - projects = data.projects.map(transform(_)(f)), - data.repository.map(rd => RepositoryData(rd.modules.map(md => - ModuleData(md.id, md.binaries.map(f), md.docs.map(f), md.sources.map(f)) - ))), - data.localCachePath.map(f), - ) + //TODO: handle URIs with escaped chars and port number + private def localResolverCaches(proj: ProjectData): List[String] = + proj.resolvers + .filter(rd => rd.root.matches("http(s)?://.*")) + .toList.sortBy(_.name) + .map(rd => new URI(rd.root)) + .map(uri => s"$coursierCachePath${uri.getScheme}/${uri.getHost}${uri.getPath}".ensureSuffix("/")) - private def unlocalize(f: File): File = { + private def unlocalize(f: File, localCachePaths: List[String]): File = { var path = unix(f.getAbsolutePath) + path = localCachePaths + .find(path.startsWith) + .fold(path)(resolverPath => s"/$AnyResolverCache/${path.stripPrefix(resolverPath)}") path = - if (path.startsWith(coursierCachePath)) "/" + CoursierCache + path.stripPrefix(coursierCachePath) - else if (path.startsWith(projectPath)) "/" + ProjectDir + path.stripPrefix(projectPath) - else if (path.startsWith(userHome)) "/" + UserHome + path.stripPrefix(userHome) + if (path.startsWith(coursierCachePath)) s"/$CoursierCache/${path.stripPrefix(coursierCachePath)}" + else if (path.startsWith(projectPath)) s"/$ProjectDir/${path.stripPrefix(projectPath)}" + else if (path.startsWith(userHome)) s"/$UserHome/${path.stripPrefix(userHome)}" else path - if (f.getName.endsWith(".jar") && !f.exists()) { - path = path + "/" + NotFoundMarker - } new File(path) } - private def localize(f: File): File = { + private def localize(f: File, localCachePaths: List[String]): File = { // The 'C:' that we're stripping appears when we create an Unix-style absolute path File on Windows // and call '.getCanonicalPath` on it, e.g. `new File("/foo").getCanonicalPath` yields `C:\foo` // We can't avoid that because `.getCanonicalPath` is used by XML serializer & deserializer of @@ -150,17 +185,20 @@ class StructureLocalizer(projectDir: File) { var path = f.getPath.stripPrefix(workingDir) path = path.stripPrefix("/").stripPrefix("C:") path = unix(path).stripPrefix("/") + if (path.startsWith(AnyResolverCache)) { + path = localCachePaths + .map(resPath => resPath + path.stripPrefix(AnyResolverCache).stripPrefix("/")) + .find(p => new File(p).exists()) + .getOrElse(throw LocalFileNotFound(new File(path))) + } path = - if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache) - else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir) - else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome) + if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache).stripPrefix("/") + else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir).stripPrefix("/") + else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome).stripPrefix("/") else path - val mustExist = !path.endsWith("/" + NotFoundMarker) - path = path.stripSuffix("/" + NotFoundMarker) val localFile = new File(path) - if (mustExist && localFile.getName.endsWith(".jar") && !localFile.exists()) { - log.warn(s"file $localFile not found, forcing full sbt refresh") - throw LocalJarNotFound + if (localFile.getName.endsWith(".jar") && !localFile.exists()) { + throw LocalFileNotFound(localFile) } localFile } From e6d5e12927c0e6168c9b35184c63af5a08d93798 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Fri, 17 Mar 2023 15:50:35 +0100 Subject: [PATCH 27/39] clearing sbt update-cache before generating to-be-cached structure --- .../org/jetbrains/sbt/project/SbtProjectResolver.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index a4cce1af1c2..c3121b8d646 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -191,6 +191,13 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti BuildDefinitionHash(hash) } + //TODO: PoC, assumes specific project structure (e.g. that someone didn't customize 'target' dir to sth else) + private def cleanSbtUpdateCache(projectRoot: File): Unit = { + FileUtil.delete(projectRoot / "target/update/update-cache") + FileUtil.delete(projectRoot / "project/target") + FileUtil.delete(projectRoot / "project/project/target") + } + private def dumpStructure(projectRoot: File, sbtLauncher: File, sbtVersion: Version, @@ -221,6 +228,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti case None => log.info("no cached structure file found") + cleanSbtUpdateCache(projectRoot) val result = doDumpStructure(structureFile) result.foreach { case (elem, _) => From 6f1258df0cf3816a8f0a9a8b18f7f0e884f11a97 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Fri, 17 Mar 2023 19:31:33 +0100 Subject: [PATCH 28/39] including SBT version into build definition hash --- .../jetbrains/sbt/project/SbtProjectResolver.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index c3121b8d646..2fc5cd894e6 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -139,7 +139,11 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti conversionResult.get // ok to throw here, that's the way ExternalSystem likes it } - private def buildDefinitionHash(projectRoot: File, settings: SbtExecutionSettings): BuildDefinitionHash = { + private def buildDefinitionHash( + projectRoot: File, + sbtVersion: Version, + settings: SbtExecutionSettings + ): BuildDefinitionHash = { val rootPath = projectRoot.toPath val filesToDigest = new ArrayBuffer[String] val extensions = List(".sbt", ".scala", ".properties") @@ -165,14 +169,16 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti def byte(b: Boolean): Byte = if (b) 1 else 0 val digest = DigestUtil.sha1() + log.info(s"Hashing SBT version ${sbtVersion.presentation}") + digest.update(sbtVersion.presentation.getBytes(StandardCharsets.UTF_8)) val settingsStr = settings.resolveClassifiers.seq("resolveClassifiers") ++ settings.resolveSbtClassifiers.seq("resolveSbtClassifiers") ++ settings.resolveJavadocs.seq("resolveJavadocs") ++ settings.preferScala2.seq("preferScala2") - log.info(s"Hashing SBT settings: ${settingsStr.mkString(",")}") + log.info(s"Hashing SBT settings: ${settingsStr.mkString(",")}") digest.update(byte(settings.resolveClassifiers)) digest.update(byte(settings.resolveSbtClassifiers)) digest.update(byte(settings.resolveJavadocs)) @@ -212,7 +218,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti case None => doDumpStructure(structureFile) case Some(cache) => log.info("sbt structure cache detected") - val key = buildDefinitionHash(projectRoot, settings) + val key = buildDefinitionHash(projectRoot, sbtVersion, settings) val localizer = new StructureLocalizer(projectRoot) log.info(s"looking for cached structure under key ${key.hash}") From a2f771be334bdb1188884a4362d7fb9efb60cbbb Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Fri, 17 Mar 2023 19:39:00 +0100 Subject: [PATCH 29/39] improved logging and coursier cache .error file awareness --- .../org/jetbrains/sbt/project/StructureLocalizer.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 9103e6ec446..4b1b8af60f0 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -4,6 +4,7 @@ import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.util.SystemInfo import com.intellij.util.SystemProperties +import org.jetbrains.plugins.scala.extensions.RichFile import org.jetbrains.sbt.structure._ import java.io.File @@ -188,7 +189,14 @@ class StructureLocalizer(projectDir: File) { if (path.startsWith(AnyResolverCache)) { path = localCachePaths .map(resPath => resPath + path.stripPrefix(AnyResolverCache).stripPrefix("/")) - .find(p => new File(p).exists()) + .find { p => + val f = new File(p) + val errorFile = f.getParentFile / s".${f.getName}.error" + f.exists() || errorFile.exists() || { + log.warn(s"file ${f.getAbsolutePath} not found") + false + } + } .getOrElse(throw LocalFileNotFound(new File(path))) } path = From 223a8b2791f810f2a9faa81b42b0b88e962a950e Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Fri, 17 Mar 2023 23:13:20 +0100 Subject: [PATCH 30/39] more logging improvements --- .../sbt/project/StructureLocalizer.scala | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index 4b1b8af60f0..f2a370defd7 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -187,28 +187,29 @@ class StructureLocalizer(projectDir: File) { path = path.stripPrefix("/").stripPrefix("C:") path = unix(path).stripPrefix("/") if (path.startsWith(AnyResolverCache)) { - path = localCachePaths - .map(resPath => resPath + path.stripPrefix(AnyResolverCache).stripPrefix("/")) - .find { p => - val f = new File(p) + val files = localCachePaths + .map(resPath => new File(resPath + path.stripPrefix(AnyResolverCache).stripPrefix("/"))) + files + .find { f => val errorFile = f.getParentFile / s".${f.getName}.error" - f.exists() || errorFile.exists() || { - log.warn(s"file ${f.getAbsolutePath} not found") - false - } + f.exists() || errorFile.exists() } - .getOrElse(throw LocalFileNotFound(new File(path))) - } - path = - if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache).stripPrefix("/") - else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir).stripPrefix("/") - else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome).stripPrefix("/") - else path - val localFile = new File(path) - if (localFile.getName.endsWith(".jar") && !localFile.exists()) { - throw LocalFileNotFound(localFile) + .getOrElse { + log.warn(s"none of the files found:\n${files.map(_.getAbsolutePath).mkString("\n")}") + throw LocalFileNotFound(new File(path)) + } + } else { + path = + if (path.startsWith(CoursierCache)) coursierCachePath + path.stripPrefix(CoursierCache).stripPrefix("/") + else if (path.startsWith(ProjectDir)) projectPath + path.stripPrefix(ProjectDir).stripPrefix("/") + else if (path.startsWith(UserHome)) userHome + path.stripPrefix(UserHome).stripPrefix("/") + else path + val localFile = new File(path) + if (localFile.getName.endsWith(".jar") && !localFile.exists()) { + throw LocalFileNotFound(localFile) + } + localFile } - localFile } private def transform(uri: URI)(f: File => File): URI = From 9ce9124054ad59293e042a637cb2906d900277f1 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Sat, 18 Mar 2023 20:19:14 +0100 Subject: [PATCH 31/39] proper project settings for structure cache instead of system properties --- .../sbt/project/SbtStructureCache.scala | 147 ------------------ .../jetbrains/sbt/settings/SbtSettings.scala | 22 ++- .../project/SbtExternalSystemManager.scala | 20 +++ .../sbt/project/SbtProjectResolver.scala | 2 +- .../sbt/project/SbtStructureCaches.scala | 142 +++++++++++++++++ .../settings/SbtExecutionSettings.scala | 4 + .../sbt/settings/SbtSettingsControl.scala | 11 +- .../sbt/settings/SbtSettingsPane.form | 78 +++++++++- .../sbt/settings/SbtSettingsPane.java | 70 ++++++++- .../sbt/shell/MaxJvmHeapParameterTest.scala | 11 +- .../formatting/settings/ImportsPanel.java | 6 +- 11 files changed, 345 insertions(+), 168 deletions(-) create mode 100644 sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala index 957fd723372..add3617bd94 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/project/SbtStructureCache.scala @@ -1,162 +1,15 @@ package org.jetbrains.sbt.project -import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.extensions.ExtensionPointName -import org.apache.commons.io.FileUtils import org.jetbrains.sbt.project.SbtStructureCache.BuildDefinitionHash -import java.io.{File, FileReader, IOException} -import java.net.http.HttpRequest.BodyPublishers -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpClient, HttpRequest} -import java.net.{Authenticator, PasswordAuthentication, URI} -import java.nio.charset.StandardCharsets -import java.time.Duration -import java.util.Properties -import scala.annotation.tailrec -import scala.concurrent.TimeoutException -import scala.util.Using - trait SbtStructureCache { def getCachedStructureXml(key: BuildDefinitionHash): Option[String] def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit } object SbtStructureCache { - object Properties { - final val LocalStructureCacheDir = "org.intellij.sbt.structure.localCacheDir" - final val RemoteStructureCacheUri = "org.intellij.sbt.structure.remoteCacheUri" - final val RemoteStructureCacheCredentialsFile = "org.intellij.sbt.structure.remoteCacheCredentialsFile" - } - - def get(): Option[SbtStructureCache] = { - val localCache = sys.props.get(Properties.LocalStructureCacheDir).map(new File(_)) - .map(new LocalStructureCache(_)) - - val remoteCache = for { - remoteCacheUri <- sys.props.get(Properties.RemoteStructureCacheUri) - .map(v => if (v.endsWith("/")) v else s"$v/") - .map(new URI(_)) - remoteCacheCredsFile <- sys.props.get(Properties.RemoteStructureCacheCredentialsFile).map(new File(_)) - } yield new RemoteStructureCache(remoteCacheUri, remoteCacheCredsFile) - - val cacheChain = localCache.toList ++ remoteCache.toList - if (cacheChain.nonEmpty) Some(new CompositeStructureCache(cacheChain)) - else None - } - - private val EpName: ExtensionPointName[SbtStructureCache] = - ExtensionPointName.create("org.intellij.sbt.structureCache") - final case class BuildDefinitionHash(hash: String) { override def toString: String = hash } } -class LocalStructureCache(cacheDir: File) extends SbtStructureCache { - private val log = Logger.getInstance(getClass) - - private def file(key: BuildDefinitionHash): File = - new File(cacheDir, s"$key.xml") - - def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = - Option(file(key)) - .filter(f => f.exists() && f.isFile) - .map { f => - log.info(s"reading $key from local cache file") - FileUtils.readFileToString(f, StandardCharsets.UTF_8) - } - - def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = { - if (!cacheDir.exists()) { - cacheDir.mkdirs() - } - log.info(s"writing $key to local cache file") - FileUtils.write(file(key), data, StandardCharsets.UTF_8) - } -} - -class RemoteStructureCache( - repositoryUri: URI, - credentialsFile: File, -) extends SbtStructureCache { - private val log = Logger.getInstance(getClass) - - private lazy val authenticator = new Authenticator { - val (user, password) = { - val props = new Properties - Using.resource(new FileReader(credentialsFile))(props.load) - val u = props.getProperty("user") - val p = props.getProperty("password") - require(u != null, "username not found in credentials file") - require(p != null, "password not found in credentials file") - (u, p.toCharArray) - } - - override def getPasswordAuthentication: PasswordAuthentication = - new PasswordAuthentication(user, password) - } - - private lazy val httpClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(30)) - .authenticator(authenticator) - .build() - - private def fileRequest(key: BuildDefinitionHash): HttpRequest.Builder = - HttpRequest - .newBuilder(repositoryUri.resolve(s"$key.xml")) - .timeout(Duration.ofSeconds(30)) - - def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = try { - val request = fileRequest(key).GET().build() - val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) - log.info(s"GET ${request.uri()}") - val resp = httpClient.send(request, bodySubscriber) - - resp.statusCode() match { - case 200 => Some(resp.body()) - case 404 => None - case other => - log.warn(s"failed to read $key from remote cache, received HTTP status $other") - None - } - } catch { - case e@(_: IOException | _: TimeoutException) => - log.warn(s"failed to read $key from remote cache", e) - None - } - - def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = try { - val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() - log.info(s"PUT ${request.uri()}") - val resp = httpClient.send(request, BodyHandlers.discarding()) - if (resp.statusCode() >= 300) { - log.warn(s"failed to save $key in remote cache, received HTTP status ${resp.statusCode()}") - } - } catch { - case e@(_: IOException | _: TimeoutException) => - log.warn(s"failed to save $key in remote cache", e) - } -} - -class CompositeStructureCache(caches: List[SbtStructureCache]) extends SbtStructureCache { - def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = { - @tailrec def loop(tried: List[SbtStructureCache], notTried: List[SbtStructureCache]): Option[String] = - notTried match { - case Nil => None - case head :: tail => - head.getCachedStructureXml(key) match { - case Some(value) => - tried.foreach(_.cacheStructureXml(key, value)) - Some(value) - case None => - loop(head :: tried, tail) - } - } - - loop(Nil, caches) - } - - def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = - caches.foreach(_.cacheStructureXml(key, data)) -} diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala b/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala index 9e750e2efad..93792359e7d 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala @@ -33,6 +33,9 @@ final class SbtSettings(project: Project) @BeanProperty var customVMEnabled: Boolean = false @BeanProperty var customVMPath: String = "" @BeanProperty var customSbtStructurePath: String = "" + @BeanProperty var localStructureCachePath: String = "" + @BeanProperty var remoteStructureCacheUri: String = "" + @BeanProperty var remoteStructureCacheCredentialsPath: String = "" override def getState: SbtSettings.State = { val state = new SbtSettings.State @@ -45,7 +48,9 @@ final class SbtSettings(project: Project) state.customVMEnabled = customVMEnabled state.customVMPath = customVMPath state.customSbtStructurePath = customSbtStructurePath - + state.localStructureCache = localStructureCachePath + state.remoteStructureCacheUri = remoteStructureCacheUri + state.remoteStructureCacheCredentialsPath = remoteStructureCacheCredentialsPath state } @@ -59,6 +64,9 @@ final class SbtSettings(project: Project) customVMEnabled = state.customVMEnabled customVMPath = state.customVMPath customSbtStructurePath = state.customSbtStructurePath + localStructureCachePath = state.localStructureCache + remoteStructureCacheUri = state.remoteStructureCacheUri + remoteStructureCacheCredentialsPath = state.remoteStructureCacheCredentialsPath } override def subscribe(listener: ExternalSystemSettingsListener[SbtProjectSettings]): Unit = { @@ -79,6 +87,9 @@ final class SbtSettings(project: Project) customVMEnabled = settings.customVMEnabled customVMPath = settings.customVMPath customSbtStructurePath = settings.customSbtStructurePath + localStructureCachePath = settings.localStructureCachePath + remoteStructureCacheUri = settings.remoteStructureCacheUri + remoteStructureCacheCredentialsPath = settings.remoteStructureCacheCredentialsPath } def getLinkedProjectSettings(module: Module): Option[SbtProjectSettings] = @@ -124,6 +135,15 @@ object SbtSettings { @BeanProperty var customSbtStructurePath: String = "" + @BeanProperty + var localStructureCache: String = "" + + @BeanProperty + var remoteStructureCacheUri: String = "" + + @BeanProperty + var remoteStructureCacheCredentialsPath: String = "" + private val linkedProjectSettings: util.TreeSet[SbtProjectSettings] = new util.TreeSet[SbtProjectSettings] @XCollection(style = XCollection.Style.v1, elementTypes = Array(classOf[SbtProjectSettings])) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala index 08c1263ac01..959dc07317d 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala @@ -23,6 +23,7 @@ import org.jetbrains.sbt.project.settings._ import org.jetbrains.sbt.settings.{SbtExternalSystemConfigurable, SbtSettings} import java.io.File +import java.net.URI import scala.jdk.CollectionConverters._ class SbtExternalSystemManager @@ -84,6 +85,22 @@ object SbtExternalSystemManager { val vmOptions = getVmOptions(settingsState, jreHome) val environment = Map.empty ++ getAndroidEnvironmentVariables(projectJdkName) + val localStructureCacheFile = + Option(settingsState.localStructureCache) + .filter(_.nonEmpty) + .map(new File(_)) + + val remoteStructureCacheUri = + Option(settingsState.remoteStructureCacheUri) + .filter(_.nonEmpty) + .map(v => if (v.endsWith("/")) v else s"$v/") + .map(new URI(_)) + + val remoteStructureCacheCredentialsFile = + Option(settingsState.remoteStructureCacheCredentialsPath) + .filter(_.nonEmpty) + .map(new File(_)) + new SbtExecutionSettings( realProjectPath, vmExecutable, @@ -92,6 +109,9 @@ object SbtExternalSystemManager { environment, customLauncher, customSbtStructureFile, + localStructureCacheFile, + remoteStructureCacheUri, + remoteStructureCacheCredentialsFile, projectJdkName, projectSettings.resolveClassifiers, projectSettings.resolveJavadocs, diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 2fc5cd894e6..8e3f916c143 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -214,7 +214,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti val options = dumpOptions(settings) def doDumpStructureCached(structureFile: File): Try[(Elem, BuildMessages)] = - SbtStructureCache.get() match { + SbtStructureCaches.get(settings) match { case None => doDumpStructure(structureFile) case Some(cache) => log.info("sbt structure cache detected") diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala new file mode 100644 index 00000000000..eaf15e9c866 --- /dev/null +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala @@ -0,0 +1,142 @@ +package org.jetbrains.sbt.project + +import com.intellij.openapi.diagnostic.Logger +import org.apache.commons.io.FileUtils +import org.jetbrains.sbt.project.SbtStructureCache.BuildDefinitionHash +import org.jetbrains.sbt.project.settings.SbtExecutionSettings + +import java.io.{File, FileReader, IOException} +import java.net.http.HttpRequest.BodyPublishers +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest} +import java.net.{Authenticator, PasswordAuthentication, URI} +import java.nio.charset.StandardCharsets +import java.time.Duration +import java.util.Properties +import scala.annotation.tailrec +import scala.concurrent.TimeoutException +import scala.util.Using +import scala.util.chaining._ + +object SbtStructureCaches { + //TODO: cache the caches per project or sth? + def get(settings: SbtExecutionSettings): Option[SbtStructureCache] = { + val localCache = settings.localStructureCacheFile.map(new LocalStructureCache(_)) + val remoteCache = settings.remoteStructureCacheUri.map { uri => + new RemoteStructureCache(uri, settings.remoteStructureCacheCredentialsFile) + } + val cacheChain = localCache.toList ++ remoteCache.toList + if (cacheChain.nonEmpty) Some(new CompositeStructureCache(cacheChain)) + else None + } +} + +class LocalStructureCache(cacheDir: File) extends SbtStructureCache { + private val log = Logger.getInstance(getClass) + + private def file(key: BuildDefinitionHash): File = + new File(cacheDir, s"$key.xml") + + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = + Option(file(key)) + .filter(f => f.exists() && f.isFile) + .map { f => + log.info(s"reading $key from local cache file") + FileUtils.readFileToString(f, StandardCharsets.UTF_8) + } + + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = { + if (!cacheDir.exists()) { + cacheDir.mkdirs() + } + log.info(s"writing $key to local cache file") + FileUtils.write(file(key), data, StandardCharsets.UTF_8) + } +} + +class RemoteStructureCache( + repositoryUri: URI, + credentialsFile: Option[File], +) extends SbtStructureCache { + private val log = Logger.getInstance(getClass) + + private lazy val authenticator = credentialsFile.map { file => + new Authenticator { + val (user, password) = { + val props = new Properties + Using.resource(new FileReader(file))(props.load) + val u = props.getProperty("user") + val p = props.getProperty("password") + require(u != null, "username not found in credentials file") + require(p != null, "password not found in credentials file") + (u, p.toCharArray) + } + + override def getPasswordAuthentication: PasswordAuthentication = + new PasswordAuthentication(user, password) + } + } + + private lazy val httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .tap(b => authenticator.fold(b)(b.authenticator)) + .build() + + private def fileRequest(key: BuildDefinitionHash): HttpRequest.Builder = + HttpRequest + .newBuilder(repositoryUri.resolve(s"$key.xml")) + .timeout(Duration.ofSeconds(30)) + + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = try { + val request = fileRequest(key).GET().build() + val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) + log.info(s"GET ${request.uri()}") + val resp = httpClient.send(request, bodySubscriber) + + resp.statusCode() match { + case 200 => Some(resp.body()) + case 404 => None + case other => + log.warn(s"failed to read $key from remote cache, received HTTP status $other") + None + } + } catch { + case e@(_: IOException | _: TimeoutException) => + log.warn(s"failed to read $key from remote cache", e) + None + } + + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = try { + val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() + log.info(s"PUT ${request.uri()}") + val resp = httpClient.send(request, BodyHandlers.discarding()) + if (resp.statusCode() >= 300) { + log.warn(s"failed to save $key in remote cache, received HTTP status ${resp.statusCode()}") + } + } catch { + case e@(_: IOException | _: TimeoutException) => + log.warn(s"failed to save $key in remote cache", e) + } +} + +class CompositeStructureCache(caches: List[SbtStructureCache]) extends SbtStructureCache { + def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = { + @tailrec def loop(tried: List[SbtStructureCache], notTried: List[SbtStructureCache]): Option[String] = + notTried match { + case Nil => None + case head :: tail => + head.getCachedStructureXml(key) match { + case Some(value) => + tried.foreach(_.cacheStructureXml(key, value)) + Some(value) + case None => + loop(head :: tried, tail) + } + } + + loop(Nil, caches) + } + + def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = + caches.foreach(_.cacheStructureXml(key, data)) +} diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala index 180efa392c2..5fed842c110 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala @@ -4,6 +4,7 @@ package project.settings import com.intellij.openapi.externalSystem.model.settings.ExternalSystemExecutionSettings import java.io.File +import java.net.URI class SbtExecutionSettings(val realProjectPath: String, val vmExecutable: File, @@ -12,6 +13,9 @@ class SbtExecutionSettings(val realProjectPath: String, val environment: Map[String,String], val customLauncher: Option[File], val customSbtStructureFile: Option[File], + val localStructureCacheFile: Option[File], + val remoteStructureCacheUri: Option[URI], + val remoteStructureCacheCredentialsFile: Option[File], val jdk: Option[String], val resolveClassifiers: Boolean, val resolveJavadocs: Boolean, diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala index 304b255f434..cfd381706a5 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala @@ -12,7 +12,10 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo pane.getMaximumHeapSize == settings.maximumHeapSize && pane.getVmParameters == settings.vmParameters && pane.isCustomVM == settings.customVMEnabled && - pane.getCustomVMPath == settings.customVMPath + pane.getCustomVMPath == settings.customVMPath && + pane.getLocalStructureCachePath == settings.localStructureCachePath && + pane.getRemoteStructureCacheUri == settings.remoteStructureCacheUri && + pane.getRemoteStructureCacheCredentialsPath == settings.remoteStructureCacheCredentialsPath } override def showUi(show: Boolean): Unit = @@ -30,6 +33,9 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo settings.vmParameters = pane.getVmParameters settings.customVMEnabled = pane.isCustomVM settings.customVMPath = pane.getCustomVMPath + settings.localStructureCachePath = pane.getLocalStructureCachePath + settings.remoteStructureCacheUri = pane.getRemoteStructureCacheUri + settings.remoteStructureCacheCredentialsPath = pane.getRemoteStructureCacheCredentialsPath } override def reset(): Unit = { @@ -38,6 +44,9 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo pane.setMaximumHeapSize(settings.maximumHeapSize) pane.setMyVmParameters(settings.vmParameters) pane.setCustomVMPath(settings.customVMPath, settings.customVMEnabled) + pane.setLocalStructureCachePath(settings.localStructureCachePath) + pane.setRemoteStructureCacheUri(settings.remoteStructureCacheUri) + pane.setRemoteStructureCacheCredentialsPath(settings.remoteStructureCacheCredentialsPath) } override def validate(settings: SbtSettings) = true diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form index b12bf4552af..b4a7d9c6ce0 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form @@ -1,6 +1,6 @@
- + @@ -8,11 +8,6 @@ - - - - - @@ -135,6 +130,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java index 829e09feaed..7643f795a24 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java @@ -32,6 +32,9 @@ public class SbtSettingsPane { private RawCommandLineEditor myVmParameters; private JPanel myContentPanel; private JrePathEditor myJrePathEditor; + private TextFieldWithBrowseButton myLocalStructureCachePath; + private JTextField myRemoteStructureCacheUri; + private TextFieldWithBrowseButton myRemoteStructureCacheCredentialsPath; private final Project myProject; @@ -51,6 +54,20 @@ public SbtSettingsPane(Project project) { SbtBundle.message("sbt.settings.choose.sbt.launch.jar"), project, FileChooserDescriptorFactory.createSingleLocalFileDescriptor()); + + myLocalStructureCachePath.addBrowseFolderListener( + "Choose Cache Directory", + "Choose local sbt structure cache directory", + project, + FileChooserDescriptorFactory.createSingleFolderDescriptor() + ); + + myRemoteStructureCacheCredentialsPath.addBrowseFolderListener( + "Choose Remote Cache Credentials", + "Choose remote sbt structure cache credentials file", + project, + FileChooserDescriptorFactory.createSingleLocalFileDescriptor() + ); } public void createUIComponents() { @@ -125,6 +142,30 @@ public void setMyVmParameters(String value) { myVmParameters.setText(value); } + public String getLocalStructureCachePath() { + return myLocalStructureCachePath.getText(); + } + + public void setLocalStructureCachePath(String path) { + myLocalStructureCachePath.setText(path); + } + + public String getRemoteStructureCacheUri() { + return myRemoteStructureCacheUri.getText(); + } + + public void setRemoteStructureCacheUri(String path) { + myRemoteStructureCacheUri.setText(path); + } + + public String getRemoteStructureCacheCredentialsPath() { + return myRemoteStructureCacheCredentialsPath.getText(); + } + + public void setRemoteStructureCacheCredentialsPath(String path) { + myRemoteStructureCacheCredentialsPath.setText(path); + } + /** * Method generated by IntelliJ IDEA GUI Designer * >>> IMPORTANT!! <<< @@ -135,9 +176,7 @@ public void setMyVmParameters(String value) { private void $$$setupUI$$$() { createUIComponents(); myContentPanel = new JPanel(); - myContentPanel.setLayout(new GridLayoutManager(6, 2, new Insets(0, 0, 0, 0), -1, -1)); - final Spacer spacer1 = new Spacer(); - myContentPanel.add(spacer1, new GridConstraints(5, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_VERTICAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_WANT_GROW, null, null, null, 0, false)); + myContentPanel.setLayout(new GridLayoutManager(8, 2, new Insets(0, 0, 0, 0), -1, -1)); final TitledSeparator titledSeparator1 = new TitledSeparator(); titledSeparator1.setText(this.$$$getMessageFromBundle$$$("messages/SbtBundle", "sbt.settings.sbtLauncher")); myContentPanel.add(titledSeparator1, new GridConstraints(3, 0, 1, 2, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_HORIZONTAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); @@ -175,8 +214,31 @@ public void setMyVmParameters(String value) { myVmParameters.setDialogCaption(this.$$$getMessageFromBundle$$$("messages/SbtBundle", "sbt.settings.vmParams")); myVmParameters.setEnabled(true); panel3.add(myVmParameters, new GridConstraints(1, 1, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, new Dimension(250, -1), new Dimension(250, -1), null, 0, false)); + final Spacer spacer1 = new Spacer(); + myContentPanel.add(spacer1, new GridConstraints(1, 1, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_HORIZONTAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, 1, null, null, null, 0, false)); + final TitledSeparator titledSeparator3 = new TitledSeparator(); + titledSeparator3.setText("Structure cache"); + myContentPanel.add(titledSeparator3, new GridConstraints(5, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); final Spacer spacer2 = new Spacer(); - myContentPanel.add(spacer2, new GridConstraints(1, 1, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_HORIZONTAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, 1, null, null, null, 0, false)); + myContentPanel.add(spacer2, new GridConstraints(7, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_VERTICAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_WANT_GROW, null, null, null, 0, false)); + final JPanel panel4 = new JPanel(); + panel4.setLayout(new GridLayoutManager(3, 2, new Insets(0, 0, 0, 0), -1, -1)); + myContentPanel.add(panel4, new GridConstraints(6, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 1, false)); + final JLabel label3 = new JLabel(); + label3.setText("Local cache directory"); + panel4.add(label3, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_FIXED, GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + myLocalStructureCachePath = new TextFieldWithBrowseButton(); + panel4.add(myLocalStructureCachePath, new GridConstraints(0, 1, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, new Dimension(250, -1), null, null, 0, false)); + final JLabel label4 = new JLabel(); + label4.setText("Remote cache URI"); + panel4.add(label4, new GridConstraints(1, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_FIXED, GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + myRemoteStructureCacheUri = new JTextField(); + panel4.add(myRemoteStructureCacheUri, new GridConstraints(1, 1, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_HORIZONTAL, GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_WANT_GROW, null, new Dimension(150, -1), null, 0, false)); + final JLabel label5 = new JLabel(); + label5.setText("Remote cache credentials file"); + panel4.add(label5, new GridConstraints(2, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_FIXED, GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); + myRemoteStructureCacheCredentialsPath = new TextFieldWithBrowseButton(); + panel4.add(myRemoteStructureCacheCredentialsPath, new GridConstraints(2, 1, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, new Dimension(250, -1), null, null, 0, false)); label1.setLabelFor(myMaximumHeapSize); ButtonGroup buttonGroup; buttonGroup = new ButtonGroup(); diff --git a/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala b/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala index f642ec3cb5b..ce73ff3561b 100644 --- a/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala +++ b/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala @@ -8,6 +8,7 @@ import org.jetbrains.sbt.project.settings.SbtExecutionSettings import java.io.File class MaxJvmHeapParameterTest extends TestCase { + import org.junit.Assert._ val hiddenDefaultSize = JvmMemorySize.Megabytes(1500) @@ -18,12 +19,12 @@ class MaxJvmHeapParameterTest extends TestCase { val workingDir = FileUtil.createTempDirectory("maxHeapJvmParamTest", getName, true) if (jvmOpts.nonEmpty) { - val jvmOptsFile = new File(workingDir,".jvmopts") + val jvmOptsFile = new File(workingDir, ".jvmopts") FileUtil.writeToFile(jvmOptsFile, jvmOpts.mkString("\n")) } - val settings = new SbtExecutionSettings(null, null, userOpts, hiddenDefaultSize, null, null, null, null, - false, false, false, false, false ,false, true) + val settings = new SbtExecutionSettings(null, null, userOpts, hiddenDefaultSize, null, null, null, None, None, None, null, + false, false, false, false, false, false, true) SbtProcessManager.buildVMParameters(settings, workingDir) } @@ -37,14 +38,14 @@ class MaxJvmHeapParameterTest extends TestCase { def testUserSettingsSmallerThanHiddenDefault(): Unit = { assertEquals( - Seq(superShellDisabled,"-Xmx4g", "-Xms4g", "-Xmx1g"), + Seq(superShellDisabled, "-Xmx4g", "-Xms4g", "-Xmx1g"), buildParamSeq("-Xmx1g")("-Xmx4g", "-Xms4g") ) } def testUserSettingsGreaterThanHiddenDefault(): Unit = { assertEquals( - Seq(superShellDisabled,"-Xmx4g", "-Xms4g", "-Xmx2g"), + Seq(superShellDisabled, "-Xmx4g", "-Xms4g", "-Xmx2g"), buildParamSeq("-Xmx2g")("-Xmx4g", "-Xms4g") ) } diff --git a/scala/scala-impl/src/org/jetbrains/plugins/scala/lang/formatting/settings/ImportsPanel.java b/scala/scala-impl/src/org/jetbrains/plugins/scala/lang/formatting/settings/ImportsPanel.java index 6ba97126430..644198bcbaf 100644 --- a/scala/scala-impl/src/org/jetbrains/plugins/scala/lang/formatting/settings/ImportsPanel.java +++ b/scala/scala-impl/src/org/jetbrains/plugins/scala/lang/formatting/settings/ImportsPanel.java @@ -272,7 +272,7 @@ private static void setValue(final JRadioButton rb, final boolean value) { final JPanel panel3 = new JPanel(); panel3.setLayout(new GridLayoutManager(1, 1, new Insets(0, 0, 0, 0), -1, -1)); panel2.add(panel3, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); - panel3.setBorder(IdeBorderFactory.PlainSmallWithIndent.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.classes.to.use.only.with.prefix"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); + panel3.setBorder(BorderFactory.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.classes.to.use.only.with.prefix"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); myImportsWithPrefixPanel = new JPanel(); myImportsWithPrefixPanel.setLayout(new BorderLayout(0, 0)); panel3.add(myImportsWithPrefixPanel, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); @@ -282,14 +282,14 @@ private static void setValue(final JRadioButton rb, final boolean value) { final JPanel panel5 = new JPanel(); panel5.setLayout(new GridLayoutManager(1, 1, new Insets(0, 0, 0, 0), -1, -1)); panel4.add(panel5, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); - panel5.setBorder(IdeBorderFactory.PlainSmallWithIndent.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.import.layout"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); + panel5.setBorder(BorderFactory.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.import.layout"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); importLayoutPanel = new JPanel(); importLayoutPanel.setLayout(new BorderLayout(0, 0)); panel5.add(importLayoutPanel, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); final JPanel panel6 = new JPanel(); panel6.setLayout(new GridLayoutManager(1, 1, new Insets(0, 0, 0, 0), -1, -1)); panel4.add(panel6, new GridConstraints(1, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); - panel6.setBorder(IdeBorderFactory.PlainSmallWithIndent.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.imports.always.marked.as.used"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); + panel6.setBorder(BorderFactory.createTitledBorder(BorderFactory.createEtchedBorder(), this.$$$getMessageFromBundle$$$("messages/ScalaBundle", "imports.panel.imports.always.marked.as.used"), TitledBorder.DEFAULT_JUSTIFICATION, TitledBorder.DEFAULT_POSITION, null, null)); myAlwaysUsedImportsPanel = new JPanel(); myAlwaysUsedImportsPanel.setLayout(new BorderLayout(0, 0)); panel6.add(myAlwaysUsedImportsPanel, new GridConstraints(0, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); From 3b4fe2cc2161739032616a36fad6009745ba8f0b Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Sat, 18 Mar 2023 20:24:08 +0100 Subject: [PATCH 32/39] bundling some strings --- sbt/sbt-impl/resources/messages/SbtBundle.properties | 4 ++++ .../src/org/jetbrains/sbt/settings/SbtSettingsPane.java | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sbt/sbt-impl/resources/messages/SbtBundle.properties b/sbt/sbt-impl/resources/messages/SbtBundle.properties index 5fe14826c6c..67fc5bded7e 100644 --- a/sbt/sbt-impl/resources/messages/SbtBundle.properties +++ b/sbt/sbt-impl/resources/messages/SbtBundle.properties @@ -169,6 +169,10 @@ sbt.settings.bundled=&Bundled ### org/jetbrains/sbt/settings/SbtSettingsPane.java sbt.settings.choose.custom.launcher=Choose a Custom Launcher sbt.settings.choose.sbt.launch.jar=Choose sbt-launch.jar +sbt.settings.choose.cache.directory=Choose Cache Directory +sbt.settings.choose.local.structure.cache.directory=Choose local sbt structure cache directory +sbt.settings.choose.remote.cache.credentials=Choose Remote Cache Credentials +sbt.settings.choose.remote.structure.cache.credentials.file=Choose remote sbt structure cache credentials file ### org/jetbrains/sbt/shell/SbtProcessManager.scala sbt.shell.disable.version.override=Disable version override diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java index 7643f795a24..3f99bdda918 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java @@ -56,15 +56,15 @@ public SbtSettingsPane(Project project) { FileChooserDescriptorFactory.createSingleLocalFileDescriptor()); myLocalStructureCachePath.addBrowseFolderListener( - "Choose Cache Directory", - "Choose local sbt structure cache directory", + SbtBundle.message("sbt.settings.choose.cache.directory"), + SbtBundle.message("sbt.settings.choose.local.structure.cache.directory"), project, FileChooserDescriptorFactory.createSingleFolderDescriptor() ); myRemoteStructureCacheCredentialsPath.addBrowseFolderListener( - "Choose Remote Cache Credentials", - "Choose remote sbt structure cache credentials file", + SbtBundle.message("sbt.settings.choose.remote.cache.credentials"), + SbtBundle.message("sbt.settings.choose.remote.structure.cache.credentials.file"), project, FileChooserDescriptorFactory.createSingleLocalFileDescriptor() ); From 719d1515e69f0431645978897aa0fae2adbda9d8 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Sun, 19 Mar 2023 01:43:48 +0100 Subject: [PATCH 33/39] IDEA can be allowed to manually download missing deps from cached structure --- .../jetbrains/sbt/settings/SbtSettings.scala | 7 ++ .../project/SbtExternalSystemManager.scala | 3 +- .../sbt/project/SbtProjectResolver.scala | 2 +- .../sbt/project/StructureLocalizer.scala | 80 +++++++++++++------ .../settings/SbtExecutionSettings.scala | 1 + .../sbt/settings/SbtSettingsControl.scala | 5 +- .../sbt/settings/SbtSettingsPane.form | 14 +++- .../sbt/settings/SbtSettingsPane.java | 16 +++- .../sbt/shell/MaxJvmHeapParameterTest.scala | 2 +- 9 files changed, 97 insertions(+), 33 deletions(-) diff --git a/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala b/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala index 93792359e7d..34c03f06517 100644 --- a/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala +++ b/sbt/sbt-api/src/org/jetbrains/sbt/settings/SbtSettings.scala @@ -36,6 +36,7 @@ final class SbtSettings(project: Project) @BeanProperty var localStructureCachePath: String = "" @BeanProperty var remoteStructureCacheUri: String = "" @BeanProperty var remoteStructureCacheCredentialsPath: String = "" + @BeanProperty var allowManualDependencyDownload: Boolean = false override def getState: SbtSettings.State = { val state = new SbtSettings.State @@ -51,6 +52,7 @@ final class SbtSettings(project: Project) state.localStructureCache = localStructureCachePath state.remoteStructureCacheUri = remoteStructureCacheUri state.remoteStructureCacheCredentialsPath = remoteStructureCacheCredentialsPath + state.allowManualDependencyDownload = allowManualDependencyDownload state } @@ -67,6 +69,7 @@ final class SbtSettings(project: Project) localStructureCachePath = state.localStructureCache remoteStructureCacheUri = state.remoteStructureCacheUri remoteStructureCacheCredentialsPath = state.remoteStructureCacheCredentialsPath + allowManualDependencyDownload = state.allowManualDependencyDownload } override def subscribe(listener: ExternalSystemSettingsListener[SbtProjectSettings]): Unit = { @@ -90,6 +93,7 @@ final class SbtSettings(project: Project) localStructureCachePath = settings.localStructureCachePath remoteStructureCacheUri = settings.remoteStructureCacheUri remoteStructureCacheCredentialsPath = settings.remoteStructureCacheCredentialsPath + allowManualDependencyDownload = settings.allowManualDependencyDownload } def getLinkedProjectSettings(module: Module): Option[SbtProjectSettings] = @@ -144,6 +148,9 @@ object SbtSettings { @BeanProperty var remoteStructureCacheCredentialsPath: String = "" + @BeanProperty + var allowManualDependencyDownload: Boolean = false + private val linkedProjectSettings: util.TreeSet[SbtProjectSettings] = new util.TreeSet[SbtProjectSettings] @XCollection(style = XCollection.Style.v1, elementTypes = Array(classOf[SbtProjectSettings])) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala index 959dc07317d..a9f42973b4b 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtExternalSystemManager.scala @@ -119,7 +119,8 @@ object SbtExternalSystemManager { projectSettings.useSbtShellForImport, projectSettings.enableDebugSbtShell, projectSettings.allowSbtVersionOverride, - projectSettings.preferScala2 + projectSettings.preferScala2, + settingsState.allowManualDependencyDownload ) } diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala index 8e3f916c143..8d7b79d5876 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtProjectResolver.scala @@ -219,7 +219,7 @@ class SbtProjectResolver extends ExternalSystemProjectResolver[SbtExecutionSetti case Some(cache) => log.info("sbt structure cache detected") val key = buildDefinitionHash(projectRoot, sbtVersion, settings) - val localizer = new StructureLocalizer(projectRoot) + val localizer = new StructureLocalizer(projectRoot, settings.allowManualDependencyDownload) log.info(s"looking for cached structure under key ${key.hash}") cache.getCachedStructureXml(key) match { diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala index f2a370defd7..08671bf03d3 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/StructureLocalizer.scala @@ -4,11 +4,13 @@ import com.intellij.ide.customize.transferSettings.db.WindowsEnvVariables import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.util.SystemInfo import com.intellij.util.SystemProperties -import org.jetbrains.plugins.scala.extensions.RichFile import org.jetbrains.sbt.structure._ -import java.io.File +import java.io.{File, IOException} import java.net.URI +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest} +import scala.concurrent.TimeoutException import scala.util.control.NoStackTrace object StructureLocalizer { @@ -41,8 +43,10 @@ object StructureLocalizer { private case class LocalFileNotFound(file: File) extends Exception with NoStackTrace + private case class ResolverInfo(uri: URI, cachePath: String) + } -class StructureLocalizer(projectDir: File) { +class StructureLocalizer(projectDir: File, allowManualDependencyDownload: Boolean) { import StructureLocalizer._ @@ -51,6 +55,8 @@ class StructureLocalizer(projectDir: File) { private val projectPath = unix(projectDir.getAbsolutePath).ensureSuffix("/") + private lazy val httpClient = HttpClient.newHttpClient() + def localize(sd: StructureData): Option[StructureData] = try Option(transform(sd)(localize)) catch { case LocalFileNotFound(file) => @@ -61,8 +67,8 @@ class StructureLocalizer(projectDir: File) { def unlocalize(sd: StructureData): StructureData = transform(sd)(unlocalize) - private def transform(data: StructureData)(f: (File, List[String]) => File): StructureData = { - val allLocalCachePaths = data.projects.toList.flatMap(localResolverCaches) + private def transform(data: StructureData)(f: (File, List[ResolverInfo]) => File): StructureData = { + val allLocalCachePaths = data.projects.toList.flatMap(resolverInfos) StructureData( data.sbtVersion, builds = data.builds.map(bd => BuildData( @@ -83,8 +89,8 @@ class StructureLocalizer(projectDir: File) { ) } - private def transform(pd: ProjectData)(f: (File, List[String]) => File): ProjectData = { - val localCachePaths = localResolverCaches(pd) + private def transform(pd: ProjectData)(f: (File, List[ResolverInfo]) => File): ProjectData = { + val localCachePaths = resolverInfos(pd) ProjectData( pd.id, transform(pd.buildURI)(f(_, localCachePaths)), @@ -156,19 +162,22 @@ class StructureLocalizer(projectDir: File) { private def on[T](v: T)(f: T => T): T = f(v) - //TODO: handle URIs with escaped chars and port number - private def localResolverCaches(proj: ProjectData): List[String] = + private def resolverInfos(proj: ProjectData): List[ResolverInfo] = proj.resolvers .filter(rd => rd.root.matches("http(s)?://.*")) .toList.sortBy(_.name) - .map(rd => new URI(rd.root)) - .map(uri => s"$coursierCachePath${uri.getScheme}/${uri.getHost}${uri.getPath}".ensureSuffix("/")) + .map { rd => + val uri = new URI(rd.root) + //TODO: handle URIs with escaped chars and port number + val cache = s"$coursierCachePath${uri.getScheme}/${uri.getHost}${uri.getPath}".ensureSuffix("/") + ResolverInfo(uri, cache) + } - private def unlocalize(f: File, localCachePaths: List[String]): File = { + private def unlocalize(f: File, resolvers: List[ResolverInfo]): File = { var path = unix(f.getAbsolutePath) - path = localCachePaths - .find(path.startsWith) - .fold(path)(resolverPath => s"/$AnyResolverCache/${path.stripPrefix(resolverPath)}") + path = resolvers + .find(r => path.startsWith(r.cachePath)) + .fold(path)(ri => s"/$AnyResolverCache/${path.stripPrefix(ri.cachePath)}") path = if (path.startsWith(coursierCachePath)) s"/$CoursierCache/${path.stripPrefix(coursierCachePath)}" else if (path.startsWith(projectPath)) s"/$ProjectDir/${path.stripPrefix(projectPath)}" @@ -177,7 +186,32 @@ class StructureLocalizer(projectDir: File) { new File(path) } - private def localize(f: File, localCachePaths: List[String]): File = { + private def download(from: URI, to: File): Option[File] = try { + log.info(s"GET $from") + to.getParentFile.mkdirs() + val req = HttpRequest.newBuilder(from).GET.build + val bodyHandler = BodyHandlers.ofFile(to.toPath) + val resp = httpClient.send(req, bodyHandler) + resp.statusCode match { + case 200 => Some(resp.body.toFile) + case 404 => None + case other => + log.warn(s"received http status $other trying to download $from") + None + } + } catch { + case e@(_: IOException | _: TimeoutException) => + log.warn(s"failure trying to download $from", e) + None + } + + private def download(depPath: String, resolver: List[ResolverInfo]): Option[File] = + if (!allowManualDependencyDownload) None + else resolver.iterator + .flatMap(ri => download(ri.uri.resolve(depPath), new File(ri.cachePath + depPath))) + .nextOption() + + private def localize(f: File, resolvers: List[ResolverInfo]): File = { // The 'C:' that we're stripping appears when we create an Unix-style absolute path File on Windows // and call '.getCanonicalPath` on it, e.g. `new File("/foo").getCanonicalPath` yields `C:\foo` // We can't avoid that because `.getCanonicalPath` is used by XML serializer & deserializer of @@ -187,15 +221,13 @@ class StructureLocalizer(projectDir: File) { path = path.stripPrefix("/").stripPrefix("C:") path = unix(path).stripPrefix("/") if (path.startsWith(AnyResolverCache)) { - val files = localCachePaths - .map(resPath => new File(resPath + path.stripPrefix(AnyResolverCache).stripPrefix("/"))) - files - .find { f => - val errorFile = f.getParentFile / s".${f.getName}.error" - f.exists() || errorFile.exists() - } + val pureDependencyPath = path.stripPrefix(AnyResolverCache).stripPrefix("/") + resolvers + .map(ri => new File(ri.cachePath + pureDependencyPath)) + .find(_.exists()) + .orElse(download(pureDependencyPath, resolvers)) .getOrElse { - log.warn(s"none of the files found:\n${files.map(_.getAbsolutePath).mkString("\n")}") + log.warn(s"could not resolve dependency $pureDependencyPath") throw LocalFileNotFound(new File(path)) } } else { diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala index 5fed842c110..5db86c3a065 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/settings/SbtExecutionSettings.scala @@ -24,4 +24,5 @@ class SbtExecutionSettings(val realProjectPath: String, val shellDebugMode: Boolean, val allowSbtVersionOverride: Boolean, val preferScala2: Boolean, + val allowManualDependencyDownload: Boolean, ) extends ExternalSystemExecutionSettings diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala index cfd381706a5..374173d1dec 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsControl.scala @@ -15,7 +15,8 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo pane.getCustomVMPath == settings.customVMPath && pane.getLocalStructureCachePath == settings.localStructureCachePath && pane.getRemoteStructureCacheUri == settings.remoteStructureCacheUri && - pane.getRemoteStructureCacheCredentialsPath == settings.remoteStructureCacheCredentialsPath + pane.getRemoteStructureCacheCredentialsPath == settings.remoteStructureCacheCredentialsPath && + pane.isAllowManualDependencyDownload == settings.allowManualDependencyDownload } override def showUi(show: Boolean): Unit = @@ -36,6 +37,7 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo settings.localStructureCachePath = pane.getLocalStructureCachePath settings.remoteStructureCacheUri = pane.getRemoteStructureCacheUri settings.remoteStructureCacheCredentialsPath = pane.getRemoteStructureCacheCredentialsPath + settings.allowManualDependencyDownload = pane.isAllowManualDependencyDownload } override def reset(): Unit = { @@ -47,6 +49,7 @@ class SbtSettingsControl(settings: SbtSettings) extends ExternalSystemSettingsCo pane.setLocalStructureCachePath(settings.localStructureCachePath) pane.setRemoteStructureCacheUri(settings.remoteStructureCacheUri) pane.setRemoteStructureCacheCredentialsPath(settings.remoteStructureCacheCredentialsPath) + pane.setAllowManualDependencyDownload(settings.allowManualDependencyDownload) } override def validate(settings: SbtSettings) = true diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form index b4a7d9c6ce0..afff69daa74 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.form @@ -1,9 +1,9 @@ - + - + @@ -140,7 +140,7 @@ - + @@ -201,6 +201,14 @@ + + + + + + + + diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java index 3f99bdda918..73a50a5467e 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/settings/SbtSettingsPane.java @@ -35,6 +35,7 @@ public class SbtSettingsPane { private TextFieldWithBrowseButton myLocalStructureCachePath; private JTextField myRemoteStructureCacheUri; private TextFieldWithBrowseButton myRemoteStructureCacheCredentialsPath; + private JCheckBox myAllowManualDependencyDownload; private final Project myProject; @@ -166,6 +167,14 @@ public void setRemoteStructureCacheCredentialsPath(String path) { myRemoteStructureCacheCredentialsPath.setText(path); } + public boolean isAllowManualDependencyDownload() { + return myAllowManualDependencyDownload.isSelected(); + } + + public void setAllowManualDependencyDownload(boolean value) { + myAllowManualDependencyDownload.setSelected(value); + } + /** * Method generated by IntelliJ IDEA GUI Designer * >>> IMPORTANT!! <<< @@ -176,7 +185,7 @@ public void setRemoteStructureCacheCredentialsPath(String path) { private void $$$setupUI$$$() { createUIComponents(); myContentPanel = new JPanel(); - myContentPanel.setLayout(new GridLayoutManager(8, 2, new Insets(0, 0, 0, 0), -1, -1)); + myContentPanel.setLayout(new GridLayoutManager(9, 2, new Insets(0, 0, 0, 0), -1, -1)); final TitledSeparator titledSeparator1 = new TitledSeparator(); titledSeparator1.setText(this.$$$getMessageFromBundle$$$("messages/SbtBundle", "sbt.settings.sbtLauncher")); myContentPanel.add(titledSeparator1, new GridConstraints(3, 0, 1, 2, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_HORIZONTAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); @@ -220,7 +229,7 @@ public void setRemoteStructureCacheCredentialsPath(String path) { titledSeparator3.setText("Structure cache"); myContentPanel.add(titledSeparator3, new GridConstraints(5, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 0, false)); final Spacer spacer2 = new Spacer(); - myContentPanel.add(spacer2, new GridConstraints(7, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_VERTICAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_WANT_GROW, null, null, null, 0, false)); + myContentPanel.add(spacer2, new GridConstraints(8, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_VERTICAL, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_WANT_GROW, GridConstraints.SIZEPOLICY_WANT_GROW, null, null, null, 0, false)); final JPanel panel4 = new JPanel(); panel4.setLayout(new GridLayoutManager(3, 2, new Insets(0, 0, 0, 0), -1, -1)); myContentPanel.add(panel4, new GridConstraints(6, 0, 1, 1, GridConstraints.ANCHOR_CENTER, GridConstraints.FILL_BOTH, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, null, null, null, 1, false)); @@ -239,6 +248,9 @@ public void setRemoteStructureCacheCredentialsPath(String path) { panel4.add(label5, new GridConstraints(2, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_FIXED, GridConstraints.SIZEPOLICY_FIXED, null, null, null, 0, false)); myRemoteStructureCacheCredentialsPath = new TextFieldWithBrowseButton(); panel4.add(myRemoteStructureCacheCredentialsPath, new GridConstraints(2, 1, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, new Dimension(250, -1), null, null, 0, false)); + myAllowManualDependencyDownload = new JCheckBox(); + myAllowManualDependencyDownload.setText("Allow IDEA to download dependencies on behalf of sbt"); + myContentPanel.add(myAllowManualDependencyDownload, new GridConstraints(7, 0, 1, 1, GridConstraints.ANCHOR_WEST, GridConstraints.FILL_NONE, GridConstraints.SIZEPOLICY_CAN_SHRINK | GridConstraints.SIZEPOLICY_CAN_GROW, GridConstraints.SIZEPOLICY_FIXED, null, null, null, 1, false)); label1.setLabelFor(myMaximumHeapSize); ButtonGroup buttonGroup; buttonGroup = new ButtonGroup(); diff --git a/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala b/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala index ce73ff3561b..c583d6f3a0e 100644 --- a/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala +++ b/sbt/sbt-impl/test/org/jetbrains/sbt/shell/MaxJvmHeapParameterTest.scala @@ -24,7 +24,7 @@ class MaxJvmHeapParameterTest extends TestCase { } val settings = new SbtExecutionSettings(null, null, userOpts, hiddenDefaultSize, null, null, null, None, None, None, null, - false, false, false, false, false, false, true) + false, false, false, false, false, false, true, false) SbtProcessManager.buildVMParameters(settings, workingDir) } From 8a8a5fcd3f0f4e8745fef77c8288c6695e5a9363 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 16:21:38 +0100 Subject: [PATCH 34/39] improved error handling --- .../sbt/project/SbtStructureCaches.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala index eaf15e9c866..dae9d0deda6 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala @@ -60,8 +60,8 @@ class RemoteStructureCache( ) extends SbtStructureCache { private val log = Logger.getInstance(getClass) - private lazy val authenticator = credentialsFile.map { file => - new Authenticator { + private lazy val authenticator = credentialsFile.flatMap { file => + try { val (user, password) = { val props = new Properties Using.resource(new FileReader(file))(props.load) @@ -71,9 +71,14 @@ class RemoteStructureCache( require(p != null, "password not found in credentials file") (u, p.toCharArray) } - - override def getPasswordAuthentication: PasswordAuthentication = - new PasswordAuthentication(user, password) + Some(new Authenticator { + override def getPasswordAuthentication: PasswordAuthentication = + new PasswordAuthentication(user, password) + }) + } catch { + case e@(_: IOException | _: IllegalArgumentException) => + log.warn(s"failed to read remote cache credentials file $file", e) + None } } From f8217462d7feef2a6f771d575d4e03c09e26dd79 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 17:08:21 +0100 Subject: [PATCH 35/39] xz compression for remote structure cache --- .../sbt/project/SbtStructureCaches.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala index dae9d0deda6..545357b777c 100644 --- a/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala +++ b/sbt/sbt-impl/src/org/jetbrains/sbt/project/SbtStructureCaches.scala @@ -4,8 +4,9 @@ import com.intellij.openapi.diagnostic.Logger import org.apache.commons.io.FileUtils import org.jetbrains.sbt.project.SbtStructureCache.BuildDefinitionHash import org.jetbrains.sbt.project.settings.SbtExecutionSettings +import org.tukaani.xz.{LZMA2Options, XZInputStream, XZOutputStream} -import java.io.{File, FileReader, IOException} +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, FileReader, IOException} import java.net.http.HttpRequest.BodyPublishers import java.net.http.HttpResponse.BodyHandlers import java.net.http.{HttpClient, HttpRequest} @@ -89,17 +90,19 @@ class RemoteStructureCache( private def fileRequest(key: BuildDefinitionHash): HttpRequest.Builder = HttpRequest - .newBuilder(repositoryUri.resolve(s"$key.xml")) + .newBuilder(repositoryUri.resolve(s"$key.xml.xz")) .timeout(Duration.ofSeconds(30)) def getCachedStructureXml(key: BuildDefinitionHash): Option[String] = try { val request = fileRequest(key).GET().build() - val bodySubscriber = BodyHandlers.ofString(StandardCharsets.UTF_8) + val bodySubscriber = BodyHandlers.ofByteArray() log.info(s"GET ${request.uri()}") val resp = httpClient.send(request, bodySubscriber) resp.statusCode() match { - case 200 => Some(resp.body()) + case 200 => + val is = new XZInputStream(new ByteArrayInputStream(resp.body())) + Some(new String(is.readAllBytes(), StandardCharsets.UTF_8)) case 404 => None case other => log.warn(s"failed to read $key from remote cache, received HTTP status $other") @@ -112,7 +115,9 @@ class RemoteStructureCache( } def cacheStructureXml(key: BuildDefinitionHash, data: String): Unit = try { - val request = fileRequest(key).PUT(BodyPublishers.ofString(data, StandardCharsets.UTF_8)).build() + val baos = new ByteArrayOutputStream + Using.resource(new XZOutputStream(baos, new LZMA2Options))(_.write(data.getBytes(StandardCharsets.UTF_8))) + val request = fileRequest(key).PUT(BodyPublishers.ofByteArray(baos.toByteArray)).build() log.info(s"PUT ${request.uri()}") val resp = httpClient.send(request, BodyHandlers.discarding()) if (resp.statusCode() >= 300) { From efee08e9970a14dddd4651c346c3885e31628693 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 18:13:36 +0100 Subject: [PATCH 36/39] temporary plugin id --- pluginXml/resources/META-INF/plugin.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index 089f46d8e75..ccffeddd713 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -16,10 +16,11 @@ - org.intellij.scala + com.avsystem.scala Scala Temporary customized version by AVSystem Adds support for the Scala language. The following features are available for free with IntelliJ IDEA Community Edition:
    From 851a31199cbff662c9e94505cc7cbc9fff7edac3 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 18:14:12 +0100 Subject: [PATCH 37/39] temporary plugin name --- pluginXml/resources/META-INF/plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index ccffeddd713..a574ebf85b3 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -17,7 +17,7 @@ xmlns:xi="http://www.w3.org/2001/XInclude" require-restart="true"> com.avsystem.scala - Scala + Scala (AVSystem) Temporary customized version by AVSystem From c64a692d2afceed4af1c427e312394d89133a7b9 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 18:15:57 +0100 Subject: [PATCH 38/39] temporary plugin description --- pluginXml/resources/META-INF/plugin.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pluginXml/resources/META-INF/plugin.xml b/pluginXml/resources/META-INF/plugin.xml index a574ebf85b3..f2fd5688d46 100644 --- a/pluginXml/resources/META-INF/plugin.xml +++ b/pluginXml/resources/META-INF/plugin.xml @@ -30,6 +30,10 @@
  • Testing frameworks support (ScalaTest, Specs2, uTest)
  • Scala debugger, worksheets and Ammonite scripts
+ AVSystem experimental extensions: +
    +
  • sbt structure caching
    • +

Support for Play Framework, Akka and Scala.js is enabled in IntelliJ IDEA Ultimate. ]]> From 57dcd90f82f765d823c4cbaec951e8652cb20228 Mon Sep 17 00:00:00 2001 From: Roman Janusz Date: Mon, 20 Mar 2023 18:21:09 +0100 Subject: [PATCH 39/39] temporary plugin name in build.sbt --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4bd89d36885..4575e195186 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ import java.nio.file.Paths // Global build settings -(ThisBuild / intellijPluginName) := "Scala" +(ThisBuild / intellijPluginName) := "Scala (AVSystem)" (ThisBuild / intellijBuild) := Versions.intellijVersion