CARGO: improve fetching actual stdlib data

build.gradle.kts

@@ -41,6 +41,7 @@ val psiViewerPlugin = "PsiViewer:${prop("psiViewerPluginVersion")}"
 val intelliLangPlugin = "org.intellij.intelliLang"
 val copyrightPlugin = "com.intellij.copyright"
 val javaPlugin = "com.intellij.java"
+val javaIdePlugin = "com.intellij.java.ide"
 val javaScriptPlugin = "JavaScript"
 val clionPlugins = listOf("com.intellij.cidr.base", "com.intellij.clion")
 val mlCompletionPlugin = "com.intellij.completion.ml.ranking"
@@ -395,7 +396,11 @@ project(":") {
 project(":idea") {
     intellij {
         version = ideaVersion
-        setPlugins(javaPlugin)
+        setPlugins(
+            javaPlugin,
+            // this plugin registers `com.intellij.ide.projectView.impl.ProjectViewPane` for IDEA that we use in tests
+            javaIdePlugin
+        )
     }
     dependencies {
         implementation(project(":"))

clion/src/test/kotlin/org/rust/clion/RsCLionProjectViewTest.kt

@@ -0,0 +1,10 @@
+/*
+ * Use of this source code is governed by the MIT license that can be
+ * found in the LICENSE file.
+ */
+
+package org.rust.clion
+
+import org.rustSlowTests.RsProjectViewTestBase
+
+class RsCLionProjectViewTest : RsProjectViewTestBase()

idea/src/test/kotlin/org/rust/ide/idea/RsIdeaProjectViewTest.kt

@@ -0,0 +1,10 @@
+/*
+ * Use of this source code is governed by the MIT license that can be
+ * found in the LICENSE file.
+ */
+
+package org.rust.ide.idea
+
+import org.rustSlowTests.RsProjectViewTestBase
+
+class RsIdeaProjectViewTest : RsProjectViewTestBase()

src/main/kotlin/org/rust/cargo/project/workspace/CargoWorkspace.kt

@@ -303,19 +303,24 @@ private class WorkspaceImpl(
                 it.id to (stdPackagesByPackageRoot[it.contentRootUrl]?.id ?: it.id)
             }
             val newStdlibCrates = stdlib.crates.map { it.copy(id = pkgIdMapping.getValue(it.id)) }
-            val newStdlibDependencies = stdlib.dependencies.map { (oldId, dependency) ->
+            val newStdlibDependencies = stdlib.workspaceData.dependencies.map { (oldId, dependency) ->
                 val newDependencies = dependency.mapToSet { it.copy(id = pkgIdMapping.getValue(it.id)) }
                 pkgIdMapping.getValue(oldId) to newDependencies
             }.toMap()
 
             Pair(
                 otherPackagesData + stdPackagesData.map { it.copy(origin = STDLIB) },
-                stdlib.copy(crates = newStdlibCrates, dependencies = newStdlibDependencies)
+                stdlib.copy(
+                    workspaceData = stdlib.workspaceData.copy(
+                        packages = newStdlibCrates,
+                        dependencies = newStdlibDependencies
+                    )
+                )
             )
         }
 
         val stdAll = stdlib.crates.associateBy { it.id }
-        val stdInternalDeps = stdlib.crates.filter { it.origin == DEPENDENCY }.mapToSet { it.id }
+        val stdInternalDeps = stdlib.crates.filter { it.origin == STDLIB_DEPENDENCY }.mapToSet { it.id }
 
         val result = WorkspaceImpl(
             manifestPath,
@@ -341,16 +346,7 @@ private class WorkspaceImpl(
                     pkg.dependencies.addAll(stdlibDependencies.filter { it.name !in explicitDeps && it.pkg.id !in stdInternalDeps })
                 } else {
                     // `pkg` is a crate from stdlib
-                    val stdCrateDeps = stdlib.dependencies[id].orEmpty().mapNotNull { dep ->
-                        val dependencyPackage = newIdToPackage[dep.id] ?: return@mapNotNull null
-                        val depName = dep.name ?: (dependencyPackage.libTarget?.normName ?: dependencyPackage.normName)
-                        DependencyImpl(
-                            dependencyPackage,
-                            depName,
-                            dep.depKinds,
-                        )
-                    }
-                    pkg.dependencies.addAll(stdCrateDeps)
+                    pkg.addDependencies(stdlib.workspaceData, newIdToPackage)
                 }
             }
         }
@@ -434,8 +430,29 @@ private class WorkspaceImpl(
             pkg.cargoEnabledFeatures.map { PackageFeature(pkg, it) }
         }
         for ((feature, state) in inferFeatureState(UserDisabledFeatures.EMPTY)) {
-            if (feature in enabledByCargo != state.isEnabled) {
-                error("Feature `${feature.name}` in package `${feature.pkg.name}` should be ${!state}, but it is $state")
+            when (feature.pkg.origin) {
+                STDLIB -> {
+                    // All features of stdlib package should be considered as enabled by default.
+                    // If `RsExperiments.FETCH_ACTUAL_STDLIB_METADATA` is enabled,
+                    // the plugin invokes `cargo metadata` for `test` crate of stdlib to get actual info.
+                    // And since stdlib packages are not a single workspace,
+                    // `cargo` considers that all stdlib crates except test one are dependencies and
+                    // as a result, enabled only features required by `test` package (i.e. `pkg.cargoEnabledFeatures` are not expected)
+                    // It doesn't make sense to fix `pkg.cargoEnabledFeatures` so let's just adjust this check
+                    if (!state.isEnabled) {
+                        error("Feature `${feature.name}` in package `${feature.pkg.name}` should be ${!state}, but it is $state")
+                    }
+                }
+                // `cargoEnabledFeatures` are not source of truth here when `RsExperiments.FETCH_ACTUAL_STDLIB_METADATA` is enabled
+                // because only `test` crate is considered as part of workspace where all features are enabled by default (see comment above).
+                // As a result, state of stdlib dependencies can differ from `cargoEnabledFeatures`.
+                // Skip check here for now
+                STDLIB_DEPENDENCY -> Unit
+                else -> {
+                    if (feature in enabledByCargo != state.isEnabled) {
+                        error("Feature `${feature.name}` in package `${feature.pkg.name}` should be ${!state}, but it is $state")
+                    }
+                }
             }
         }
     }
@@ -495,33 +512,7 @@ private class WorkspaceImpl(
             // Fill package dependencies
             run {
                 val idToPackage = result.packages.associateBy { it.id }
-                idToPackage.forEach { (id, pkg) ->
-                    val pkgDeps = data.dependencies[id].orEmpty()
-                    val pkgRawDeps = data.rawDependencies[id].orEmpty()
-                    pkg.dependencies += pkgDeps.mapNotNull { dep ->
-                        val dependencyPackage = idToPackage[dep.id] ?: return@mapNotNull null
-
-                        // There can be multiple appropriate raw dependencies because a dependency can be mentioned
-                        // in `Cargo.toml` in different sections, e.g. [dev-dependencies] and [build-dependencies]
-                        val rawDeps = pkgRawDeps.filter { rawDep ->
-                            rawDep.name == dependencyPackage.name && dep.depKinds.any {
-                                it.kind == CargoWorkspace.DepKind.Unclassified ||
-                                    it.target == rawDep.target && it.kind.cargoName == rawDep.kind
-                            }
-                        }
-
-                        val depName = dep.name ?: (dependencyPackage.libTarget?.normName ?: dependencyPackage.normName)
-
-                        DependencyImpl(
-                            dependencyPackage,
-                            depName,
-                            dep.depKinds,
-                            rawDeps.any { it.optional },
-                            rawDeps.any { it.uses_default_features },
-                            rawDeps.flatMap { it.features }.toSet()
-                        )
-                    }
-                }
+                idToPackage.forEach { (_, pkg) -> pkg.addDependencies(data, idToPackage) }
             }
 
             return result
@@ -579,6 +570,33 @@ private class PackageImpl(
     override fun toString() = "Package(name='$name', contentRootUrl='$contentRootUrl', outDirUrl='$outDirUrl')"
 }
 
+private fun PackageImpl.addDependencies(workspaceData: CargoWorkspaceData, packagesMap: Map<PackageId, PackageImpl>) {
+    val pkgDeps = workspaceData.dependencies[id].orEmpty()
+    val pkgRawDeps = workspaceData.rawDependencies[id].orEmpty()
+    dependencies += pkgDeps.mapNotNull { dep ->
+        val dependencyPackage = packagesMap[dep.id] ?: return@mapNotNull null
+
+        // There can be multiple appropriate raw dependencies because a dependency can be mentioned
+        // in `Cargo.toml` in different sections, e.g. [dev-dependencies] and [build-dependencies]
+        val rawDeps = pkgRawDeps.filter { rawDep ->
+            rawDep.name == dependencyPackage.name && dep.depKinds.any {
+                it.kind == CargoWorkspace.DepKind.Unclassified ||
+                    it.target == rawDep.target && it.kind.cargoName == rawDep.kind
+            }
+        }
+
+        val depName = dep.name ?: (dependencyPackage.libTarget?.normName ?: dependencyPackage.normName)
+
+        DependencyImpl(
+            dependencyPackage,
+            depName,
+            dep.depKinds,
+            rawDeps.any { it.optional },
+            rawDeps.any { it.uses_default_features },
+            rawDeps.flatMap { it.features }.toSet()
+        )
+    }
+}
 
 private class TargetImpl(
     override val pkg: PackageImpl,

src/main/kotlin/org/rust/cargo/project/workspace/PackageOrigin.kt

@@ -20,7 +20,12 @@ enum class PackageOrigin {
     WORKSPACE,
 
     /**
-     * Other external dependencies (that are not [WORKSPACE])
+     * External dependency of [WORKSPACE] or other [DEPENDENCY] package
      */
-    DEPENDENCY;
+    DEPENDENCY,
+
+    /**
+     * External dependency of [STDLIB] or other [STDLIB_DEPENDENCY] package
+     */
+    STDLIB_DEPENDENCY
 }

src/main/kotlin/org/rust/cargo/project/workspace/RsAdditionalLibraryRootsProvider.kt

@@ -72,7 +72,7 @@ private val CargoProject.ideaLibraries: Collection<CargoLibrary>
         val dependencyPackages = mutableListOf<CargoWorkspace.Package>()
         for (pkg in workspace.packages) {
             when (pkg.origin) {
-                STDLIB -> stdlibPackages += pkg
+                STDLIB, STDLIB_DEPENDENCY -> stdlibPackages += pkg
                 DEPENDENCY -> dependencyPackages += pkg
                 WORKSPACE -> Unit
             }.exhaustive

src/main/kotlin/org/rust/cargo/project/workspace/StandardLibrary.kt

@@ -31,11 +31,12 @@ import org.rust.stdext.toPath
 import java.nio.file.Path
 
 data class StandardLibrary(
-    val crates: List<CargoWorkspaceData.Package>,
-    val dependencies: Map<PackageId, Set<CargoWorkspaceData.Dependency>>,
+    val workspaceData: CargoWorkspaceData,
     val isHardcoded: Boolean,
     val isPartOfCargoProject: Boolean = false
 ) {
+    val crates: List<CargoWorkspaceData.Package> get() = workspaceData.packages
+
     companion object {
         private val LOG: Logger = logger<StandardLibrary>()
 
@@ -135,7 +136,8 @@ data class StandardLibrary(
             }
 
             if (crates.isEmpty()) return null
-            return StandardLibrary(crates.values.toList(), dependencies, isHardcoded = true)
+            val data = CargoWorkspaceData(crates.values.toList(), dependencies, emptyMap())
+            return StandardLibrary(workspaceData = data, isHardcoded = true)
         }
     }
 }
@@ -181,10 +183,10 @@ private class StdlibDataFetcher private constructor(
         )
         val stdlibWorkspaceData = CargoMetadata.clean(stdlibMetadataProject)
         val stdlibPackages = stdlibWorkspaceData.packages.map {
-            // TODO: introduce PackageOrigin.STDLIB_DEPENDENCY
-            if (it.source == null) it.copy(origin = PackageOrigin.STDLIB) else it
+            val newOrigin = if (it.source == null) PackageOrigin.STDLIB else PackageOrigin.STDLIB_DEPENDENCY
+            it.copy(origin = newOrigin)
         }
-        return StandardLibrary(stdlibPackages, stdlibWorkspaceData.dependencies, isHardcoded = false)
+        return StandardLibrary(stdlibWorkspaceData.copy(packages = stdlibPackages), isHardcoded = false)
     }
 
 
@@ -282,7 +284,7 @@ private class StdlibDataFetcher private constructor(
 
             val stdlibHash = stdlibHash(srcDir, version)
 
-            val stdlibVendor = RsPathManager.pluginDirInSystem().resolve("stdlib/${version.semver.parsedVersion}-$stdlibHash/vendor")
+            val stdlibVendor = RsPathManager.stdlibDependenciesDir().resolve("${version.semver.parsedVersion}-$stdlibHash/vendor")
             if (!stdlibVendor.exists()) {
                 try {
                     // `test` package depends on all other stdlib packages,

src/main/kotlin/org/rust/ide/docs/RsDocumentationProvider.kt

@@ -13,8 +13,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapiext.Testmark
 import com.intellij.openapiext.hitOnFalse
 import com.intellij.psi.*
-import org.rust.cargo.project.workspace.PackageOrigin.DEPENDENCY
-import org.rust.cargo.project.workspace.PackageOrigin.STDLIB
+import org.rust.cargo.project.workspace.PackageOrigin.*
 import org.rust.cargo.util.AutoInjectedCrates.STD
 import org.rust.ide.presentation.presentableQualifiedName
 import org.rust.ide.presentation.presentationInfo
@@ -169,7 +168,7 @@ class RsDocumentationProvider : AbstractDocumentationProvider() {
 
         val pagePrefix = when (origin) {
             STDLIB -> STD_DOC_HOST
-            DEPENDENCY -> {
+            DEPENDENCY, STDLIB_DEPENDENCY -> {
                 val pkg = (element as? RsElement)?.containingCargoPackage ?: return emptyList()
                 // Packages without source don't have documentation at docs.rs
                 if (pkg.source == null) {

src/main/kotlin/org/rust/ide/inspections/import/ui.kt

@@ -152,7 +152,7 @@ private class RsElementCellRenderer : DefaultPsiElementCellRenderer() {
         private fun textWithIcon(): Pair<String, Icon>? {
             val crate = importCandidate?.qualifiedNamedItem?.containingCrate ?: return null
             return when (crate.origin) {
-                PackageOrigin.STDLIB -> crate.normName to RsIcons.RUST
+                PackageOrigin.STDLIB, PackageOrigin.STDLIB_DEPENDENCY -> crate.normName to RsIcons.RUST
                 PackageOrigin.DEPENDENCY -> crate.normName to CargoIcons.ICON
                 else -> null
             }

src/main/kotlin/org/rust/ide/refactoring/RsImportOptimizer.kt

@@ -148,7 +148,7 @@ private class UseItemWrapper(val useItem: RsUseItem) {
         else -> when (basePath?.reference?.resolve()?.containingCrate?.origin) {
             PackageOrigin.WORKSPACE -> 3
             PackageOrigin.DEPENDENCY -> 2
-            PackageOrigin.STDLIB -> 1
+            PackageOrigin.STDLIB, PackageOrigin.STDLIB_DEPENDENCY -> 1
             null -> 3
         }
     }

src/main/kotlin/org/rust/lang/core/CompilerFeature.kt

@@ -50,7 +50,9 @@ class CompilerFeature(
         }
 
         val crate = rsElement.containingCrate ?: return UNKNOWN
-        if (version.channel != RustChannel.NIGHTLY && crate.origin != PackageOrigin.STDLIB) return NOT_AVAILABLE
+        val origin = crate.origin
+        val isStdlibPart = origin == PackageOrigin.STDLIB || origin == PackageOrigin.STDLIB_DEPENDENCY
+        if (version.channel != RustChannel.NIGHTLY && !isStdlibPart) return NOT_AVAILABLE
 
         val cfgEvaluator = CfgEvaluator.forCrate(crate)
         val attrs = RsFeatureIndex.getFeatureAttributes(element.project, name)

src/main/kotlin/org/rust/lang/core/resolve/NameResolution.kt

@@ -300,7 +300,7 @@ fun processExternCrateResolveVariants(
                 NameResolutionTestmarks.shadowingStdCrates.hit()
                 0
             }
-            PackageOrigin.STDLIB -> 1
+            PackageOrigin.STDLIB, PackageOrigin.STDLIB_DEPENDENCY -> 1
         }
     }
     for (dependency in explicitDepsFirst) {

src/main/kotlin/org/rust/lang/utils/evaluation/CfgEvaluator.kt

@@ -164,7 +164,7 @@ class CfgEvaluator(
         fun forCrate(crate: Crate): CfgEvaluator {
             // `cfg(test)` evaluates to true only if there are no `cfg(not(test))` in the package
             val cfgTest = when (crate.origin) {
-                PackageOrigin.STDLIB -> False
+                PackageOrigin.STDLIB, PackageOrigin.STDLIB_DEPENDENCY -> False
 
                 PackageOrigin.DEPENDENCY -> ThreeValuedLogic.fromBoolean(
                     crate.cargoTarget?.pkg?.let { !RsCfgNotTestIndex.hasCfgNotTest(crate.cargoProject.project, it) } ?: false

src/main/kotlin/org/rust/openapiext/RsPathManager.kt

@@ -39,5 +39,6 @@ object RsPathManager {
     }
 
     fun pluginDirInSystem(): Path = Paths.get(PathManager.getSystemPath()).resolve("intellij-rust")
+    fun stdlibDependenciesDir(): Path = pluginDirInSystem().resolve("stdlib")
     fun tempPluginDirInSystem(): Path = Paths.get(PathManager.getTempPath()).resolve("intellij-rust")
 }

src/main/resources-nightly/META-INF/experiments.xml

@@ -12,6 +12,9 @@
         <experimentalFeature id="org.rust.cargo.evaluate.build.scripts" percentOfUsers="0">
             <description>Run build scripts while project refresh. It allows collecting information about generated items like `cfg` options, environment variables, etc</description>
         </experimentalFeature>
+        <experimentalFeature id="org.rust.cargo.fetch.actual.stdlib.metadata" percentOfUsers="100">
+            <description>Fetch metadata of actual stdlib crates instead of using hardcoded data</description>
+        </experimentalFeature>
         <experimentalFeature id="org.rust.macros.new.engine" percentOfUsers="100">
             <description>Enables new macro expansion engine by default</description>
         </experimentalFeature>

src/main/resources-stable/META-INF/experiments.xml

@@ -12,6 +12,9 @@
         <experimentalFeature id="org.rust.cargo.evaluate.build.scripts" percentOfUsers="0">
             <description>Run build scripts while project refresh. It allows collecting information about generated items like `cfg` options, environment variables, etc</description>
         </experimentalFeature>
+        <experimentalFeature id="org.rust.cargo.fetch.actual.stdlib.metadata" percentOfUsers="0">
+            <description>Fetch metadata of actual stdlib crates instead of using hardcoded data</description>
+        </experimentalFeature>
         <experimentalFeature id="org.rust.macros.new.engine" percentOfUsers="100">
             <description>Enables new macro expansion engine by default</description>
         </experimentalFeature>

src/main/resources/META-INF/rust-core.xml

@@ -1073,9 +1073,6 @@
         <experimentalFeature id="org.rust.cargo.features.settings.gutter" percentOfUsers="0">
             <description>Show settings gutter icon near [features] section in `Cargo.toml` where you can enable or disable all features</description>
         </experimentalFeature>
-        <experimentalFeature id="org.rust.cargo.fetch.actual.stdlib.metadata" percentOfUsers="0">
-            <description>Fetch metadata of actual stdlib crates instead of using hardcoded data</description>
-        </experimentalFeature>
 
     </extensions>