Skip to content

Commit

Permalink
Merge pull request #2084 from JetBrains/net211-mte-resolving-packages
Browse files Browse the repository at this point in the history 
Fix issues resolving packages
  • Loading branch information
@citizenmatt
citizenmatt committed May 7, 2021
2 parents cc44905 + af73311 commit 32c7f4d
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 37 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Since 2018.1, the version numbers and release cycle match Rider's versions and r
- Rider: Remove Unity built in package folders from Perforce content roots ([RIDER-61551](https://youtrack.jetbrains.com/issue/RIDER-61551))
- Rider: Only add Unity folders to index for Unity packages ([#2076](https://github.com/JetBrains/resharper-unity/pull/2076))
- Rider: Clean up indexed folders added in previous versions ([#2076](https://github.com/JetBrains/resharper-unity/pull/2076))
- Rider: Fix unresolved git packages when the `packages-lock.json` is disabled ([#2084](https://github.com/JetBrains/resharper-unity/pull/2084))
- Rider: Fix unresolved or incorrectly resolved package used as a dependency when `packages-lock.json` is disabled ([#2084](https://github.com/JetBrains/resharper-unity/pull/2084))



Expand Down
4 changes: 2 additions & 2 deletions resharper/resharper-unity/src/Packages/PackageData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ internal static PackageDetails FromPackageJson([NotNull] PackageJson packageJson
public class GitDetails
{
[NotNull] public readonly string Url;
[NotNull] public readonly string Hash;
[CanBeNull] public readonly string Hash;
[CanBeNull] public readonly string Revision;

public GitDetails([NotNull] string url, [NotNull] string hash, [CanBeNull] string revision)
public GitDetails([NotNull] string url, [CanBeNull] string hash, [CanBeNull] string revision)
{
Url = url;
Hash = hash;
Expand Down
153 changes: 118 additions & 35 deletions resharper/resharper-unity/src/Packages/PackageManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public class PackageManager
private readonly FileSystemPath myPackagesFolder;
private readonly FileSystemPath myPackagesLockPath;
private readonly FileSystemPath myManifestPath;
private readonly FileSystemPath myLocalPackageCacheFolder;

[CanBeNull] private FileSystemPath myLastReadGlobalManifestPath;
[CanBeNull] private EditorManifestJson myGlobalManifest;
Expand All @@ -92,6 +93,7 @@ public class PackageManager
myPackagesFolder = mySolution.SolutionDirectory.Combine("Packages");
myPackagesLockPath = myPackagesFolder.Combine("packages-lock.json");
myManifestPath = myPackagesFolder.Combine("manifest.json");
myLocalPackageCacheFolder = mySolution.SolutionDirectory.Combine("Library/PackageCache");

unitySolutionTracker.IsUnityProject.AdviseUntil(lifetime, value =>
{
Expand All @@ -102,12 +104,9 @@ public class PackageManager
// Track changes to manifest.json and packages-lock.json. Also track changes in the Packages folder, but
// only top level, not recursively. We only want to update the packages if a new package has been added
// or removed
var packagesFolder = mySolution.SolutionDirectory.Combine("Packages");
fileSystemTracker.AdviseFileChanges(lifetime, packagesFolder.Combine("packages-lock.json"),
_ => ScheduleRefresh());
fileSystemTracker.AdviseFileChanges(lifetime, packagesFolder.Combine("manifest.json"),
_ => ScheduleRefresh());
fileSystemTracker.AdviseDirectoryChanges(lifetime, packagesFolder, false, _ => ScheduleRefresh());
fileSystemTracker.AdviseFileChanges(lifetime, myPackagesLockPath, _ => ScheduleRefresh());
fileSystemTracker.AdviseFileChanges(lifetime, myManifestPath, _ => ScheduleRefresh());
fileSystemTracker.AdviseDirectoryChanges(lifetime, myPackagesFolder, false, _ => ScheduleRefresh());
// We're all set up, terminate the advise
return true;
Expand Down Expand Up @@ -308,6 +307,9 @@ private List<PackageData> GetPackagesFromManifestJson()
myGlobalManifest = SafelyReadGlobalManifestFile(globalManifestPath);
}

// TODO: Support registry scopes
// Not massively important. We need the registry for a pre-2018.3 cache folder, which I think predates
// scopes. Post 2018.3, we should get the package from the project local cache
var registry = projectManifest.Registry ?? DefaultRegistryUrl;

var packages = new Dictionary<string, PackageData>();
Expand All @@ -321,8 +323,7 @@ private List<PackageData> GetPackagesFromManifestJson()
lockDetails);
}

// From observation, Unity treats package folders in the Packages folder as actual packages, even if they're
// not registered in manifest.json. They must have a */package.json file, in the root of the package itself
// If a child folder of Packages has a package.json file, then it's a package
foreach (var child in myPackagesFolder.GetChildDirectories())
{
// The folder name is not reliable to act as ID, so we'll use the ID from package.json. All other
Expand All @@ -333,18 +334,41 @@ private List<PackageData> GetPackagesFromManifestJson()
packages[packageData.Id] = packageData;
}

// Calculate the transitive dependencies. Based on observation, we simply go with the highest available
// We currently have the project dependencies. These will usually be the version requested, and will
// therefore have package data, as long as that data exists in the cache. However, a transitive
// dependency might get resolved to a higher version, so the project dependency version won't be in the
// cache, and that package data will be missing.

// Let's calculate the transitive dependencies.
// This is a very naive implementation, initially based on observation. UPM will try to resolve
// dependencies based on a resolution strategy. Note that this is not a conflict resolution strategy. It
// applies to all dependencies, even if there is only usage of that package.
// The default resolution strategy is "lowest". For a single package, this means get that version. For
// multiple packages it means get the lowest version that meets all version requirements, which
// translates to the highest common version.
// With one of the "highest*" resolution strategies, UPM will choose the highest patch, minor or major
// version that's available on the server. E.g. if two packages require dependency A@1.0.0 and A@1.0.5,
// then UPM can resolve this to A@1.0.7 or A@1.1.0 or A@20.0.0. This causes us problems because we don't
// have that information (although it is cached elsewhere on disk). If this dependency is used as a
// project dependency, then it also updates the project dependency.
// We fake "highest*" resolution by getting whatever version is available in Library/PackagesCache.
var packagesToProcess = new List<PackageData>(packages.Values);
while (packagesToProcess.Count > 0)
{
var foundDependencies = GetPackagesFromDependencies(registry, builtInPackagesFolder,
packages, packagesToProcess);
var foundDependencies = GetPackagesFromDependencies(registry, packages, packagesToProcess);
foreach (var package in foundDependencies)
packages[package.Id] = package;

packagesToProcess = foundDependencies;
}

// TODO: Strip unused packages
// There is a chance we have introduced an extra package via a dependency that is subsequently updated.
// E.g. a dependency introduces A@1.0.0 which introduces B@1.0.0. If we have another package that
// depends on A@2.0.0 which no longer uses B, then we have an orphaned package
// This is an unlikely edge case, as it means we'd have to resolve the old version correctly as well as
// the new one. And the worst that can happen is we show an extra package in the UI

return new List<PackageData>(packages.Values);
}
catch (Exception e)
Expand Down Expand Up @@ -454,37 +478,35 @@ private EditorManifestJson SafelyReadGlobalManifestFile(FileSystemPath globalMan
[CanBeNull]
private PackageData GetEmbeddedPackage(string id, string filePath)
{
// Embedded packages live in the Packages folder. When reading from packages-lock.json, the filePath has a
// 'file:' prefix. We make sure it's the folder name when there is no packages-lock.json
// Embedded packages live in the Packages folder. When reading from manifest.json, filePath is the same as
// ID. When reading from packages-lock.json, we already know it's an embedded folder, and use the version,
// which has a 'file:' prefix
var packageFolder = myPackagesFolder.Combine(filePath.TrimFromStart("file:"));
return GetPackageDataFromFolder(id, packageFolder, PackageSource.Embedded);
}

[CanBeNull]
private PackageData GetRegistryPackage(string id, string version, string registryUrl)
{
// The version parameter isn't necessarily a version, and might not parse correctly. When using
// manifest.json to load packages, we will try to match a registry package before we try to match a git
// package, so the version might even be a URL
// When parsing manifest.json, version might be a version, or it might even be a URL for a git package
var cacheFolder = RelativePath.TryParse($"{id}@{version}");
if (cacheFolder.IsEmpty)
return null;

// Unity 2018.3 introduced an additional layer of caching for registry based packages, local to the
// project, so that any edits to the files in the package only affect this project. This is primarily
// for the API updater, which would otherwise modify files in the product wide cache
var packageCacheFolder = mySolution.SolutionDirectory.Combine("Library/PackageCache");
var packageFolder = packageCacheFolder.Combine(cacheFolder);
var packageData = GetPackageDataFromFolder(id, packageFolder, PackageSource.Registry);
var packageData = GetPackageDataFromFolder(id, myLocalPackageCacheFolder.Combine(cacheFolder),
PackageSource.Registry);
if (packageData != null)
return packageData;

// Fall back to the product wide cache
packageCacheFolder = UnityCachesFinder.GetPackagesCacheFolder(registryUrl);
var packageCacheFolder = UnityCachesFinder.GetPackagesCacheFolder(registryUrl);
if (packageCacheFolder == null || !packageCacheFolder.ExistsDirectory)
return null;

packageFolder = packageCacheFolder.Combine(cacheFolder);
var packageFolder = packageCacheFolder.Combine(cacheFolder);
return GetPackageDataFromFolder(id, packageFolder, PackageSource.Registry);
}

Expand Down Expand Up @@ -516,22 +538,33 @@ private PackageData GetBuiltInPackage(string id, string version, FileSystemPath
private PackageData GetGitPackage(string id, string version, [CanBeNull] string hash,
[CanBeNull] string revision = null)
{
// If we don't have a hash, we know this isn't a git package
if (hash == null)
// For older Unity versions, manifest.json will have a hash for any git based package. For newer Unity
// versions, this is stored in packages-lock.json. If the lock file is disabled, then we don't get a hash
// and have to figure it out based on whatever is in Library/PackagesCache. We check the vesion as a git
// URL based on the docs: https://docs.unity3d.com/Manual/upm-git.html
if (hash == null && !IsGitUrl(version))
return null;

// This must be a git package, make sure we return something
try
{
var packageFolder = mySolution.SolutionDirectory.Combine($"Library/PackageCache/{id}@{hash}");
if (!packageFolder.ExistsDirectory)
var packageFolder = myLocalPackageCacheFolder.Combine($"{id}@{hash}");
if (!packageFolder.ExistsDirectory && hash != null)
{
var shortHash = hash.Substring(0, Math.Min(hash.Length, 10));
packageFolder = mySolution.SolutionDirectory.Combine($"Library/PackageCache/{id}@{shortHash}");
packageFolder = myLocalPackageCacheFolder.Combine($"{id}@{shortHash}");
}

return GetPackageDataFromFolder(id, packageFolder, PackageSource.Git,
new GitDetails(version, hash, revision));
if (!packageFolder.ExistsDirectory)
packageFolder = myLocalPackageCacheFolder.GetChildDirectories($"{id}@*").FirstOrDefault();

if (packageFolder != null && packageFolder.ExistsDirectory)
{
return GetPackageDataFromFolder(id, packageFolder, PackageSource.Git,
new GitDetails(version, hash, revision));
}

return null;
}
catch (Exception e)
{
Expand All @@ -540,6 +573,13 @@ private PackageData GetBuiltInPackage(string id, string version, FileSystemPath
}
}

private static bool IsGitUrl(string version)
{
return Uri.TryCreate(version, UriKind.Absolute, out var url) &&
(url.Scheme.StartsWith("git+") ||
url.AbsolutePath.EndsWith(".git", StringComparison.InvariantCultureIgnoreCase));
}

[CanBeNull]
private PackageData GetLocalPackage(string id, string version)
{
Expand Down Expand Up @@ -586,7 +626,7 @@ private PackageData GetLocalTarballPackage(string id, string version)
var timestamp = (long) (tarballPath.FileModificationTimeUtc - DateTimeEx.UnixEpoch).TotalMilliseconds;
var hash = GetMd5OfString(tarballPath.FullPath).Substring(0, 12).ToLowerInvariant();

var packageFolder = mySolution.SolutionDirectory.Combine($"Library/PackageCache/{id}@{hash}-{timestamp}");
var packageFolder = myLocalPackageCacheFolder.Combine($"{id}@{hash}-{timestamp}");
var tarballLocation = tarballPath.StartsWith(mySolution.SolutionDirectory)
? tarballPath.RemovePrefix(mySolution.SolutionDirectory.Parent)
: tarballPath;
Expand Down Expand Up @@ -652,7 +692,6 @@ private static string GetMd5OfString(string value)
}

private List<PackageData> GetPackagesFromDependencies([NotNull] string registry,
[NotNull] FileSystemPath builtInPackagesFolder,
Dictionary<string, PackageData> resolvedPackages,
List<PackageData> packagesToProcess)
{
Expand All @@ -664,8 +703,7 @@ private static string GetMd5OfString(string value)
{
foreach (var (id, versionString) in packageData.PackageDetails.Dependencies)
{
// Embedded packages take precedence over any version
if (IsEmbeddedPackage(id, resolvedPackages))
if (DoesResolvedPackageTakePrecedence(id, resolvedPackages))
continue;

if (!JetSemanticVersion.TryParse(versionString, out var dependencyVersion))
Expand All @@ -678,20 +716,65 @@ private static string GetMd5OfString(string value)
}
}

ICollection<FileSystemPath> cachedPackages = null;
var newPackages = new List<PackageData>();
foreach (var (id, version) in dependencies)
{
if (version > GetResolvedVersion(id, resolvedPackages))
newPackages.Add(GetPackageData(id, version.ToString(), registry, builtInPackagesFolder, null));
{
if (cachedPackages == null) cachedPackages = myLocalPackageCacheFolder.GetChildDirectories();

// We know this is a registry package, so try to get it from the local cache. It might be missing:
// 1) the cache hasn't been built yet
// 2) the package has been resolved with one of the "highest*" strategies and a newer version has
// been downloaded from the UPM server. Check for any "id@" folders, and use that as the version.
// If it's in the local cache, it's the (last) resolved version. If Unity isn't running and the
// manifest is out of date, we can only do a best effort attempt at showing the right packages.
// We need Unity to resolve. We'll refresh once Unity has started again.
// So:
// 1) Check for the exact version in the local cache
// 2) Check for any version in the local cache
// 3) Check for the exact version in the global cache
PackageData packageData = null;
var exact = $"{id}@{version}";
var prefix = $"{id}@";
foreach (var packageFolder in cachedPackages)
{
if (packageFolder.Name.Equals(exact, StringComparison.InvariantCultureIgnoreCase)
|| packageFolder.Name.StartsWith(prefix, StringComparison.InvariantCultureIgnoreCase))
{
packageData = GetPackageDataFromFolder(id, packageFolder, PackageSource.Registry);
if (packageData != null)
break;
}
}

if (packageData == null)
{
var packageFolder = UnityCachesFinder.GetPackagesCacheFolder(registry)?.Combine(exact);
if (packageFolder != null)
packageData = GetPackageDataFromFolder(id, packageFolder, PackageSource.Registry);
}

if (packageData != null)
newPackages.Add(packageData);
}
}

return newPackages;
}

private static bool IsEmbeddedPackage(string id, Dictionary<string, PackageData> resolvedPackages)
private static bool DoesResolvedPackageTakePrecedence(string id,
IReadOnlyDictionary<string, PackageData> resolvedPackages)
{
// Some package types take precedence over any requests for another version. Basically any package that is
// built in or pointing at actual files
return resolvedPackages.TryGetValue(id, out var packageData) &&
packageData.Source == PackageSource.Embedded;
(packageData.Source == PackageSource.Embedded
|| packageData.Source == PackageSource.BuiltIn
|| packageData.Source == PackageSource.Git
|| packageData.Source == PackageSource.Local
|| packageData.Source == PackageSource.LocalTarball);
}

private static JetSemanticVersion GetCurrentMaxVersion(
Expand Down

0 comments on commit 32c7f4d

Please sign in to comment.