Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PackageManager: Store packages hierarchically, by version #2610

Merged
merged 1 commit into from
Mar 7, 2023
Merged

PackageManager: Store packages hierarchically, by version #2610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions changelog/hierarchy.dd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
The way packages are stored internally has changed

Previous versions of dub stored packages in the following format:
`$CACHE_PATH/$PACKAGE_NAME-$PACKAGE_VERSION/$PACKAGE_NAME/`
Starting from this version, the format will be:
`$CACHE_PATH/$PACKAGE_NAME/$PACKAGE_VERSION/$PACKAGE_NAME`.

Introducing a new level will help users quickly list what packages
they actually have installed, and reduce visibility of packages that
might update frequently. It will render various commands (e.g. `du`)
more useful, pave the way for a package GC function, and make manual
browsing easier.

More importantly, it will allow future version of dub to infer the
version from the path to the package, removing the need to read (or edit)
the recipe file on every dub invocation.
61 changes: 35 additions & 26 deletions source/dub/packagemanager.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1298,29 +1298,8 @@ private struct Location {
if (!path.existsDirectory())
return;

logDebug("iterating dir %s", path.toNativeString());
try foreach (pdir; iterateDirectory(path)) {
logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
if (!pdir.isDirectory) continue;

// Old / flat directory structure, used in non-standard path
// Packages are stored in $ROOT/$SOMETHING/`
auto pack_path = path ~ (pdir.name ~ "/");
auto packageFile = Package.findPackageFile(pack_path);

// New (since 2015) managed structure:
// $ROOT/$NAME-$VERSION/$NAME
// This is the most common code path
if (mgr.isManagedPath(path) && packageFile.empty) {
foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/")))
if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {
pack_path ~= subdir.name ~ "/";
packageFile = Package.findPackageFile(pack_path);
break;
}
}

if (packageFile.empty) continue;
void loadInternal (NativePath pack_path, NativePath packageFile)
{
Package p;
try {
foreach (pp; existing_packages)
Expand All @@ -1338,6 +1317,38 @@ private struct Location {
logDiagnostic("Full error: %s", e.toString().sanitize());
}
}

logDebug("iterating dir %s", path.toNativeString());
try foreach (pdir; iterateDirectory(path)) {
logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
if (!pdir.isDirectory) continue;

// Old / flat directory structure, used in non-standard path
// Packages are stored in $ROOT/$SOMETHING/`
const pack_path = path ~ (pdir.name ~ "/");
auto packageFile = Package.findPackageFile(pack_path);
if (!packageFile.empty) {
// Deprecated unmanaged directory structure
logWarn("Package at path '%s' should be under '%s'",
pack_path.toNativeString().color(Mode.bold),
(pack_path ~ "$VERSION" ~ pdir.name).toNativeString().color(Mode.bold));
logWarn("The package will no longer be detected starting from v1.42.0");
loadInternal(pack_path, packageFile);
}

// Managed structure: $ROOT/$NAME/$VERSION/$NAME
// This is the most common code path
else if (mgr.isManagedPath(path)) {
// Iterate over versions of a package
foreach (versdir; iterateDirectory(pack_path)) {
if (!versdir.isDirectory) continue;
auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/");
if (!vers_path.existsDirectory()) continue;
packageFile = Package.findPackageFile(vers_path);
loadInternal(vers_path, packageFile);
}
}
}
catch (Exception e)
logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
}
Expand Down Expand Up @@ -1417,9 +1428,7 @@ private struct Location {
*/
private NativePath getPackagePath (string name, string vers)
{
// + has special meaning for Optlink
string clean_vers = vers.chompPrefix("~").replace("+", "_");
NativePath result = this.packagePath ~ (name ~ "-" ~ clean_vers);
NativePath result = this.packagePath ~ name ~ vers;
result.endsWithSlash = true;
return result;
}
Expand Down
4 changes: 2 additions & 2 deletions test/dpath-variable.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ rm -rf "$DPATH"
cd "${CURR_DIR}/dpath-variable"
"${DUB}" upgrade

if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
die $LINENO 'Did not get dependencies installed into $DPATH.'
fi

Expand All @@ -24,6 +24,6 @@ trap cleanup EXIT

"${DUB}" upgrade

if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
die $LINENO 'Did not get dependencies installed into dubHome (set from config).'
fi
18 changes: 9 additions & 9 deletions test/interactive-remove.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,32 @@
# we need to nuke every `dub` version in the user cache...
$DUB remove dub -n || true

$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]

echo 1 | $DUB remove dub | tr -d '\n' | grep --ignore-case 'select.*1\.9\.0.*1\.10\.0.*'
if [ -d $HOME/.dub/packages/dub-1.9.0/dub ]; then
if [ -d $HOME/.dub/packages/dub/1.9.0/dub ]; then
die $LINENO 'Failed to remove dub-1.9.0'
fi

$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
# EOF aborts remove
echo -xn '' | $DUB remove dub
if [ ! -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
if [ ! -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
die $LINENO 'Aborted dub still removed a package'
fi

# validates input
echo -e 'abc\n4\n-1\n3' | $DUB remove dub
if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
die $LINENO 'Failed to remove all version of dub'
fi

$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
# is non-interactive with a <version-spec>
$DUB remove dub@1.9.0
$DUB remove dub@1.10.0
if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
die $LINENO 'Failed to non-interactively remove specified versions'
fi
0 ...m-cache-paths/cache/foo-1.0.0/foo/dub.sdl → ...m-cache-paths/cache/foo/1.0.0/foo/dub.sdl
View file
Open in desktop
File renamed without changes.
6 changes: 3 additions & 3 deletions test/version-spec.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ $DUB add-local "$CURR_DIR/version-spec/oldfoo"
$DUB remove-local "$CURR_DIR/version-spec/newfoo"
$DUB remove-local "$CURR_DIR/version-spec/oldfoo"

$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
$DUB remove dub@1.9.0
$DUB remove dub=1.10.0
if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
die $LINENO 'Failed to remove specified versions'
fi