Skip to content
Permalink
Browse files

Improve storages, packages, source download destinations.

  • Loading branch information...
egorpugin committed Aug 23, 2019
1 parent 3924f8e commit 3e8f7850c68a36ba82179e788bf7c8a5709d0cce
@@ -544,7 +544,7 @@ static void isolated_build1(sw::SwContext &swctx)
s->applyVersion(pkg.getVersion());
if (srcs.find(s->getHash()) != srcs.end())
continue;
srcs[s->getHash()] = fs::current_path();
srcs[s->getHash()].requested_dir = fs::current_path();
}

LOG_INFO(logger, "Copying files");
@@ -66,7 +66,7 @@ static sw::SourceDirMap getSources(sw::SwContext &swctx)
s->applyVersion(pkg.getVersion());
if (srcs.find(s->getHash()) != srcs.end())
continue;
srcs[s->getHash()] = d / s->getHash();
srcs[s->getHash()].root_dir = d / s->getHash();
sources.emplace(std::move(s));
}

@@ -89,7 +89,7 @@ std::pair<sw::SourceDirMap, const sw::Input &> fetch(sw::SwBuild &b)

auto ts = createInitialSettings(b.getContext());
for (auto &[h, d] : srcs)
ts["driver"]["source-dir-for-source"][h] = normalize_path(d);
ts["driver"]["source-dir-for-source"][h] = normalize_path(d.getRequestedDirectory());

auto &ii = getInput(b);
sw::InputWithSettings i(ii);
@@ -42,7 +42,7 @@ SUBCOMMAND_DECL(open)
auto &p = pkgs.find(open_arg)->second;

#ifdef _WIN32
if (sdb.isPackageInstalled(p))
if (sdb.isPackageInstalled(*p))
{
auto p = swctx->resolve(open_arg);

@@ -68,7 +68,7 @@ SUBCOMMAND_DECL(open)
}
else
{
LOG_INFO(logger, "Package '" + p.toString() + "' not installed");
LOG_INFO(logger, "Package '" + p->toString() + "' not installed");
}
#endif
}
@@ -77,7 +77,7 @@ sw::PackageDescriptionMap getPackages(const sw::SwBuild &b, const sw::SourceDirM
auto si = sources.find(src->getHash());
if (si == sources.end())
throw SW_RUNTIME_ERROR("no such source");
rd = si->second;
rd = si->second.getRequestedDirectory();
}
j["root_dir"] = normalize_path(rd);

@@ -277,12 +277,12 @@ void SwContext::loadEntryPoints(const std::set<Input*> &inputs, bool set_eps)
// which are located in this config AND which are deps of this input package id
auto m = resolve(UnresolvedPackages{ inputs[i].getPackageId() });
auto &p = m.find(inputs[i].getPackageId())->second;
for (auto &d : p.getData().dependencies)
for (auto &d : p->getData().dependencies)
{
auto &p2 = m.find(d)->second;
if (p2.getData().group_number != p.getData().group_number)
if (p2->getData().group_number != p->getData().group_number)
continue;
getTargetData(p2).setEntryPoint(ep);
getTargetData(*p2).setEntryPoint(ep);
}
}
}
@@ -153,7 +153,7 @@ std::optional<path> Build::getSourceDir(const Source &s, const Version &v) const
s2->applyVersion(v);
auto i = source_dirs_by_source.find(s2->getHash());
if (i != source_dirs_by_source.end())
return i->second;
return i->second.getRequestedDirectory();
return {};
}

@@ -219,7 +219,7 @@ std::vector<ITargetPtr> NativeTargetEntryPoint::loadPackages(SwBuild &swb, const

b.DryRun = settings["driver"]["dry-run"] == "true";
for (auto &[h, d] : settings["driver"]["source-dir-for-source"].getSettings())
b.source_dirs_by_source[h] = d.getValue();
b.source_dirs_by_source[h].requested_dir = d.getValue();
for (auto &[pkg, p] : settings["driver"]["source-dir-for-package"].getSettings())
b.source_dirs_by_package[pkg] = p.getValue();
settings.erase("driver");
@@ -339,14 +339,15 @@ void Target::fetch()
s2->applyVersion(getPackage().getVersion());
s2->download(d);
}
fetched_dirs[s2->getHash()].root_dir = d;
d = d / findRootDirectory(d);
setSourceDirectory(d);

fetched_dirs.emplace(s2->getHash(), d);
fetched_dirs[s2->getHash()].requested_dir = d;
}
else
{
setSourceDirectory(i->second);
setSourceDirectory(i->second.getRequestedDirectory());
}
}

@@ -52,7 +52,7 @@ struct SW_MANAGER_API Database
void setValue(const String &key, const T &v) const;
};

struct SW_MANAGER_API PackagesDatabase : public Database
struct SW_MANAGER_API PackagesDatabase : Database
{
PackagesDatabase(const path &db_fn);
~PackagesDatabase();
@@ -35,75 +35,62 @@ String getSourceDirectoryName()
return "sdir";
}

Package::Package(const Storage &storage, const String &s)
: storage(storage), PackageId(s)
Package::Package(const IStorage &storage, const PackagePath &p, const Version &v)
: storage(storage), PackageId(p, v)
{
}

Package::Package(const Storage &storage, const PackagePath &p, const Version &v)
: storage(storage), PackageId(p, v)
Package::Package(const IStorage &storage, const PackageId &id)
: storage(storage), PackageId(id)
{
}

Package::Package(const Storage &storage, const PackageId &id)
: storage(storage), PackageId(id)
Package::Package(const Package &rhs)
: PackageId(rhs), storage(rhs.storage), data(rhs.data ? rhs.data->clone() : nullptr)
{
}

String Package::getHash() const
{
// move these calculations to storage?
switch (storage.getHashSchemaVersion())
switch (storage.getSchema().getHashVersion())
{
case 1:
return blake2b_512(getPath().toStringLower() + "-" + getVersion().toString());
}

throw SW_RUNTIME_ERROR("Unknown hash schema version: " + std::to_string(storage.getHashSchemaVersion()));
throw SW_RUNTIME_ERROR("Unknown hash schema version: " + std::to_string(storage.getSchema().getHashVersion()));
}

path Package::getHashPath() const
{
// move these calculations to storage?
switch (storage.getHashPathFromHashSchemaVersion())
switch (storage.getSchema().getHashPathFromHashVersion())
{
case 1:
return ::sw::getHashPathFromHash(getHash(), 4, 2); // remote consistent storage paths
case 2:
return ::sw::getHashPathFromHash(getHashShort(), 2, 2); // local storage is more relaxed
}

throw SW_RUNTIME_ERROR("unreachable");
throw SW_RUNTIME_ERROR("Unknown hash path schema version: " + std::to_string(storage.getSchema().getHashPathFromHashVersion()));
}

String Package::getHashShort() const
{
return shorten_hash(getHash(), 8);
}

/*void Package::setData(const PackageData &d) const
{
data = std::make_unique<PackageData>(d);
}*/

const PackageData &Package::getData() const
{
return storage.loadData(*this);
if (!data)
data = std::move(storage.loadData(*this));
return *data;
}

/*LocalPackage Package::download() const
{
return storage.download(*this);
}*/

/*LocalPackage Package::install() const
{
return storage.install(*this);
}*/

LocalPackage::LocalPackage(const LocalStorage &storage, const String &s)
: Package(storage, s)
const IStorage &Package::getStorage() const
{
return storage;
}

LocalPackage::LocalPackage(const LocalStorage &storage, const PackagePath &p, const Version &v)
@@ -118,7 +105,7 @@ LocalPackage::LocalPackage(const LocalStorage &storage, const PackageId &id)

const LocalStorage &LocalPackage::getLocalStorage() const
{
return static_cast<const LocalStorage &>(storage);
return static_cast<const LocalStorage &>(getStorage());
}

bool LocalPackage::isOverridden() const
@@ -12,7 +12,7 @@
namespace sw
{

struct Storage;
struct IStorage;
struct LocalStorage;

struct PackageData
@@ -41,45 +41,57 @@ struct PackageData

//
// PackageId driver

virtual ~PackageData() = default;

virtual std::unique_ptr<PackageData> clone() const { return std::make_unique<PackageData>(*this); }
};

using PackageDataPtr = std::unique_ptr<PackageData>;

struct LocalPackage;
struct OverriddenPackagesStorage;

struct SW_MANAGER_API Package : PackageId
{
const Storage &storage;
Package(const IStorage &, const PackagePath &, const Version &);
Package(const IStorage &, const PackageId &);

Package(const Storage &, const String &);
Package(const Storage &, const PackagePath &, const Version &);
Package(const Storage &, const PackageId &);
Package(const Package &) = default;
Package &operator=(const Package &) = default;
Package(const Package &);
Package &operator=(const Package &) = delete;
Package(Package &&) = default;
Package &operator=(Package &&) = default;
~Package() = default;
virtual ~Package() = default;

String getHash() const;
String getHashShort() const;
path getHashPath() const;

//void setData(const PackageData &) const;
const PackageData &getData() const;
//LocalPackage download(file type) const;
//LocalPackage install() const;
const IStorage &getStorage() const;

virtual std::unique_ptr<Package> clone() const { return std::make_unique<Package>(*this); }

private:
const IStorage &storage;
mutable PackageDataPtr data;
};

using PackagePtr = std::unique_ptr<Package>;
//using Packages = std::unordered_set<Package>;

struct SW_MANAGER_API LocalPackage : Package
{
LocalPackage(const LocalStorage &, const String &);
LocalPackage(const LocalStorage &, const PackagePath &, const Version &);
LocalPackage(const LocalStorage &, const PackageId &);

LocalPackage(const LocalPackage &) = default;
LocalPackage &operator=(const LocalPackage &) = default;
LocalPackage &operator=(const LocalPackage &) = delete;
LocalPackage(LocalPackage &&) = default;
LocalPackage &operator=(LocalPackage &&) = default;
~LocalPackage() = default;
virtual ~LocalPackage() = default;

virtual std::unique_ptr<Package> clone() const { return std::make_unique<LocalPackage>(*this); }

bool isOverridden() const;
std::optional<path> getOverriddenDir() const;
@@ -102,7 +114,10 @@ struct SW_MANAGER_API LocalPackage : Package
const LocalStorage &getLocalStorage() const;
};

using Packages = std::unordered_set<Package>;
using LocalPackagePtr = std::unique_ptr<LocalPackage>;

SW_MANAGER_API
String getSourceDirectoryName();

}

@@ -45,9 +45,6 @@ using PackageIdSet = std::unordered_set<PackageId>;
SW_MANAGER_API
PackageId extractPackageIdFromString(const String &target);

SW_MANAGER_API
String getSourceDirectoryName();

}

namespace std
@@ -32,6 +32,12 @@ bool Git::isValid()
return i == 1;
}

void detail::DownloadData::remove() const
{
fs::remove_all(root_dir);
fs::remove(stamp_file);
}

bool download(const std::unordered_set<SourcePtr> &sset, SourceDirMap &source_dirs, const SourceDownloadOptions &opts)
{
std::atomic_bool downloaded = false;
@@ -41,24 +47,25 @@ bool download(const std::unordered_set<SourcePtr> &sset, SourceDirMap &source_di
{
fs.push_back(e.push([src = src.get(), &d = source_dirs[src->getHash()], &opts, &downloaded]
{
path t = d;
auto &t = d.stamp_file;
t = d.root_dir;
t += ".stamp";

auto dl = [&src, d, &t, &downloaded]()
{
downloaded = true;
LOG_INFO(logger, "Downloading source:\n" << src->print());
src->download(d);
src->download(d.root_dir);
write_file(t, timepoint2string(getUtc()));
};

if (!fs::exists(d))
if (!fs::exists(d.root_dir))
{
dl();
}
else if (!opts.ignore_existing_dirs)
{
throw SW_RUNTIME_ERROR("Directory exists " + normalize_path(d) + " for source " + src->print());
throw SW_RUNTIME_ERROR("Directory exists " + normalize_path(d.root_dir) + " for source " + src->print());
}
else
{
@@ -73,13 +80,14 @@ bool download(const std::unordered_set<SourcePtr> &sset, SourceDirMap &source_di
{
if (e)
LOG_INFO(logger, "Download data is stale, re-downloading");
fs::remove_all(d);
fs::remove_all(d.root_dir);
dl();
}
}
}
}
d.requested_dir = d.root_dir;
if (opts.adjust_root_dir)
d = d / findRootDirectory(d); // pass found regex or files for better root dir lookup
d.requested_dir /= findRootDirectory(d.requested_dir); // pass found regex or files for better root dir lookup
}));
}
waitAndGet(fs);
@@ -90,7 +98,7 @@ SourceDirMap download(const std::unordered_set<SourcePtr> &sset, const SourceDow
{
SourceDirMap sources;
for (auto &s : sset)
sources[s->getHash()] = opts.root_dir.empty() ? get_temp_filename("dl") : (opts.root_dir / s->getHash());
sources[s->getHash()].root_dir = opts.root_dir.empty() ? get_temp_filename("dl") : (opts.root_dir / s->getHash());
download(sset, sources, opts);
return sources;
}

0 comments on commit 3e8f785

Please sign in to comment.
You can’t perform that action at this time.