Skip to content
Permalink
Browse files

Fixes in resolving overridden packages.

  • Loading branch information...
egorpugin committed Dec 16, 2018
1 parent c5eca64 commit c5cb4a96ca2cba087812e7d74488f809efdb1729
@@ -287,7 +287,7 @@ static cl::opt<bool> list_overridden_packages("list-overridden-remote-packages",
static cl::opt<String> delete_overridden_package("delete-overridden-remote-package", cl::value_desc("package"), cl::desc("Delete overridden package from index"));
static cl::opt<path> delete_overridden_package_dir("delete-overridden-remote-package-dir", cl::value_desc("sdir"), cl::desc("Delete overridden dir packages"));

static cl::opt<bool> use_lock_file("l", cl::desc("Use lock file"));
static cl::opt<bool> use_lock_file("l", cl::desc("Use lock file"), cl::init(true));

//static cl::list<String> builtin_function("internal-call-builtin-function", cl::desc("Call built-in function"), cl::Hidden);

@@ -346,7 +346,7 @@ int sw_main(const Strings &args)
return 0;
}

if (/*use_lock_file && */fs::exists(fs::current_path() / "sw.lock"))
if (use_lock_file && fs::exists(fs::current_path() / "sw.lock"))
getPackageStore().loadLockFile(fs::current_path() / "sw.lock");

/*if (!build_arg0.empty())
@@ -368,9 +368,8 @@ void stop()
//getFileMonitor().stop();
//getExecutor().join();
//getFileStorages().clear();
//if (use_lock_file)
// create always, use asap
getPackageStore().saveLockFile(fs::current_path() / "sw.lock");
if (use_lock_file)
getPackageStore().saveLockFile(fs::current_path() / "sw.lock");
}

static cl::opt<bool> write_log_to_file("log-to-file");
@@ -292,6 +292,21 @@ bool Solution::skipTarget(TargetScope Scope) const
)
return !with_testing;
return false;
}

TargetBaseTypePtr Solution::resolveTarget(const UnresolvedPackage &pkg) const
{
if (resolved_targets.find(pkg) == resolved_targets.end())
{
for (const auto &[p, t] : getChildren())
{
if (pkg.canBe(p))
{
resolved_targets[pkg] = t;
}
}
}
return resolved_targets[pkg];
}

void Solution::addTest(const ExecutableTarget &t)
@@ -798,7 +813,7 @@ void Solution::execute(ExecutionPlan<builder::Command> &p) const
}
}

void Solution::build_and_resolve()
void Solution::build_and_resolve(int n_runs)
{
auto ud = gatherUnresolvedDependencies();
if (ud.empty())
@@ -809,6 +824,9 @@ void Solution::build_and_resolve()
for (auto &[pkg, d] : ud)
pkgs.insert(pkg);

if (n_runs > 1)
LOG_ERROR(logger, "You are here for the third time. This is not intended. Failures are imminent.");

// resolve only deps needed
Resolver r;
r.resolve_dependencies(pkgs, true);
@@ -834,6 +852,10 @@ void Solution::build_and_resolve()
if (cfgs.empty())
return;

// all deps must be resolved in the first run!
if (n_runs > 0)
LOG_ERROR(logger, "You are here for the second time. This is not intended. Expect failures.");

Build b;
b.Local = false;
auto dll = b.build_configs(cfgs);
@@ -853,17 +875,6 @@ void Solution::build_and_resolve()

sr.restoreNow(true);

int retries = 0;
if (retries++ > 10)
{
String s = "Too many attempts on resolving packages, probably something wrong. Unresolved dependencies (" +
std::to_string(ud.size()) + ") are: ";
for (auto &p : ud)
s += p.first.toString() + ", ";
s.resize(s.size() - 2);
throw std::logic_error(s);
}

auto rd = r.resolved_packages;
for (auto &[porig, p] : rd)
{
@@ -893,7 +904,7 @@ void Solution::build_and_resolve()
// or to unregistered deps in sw - probably something wrong or
// malicious

build_and_resolve();
build_and_resolve(n_runs + 1);
}

void Solution::prepare()
@@ -937,6 +948,8 @@ UnresolvedDependenciesType Solution::gatherUnresolvedDependencies() const
for (const auto &p : getChildren())
{
auto c = p.second->gatherUnresolvedDependencies();
if (c.empty())
continue;
std::unordered_set<UnresolvedPackage> rm;
for (auto &[up, dptr] : c)
{
@@ -1390,6 +1403,7 @@ FilesMap Build::build_configs_separate(const Files &files)
check_self(solution.Checks);
solution.performChecks();
build_self(solution);
addDeps(lib, solution);
once = true;
}

@@ -1481,7 +1495,6 @@ FilesMap Build::build_configs_separate(const Files &files)
//#endif
}

addDeps(lib, solution);
for (auto &d : udeps)
lib += std::make_shared<Dependency>(d);

@@ -1541,6 +1554,7 @@ path Build::build_configs(const std::unordered_set<ExtendedPackageData> &pkgs)
check_self(solution.Checks);
solution.performChecks();
build_self(solution);
addDeps(lib, solution);

#if defined(CPPAN_OS_WINDOWS)
lib += implib;
@@ -1731,8 +1745,6 @@ path Build::build_configs(const std::unordered_set<ExtendedPackageData> &pkgs)
//#endif
}

addDeps(lib, solution);

auto i = solution.children.find(lib.pkg);
if (i == solution.children.end())
throw std::logic_error("config target not found");
@@ -93,7 +93,8 @@ struct SW_DRIVER_CPP_API Solution : TargetBase

// target data
TargetMap children;
TargetMap dummy_children;
TargetMap dummy_children; // contain dirs, projects,
TargetMap weak_children; // trash & garbage, used for storage internally, do not use

SourceDirMap source_dirs_by_source;

@@ -141,6 +142,8 @@ struct SW_DRIVER_CPP_API Solution : TargetBase

bool skipTarget(TargetScope Scope) const;

TargetBaseTypePtr resolveTarget(const UnresolvedPackage &) const;

// tests
// TODO: implement some of https://cmake.org/cmake/help/latest/manual/cmake-properties.7.html#properties-on-tests
Commands tests;
@@ -186,10 +189,11 @@ struct SW_DRIVER_CPP_API Solution : TargetBase
std::unordered_set<ExtendedPackageData> known_cfgs;
std::vector<detail::EventCallback> events;
//Files used_modules;
mutable std::unordered_map<UnresolvedPackage, TargetBaseTypePtr> resolved_targets;

void checkPrepared() const;
UnresolvedDependenciesType gatherUnresolvedDependencies() const;
void build_and_resolve();
void build_and_resolve(int n_runs = 0);

path getChecksFilename() const;
void loadChecks();
@@ -310,7 +310,8 @@ void TargetBase::addChild(const TargetBaseTypePtr &t)

void TargetBase::setupTarget(TargetBaseType *t) const
{
if (getSolution()->exists(t->pkg))
bool exists = getSolution()->exists(t->pkg);
if (exists)
throw SW_RUNTIME_EXCEPTION("Target already exists: " + t->pkg.target_name);

// find automatic way of copying data?
@@ -828,7 +829,7 @@ UnresolvedDependenciesType NativeExecutedTarget::gatherUnresolvedDependencies()
{
for (auto &d : v.Dependencies)
{
if (!d->target.lock())
if (!getSolution()->resolveTarget(d->package) && !d->target.lock())
deps.insert({ d->package, d });
}
});
@@ -1110,7 +1111,7 @@ void NativeExecutedTarget::addPrecompiledHeader(const PrecompiledHeader &p)
c->PrecompiledHeader().create = p.header;
c->PDBFilename = pdb_fn;
c->PDBFilename.intermediate_file = false;
c->PDBFilename.output_dependency = true;
//c->PDBFilename.output_dependency = true;
}
else if (auto c = sf->compiler->as<ClangClCompiler>())
{
@@ -1883,7 +1884,17 @@ bool NativeExecutedTarget::prepare()
}
}
if (!added)*/
throw std::logic_error("Unresolved package on stage 1: " + d->getPackage().toString());

auto err = "Package: " + pkg.toString() + ": Unresolved package on stage 1: " + d->getPackage().toString();
if (auto d = pkg.getOverriddenDir(); d)
{
err += ".\nPackage: " + pkg.toString() + " is overridden locally. "
"This means you have new dependency that is not in db.\n"
"Run following command in attempt to fix this issue:"
"'sw -d " + normalize_path(d.value()) + " -override-remote-package " +
pkg.ppath.slice(0, getServiceDatabase().getOverriddenPackage(pkg).value().prefix).toString() + "'";
}
throw std::logic_error(err);
}
}
});
@@ -1941,8 +1952,8 @@ bool NativeExecutedTarget::prepare()
{
if (d->target.lock() == nullptr)
{
throw std::logic_error("Unresolved package on stage 2: " + d->package.toString());
/*LOG_ERROR(logger, "Unresolved package on stage 2: " + d->package.toString() + ". Resolving inplace");
throw std::logic_error("Package: " + pkg.toString() + ": Unresolved package on stage 2: " + d->package.toString());
/*LOG_ERROR(logger, "Package: " + pkg.toString() + ": Unresolved package on stage 2: " + d->package.toString() + ". Resolving inplace");
auto id = d->package.resolve();
d->target = std::static_pointer_cast<NativeTarget>(getSolution()->getTargetPtr(id));*/
}
@@ -2034,6 +2045,7 @@ bool NativeExecutedTarget::prepare()
// but still in use.
// We add them back to children.
// Example: helpers, small tools, code generators.
// TODO: maybe reconsider
{
auto &c = getSolution()->children;
auto &dc = getSolution()->dummy_children;
@@ -546,6 +546,15 @@ void ServiceDatabase::setInstalledPackageFlags(const PackageId &p, const String
"' where id = '" + std::to_string(getInstalledPackageConfigId(p, config)) + "'");*/
}

optional<ServiceDatabase::OverriddenPackage> ServiceDatabase::getOverriddenPackage(const PackageId &pkg) const
{
const auto &pkgs = getOverriddenPackages();
auto i = pkgs.find(pkg);
if (i == pkgs.end(pkg))
return {};
return i->second;
}

const ServiceDatabase::OverriddenPackages &ServiceDatabase::getOverriddenPackages() const
{
// maybe move them to packages db?
@@ -978,6 +987,8 @@ IdDependencies PackagesDatabase::findDependencies(const UnresolvedPackages &deps

preInitFindDependencies();

const auto &overridden = getServiceDatabase().getOverriddenPackages();

DependenciesMap all_deps;
for (auto &dep : deps)
{
@@ -993,7 +1004,7 @@ IdDependencies PackagesDatabase::findDependencies(const UnresolvedPackages &deps
);
if (q.empty())
{
auto &pkgs = getServiceDatabase().getOverriddenPackages();
auto &pkgs = overridden;
PackageId pkg{ dep.ppath, dep.range.toString() };
auto i = pkgs.find(pkg);
if (i != pkgs.end(pkg))
@@ -1017,12 +1028,11 @@ IdDependencies PackagesDatabase::findDependencies(const UnresolvedPackages &deps
project.id = getExactProjectVersionId(project, project.version, project.flags, project.hash, project.group_number, project.prefix);
//project.flags.set(pfDirectDependency);
all_deps[project] = project; // assign first, deps assign second
all_deps[project].db_dependencies = getProjectDependencies(project.id, all_deps);
all_deps[project].db_dependencies = getProjectDependencies(project.id, all_deps, getServiceDatabase().getOverriddenPackageVersionDependencies(-project.id));
//all_deps[project].setDependencyIds(getProjectDependencies(project.id, all_deps)); // see dependency.h note
}

// mark local deps
const auto &overridden = getServiceDatabase().getOverriddenPackages();
if (!overridden.empty())
{
for (auto &[pkg, d] : all_deps)
@@ -1051,7 +1061,7 @@ void check_version_age(const std::string &created)
// and during the second one, the packet is really young
//LOG_INFO(logger, "mins " << mins);
if (mins < PACKAGES_DB_REFRESH_TIME_MINUTES * 2)
throw SW_RUNTIME_EXCEPTION("One of the queried packages is 'young'. Young packages must be retrieved from server.");
throw std::runtime_error("One of the queried packages is 'young'. Young packages must be retrieved from server.");
}

db::PackageVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDependency &project, Version &version, SomeFlags &flags, String &hash, PackageVersionGroupNumber &gn, int &prefix) const
@@ -1075,26 +1085,30 @@ db::PackageVersionId PackagesDatabase::getExactProjectVersionId(const DownloadDe
version_ids[row.version.value()] = row.packageVersionId.value();
}

auto v = project.range.getMaxSatisfyingVersion(versions);
if (!v)
// also check local db
auto &o = getServiceDatabase().getOverriddenPackages();
auto i = o.find(project.ppath);
if (i != o.end())
{
auto &o = getServiceDatabase().getOverriddenPackages();
auto i = o.find(project.ppath);
if (i != o.end())
for (auto &[v, d] : i->second)
{
for (auto &[v, d] : i->second)
{
versions.insert(v);
version_ids[v] = d.id;
}
versions.insert(v);
version_ids[v] = d.id;
}
v = project.range.getMaxSatisfyingVersion(versions);
if (!v)
throw err(project.ppath, project.range);
}

auto v = project.range.getMaxSatisfyingVersion(versions);
if (!v)
throw err(project.ppath, project.range);

PackageId pkg{ project.ppath, v.value() };
auto i2 = o.find(pkg);
if (i2 != o.end(pkg))
{
id = version_ids[v.value()];
version = v.value();
prefix = i->second.find(version)->second.prefix;
prefix = i2->second.prefix;
gn = i2->second.getGroupNumber();
return id;
}

@@ -82,6 +82,14 @@ struct SW_MANAGER_API ServiceDatabase : public Database
// extended
db::PackageVersionId id = 0; // overridden id is less than 0
int prefix = 2;

int64_t getGroupNumber() const
{
auto gn = std::hash<path>()(sdir);
if (gn > 0)
gn = -gn;
return gn;
}
};
using OverriddenPackages = PackageVersionMapBase<OverriddenPackage, std::unordered_map, std::map>;

@@ -121,6 +129,7 @@ struct SW_MANAGER_API ServiceDatabase : public Database
bool isPackageInstalled(const PackageId &p) const { return getInstalledPackageId(p) != 0; }
Packages getInstalledPackages() const;

optional<OverriddenPackage> getOverriddenPackage(const PackageId &pkg) const;
const OverriddenPackages &getOverriddenPackages() const;
void overridePackage(const PackageId &pkg, const OverriddenPackage &opkg) const;
void deleteOverriddenPackage(const PackageId &pkg) const;
@@ -18,6 +18,7 @@ struct ExtendedPackageData : Package
db::PackageVersionId id = 0;
String hash;
PackageVersionGroupNumber group_number = 0;
PackageVersionGroupNumber group_number_from_lock_file = 0;
int prefix = 2;
const Remote *remote = nullptr;
bool local_override = false;
Oops, something went wrong.

0 comments on commit c5cb4a9

Please sign in to comment.
You can’t perform that action at this time.