Skip to content
Permalink
Browse files

Implement upload.

  • Loading branch information...
egorpugin committed Jul 11, 2019
1 parent d5a48f0 commit ad1844d49730216af9e83b0f71ac45b5fb47dd26
@@ -331,6 +331,8 @@ void Command::prepare()
// program is an input!
inputs.insert(getProgram());

// stable sort args by their positions

getHashAndSave();

// add more deps
@@ -63,10 +63,12 @@ struct SwBuilderContext;
template <class T>
struct CommandData
{
std::unordered_set<std::shared_ptr<T>> dependencies;
using SPtr = std::shared_ptr<T>;

std::unordered_set<SPtr> dependencies;

std::atomic_size_t dependencies_left = 0;
std::unordered_set<std::shared_ptr<T>> dependent_commands;
std::unordered_set<SPtr> dependent_commands;

std::atomic_size_t *current_command = nullptr;
std::atomic_size_t *total_commands = nullptr;
@@ -77,6 +79,7 @@ struct CommandData

virtual void execute() = 0;
virtual void prepare() = 0;
// virtual bool lessDuringExecution(const Command &rhs) const;

void clear()
{
@@ -30,12 +30,4 @@ struct SwContext;

void run(const SwContext &swctx, const PackageId &package);

struct FetchOptions : SourceDownloadOptions
{
String name_prefix;
bool apply_version_to_source = false;
bool dry_run = true;
bool parallel = true;
};

}
@@ -143,18 +143,15 @@ SUBCOMMAND_DECL2(fetch)
{
using namespace sw;

sw::FetchOptions opts;
//opts.name_prefix = upload_prefix;
opts.dry_run = !build_after_fetch;
sw::SourceDownloadOptions opts;
opts.root_dir = fs::current_path() / SW_BINARY_DIR;
opts.ignore_existing_dirs = true;
opts.existing_dirs_age = std::chrono::hours(1);
//opts.apply_version_to_source = true;

auto &i = swctx.addInput(fs::current_path());
auto ts = swctx.getHostSettings();
//ts["name-prefix"] = "true";
ts["dry-run"] = "true";
ts["driver"]["dry-run"] = "true";
i.addSettings(ts);
swctx.load();

@@ -185,9 +182,9 @@ SUBCOMMAND_DECL2(fetch)
download(sources, srcs, opts);

i.clearSettings();
ts["dry-run"] = "false";
ts["driver"]["dry-run"] = "false";
for (auto &[h, d] : srcs)
ts["source-dir-for-source"][h] = normalize_path(d);
ts["driver"]["source-dir-for-source"][h] = normalize_path(d);
i.addSettings(ts);
swctx.load();

@@ -23,7 +23,7 @@ extern ::cl::list<String> build_arg;

static ::cl::opt<String> build_arg_generate(::cl::Positional, ::cl::desc("File or directory to use to generate projects"), ::cl::init("."), ::cl::sub(subcommand_generate));

extern String gGenerator;
String gGenerator;
::cl::opt<String, true> cl_generator("G", ::cl::desc("Generator"), ::cl::location(gGenerator), ::cl::sub(subcommand_generate));
::cl::alias generator2("g", ::cl::desc("Alias for -G"), ::cl::aliasopt(cl_generator));
extern bool gPrintDependencies;
@@ -21,13 +21,17 @@

#include <sw/driver/build.h>
#include <sw/manager/settings.h>
#include <sw/manager/api.h>

#include <primitives/log.h>
DECLARE_STATIC_LOGGER(logger, "upload");

extern ::cl::opt<String> build_arg_update;

static ::cl::opt<String> upload_remote(::cl::Positional, ::cl::desc("Remote name"), ::cl::sub(subcommand_upload));
String gUploadPrefix;
static ::cl::opt<String, true> upload_prefix(::cl::Positional, ::cl::desc("Prefix path"), ::cl::sub(subcommand_upload), ::cl::Required, ::cl::location(gUploadPrefix));
static ::cl::opt<bool> build_before_upload("build", ::cl::desc("Build before upload"), ::cl::sub(subcommand_upload));
static ::cl::opt<String, true> upload_prefix(::cl::Positional, ::cl::desc("Prefix path"), ::cl::sub(subcommand_upload),
::cl::Required, ::cl::location(gUploadPrefix));

sw::Remote *find_remote(sw::Settings &s, const String &name);

@@ -39,43 +43,26 @@ SUBCOMMAND_DECL(upload)

SUBCOMMAND_DECL2(upload)
{
// select remote first
auto &us = sw::Settings::get_user_settings();
auto current_remote = &*us.remotes.begin();
if (!upload_remote.empty())
current_remote = find_remote(us, upload_remote);

sw::FetchOptions opts;
//opts.name_prefix = upload_prefix;
opts.dry_run = !build_before_upload;
opts.root_dir = fs::current_path() / SW_BINARY_DIR;
opts.ignore_existing_dirs = true;
opts.existing_dirs_age = std::chrono::hours(8);
//opts.apply_version_to_source = true;
//swctx.
SW_UNIMPLEMENTED;
//auto s = sw::fetch_and_load(swctx, build_arg_update.getValue(), opts);
if (build_before_upload)
{
//s->execute();
cli_fetch(swctx);

// after execution such solution has resolved deps and deps of the deps
// we must not add them
SW_UNIMPLEMENTED;
}

/*auto m = s->getPackages();
auto m = swctx.getPackages();
// dbg purposes
for (auto &[id, d] : m)
{
write_file(fs::current_path() / SW_BINARY_DIR / "upload" / id.toString() += ".json", d->getString());
auto id2 = id;
id2.ppath = PackagePath(upload_prefix) / id2.ppath;
id2.ppath = sw::PackagePath(upload_prefix) / id2.ppath;
LOG_INFO(logger, "Uploading " + id2.toString());
}

// select remote first
auto &us = sw::Settings::get_user_settings();
auto current_remote = &*us.remotes.begin();
if (!upload_remote.empty())
current_remote = find_remote(us, upload_remote);

// send signatures (gpg)
// -k KEY1 -k KEY2
auto api = current_remote->getApi();
api->addVersion(upload_prefix, m, sw::read_config(build_arg_update.getValue()).value());*/
api->addVersion(gUploadPrefix, m, swctx.getSpecification());
}
@@ -23,6 +23,7 @@ struct SW_CORE_API IDriver

virtual bool canLoad(const Input &) const = 0;
virtual void load(const std::set<Input> &) = 0;
virtual String getSpecification() const = 0;

//virtual void execute() = 0;
//virtual bool prepareStep() = 0;
@@ -106,6 +106,11 @@ const TargetSetting &TargetSetting::operator[](const TargetSettingKey &k) const
return s;
}
return (*settings)[k];*/
if (value.index() != 3)
{
static TargetSetting s("");
return s;
}
return std::get<TargetSettings>(value)[k];
}

@@ -29,7 +29,7 @@ IDriver::~IDriver() = default;
SwCoreContext::SwCoreContext(const path &local_storage_root_dir)
: SwBuilderContext(local_storage_root_dir)
{
source_dir = fs::canonical(fs::current_path());
source_dir = primitives::filesystem::canonical(fs::current_path());

detectCompilers(*this);
predefined_targets = targets; // save
@@ -461,6 +461,15 @@ SwContext::CommandExecutionPlan SwContext::getExecutionPlan(const Commands &cmds
throw SW_RUNTIME_ERROR(error);
}

String SwContext::getSpecification() const
{
if (inputs.empty())
throw SW_RUNTIME_ERROR("Empty inputs");
if (inputs.size() > 1)
throw SW_RUNTIME_ERROR("More than 1 input");
return inputs.begin()->getDriver().getSpecification();
}

PackageDescriptionMap SwContext::getPackages() const
{
PackageDescriptionMap m;
@@ -469,6 +478,8 @@ PackageDescriptionMap SwContext::getPackages() const
// deps
if (pkg.ppath.isAbsolute())
continue;
if (tgts.empty())
continue;

auto &t = *tgts.begin();
if (!t->isReal())
@@ -62,6 +62,7 @@ struct SW_CORE_API SwContext : SwCoreContext

CommandExecutionPlan getExecutionPlan() const;
PackageDescriptionMap getPackages() const;
String getSpecification() const;

String getBuildHash() const;

@@ -55,6 +55,11 @@ TargetData::Base::const_iterator TargetData::find(const TargetSettings &s) const
});
}

bool TargetData::empty() const
{
return targets.empty();
}

TargetMap::~TargetMap()
{
}
@@ -106,6 +106,7 @@ struct SW_CORE_API TargetData

void push_back(const ITargetPtr &);
void clear();
bool empty() const;

auto begin() { return targets.begin(); }
auto end() { return targets.end(); }
@@ -44,10 +44,8 @@
#include <primitives/log.h>
DECLARE_STATIC_LOGGER(logger, "build");

String gGenerator;
cl::opt<bool> dry_run("n", cl::desc("Dry run"));
static cl::opt<bool> debug_configs("debug-configs", cl::desc("Build configs in debug mode"));
static cl::opt<bool> fetch_sources("fetch", cl::desc("Fetch files in process"));

static cl::opt<int> config_jobs("jc", cl::desc("Number of config jobs"));

@@ -1132,11 +1130,6 @@ void Build::load_spec_file(const path &fn)
{
auto dll = build(fn);

if (fetch_sources)
{
fetch_dir = BinaryDir / "src";
}

auto fe = selectFrontendByFilename(fn);
if (!fe)
throw SW_RUNTIME_ERROR("frontend was not found for file: " + normalize_path(fn));
@@ -1175,30 +1168,11 @@ void Build::load(const path &fn, bool configless)
if (!fs::exists(fn))
throw SW_RUNTIME_ERROR("path does not exists: " + normalize_path(fn));

/*if (!gGenerator.empty())
{
generator = Generator::create(gGenerator);
// set early, before prepare
// also add tests to solution
// protect with option
with_testing = true;
}*/

if (configless)
return load_configless(fn);

auto dll = build(fn);

//fs->save(); // remove?
//fs->reset();

if (fetch_sources)
{
fetch_dir = BinaryDir / "src";
}

auto fe = selectFrontendByFilename(fn);
if (!fe)
throw SW_RUNTIME_ERROR("frontend was not found for file: " + normalize_path(fn));
@@ -40,6 +40,18 @@ void Build::build_self()
for (auto &[u, p] : m)
{
auto &ep = epm[p.getData().group_number];
// may be empty when different versions is requested also
// example:
// we request sqlite3-3.28.0
// some lib requests sqlite3-*
// now we got 3.28.0 and 3.29.0
// for 3.29.0 we do not have ep
if (!ep)
{
continue;
// actually it's better throw here?
//throw SW_RUNTIME_ERROR();
}
ep->module_data.known_targets.insert(p);
getChildren()[p].setEntryPoint(ep);
}
@@ -32,8 +32,6 @@ Driver::Driver(SwCoreContext &swctx)
: swctx(swctx)
{
build = std::make_unique<Build>(swctx, *this);

//source_dir = fs::canonical(fs::current_path());
module_storage = std::make_unique<ModuleStorage>();
}

@@ -74,6 +72,8 @@ bool Driver::canLoad(const Input &i) const
return false;
}

static String spec;

void Driver::load(const std::set<Input> &inputs)
{
PackageIdSet pkgsids;
@@ -93,17 +93,16 @@ void Driver::load(const std::set<Input> &inputs)
build->settings.clear();
for (auto s : i.getSettings())
{
s.erase("name-prefix");
s.erase("dry-run");
s.erase("source-dir-for-source");
s.erase("driver");
build->addSettings(s);
}

build->DryRun = (*i.getSettings().begin())["dry-run"] == "true";
build->DryRun = (*i.getSettings().begin())["driver"]["dry-run"] == "true";

for (auto &[h, d] : (*i.getSettings().begin())["source-dir-for-source"].getSettings())
for (auto &[h, d] : (*i.getSettings().begin())["driver"]["source-dir-for-source"].getSettings())
build->source_dirs_by_source[h] = d.getValue();

spec = read_file(p);
build->load_spec_file(p);
break;
}
@@ -126,6 +125,11 @@ bool Driver::prepareStep()
return build->prepareStep();
}

String Driver::getSpecification() const
{
return spec;
}

ChecksStorage &Driver::getChecksStorage(const String &config) const
{
auto i = checksStorages.find(config);
@@ -27,6 +27,7 @@ struct SW_DRIVER_CPP_API Driver : IDriver
PackageId getPackageId() const override;
bool canLoad(const Input &) const override;
void load(const std::set<Input> &) override;
String getSpecification() const override;
void execute();
bool prepareStep();

0 comments on commit ad1844d

Please sign in to comment.
You can’t perform that action at this time.