Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Draft] feat: source add build field #280

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

[Draft] feat: source add build field #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7280145
feat: mount build helper
myml Mar 13, 2024
97ae5a5
feat: source add build field
myml Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 100 additions & 34 deletions src/linglong/builder/linglong_builder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -745,45 +745,73 @@ LinglongBuilder::buildStageSource(ocppi::runtime::config::types::Config &r, Proj
LINGLONG_TRACE("process source");

printer.printMessage("[Processing Source]");
QSharedPointer<SourceFetcher> sf;
for (const auto source : project->sources) {
sf.reset(new SourceFetcher(source, printer, project));
if (source) {
auto err = sf->fetch();
if (err) {
return LINGLONG_ERR("fetch source failed");
auto containerSourcesPath = QDir("/sources");
auto hostSourcesPath = QDir(BuilderConfig::instance()->targetSourcePath());
// 挂载/sources为tmpfs
utils::command::AddMount(r,
"",
containerSourcesPath.path(),
{ "nosuid", "strictatime" },
"tmpfs");

for (const auto &source : project->sources) {
// 过滤path中可能包含的..，避免路径注入
source->path = QDir::cleanPath(source->path);
// local类型的source挂载点随机生成，也可以使用path参数手动指定
if (source->kind == "local") {
if (source->path.isEmpty()) {
source->path = "local_" + util::genUuid();
}
utils::command::AddMount(r,
BuilderConfig::instance()->getProjectRoot(),
containerSourcesPath.filePath(source->path),
{ "rbind", "nosuid", "nodev" });
continue;
}
// 其他类型的source挂载点使用哈希值，也可以使用path参数手动指定
if (source->path.isEmpty()) {
if (source->digest.isEmpty()) {
source->path = "commit_" + source->commit;
} else {
source->path = "digest_" + source->digest;
}
}
// 避免digest或commit里面有路径注入
source->path = QDir::cleanPath(source->path);
// 为远程source创建目录
hostSourcesPath.mkpath(source->path);
// 拉去远程source
SourceFetcher sf(source, printer, project);
sf.setSourceRoot(hostSourcesPath.filePath(source->path));
auto err = sf.fetch();
if (err) {
return LINGLONG_ERR("fetch source failed");
}
// 添加挂载
utils::command::AddMount(r,
hostSourcesPath.filePath(source->path),
containerSourcesPath.filePath(source->path),
{ "rbind", "nosuid", "nodev" });
}
// 挂载tmp
ocppi::runtime::config::types::Mount m;
m.type = "tmpfs";
m.source = "tmp";
m.destination = "/tmp";
m.options = { "nosuid", "strictatime", "mode=777" };
r.mounts->push_back(m);
utils::command::AddMount(r, "", "/tmp", { "nosuid", "strictatime", "mode=777" }, "tmpfs");
// 挂载构建文件
auto containerSourcePath = "/source";
QList<QPair<QString, QString>> mountMap = {
// 源码目录
{ sf->sourceRoot(), containerSourcePath },
// 构建脚本
{ project->buildScriptPath(), BuildScriptPath },
};
for (const auto &pair : mountMap) {
ocppi::runtime::config::types::Mount m;
m.type = "bind";
m.source = pair.first.toStdString();
m.destination = pair.second.toStdString();
m.options = { "rbind", "nosuid", "nodev" };
r.mounts->push_back(m);
}

if (!BuilderConfig::instance()->getExec().empty()) {
project->generateBuildScript();
utils::command::AddMount(r,
project->buildScriptPath(),
BuildScriptPath,
{ "rbind", "nosuid", "nodev" });
// 如果手动指定exec，不再执行构建脚本
if (BuilderConfig::instance()->getExec().empty()) {
r.process->args = { "/bin/bash", "-e", BuildScriptPath };
} else {
r.process->terminal = true;
for (auto exec : BuilderConfig::instance()->getExec()) {
r.process->args->push_back(exec.toStdString());
}
}
r.process->args = { "/bin/bash", "-e", BuildScriptPath };
r.process->cwd = containerSourcePath;
r.process->cwd = containerSourcesPath.path().toStdString();
r.process->env->push_back("SOURCES=" + r.process->cwd);
r.process->user = ocppi::runtime::config::types::User();
r.process->user->uid = getuid();
r.process->user->gid = getgid();
Expand Down Expand Up @@ -829,6 +857,41 @@ LinglongBuilder::buildStageEnvrionment(ocppi::runtime::config::types::Config &r,
return LINGLONG_OK;
}

// 挂载构建助手目录到容器，构建助手目录存放一些脚本和二进制，目的是简化构建过程
linglong::utils::error::Result<void>
LinglongBuilder::buildStageMountHelper(ocppi::runtime::config::types::Config &r)
{
QStringList mountOption = { "ro", "rbind", "nosuid", "nodev" };
QStringList path;
// 挂载系统helper目录
// /usr/libexec/linglong/builder/helper 挂载到 /usr/libexec/linglong/builder/helper/system
auto helperPath = QDir(LINGLONG_LIBEXEC_DIR).filePath("builder/helper");
if (QDir(helperPath).exists()) {
path.push_back(helperPath);
auto mountPoint = QDir(helperPath).filePath("system");
utils::command::AddMount(r, helperPath, mountPoint, mountOption);
}
// 挂载用户helper目录
// $LINGLONG_BUILDER_HELPER_PATH 挂载到 /usr/libexec/linglong/builder/helper/user
auto helperUserPath =
QProcessEnvironment::systemEnvironment().value("LINGLONG_BUILDER_HELPER_PATH");
if (!helperUserPath.isEmpty()) {
auto mountPoint = QDir(helperPath).filePath("user");
path.push_front(mountPoint); // 用户的helper优先级更高
utils::command::AddMount(r, helperUserPath, mountPoint, mountOption);
}
// 将挂载点添加到PATH环境变量中
if (!path.empty()) {
for (auto &env : r.process.value().env.value()) {
if (QString::fromStdString(env).startsWith("PATH=")) {
env += ":" + path.join(":").toStdString();
}
}
}
return LINGLONG_OK;
}

// 映射用户身份
linglong::utils::error::Result<void>
LinglongBuilder::buildStageIDMapping(ocppi::runtime::config::types::Config &r)
{
Expand All @@ -849,7 +912,7 @@ LinglongBuilder::buildStageIDMapping(ocppi::runtime::config::types::Config &r)
}
// 映射gid
QList<QList<quint64>> gidMaps = {
{ getgid(), getuid(), 1 },
{ getgid(), getgid(), 1 },
};
for (auto const &gidMap : gidMaps) {
ocppi::runtime::config::types::IdMapping idMap;
Expand Down Expand Up @@ -933,7 +996,6 @@ linglong::utils::error::Result<QSharedPointer<Project>> LinglongBuilder::buildSt
return LINGLONG_ERR("unknown package kind");
}

project->generateBuildScript();
project->configFilePath = projectConfigPath;

linglong::builder::BuilderConfig::instance()->setProjectName(project->package->id);
Expand Down Expand Up @@ -1013,6 +1075,10 @@ linglong::utils::error::Result<void> LinglongBuilder::build()
if (!voidRet) {
return LINGLONG_ERR(voidRet);
}
voidRet = buildStageMountHelper(containerConfig);
if (!voidRet) {
return LINGLONG_ERR(voidRet);
}
// 配置容器rootfs
QTemporaryDir tmpDir;
tmpDir.setAutoRemove(false);
Expand Down
11 changes: 6 additions & 5 deletions src/linglong/builder/linglong_builder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,14 @@ class LinglongBuilder : public QObject, public Builder

util::Error exportLayer(const QString &destination) override;

util::Error extractLayer(const QString &layerPath,
const QString &destination) override;
util::Error extractLayer(const QString &layerPath, const QString &destination) override;

util::Error exportBundle(const QString &outputFilepath, bool useLocalDir) override;

util::Error push(const QString &repoUrl,
const QString &repoName,
const QString &channel,
bool pushWithDevel) override;
const QString &repoName,
const QString &channel,
bool pushWithDevel) override;

util::Error import() override;

Expand Down Expand Up @@ -85,6 +84,8 @@ class LinglongBuilder : public QObject, public Builder
const Project &project,
const BuilderConfig &config);
linglong::utils::error::Result<void>
buildStageMountHelper(ocppi::runtime::config::types::Config &r);
linglong::utils::error::Result<void>
buildStageIDMapping(ocppi::runtime::config::types::Config &r);
linglong::utils::error::Result<void> buildStageRootfs(ocppi::runtime::config::types::Config &r,
const QDir &workdir,
Expand Down
19 changes: 8 additions & 11 deletions src/linglong/builder/project.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,6 @@ int Project::generateBuildScript(const QString &path)
return -1;
}

if (!BuilderConfig::instance()->getExec().isEmpty()) {
auto exec = BuilderConfig::instance()->getExec();
for (const auto &arg : exec) {
command += '\'' + arg + '\'' + ' ';
}
command = command.trimmed();
command += "\n";
scriptFile.write(command.toLocal8Bit());
scriptFile.close();
return 0;
}
// TODO: generate global config, load from builder config file.
command += "#global variable\n";
command += QString("JOBS=%1\n").arg("6");
Expand Down Expand Up @@ -185,6 +174,14 @@ int Project::generateBuildScript(const QString &path)
}
}

command += "#source build commands\n";
for (auto source : sources) {
command += QString("# source kind: %1, url: %2\n").arg(source->kind).arg(source->url);
command += QString("cd $SOURCES/%1\n").arg(source->path);
command += source->build;
command += "\n";
}

command += "#build commands\n";
command +=
build->manual->configure.isNull() ? temp->build->manual->configure : build->manual->configure;
Expand Down
4 changes: 3 additions & 1 deletion src/linglong/builder/project.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
#ifndef LINGLONG_SRC_BUILDER_BUILDER_PROJECT_H_
#define LINGLONG_SRC_BUILDER_BUILDER_PROJECT_H_

#include "linglong/builder/builder_config.h"
#include "linglong/package/ref.h"
#include "linglong/util/qserializer/deprecated.h"
#include "linglong/builder/builder_config.h"

#include <QObject>
#include <QScopedPointer>
Expand Down Expand Up @@ -91,6 +91,8 @@ class Source : public JsonSerialize
Q_JSON_PROPERTY(QString, commit);

Q_JSON_PROPERTY(QStringList, patch);
Q_JSON_PROPERTY(QString, build);
Q_JSON_PROPERTY(QString, path);
};

class BuildManual : public JsonSerialize
Expand Down
14 changes: 7 additions & 7 deletions src/linglong/builder/source_fetcher.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,6 @@ std::tuple<QString, linglong::util::Error> SourceFetcherPrivate::fetchArchiveFil
{
Q_Q(SourceFetcher);

q->setSourceRoot(sourceTargetPath());

QUrl url(source->url);
auto path = BuilderConfig::instance()->targetFetchCachePath() + "/" + filename();

Expand Down Expand Up @@ -165,9 +163,7 @@ util::Error SourceFetcherPrivate::fetchGitRepo()
{
Q_Q(SourceFetcher);

q->setSourceRoot(sourceTargetPath());

q->printer.printMessage(QString("Path: %1").arg(sourceTargetPath()), 2);
q->printer.printMessage(QString("Path: %1").arg(q->sourceRoot()), 2);

// 如果二进制安装在系统目录中，优先使用系统中安装的脚本文件（便于用户更改），否则使用二进制内嵌的脚本（便于开发调试）
auto scriptFile = QString(LINGLONG_LIBEXEC_DIR) + "/fetch-git-repo.sh";
Expand All @@ -182,7 +178,7 @@ util::Error SourceFetcherPrivate::fetchGitRepo()
QFile::copy(":/scripts/fetch-git-repo", scriptFile);
}
QStringList args = {
scriptFile, sourceTargetPath(), source->url, source->commit, source->version,
scriptFile, q->sourceRoot(), source->url, source->commit, source->version,
};
q->printer.printMessage(QString("Command: sh %1").arg(args.join(" ")), 2);

Expand Down Expand Up @@ -265,7 +261,7 @@ linglong::util::Error SourceFetcher::fetch()
if (err) {
return err;
}
err = d->extractFile(s, d->sourceTargetPath());
err = d->extractFile(s, sourceRoot());
} else if (d->source->kind == "local") {
err = d->handleLocalSource();
printer.printMessage(QString("Source: %1").arg(sourceRoot()), 2);
Expand Down Expand Up @@ -307,6 +303,10 @@ SourceFetcher::SourceFetcher(QSharedPointer<Source> s, cli::Printer &p, Project
Q_D(SourceFetcher);

d->project = project;
if (d->source->kind == "git" && d->source->version.isEmpty()) {
d->source->version = project->package->version;
}
setSourceRoot(d->sourceTargetPath());
}

SourceFetcher::~SourceFetcher() { }
Expand Down
Loading