Permalink
Fetching contributors…
Cannot retrieve contributors at this time
executable file 1152 lines (1063 sloc) 34.3 KB
#!/usr/bin/env lua
--[[
This file is part of MXE. See LICENSE.md for licensing information.
build-pkg, Build binary packages from MXE packages
Instructions: http://pkg.mxe.cc
Requirements (see bootstrapped build below for non-debian systems):
MXE
apt-get install lua5.1 fakeroot dpkg dpkg-dev
Usage: lua tools/build-pkg.lua
Packages are written to `*.tar.xz` files.
Debian packages are written to `*.deb` files.
Build in directory /usr/lib/mxe
This directory can not be changed in .deb packages.
To prevent build-pkg from creating deb packages,
set environment variable MXE_BUILD_PKG_NO_DEBS to 1
In this case fakeroot and dpkg-deb are not needed.
To do a dry run without actually building any packages,
set environment variable MXE_BUILD_DRY_RUN to any value
Packages will be downloaded, but builds will be skipped.
To switch off the second pass, set
MXE_BUILD_PKG_NO_SECOND_PASS to 1.
See https://github.com/mxe/mxe/issues/1111
To limit number of packages being built to x,
set environment variable MXE_BUILD_PKG_MAX_ITEMS to x.
To set list of MXE targets to build,
set environment variable MXE_BUILD_PKG_TARGETS to
the list of targets separated by space.
By default, all 4 major targets are built.
To set list of MXE packages to build,
set environment variable MXE_BUILD_PKG_PKGS to
the list of packages separated by space. This is similar
to a normal `make` invocation in that all dependencies
will be built, so list just the packages you require.
By default, all packages are built.
The following error:
> fakeroot, while creating message channels: Invalid argument
> This may be due to a lack of SYSV IPC support.
> fakeroot: error while starting the `faked' daemon.
can be caused by leaked ipc resources originating in fakeroot.
How to remove them: https://stackoverflow.com/a/4262545
Alternatively, to switch off using fakeroot (e.g. inside docker),
set MXE_BUILD_PKG_NO_FAKEROOT to 1.
Bootstrapped build (non-debian systems building w/o deb pkgs):
export MXE_DIR=/path/to/mxe && \
export BUILD=`$MXE_DIR/ext/config.guess` && \
rm -rf $MXE_DIR/usr* && \
make -C $MXE_DIR lua \
MXE_TARGETS=$BUILD \
lua_TARGETS=$BUILD \
PREFIX=$MXE_DIR/usr.lua && \
MXE_BUILD_PKG_TARGETS="`echo {i686-w64-mingw32,x86_64-w64-mingw32}.{static,shared}`" \
MXE_BUILD_PKG_PKGS= \
MXE_BUILD_DRY_RUN=1 \
MXE_BUILD_PKG_MAX_ITEMS= \
MXE_BUILD_PKG_NO_DEBS=1 \
MXE_BUILD_PKG_NO_SECOND_PASS=0 \
$MXE_DIR/usr.lua/$BUILD/bin/lua $MXE_DIR/tools/build-pkg.lua
]]
local max_items = tonumber(os.getenv('MXE_BUILD_PKG_MAX_ITEMS'))
local no_debs = os.getenv('MXE_BUILD_PKG_NO_DEBS')
local no_fakeroot = os.getenv('MXE_BUILD_PKG_NO_FAKEROOT')
local no_second_pass = os.getenv('MXE_BUILD_PKG_NO_SECOND_PASS')
local build_targets = os.getenv('MXE_BUILD_PKG_TARGETS')
local TODAY = os.date("%Y%m%d")
local MAX_TRIES = 10
local GIT = 'git --work-tree=./usr/ --git-dir=./usr/.git '
local GIT_USER = '-c user.name="build-pkg" ' ..
'-c user.email="build-pkg@mxe" '
local BLACKLIST = {
'^usr/installed/check%-requirements$',
-- usr/share/cmake is useful
'^usr/share/doc/',
'^usr/share/info/',
'^usr/share/man/',
'^usr/share/gcc',
'^usr/share/gtk-doc',
'^usr/[^/]+/share/doc/',
'^usr/[^/]+/share/info/',
-- usr/lib/nonetwork.so and
-- usr/x86_64-unknown-linux-gnu/lib/nonetwork.so
'lib/nonetwork.so',
-- https://github.com/mxe/mxe/issues/1886#issuecomment-331719282
'installed/.gitkeep',
'lib/.gitkeep',
}
local TARGETS = {
'i686-w64-mingw32.static',
'x86_64-w64-mingw32.static',
'i686-w64-mingw32.shared',
'x86_64-w64-mingw32.shared',
}
if build_targets then
TARGETS = {}
for target in build_targets:gmatch('(%S+)') do
table.insert(TARGETS, target)
end
end
local function echo(fmt, ...)
print(fmt:format(...))
io.stdout:flush()
end
local function log(fmt, ...)
echo('[build-pkg]\t' .. fmt, ...)
end
-- based on http://lua-users.org/wiki/SplitJoin
local function split(self, sep, nMax, plain)
if not sep then
sep = '%s+'
end
assert(sep ~= '')
assert(nMax == nil or nMax >= 1)
local aRecord = {}
if self:len() > 0 then
nMax = nMax or -1
local nField = 1
local nStart = 1
local nFirst, nLast = self:find(sep, nStart, plain)
while nFirst and nMax ~= 0 do
aRecord[nField] = self:sub(nStart, nFirst - 1)
nField = nField + 1
nStart = nLast + 1
nFirst, nLast = self:find(sep, nStart, plain)
nMax = nMax - 1
end
aRecord[nField] = self:sub(nStart)
end
return aRecord
end
local function trim(str)
local text = str:gsub("%s+$", "")
text = text:gsub("^%s+", "")
return text
end
local function isInArray(element, array)
for _, member in ipairs(array) do
if member == element then
return true
end
end
return false
end
local function sliceArray(list, nelements)
nelements = nelements or #list
local new_list = {}
for i = 1, nelements do
new_list[i] = list[i]
end
return new_list
end
local function concatArrays(...)
local result = {}
for _, array in ipairs({...}) do
for _, elem in ipairs(array) do
table.insert(result, elem)
end
end
return result
end
local function isInString(substring, string)
return string:find(substring, 1, true)
end
local function shell(cmd)
local f = io.popen(cmd, 'r')
local text = f:read('*all')
f:close()
return text
end
local function execute(cmd)
if _VERSION == 'Lua 5.1' then
return os.execute(cmd) == 0
else
-- Lua >= 5.2
return os.execute(cmd)
end
end
local MXE_DIR = trim(shell('pwd'))
-- for tar, try gtar and gnutar first
local tools = {}
local function tool(name)
if tools[name] then
return tools[name]
end
if execute(("g%s --help > /dev/null 2>&1"):format(name)) then
tools[name] = 'g' .. name
elseif execute(("gnu%s --help > /dev/null 2>&1"):format(name)) then
tools[name] = 'gnu' .. name
else
tools[name] = name
end
return tools[name]
end
local function fileExists(name)
local f = io.open(name, "r")
if f ~= nil then
io.close(f)
return true
else
return false
end
end
local function fileSize(name)
local f = io.open(name, "r")
local size = f:seek("end")
io.close(f)
return size
end
local function isSymlink(name)
return shell(("ls -l %q"):format(name)):sub(1, 1) == "l"
end
local function writeFile(filename, data)
local file = io.open(filename, 'w')
file:write(data)
file:close()
end
local NATIVE_TARGET = trim(shell("ext/config.guess"))
local function isCross(target)
return target ~= NATIVE_TARGET
end
local function getArch()
local cmd = "dpkg-architecture -qDEB_BUILD_ARCH 2> /dev/null"
return trim(shell(cmd))
end
local ARCH = getArch()
-- return target and package from item name
local function parseItem(item)
return item:match("([^~]+)~([^~]+)")
end
-- return item name from target and package
local function makeItem(target, package)
return target .. '~' .. package
end
-- return several tables describing packages for all targets
-- * list of items
-- * map from item to list of deps (which are also items)
-- * map from item to version
-- Item is a string like "target~pkg"
local function getItems()
local items = {}
local item2deps = {}
local item2ver = {}
local cmd = '%s print-deps-for-build-pkg MXE_TARGETS=%q'
cmd = cmd:format(tool 'make', table.concat(TARGETS, ' '))
local make = io.popen(cmd)
for line in make:lines() do
local deps = split(trim(line))
if deps[1] == 'for-build-pkg' then
-- first value is marker 'for-build-pkg'
table.remove(deps, 1)
-- first value is name of package which depends on
local item = table.remove(deps, 1)
-- second value is version of package
local ver = table.remove(deps, 1)
table.insert(items, item)
item2deps[item] = deps
item2ver[item] = ver
local target, _ = parseItem(item)
for _, dep_item in ipairs(deps) do
local target2, _ = parseItem(dep_item)
if isCross(target2) and target2 ~= target then
log("Cross-target dependency %s -> %s",
target2, target)
end
end
end
end
make:close()
return items, item2deps, item2ver
end
local function getInstalled()
local installed = {}
local f = io.popen('ls usr/*/installed/*')
local pattern = '/([^/]+)/installed/([^/]+)'
for file in f:lines() do
local target, pkg = assert(file:match(pattern))
table.insert(installed, makeItem(target, pkg))
end
f:close()
return installed
end
-- graph is a map from item to a list of destinations
local function transpose(graph)
local transposed = {}
for item, destinations in pairs(graph) do
for _, dest in ipairs(destinations) do
if not transposed[dest] then
transposed[dest] = {}
end
table.insert(transposed[dest], item)
end
end
return transposed
end
local function reverse(list)
local n = #list
local reversed = {}
for i = 1, n do
reversed[i] = list[n - i + 1]
end
return reversed
end
-- return items ordered in build order
-- this means, if item depends on item2, then
-- item2 precedes item1 in the list
local function sortForBuild(items, item2deps)
local n = #items
local item2followers = transpose(item2deps)
-- Tarjan's algorithm
-- https://en.wikipedia.org/wiki/Topological_sorting
local build_list_reversed = {}
local marked_permanently = {}
local marked_temporarily = {}
local function visit(item1)
assert(not marked_temporarily[item1], 'not a DAG')
if not marked_permanently[item1] then
marked_temporarily[item1] = true
local followers = item2followers[item1] or {}
for _, item2 in ipairs(followers) do
visit(item2)
end
marked_permanently[item1] = true
marked_temporarily[item1] = false
table.insert(build_list_reversed, item1)
end
end
for _, item in ipairs(items) do
if not marked_permanently[item] then
visit(item)
end
end
assert(#build_list_reversed == n)
local build_list = reverse(build_list_reversed)
assert(#build_list == n)
return build_list
end
local function isDependency(item, dependency, item2deps)
for _, dep in ipairs(item2deps[item]) do
if dep == dependency then
return true
end
if isDependency(dep, dependency, item2deps) then
return true
end
end
return false
end
local function makeItem2Index(build_list)
local item2index = {}
for index, item in ipairs(build_list) do
assert(not item2index[item], 'Duplicate item')
item2index[item] = index
end
return item2index
end
-- return if build_list is ordered topologically
local function isTopoOrdered(build_list, items, item2deps)
if #build_list ~= #items then
return false, 'Length of build_list is wrong'
end
local item2index = makeItem2Index(build_list)
for item, deps in pairs(item2deps) do
for _, dep in ipairs(deps) do
if item2index[item] < item2index[dep] then
return false, 'Item ' .. item ..
'is built before its dependency ' ..
dep
end
end
end
return true
end
local function isListed(file, list)
for _, pattern in ipairs(list) do
if file:match(pattern) then
return true
end
end
return false
end
local function isBlacklisted(file)
return isListed(file, BLACKLIST)
end
local GIT_INITIAL = 'initial'
local GIT_ALL_PSEUDOITEM = 'all'
local function itemToBranch(item, pass)
return pass .. '-' .. item:gsub('~', '_')
end
-- creates git repo in ./usr
local function gitInit()
os.execute('mkdir -p ./usr')
os.execute(GIT .. 'init --quiet')
end
local function gitTag(name)
os.execute(GIT .. 'tag ' .. name)
end
local function gitConflicts()
local cmd = GIT .. 'diff --name-only --diff-filter=U'
local f = io.popen(cmd, 'r')
local conflicts = {}
for conflict in f:lines() do
table.insert(conflicts, conflict)
end
f:close()
return conflicts
end
-- git commits changes in ./usr
local function gitCommit(message)
local cmd = GIT .. GIT_USER .. 'commit -a -m %q --quiet'
assert(execute(cmd:format(message)))
end
local function gitCheckout(new_branch, deps, item2index, pass_of_deps)
local main_dep = deps[1]
if main_dep then
main_dep = itemToBranch(main_dep, pass_of_deps)
else
main_dep = GIT_INITIAL
end
local cmd = '%s checkout -q -b %s %s'
assert(execute(cmd:format(GIT, new_branch, main_dep)))
-- merge with other dependencies
for i = 2, #deps do
local message = 'Merge with ' .. deps[i]
local cmd2 = '%s %s merge -q %s -m %q'
if not execute(cmd2:format(GIT,
GIT_USER,
itemToBranch(deps[i], pass_of_deps),
message))
then
-- probably merge conflict
local conflicts = table.concat(gitConflicts(), ' ')
log('Merge conflicts: %s', conflicts)
local cmd3 = '%s checkout --ours %s'
assert(execute(cmd3:format(GIT, conflicts)))
gitCommit(message)
end
end
if #deps > 0 then
-- prevent accidental rebuilds
-- touch usr/*/installed/* files in build order
-- see https://git.io/vuDJY
local installed = getInstalled()
table.sort(installed, function(x, y)
return item2index[x] < item2index[y]
end)
for _, item in ipairs(installed) do
local target, pkg = assert(parseItem(item))
local cmd4 = 'touch -c usr/%s/installed/%s'
execute(cmd4:format(target, pkg))
end
end
end
local function gitAdd()
os.execute(GIT .. 'add --all --force .')
end
-- return two lists of filepaths under ./usr/
-- 1. new files
-- 2. changed files
local function gitStatus(item, item2deps, file2item)
local new_files = {}
local changed_files = {}
local git_st = io.popen(GIT .. 'status --porcelain', 'r')
for line in git_st:lines() do
local status, file = line:match('(..) (.*)')
assert(status:sub(2, 2) == ' ')
status = trim(status)
if file:sub(1, 1) == '"' then
-- filename with a space is quoted by git
file = file:sub(2, -2)
end
file = 'usr/' .. file
if not fileExists(file) then
if status == 'D' then
local prev_owner = assert(file2item[file])
if prev_owner == item then
log('Item %s removed %q installed by itself',
item, file)
elseif isDependency(prev_owner, item, item2deps) then
log('Item %s removed %q installed by its follower %s',
item, file, prev_owner)
else
log('Item %s removed %q installed by %s',
item, file, prev_owner)
end
elseif isSymlink(file) then
log('Broken symlink: %q', file)
else
log('Missing file: %q', file)
end
elseif not isBlacklisted(file) then
if status == 'A' then
table.insert(new_files, file)
elseif status == 'M' then
table.insert(changed_files, file)
else
log('Strange git status: %q of %q',
status, file)
end
end
end
git_st:close()
return new_files, changed_files
end
local function isValidBinary(target, file)
local cmd = './usr/bin/%s-objdump -t %s > /dev/null 2>&1'
return execute(cmd:format(target, file))
end
local function checkFile(file, item)
local target, _ = parseItem(item)
-- if it is PE32 file, it must have '.exe' in name
local ext = file:sub(-4):lower()
local cmd = 'file --dereference --brief %q'
local file_type = trim(shell(cmd:format(file)))
if ext == '.exe' then
if not file_type:match('PE32') then
log('File %s (%s) is %q. Remove .exe',
file, item, file_type)
end
elseif ext == '.dll' then
if not file_type:match('PE32.*DLL') then
log('File %s (%s) is %q. Remove .dll',
file, item, file_type)
end
elseif ext ~= '.bin' then
-- .bin can be an executable or something else (font)
if file_type:match('PE32') then
log('File %s (%s) is %q. Add exe or dll',
file, item, file_type)
end
end
for _, t in ipairs(TARGETS) do
if t ~= target and isInString(t, file) then
log('File %s (%s): other target %s in name',
file, item, t)
end
end
if file:match('/lib/.*%.dll$') then
log('File %s (%s): DLL in /lib/', file, item)
end
if file:match('%.dll$') or file:match('%.a$') then
if isInString(target, file) and isCross(target) then
-- cross-compiled
if not isValidBinary(target, file) then
log('File %s (%s): not recognized library',
file, item)
end
end
end
end
local function checkFileList(files, item)
local target, _ = parseItem(item)
if target:match('shared') then
local has_a, has_dll
for _, file in ipairs(files) do
file = file:lower()
if file:match('%.a') then
has_a = true
end
if file:match('%.dll') then
has_dll = true
end
end
if has_a and not has_dll then
log('Shared item %s installs .a file ' ..
'but no .dll', item)
end
end
end
local function removeEmptyDirs(item)
-- removing an empty dir can reveal another one (parent)
-- don't pass item to mute the log message
local go_on = true
while go_on do
go_on = false
local f = io.popen('find usr/* -empty -type d', 'r')
for dir in f:lines() do
if item then
log("Remove empty directory %s created by %s",
dir, item)
end
os.remove(dir)
go_on = true
end
f:close()
end
end
local function prepareTree(pass, item, item2deps, prev_files, item2index)
if pass == 'first' then
gitCheckout(
itemToBranch(item, pass),
item2deps[item],
item2index,
pass
)
elseif pass == 'second' then
-- Build item second time to check if it builds correctly if
-- its followers and unrelated packages have been built.
gitCheckout(
itemToBranch(item, 'second'),
{GIT_ALL_PSEUDOITEM},
item2index,
'first'
)
removeEmptyDirs()
if prev_files then
-- Remove files of item from previous build.
for _, file in ipairs(prev_files) do
os.remove(file)
end
gitAdd()
gitCommit(("Remove %s to rebuild it"):format(item, pass))
end
else
error("Unknown pass: " .. pass)
end
end
local function comparePasses(item, new_files, prev_file2item, prev_files)
local files_set = {}
for _, file in ipairs(new_files) do
if not prev_file2item[file] then
log('Item %s installs a file on second pass only: %s',
item, file)
elseif prev_file2item[file] ~= item then
log('File %s was installed by %s on first pass ' ..
'and by %s - on the second pass',
file, prev_file2item[file], item)
end
files_set[file] = true
end
if prev_files then
-- prev_files is nil, if the first pass failed
for _, file in ipairs(prev_files) do
if not files_set[file] then
log('Item %s installs a file on first pass only: %s',
item, file)
end
end
end
-- TODO compare contents of files (nm for binaries)
end
local function isBuilt(item, files)
local target, pkg = parseItem(item)
local INSTALLED = 'usr/%s/installed/%s'
local installed = INSTALLED:format(target, pkg)
for _, file in ipairs(files) do
if file == installed then
return true
end
end
return false
end
-- builds package, returns list of new files
-- prev_files is passed only to second pass.
local function buildItem(item, item2deps, file2item, item2index, pass, prev_files)
prepareTree(pass, item, item2deps, prev_files, item2index)
local target, pkg = parseItem(item)
local cmd = '%s %s~%s MXE_TARGETS=%s --jobs=1'
os.execute(cmd:format(tool 'make', pkg, target, target))
gitAdd()
local new_files, changed_files = gitStatus(item, item2deps, file2item)
if #new_files + #changed_files > 0 then
gitCommit(("Build %s, pass %s"):format(item, pass))
end
if pass == 'first' then
for _, file in ipairs(new_files) do
checkFile(file, item)
file2item[file] = item
end
elseif isBuilt(item, new_files) then
comparePasses(item, new_files, file2item, prev_files)
end
for _, file in ipairs(changed_files) do
checkFile(file, item)
-- add a dependency on a package created this file
local creator_item = assert(file2item[file])
if not isInArray(creator_item, item2deps[item]) then
table.insert(item2deps[item], creator_item)
end
log('Item %s (pass %s) changes %s, created by %s',
item, pass, file, creator_item)
end
checkFileList(concatArrays(new_files, changed_files), item)
removeEmptyDirs(item)
return new_files
end
local function nameToDebian(item)
item = item:gsub('[~_]', '-')
local name = 'mxe-%s'
return name:format(item)
end
local function protectVersion(ver)
ver = ver:gsub('_', '.')
if ver:sub(1, 1):match('%d') then
return ver
else
-- version number does not start with digit
return '0.' .. ver
end
end
local CONTROL = [[Package: %s
Version: %s
Section: devel
Priority: optional
Architecture: %s%s
Installed-Size: %d
Maintainer: Boris Nagaev <bnagaev@gmail.com>
Homepage: https://mxe.cc/
Description: %s
MXE (M cross environment) is a Makefile that compiles
a cross compiler and cross compiles many free libraries
such as SDL and Qt for various target platforms (MinGW).
.
%s
]]
local function debianControl(options)
local deb_deps_str = ''
if options.deps and #options.deps >= 1 then
deb_deps_str = deb_deps_str ..
'\n' .. 'Depends: ' ..
table.concat(options.deps, ', ')
end
if options.recommends and #options.recommends >= 1 then
deb_deps_str = deb_deps_str ..
'\n' .. 'Recommends: ' ..
table.concat(options.recommends, ', ')
end
local version = options.version .. '-' .. TODAY
return CONTROL:format(
options.package,
version,
options.arch,
deb_deps_str,
math.ceil(options.size_bytes / 1024),
options.description1,
options.description2
)
end
local function makePackage(name, files, deps, ver, d1, d2, dst, recommends)
-- calculate size_bytes
local size_bytes = 0
for _, f in ipairs(files) do
local size = math.ceil(fileSize(f) / 4096) * 4096
size_bytes = size_bytes + size
end
-- dirname
dst = dst or '.'
local dirname = ('%s/%s_%s'):format(dst, name,
protectVersion(ver))
-- make .list file
local list_path = ('%s/%s.list'):format(dst, name)
writeFile(list_path, table.concat(files, "\n") .. "\n")
-- make .tar.xz file
local tar_name = dirname .. '.tar.xz'
local cmd1 = '%s -T %s --owner=root --group=root -cJf %s'
os.execute(cmd1:format(tool 'tar', list_path, tar_name))
-- update list of files back from .tar.xz (see #1067)
local cmd2 = '%s -tf %s'
cmd2 = cmd2:format(tool 'tar', tar_name)
local tar_reader = io.popen(cmd2, 'r')
local files_str = tar_reader:read('*all')
tar_reader:close()
writeFile(list_path, files_str)
-- make DEBIAN/control file
local control_text = debianControl {
package = name,
version = protectVersion(ver),
arch = ARCH,
deps = deps,
recommends = recommends,
size_bytes = size_bytes,
description1 = d1,
description2 = d2,
}
writeFile(dirname .. ".deb-control", control_text)
if not no_debs then
-- unpack .tar.xz to the path for Debian
local usr = dirname .. MXE_DIR
os.execute(('mkdir -p %s'):format(usr))
os.execute(('mkdir -p %s/DEBIAN'):format(dirname))
-- use tar to copy files with paths
local cmd3 = '%s -C %s -xf %s'
if not no_fakeroot then
cmd3 = 'fakeroot -s deb.fakeroot ' .. cmd3
end
os.execute(cmd3:format(tool 'tar', usr, tar_name))
-- make DEBIAN/control file
local control_fname = dirname .. '/DEBIAN/control'
writeFile(control_fname, control_text)
-- make .deb file
local cmd4 = 'dpkg-deb -Zxz -b %s'
if not no_fakeroot then
cmd4 = 'fakeroot -i deb.fakeroot ' .. cmd4
end
os.execute(cmd4:format(dirname))
-- cleanup
os.execute(('rm -fr %s deb.fakeroot'):format(dirname))
end
end
local D1 = "MXE package %s for %s"
local D2 = "This package contains the files for MXE package %s"
local function makeDeb(item, files, deps, ver)
local target, pkg = parseItem(item)
local deb_pkg = nameToDebian(item)
local d1 = D1:format(pkg, target)
local d2 = D2:format(pkg)
local deb_deps = {'mxe-requirements', 'mxe-source'}
for _, dep in ipairs(deps) do
table.insert(deb_deps, nameToDebian(dep))
end
makePackage(deb_pkg, files, deb_deps, ver, d1, d2)
end
local function findForeignInstalls(item, files)
for _, file in ipairs(files) do
local pattern = 'usr/([^/]+)/installed/([^/]+)'
local t, p = file:match(pattern)
if t and p ~= '.gitkeep' then
local item1 = makeItem(t, p)
if item1 ~= item then
log('Item %s built item %s', item, item1)
end
end
end
end
-- script building HUGE_TIMES from MXE main log
-- https://gist.github.com/starius/3ea9d953b0c30df88aa7
local HUGE_TIMES = {
[7] = {"ocaml-native", "ffmpeg", "boost"},
[9] = {"openssl", "qtdeclarative", "ossim", "wxwidgets"},
[12] = {"ocaml-core", "itk", "wt"},
[19] = {"gcc", "qtbase", "llvm"},
[24] = {"vtk", "vtk6", "openscenegraph"},
[36] = {"openblas", "pcl", "oce"},
[51] = {"qt"},
}
local PROGRESS = "[%3d/%d] " ..
"The build is expected to complete in %0.1f hours, " ..
"on %s"
local function progressPrinter(items)
local pkg2time = {}
for time, pkgs in pairs(HUGE_TIMES) do
for _, pkg in ipairs(pkgs) do
pkg2time[pkg] = time
end
end
--
local started_at = os.time()
local sums = {}
for i, item in ipairs(items) do
local _, pkg = parseItem(item)
local expected_time = pkg2time[pkg] or 1
sums[i] = (sums[i - 1] or 0) + expected_time
end
local total_time = sums[#sums]
local time_done = 0
local pkgs_done = 0
local printer = {}
--
function printer.advance(_, i)
pkgs_done = i
time_done = sums[i]
end
function printer.status(_)
local now = os.time()
local spent = now - started_at
local predicted_duration = spent * total_time / time_done
local predicted_end = started_at + predicted_duration
local predicted_end_str = os.date("%c", math.floor(predicted_end + 0.5))
local predicted_wait = predicted_end - now
local predicted_wait_hours = predicted_wait / 3600.0
return PROGRESS:format(pkgs_done, #items,
predicted_wait_hours, predicted_end_str)
end
return printer
end
-- build all packages, save filelist to list file
-- prev_files is passed only to second pass.
local function buildPackages(items, item2deps, pass, prev_item2files)
local broken = {}
local unbroken = {}
local file2item = {}
local item2files = {}
local function brokenDep(item)
for _, dep in ipairs(item2deps[item]) do
if broken[dep] then
return dep
end
end
return false
end
if pass == 'second' then
assert(prev_item2files)
-- fill file2item with data from prev_item2files
for item, files in pairs(prev_item2files) do
for _, file in ipairs(files) do
file2item[file] = item
end
end
end
local item2index = makeItem2Index(items)
local progress_printer = progressPrinter(items)
for i, item in ipairs(items) do
if not brokenDep(item) then
local prev_files = prev_item2files and prev_item2files[item]
local files = buildItem(
item, item2deps, file2item, item2index, pass, prev_files
)
findForeignInstalls(item, files)
if isBuilt(item, files) then
item2files[item] = files
table.insert(unbroken, item)
else
-- broken package
broken[item] = true
log('Item is broken: %s', item)
end
else
broken[item] = true
log('Item %s depends on broken item %s',
item, brokenDep(item))
end
progress_printer:advance(i)
echo(progress_printer:status())
end
return unbroken, item2files
end
local function makeDebs(items, item2deps, item2ver, item2files)
for _, item in ipairs(items) do
local deps = assert(item2deps[item], item)
local ver = assert(item2ver[item], item)
local files = assert(item2files[item], item)
makeDeb(item, files, deps, ver)
end
end
local function getMxeVersion()
local index_html = io.open 'docs/index.html'
local text = index_html:read('*all')
index_html:close()
return text:match('Release ([^<]+)')
end
local MXE_REQUIREMENTS_DESCRIPTION2 =
[[This package depends on all Debian dependencies of MXE.
Other MXE packages depend on this package.]]
local function makeMxeRequirementsPackage(release)
os.execute(('mkdir -p %s'):format(release))
local name = 'mxe-requirements'
local ver = getMxeVersion() .. release
-- MXE build requirements should not be strict dependencies here
-- See https://github.com/mxe/mxe/issues/1537
local deps = {}
local recommends = {
'autoconf', 'automake', 'autopoint', 'bash', 'bison',
'bzip2', 'cmake', 'flex', 'gettext', 'git', 'g++',
'gperf', 'intltool', 'libffi-dev', 'libtool',
'libltdl-dev', 'libssl-dev', 'libxml-parser-perl',
'make', 'openssl', 'patch', 'perl', 'p7zip-full',
'pkg-config', 'python', 'ruby', 'scons', 'sed',
'unzip', 'wget', 'xz-utils',
'g++-multilib', 'libc6-dev-i386',
}
if release ~= 'wheezy' then
-- Jessie+
table.insert(recommends, 'libtool-bin')
end
local dummy_name = 'mxe-requirements.dummy.' .. release
local dummy = io.open(dummy_name, 'w')
dummy:close()
local files = {dummy_name}
local d1 = "MXE requirements package"
local d2 = MXE_REQUIREMENTS_DESCRIPTION2
local dst = release
makePackage(name, files, deps, ver, d1, d2, dst, recommends)
os.remove(dummy_name)
end
local MXE_SOURCE_DESCRIPTION2 =
[[This package contains MXE source files.
Other MXE packages depend on this package.]]
local function makeMxeSourcePackage()
local name = 'mxe-source'
local ver = getMxeVersion()
-- dependencies
local deps = {}
local files = {
'docs',
'ext',
'LICENSE.md',
'Makefile',
'mxe.github.mk',
'mxe.patch.mk',
'mxe.updates.mk',
'plugins',
'README.md',
'src',
'tools',
}
local d1 = "MXE source"
local d2 = MXE_SOURCE_DESCRIPTION2
makePackage(name, files, deps, ver, d1, d2)
end
local function downloadPackages()
local cmd = ('%s download -j 6 -k'):format(tool 'make')
for i = 1, MAX_TRIES do
log("Downloading packages. Attempt %d.", i)
if execute(cmd) then
log("All packages were downloaded.")
return
end
log("Some packages failed to download.")
end
log("%d downloading attempts failed. Giving up.", MAX_TRIES)
error('downloading failed')
end
local function main()
assert(not io.open('usr/.git'), 'Remove usr/')
local MXE_DIR_EXPECTED = '/usr/lib/mxe'
if MXE_DIR ~= MXE_DIR_EXPECTED then
log("Warning! Building in dir %s, not in %s",
MXE_DIR, MXE_DIR_EXPECTED)
end
gitInit()
assert(execute(("%s check-requirements nonet-lib print-git-oneline MXE_TARGETS=%q"):format(
tool 'make', table.concat(TARGETS, ' '))))
if not max_items then
downloadPackages()
end
gitAdd()
gitCommit('Initial commit')
gitTag(GIT_INITIAL)
local items, item2deps, item2ver = getItems()
local build_list = sortForBuild(items, item2deps)
assert(isTopoOrdered(build_list, items, item2deps))
build_list = sliceArray(build_list, max_items)
local first_pass_failed, second_pass_failed
local unbroken, item2files = buildPackages(
build_list, item2deps, 'first'
)
if #unbroken < #build_list then
first_pass_failed = true
end
gitCheckout(
itemToBranch(GIT_ALL_PSEUDOITEM, 'first'),
unbroken,
makeItem2Index(build_list),
'first'
)
makeDebs(unbroken, item2deps, item2ver, item2files)
if not no_debs then
makeMxeRequirementsPackage('wheezy')
makeMxeRequirementsPackage('jessie')
end
makeMxeSourcePackage()
if not no_second_pass then
local unbroken_second = buildPackages(
build_list, item2deps, 'second', item2files
)
if #unbroken_second < #build_list then
second_pass_failed = true
end
end
if first_pass_failed or second_pass_failed then
local code = 1
local close = true
os.exit(code, close)
end
end
main()