Skip to content


Subversion checkout URL

You can clone with
Download ZIP
Tree: 6c807d5843
Fetching contributors…

Cannot retrieve contributors at this time

557 lines (463 sloc) 19.703 kB
# tools python has no gdbm, breaks simple home python/lilypond build
# but dbhash seems to break in odd ways:
# File "bsddb/", line 62, in DeadlockWrap
# DBPageNotFoundError: (-30987, 'DB_PAGE_NOTFOUND: Requested page not found')
import fcntl
import glob
import inspect
import os
import pickle
import re
import string
import sys
from gub.syntax import printf
from gub import build
from gub import cross
from gub.db import db
from gub import dependency
from gub import locker
from gub import logging
from gub import loggedos
from gub import misc
import gub.settings
from gub import target
from gub import cygwin
from gub import debian
class GupException (Exception):
class FileManager:
"""FileManager handles a tree, and keeps track of files,
associating files with a package name"""
def __init__ (self, root, dbdir=None, clean=False):
self.root = os.path.normpath (root)
if dbdir:
self.config = dbdir
self.config = self.root + '/etc/gup'
self.config = os.path.normpath (self.config)
self.verbose = True
self.is_distro = False
## lock must be outside of root, otherwise we can't rm -rf root
self.lock_file = self.root + '.lock'
self.lock = locker.Locker (self.lock_file)
if clean:
loggedos.system (logging.default_logger,
'rm -fr %s' % self.config)
self.make_dirs ()
files_db = self.config + '/files.db'
packages_db = self.config + '/packages.db'
self._file_package_db = (files_db, 'c')
self._package_file_db = (packages_db, 'c')
#except DBInvalidArgError:
# import gdmb
# file_db = (file_db, 'c')
# packages_db = (packages_db, 'c')
def __repr__ (self):
name = self.__class__.__name__
root = self.root
distro = self.is_distro
return '%(name)s: %(root)s, distro: %(distro)d' % locals ()
def make_dirs (self):
if not os.path.isdir (self.config):
loggedos.system (logging.default_logger,
'mkdir -p %s' % self.config)
if not os.path.isdir (self.root):
loggedos.system (logging.default_logger,
'mkdir -p %s' % self.root)
def package_installed_files (self, name):
lst = self._package_file_db[name]
if lst:
return [file_name for file_name in lst.decode ('utf8').split ('\n')]
return []
def installed_packages (self):
return [name.decode ('utf8') for name in list (self._package_file_db.keys ())]
def is_installed (self, name):
return name in self.installed_packages ()
def installed_files (self):
return [file_name.decode ('utf8') for file_name in list (self._file_package_db.keys ())]
def is_installed_file (self, name):
return name in self.installed_files ()
def install_tarball (self, ball, name, prefix_dir):
logging.action ('untarring: %(ball)s\n' % locals ())
_z = misc.compression_flag (ball)
_v = '' # self.os_interface.verbose_flag ()
lst = loggedos.read_pipe (logging.default_logger,
'tar -t%(_z)s -f "%(ball)s"'
% locals ()).split ('\n')
conflicts = False
installed_files = self.installed_files ()
installed_files_string = ':'.join ([''] + installed_files + [''])
misc.timing ()
for f in lst:
if (':' + f + ':' in installed_files_string
and not os.path.isdir (os.path.join (self.root, f))):
package = self._file_package_db[f]
logging.error ('already have file %(f)s: %(package)s\n'
% locals ())
conflicts = True
logging.command ('GUP: for f in lst:' + misc.timing () + '\n')
if conflicts and not self.is_distro:
raise Exception ('Duplicate files found.')
root = self.root
loggedos.system (logging.default_logger,
# cd %(root)s to avoid open(2) of cwd, see
'cd %(root)s && tar -C %(root)s -p -x%(_z)s%(_v)s -f %(ball)s'
% locals ())
for f in lst:
if f.endswith ('.la'):
self.libtool_la_fixup (root, f)
if f.endswith ('.pc'):
self.pkgconfig_pc_fixup (root, f, prefix_dir)
self._package_file_db[name] = '\n'.join (lst)
for f in lst:
# ignore directories.
if not f.endswith ('/'):
self._file_package_db[f] = name
if os.path.exists ('%(root)s/usr/etc/postinstall/%(name)s' % locals ()):
loggedos.system (logging.default_logger,
'PATH=%(root)s/usr/bin:$PATH %(root)s/usr/etc/postinstall/%(name)s && mv %(root)s/usr/etc/postinstall/%(name)s %(root)s/usr/etc/postinstall/%(name)s.done || :' % locals ())
def libtool_la_fixup (self, root, file):
# avoid using libs from build platform, by adding
# %(system_root)s
if file.startswith ('./'):
file = file[2:]
dir = os.path.dirname (file)
loggedos.file_sub (logging.default_logger,
"""libdir='%(root)s/%(dir)s'""" % locals ()
'%(root)s/%(file)s' % locals (),
must_succeed=('tools/root' not in self.root
and 'tools32/root' not in self.root
and 'cross' not in dir
and '/GUB' not in self.root))
def pkgconfig_pc_fixup (self, root, file, prefix_dir):
# avoid using libs from build platform, by adding
# %(system_root)s
if file.startswith ('./'):
file = file[2:]
dir = os.path.dirname (file)
if '%' in prefix_dir or not prefix_dir:
loggedos.file_sub (logging.default_logger,
[('(-I|-L) */usr',
'''\\1%(root)s%(prefix_dir)s''' % locals ()
'%(root)s/%(file)s' % locals ())
def uninstall_package (self, name):
logging.action ('uninstalling package: %s\n' % name)
lst = self.package_installed_files (name)
dirs = []
files = []
for i in lst:
f = os.path.join (self.root, i)
if os.path.islink (f):
files.append (f)
elif (not os.path.exists (f)
and not self.is_distro):
printf ('FileManager: uninstall: %s' % name)
printf ('FileManager: no such file: %s' % f)
elif os.path.isdir (f):
dirs.append (f)
files.append (f)
for f in files:
os.unlink (f)
for d in reversed (dirs):
os.rmdir (d)
except OSError:
logging.verbose ('warning: %(d)s not empty\n' % locals ())
for f in lst:
## fixme (?) -- when is f == ''
if not f or f.endswith ('/'):
del self._file_package_db[f]
printf ('db delete failing for ', f)
del self._package_file_db[name]
class PackageDictManager:
A dict of PackageName -> (Key->Value dict)
which can be read off the disk.
def __init__ (self):
self._packages = {}
## ugh mi
self.verbose = False
def register_package_dict (self, d):
nm = d['name']
if 'split_name' in d:
nm = d['split_name']
if 0 and (nm in self._packages):
if self._packages[nm]['spec_checksum'] != d['spec_checksum']: ('******** checksum of %s changed!\n\n' % nm)
if self._packages[nm]['cross_checksum'] != d['cross_checksum']: ('******** checksum of cross changed for %s\n' % nm)
self._packages[nm] = d
def register_package_header (self, package_hdr, branch_dict):
if self.verbose: ('reading package header: %s\n'
% package_hdr.__repr__ ())
str = open (package_hdr).read ()
header_name = os.path.basename (package_hdr)
d = dict (pickle.loads (str))
name = d['basename']
vc_branch = d.get ('vc_branch', '')
if name in branch_dict:
branch = branch_dict[name]
if ':' in branch:
(remote_branch, branch) = tuple (branch.split (':'))
if branch != vc_branch:
logging.error ('package of branch: %(vc_branch)s, expecting: %(branch)s\n' % locals ())
logging.error ('ignoring header: %(header_name)s\n' % locals ())
elif d['vc_branch']:
logging.error ('no branch for package: %(name)s\n' % locals ())
logging.error ('ignoring header: %(header_name)s\n' % locals ())
logging.error ('available branch: %(vc_branch)s\n' % locals ())
name = d['split_name']
if 0:
## FIXME ?
if name in self._package_dict_db:
if str != self._package_dict_db[name]: ("package header changed for %s\n" % name)
self.register_package_dict (d)
def unregister_package_dict (self, name):
del self._packages[name]
def is_registered (self, name):
return name in self._packages
def package_dict (self, name):
return self._packages.get (name, dict ())
def available_packages (self):
return list (self._packages.keys ())
def get_all_packages (self):
return list (self._packages.values ())
def is_installable (self, name):
d = self.package_dict (name)
ball = '%(split_ball)s' % d
hdr = '%(split_hdr)s' % d
return os.path.exists (ball) and os.path.exists (hdr)
def read_package_headers (self, s, branch_dict):
if os.path.isdir (s) and not s.endswith ('/'):
s += '/'
for f in glob.glob ('%(s)s*hdr' % locals ()):
self.register_package_header (f, branch_dict)
class PackageManager (FileManager, PackageDictManager):
"""PackageManager is a FileManager, which also associates a
key/value dict with each package.
Such dicts come either from either
1. A build spec (ie. a python object)
2. A pickled dict on disk, a package header
def __init__ (self, root, **kwargs):
FileManager.__init__ (self, root, **kwargs)
PackageDictManager.__init__ (self)
dicts_db = self.config + '/dicts.db'
self._package_dict_db = (dicts_db, 'c')
for k in list (self._package_dict_db.keys ()):
v = self._package_dict_db[k]
self.register_package_dict (pickle.loads (v))
def installed_package_dicts (self):
return [self._packages[n] for n in self.installed_packages ()]
def install_package (self, name):
if self.is_installed (name):
logging.action ('installing package: %s\n' % name)
if self.is_installed (name):
message = 'already have package: ' + name + '\n'
logging.error (message)
raise Exception (message)
d = self._packages[name]
ball = '%(split_ball)s' % d
self.install_tarball (ball, name, d['prefix_dir'])
self._package_dict_db[name] = pickle.dumps (d, protocol=2)
def uninstall_package (self, name):
FileManager.uninstall_package (self, name)
del self._package_dict_db[name]
def source_name (self, name):
return self._packages [name]['source_name']
class DependencyManager (PackageManager):
"""Manage packages that have dependencies and
build_dependencies in their package dicts"""
def __init__ (self, *args, **kwargs):
PackageManager.__init__ (self, *args, **kwargs)
self.include_build_deps = True
def dependencies (self, name):
assert type (name) == str
return [misc.strip_platform (x)
for x in self.dict_dependencies (self._packages[name])]
except KeyError:
logging.error ('no such package: %(name)s\n' % locals ())
return list ()
def dict_dependencies (self, dict):
deps = dict['dependencies_string'].split (';')
if self.include_build_deps:
deps += dict['build_dependencies_string'].split (';')
deps = [d for d in deps if d]
return deps
# UGh moveme
def topologically_sorted_one (todo, done, dependency_getter,
sorted = []
if todo in done:
return sorted
done[todo] = 1
def type_equal (a, b):
return ((type (a) == type (b))
or inspect.isclass (type (a)) == inspect.isclass (type (b)))
deps = dependency_getter (todo)
for d in deps:
if recurse_stop_predicate and recurse_stop_predicate (d):
if not type_equal (d, todo):
printf (type (d), '!=', type (todo))
printf (d.__class__, todo.__class__)
printf (d.__dict__, todo.__dict__)
printf (inspect.isclass (type (d)), inspect.isclass (type (todo)))
assert type_equal (a, b)
sorted += topologically_sorted_one (d, done, dependency_getter,
sorted.append (todo)
return sorted
def topologically_sorted (todo, done, dependency_getter,
s = []
for t in todo:
s += topologically_sorted_one (t, done, dependency_getter,
return s
# this is too hairy. --hwn
def gub_to_distro_deps (deps, gub_to_distro_dict):
distro = []
for i in deps:
if i in list (gub_to_distro_dict.keys ()):
distro += gub_to_distro_dict[i]
distro += [i]
return distro
def get_base_package_name (name):
# FIXME: rename packages, fragile
if [True for x in [
] if x in name]:
return name
name = re.sub ('(gcc(-.*)?)-c[+][+]-runtime', r'\1', name)
name = re.sub ('-devel$', '', name)
name = re.sub ('-runtime$', '', name)
name = re.sub ('-doc$', '', name)
return name
def get_source_packages (settings, const_todo):
"""TODO is a list of (source) builds.
Generate a list of AutoBuild needed to build TODO, in
topological order
# Do not confuse caller, do not modify caller's todo
todo = const_todo[:]
specs = dict ()
sets = {settings.platform: settings}
def with_platform (s, platform=settings.platform):
return misc.with_platform (s, platform)
def split_platform (u):
return misc.split_platform (u, settings.platform)
def name_to_dependencies_via_gub (url):
platform, url = split_platform (url)
if ':' in url:
base, unused_parameters = misc.dissect_url (url)
name = (os.path.basename (base)
.replace ('.git', ''))
key = url
name = get_base_package_name (url)
url = None
key = name
key = with_platform (key, platform)
if key in specs:
spec = specs[key]
if platform not in sets:
sets[platform] = gub.settings.Settings (platform)
spec = dependency.Dependency (sets[platform], name, url).build ()
specs[key] = spec
return list (map (get_base_package_name, spec.get_platform_build_dependencies ()))
def name_to_dependencies_via_distro (distro_packages, url):
platform, url = split_platform (url)
if ':' in url:
base, unused_parameters = misc.dissect_url (url)
name = (os.path.basename (base)
.replace ('.git', ''))
key = url
name = url #get_base_package_name (url)
url = None
key = name
key = with_platform (key, platform)
if key in specs:
spec = specs[key]
if name in todo or name not in list (distro_packages.keys ()):
if platform not in sets:
sets[platform] = gub.settings.Settings (platform)
spec = dependency.Dependency (sets[platform], name).build ()
spec = distro_packages[name]
specs[key] = spec
return list (map (get_base_package_name, spec.get_platform_build_dependencies ()))
def name_to_dependencies_via_cygwin (name):
if not cygwin.dependency_resolver:
cygwin.init_dependency_resolver (settings, const_todo[:])
return name_to_dependencies_via_distro (cygwin.get_packages (), name)
def name_to_dependencies_via_debian (name):
if not debian.dependency_resolver:
debian.init_dependency_resolver (settings, const_todo[:])
return name_to_dependencies_via_distro (debian.get_packages (), name)
name_to_dependencies = {
'cygwin': name_to_dependencies_via_cygwin,
'debian': name_to_dependencies_via_debian,
def name_to_dependencies_broker (url):
platform, x = split_platform (url)
return name_to_dependencies.get (platform,
name_to_dependencies_via_gub) (url)
# Must iterate, try:
# bin/gub -p tools tar
# bin/gub -p tools make tar
# or
# bin/gub darwin-ppc::libtool freebsd-64::libtool
last_count = len (todo)
while last_count != len (list (specs.keys ())):
add = cross.set_cross_dependencies (specs)
todo += [a for a in add if a not in todo]
last_count = len (list (specs.keys ()))
topologically_sorted (todo, {}, name_to_dependencies_broker)
# Fixup for build from url: specs key is full url, change to
# base name. Must use list (dict.keys()), since dict changes during
# iteration.
for name in list (specs.keys ()):
spec = specs[name]
if name != spec.platform_name ():
specs[spec.platform_name ()] = spec
def obj_to_dependency_objects (obj):
return [specs[get_base_package_name (n)]
for n in obj.get_platform_build_dependencies ()]
sorted_specs = topologically_sorted (list (specs.values ()), {},
# Make sure we build dependencies in order
sorted_names = [o.platform_name () for o in sorted_specs]
return (sorted_names, specs)
Jump to Line
Something went wrong with that request. Please try again.