diff --git a/.travis.yml b/.travis.yml index 59f17daf040..a3802e3e2ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_script: - sudo lxc config device add xenial /dev/sda1 disk source=$(pwd) path=$(pwd) # Install the snapcraft dependencies. - sudo lxc exec xenial -- apt-get update - - sudo lxc exec xenial -- apt-get install -y lxd pyflakes python-flake8 python3.5 python3-apt python3-docopt python3-coverage python3-fixtures python3-flake8 python3-jsonschema python3-mccabe python3-mock python3-pep8 python3-pexpect python3-pip python3-requests python3-requests-oauthlib python3-responses python3-ssoclient python3-testscenarios python3-testtools python3-xdg python3-yaml python3-lxml squashfs-tools python3-progressbar python3-requests-toolbelt python3-petname + - sudo lxc exec xenial -- apt-get install -y lxd pyflakes python-flake8 python3.5 python3-apt python3-magic python3-docopt python3-coverage python3-fixtures python3-flake8 python3-jsonschema python3-mccabe python3-mock python3-pep8 python3-pexpect python3-pip python3-requests python3-requests-oauthlib python3-responses python3-ssoclient python3-testscenarios python3-testtools python3-xdg python3-yaml python3-lxml squashfs-tools python3-progressbar python3-requests-toolbelt python3-petname - sudo lxc exec xenial -- usermod ubuntu -a -G lxd script: - sudo -E lxc exec xenial -- su - ubuntu -c "cd $(pwd); TEST_USER_PASSWORD=$TEST_USER_PASSWORD ./runtests.sh $TEST_SUITE" diff --git a/debian/control b/debian/control index a3250365b46..492a4f1b435 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 9), python3-fixtures, python3-jsonschema, python3-lxml, + python3-magic, python3-petname, python3-pkg-resources, python3-progressbar, @@ -33,6 +34,7 @@ Depends: lxd, python3-docopt, python3-jsonschema, python3-lxml, + python3-magic, python3-petname, python3-pkg-resources, python3-progressbar, diff --git a/examples/godd/snapcraft.yaml b/examples/godd/snapcraft.yaml index fd944e6658f..060ecf0e2d0 100644 --- a/examples/godd/snapcraft.yaml +++ b/examples/godd/snapcraft.yaml @@ -14,11 +14,4 @@ parts: plugin: go source: https://github.com/mvo5/godd source-type: git - build-packages: [gcc] - stage-packages: [libgudev-1.0-dev] - snap: - - usr/lib/x86_64-linux-gnu/libgudev-1.0.so* - - usr/lib/x86_64-linux-gnu/libobject-2.0.so* - - usr/lib/x86_64-linux-gnu/libglib-2.0.so* - - lib/x86_64-linux-gnu/libglib-2.0.so* - - bin/godd* + build-packages: [gcc, libgudev-1.0-dev] diff --git a/libraries/16.04 b/libraries/16.04 new file mode 100644 index 00000000000..d41bd1520cb --- /dev/null +++ b/libraries/16.04 @@ -0,0 +1,513 @@ +libBrokenLocale-2.21.so +libBrokenLocale.so.1 +libSegFault.so +libanl-2.21.so +libanl.so.1 +libc-2.21.so +libc.so.6 +libcidn-2.21.so +libcidn.so.1 +libcrypt-2.21.so +libcrypt.so.1 +libdl-2.21.so +libdl.so.2 +libgcc_s.so.1 +libm-2.21.so +libm.so.6 +libmemusage.so +libnsl-2.21.so +libnsl.so.1 +libnss_compat-2.21.so +libnss_compat.so.2 +libnss_dns-2.21.so +libnss_dns.so.2 +libnss_files-2.21.so +libnss_files.so.2 +libnss_hesiod-2.21.so +libnss_hesiod.so.2 +libnss_nis-2.21.so +libnss_nis.so.2 +libnss_nisplus-2.21.so +libnss_nisplus.so.2 +libpcprofile.so +libpthread-2.21.so +libpthread.so.0 +libresolv-2.21.so +libresolv.so.2 +librt-2.21.so +librt.so.1 +libthread_db-1.0.so +libthread_db.so.1 +libutil-2.21.so +libutil.so.1 +libip4tc.so.0 +libip4tc.so.0.1.0 +libip6tc.so.0 +libip6tc.so.0.1.0 +libiptc.so.0 +libiptc.so.0.0.0 +libxtables.so.10 +libxtables.so.10.0.0 +libBrokenLocale-2.21.so +libBrokenLocale.so.1 +libSegFault.so +libacl.so.1 +libacl.so.1.1.0 +libanl-2.21.so +libanl.so.1 +libapparmor.so.1 +libapparmor.so.1.3.0 +libattr.so.1 +libattr.so.1.1.0 +libaudit.so.1 +libaudit.so.1.0.0 +libblkid.so.1 +libblkid.so.1.1.0 +libbsd.so.0 +libbsd.so.0.8.2 +libbz2.so.1 +libbz2.so.1.0 +libbz2.so.1.0.4 +libc-2.21.so +libc.so.6 +libcap.so.2 +libcap.so.2.24 +libcgmanager.so.0 +libcgmanager.so.0.0.0 +libcidn-2.21.so +libcidn.so.1 +libcom_err.so.2 +libcom_err.so.2.1 +libcrypt-2.21.so +libcrypt.so.1 +libcrypto.so.1.0.0 +libcryptsetup.so.4 +libcryptsetup.so.4.6.0 +libdbus-1.so.3 +libdbus-1.so.3.14.6 +libdevmapper.so.1.02.1 +libdl-2.21.so +libdl.so.2 +libdns-export.so.100 +libdns-export.so.100.2.2 +libe2p.so.2 +libe2p.so.2.3 +libexpat.so.1 +libexpat.so.1.6.0 +libext2fs.so.2 +libext2fs.so.2.4 +libfdisk.so.1 +libfdisk.so.1.1.0 +libfuse.so.2 +libfuse.so.2.9.4 +libgcc_s.so.1 +libgcrypt.so.20 +libgcrypt.so.20.0.4 +libglib-2.0.so.0 +libglib-2.0.so.0.4705.0 +libgpg-error.so.0 +libgpg-error.so.0.17.0 +libhistory.so.6 +libhistory.so.6.3 +libisc-export.so.95 +libisc-export.so.95.5.0 +libjson-c.so.2 +libjson-c.so.2.0.0 +libkeyutils.so.1 +libkeyutils.so.1.5 +libkmod.so.2 +libkmod.so.2.2.11 +liblzma.so.5 +liblzma.so.5.0.0 +liblzo2.so.2 +liblzo2.so.2.0.0 +libm-2.21.so +libm.so.6 +libmemusage.so +libmnl.so.0 +libmnl.so.0.1.0 +libmount.so.1 +libmount.so.1.1.0 +libncurses.so.5 +libncurses.so.5.9 +libncursesw.so.5 +libncursesw.so.5.9 +libnewt.so.0.52 +libnewt.so.0.52.18 +libnih-dbus.so.1 +libnih-dbus.so.1.0.0 +libnih.so.1 +libnih.so.1.0.0 +libnl-3.so.200 +libnl-3.so.200.22.0 +libnl-genl-3.so.200 +libnl-genl-3.so.200.22.0 +libnsl-2.21.so +libnsl.so.1 +libnss_compat-2.21.so +libnss_compat.so.2 +libnss_dns-2.21.so +libnss_dns.so.2 +libnss_files-2.21.so +libnss_files.so.2 +libnss_hesiod-2.21.so +libnss_hesiod.so.2 +libnss_myhostname.so.2 +libnss_nis-2.21.so +libnss_nis.so.2 +libnss_nisplus-2.21.so +libnss_nisplus.so.2 +libpam.so.0 +libpam.so.0.83.1 +libpam_misc.so.0 +libpam_misc.so.0.82.0 +libpamc.so.0 +libpamc.so.0.82.1 +libparted.so.2 +libparted.so.2.0.1 +libpcprofile.so +libpcre.so.3 +libpcre.so.3.13.2 +libpcsclite.so.1 +libpcsclite.so.1.0.0 +libpng12.so.0 +libpng12.so.0.54.0 +libpopt.so.0 +libpopt.so.0.0.0 +libprocps.so.4 +libprocps.so.4.0.0 +libpthread-2.21.so +libpthread.so.0 +libreadline.so.6 +libreadline.so.6.3 +libresolv-2.21.so +libresolv.so.2 +librt-2.21.so +librt.so.1 +libseccomp.so.2 +libseccomp.so.2.2.3 +libselinux.so.1 +libsepol.so.1 +libslang.so.2 +libslang.so.2.3.0 +libsmartcols.so.1 +libsmartcols.so.1.1.0 +libss.so.2 +libss.so.2.0 +libssl.so.1.0.0 +libsystemd.so.0 +libsystemd.so.0.13.0 +libthread_db-1.0.so +libthread_db.so.1 +libtinfo.so.5 +libtinfo.so.5.9 +libudev.so.1 +libudev.so.1.6.4 +libulockmgr.so.1 +libulockmgr.so.1.0.1 +libusb-0.1.so.4 +libusb-0.1.so.4.4.4 +libutil-2.21.so +libutil.so.1 +libuuid.so.1 +libuuid.so.1.3.0 +libwrap.so.0 +libwrap.so.0.7.6 +libz.so.1 +libz.so.1.2.8 +libip6t_DNAT.so +libip6t_DNPT.so +libip6t_HL.so +libip6t_LOG.so +libip6t_MASQUERADE.so +libip6t_NETMAP.so +libip6t_REDIRECT.so +libip6t_REJECT.so +libip6t_SNAT.so +libip6t_SNPT.so +libip6t_ah.so +libip6t_dst.so +libip6t_eui64.so +libip6t_frag.so +libip6t_hbh.so +libip6t_hl.so +libip6t_icmp6.so +libip6t_ipv6header.so +libip6t_mh.so +libip6t_rt.so +libipt_CLUSTERIP.so +libipt_DNAT.so +libipt_ECN.so +libipt_LOG.so +libipt_MASQUERADE.so +libipt_MIRROR.so +libipt_NETMAP.so +libipt_REDIRECT.so +libipt_REJECT.so +libipt_SAME.so +libipt_SNAT.so +libipt_TTL.so +libipt_ULOG.so +libipt_ah.so +libipt_icmp.so +libipt_realm.so +libipt_ttl.so +libipt_unclean.so +libxt_AUDIT.so +libxt_CHECKSUM.so +libxt_CLASSIFY.so +libxt_CONNMARK.so +libxt_CONNSECMARK.so +libxt_CT.so +libxt_DSCP.so +libxt_HMARK.so +libxt_IDLETIMER.so +libxt_LED.so +libxt_MARK.so +libxt_NFLOG.so +libxt_NFQUEUE.so +libxt_NOTRACK.so +libxt_RATEEST.so +libxt_SECMARK.so +libxt_SET.so +libxt_SYNPROXY.so +libxt_TCPMSS.so +libxt_TCPOPTSTRIP.so +libxt_TEE.so +libxt_TOS.so +libxt_TPROXY.so +libxt_TRACE.so +libxt_addrtype.so +libxt_bpf.so +libxt_cluster.so +libxt_comment.so +libxt_connbytes.so +libxt_connlabel.so +libxt_connlimit.so +libxt_connmark.so +libxt_conntrack.so +libxt_cpu.so +libxt_dccp.so +libxt_devgroup.so +libxt_dscp.so +libxt_ecn.so +libxt_esp.so +libxt_hashlimit.so +libxt_helper.so +libxt_iprange.so +libxt_ipvs.so +libxt_length.so +libxt_limit.so +libxt_mac.so +libxt_mark.so +libxt_multiport.so +libxt_nfacct.so +libxt_osf.so +libxt_owner.so +libxt_physdev.so +libxt_pkttype.so +libxt_policy.so +libxt_quota.so +libxt_rateest.so +libxt_recent.so +libxt_rpfilter.so +libxt_sctp.so +libxt_set.so +libxt_socket.so +libxt_standard.so +libxt_state.so +libxt_statistic.so +libxt_string.so +libxt_tcp.so +libxt_tcpmss.so +libxt_time.so +libxt_tos.so +libxt_u32.so +libxt_udp.so +libstdbuf.so +libinproctrace.so +libnss_extrausers.so.2 +libopencryptoki.so.0 +libopencryptoki.so.0.0.0 +libpkcs11_sw.so +libpkcs11_sw.so.0 +libpkcs11_sw.so.0.0.0 +libpkcs11_tpm.so +libpkcs11_tpm.so.0 +libpkcs11_tpm.so.0.0.0 +libxmlparse.so.1 +libxmlparse.so.1.2 +libxmltok.so.1 +libxmltok.so.1.2 +libopencryptoki.so.0 +libopencryptoki.so.0.0.0 +libpkcs11_sw.so +libpkcs11_sw.so.0 +libpkcs11_sw.so.0.0.0 +libpkcs11_tpm.so +libpkcs11_tpm.so.0 +libpkcs11_tpm.so.0.0.0 +libopencryptoki.so +libsudo_util.so +libsudo_util.so.0 +libsudo_util.so.0.0.0 +libQt5Core.so.5 +libQt5Core.so.5.5 +libQt5Core.so.5.5.1 +libQt5DBus.so.5 +libQt5DBus.so.5.5 +libQt5DBus.so.5.5.1 +libQt5Network.so.5 +libQt5Network.so.5.5 +libQt5Network.so.5.5.1 +libQt5Sql.so.5 +libQt5Sql.so.5.5 +libQt5Sql.so.5.5.1 +libapt-inst.so.2.0 +libapt-inst.so.2.0.0 +libapt-pkg.so.5.0 +libapt-pkg.so.5.0.0 +libasprintf.so.0 +libasprintf.so.0.0.0 +libboost_program_options.so.1.58.0 +libcap-ng.so.0 +libcap-ng.so.0.0.0 +libdb-5.3.so +libdbus-glib-1.so.2 +libdbus-glib-1.so.2.3.3 +libdebconfclient.so.0 +libdebconfclient.so.0.0.0 +libedit.so.2 +libedit.so.2.0.53 +libefiboot.so.0 +libefiboot.so.0.21 +libefivar.so.0 +libefivar.so.0.21 +libestr.so.0 +libestr.so.0.0.0 +libexpatw.so.1 +libexpatw.so.1.6.0 +libffi.so.6 +libffi.so.6.0.4 +libform.so.5 +libform.so.5.9 +libformw.so.5 +libformw.so.5.9 +libfreetype.so.6 +libfreetype.so.6.11.1 +libfwup.so.0 +libfwup.so.0.5 +libgdbm.so.3 +libgdbm.so.3.0.0 +libgdbm_compat.so.3 +libgdbm_compat.so.3.0.0 +libgflags.so.2 +libgflags.so.2.1.2 +libgflags_nothreads.so.2 +libgflags_nothreads.so.2.1.2 +libgio-2.0.so.0 +libgio-2.0.so.0.4705.0 +libgirepository-1.0.so.1 +libgirepository-1.0.so.1.0.0 +libglog.so.0 +libglog.so.0.0.0 +libgmodule-2.0.so.0 +libgmodule-2.0.so.0.4705.0 +libgmp.so.10 +libgmp.so.10.3.0 +libgnutls-deb0.so.28 +libgnutls-deb0.so.28.41.12 +libgnutls-openssl.so.27 +libgnutls-openssl.so.27.0.2 +libgobject-2.0.so.0 +libgobject-2.0.so.0.4705.0 +libgssapi_krb5.so.2 +libgssapi_krb5.so.2.2 +libgthread-2.0.so.0 +libgthread-2.0.so.0.4705.0 +libhogweed.so.4 +libhogweed.so.4.1 +libicudata.so.55 +libicudata.so.55.1 +libicui18n.so.55 +libicui18n.so.55.1 +libicuio.so.55 +libicuio.so.55.1 +libicule.so.55 +libicule.so.55.1 +libiculx.so.55 +libiculx.so.55.1 +libicutest.so.55 +libicutest.so.55.1 +libicutu.so.55 +libicutu.so.55.1 +libicuuc.so.55 +libicuuc.so.55.1 +libidn.so.11 +libidn.so.11.6.15 +libk5crypto.so.3 +libk5crypto.so.3.1 +libkrb5.so.3 +libkrb5.so.3.3 +libkrb5support.so.0 +libkrb5support.so.0.1 +liblockfile.so.1 +liblockfile.so.1.0 +libmenu.so.5 +libmenu.so.5.9 +libmenuw.so.5 +libmenuw.so.5.9 +libmpdec.so.2 +libmpdec.so.2.4.1 +libnetfilter_conntrack.so.3 +libnetfilter_conntrack.so.3.5.0 +libnettle.so.6 +libnettle.so.6.1 +libnfnetlink.so.0 +libnfnetlink.so.0.2.0 +libp11-kit.so.0 +libp11-kit.so.0.1.0 +libpanel.so.5 +libpanel.so.5.9 +libpanelw.so.5 +libpanelw.so.5.9 +libpcap.so.0.8 +libpcap.so.1.7.4 +libpcre16.so.3 +libpcre16.so.3.13.2 +libpcreposix.so.3 +libpcreposix.so.3.13.2 +libpng12.so.0 +libproxy.so.1 +libproxy.so.1.0.0 +libsemanage.so.1 +libsqlite3.so.0 +libsqlite3.so.0.8.6 +libstdc++.so.6 +libstdc++.so.6.0.21 +libtasn1.so.6 +libtasn1.so.6.5.1 +libtic.so.5 +libtic.so.5.9 +libtpm_unseal.so.1 +libtpm_unseal.so.1.0.0 +libtspi.so.1 +libtspi.so.1.2.0 +libubuntu-download-manager-common.so.1 +libubuntu-download-manager-common.so.1.2.0 +libudm-common.so.1 +libudm-common.so.1.2.0 +libudm-priv-common.so.1 +libudm-priv-common.so.1.2.0 +libunwind-coredump.so.0 +libunwind-coredump.so.0.0.0 +libunwind-ptrace.so.0 +libunwind-ptrace.so.0.0.0 +libunwind-x86_64.so.8 +libunwind-x86_64.so.8.0.1 +libunwind.so.8 +libunwind.so.8.0.1 +libustr-1.0.so.1 +libustr-1.0.so.1.0.4 +libyaml-0.so.2 +libyaml-0.so.2.0.4 diff --git a/snapcraft/common.py b/snapcraft/common.py index 2e7f8c0b03c..39745ff28dc 100644 --- a/snapcraft/common.py +++ b/snapcraft/common.py @@ -20,6 +20,7 @@ import os import platform import subprocess +import sys import tempfile import urllib @@ -30,6 +31,9 @@ _plugindir = _DEFAULT_PLUGINDIR _DEFAULT_SCHEMADIR = '/usr/share/snapcraft/schema' _schemadir = _DEFAULT_SCHEMADIR +_DEFAULT_LIBRARIESDIR = 'usr/share/snapcraft/libraries' +_librariesdir = _DEFAULT_LIBRARIESDIR + _arch = None env = [] @@ -58,8 +62,11 @@ def run_output(cmd, **kwargs): f.write('\n') f.write('exec $*') f.flush() - return subprocess.check_output(['/bin/sh', f.name] + cmd, - **kwargs).decode('utf8').strip() + output = subprocess.check_output(['/bin/sh', f.name] + cmd, **kwargs) + try: + return output.decode(sys.getfilesystemencoding()).strip() + except UnicodeEncodeError: + return output.decode('latin-1', 'surrogateescape').strip() _DEB_TRANSLATIONS = { @@ -116,6 +123,12 @@ def format_snap_name(snap): return '{name}_{version}_{arch}.snap'.format(**snap) +def get_projectdir(): + # TODO this needs to be initialized right after figuring out + # where snapcraft.yaml is. + return os.getcwd() + + def get_partsdir(): return os.path.join(os.getcwd(), 'parts') @@ -146,6 +159,15 @@ def get_schemadir(): return _schemadir +def set_librariesdir(librariesdir): + global _librariesdir + _librariesdir = librariesdir + + +def get_librariesdir(): + return _librariesdir + + def get_python2_path(root): """Return a valid PYTHONPATH or raise an exception.""" python_paths = glob.glob(os.path.join( diff --git a/snapcraft/dirs.py b/snapcraft/dirs.py index 29d5465a3f5..fcfc9d65402 100644 --- a/snapcraft/dirs.py +++ b/snapcraft/dirs.py @@ -28,3 +28,4 @@ def setup_dirs(): if os.path.exists(os.path.join(topdir, 'setup.py')): common.set_plugindir(os.path.join(topdir, 'plugins')) common.set_schemadir(os.path.join(topdir, 'schema')) + common.set_librariesdir(os.path.join(topdir, 'libraries')) diff --git a/snapcraft/libraries.py b/snapcraft/libraries.py index b930509fb20..6254e3e959e 100644 --- a/snapcraft/libraries.py +++ b/snapcraft/libraries.py @@ -16,6 +16,14 @@ import re import glob +import logging +import os +import platform + +from snapcraft import common + + +logger = logging.getLogger(__name__) def determine_ld_library_path(root): @@ -48,3 +56,41 @@ def _extract_ld_library_paths(ld_conf_file): paths.extend(path_delimiters.split(line)) return paths + + +_libraries = None + + +def _get_system_libs(): + global _libraries + if _libraries: + return _libraries + + release = platform.linux_distribution()[1] + lib_path = os.path.join(common.get_librariesdir(), release) + + if not os.path.exists(lib_path): + logger.debug('No libraries to exclude from this release') + + with open(lib_path) as fn: + _libraries = frozenset(fn.read().split()) + + return _libraries + + +def get_dependencies(elf): + """Return a list of libraries that are needed to satisfy elf's runtime.""" + logger.debug('Getting dependencies for {!r}'.format(elf)) + ldd_out = common.run_output(['ldd', elf]).split('\n') + ldd_out = [l.split() for l in ldd_out] + ldd_out = [l[2] for l in ldd_out if len(l) > 2 and os.path.exists(l[2])] + + # Anything that is part of the project we don't need to check for + project_dir = common.get_projectdir() + libs = [l for l in ldd_out if not l.startswith(project_dir)] + + # Now lets filter out what would be on the system + system_libs = _get_system_libs() + libs = [l for l in libs if not os.path.basename(l) in system_libs] + + return libs diff --git a/snapcraft/pluginhandler.py b/snapcraft/pluginhandler.py index f982c4bbd7f..ee03e241929 100644 --- a/snapcraft/pluginhandler.py +++ b/snapcraft/pluginhandler.py @@ -18,17 +18,20 @@ import filecmp import glob import importlib +import itertools import logging import os import shutil import sys import jsonschema +import magic import yaml import snapcraft from snapcraft import ( common, + libraries, repo, ) @@ -64,6 +67,7 @@ def __init__(self, plugin_name, part_name, properties): parts_dir = common.get_partsdir() self.ubuntudir = os.path.join(parts_dir, part_name, 'ubuntu') + self.bindir = os.path.join(parts_dir, part_name, 'bin') self.stagedir = os.path.join(os.getcwd(), 'stage') self.snapdir = os.path.join(os.getcwd(), 'snap') self.statefile = os.path.join(parts_dir, part_name, 'state') @@ -132,8 +136,6 @@ def _setup_stage_packages(self): ubuntu.unpack(self.installdir) snap_files, snap_dirs = self.migratable_fileset_for('stage') - _migrate_files(snap_files, snap_dirs, self.code.installdir, - self.stagedir, missing_ok=True) def pull(self, force=False): if not self.should_step_run('pull', force): @@ -191,6 +193,8 @@ def stage(self, force=False): snap_files, snap_dirs = self.migratable_fileset_for('stage') _migrate_files(snap_files, snap_dirs, self.code.installdir, self.stagedir) + # TODO once `snappy try` is in place we will need to copy + # dependencies here too self.mark_done('stage') @@ -203,6 +207,7 @@ def strip(self, force=False): self.notify_stage('Stripping') snap_files, snap_dirs = self.migratable_fileset_for('snap') _migrate_files(snap_files, snap_dirs, self.stagedir, self.snapdir) + _copy_dependencies(self.snapdir) self.mark_done('strip') @@ -334,6 +339,33 @@ def _migrate_files(snap_files, snap_dirs, srcdir, dstdir, missing_ok=False): os.link(src, dst, follow_symlinks=False) +def _copy_dependencies(workdir): + ms = magic.open(magic.NONE) + if ms.load() != 0: + raise RuntimeError('Cannot load magic header detection') + + elf_files = set() + for root, dirs, files in os.walk(workdir): + for entry in itertools.chain(files, dirs): + path = os.path.join(root, entry) + if os.path.islink(path): + logger.debug('Skipped link {!r} when parsing {!r}'.format( + path, workdir)) + continue + file_m = ms.file(path) + if file_m.startswith('ELF') and 'dynamically linked' in file_m: + elf_files.add(path) + + dependencies = [] + for elf_file in elf_files: + dependencies += libraries.get_dependencies(elf_file) + + for src in set(dependencies): + dst = os.path.join(workdir, src.lstrip('/')) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy(src, dst) + + def _get_file_list(stage_set): includes = [] excludes = [] diff --git a/snapcraft/tests/test_yaml.py b/snapcraft/tests/test_yaml.py index 9612b83fca7..d56997cdb21 100644 --- a/snapcraft/tests/test_yaml.py +++ b/snapcraft/tests/test_yaml.py @@ -17,6 +17,7 @@ import copy import logging import os +import subprocess import unittest import unittest.mock @@ -570,15 +571,7 @@ def test_config_stage_environment(self, mock_stagedir): self.assertTrue( 'LDFLAGS="-Lfoo/lib -Lfoo/usr/lib -Lfoo/lib/x86_64-linux-gnu ' '-Lfoo/usr/lib/x86_64-linux-gnu $LDFLAGS"' in environment) - self.assertTrue( - 'PKG_CONFIG_PATH=foo/lib/pkgconfig:' - 'foo/lib/x86_64-linux-gnu/pkgconfig:foo/usr/lib/pkgconfig:' - 'foo/usr/lib/x86_64-linux-gnu/pkgconfig:foo/usr/share/pkgconfig:' - 'foo/usr/local/lib/pkgconfig:' - 'foo/usr/local/lib/x86_64-linux-gnu/pkgconfig:' - 'foo/usr/local/share/pkgconfig:$PKG_CONFIG_PATH' in environment) self.assertTrue('PERL5LIB=foo/usr/share/perl5/' in environment) - self.assertTrue('PKG_CONFIG_SYSROOT_DIR=foo' in environment) class TestValidation(tests.TestCase): @@ -953,3 +946,76 @@ def test_invalid_expansion(self): raised.exception.message, '\'$3\' referred to in the \'stage\' fileset but it is not ' 'in filesets') + + +class TestPkgConfig(tests.TestCase): + + _PC_TEMPLATE = """prefix=/usr +exec_prefix=${{prefix}} +libdir=${{prefix}}/lib/x86_64-linux-gnu +includedir=${{prefix}}/include + +Name: {module} +Description: test +Version: 1.0 +Libs: -L${{libdir}} -llib{module} +Cflags: -I${{includedir}}/{module} +""" + + def setUp(self): + super().setUp() + + self.installdir = os.path.join(os.getcwd(), 'installdir') + os.makedirs(self.installdir) + + self.stagedir = os.path.join(os.getcwd(), 'stagedir') + os.makedirs(self.stagedir) + + self.bindir = os.path.join(os.getcwd(), 'bin') + os.makedirs(self.bindir) + + env = snapcraft.yaml._create_pkg_config_override( + self.bindir, self.installdir, self.stagedir) + self.assertEqual(env, ['PATH={}:$PATH'.format(self.bindir)]) + + self.pkg_config_bin = os.path.join(self.bindir, 'pkg-config') + + def _create_pc_file(self, workdir, module): + pkgconfig_dir = os.path.join(workdir, 'usr', 'lib', 'pkgconfig') + os.makedirs(pkgconfig_dir, exist_ok=True) + pc_module = os.path.join(pkgconfig_dir, '{}.pc'.format(module)) + with open(pc_module, 'w') as fn: + fn.write(self._PC_TEMPLATE.format(module=module)) + + def test_pkg_config_prefers_installdir(self): + self._create_pc_file(self.installdir, 'module1') + self._create_pc_file(self.stagedir, 'module1') + + out = subprocess.check_output([ + self.pkg_config_bin, '--cflags-only-I', 'module1']).decode('utf-8') + + self.assertEqual( + out, '-I{}/usr/include/module1\n'.format(self.installdir)) + + def test_pkg_config_finds_in_stagedir(self): + self._create_pc_file(self.installdir, 'module2') + self._create_pc_file(self.stagedir, 'module1') + + out = subprocess.check_output([ + self.pkg_config_bin, '--cflags-only-I', 'module1']).decode('utf-8') + + self.assertEqual( + out, '-I{}/usr/include/module1\n'.format(self.stagedir)) + + def test_pkg_config_works_with_two_modules(self): + self._create_pc_file(self.installdir, 'module1') + self._create_pc_file(self.installdir, 'module2') + + out = subprocess.check_output([ + self.pkg_config_bin, '--cflags-only-I', + 'module1', 'module2']).decode('utf-8') + + self.assertEqual(out, + '-I{dir}/usr/include/module1 ' + '-I{dir}/usr/include/module2\n'.format( + dir=self.installdir)) diff --git a/snapcraft/yaml.py b/snapcraft/yaml.py index e8bb6a9c051..98b4b99a15e 100644 --- a/snapcraft/yaml.py +++ b/snapcraft/yaml.py @@ -229,6 +229,9 @@ def build_env_for_part(self, part, root_part=True): env += self.build_env_for_part(dep_part, root_part=False) if root_part: + # this has to come before any {}/usr/bin + env += _create_pkg_config_override( + part.bindir, part.installdir, stagedir) env += part.env(part.installdir) env += _runtime_env(stagedir) env += _runtime_env(part.installdir) @@ -320,30 +323,97 @@ def _build_env(root): '-L{0}/usr/lib/{1}', '$LDFLAGS' ]).format(root, arch_triplet) + '"') - env.append('PKG_CONFIG_PATH=' + ':'.join([ - '{0}/lib/pkgconfig', - '{0}/lib/{1}/pkgconfig', - '{0}/usr/lib/pkgconfig', - '{0}/usr/lib/{1}/pkgconfig', - '{0}/usr/share/pkgconfig', - '{0}/usr/local/lib/pkgconfig', - '{0}/usr/local/lib/{1}/pkgconfig', - '{0}/usr/local/share/pkgconfig', - '$PKG_CONFIG_PATH' - ]).format(root, arch_triplet)) return env def _build_env_for_stage(stagedir): env = _build_env(stagedir) - env.append('PERL5LIB={0}/usr/share/perl5/'.format(stagedir)) - env.append('PKG_CONFIG_SYSROOT_DIR={0}'.format(stagedir)) return env +_PKG_CONFIG_TEMPLATE = """#!/usr/bin/env python3 + +import os +import sys +from subprocess import check_call, CalledProcessError + +real_env = os.environ.copy() + + +def run_pkg_config(args, env): + check_call(['/usr/bin/pkg-config'] + args, env=env) + + +def get_pkg_env_for(basedir): + current_env = real_env.copy() + env = {{}} + env['PKG_CONFIG_PATH'] = ':'.join([ + '{{basedir}}/lib/pkgconfig', + '{{basedir}}/lib/{arch}/pkgconfig', + '{{basedir}}/usr/lib/pkgconfig', + '{{basedir}}/usr/lib/{arch}/pkgconfig', + '{{basedir}}/usr/share/pkgconfig', + '{{basedir}}/usr/local/lib/pkgconfig', + '{{basedir}}/usr/local/lib/{arch}/pkgconfig', + '{{basedir}}/usr/local/share/pkgconfig']).format(basedir=basedir) + if 'PKG_CONFIG_PATH' in current_env: + env['PKG_CONFIG_PATH'] = '{{}}:{{}}'.format( + env['PKG_CONFIG_PATH'], current_env['PKG_CONFIG_PATH']) + env['PKG_CONFIG_SYSROOT_DIR'] = basedir + + current_env.update(env) + + return current_env + + +def modules_exist(modules, env): + try: + check_call(['/usr/bin/pkg-config', '--exists'] + modules, env=env) + return True + except CalledProcessError: + return False + + +def main(): + args = sys.argv[1:] + modules = list(filter(lambda x: not x.startswith('-'), args)) + + env = get_pkg_env_for('{installdir}') + if modules_exist(modules, env): + run_pkg_config(args, env) + return + + env = get_pkg_env_for('{stagedir}') + if modules_exist(modules, env): + run_pkg_config(args, env) + return + + run_pkg_config(args, env=real_env) + + +if __name__ == '__main__': + main() +""" + + +def _create_pkg_config_override(bindir, installdir, stagedir): + pkg_config_path = os.path.join(bindir, 'pkg-config') + os.makedirs(os.path.dirname(pkg_config_path), exist_ok=True) + + arch = common.get_arch_triplet() + pkg_config_content = _PKG_CONFIG_TEMPLATE.format( + installdir=installdir, stagedir=stagedir, arch=arch) + + with open(pkg_config_path, 'w') as fn: + fn.write(pkg_config_content) + os.chmod(pkg_config_path, 0o755) + + return ['PATH={}:$PATH'.format(bindir)] + + def _validate_snapcraft_yaml(snapcraft_yaml): schema_file = os.path.abspath(os.path.join(common.get_schemadir(), 'snapcraft.yaml'))