From cc9aa47aca4fc63347eba796bbac5e59f5aa49ce Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 29 Jan 2017 20:40:58 +0000 Subject: [PATCH 1/5] Build python3.6 and add it into docker-snmp-sv2 Squashed commit of the following: commit dfa48862586c6c56fc7425aeb802f67f3245077b Merge: 4a70bde f2eed8f Author: Qi Luo Date: Sun Jan 29 20:38:22 2017 +0000 Merge branch 'github_master' into qiluo/python36 commit 4a70bde906bff141bcb1d6578826c009f9bb9168 Merge: 00ec4e1 9b940c3 Author: Qi Luo Date: Sun Jan 22 23:56:13 2017 +0000 Merge branch 'github_master' into qiluo/python36 commit 00ec4e1a9295dd9420da492be60a297d55bbdb5a Author: Qi Luo Date: Sun Jan 22 23:40:23 2017 +0000 Fix python3 command commit 04daf5826dea73fa68a7c0ad6799e5f0fdafdded Author: Qi Luo Date: Sat Jan 21 02:16:48 2017 +0000 Remove build shell script commit 5988810c4537f5a3350fc3d01e399ea25006e191 Author: Qi Luo Date: Sat Jan 21 02:16:27 2017 +0000 Refine python3 build.sh commit 1c4ba20ef944131e5d8c35d08f9454024d9eea14 Author: Qi Luo Date: Sat Jan 21 02:15:08 2017 +0000 Add missing supervisor commit 70369a596099cbebb12a1052bb3dc322cb72caf1 Author: Qi Luo Date: Fri Jan 20 23:23:52 2017 +0000 Fix DERIVED_TARGETS and cp command commit 5f52e884620c59239a55102fc1ce988ed85fe13d Author: Qi Luo Date: Fri Jan 20 06:37:13 2017 +0000 Reduce docker image size commit 3985310970bc8f8ab0b84358cef61834153a69fd Author: Qi Luo Date: Fri Jan 20 03:50:30 2017 +0000 snmp docker images builds with python debs commit 0c340ec460b7172fe3662371f51bfb13cb7dacc5 Author: Qi Luo Date: Thu Jan 12 02:50:57 2017 +0000 (comment) commit 0c2353f6c93678b00a01ea2104e145bb822ad542 Author: Qi Luo Date: Thu Jan 12 00:53:32 2017 +0000 (comment) commit 22836d960e97f070b99543f255f0a61ccdcfb1c8 Author: Qi Luo Date: Wed Jan 11 23:12:55 2017 +0000 Fix bug: dpkg and force install commit d26da9607af702b8029a7b9aaf9f73ec1ae7734d Author: Qi Luo Date: Wed Jan 11 22:43:09 2017 +0000 Remove wget timestamp option commit 363aeba0aaf5bf53f557385355d34c3824a4a778 Author: Qi Luo Date: Wed Jan 11 22:18:22 2017 +0000 Build debian package python3.6 --- dockers/docker-snmp-sv2/Dockerfile.j2 | 61 +++++++++++------------- dockers/docker-snmp-sv2/supervisord.conf | 2 +- rules/docker-snmp-sv2.mk | 3 +- rules/python3.mk | 38 +++++++++++++++ src/python3.5/build.sh | 18 ------- src/python3/Makefile | 34 +++++++++++++ 6 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 rules/python3.mk delete mode 100755 src/python3.5/build.sh create mode 100644 src/python3/Makefile diff --git a/dockers/docker-snmp-sv2/Dockerfile.j2 b/dockers/docker-snmp-sv2/Dockerfile.j2 index 33a2bdaa74a8..ba5ece567181 100644 --- a/dockers/docker-snmp-sv2/Dockerfile.j2 +++ b/dockers/docker-snmp-sv2/Dockerfile.j2 @@ -1,45 +1,40 @@ FROM docker-config-engine -COPY \ +COPY [ \ {% for deb in docker_snmp_sv2_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ + "debs/{{ deb }}", +{%- endfor %} \ + "/debs/"] -COPY python-wheels /python-wheels +COPY python-wheels/sswsdk-*-py3-*.whl /python-wheels/ +COPY python-wheels/asyncsnmp-*-py3-*.whl /python-wheels/ # enable -O for all Python calls ENV PYTHONOPTIMIZE 1 -## Pre-install the fundamental packages -## Install Python SSWSDK (SNMP subagent dependency) -## Install SNMP subagent -## Clean up -RUN apt-get update && apt-get install -y libmysqlclient-dev libmysqld-dev libperl-dev libpci-dev libpci3 libsensors4 libsensors4-dev libwrap0-dev - -RUN dpkg -i \ -{% for deb in docker_snmp_sv2_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor %} - -RUN rm -rf /debs - +# install supervisor +# install libsnmp30 dependencies +# install dependencies for sonic-cfggen +# install libpython3.6-dev and pip dependencies +# TODO: remove libpython3.6-dev, its and pip's dependencies if we can get pip3 directly # install subagent -RUN apt-get -y install build-essential wget libssl-dev openssl supervisor && \ - rm -rf /var/lib/apt/lists/* && \ - wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz && \ - tar xvf Python-3.5.2.tgz && cd Python-3.5.2 && \ - ./configure --without-doc-strings --prefix=/usr --without-pymalloc --enable-shared && \ - make && make install && \ - ldconfig && \ - cd .. && rm -rf Python-3.5.2 && rm Python-3.5.2.tgz && \ - pip3 install --no-cache-dir /python-wheels/*py3*.whl hiredis && \ +# clean up +RUN apt-get update && apt-get install -y supervisor \ + libperl5.20 libpci3 libwrap0 \ + libexpat1-dev \ + python-lxml python-jinja2 python-netaddr python-ipaddr python-yaml \ + curl gcc && \ + dpkg -i \ +{% for deb in docker_snmp_sv2_debs.split(' ') -%} + debs/{{ deb }}{{' '}} +{%- endfor %} && \ + rm -rf /debs && \ + curl https://bootstrap.pypa.io/get-pip.py | python3.6 && \ + python3.6 -m pip install --no-cache-dir /python-wheels/*py3*.whl hiredis && \ rm -rf /python-wheels && \ - python3 -m sonic_ax_impl install && \ - python3 -m pip uninstall -y pip setuptools && \ - /bin/bash -c "rm -rf /usr/lib/python3.5/{unittest,lib2to3,tkinter,idlelib,email,test}" && \ - apt-get -y purge build-essential libssl-dev openssl && \ - apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y && \ + python3.6 -m sonic_ax_impl install && \ + apt-get -y purge libpython3.6-dev libexpat1-dev curl gcc && \ + apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y --purge && \ find / | grep -E "__pycache__" | xargs rm -rf && \ rm -rf ~/.cache @@ -51,4 +46,4 @@ COPY ["config.sh", "/usr/bin/"] ## Although exposing ports is not need for host net mode, keep it for possible bridge mode EXPOSE 161/udp 162/udp -ENTRYPOINT /usr/bin/config.sh && /usr/bin/supervisord +ENTRYPOINT /usr/bin/config.sh && /usr/bin/supervisord diff --git a/dockers/docker-snmp-sv2/supervisord.conf b/dockers/docker-snmp-sv2/supervisord.conf index 50b510063d75..689cf5d91948 100644 --- a/dockers/docker-snmp-sv2/supervisord.conf +++ b/dockers/docker-snmp-sv2/supervisord.conf @@ -6,7 +6,7 @@ command=/usr/sbin/snmpd -f -LS4d -u Debian-snmp -g Debian-snmp -I -smux,mteTrigg priority=100 [program:snmp-subagent] -command=/usr/bin/env python3 -m sonic_ax_impl +command=/usr/bin/env python3.6 -m sonic_ax_impl priority=200 [program:rsyslogd] diff --git a/rules/docker-snmp-sv2.mk b/rules/docker-snmp-sv2.mk index 0cad7cb43407..95fdcc25775c 100644 --- a/rules/docker-snmp-sv2.mk +++ b/rules/docker-snmp-sv2.mk @@ -2,7 +2,8 @@ DOCKER_SNMP_SV2 = docker-snmp-sv2.gz $(DOCKER_SNMP_SV2)_PATH = $(DOCKERS_PATH)/docker-snmp-sv2 -$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD) +## TODO: remove LIBPY3_DEV if we can get pip3 directly +$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD) $(PY3) $(LIBPY3_DEV) $(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(ASYNCSNMP_PY3) $(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE) SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2) diff --git a/rules/python3.mk b/rules/python3.mk new file mode 100644 index 000000000000..656c4b35c876 --- /dev/null +++ b/rules/python3.mk @@ -0,0 +1,38 @@ +# python3.5.2+ + +PYTHON_VER=3.6.0-1 +PYTHON_PNAME=python3.6 + +export PYTHON_VER +export PYTHON_PNAME + +LIBPY3_MIN = lib$(PYTHON_PNAME)-minimal_$(PYTHON_VER)_amd64.deb +$(LIBPY3_MIN)_SRC_PATH = $(SRC_PATH)/python3 +$(LIBPY3_MIN)_DEPENDS += +$(LIBPY3_MIN)_RDEPENDS += +SONIC_MAKE_DEBS += $(LIBPY3_MIN) + +LIBPY3_STD = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)_amd64.deb +$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_STD))) +$(LIBPY3_STD)_DEPENDS += $(LIBMPDECIMAL) +$(LIBPY3_STD)_RDEPENDS += $(LIBPY3_MIN) $(LIBMPDECIMAL) + +LIBPY3 = lib$(PYTHON_PNAME)_$(PYTHON_VER)_amd64.deb +$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3))) +$(LIBPY3)_DEPENDS += $(LIBPY3_STD) +$(LIBPY3)_RDEPENDS += $(LIBPY3_MIN) $(LIBPY3_STD) + +PY3_MIN = $(PYTHON_PNAME)-minimal_$(PYTHON_VER)_amd64.deb +$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3_MIN))) +$(PY3_MIN)_RDEPENDS += $(LIBPY3_MIN) + +PY3 = $(PYTHON_PNAME)_$(PYTHON_VER)_amd64.deb +$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3))) +$(PY3)_DEPENDS += $(PY3_MIN) $(LIBPY3) +$(PY3)_RDEPENDS += $(PY3_MIN) $(LIBPY3) $(LIBPY3_MIN) + +LIBPY3_DEV = lib$(PYTHON_PNAME)-dev_$(PYTHON_VER)_amd64.deb +$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_DEV))) +$(LIBPY3_DEV)_DEPENDS += $(LIBPY3) $($(LIBPY3)_DEPENDS) +$(LIBPY3_DEV)_RDEPENDS += $(LIBPY3) $($(LIBPY3)_RDEPENDS) + diff --git a/src/python3.5/build.sh b/src/python3.5/build.sh deleted file mode 100755 index ebada5397e1b..000000000000 --- a/src/python3.5/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -PYTHON_VER=3.5.2 -PYTHON_DEB_VER=8 - -wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}.orig.tar.xz -wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.debian.tar.xz -wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.dsc - -dpkg-source -x python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.dsc - -pushd python3.5-${PYTHON_VER} - -dpkg-buildpackage -us -uc -b - -popd - -cp *.deb ../ diff --git a/src/python3/Makefile b/src/python3/Makefile new file mode 100644 index 000000000000..841b393730d5 --- /dev/null +++ b/src/python3/Makefile @@ -0,0 +1,34 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +PYTHON_VER=3.6.0 +PYTHON_DEB_VER=1 +PYTHON_PNAME=python3.6 + +MAIN_TARGET = lib$(PYTHON_PNAME)-minimal_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb +DERIVED_TARGETS = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \ + lib$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \ + $(PYTHON_PNAME)-minimal_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \ + $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \ + lib$(PYTHON_PNAME)-dev_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb + #$(PYTHON_PNAME)-dev_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + ## Obtaining the python3 + wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz + wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz + wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc + + ## Build + dpkg-source -x $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc + pushd $(PYTHON_PNAME)-$(PYTHON_VER) + sudo apt-get install devscripts + mk-build-deps + sudo dpkg -i $(PYTHON_PNAME)-build-deps_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb || sudo apt-get install -f + dpkg-buildpackage -us -uc -b + popd + + cp $(DERIVED_TARGETS) $* $(DEST)/ + +$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) From 80d2923511d6128f9f1bce87487c5af0f2cfebe2 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 29 Jan 2017 20:43:12 +0000 Subject: [PATCH 2/5] (reformat) --- src/python3/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python3/Makefile b/src/python3/Makefile index 841b393730d5..f7ab6328ac3c 100644 --- a/src/python3/Makefile +++ b/src/python3/Makefile @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc - + ## Build dpkg-source -x $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc pushd $(PYTHON_PNAME)-$(PYTHON_VER) @@ -28,7 +28,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : sudo dpkg -i $(PYTHON_PNAME)-build-deps_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb || sudo apt-get install -f dpkg-buildpackage -us -uc -b popd - + cp $(DERIVED_TARGETS) $* $(DEST)/ $(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) From c77e64b220d06652bf3efc8ec7ebe0ff5d332bd3 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 4 Feb 2017 01:17:04 +0000 Subject: [PATCH 3/5] Download python3 source from azure storage mirror --- src/python3/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python3/Makefile b/src/python3/Makefile index f7ab6328ac3c..63941810677b 100644 --- a/src/python3/Makefile +++ b/src/python3/Makefile @@ -16,9 +16,9 @@ DERIVED_TARGETS = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd6 $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : ## Obtaining the python3 - wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz - wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz - wget http://http.debian.net/debian/pool/main/p/$(PYTHON_PNAME)/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc + wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz?sv=2015-04-05&sr=b&sig=d42Wh1CA9NZvlskhW4fpWcHVgc7N3IKhdFzyeO2zbRA%3D&se=2027-02-02T01%3A00%3A57Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz + wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz?sv=2015-04-05&sr=b&sig=KLX9pMJ3zpQvGBo6ZjzoZXgooMJRUUwMx8ZaTJtywK0%3D&se=2027-02-02T00%3A59%3A34Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz + wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc?sv=2015-04-05&sr=b&sig=95s%2FC4vKY6bRKtkUTz%2BmHLqOllBOYbfP3zV5ayAuzSM%3D&se=2027-02-02T01%3A00%3A26Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc ## Build dpkg-source -x $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc From 785a24296a1860ac4dcf0068a7c4e2b8027dace0 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Wed, 8 Feb 2017 19:21:55 +0000 Subject: [PATCH 4/5] Remove unused dependencies --- dockers/docker-snmp-sv2/Dockerfile.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dockers/docker-snmp-sv2/Dockerfile.j2 b/dockers/docker-snmp-sv2/Dockerfile.j2 index ba5ece567181..4e440d7f5e3a 100644 --- a/dockers/docker-snmp-sv2/Dockerfile.j2 +++ b/dockers/docker-snmp-sv2/Dockerfile.j2 @@ -14,15 +14,14 @@ ENV PYTHONOPTIMIZE 1 # install supervisor # install libsnmp30 dependencies -# install dependencies for sonic-cfggen -# install libpython3.6-dev and pip dependencies +# install libpython3.6-dev dependencies +# install pip dependencies # TODO: remove libpython3.6-dev, its and pip's dependencies if we can get pip3 directly # install subagent # clean up RUN apt-get update && apt-get install -y supervisor \ libperl5.20 libpci3 libwrap0 \ libexpat1-dev \ - python-lxml python-jinja2 python-netaddr python-ipaddr python-yaml \ curl gcc && \ dpkg -i \ {% for deb in docker_snmp_sv2_debs.split(' ') -%} From 257f0911afa77682ee682d0388b4ca3e5624a594 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Wed, 8 Feb 2017 21:24:56 +0000 Subject: [PATCH 5/5] (comment) --- rules/python3.mk | 2 -- 1 file changed, 2 deletions(-) diff --git a/rules/python3.mk b/rules/python3.mk index 656c4b35c876..8fa02a26a462 100644 --- a/rules/python3.mk +++ b/rules/python3.mk @@ -1,5 +1,3 @@ -# python3.5.2+ - PYTHON_VER=3.6.0-1 PYTHON_PNAME=python3.6