From 6320c00217c64d2e3c1bb8af2c45704b5ae5daa4 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Tue, 21 Nov 2017 08:21:29 +0000 Subject: [PATCH] Implement pypiserver and pypi proxy cache This patch implements nginx as a reverse proxy for python packages. The initial query will be to a local deployment of pypiserver in order to serve any locally built packages, but if the package is not available locally it will retry the query against pypi and cache the response. Depends-On: Id20a43fed833d53ca0f147f517deafba6587352d Change-Id: Ic4fd64f4dc82121a65088f3d7f4ae53f373df608 Implements: blueprint python-build-install-simplification Signed-off-by: Jesse Pretorius --- defaults/main.yml | 45 ++++++++++++++++++- handlers/main.yml | 11 +++++ meta/main.yml | 1 + ...ypiserver-pypi-cache-216e9e087f6d3f24.yaml | 8 ++++ tasks/repo_install.yml | 35 +++++++++++++-- tasks/repo_post_install.yml | 15 +++++++ tasks/repo_pre_install.yml | 6 +++ templates/nginx-pypi.conf.j2 | 12 +++++ templates/openstack-slushee.vhost.j2 | 34 ++++++++++++++ templates/pypiserver-systemd-init.j2 | 32 +++++++++++++ 10 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml create mode 100644 templates/nginx-pypi.conf.j2 create mode 100644 templates/pypiserver-systemd-init.j2 diff --git a/defaults/main.yml b/defaults/main.yml index b091fa8..3152806 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +## Verbosity Options +debug: False + ## Cap the maximum number of threads / workers when a user value is unspecified. repo_nginx_threads_max: 16 repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, repo_nginx_threads_max] | min }}" @@ -20,9 +23,10 @@ repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, rep ## APT Cache Options cache_timeout: 600 -# Set the package install state for distribution packages +# Set the package install state for distribution and pip packages # Options are 'present' and 'latest' repo_server_package_state: "latest" +repo_server_pip_package_state: "latest" repo_worker_connections: 1024 repo_server_name: openstack-slushee @@ -61,3 +65,42 @@ repo_pkg_cache_group: apt-cacher-ng # Set the log directory repo_service_log_dir: /var/log/apt-cacher-ng + +# Required packages to install on the host +repo_requires_pip_packages: + - virtualenv + - virtualenv-tools + +# Set the list of packages for the pypiserver +repo_pypiserver_pip_packages: + - "pypiserver==1.2.0" + +# Set the path to place all built python wheels +# This is used by pypiserver to serve them +repo_pypiserver_package_path: "{{ repo_service_home_folder }}/repo/python_packages" + +# Path to the pypiserver python virtualenv binaries +repo_pypiserver_bin: "/openstack/venvs/pypiserver-1.2.0/bin" + +# Path to the pypiserver working directory +repo_pypiserver_working_dir: "{{ repo_service_home_folder }}/pypiserver" + +# pypiserver service start options +repo_pypiserver_start_options: >- + -i localhost + -p 8080 + --log-file /var/log/pypiserver/pypiserver.log + --disable-fallback + {{ (debug | bool) | ternary('-vv', '-v') }} + {{ repo_pypiserver_package_path }} + +# config override var for systemd init file +repo_pypiserver_init_overrides: {} + +# Set the options for the nginx proxy_cache_path directive. +# The proxy cache is used for data downloaded from pypi. +# The default is set to cache up to 1G worth of packages +# for up to 1 month +repo_nginx_proxy_cache_path: >- + /var/lib/nginx/pypi levels=1:2 keys_zone=pypi:16m inactive=1M max_size=1G + diff --git a/handlers/main.yml b/handlers/main.yml index 850cfd7..37a8ee1 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -79,3 +79,14 @@ retries: 5 delay: 2 +- name: reload pypiserver + service: + name: "pypiserver" + enabled: yes + state: restarted + daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}" + register: _restart + until: _restart | success + retries: 5 + delay: 2 + diff --git a/meta/main.yml b/meta/main.yml index 4ac4f49..c135c74 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -38,3 +38,4 @@ dependencies: - role: apt_package_pinning when: - ansible_pkg_mgr == 'apt' + - role: pip_install diff --git a/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml b/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml new file mode 100644 index 0000000..f95a59b --- /dev/null +++ b/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The repo server now implements nginx as a reverse proxy for python + packages sourced from pypi. The initial query will be to a local + deployment of pypiserver in order to serve any locally built packages, + but if the package is not available locally it will retry + the query against pypi and cache the response. diff --git a/tasks/repo_install.yml b/tasks/repo_install.yml index 0f05f13..3467f60 100644 --- a/tasks/repo_install.yml +++ b/tasks/repo_install.yml @@ -13,15 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install repo server packages +- name: Install distro packages package: name: "{{ repo_server_distro_packages }}" state: "{{ repo_server_package_state }}" update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}" cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}" register: install_packages - until: install_packages|success + until: install_packages | success retries: 5 delay: 5 - tags: - - repo-packages + +- name: Install required pip packages + pip: + name: "{{ repo_requires_pip_packages }}" + state: "{{ repo_server_pip_package_state }}" + extra_args: >- + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages | success + retries: 5 + delay: 2 + +- name: Install pip packages + pip: + name: "{{ repo_pypiserver_pip_packages }}" + state: "{{ repo_server_pip_package_state }}" + virtualenv: "{{ repo_pypiserver_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages | success + retries: 5 + delay: 2 + notify: + - reload pypiserver + diff --git a/tasks/repo_post_install.yml b/tasks/repo_post_install.yml index f7feca5..99c655b 100644 --- a/tasks/repo_post_install.yml +++ b/tasks/repo_post_install.yml @@ -39,6 +39,8 @@ dest: "/etc/rsyncd.conf" - src: "openstack-slushee.vhost.j2" dest: "/etc/nginx/sites-available/openstack-slushee.vhost" + - src: "nginx-pypi.conf.j2" + dest: "/etc/nginx/conf.d/pypi.conf" notify: - reload nginx @@ -64,3 +66,16 @@ dest: "{{ systemd_utils_prefix }}/system/git.socket" notify: - reload git socket + +- name: Place the pypiserver systemd init script + config_template: + src: "pypiserver-systemd-init.j2" + dest: "/etc/systemd/system/pypiserver.service" + mode: "0644" + owner: "root" + group: "root" + config_overrides: "{{ repo_pypiserver_init_overrides }}" + config_type: "ini" + notify: + - reload pypiserver + diff --git a/tasks/repo_pre_install.yml b/tasks/repo_pre_install.yml index ceac285..8f2611e 100644 --- a/tasks/repo_pre_install.yml +++ b/tasks/repo_pre_install.yml @@ -62,6 +62,10 @@ - path: "{{ repo_service_home_folder }}/repo/venvs" - path: "/var/log/nginx" mode: "0775" + - path: "/var/log/pypiserver" + mode: "0775" + - path: "{{ repo_pypiserver_working_dir }}" + mode: "0775" - name: Drop repo pre/post command script template: @@ -96,6 +100,8 @@ - path: "/var/log/lsyncd" - path: "/etc/nginx/sites-enabled/default" state: "absent" + - path: "/etc/nginx/conf.d" - path: "/etc/nginx/sites-available" - path: "/etc/nginx/sites-enabled" + - path: "{{ repo_pypiserver_package_path }}" diff --git a/templates/nginx-pypi.conf.j2 b/templates/nginx-pypi.conf.j2 new file mode 100644 index 0000000..019e33d --- /dev/null +++ b/templates/nginx-pypi.conf.j2 @@ -0,0 +1,12 @@ +# {{ ansible_managed }} + +proxy_cache_path {{ repo_nginx_proxy_cache_path }}; + +upstream pypiserver { + server localhost:8080; +} + +upstream pypi { + server pypi.python.org:443; + keepalive 16; +} diff --git a/templates/openstack-slushee.vhost.j2 b/templates/openstack-slushee.vhost.j2 index 714582e..d9639eb 100644 --- a/templates/openstack-slushee.vhost.j2 +++ b/templates/openstack-slushee.vhost.j2 @@ -6,6 +6,40 @@ server { access_log /var/log/nginx/{{ repo_server_name }}.access.log gzip buffer=32k; error_log /var/log/nginx/{{ repo_server_name }}.error.log notice; + # Allow cached content to be used even when the upstream source is not available. + proxy_cache pypi; + proxy_cache_key $uri; + proxy_cache_lock on; + proxy_cache_revalidate on; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + + proxy_http_version 1.1; + proxy_set_header Host $host:$server_port; + proxy_set_header Connection ""; + proxy_set_header Accept-Encoding ""; + + # Rewrite any http redirects to use relative to proxy + proxy_redirect ~https?://pypi.python.org(.*) $1; + + # Fallback mechanism from: + # http://linuxplayer.org/2013/06/nginx-try-files-on-multiple-named-location-or-server + location @pypi { + proxy_set_header Host pypi.python.org; + proxy_pass https://pypi; + } + + location /simple { + proxy_intercept_errors on; + proxy_pass http://pypiserver; + error_page 404 = @pypi; + } + + location /packages { + proxy_intercept_errors on; + proxy_pass http://pypiserver; + error_page 404 = @pypi; + } + location / { root {{ repo_service_home_folder }}/repo/; autoindex on; diff --git a/templates/pypiserver-systemd-init.j2 b/templates/pypiserver-systemd-init.j2 new file mode 100644 index 0000000..18754b5 --- /dev/null +++ b/templates/pypiserver-systemd-init.j2 @@ -0,0 +1,32 @@ +# {{ ansible_managed }} + +[Unit] +Description=pypiserver +After=network.target + +[Service] +Type=simple +User={{ repo_service_user_name }} +Group={{ repo_service_group_name }} + +ExecStart={{ repo_pypiserver_bin }}/pypi-server {{ repo_pypiserver_start_options }} +ExecStop=/bin/kill -TERM $MAINPID +WorkingDirectory={{ repo_pypiserver_working_dir }} + +# Give a reasonable amount of time for the server to start up/shut down +TimeoutSec=120 +TimeoutStartSec=3 +Restart=on-failure +RestartSec=2 + +# This creates a specific slice which all services will operate from +# The accounting options give us the ability to see resource usage through +# the `systemd-cgtop` command. +Slice=pypiserver.slice +CPUAccounting=true +BlockIOAccounting=true +MemoryAccounting=false +TasksAccounting=true + +[Install] +WantedBy=multi-user.target