diff --git a/defaults/main.yml b/defaults/main.yml index b091fa8..3152806 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +## Verbosity Options +debug: False + ## Cap the maximum number of threads / workers when a user value is unspecified. repo_nginx_threads_max: 16 repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, repo_nginx_threads_max] | min }}" @@ -20,9 +23,10 @@ repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, rep ## APT Cache Options cache_timeout: 600 -# Set the package install state for distribution packages +# Set the package install state for distribution and pip packages # Options are 'present' and 'latest' repo_server_package_state: "latest" +repo_server_pip_package_state: "latest" repo_worker_connections: 1024 repo_server_name: openstack-slushee @@ -61,3 +65,42 @@ repo_pkg_cache_group: apt-cacher-ng # Set the log directory repo_service_log_dir: /var/log/apt-cacher-ng + +# Required packages to install on the host +repo_requires_pip_packages: + - virtualenv + - virtualenv-tools + +# Set the list of packages for the pypiserver +repo_pypiserver_pip_packages: + - "pypiserver==1.2.0" + +# Set the path to place all built python wheels +# This is used by pypiserver to serve them +repo_pypiserver_package_path: "{{ repo_service_home_folder }}/repo/python_packages" + +# Path to the pypiserver python virtualenv binaries +repo_pypiserver_bin: "/openstack/venvs/pypiserver-1.2.0/bin" + +# Path to the pypiserver working directory +repo_pypiserver_working_dir: "{{ repo_service_home_folder }}/pypiserver" + +# pypiserver service start options +repo_pypiserver_start_options: >- + -i localhost + -p 8080 + --log-file /var/log/pypiserver/pypiserver.log + --disable-fallback + {{ (debug | bool) | ternary('-vv', '-v') }} + {{ repo_pypiserver_package_path }} + +# config override var for systemd init file +repo_pypiserver_init_overrides: {} + +# Set the options for the nginx proxy_cache_path directive. +# The proxy cache is used for data downloaded from pypi. +# The default is set to cache up to 1G worth of packages +# for up to 1 month +repo_nginx_proxy_cache_path: >- + /var/lib/nginx/pypi levels=1:2 keys_zone=pypi:16m inactive=1M max_size=1G + diff --git a/handlers/main.yml b/handlers/main.yml index 850cfd7..37a8ee1 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -79,3 +79,14 @@ retries: 5 delay: 2 +- name: reload pypiserver + service: + name: "pypiserver" + enabled: yes + state: restarted + daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}" + register: _restart + until: _restart | success + retries: 5 + delay: 2 + diff --git a/meta/main.yml b/meta/main.yml index 4ac4f49..c135c74 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -38,3 +38,4 @@ dependencies: - role: apt_package_pinning when: - ansible_pkg_mgr == 'apt' + - role: pip_install diff --git a/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml b/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml new file mode 100644 index 0000000..f95a59b --- /dev/null +++ b/releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The repo server now implements nginx as a reverse proxy for python + packages sourced from pypi. The initial query will be to a local + deployment of pypiserver in order to serve any locally built packages, + but if the package is not available locally it will retry + the query against pypi and cache the response. diff --git a/tasks/repo_install.yml b/tasks/repo_install.yml index 0f05f13..3467f60 100644 --- a/tasks/repo_install.yml +++ b/tasks/repo_install.yml @@ -13,15 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install repo server packages +- name: Install distro packages package: name: "{{ repo_server_distro_packages }}" state: "{{ repo_server_package_state }}" update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}" cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}" register: install_packages - until: install_packages|success + until: install_packages | success retries: 5 delay: 5 - tags: - - repo-packages + +- name: Install required pip packages + pip: + name: "{{ repo_requires_pip_packages }}" + state: "{{ repo_server_pip_package_state }}" + extra_args: >- + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages | success + retries: 5 + delay: 2 + +- name: Install pip packages + pip: + name: "{{ repo_pypiserver_pip_packages }}" + state: "{{ repo_server_pip_package_state }}" + virtualenv: "{{ repo_pypiserver_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages | success + retries: 5 + delay: 2 + notify: + - reload pypiserver + diff --git a/tasks/repo_post_install.yml b/tasks/repo_post_install.yml index f7feca5..99c655b 100644 --- a/tasks/repo_post_install.yml +++ b/tasks/repo_post_install.yml @@ -39,6 +39,8 @@ dest: "/etc/rsyncd.conf" - src: "openstack-slushee.vhost.j2" dest: "/etc/nginx/sites-available/openstack-slushee.vhost" + - src: "nginx-pypi.conf.j2" + dest: "/etc/nginx/conf.d/pypi.conf" notify: - reload nginx @@ -64,3 +66,16 @@ dest: "{{ systemd_utils_prefix }}/system/git.socket" notify: - reload git socket + +- name: Place the pypiserver systemd init script + config_template: + src: "pypiserver-systemd-init.j2" + dest: "/etc/systemd/system/pypiserver.service" + mode: "0644" + owner: "root" + group: "root" + config_overrides: "{{ repo_pypiserver_init_overrides }}" + config_type: "ini" + notify: + - reload pypiserver + diff --git a/tasks/repo_pre_install.yml b/tasks/repo_pre_install.yml index ceac285..8f2611e 100644 --- a/tasks/repo_pre_install.yml +++ b/tasks/repo_pre_install.yml @@ -62,6 +62,10 @@ - path: "{{ repo_service_home_folder }}/repo/venvs" - path: "/var/log/nginx" mode: "0775" + - path: "/var/log/pypiserver" + mode: "0775" + - path: "{{ repo_pypiserver_working_dir }}" + mode: "0775" - name: Drop repo pre/post command script template: @@ -96,6 +100,8 @@ - path: "/var/log/lsyncd" - path: "/etc/nginx/sites-enabled/default" state: "absent" + - path: "/etc/nginx/conf.d" - path: "/etc/nginx/sites-available" - path: "/etc/nginx/sites-enabled" + - path: "{{ repo_pypiserver_package_path }}" diff --git a/templates/nginx-pypi.conf.j2 b/templates/nginx-pypi.conf.j2 new file mode 100644 index 0000000..019e33d --- /dev/null +++ b/templates/nginx-pypi.conf.j2 @@ -0,0 +1,12 @@ +# {{ ansible_managed }} + +proxy_cache_path {{ repo_nginx_proxy_cache_path }}; + +upstream pypiserver { + server localhost:8080; +} + +upstream pypi { + server pypi.python.org:443; + keepalive 16; +} diff --git a/templates/openstack-slushee.vhost.j2 b/templates/openstack-slushee.vhost.j2 index 714582e..d9639eb 100644 --- a/templates/openstack-slushee.vhost.j2 +++ b/templates/openstack-slushee.vhost.j2 @@ -6,6 +6,40 @@ server { access_log /var/log/nginx/{{ repo_server_name }}.access.log gzip buffer=32k; error_log /var/log/nginx/{{ repo_server_name }}.error.log notice; + # Allow cached content to be used even when the upstream source is not available. + proxy_cache pypi; + proxy_cache_key $uri; + proxy_cache_lock on; + proxy_cache_revalidate on; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + + proxy_http_version 1.1; + proxy_set_header Host $host:$server_port; + proxy_set_header Connection ""; + proxy_set_header Accept-Encoding ""; + + # Rewrite any http redirects to use relative to proxy + proxy_redirect ~https?://pypi.python.org(.*) $1; + + # Fallback mechanism from: + # http://linuxplayer.org/2013/06/nginx-try-files-on-multiple-named-location-or-server + location @pypi { + proxy_set_header Host pypi.python.org; + proxy_pass https://pypi; + } + + location /simple { + proxy_intercept_errors on; + proxy_pass http://pypiserver; + error_page 404 = @pypi; + } + + location /packages { + proxy_intercept_errors on; + proxy_pass http://pypiserver; + error_page 404 = @pypi; + } + location / { root {{ repo_service_home_folder }}/repo/; autoindex on; diff --git a/templates/pypiserver-systemd-init.j2 b/templates/pypiserver-systemd-init.j2 new file mode 100644 index 0000000..18754b5 --- /dev/null +++ b/templates/pypiserver-systemd-init.j2 @@ -0,0 +1,32 @@ +# {{ ansible_managed }} + +[Unit] +Description=pypiserver +After=network.target + +[Service] +Type=simple +User={{ repo_service_user_name }} +Group={{ repo_service_group_name }} + +ExecStart={{ repo_pypiserver_bin }}/pypi-server {{ repo_pypiserver_start_options }} +ExecStop=/bin/kill -TERM $MAINPID +WorkingDirectory={{ repo_pypiserver_working_dir }} + +# Give a reasonable amount of time for the server to start up/shut down +TimeoutSec=120 +TimeoutStartSec=3 +Restart=on-failure +RestartSec=2 + +# This creates a specific slice which all services will operate from +# The accounting options give us the ability to see resource usage through +# the `systemd-cgtop` command. +Slice=pypiserver.slice +CPUAccounting=true +BlockIOAccounting=true +MemoryAccounting=false +TasksAccounting=true + +[Install] +WantedBy=multi-user.target