Merge pull request #7308 from kalefranz/proxy-tests

proxy test routine

circle.yml

@@ -66,7 +66,7 @@ conda_build_test: &conda_build_test
           conda create -n blarg -yq --download-only python=3.4
           conda create -n blarg -yq --download-only python=3.5
           conda create -n blarg -yq --download-only python=3.6
-          conda create -n blarg -yq --download-only python setuptools cython
+          conda create -n blarg -yq --download-only python setuptools cython certifi
           conda create -n blarg -yq --download-only libpng=1.6.17
 
     - run:

conda/cli/main_search.py

@@ -6,12 +6,12 @@
 from .install import calculate_channel_urls
 from ..base.context import context
 from ..cli.common import stdout_json
+from ..common.compat import text_type
 from ..common.io import Spinner
-from ..compat import text_type
 from ..core.envs_manager import query_all_prefixes
 from ..core.subdir_data import SubdirData
-from ..models.records import PackageRecord
 from ..models.match_spec import MatchSpec
+from ..models.records import PackageRecord
 from ..models.version import VersionOrder
 from ..resolve import dashlist
 from ..utils import human_bytes

tests/integration/proxy/.gitignore

@@ -0,0 +1,2 @@
+squid_log/
+temp/

tests/integration/proxy/condarc.proxybad

@@ -0,0 +1,4 @@
+report_errors: false
+proxy_servers:
+  http: http://condauser:BAD_PASSWORD@127.0.0.1:3128
+  https: http://condauser:BAD_PASSWORD@127.0.0.1:3128

tests/integration/proxy/condarc.proxygood

@@ -0,0 +1,4 @@
+report_errors: false
+proxy_servers:
+  http: http://condauser:condapass@127.0.0.1:3128
+  https: http://condauser:condapass@127.0.0.1:3128

tests/integration/proxy/proxy-test.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+set -uo pipefail
+
+# Invoke with:
+#   $ bash ./tests/integration/proxy/proxy-test.sh
+#
+# NOTES:
+#   * PWD should be the conda project root
+#   * docker is required, and executable without sudo
+#   * squidusers file has condauser:condapass for credentials
+#   * to look at logs for squid proxy, use:
+#       $ cat ./tests/integration/proxy/squid_log/*
+#
+# REFERENCES:
+#   https://veesp.com/en/blog/squid-authentication
+#   https://wiki.squid-cache.org/Features/Authentication
+
+
+SRC_DIR="$PWD"
+
+[ -f "$SRC_DIR/conda/__main__.py" ] && [ -f "$SRC_DIR/conftest.py" ] || (echo "Current working directory must be conda project root." && exit 1)
+which docker > /dev/null || (echo "docker required but not found" && exit 1)
+docker --version > /dev/null || (echo "Cannot execute docker. Apparently needs sudo?" && exit 1)
+
+
+rm -rf "$SRC_DIR"/tests/integration/proxy/squid_log/*
+CID=$(docker run \
+    --detach \
+    --rm \
+    -v $SRC_DIR/tests/integration/proxy/squid.conf:/etc/squid/squid.conf:ro \
+    -v $SRC_DIR/tests/integration/proxy/squidusers:/etc/squid/squidusers:ro \
+    -v $SRC_DIR/tests/integration/proxy/squid_log:/var/log/squid:rw \
+    -p 3128:3128 \
+    kalefranz/squid)
+
+echo "waiting for proxy to start"
+( tail -f -n0 "$SRC_DIR/tests/integration/proxy/squid_log/cache.log" & ) | grep -q "Accepting HTTP Socket connections at"
+
+
+_fail() {
+  echo -e "$1"
+  echo "removing container $CID"
+  docker rm --force $CID > /dev/null
+  exit 1
+}
+
+
+# Don't use repodata Cache-Control
+export CONDA_LOCAL_REPODATA_TTL=0
+
+# Ensure we have an empty package cache
+export CONDA_PKGS_DIRS="$SRC_DIR/tests/integration/proxy/temp"
+mkdir -p "$CONDA_PKGS_DIRS" || _fail "permissions error"
+touch "$CONDA_PKGS_DIRS/permissions-check" || _fail "permissions error"
+rm -rf "$CONDA_PKGS_DIRS"/*
+
+
+# ###########################################################
+# Test that we have failures when directing traffic through proxy with wrong password
+# ###########################################################
+export CONDARC="$SRC_DIR/tests/integration/proxy/condarc.proxybad"
+
+# test for repodata failure
+echo "test expecting repodata failure"
+captured="$(conda search zlib 2>&1)"
+rc=$?
+[ $rc -eq 1 ] || _fail "'conda search zlib' was expected to fail\n$captured"
+rm -rf "$CONDA_PKGS_DIRS"/*
+
+# test for package download failure
+echo "test expecting package download failure"
+captured="$(conda install --mkdir -y -q -p $CONDA_PKGS_DIRS/test-env https://repo.continuum.io/pkgs/main/osx-64/six-1.11.0-py36h0e22d5e_1.tar.bz2 2>&1)"
+rc=$?
+[ $rc -eq 1 ] || _fail "'conda install' was expected to fail\n$captured"
+rm -rf "$CONDA_PKGS_DIRS"/*
+
+
+# ###########################################################
+# Test that directing traffic through proxy with correct password succeeds
+# ###########################################################
+export CONDARC="$SRC_DIR/tests/integration/proxy/condarc.proxygood"
+
+# test for repodata success
+echo "test expecting repodata success"
+captured="$(conda search zlib 2>&1)"
+rc=$?
+[ $rc -eq 0 ] || _fail "'conda search zlib' was expected to succeed\n$captured"
+echo "$captured" | grep -q 1.2.11 || _fail "'conda search zlib' was expected to contain zlib version 1.2.11"\n$captured
+rm -rf "$CONDA_PKGS_DIRS"/*
+
+# test for package download success
+echo "test expecting package download success"
+captured="$(conda install --mkdir -y -q -p $CONDA_PKGS_DIRS/test-env https://repo.continuum.io/pkgs/main/osx-64/six-1.11.0-py36h0e22d5e_1.tar.bz2 2>&1)"
+rc=$?
+[ $rc -eq 0 ] || _fail "'conda install' was expected to succeed\n$captured"
+[ -f "$CONDA_PKGS_DIRS/test-env/conda-meta/history" ] || _fail "history file expected\n$captured"
+[ -f "$CONDA_PKGS_DIRS/test-env/lib/python3.6/site-packages/six.py" ] || _fail "six.py file expected\n$captured"
+rm -rf "$CONDA_PKGS_DIRS"/*
+
+
+# ###########################################################
+# clean up
+# ###########################################################
+
+echo "removing container $CID"
+docker rm --force $CID > /dev/null
+
+echo
+echo ">>>>> ALL TESTS COMPLETED <<<<<"
+echo

tests/integration/proxy/squid.conf

@@ -0,0 +1,116 @@
+## Tested and working on squid 3.3.10-r0 and Alpine 2.7.1 (kernel 3.10.19-0-grsec), 64-bit
+## Example rule allowing access from your local networks.
+## Adapt to list your (internal) IP networks from where browsing
+## should be allowed
+
+acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
+acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
+acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
+## Allow anyone to use the proxy (you should lock this down to client networks only!):
+# acl localnet src all
+## IPv6 local addresses:
+acl localnet src fc00::/7       # RFC 4193 local private network range
+acl localnet src fe80::/10      # RFC 4291 link-local (directly plugged) machines
+
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 443		# https
+acl CONNECT method CONNECT
+
+auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/squidusers
+## here /usr/lib/squid3/ncsa_auth is a helper path and  /etc/squid3/squidusers is a path to the file containing login/pass combinations.
+#acl localnet proxy_auth REQUIRED  # creation of access list of all authenticated users
+#acl localhost proxy_auth REQUIRED  # creation of access list of all authenticated users
+acl lan proxy_auth REQUIRED
+http_access allow lan
+
+
+## Only allow access to the defined safe ports whitelist
+http_access deny !Safe_ports
+
+## Deny CONNECT to other than secure SSL ports
+http_access deny CONNECT !SSL_ports
+
+## Only allow cachemgr access from localhost
+http_access allow localhost manager
+http_access deny manager
+
+## We strongly recommend the following be uncommented to protect innocent
+## web applications running on the proxy server who think the only
+## one who can access services on "localhost" is a local user
+http_access deny to_localhost
+
+##
+## INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
+##
+
+## Example rule allowing access from your local networks.
+## Adapt localnet in the ACL section to list your (internal) IP networks
+## from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+
+## And finally deny all other access to this proxy
+http_access deny all
+
+## Squid normally listens to port 3128
+http_port 3128
+## If you have multiple interfaces you can specify to listen on one IP like this:
+#http_port 1.2.3.4:3128
+
+## Uncomment and adjust the following to add a disk cache directory.
+## 1024 is the disk space to use for cache in MB, adjust as you see fit!
+## Default is no disk cache
+#cache_dir ufs /var/cache/squid 1024 16 256
+## Better, use 'aufs' cache type, see
+##http://www.squid-cache.org/Doc/config/cache_dir/ for info.
+#cache_dir aufs /var/cache/squid 1024 16 256
+## Recommended to only change cache type when squid is stopped, and use 'squid -z' to
+## ensure cache is (re)created correctly
+
+## Leave coredumps in the first cache dir
+#coredump_dir /var/cache/squid
+
+## Where does Squid log to?
+logformat combined [%tl] %>A %{Host}>h "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
+access_log stdio:/var/log/squid/access.log combined
+cache_store_log stdio:/var/log/squid/store.log
+cache_log stdio:/var/log/squid/cache.log
+logfile_rotate 8
+## Use the below to turn off access logging
+#access_log none
+## When logging, web auditors want to see the full uri, even with the query terms
+#strip_query_terms off
+## Keep 7 days of logs
+#logfile_rotate 7
+
+## How much RAM, in MB, to use for cache? Default since squid 3.1 is 256 MB
+cache_mem 64 MB
+
+## Maximum size of individual objects to store in cache
+maximum_object_size 1 MB
+
+## Amount of data to buffer from server to client
+read_ahead_gap 64 KB
+
+## Use X-Forwarded-For header?
+## Some consider this a privacy/security risk so it is often disabled
+## However it can be useful to identify misbehaving/problematic clients
+forwarded_for on
+forwarded_for delete
+
+## Suppress sending squid version information
+httpd_suppress_version_string on
+
+## How long to wait when shutting down squid
+shutdown_lifetime 10 seconds
+
+## Replace the User Agent header.  Be sure to deny the header first, then replace it :)
+#request_header_access User-Agent deny all
+#request_header_replace User-Agent Mozilla/5.0 (Windows; MSIE 9.0; Windows NT 9.0; en-US)
+
+## What hostname to display? (defaults to system hostname)
+#visible_hostname a_proxy
+
+## Use a different hosts file?
+#hosts_file /path/to/file

tests/integration/proxy/squidusers

@@ -0,0 +1 @@
+condauser:$apr1$XQCp9LE8$nhodpF/NThFB0LsO1youT0

tests/test_create.py

@@ -739,7 +739,7 @@ def test_allow_softlinks(self, hardlink_supported_mock):
 
     @pytest.mark.skipif(on_win, reason="nomkl not present on windows")
     def test_remove_features(self):
-        with make_temp_env("python=2 numpy nomkl") as prefix:
+        with make_temp_env("python=2 numpy=1.13 nomkl") as prefix:
             assert exists(join(prefix, PYTHON_BINARY))
             assert package_is_installed(prefix, 'numpy')
             assert package_is_installed(prefix, 'nomkl')
@@ -846,11 +846,11 @@ def test_install_freeze_installed_flag(self):
 
     @pytest.mark.skipif(on_win, reason="mkl package not available on Windows")
     def test_install_features(self):
-        with make_temp_env("python=2 numpy nomkl") as prefix:
+        with make_temp_env("python=2 numpy=1.13 nomkl") as prefix:
             numpy_details = get_conda_list_tuple(prefix, "numpy")
             assert len(numpy_details) == 4 and 'nomkl' in numpy_details[3]
 
-        with make_temp_env("python=2 numpy") as prefix:
+        with make_temp_env("python=2 numpy=1.13") as prefix:
             numpy_details = get_conda_list_tuple(prefix, "numpy")
             assert len(numpy_details) == 3 or 'nomkl' not in numpy_details[3]