Permalink
Browse files

Make online tests use localhost only, and factor out tests that requi…

…re no networking at all
  • Loading branch information...
ivan committed Dec 14, 2018
1 parent 8d1402e commit 2524e9c5e7b41743531767e43e3b121b7c240480
Showing with 45 additions and 16 deletions.
  1. +2 −16 .travis.yml
  2. +8 −0 tests/offline-tests
  3. +35 −0 tests/online-tests
@@ -13,19 +13,5 @@ install:
- travis_retry pip install --process-dependency-links --no-binary --upgrade .

script:
- grab-site --help
- grab-site --version
- grab-site --1 https://www.google.com/
- gs-dump-urls --help
- ls -l www.google.com*
- gs-dump-urls www.google.com*/wpull.db done
- grab-site --1 --permanent-error-status-codes=404 https://www.google.com/
- echo '.*' > ignores
- grab-site --import-ignores ignores https://www.google.com/
- grab-site --1 --id my-id --no-dupespotter --no-video --concurrent 3 https://www.google.com/ https://www.google.com/intl/en/ads/
- grab-site --1 -i https://gist.githubusercontent.com/ivan/1d437237746cf067e134/raw/47d2253c0caf4a6b964dc657840ddce22eb9eac7/grab-site%2520test%2520urls.txt
- curl https://gist.githubusercontent.com/ivan/1d437237746cf067e134/raw/47d2253c0caf4a6b964dc657840ddce22eb9eac7/grab-site%2520test%2520urls.txt > local-url-list
- grab-site --1 -i local-url-list
- grab-site --1 -i local-url-list --which-wpull-args-partial
- grab-site --1 -i local-url-list --which-wpull-command
- python3 -c 'import libgrabsite.server'
- ./tests/offline-tests
- ./tests/online-tests
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

set -eu -o pipefail -o verbose

grab-site --help
grab-site --version
gs-dump-urls --help
python -c 'import libgrabsite.server'
@@ -0,0 +1,35 @@
#!/usr/bin/env bash

set -eu -o pipefail -o verbose

server_log=$(mktemp)
python -u -m http.server 0 > "$server_log" &
port=
# Try until server starts up
while [[ "$port" = "" ]]; do
port=$(grep -P -o 'port \d+' "$server_log" | cut -d ' ' -f 2 || true)
sleep 0.1
done
server="http://127.0.0.1:$port/"

trash=$(mktemp -d)
cd "$trash"

nodnspython=--wpull-args=--no-skip-getaddrinfo

grab-site --1 "$server"
grab-site $nodnspython --1 "$server"
ls -l 127.0.0.1*
gs-dump-urls 127.0.0.1*/wpull.db done
grab-site $nodnspython --1 --permanent-error-status-codes=404 "$server"
echo '.*' > ignores
grab-site $nodnspython --import-ignores ignores "$server"
grab-site $nodnspython --1 --id my-id --no-dupespotter --no-video --concurrent 3 "$server" "$server/another"
# TODO: test -i with remote URL list
echo "$server" > local-url-list
grab-site $nodnspython --1 -i local-url-list
grab-site $nodnspython --1 -i local-url-list --which-wpull-args-partial
grab-site $nodnspython --1 -i local-url-list --which-wpull-command

# kill http.server
kill $!

0 comments on commit 2524e9c

Please sign in to comment.