diff --git a/docs/conf.py b/docs/conf.py index e71c4723a9..8172d57f55 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -293,6 +293,7 @@ (_py_class_role, 'proxy.core.pool.AcceptorPool'), (_py_class_role, 'proxy.core.executors.ThreadlessPool'), (_py_class_role, 'proxy.core.acceptor.threadless.T'), + (_py_class_role, 'proxy.core.acceptor.work.T'), (_py_class_role, 'queue.Queue[Any]'), (_py_class_role, 'TcpClientConnection'), (_py_class_role, 'TcpServerConnection'), @@ -303,4 +304,5 @@ (_py_class_role, 'WebsocketFrame'), (_py_class_role, 'Work'), (_py_obj_role, 'proxy.core.acceptor.threadless.T'), + (_py_obj_role, 'proxy.core.acceptor.work.T'), ] diff --git a/examples/web_scraper.py b/examples/web_scraper.py index 4b925876c5..b3dae1aa2d 100644 --- a/examples/web_scraper.py +++ b/examples/web_scraper.py @@ -14,10 +14,11 @@ from proxy import Proxy from proxy.core.acceptor import Work +from proxy.core.connection import TcpClientConnection from proxy.common.types import Readables, Writables -class WebScraper(Work): +class WebScraper(Work[TcpClientConnection]): """Demonstrates how to orchestrate a generic work acceptors and executors workflow using proxy.py core. diff --git a/proxy/core/acceptor/executors.py b/proxy/core/acceptor/executors.py index e4e165b97b..9512762671 100644 --- a/proxy/core/acceptor/executors.py +++ b/proxy/core/acceptor/executors.py @@ -125,7 +125,7 @@ def start_threaded_work( addr: Optional[Tuple[str, int]], event_queue: Optional[EventQueue] = None, publisher_id: Optional[str] = None, - ) -> Tuple[Work, threading.Thread]: + ) -> Tuple[Work[TcpClientConnection], threading.Thread]: """Utility method to start a work in a new thread.""" work = flags.work_klass( TcpClientConnection(conn, addr), diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index cfd2f12086..5140c9345b 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -18,7 +18,7 @@ import multiprocessing from abc import abstractmethod, ABC -from typing import Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union +from typing import Any, Dict, Optional, Tuple, List, Set, Generic, TypeVar, Union from ...common.logger import Logger from ...common.types import Readables, Writables @@ -71,7 +71,7 @@ def __init__( self.event_queue = event_queue self.running = multiprocessing.Event() - self.works: Dict[int, Work] = {} + self.works: Dict[int, Work[Any]] = {} self.selector: Optional[selectors.DefaultSelector] = None # If we remove single quotes for typing hint below, # runtime exceptions will occur for < Python 3.9. diff --git a/proxy/core/acceptor/work.py b/proxy/core/acceptor/work.py index 0152e1b2a6..5a7ba0723d 100644 --- a/proxy/core/acceptor/work.py +++ b/proxy/core/acceptor/work.py @@ -16,19 +16,20 @@ from abc import ABC, abstractmethod from uuid import uuid4 -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, TypeVar, Generic from ..event import eventNames, EventQueue -from ..connection import TcpClientConnection from ...common.types import Readables, Writables +T = TypeVar('T') -class Work(ABC): + +class Work(ABC, Generic[T]): """Implement Work to hook into the event loop provided by Threadless process.""" def __init__( self, - work: TcpClientConnection, + work: T, flags: argparse.Namespace, event_queue: Optional[EventQueue] = None, uid: Optional[str] = None, diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index 236d5b764e..80f795e36f 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -19,12 +19,13 @@ from typing import Dict, Any, Optional from ...core.acceptor import Work +from ...core.connection import TcpClientConnection from ...common.types import Readables, Writables logger = logging.getLogger(__name__) -class BaseTcpServerHandler(Work): +class BaseTcpServerHandler(Work[TcpClientConnection]): """BaseTcpServerHandler implements Work interface. BaseTcpServerHandler lifecycle is controlled by Threadless core diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh index b31b8452bf..64cc3ae007 100755 --- a/tests/integration/test_integration.sh +++ b/tests/integration/test_integration.sh @@ -15,6 +15,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then exit 1 fi +PROXY_URL="127.0.0.1:$PROXY_PY_PORT" + # Wait for server to come up WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '" while true; do @@ -31,8 +33,8 @@ while true; do curl -v \ --max-time 1 \ --connect-timeout 1 \ - -x 127.0.0.1:$PROXY_PY_PORT \ - http://127.0.0.1:$PROXY_PY_PORT/ 2>/dev/null + -x $PROXY_URL \ + http://$PROXY_URL/ 2>/dev/null if [[ $? == 0 ]]; then break fi @@ -40,6 +42,23 @@ while true; do sleep 1 done +verify_response() { + if [ "$1" == "" ]; + then + echo "Empty response"; + return 1; + else + if [ "$1" == "$2" ]; + then + echo "Ok"; + return 0; + else + echo "Invalid response: '$1', expected: '$2'"; + return 1; + fi + fi; +} + # Check if proxy was started with integration # testing web server plugin. If detected, use # internal web server for integration testing. @@ -47,28 +66,56 @@ done # If integration testing plugin is not found, # detect if we have internet access. If we do, # then use httpbin.org for integration testing. -curl -v \ - -x 127.0.0.1:$PROXY_PY_PORT \ - http://httpbin.org/get -if [[ $? != 0 ]]; then - echo "http request failed" - exit 1 -fi +read -r -d '' ROBOTS_RESPONSE << EOM +User-agent: * +Disallow: /deny +EOM -curl -v \ - -x 127.0.0.1:$PROXY_PY_PORT \ - https://httpbin.org/get -if [[ $? != 0 ]]; then - echo "https request failed" - exit 1 -fi +echo "[Test HTTP Request via Proxy]" +CMD="curl -v -x $PROXY_URL http://httpbin.org/robots.txt" +RESPONSE=$($CMD 2> /dev/null) +verify_response "$RESPONSE" "$ROBOTS_RESPONSE" +VERIFIED1=$? + +echo "[Test HTTPS Request via Proxy]" +CMD="curl -v -x $PROXY_URL https://httpbin.org/robots.txt" +RESPONSE=$($CMD 2> /dev/null) +verify_response "$RESPONSE" "$ROBOTS_RESPONSE" +VERIFIED2=$? +echo "[Test Internal Web Server via Proxy]" curl -v \ - -x 127.0.0.1:$PROXY_PY_PORT \ - http://127.0.0.1:$PROXY_PY_PORT/ -if [[ $? != 0 ]]; then - echo "http request to built in webserver failed" - exit 1 + -x $PROXY_URL \ + http://$PROXY_URL/ +VERIFIED3=$? + +SHASUM=sha256sum +if [ "$(uname)" = "Darwin" ]; +then + SHASUM="shasum -a 256" fi -exit 0 +echo "[Test Download File Hash Verifies 1]" +touch downloaded.hash +echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc -" > downloaded.hash +curl -vL \ + -o downloaded.whl \ + -x $PROXY_URL \ + https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc +cat downloaded.whl | $SHASUM -c downloaded.hash +VERIFIED4=$? +rm downloaded.whl downloaded.hash + +echo "[Test Download File Hash Verifies 2]" +touch downloaded.hash +echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 -" > downloaded.hash +curl -vL \ + -o downloaded.whl \ + -x $PROXY_URL \ + https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 +cat downloaded.whl | $SHASUM -c downloaded.hash +VERIFIED5=$? +rm downloaded.whl downloaded.hash + +EXIT_CODE=$(( $VERIFIED1 || $VERIFIED2 || $VERIFIED3 || $VERIFIED4 || $VERIFIED5 )) +exit $EXIT_CODE