diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..38aaa80 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,41 @@ +# Effect Examples + +## http + +The `http` directory contains a very simple `HTTPRequest` intent and performers +using common HTTP client libraries: +[requests](http://warehouse.python.org/project/requests/) and +[treq](https://warehouse.python.org/project/treq/). + + +## readline_intent + +The `readline_intent.py` file has a simple `ReadLine` intent that uses +`raw_input` (or `input` in Py3) to prompt the user for input. + +## github + +The `github` directory contains a simple application that lets the user input a +GitHub username and prints out a list of all repositories that that user has +access to. It depends on the `http` and `readline_intent` modules. + +There are two entrypoints into the example: +[`examples.github.sync_main`](github/sync_main.py) and +[`examples.github.twisted_main`](github/twisted_main.py). `sync_main` does +typical blocking IO, and `twisted_main` uses asynchronous IO. Note that the +vast majority of the code doesn't need to care about this difference; the only +part that cares about it is the `*_main.py` files. All of the logic in +[`core.py`](github/core.py) is generic. Tests are in +[`test_core.py`](github/test_core.py). + +To run them: + + python -m examples.github.sync_main + +or + + python -m examples.github.twisted_main + + +Note that the twisted example does not run on Python 3, but all other examples +do. diff --git a/examples/github/__init__.py b/examples/github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/github/core.py b/examples/github/core.py new file mode 100644 index 0000000..f3edb84 --- /dev/null +++ b/examples/github/core.py @@ -0,0 +1,69 @@ +""" +Core application / API interaction logic of the GitHub example. + +None of this code needs to change based on your I/O strategy -- you can +use blocking API (e.g. the ``requests`` library) or Twisted, asyncio, +or Tornado implementations of an HTTP client with this code, by providing +different performers for the :obj:`HTTPRequest` intent. +""" + +from __future__ import print_function + +from functools import reduce +import json +import operator + +from effect import Effect, parallel + +from ..readline_intent import ReadLine +from ..http.http_intent import HTTPRequest + + +def get_orgs(name): + """ + Fetch the organizations a user belongs to. + + :return: An Effect resulting in a list of strings naming the user's + organizations. + """ + req = Effect( + HTTPRequest("get", + "https://api.github.com/users/{0}/orgs".format(name))) + return req.on(success=lambda x: [org['login'] for org in json.loads(x)]) + + +def get_org_repos(name): + """ + Fetch the repos that belong to an organization. + + :return: An Effect resulting in a list of strings naming the repositories. + """ + req = Effect( + HTTPRequest("get", + "https://api.github.com/orgs/{0}/repos".format(name))) + return req.on(success=lambda x: [repo['name'] for repo in json.loads(x)]) + + +def get_orgs_repos(name): + """ + Fetch ALL of the repos that a user has access to, in any organization. + + :return: An Effect resulting in a list of repositories. + """ + req = get_orgs(name) + req = req.on(lambda org_names: parallel(map(get_org_repos, org_names))) + req = req.on(lambda repo_lists: reduce(operator.add, repo_lists)) + return req + + +def main_effect(): + """ + Request a username from the keyboard, and look up all repos in all of + that user's organizations. + + :return: an Effect resulting in a list of repositories. + """ + intent = ReadLine("Enter Github Username> ") + read_eff = Effect(intent) + org_repos_eff = read_eff.on(success=get_orgs_repos) + return org_repos_eff diff --git a/examples/github/sync_main.py b/examples/github/sync_main.py new file mode 100644 index 0000000..c45908a --- /dev/null +++ b/examples/github/sync_main.py @@ -0,0 +1,62 @@ +""" +Run this example with: + python -m examples.github.sync_main + +This is an example of using Effect in a normal program that uses +synchronous/blocking functions to do I/O. + +This code has these responsibilities: + +- set up a dispatcher that knows how to find performers for all intents + used in this application. The application uses ReadLine, HTTPRequest, and + ParallelEffects. +- use :func:`effect.sync_perform` to perform an effect, which returns the + result of the effect (or raises an exception it it failed). +""" + +from __future__ import print_function + +from functools import partial +from multiprocessing.pool import ThreadPool + +from effect import ( + ComposedDispatcher, + ParallelEffects, + TypeDispatcher, + sync_perform) +from effect.threads import perform_parallel_with_pool + + +from ..http.http_intent import HTTPRequest +from ..http.sync_http import perform_request_requests +from ..readline_intent import ReadLine, perform_readline_stdin + +from .core import main_effect + + +def get_dispatcher(): + """ + Create a dispatcher that can find performers for :obj:`ReadLine`, + :obj:`HTTPRequest`, and :obj:`ParallelEffects`. There's a built-in + performer for ParallelEffects that uses a multiprocessing ThreadPool, + :func:`effect.perform_parallel_with_pool`. + """ + my_pool = ThreadPool() + pool_performer = partial(perform_parallel_with_pool, my_pool) + return ComposedDispatcher([ + TypeDispatcher({ + ReadLine: perform_readline_stdin, + HTTPRequest: perform_request_requests, + ParallelEffects: pool_performer, + }) + ]) + + +def main(): + dispatcher = get_dispatcher() + eff = main_effect() + print(sync_perform(dispatcher, eff)) + + +if __name__ == '__main__': + main() diff --git a/examples/test_github_example.py b/examples/github/test_core.py similarity index 87% rename from examples/test_github_example.py rename to examples/github/test_core.py index 268fc22..504710b 100644 --- a/examples/test_github_example.py +++ b/examples/github/test_core.py @@ -1,5 +1,5 @@ -# Run these tests with "trial examples.test_github_example" -# or "python -m testtools.run examples.test_github_example" +# Run these tests with "trial examples.github.test_core" +# or "python -m testtools.run examples.github.test_core" import json @@ -7,7 +7,8 @@ from effect import ParallelEffects from effect.testing import resolve_effect -from . import github_example + +from .core import get_orgs, get_org_repos, get_orgs_repos class GithubTests(TestCase): @@ -16,14 +17,14 @@ def test_get_orgs_request(self): get_orgs returns an effect that makes an HTTP request to the GitHub API to look up organizations for a user. """ - eff = github_example.get_orgs('radix') + eff = get_orgs('radix') http = eff.intent self.assertEqual(http.method, 'get') self.assertEqual(http.url, 'https://api.github.com/users/radix/orgs') def test_get_orgs_success(self): """get_orgs extracts the result into a simple list of orgs.""" - eff = github_example.get_orgs('radix') + eff = get_orgs('radix') self.assertEqual( resolve_effect(eff, json.dumps([{'login': 'twisted'}, {'login': 'rackerlabs'}])), @@ -34,14 +35,14 @@ def test_get_org_repos_request(self): get_org_repos returns an effect that makes an HTTP request to the GitHub API to look up repos in an org. """ - eff = github_example.get_org_repos('twisted') + eff = get_org_repos('twisted') http = eff.intent self.assertEqual(http.method, 'get') self.assertEqual(http.url, 'https://api.github.com/orgs/twisted/repos') def test_get_org_repos_success(self): """get_org_repos extracts the result into a simple list of repos.""" - eff = github_example.get_org_repos('radix') + eff = get_org_repos('radix') self.assertEqual( resolve_effect(eff, json.dumps([{'name': 'twisted'}, {'name': 'txstuff'}])), @@ -53,7 +54,7 @@ def test_get_orgs_repos(self): a user, and then looks up all of the repositories of those orgs in parallel, and returns a single flat list of all repos. """ - effect = github_example.get_orgs_repos('radix') + effect = get_orgs_repos('radix') self.assertEqual(effect.intent.method, 'get') self.assertEqual(effect.intent.url, 'https://api.github.com/users/radix/orgs') diff --git a/examples/github/twisted_main.py b/examples/github/twisted_main.py new file mode 100644 index 0000000..ee0d791 --- /dev/null +++ b/examples/github/twisted_main.py @@ -0,0 +1,59 @@ +""" +Run this example with: + python -m examples.github.twisted_main + +This is an example of using Effect with Twisted. + +It's important to note that none of the application code relies on Twisted -- +this is the only file that has any dependencies on Twisted. There's also an +example of running the same application code in ``sync_main.py`` in the same +directory. + +This code has these responsibilities: + +- set up a dispatcher that knows how to find performers for all intents + used in this application. The application uses ReadLine, HTTPRequest, and + ParallelEffects. +- use :func:`effect.twisted.perform` to perform an effect, which returns a + Deferred that we can return from our react function. +""" + +from __future__ import print_function + +from twisted.internet.task import react + +from effect.twisted import make_twisted_dispatcher, perform +from effect import ( + ComposedDispatcher, + TypeDispatcher) + +from ..http.http_intent import HTTPRequest +from ..http.twisted_http import perform_request_with_treq +from ..readline_intent import ReadLine, perform_readline_stdin + +from .core import main_effect + + +def get_dispatcher(reactor): + """ + Create a dispatcher that can find performers for :obj:`ReadLine`, + :obj:`HTTPRequest`, and :obj:`ParallelEffects`. + :func:`make_twisted_dispatcher` is able to provide the ``ParallelEffects`` + performer, so we compose it with our own custom :obj:`TypeDispatcher`. + """ + return ComposedDispatcher([ + TypeDispatcher({ + ReadLine: perform_readline_stdin, + HTTPRequest: perform_request_with_treq, + }), + make_twisted_dispatcher(reactor), + ]) + + +def main(reactor): + dispatcher = get_dispatcher(reactor) + eff = main_effect() + return perform(dispatcher, eff).addCallback(print) + +if __name__ == '__main__': + react(main, []) diff --git a/examples/github_example.py b/examples/github_example.py deleted file mode 100644 index 774bcdf..0000000 --- a/examples/github_example.py +++ /dev/null @@ -1,85 +0,0 @@ -# python -m examples.github_example - -# An example that shows -# - usage of HTTP effects -# - chaining effects with callbacks that return more effects -# - a custom effect that reads from the console -# - very simple Twisted-based usage - -# Unfortunately python3 is not yet supported by this example since treq -# has not been ported. - -from __future__ import print_function - -import operator -import json -from functools import reduce - -from effect import ( - ComposedDispatcher, - Effect, - TypeDispatcher, - parallel) -from effect.twisted import perform, make_twisted_dispatcher -from .http_example import HTTPRequest, treq_http_request -from .readline_example import ReadLine, stdin_read_line - - -def get_orgs(name): - """ - Fetch the organizations a user belongs to. - - :return: An Effect resulting in a list of strings naming the user's - organizations. - """ - req = Effect( - HTTPRequest("get", - "https://api.github.com/users/{0}/orgs".format(name))) - return req.on(success=lambda x: [org['login'] for org in json.loads(x)]) - - -def get_org_repos(name): - """ - Fetch the repos that belong to an organization. - - :return: An effect resulting in a list of strings naming the repositories. - """ - req = Effect( - HTTPRequest("get", - "https://api.github.com/orgs/{0}/repos".format(name))) - return req.on(success=lambda x: [repo['name'] for repo in json.loads(x)]) - - -def get_orgs_repos(name): - """ - Fetch ALL of the repos that a user has access to, in any organization. - """ - req = get_orgs(name) - req = req.on(lambda org_names: parallel(map(get_org_repos, org_names))) - req = req.on(lambda repo_lists: reduce(operator.add, repo_lists)) - return req - - -def main_effect(): - """ - Let the user enter a username, and then list all repos in all of that - username's organizations. - """ - return Effect(ReadLine("Enter GitHub Username> ")).on( - success=get_orgs_repos) - - -# Only the code below here depends on Twisted. -def main(reactor): - dispatcher = ComposedDispatcher([ - TypeDispatcher({ - ReadLine: stdin_read_line, - HTTPRequest: treq_http_request, - }), - make_twisted_dispatcher(reactor) - ]) - return perform(dispatcher, main_effect()).addCallback(print) - -if __name__ == '__main__': - from twisted.internet.task import react - react(main, []) diff --git a/examples/http/__init__.py b/examples/http/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/http/http_intent.py b/examples/http/http_intent.py new file mode 100644 index 0000000..406f87b --- /dev/null +++ b/examples/http/http_intent.py @@ -0,0 +1,14 @@ +class HTTPRequest(object): + """ + An HTTP request intent. + """ + + def __init__(self, method, url, headers=None, data=None): + self.method = method + self.url = url + self.headers = headers + self.data = data + + def __repr__(self): + return "HTTPRequest(%r, %r, headers=%r, data=%r)" % ( + self.method, self.url, self.headers, self.data) diff --git a/examples/http/sync_http.py b/examples/http/sync_http.py new file mode 100644 index 0000000..a07d411 --- /dev/null +++ b/examples/http/sync_http.py @@ -0,0 +1,22 @@ +from effect import sync_performer + +import requests + + +@sync_performer +def perform_request_requests(dispatcher, http_request): + """ + A performer for :obj:`HTTPRequest` that uses the ``requests`` library. + """ + headers = ( + http_request.headers.copy() + if http_request.headers is not None + else {}) + if 'user-agent' not in headers: + headers['user-agent'] = 'Effect example' + response = requests.request( + http_request.method.lower(), + http_request.url, + headers=headers, + data=http_request.data) + return response.content.decode('utf-8') diff --git a/examples/http_example.py b/examples/http/twisted_http.py similarity index 53% rename from examples/http_example.py rename to examples/http/twisted_http.py index b02f537..dda2947 100644 --- a/examples/http_example.py +++ b/examples/http/twisted_http.py @@ -1,26 +1,11 @@ from effect.twisted import deferred_performer - -class HTTPRequest(object): - """ - An HTTP request intent. - """ - - def __init__(self, method, url, headers=None, data=None): - self.method = method - self.url = url - self.headers = headers - self.data = data - - def __repr__(self): - return "HTTPRequest(%r, %r, headers=%r, data=%r)" % ( - self.method, self.url, self.headers, self.data) +import treq @deferred_performer -def treq_http_request(dispatcher, http_request): +def perform_request_with_treq(dispatcher, http_request): """A performer for :obj:`HTTPRequest` that uses the ``treq`` library.""" - import treq headers = ( http_request.headers.copy() if http_request.headers is not None diff --git a/examples/readline_example.py b/examples/readline_intent.py similarity index 69% rename from examples/readline_example.py rename to examples/readline_intent.py index d02adb7..c871bb5 100644 --- a/examples/readline_example.py +++ b/examples/readline_intent.py @@ -11,6 +11,6 @@ def __init__(self, prompt): @sync_performer -def stdin_read_line(dispatcher, readline): - """Perform a :obj:`ReadLine` intent by reading from STDIN.""" +def perform_readline_stdin(dispatcher, readline): + """Perform a :obj:`ReadLine` intent by reading from stdin.""" return input(readline.prompt)