Documented asyncio with aiopg usage in user's guide

docs/conf.py

@@ -3,7 +3,13 @@
 import sphinx_rtd_theme
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-extensions = ['sphinx.ext.autodoc']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3.6', None),
+    'aiopg': ('http://aiopg.readthedocs.io/en/stable', None),
+    'sqlalchemy': ('http://docs.sqlalchemy.org/en/rel_1_1', None),
+}
 
 source_suffix = '.rst'
 master_doc = 'index'

docs/guide/asyncio.rst

@@ -0,0 +1,72 @@
+Using asyncio
+=============
+
+Hiku has several executors, previous examples were using
+:py:class:`hiku.executors.sync.SyncExecutor` and they can also be used with
+:py:class:`hiku.executors.threads.ThreadsExecutor` to execute query
+concurrently using :py:class:`python:concurrent.futures.ThreadPoolExecutor`
+without any change in the graph definition.
+
+But, to be able to load data using :py:mod:`python:asyncio` library, all data
+loading functions should be coroutines. We will translate one of the
+:doc:`previous examples <database>` to show how to use :py:mod:`python:asyncio`
+and :doc:`aiopg:index` libraries.
+
+Prerequisites
+~~~~~~~~~~~~~
+
+.. note:: Source code of this example can be found
+    `on GitHub <https://github.com/vmagamedov/hiku/blob/master/docs/guide/test_asyncio.py>`_.
+
+.. literalinclude:: test_asyncio.py
+    :lines: 7-27
+
+.. literalinclude:: test_asyncio.py
+    :lines: 41-58
+    :dedent: 4
+
+Graph definition
+~~~~~~~~~~~~~~~~
+
+.. literalinclude:: test_asyncio.py
+    :lines: 83-135
+    :linenos:
+    :emphasize-lines: 4,16,20,27
+
+Note that we are using :py:mod:`hiku.sources.aiopg` source :sup:`[4]`
+in our graph definition, instead of :py:mod:`hiku.sources.sqlalchemy`.
+
+All our custom data loading functions :sup:`[16,20,27]` are coroutine
+functions now and using :py:class:`aiopg:aiopg.sa.Engine` instead of
+:py:class:`sqlalchemy:sqlalchemy.engine.Engine` to execute SQL queries.
+
+Querying graph
+~~~~~~~~~~~~~~
+
+For testing purposes let's define helper coroutine function ``execute``:
+
+.. literalinclude:: test_asyncio.py
+    :lines: 139-149
+    :linenos:
+    :emphasize-lines: 10
+
+Note that :py:meth:`hiku.engine.Engine.execute` method :sup:`[10]`
+returns "awaitable" object, when it is using with
+:py:class:`hiku.executors.asyncio.AsyncIOExecutor`. Here is how it
+should be constructed:
+
+.. literalinclude:: test_asyncio.py
+    :lines: 153
+    :dedent: 4
+
+Testing one to many link:
+
+.. literalinclude:: test_asyncio.py
+    :lines: 154-181
+    :dedent: 4
+
+Testing many to one link:
+
+.. literalinclude:: test_asyncio.py
+    :lines: 186-216
+    :dedent: 4

docs/guide/database.rst

@@ -1,5 +1,5 @@
-Using database
-==============
+Using sqlalchemy
+================
 
 Hiku provides support for loading data from SQL databases using SQLAlchemy_
 library, but Hiku doesn't requires to use it's ORM layer, it requires only Core

docs/guide/index.rst

@@ -7,3 +7,4 @@ User's Guide
     basics
     database
     subgraph
+    asyncio

docs/guide/test_asyncio.py

@@ -4,11 +4,9 @@
 
 # setup storage
 
-import aiopg.sa
-
 from sqlalchemy import MetaData, Table, Column
 from sqlalchemy import Integer, String, ForeignKey, select
-
+from sqlalchemy.sql.ddl import CreateTable
 
 metadata = MetaData()
 
@@ -30,8 +28,7 @@
 
 # setup test environment
 
-from sqlalchemy.sql.ddl import CreateTable
-
+import aiopg.sa
 
 async def init_db(pg_dsn, *, loop):
     db_name = 'test_{}'.format(uuid.uuid4().hex)
@@ -40,7 +37,6 @@ async def init_db(pg_dsn, *, loop):
             await conn.execute('CREATE DATABASE {0}'.format(db_name))
             return db_name
 
-
 async def setup_db(db_dsn, *, loop):
     async with aiopg.sa.create_engine(db_dsn, loop=loop) as db_engine:
         async with db_engine.acquire() as conn:
@@ -61,13 +57,11 @@ async def setup_db(db_dsn, *, loop):
                 dict(id=6, character_id=3, name='Karl Urban'),
             ]))
 
-
 async def drop_db(pg_dsn, db_name, *, loop):
     async with aiopg.sa.create_engine(pg_dsn, loop=loop) as db_engine:
         async with db_engine.acquire() as conn:
             await conn.execute('DROP DATABASE {0}'.format(db_name))
 
-
 @pytest.fixture(scope='session', name='db_dsn')
 def db_dsn_fixture(request):
     loop = asyncio.get_event_loop()
@@ -101,27 +95,23 @@ def fin():
                                          from_column=actor_table.c.character_id,
                                          to_column=actor_table.c.id)
 
-
 async def direct_link(ids):
     return ids
 
-
 @pass_context
 async def to_characters_query(ctx):
     query = select([character_table.c.id])
     async with ctx[SA_ENGINE_KEY].acquire() as conn:
         rows = await conn.execute(query)
     return [row.id for row in rows]
 
-
 @pass_context
 async def to_actors_query(ctx):
     query = select([actor_table.c.id])
     async with ctx[SA_ENGINE_KEY].acquire() as conn:
         rows = await conn.execute(query)
     return [row.id for row in rows]
 
-
 GRAPH = Graph([
     Node('character', [
         sa.Field('id', character_query),
@@ -146,18 +136,18 @@ async def to_actors_query(ctx):
 
 # test graph
 
+import aiopg.sa
+
 from hiku.engine import Engine
 from hiku.result import denormalize
 from hiku.readers.simple import read
 from hiku.executors.asyncio import AsyncIOExecutor
 
-
 async def execute(hiku_engine, sa_engine, graph, query_string):
     query = read(query_string)
     result = await hiku_engine.execute(graph, query, {SA_ENGINE_KEY: sa_engine})
     return denormalize(graph, result, query)
 
-
 @pytest.mark.asyncio(forbid_global_loop=True)
 async def test_character_to_actors(db_dsn, event_loop):
     hiku_engine = Engine(AsyncIOExecutor(event_loop))
@@ -190,7 +180,6 @@ async def test_character_to_actors(db_dsn, event_loop):
             ],
         }
 
-
 @pytest.mark.asyncio(forbid_global_loop=True)
 async def test_actor_to_character(db_dsn, event_loop):
     hiku_engine = Engine(AsyncIOExecutor(event_loop))

pi.yaml

@@ -44,9 +44,11 @@
   from: env
   repository: reg.local/hiku/docs
   provision-with: !AnsibleTasks
+    - raw: apk add --no-cache ca-certificates
+    - raw: update-ca-certificates
     - pip: name={{item}} executable=pip3 extra_args='--no-cache-dir'
       with_items:
-        - sphinx==1.4.6
+        - sphinx==1.4.9
         - sphinx_rtd_theme==0.1.10a0
 
 - !Service