Lists not returned with __iter__ and not compatible with an async for loop

[SelfhostedPro]

Long story short

I'm attempting to gather stats on all of my running docker containers then sending them (via a websocket) back to my frontend. I can't get an async for loop going using docker.containers.list() as it returns a standard list without an aiter method.

• Expected behaviour: Returns a list that is iterable in an async for loop
• Actual behaviour: Returns a standard list that is blocking

How to reproduce (Ignore the fastapi/websocket stuff)

Running the following (without websocket/fastapi stuff) will only stream stats for a single container (if you remove async from the first for loop), not all of them at the same time.

@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
    # auth_success = await websocket_auth(websocket=websocket)
    # if auth_success:
    await websocket.accept()
    dclient = aiodocker.Docker()
    async for app in await dclient.containers.list():
        async with aiodocker.Docker() as docker:
            cpu_total = 0.0
            cpu_system = 0.0
            cpu_percent = 0.0

            _name = app._container['Names'][0][1:]
            container: DockerContainer = await docker.containers.get(_name)
            stats = container.stats(stream=True)
            async for line in stats:
                mem_current = line["memory_stats"]["usage"]
                mem_total = line["memory_stats"]["limit"]

                try:
                    cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
                except KeyError as e:
                    print("error while getting new CPU stats: %r, falling back")
                    cpu_percent = await calculate_cpu_percent(line)

                full_stats = {
                    "name": _name,
                    "time": line['read'],
                    "cpu_percent": cpu_percent,
                    "mem_current": mem_current,
                    "mem_total": line["memory_stats"]["limit"],
                    "mem_percent": (mem_current / mem_total) * 100.0,
                }
                await websocket.send_text(json.dumps(full_stats))

Your environment

Running the latest version of Manjaro, Python 3.8.3, FastAPI. Not sure if anything in my env is going to be useful but if you need anything specifically feel free to ask.

[SelfhostedPro]

Tried changing things around some but still am not having any luck:

@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
    # auth_success = await websocket_auth(websocket=websocket)
    # if auth_success:
    await websocket.accept()
    tasks = []
    async with aiodocker.Docker() as docker: 
        containers = await docker.containers.list()
        for app in containers:
            _name = app._container['Names'][0][1:]
            container: DockerContainer = await docker.containers.get(_name)
            stats = container.stats(stream=True)
            tasks.append(process_container(_name, stats))
    await asyncio.gather(*tasks)

async def process_container(name, stats):
    cpu_total = 0.0
    cpu_system = 0.0
    cpu_percent = 0.0
    async for line in stats:
        mem_current = line["memory_stats"]["usage"]
        mem_total = line["memory_stats"]["limit"]

        try:
            cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
        except KeyError as e:
            print("error while getting new CPU stats: %r, falling back")
            cpu_percent = await calculate_cpu_percent(line)

        full_stats = {
            "name": _name,
            "time": line['read'],
            "cpu_percent": cpu_percent,
            "mem_current": mem_current,
            "mem_total": line["memory_stats"]["limit"],
            "mem_percent": (mem_current / mem_total) * 100.0,
        }
        await websocket.send_text(json.dumps(full_stats))

ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 154, in run_asgi
    result = await self.app(self.scope, self.asgi_receive, self.asgi_send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
    return await self.app(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/fastapi/applications.py", line 180, in __call__
    await super().__call__(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/applications.py", line 111, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 146, in __call__
    await self.app(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/exceptions.py", line 58, in __call__
    await self.app(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 566, in __call__
    await route.handle(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 283, in handle
    await self.app(scope, receive, send)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 57, in app
    await func(session)
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/fastapi/routing.py", line 242, in app
    await dependant.call(**values)
  File "./backend/api/routers/apps.py", line 133, in dashboard
    await asyncio.gather(*tasks)
  File "./backend/api/routers/apps.py", line 139, in process_container
    async for line in stats:
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/containers.py", line 299, in _stats_stream
    async with cm as response:
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/utils.py", line 309, in __aenter__
    resp = await self._coro
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/docker.py", line 241, in _do_query
    response = await self.session.request(
  File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiohttp/client.py", line 357, in _request
    raise RuntimeError('Session is closed')
RuntimeError: Session is closed

[SelfhostedPro]

This fixed it (indentation error)

@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
    # auth_success = await websocket_auth(websocket=websocket)
    # if auth_success:
    await websocket.accept()
    tasks = []
    async with aiodocker.Docker() as docker: 
        containers = await docker.containers.list()
        for app in containers:
            _name = app._container['Names'][0][1:]
            container: DockerContainer = await docker.containers.get(_name)
            stats = container.stats(stream=True)
            tasks.append(process_container(_name, stats, websocket))
        await asyncio.gather(*tasks)

async def process_container(name, stats, websocket):
    cpu_total = 0.0
    cpu_system = 0.0
    cpu_percent = 0.0
    async for line in stats:
        mem_current = line["memory_stats"]["usage"]
        mem_total = line["memory_stats"]["limit"]

        try:
            cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
        except KeyError as e:
            print("error while getting new CPU stats: %r, falling back")
            cpu_percent = await calculate_cpu_percent(line)

        full_stats = {
            "name": name,
            "time": line['read'],
            "cpu_percent": cpu_percent,
            "mem_current": mem_current,
            "mem_total": line["memory_stats"]["limit"],
            "mem_percent": (mem_current / mem_total) * 100.0,
        }
        await websocket.send_text(json.dumps(full_stats))