Using httpx with httpcore newer than 0.14.5 results in `The connection pool was closed while XXX HTTP requests/responses were still in-flight.` . Need help figuring it out.

[teodoryantcheff]

Using httpx with httpcore newer than 0.14.5 results in RuntimeError: The connection pool was closed while XXX HTTP requests/responses were still in-flight. Anything newer and it breaks, but I need help figuring out why.

So, I have this code:

async def main():
    async with httpx.AsyncClient(proxies=proxyconfig.proxies) as client:
        .
        .
        .
        tasks = []
        for y, yms in year_makes:
            for make_slug in yms:
                tasks.append(get_models(year=y, make_slug=make_slug, client=client))

        # here tasks has ~2000 API call croutine objects in it

        print(f'making {len(tasks)} calls')     
        responses = await asyncio.gather(*tasks)
        .
        .

async def get_models(year: int | str, make_slug: tuple[str, str], *, client=None) -> list[list]:
    r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
    ret = [[year, make_slug[0], make_slug[1], i['model'], i['value']] for i in r.json()['items']]
    return ret

@backoff.on_exception(backoff.expo,
                      (httpx.HTTPStatusError,  # 429s
                       httpx.RemoteProtocolError,
                       httpx.ReadTimeout,  # fsck the proxy
                       httpx.ConnectTimeout),
                      on_backoff=backoff_hdlr,
                      max_time=240)
async def api_call(url: str, *, params: dict[str, ...] = None, client: httpx.AsyncClient = None) -> httpx.Response:
    params = params if params else {}
    if client:
        r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
    else:
        r = await httpx.AsyncClient().get(URL_BASE + url, params=DFL_PARAMS | params)
    r.raise_for_status()
    return r

and this combination of httpcore and httpx:

httpcore==0.14.5
httpx==0.23.1

on

(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ python -V 
Python 3.11.0
(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ lsb_release -a 
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 22.04.1 LTS
Release:	22.04
Codename:	jammy
(venv) teo@thinkpad:~/src/sgt--ymm-scraper$ 

results in smooth sailing -- It all works as expected and life is good and all that.

While using

httpcore==0.16.1
httpx==0.23.1

Suffers a horrible death with:

unhandled exception during asyncio.run() shutdown
task: <Task finished name='Task-1464' coro=<get_models() done, defined at /home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py:84> exception=ValueError('list.remove(x): x not in list')>
Traceback (most recent call last):
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 110, in main
    responses = await asyncio.gather(*tasks)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
    r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
    ret = await target(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
    r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
    return await self.request(
           ^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 221, in handle_async_request
    await self._attempt_to_acquire_connection(status)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 178, in _attempt_to_acquire_connection
    status.set_connection(connection)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 22, in set_connection
    assert self.connection is None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.11/asyncio/runners.py", line 190, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/asyncio/base_events.py", line 650, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 91, in main
    async with httpx.AsyncClient(proxies=proxyconfig.proxies) as client:
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 2006, in __aexit__
    await proxy.__aexit__(exc_type, exc_value, traceback)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 332, in __aexit__
    await self._pool.__aexit__(exc_type, exc_value, traceback)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 326, in __aexit__
    await self.aclose()
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 312, in aclose
    raise RuntimeError(
RuntimeError: The connection pool was closed while 170 HTTP requests/responses were still in-flight.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 227, in handle_async_request
    connection = await status.wait_for_connection(timeout=timeout)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 34, in wait_for_connection
    await self._connection_acquired.wait(timeout=timeout)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_synchronization.py", line 38, in wait
    await self._event.wait()
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 1842, in wait
    if await self._event.wait():
       ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/asyncio/locks.py", line 213, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
    r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
    ret = await target(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
    r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
    return await self.request(
           ^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 232, in handle_async_request
    self._requests.remove(status)
ValueError: list.remove(x): x not in list
unhandled exception during asyncio.run() shutdown
task: <Task finished name='Task-1466' coro=<get_models() done, defined at /home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py:84> exception=ValueError('list.remove(x): x not in list')>
Traceback (most recent call last):
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 110, in main
    responses = await asyncio.gather(*tasks)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 85, in get_models
    r = await api_call('Models', params={'year': year, 'make': make_slug[0]}, client=client)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
    ret = await target(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/ymm-scrape.py", line 65, in api_call
    r = await client.get(URL_BASE + url, params=DFL_PARAMS | params)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
    return await self.request(
           ^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 221, in handle_async_request
    await self._attempt_to_acquire_connection(status)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 178, in _attempt_to_acquire_connection
    status.set_connection(connection)
  File "/home/teo/src/sgt-uship-ymm-scraper/venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 22, in set_connection
    assert self.connection is None
AssertionError

And this happens always near the end of the request pile. Tested with splitting tasks in chunks of various sizes but it always dies at the last chunk.
Same behavior using any version of httpcore newer than httpcore==0.14.5.

I read @tomchristie 's post hinting at "oddly structured" code, but I honestly cannot see anything wrong with it.

I have looked at:

• httpx 0.21.0 showing pool-connection traceback when 0.20.0 does not. #510
• this
• this
• this

And still have no idea what (if anything) I'm doing wrong.

Please advice on what I'm doing wrong.

Many thanks in advance!

[obendidi]

+1
on python3.8
fails with httpcore=0.16.1 but works with httpcore=0.15.0, both with httpx=0.23

I'm using a similar code to @teodoryantcheff: 1 async client across different tasks used with asyncio.gather

The error I'm getting:

[2022-11-24T18:21:12Z] Traceback (most recent call last):
--
  | [2022-11-24T18:21:12Z]   File "scripts/cli.py", line 43, in <module>
  | [2022-11-24T18:21:12Z]     main()
  | [2022-11-24T18:21:12Z]   File "scripts/cli.py", line 30, in main
  | [2022-11-24T18:21:12Z]     FCVCli(
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/cli.py", line 350, in __init__
  | [2022-11-24T18:21:12Z]     self._run_subcommand(self.subcommand)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/cli.py", line 626, in _run_subcommand
  | [2022-11-24T18:21:12Z]     fn(**fn_kwargs)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
  | [2022-11-24T18:21:12Z]     self._call_and_handle_interrupt(
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _call_and_handle_interrupt
  | [2022-11-24T18:21:12Z]     return trainer_fn(*args, **kwargs)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 737, in _fit_impl
  | [2022-11-24T18:21:12Z]     results = self._run(model, ckpt_path=self.ckpt_path)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1097, in _run
  | [2022-11-24T18:21:12Z]     self._data_connector.prepare_data()
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 114, in prepare_data
  | [2022-11-24T18:21:12Z]     self.trainer._call_lightning_datamodule_hook("prepare_data")
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1571, in _call_lightning_datamodule_hook
  | [2022-11-24T18:21:12Z]     return fn(*args, **kwargs)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/fcv/service/lightning/data/dali/data_module.py", line 268, in prepare_data
  | [2022-11-24T18:21:12Z]     super().prepare_data(*args, **kwargs)
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/fcv/service/lightning/data/base/data_module.py", line 58, in prepare_data
  | [2022-11-24T18:21:12Z]     self._image_keys = loop.run_until_complete(
  | [2022-11-24T18:21:12Z]   File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
  | [2022-11-24T18:21:12Z]     return future.result()
  | [2022-11-24T18:21:12Z]   File "/repo/folder/libs/datasets/fcv/lib/datasets/download.py", line 114, in download_datasets
  | [2022-11-24T18:21:12Z]     await client.aclose()
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpx/_client.py", line 1974, in aclose
  | [2022-11-24T18:21:12Z]     await self._transport.aclose()
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpx/_transports/default.py", line 365, in aclose
  | [2022-11-24T18:21:12Z]     await self._pool.aclose()
  | [2022-11-24T18:21:12Z]   File "/repo/folder/subfolder/lightning/.venv/lib/python3.8/site-packages/httpcore/_async/connection_pool.py", line 312, in aclose
  | [2022-11-24T18:21:12Z]     raise RuntimeError(
  | [2022-11-24T18:21:12Z] RuntimeError: The connection pool was closed while 15 HTTP requests/responses were still in-flight.

[lovelydinosaur]

Closed by #627 and released as 0.16.2