Python websockets error handling: growing number of threads

Question

I have a very simple MWE - two WS connections, one of which will throw an error. Both are executed as tasks and should each recover. Unfortunately, my implementation leads to growing number of threads (one per error). What am I doing wrong here?

import websockets
import asyncio
import json
uri = "wss://echo.websocket.org"
async def ws(error):
    while True:
        try:
            async with websockets.connect(uri=uri) as websocket:
                await websocket.send("foo")
                async for message in websocket:
                    if error:
                        raise ValueError
                    print(message)
        except:
            print('caught a WS exception - going to sleep for 30 seconds')
            websocket.close()
            await asyncio.sleep(30)


async def main():
    # try:
    ws_task_error = loop.create_task(ws(error=True))
    ws_task_fine = loop.create_task(ws(False))
    await asyncio.wait([ws_task_error, ws_task_fine], return_when=asyncio.FIRST_EXCEPTION)

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())

It was my understanding that as soon as I leave the context manager of websockets.connect(), the thread in charge should be removed. I'm clearly not implementing it correctly here. Ideas?

----- EDIT ------

I feel the one above is WE, but not really M. This example forces an exception immediately after the connection. In logs, the connection shuts down correctly. Still, I end up creating a new thread upon every connection.

import websockets
import asyncio
import logging
uri = "wss://echo.websocket.org"

logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s')
logger = logging.getLogger('websockets')
logger.setLevel(logging.DEBUG)

async def ws():
    while True:
        async with websockets.connect(uri=uri) as websocket:
            try:
                raise ValueError
            except:
                print('Exception thrown')
            print('Exiting...')

async def main():
    ws_task_error = loop.create_task(ws())
    await asyncio.wait([ws_task_error], return_when=asyncio.FIRST_EXCEPTION)

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())

OUTPUT:

2020-01-21 10:44:04,710 DEBUG websockets.protocol: client - state = CONNECTING
2020-01-21 10:44:04,908 DEBUG websockets.protocol: client - event = connection_made(<asyncio.sslproto._SSLProtocolTransport object at 0x00000210389E6828>)
2020-01-21 10:44:05,001 DEBUG websockets.protocol: client - state = OPEN
Exception thrown
Exiting...
2020-01-21 10:44:05,003 DEBUG websockets.protocol: client - state = CLOSING
2020-01-21 10:44:05,003 DEBUG websockets.protocol: client > Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:05,098 DEBUG websockets.protocol: client < Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:05,099 DEBUG websockets.protocol: client - event = connection_lost(None)
2020-01-21 10:44:05,100 DEBUG websockets.protocol: client - state = CLOSED
2020-01-21 10:44:05,100 DEBUG websockets.protocol: client x code = 1000, reason = [no reason]
2020-01-21 10:44:05,100 DEBUG websockets.protocol: client x closing TCP connection
2020-01-21 10:44:05,193 DEBUG websockets.protocol: client - state = CONNECTING
2020-01-21 10:44:05,393 DEBUG websockets.protocol: client - event = connection_made(<asyncio.sslproto._SSLProtocolTransport object at 0x0000021038AB2908>)
2020-01-21 10:44:05,482 DEBUG websockets.protocol: client - state = OPEN
Exception thrown
Exiting...
2020-01-21 10:44:05,489 DEBUG websockets.protocol: client - state = CLOSING
2020-01-21 10:44:05,489 DEBUG websockets.protocol: client > Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:05,580 DEBUG websockets.protocol: client < Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:05,581 DEBUG websockets.protocol: client - event = connection_lost(None)
2020-01-21 10:44:05,582 DEBUG websockets.protocol: client - state = CLOSED
2020-01-21 10:44:05,582 DEBUG websockets.protocol: client x code = 1000, reason = [no reason]
2020-01-21 10:44:05,583 DEBUG websockets.protocol: client x closing TCP connection
2020-01-21 10:44:05,670 DEBUG websockets.protocol: client - state = CONNECTING
2020-01-21 10:44:05,863 DEBUG websockets.protocol: client - event = connection_made(<asyncio.sslproto._SSLProtocolTransport object at 0x0000021038AB2B70>)
2020-01-21 10:44:05,950 DEBUG websockets.protocol: client - state = OPEN
Exception thrown
Exiting...
2020-01-21 10:44:05,953 DEBUG websockets.protocol: client - state = CLOSING
2020-01-21 10:44:05,953 DEBUG websockets.protocol: client > Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:06,036 DEBUG websockets.protocol: client < Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:06,037 DEBUG websockets.protocol: client - event = connection_lost(None)
2020-01-21 10:44:06,037 DEBUG websockets.protocol: client - state = CLOSED
2020-01-21 10:44:06,038 DEBUG websockets.protocol: client x code = 1000, reason = [no reason]
2020-01-21 10:44:06,039 DEBUG websockets.protocol: client x closing TCP connection
2020-01-21 10:44:06,124 DEBUG websockets.protocol: client - state = CONNECTING
2020-01-21 10:44:06,308 DEBUG websockets.protocol: client - event = connection_made(<asyncio.sslproto._SSLProtocolTransport object at 0x0000021038AC6C50>)
2020-01-21 10:44:06,400 DEBUG websockets.protocol: client - state = OPEN
Exception thrown
Exiting...
2020-01-21 10:44:06,402 DEBUG websockets.protocol: client - state = CLOSING
2020-01-21 10:44:06,402 DEBUG websockets.protocol: client > Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:06,495 DEBUG websockets.protocol: client < Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:06,496 DEBUG websockets.protocol: client - event = connection_lost(None)
2020-01-21 10:44:06,496 DEBUG websockets.protocol: client - state = CLOSED
2020-01-21 10:44:06,496 DEBUG websockets.protocol: client x code = 1000, reason = [no reason]
2020-01-21 10:44:06,497 DEBUG websockets.protocol: client x closing TCP connection
2020-01-21 10:44:06,600 DEBUG websockets.protocol: client - state = CONNECTING
2020-01-21 10:44:06,827 DEBUG websockets.protocol: client - event = connection_made(<asyncio.sslproto._SSLProtocolTransport object at 0x0000021038AC6B70>)
2020-01-21 10:44:06,934 DEBUG websockets.protocol: client - state = OPEN
Exception thrown
Exiting...
2020-01-21 10:44:06,936 DEBUG websockets.protocol: client - state = CLOSING
2020-01-21 10:44:06,936 DEBUG websockets.protocol: client > Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:07,042 DEBUG websockets.protocol: client < Frame(fin=True, opcode=8, data=b'\x03\xe8', rsv1=False, rsv2=False, rsv3=False)
2020-01-21 10:44:07,043 DEBUG websockets.protocol: client - event = connection_lost(None)
2020-01-21 10:44:07,044 DEBUG websockets.protocol: client - state = CLOSED
2020-01-21 10:44:07,044 DEBUG websockets.protocol: client x code = 1000, reason = [no reason]
2020-01-21 10:44:07,045 DEBUG websockets.protocol: client x closing TCP connection

Answer 1

while True: should go under async with websockets.connect(uri=uri) as websocket:

In fact in your example the ws_task_fine task locks in async for message in websocket: after the first message. And the ws_task_error task loops, multiplying threads.

Here is quite a crude example of how it could be implemented:

import websockets
import asyncio

uri = "wss://echo.websocket.org"


async def ws(error):
    async with websockets.connect(uri=uri) as websocket:
        while True:
            print('Starting...' + str(error))
            try:
                await websocket.send("foo")
                message = await websocket.recv()
                print(message)
                if error:
                    raise ValueError('Exception')
            except Exception as e:
                print(e)
            print('Finishing...' + str(error))
            await asyncio.sleep(2)


async def main():
    # try:
    ws_task_error = loop.create_task(ws(error=True))
    # ws_task_fine = loop.create_task(ws(False))
    await asyncio.wait([ws_task_error])

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())
    loop.run_forever()

loop.run_forever() allows us to restart the main() coroutine every time it finished

Output:

Starting...True
foo
Exception
Finishing...True
Starting...True
foo
Exception
Finishing...True
Starting...True
foo
Exception
Finishing...True