Sequencing issue when upgrading Python asyncio connection to TLS

Question

I have a Python server that needs to upgrade an existing connection to TLS. The connection is created by asyncio.start_server .

It needs to upgrade to TLS in the middle of a connection because it is emulating MySQL's wire protocol .

The project is mysql-mimic . Here is what I've tried so far: https://github.com/kelsin/mysql-mimic/pull/17

There appears to be a some kind of sequencing issue: if the client starts TLS before the server, the server never completes the handshake.

Here is a script to reproduce the issue:

import asyncio
import ssl

# Self-signed certs for testing
CERT_FILE = "cert.pem"
CERT = """
-----BEGIN CERTIFICATE-----
MIIEIzCCAwugAwIBAgIUUMROPFb3ZMssWfl9/tJVMB0Zl0owDQYJKoZIhvcNAQEL
BQAwgZ8xCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQH
DA1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKDAtteXNxbC1taW1pYzEUMBIGA1UECwwL
bXlzcWwtbWltaWMxFDASBgNVBAMMC215c3FsLW1pbWljMSEwHwYJKoZIhvcNAQkB
FhJleGFtcGxlQGRvbWFpbi5jb20wIBcNMjIxMDI0MTgyNzQ0WhgPMjEyMjA5MzAx
ODI3NDRaMIGfMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQG
A1UEBwwNU2FuIEZyYW5jaXNjbzEUMBIGA1UECgwLbXlzcWwtbWltaWMxFDASBgNV
BAsMC215c3FsLW1pbWljMRQwEgYDVQQDDAtteXNxbC1taW1pYzEhMB8GCSqGSIb3
DQEJARYSZXhhbXBsZUBkb21haW4uY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
MIIBCgKCAQEAsnmS0k1Q0f+mv7izbT5K8zqYP54KwyW000qMhYLW/Ir5fGAw2QHg
Q1vMBY+1oaHcO1DxjriG4l8P8ho9AShAD63Q3PVUiy3Prxi3PaZ/jPsI5rN/8s7s
TXai6Po6gD56uYtZACl4cjF2ob3Vy/qzIPitW3D7UVEL+nqDEZUSmbFnT+NAMaCv
Bq+Zf94vQSpQXgUSbTNAuFqjwMLeb8VX31e5yFOvhRd1Y65MOwmeCX0hMZ+XtcPc
xtgCyQ9uT2OaKcqcngE/LtGnC4UJy0u5bcI8pPgwenGWNhQ6LrqLWAUizEXoprY8
PUVoE42Itm7j1ODKm71OpNPqEjXxvs924wIDAQABo1MwUTAdBgNVHQ4EFgQUSVIG
PQ1FTdvJeup85cCLpp/5wGIwHwYDVR0jBBgwFoAUSVIGPQ1FTdvJeup85cCLpp/5
wGIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAJcoALjwItgWa
Qpr/hJCHhtf1gS6DCbtQoc+Owvz0iJkC+zqeaFlOYwVQz/n8iQ45g03cLMxHfMEv
lj1ctHfVGI+2exp2hesMmU+CT49D9qFS2vQnaKxkeUqv5ia2d+V49jE6fj3CxIPP
T3yI9+K2Ojx+7/WVVxCgz+eIZJbTCugoypDksldfuY4mx48E9qefOlBNcYyBzHt4
4iymCFFrfQHfMX+PnKDGHaoh/T/oZxdZRyDxNqTLfiyZ1PtAbueeT4Jf6CmamQZh
IadFPXnA1YlEqreMk6nDrF8pqgOosHIngLhhUXHAVj/Br3UaDTaUGzlrlL7rLpRL
oToCkplUgg==
-----END CERTIFICATE-----
"""
KEY_FILE = "key.pem"
KEY = """
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyeZLSTVDR/6a/
uLNtPkrzOpg/ngrDJbTTSoyFgtb8ivl8YDDZAeBDW8wFj7Whodw7UPGOuIbiXw/y
Gj0BKEAPrdDc9VSLLc+vGLc9pn+M+wjms3/yzuxNdqLo+jqAPnq5i1kAKXhyMXah
vdXL+rMg+K1bcPtRUQv6eoMRlRKZsWdP40AxoK8Gr5l/3i9BKlBeBRJtM0C4WqPA
wt5vxVffV7nIU6+FF3Vjrkw7CZ4JfSExn5e1w9zG2ALJD25PY5opypyeAT8u0acL
hQnLS7ltwjyk+DB6cZY2FDouuotYBSLMReimtjw9RWgTjYi2buPU4MqbvU6k0+oS
NfG+z3bjAgMBAAECggEBAI1wPUO+k/soUByGImPDxyAE4p0gAUVv/2KnJL+11exj
sp23mV6Q1wpqmEAcCIQkQuUbG6PQZszFK1zhIFFndYU3aVuCbNKzpnAL9UO9TD4M
v5wcypxBEhG9oBNkIrJ5UUbzwL+ZHePZgTtitykk74qEqNXbrr9drFF/f5mSeyAi
nUkizI/aGJVp2+NVpsFvZd0M5gM1tEWoBq/5fny/b1oWM/blh1ZwWQII3bTcDABH
1h30oTuGWYyeQExUjiS4SDDJ8BH9PZxUGQhQsK1ZuJczJ5B+jh0OWicdD+P+V2EB
npoWQVHKwqwfDQisx2T04TAb65QcbM7V/EwgQPpcd5ECgYEA3bQbff5bIvbgJwCo
1bHxj3MMmu1Qs+Km9nR8FbEtFaF1lbXzPLCj98VDpT3tBNw4VwpbVYn5rd4ZSs6h
jlK2zEakFRvU/qgtTrUviW4oeSrEtUt8jI0QkdobUWPM8HuZih6Lgm5DTU6HwIBR
dxkhojx7c1zwSm66Bojvn/CjVakCgYEAzhWHAxkNAj97XvjUe9XsV41jiksheNS2
7vAm1PASq9MGFpSazq2NeXTxuChvMK8vN9r2FngnFKW4hLGMi8wpReT+S1C32sNP
ZWRq6w2JWgQvh7dNSNmn0mBc7zwUB6sFeS72q3Ge355dPJiuomRLxZmm+P3suHIV
b+4Fj6q0p6sCgYEAqj5EojJwn1+97pVGEJqM6N+qvUkgoJGaLkRyiGG+Qg7y8RyA
BImLz5ZuBHSSDhphNQ1h50SFMusKtvQHAPgpIKHaG898dnSEHh1pvHmXoLujw6eM
o40rPSSjt5MQa1YuJ+6eqHCtQ67a9YpThEYLGr6g+YxThISUWrJKd6HceskCgYB0
gWMUc0MRdEYQyOeHIsc8L+iINDU2FDtfFVE+rIJBtUkJ1vU1xpPmiCBnFiTWBxPQ
pe7dgQvG9nE8QwvLtJ3Yr767YWSvPh9SmNSBEeQGibs9JHmCp9niayve673/H8Y2
XkCBZ/iDPwpCyaZglAbqLRViSltbYtOPtaZbNAxxhQKBgH8Gj0yg3A5DwWEyeUKR
KJBS2rPiKgrhJs8Kd8GZZUb3H5WGqzfgrRK1p9j5Ug8UXSWbB9e5jg4ymVtcAAhc
tRz4rCvYNY8fHlA2TfzrOeEuuxtoMFyxd26eFjZvS/w2VdFrIQZbSvDkV4hMqhpS
CCslSFIIZOMCzqgQtM+NTQJ6
-----END PRIVATE KEY-----
"""


async def start_tls(reader, writer, sslcontext, server_side=False):
    # Borrowed from https://stackoverflow.com/questions/62851407/how-do-i-enable-tls-on-an-already-connected-python-asyncio-stream
    transport = writer.transport
    protocol = transport.get_protocol()
    loop = asyncio.get_event_loop()
    new_transport = await loop.start_tls(
        transport=transport,
        protocol=protocol,
        sslcontext=sslcontext,
        server_side=server_side,
        ssl_handshake_timeout=5,  # arbitrarily small timeout so the test fails faster
    )
    reader._transport = new_transport
    writer._transport = new_transport


async def server_cb(reader, writer):
    # Sleeping 1 second seems to reliably reproduce the error
    await asyncio.sleep(1)

    sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
    sslcontext.load_cert_chain(CERT_FILE, KEY_FILE)
    await start_tls(reader, writer, sslcontext, server_side=True)

    # We never get here
    print(await reader.read(4))
    writer.write(b"pong")
    await writer.drain()


async def client(reader, writer):
    sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    sslcontext.check_hostname = False
    sslcontext.verify_mode = ssl.VerifyMode.CERT_NONE
    await start_tls(reader, writer, sslcontext, server_side=False)

    # We never get here
    writer.write(b"ping")
    await writer.drain()
    print(await reader.read(4))


async def main():
    with open(CERT_FILE, "w") as fp:
        fp.write(CERT)
    with open(KEY_FILE, "w") as fp:
        fp.write(KEY)

    server = await asyncio.start_server(
        client_connected_cb=server_cb,
        host="localhost",
        port=0,  # Let the OS pick an available port
    )
    task = asyncio.create_task(server.serve_forever())
    try:
        port = server.sockets[0].getsockname()[1]
        reader, writer = await asyncio.open_connection(host="localhost", port=port)
        await client(reader, writer)
        writer.close()
        await writer.wait_closed()
    finally:
        task.cancel()

asyncio.run(main())

The output:

Task exception was never retrieved
future: <Task finished name='Task-5' coro=<server_cb() done, defined at /Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py:82> exception=ConnectionResetError()>
Traceback (most recent call last):
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 88, in server_cb
    await start_tls(reader, writer, sslcontext, server_side=True)
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 71, in start_tls
    new_transport = await loop.start_tls(
  File "/Users/barak_alon/.pyenv/versions/3.9.7/lib/python3.9/asyncio/base_events.py", line 1231, in start_tls
    await waiter
ConnectionResetError
Traceback (most recent call last):
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 129, in <module>
    asyncio.run(main())
  File "/Users/barak_alon/.pyenv/versions/3.9.7/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/Users/barak_alon/.pyenv/versions/3.9.7/lib/python3.9/asyncio/base_events.py", line 642, in run_until_complete
    return future.result()
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 123, in main
    await client(reader, writer)
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 100, in client
    await start_tls(reader, writer, sslcontext, server_side=False)
  File "/Users/barak_alon/Library/Application Support/JetBrains/PyCharmCE2022.2/scratches/tls_race_condition.py", line 71, in start_tls
    new_transport = await loop.start_tls(
  File "/Users/barak_alon/.pyenv/versions/3.9.7/lib/python3.9/asyncio/base_events.py", line 1231, in start_tls
    await waiter
ConnectionAbortedError: SSL handshake is taking longer than 5 seconds: aborting the connection

Anyone have any idea what's going on here? Is there a fix that doesn't require a custom asyncio Protocol ?

Answer 1

calling.pause_reading() on the transport before you try to elevate it to TLS might help