如何正确处理 tcp stream 并在 python 中按行拆分？

Question

I am using the code below to handle the date coming in from tcp connections. But sometimes, on slower connections, the data is not correctly split into lines and i get handle_line() called with half of the line I want to deal with. The second part of the line get called as next. Any ideas how to solve this? This is used for HTTP so each line is ending with \r\n.

from io import BytesIO
import time
import logging
import errno
import threading
import socket
import os

def listener(client, address):
   try:
      with BytesIO() as buffer:
         while True:
            time.sleep(0.5) #slow down for low speed of data transmission to get fill lines, still does not solve
            resp = client.recv(1024)
            if not resp:
               logging.info('Connection closed by ' + str(address))
               break
            else:
               buffer.write(resp)          # Write to the BytesIO object
               buffer.seek(0)              # Set the file pointer to the SoF
               start_index = 0             # Count the number of characters processed
               for line in buffer:
                   start_index += len(line)
                   handle_line(client,line)       # Do something with your line
               if start_index:
                   buffer.seek(start_index)
                   remaining = buffer.read()
                   buffer.truncate(0)
                   buffer.seek(0)
                   buffer.write(remaining)
               else:
                   buffer.seek(0, 2)
   except IOError as e:
      if e.errno == errno.EPIPE:
         logging.warning(e)
      else:
         logging.warning(e)
   finally:
      client.close()


class nc(threading.Thread):
   def run(self):
      s = socket.socket()
      s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
      try:
         s.bind((tcp_host,tcp_port))
      except:
         logging.error('Can not bind address')
         os._exit(1)
      s.listen(1)
      th = []
      while True:
         client, address = s.accept()
         logging.info('Incoming connection from ' + str(address))
         th.append(threading.Thread(target=listener, args = (client,address)).start())
      s.close()


tcp_host = 'localhost'
tcp_port = 8080

server = nc()
server.daemon = True
server.start()

Answer 1

This is how TCP works. If you want complete messages, buffer the data stream until you have a complete message and extract it from the buffer. There is a built-in "file-like" wrapper to sockets that has a .readline() method that you can use. Also use with to automatically close the reader and client objects.

Untested:

def listener(client, address):
    with client, client.makefile('rb') as reader:
        try:
            while True:
                resp = reader.readline()
                if not resp:
                   logging.info('Connection closed by ' + str(address))
                   break
                handle_line(client,line)       # Do something with your line
        except IOError as e:
            if e.errno == errno.EPIPE:
                logging.warning(e)
            else:
                logging.warning(e)