python3 tcp client server communication

Question

I want to send an image (.pgm) via TCP as soon as it is written to the ramdisk. For this I'm working with pyinotify and sockets. After the picture is sent I would like to tell the server to stop now. Everything works fine but the last part gives me following error:

if data.decode('utf-8') == 'stop': UnicodeDecodeError: 'utf-8' codec can't
decode byte 0x88 in position 319: invalid start byte

Client:

import pyinotify
import socket
import traceback
import sys


class ModHandler(pyinotify.ProcessEvent):
    def __init__(self, socket, buffer_size):
        self.socket = socket
        self.buffer_size = buffer_size

    def process_IN_CLOSE_WRITE(self, event):
        try:
            self.socket.send(bytes(event.pathname, encoding='utf-8'))
            file = open(event.pathname, "rb")
            line = file.read(self.buffer_size) 
            while(line):
                self.socket.send(line)  
                line = file.read(self.buffer_size)

        except Exception:
            traceback.print_exc()
        finally:
            try:
                self.socket.send(bytes('stop', encoding='utf-8'))
                print("done")
                file.close
            except Exception:
                traceback.print_exc()

class TCPStream():
    def __init__(self, ip, port, buffer_size):
        self.ip = ip
        self.port = port
        self.buffer_size = buffer_size
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self.socket.connect((self.ip, self.port))
        except Exception:
            traceback.print_exc()

    def __del__(self):
        try:
            self.socket.close()
        except Exception:
            traceback.print_exc()


stream = TCPStream('127.0.0.1', 5005, 1024)

handler = ModHandler(stream.socket, stream.buffer_size)
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm, handler)
wd_value = wm.add_watch("/media/ram_disk", pyinotify.IN_CLOSE_WRITE)
if wd_value["/media/ram_disk"] <= 0:
    print("can't add watchmanager to the ram_disk... insufficient
    authorization? another watchmanager already running?")
    sys.exit(0)
notifier.loop()

Server:

import socket


TCP_IP = '127.0.0.1'
TCP_PORT = 5005
BUFFER_SIZE = 1024 

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(1)
conn, addr = s.accept()
print("connection address: ", addr)

path = conn.recv(BUFFER_SIZE).decode('utf-8')
filename = path.split("/")

with open(filename[3], 'wb') as f:
    data = conn.recv(BUFFER_SIZE)
    while data:
        print("receiving...")
        f.write(data)
        data = conn.recv(BUFFER_SIZE)

        if not data: 
            break

        if data.decode('utf-8') == 'stop':
            f.close()
            print("done")
            break

conn.close()

The goal is to have a constant TCP stream of images written to the ramdisk. Therefore I wanted to communicate via Bytes with the server to tell him what to do. It seems that after the first picture gets transmitted it breaks somehow. Any help is appreciated!

Answer 1

What if four consecutive bytes in your image at the beginning of a buffer happen to match the ASCII (and UTF-8) characters stop ? Also, how does the receiving side know when the file name ends, and the file data starts?

You should create a binary encoding that frames the various bits of your data. This makes the whole process nicely deterministic. That's often best done with the struct module's pack and unpack methods. Given a file you want to send, client side:

import os
import struct

    ...
    pathname = event.pathname.encode('utf-8')  # Encode pathname into bytes
    pathname_len = len(pathname)
    file = open(event.pathname, "rb")
    filesize = os.path.getsize(event.filename) # Get file size
    # Encode size of file name, and size of file into a binary header
    header_format = struct.Struct("!II")
    header = header_format.pack(pathname_len, filesize)
    self.socket.sendall(header)
    self.socket.sendall(pathname)
    while True:
        line = file.read(self.buffer_size)
        if not line: break # EOF
        self.socket.sendall(line)

    # (Remove sending of 'stop' from finally block)

Note the use of sendall to ensure that the entire buffer gets sent (it is legal for send to send only part of a buffer but that can result in missing bytes if you don't account for it).

Server side will look something like this:

import struct

    ...
    def recv_exactly(s, buffer_len):
        """ This is the converse of sendall """
        data = b''
        rem_bytes = buffer_len
        while rem_bytes > 0:
            buf = s.recv(rem_bytes)
            if not buf:
                raise Exception("Received EOF in middle of block")
            data += buf
            rem_bytes -= len(buf)
        return data

    conn, addr = s.accept()
    ...

    header_format = struct.Struct("!II")
    # Receive exactly the bytes of the header
    header = recv_exactly(conn, header_format.size)
    pathname_len, file_len = header_format.unpack(header)
    path = recv_exactly(conn, pathname_len)
    filename = path.split("/")
    ...
    rem_bytes = file_len
    while rem_bytes > 0:
        data = conn.recv(min(rem_bytes, BUFFER_SIZE))
        if not data:
            raise Exception("Received EOF in middle of file")
        f.write(data)
        rem_bytes -= len(data)

Another important advantage of this model is that you now have a clear notion of the boundary between one file and the next ( without having a "signal value" that might appear in the data). The receiver always knows exactly how many bytes remain until the end of the current file, and the sender can simply move on to send a new header, pathname, and file without opening a new connection.