Running different tasks simultaneously in python?

Question

I've written a python program to continuously do the following on a Raspberry Pi 3B+:

• Check date/time.
• Check if a directory is available with the current date/time as name, if not make a directory.
• Read serial data from a sensor.
• Convert serial data to readable results.
• Check results for triggers.
• Check GPIO pins for an extra, external trigger.
• IF triggered: take a picture using a PiCamera module, save the picture with date/time as name in the directory, create/append results in a .txt file.

The program mostly does what I intended it to do, but when I check the results in the .txt file, some results are 'impossible' and indicate I'm not reading/converting my serial data properly. The sensor also outputs data at 100 telegrams/second, which is nowhere near the actual rate at which the Raspberry Pi is saving results (around 2 measurements/second).

I've tried writing a program that only reads the serial data, nothing else, and this is able to keep up with the sensor. I also tried replacing the checking of the header with a ser.read_until() (leaving the useless headers at the end of each telegram). However this resulted in telegrams of varying lengths, making the parsing into the 6 variables harder. Check this link to see what a telegram consists of according to specs. The code I use now (below) reads the telegram in pieces (2, 2, 1, 2, 2, 1 bytes) but sometimes returns values that seem misinterpreted.

#!/usr/bin/env python
import time, serial, os, sys, stat, os.path, subprocess                                 
import RPi.GPIO as GPIO
from picamera import PiCamera
from datetime import datetime

ser = serial.Serial(                                                                    
    port='/dev/serial0',
    baudrate = 19200,
    parity=serial.PARITY_NONE,
    stopbits=serial.STOPBITS_ONE,
    bytesize=serial.EIGHTBITS,
    timeout=0)

signaltrigger = 60
trigger = 24
GPIO.setmode(GPIO.BCM)                                                                  
GPIO.setup(trigger, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)                                

now = datetime.now()                                                                    
data_dir = now.strftime("%d-%m-%Y")
save_path = "/home/pi/serial/"+data_dir+"/"                                            
test = os.path.isdir(save_path)                                                         
if test == False:                                                                       
    os.mkdir(save_path)                                                                 
    subprocess.call(['chmod', '-R', '777', '/home/pi/serial'])                          
else:
    pass

cam = PiCamera()                                                                        
cam.rotation = 90                                                                       
cam.resolution = (480, 360)                                                             
cam.start_preview()                                                                     
signaltriggerA = 0
signaltriggerB = 0

while 1:
    now = datetime.now()                                                                    
    time_exact = now.strftime(str(now)[11:23])                                          
    date_time = now.strftime("%d-%m-%Y %H:%M")                                          
    data_dir = now.strftime("%d-%m-%Y")                                                 
    save_path = "/home/pi/serial/"+data_dir+"/"                                         
    completename = os.path.join(save_path, date_time+".txt")                            
    namepic = str(time_exact+".jpg")
    completenamepic = os.path.join(save_path, data_dir+" "+time_exact+".jpg")           
    test = os.path.isdir(save_path)                                                     
    if test == False:                                                                   
        os.mkdir(save_path)                                                             
        subprocess.call(['chmod', '-R', '777', '/home/pi/serial'])
        pass
    else:
        pass
    x = ser.read(3)                                                                     
    if x != b'~~~':                                                                     
        print("Header mismatched")
        x = ser.read(9)                                                                 
        d = 0
        j = 0
    elif x == b'~~~':
        print("Serial communication started...")
        y = ser.read(2)                                                                 
        a = ser.read(2)
        c = ser.read(1)
        e = ser.read(2)
        g = ser.read(2)
        i = ser.read(1)
        z = int.from_bytes(y, byteorder='little', signed=False)/100                     
        b = round(int.from_bytes(a, byteorder='little', signed=True)*0.0367, 2)
        d = int.from_bytes(c, byteorder='little', signed=False)                         
        f = int.from_bytes(e, byteorder='little', signed=False)/100                     
        h = round(int.from_bytes(g, byteorder='little', signed=True)*0.0367, 2)
        j = int.from_bytes(i, byteorder='little', signed=False)                         
    if d >= signaltrigger:
        signaltriggerA = 1
    else:
        signaltriggerA = 0
    if j >= signaltrigger:
        signaltriggerB = 1
    else:
        signaltriggerB = 0
    if signaltriggerA or signaltriggerB or GPIO.input(trigger):        
        cam.capture(completenamepic)
        M = open(completename,"a+",encoding='utf-8')                                    
        M.write('[%s]: Distance(a) = %s [m], Velocity(a) = %s [km/h], Signal(a) = %s [dB], Distance(r) = %s [m], Velocity(r) = %s [km/h], Signal(r) = %s [dB]. Trigger(a) = %s, Trigger(r) = %s, Trigger(ETZ) = %s. Picture: %s\n'%(time_exact,z,b,d,f,h,j,signaltriggerA,signaltriggerB,GPIO.input(trigger),namepic))
        M.close()
        pass
    else:
        print("No triggers detected")

I expect the program to parse each incoming telegram into 6 pieces and translate these pieces into results (integers or floats), but sometimes the values of these results turn out way too high (impossible for the sensor). I expect this to be caused by manually reading the different bytes (taking too much time and missing parts of the telegram as the sensor continuously spits out data).
How could I ensure the correct reading and converting of the serial data? Also if the sensor outputs data at 100 telegrams/second but the camera can't keep up with 100 pictures/second, is it possible to calculate a moving average instead of using just one of many telegrams? Edit: Added picture of telegram's specifications to help understand the serial data + added complete python program.

Answer 1

I would use read_until() to read up to the three header bytes and some saner and more readable way to turn the data bytes into values.

#!/usr/bin/env python3
from ctypes import LittleEndianStructure, c_int16, c_uint16, c_uint8, sizeof
from datetime import datetime
import os


class SensorData(LittleEndianStructure):
    _pack_ = 1
    _fields_ = [
        ('_distance', c_uint16),
        ('_velocity', c_int16),
        ('signal_strength', c_uint8),
    ]

    @property
    def distance(self):
        return self._distance / 100

    @property
    def velocity(self):
        return self._velocity * 0.0367

    def to_string(self, name):
        return (
            f'Distance({name}) = {self.distance} [m],'
            f' Velocity({name}) = {self.velocity} [km/h],'
            f' Signal({name}) = {self.signal_strength} [dB]'
        )


class Message(LittleEndianStructure):
    _pack_ = 1
    _fields_ = [
        ('sensor_a', SensorData),
        ('sensor_b', SensorData),
    ]


def main():
    # ...

    while True:
        now = datetime.now()
        save_path = f'/home/pi/serial/{now:%d-%m-%Y}'
        log_path = os.path.join(save_path, f'{now:%d-%m-%Y %H:%M}.txt')
        picture_filename = f'{now:%H:%M:%S.%f}.jpg'
        picture_path = os.path.join(save_path, picture_filename)
        os.makedirs(save_path, exist_ok=True)

        ser.read_until(b'~~~')
        message = Message.from_buffer_copy(ser.read(sizeof(Message)))
        signal_trigger_a = message.sensor_a.signal_strength >= signal_trigger
        signal_trigger_b = message.sensor_b.signal_strength >= signal_trigger
        gpio_trigger = GPIO.input(trigger)
        if signal_trigger_a or signal_trigger_b or gpio_trigger:
            cam.capture(picture_path)
            with open(log_path, 'a', encoding='utf-8') as log_file:
                log_file.write(
                    f'[{now}]:'
                    f' {message.sensor_a.to_string("a")},'
                    f' {message.sensor_a.to_string("r")}.'
                    f' Trigger(a) = {signal_trigger_a},'
                    f' Trigger(r) = {signal_trigger_b},'
                    f' Trigger(ETZ) = {gpio_trigger}.'
                    f' Picture: {picture_filename}\n'
                )
        else:
            print('No triggers detected')


if __name__ == '__main__':
    main()