difference detection in live camera feed

Question

I am facing a challenge where i am given a picture and i need to find the difference between an object in that picture and an object in front of me that i will see through an IP camera using python. this is the originally handed photo

this is what the feed looks like from the IP camera

now i should write a program to detect the differences between the two objects and color code every different part. its all easy if it included only those 2 pictures but the latter should be a live feed from the camera so the program i wrote gets overwhelmed with differences due to different camera positioning or different light exposure.

#!/usr/bin/env python
import cv2
from skimage import measure
import imutils

cap = cv2.VideoCapture(0)
img_gray = cv2.imread("pic1.png", 0)
img = cv2.imread("pic1.png")

while True:
    _, frame = cap.read()

    frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    (score, diff) = measure.compare_ssim(img_gray, frame_gray, full=True)#we won't use the score
    diff = (diff * 255).astype("uint8") # converting the float score from a (-1,1) range to 8 bit 0-255 range

    thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]

    contours = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    contours = imutils.grab_contours(contours)



    for cnt in contours:
        area = cv2.contourArea(cnt)
        if 3150 < area:
            (x,y,w,h) = cv2.boundingRect(cnt)
            cv2.rectangle(frame, (x,y), (x+w, y+h), (0,255,0))

    cv2.imshow("Current", frame)
    cv2.imshow("Back then", img)
    cv2.imshow("mask", thresh)


    if cv2.waitKey(1) & 0xFF == ord('q'):
        break



cap.release()
cv2.destroyAllWindows()

now the output on the colored video feed should be like this

the color coding is not a problem

i tried giving the drawn contours a set area to draw in and obviously it didn't work neither did the ssim score calculation. how should i approach this as it feels like i am trying to reinvent something already built but i couldn't find after 2 weeks of trying.

Answer 1

I write again the answer that I put on your other post . You can use SimpleElastix . I updated the code for those new images. Note that a part of the image is missing with the top pipe, so we actually don't know what's happening after the image border and the current algorithm only estimate the difference for the small part.

    import cv2
    import numpy as np
    import matplotlib.pyplot as plt
    import SimpleITK as sitk

    movingImage = sitk.ReadImage('b1.png', sitk.sitkFloat32)
    fixedImage = sitk.ReadImage('b2.png', sitk.sitkFloat32)

    elastixImageFilter = sitk.ElastixImageFilter()

    affine_registration_parameters = sitk.GetDefaultParameterMap('affine')
    affine_registration_parameters["NumberOfResolutions"] = ['6']
    affine_registration_parameters["WriteResultImage"] = ['false']
    affine_registration_parameters["MaximumNumberOfSamplingAttempts"] = ['4']

    spline_registration_parameters = sitk.GetDefaultParameterMap('bspline')
    spline_registration_parameters["NumberOfResolutions"] = ['1']
    spline_registration_parameters["WriteResultImage"] = ['false']
    spline_registration_parameters["MaximumNumberOfSamplingAttempts"] = ['4']
    spline_registration_parameters["Metric"] = ['AdvancedMattesMutualInformation']
    spline_registration_parameters["GridSpacingSchedule"] = ['3']

    print(spline_registration_parameters["Registration"])

    parameterMapVector = sitk.VectorOfParameterMap()
    parameterMapVector.append(affine_registration_parameters)
    parameterMapVector.append(spline_registration_parameters)

    elastixImageFilter.SetFixedImage(fixedImage)
    elastixImageFilter.SetMovingImage(movingImage)
    elastixImageFilter.SetParameterMap(parameterMapVector)
    elastixImageFilter.Execute()

    registeredImage = elastixImageFilter.GetResultImage()
    transformParameterMap = elastixImageFilter.GetTransformParameterMap()

    resultImage = sitk.Subtract(registeredImage, fixedImage)
    resultImageNp = (np.sqrt(sitk.GetArrayFromImage(resultImage) ** 2)  > 60)*255

    cv2.imwrite('b_1.png', sitk.GetArrayFromImage(fixedImage))
    cv2.imwrite('b_2.png', sitk.GetArrayFromImage(movingImage))
    cv2.imwrite('b_2r.png', sitk.GetArrayFromImage(registeredImage))
    cv2.imwrite('b_diff.png', resultImageNp.astype(np.uint8))

Reference image:

Registered second images:

Thresholded difference:

Answer 2

sorry for the later answer but one of my team members used @fmw42 's comment and it worked.

and sorry for having no details as the script was lost