Display and EDIT opencv image using Tkinter

Question

I have a simple GUI app to display image from chosen camera and transformed image. To do transformation I need to choose 4 points on image, so I would like to pick them by clicking. But in order to do that the picture must be an opencv image. So my idea was to "freeze" the image from camera and then edit it (for example by drawing on it 4 circles by function set_default_points()).

Here is my code

(The key function for now are show_frame(), freeze_camera() and set_default_points())

from tkinter import *
from tkinter import ttk
import cv2
import numpy as np
from PIL import Image, ImageTk

# ----- SET -----
GREY = "#D8D8D8"
BLUE = "#81F7F3"
AQUA = "#9CC3D5"
ICE = "#C7D3D4"

canvas_width = 500
canvas_height = 600


def camera_amount():
    '''Returns int value of available camera devices connected to the host device
    from url: https://www.codegrepper.com/code-examples/python/how+to+count+how+many+cameras+you+have+with+python
    '''
    camera = 0
    while True:
        if (cv2.VideoCapture(camera).grab()) is True:
            camera = camera + 1
        else:
            cv2.destroyAllWindows()
            return camera



# ----- default start images -----
img1 = cv2.imread("d1.png")
img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
resized1 = cv2.resize(img1, (canvas_width, canvas_height))

img2 = cv2.imread("ipm.png")
img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
resized2 = cv2.resize(img2, (canvas_width, canvas_height))


# ----- Application ------
class App(Tk):
    def __init__(self):
        super().__init__()

        self.title("Image TOP-DOWN Tranformation")
        self.minsize(width=1200, height=700)
        self.config(padx=5, pady=5, bg=ICE)

        im1 = Image.fromarray(resized1)
        im2 = Image.fromarray(resized2)
        self.org_img = ImageTk.PhotoImage(im1)
        self.transf_img = ImageTk.PhotoImage(im2)

        self.canvas1 = Canvas(width=500, height=600, bg=ICE, highlightthickness=2)
        self.org_image_container = self.canvas1.create_image(250, 300, anchor="center", image=self.org_img)
        self.canvas1.grid(padx=10, pady=10, row=1, rowspan=8, column=0)

        self.canvas2 = Canvas(width=500, height=600, bg=ICE, highlightthickness=2)
        self.trf_image_container = self.canvas2.create_image(250, 300, anchor="center", image=self.transf_img)
        self.canvas2.grid(padx=10, pady=10, row=1, rowspan=8, column=1)

        # ----- Labels -----
        self.label1 = Label(text="Original image")
        self.label1.grid(row=0, column=0)

        self.label2 = Label(text="Transformed image")
        self.label2.grid(row=0, column=1)

        self.label3 = Label(text="Functionalities")
        self.label3.grid(row=0, column=2)

        self.move_label = Label(text="Move points")
        self.move_label.grid(row=8, column=2, columnspan=3)

        # ----- Buttons -----

        self.selected_camera = StringVar()
        self.camera_cb = ttk.Combobox(self, textvariable=self.selected_camera)
        self.camera_cb.grid(padx=5, pady=5, row=1, column=2, columnspan=3)
        amount_of_cameras = camera_amount()
        self.camera_cb['values'] = [i for i in range(amount_of_cameras)]
        self.camera_cb.current(0)

        self.change_camera = Button(text="Change Camera", command=self.choose_camera)
        self.change_camera.grid(padx=5, pady=5, row=2, column=2, columnspan=3)

        self.freeze_cam = Button(text="Freeze camera", command=self.freeze_camera)
        self.freeze_cam.grid(padx=5, pady=5, row=3, column=2, columnspan=3)

        self.default_points = Button(text="Set default points", command=self.set_default_points)
        self.default_points.grid(padx=5, pady=5, row=4, column=2, columnspan=3)

        self.transform_button = Button(text="Transform", command=self.transform)
        self.transform_button.grid(padx=5, pady=5, row=5, column=2, columnspan=3)

        self.clear_button = Button(text="CLEAR", command=self.clear)
        self.clear_button.grid(padx=5, pady=5, row=6, column=2, columnspan=3)

        self.save_button = Button(text="SAVE", command=self.save)
        self.save_button.grid(padx=5, pady=5, row=7, column=2, columnspan=3)

        self.up_butt = Button(text="↑", command=self.up)
        self.up_butt.grid(padx=5, pady=5, row=9, column=3)

        self.down_butt = Button(text="↓", command=self.down)
        self.down_butt.grid(padx=5, pady=5, row=10, column=3)

        self.left_butt = Button(text="←", command=self.left)
        self.left_butt.grid(padx=5, pady=5, row=10, column=2, columnspan=2)

        self.right_butt = Button(text="→", command=self.right)
        self.right_butt.grid(padx=5, pady=5, row=10, column=4)

        # ----- fields -----
        self.cap = cv2.VideoCapture(int(self.selected_camera.get()))
        self.imgtk = None
        self.after_id = None
        self.resized11 = None

        self.input_points = []
        self.output_points = []

    def choose_camera(self):
        self.freeze_camera()
        self.cap = cv2.VideoCapture(int(self.selected_camera.get()))
        self.show_frame()

    # po upakowaniu w klasę po prostu zmienić jedno pole, a w funkcji odpowiedzialnej za wyświetlanie kamery dac ifa
    def freeze_camera(self):
        self.canvas1.after_cancel(self.after_id)
        # now, the image on canvas is freezed,
        # it means, that the picture is last imgtk from show_frame function,
        # but it's not like cv2 image, and it cannot be editable
        # I tried example like this, but it doesn't work
        # pil_image = PIL.Image.open('Image.jpg').convert('RGB')
        # open_cv_image = numpy.array(pil_image)
        # # Convert RGB to BGR
        # open_cv_image = open_cv_image[:, :, ::-1].copy()

    def set_default_points(self):
        default_points = [[448, 609], [580,609], [580,741], [448,741]]
        for pts in default_points:
            cv2.circle(self.imgtk, pts, 5, (0, 0, 255), -2)
        pass

    def transform(self):
        pass

    def clear(self):
        pass

    def save(self):
        pass

    def up(self):
        pass

    def down(self):
        pass

    def left(self):
        pass

    def right(self):
        pass

    def draw_circle(self, event, x, y, flags, param):
        if event == cv2.EVENT_LBUTTONDBLCLK:
            cv2.circle(self.imgtk, (x, y), 5, (255, 0, 0), -2)
            self.input_points.append([x, y])
        if event == cv2.EVENT_RBUTTONDBLCLK:
            cv2.circle(self.imgtk, (x, y), 5, (0, 0, 255), -2)
            self.output_points.append([x, y])

    def show_frame(self):
        """
        https://www.tutorialspoint.com/how-to-show-webcam-in-tkinter-window
        """
        img11 = self.cap.read()[1]
        cv2image = cv2.cvtColor(img11, cv2.COLOR_BGR2RGB)
        self.resized11 = cv2.resize(cv2image, (canvas_width, canvas_height))
        im11 = Image.fromarray(self.resized11)
        self.imgtk = ImageTk.PhotoImage(im11)
        self.canvas1.imgtk = self.imgtk
        self.canvas1.itemconfig(self.org_image_container, image=self.imgtk)
        self.after_id = self.canvas1.after(10, self.show_frame)




if __name__ == "__main__":
    app = App()
    app.show_frame()
    app.mainloop()

The problem is when I try to edit that frozen picture, by using function set_default_points I occure this error:

Exception in Tkinter callback
Traceback (most recent call last):
  File "D:\Python_versions\lib\tkinter\__init__.py", line 1921, in __call__
    return self.func(*args)
  File "D:\DUCKIETOWN\aplikacja_do_transformacji_obrazu\APP.py", line 146, in set_default_points
    cv2.circle(self.imgtk, pts, 5, (0, 0, 255), -2)
cv2.error: OpenCV(4.5.5) :-1: error: (-5:Bad argument) in function 'circle'
> Overload resolution failed:
>  - img is not a numpy array, neither a scalar
>  - Expected Ptr<cv::UMat> for argument 'img'

EDIT: So it seems, like only way to achieve my goal is to do operations like choose transformation points on detached opencv window with opencv image, and then upload it to my tkinter display window.

Here is corrected code:

import tkinter.messagebox
from tkinter import *
from tkinter import ttk
import cv2
import numpy as np
from PIL import Image, ImageTk
import transform


# ----- SET -----
GREY = "#D8D8D8"
BLUE = "#81F7F3"
AQUA = "#9CC3D5"
ICE = "#C7D3D4"

canvas_width = 500
canvas_height = 600


def camera_amount() -> int:
    '''Returns int value of available camera devices connected to the host device
    from url: https://www.codegrepper.com/code-examples/python/how+to+count+how+many+cameras+you+have+with+python
    '''
    camera = 0
    while True:
        if (cv2.VideoCapture(camera).grab()) is True:
            camera = camera + 1
        else:
            cv2.destroyAllWindows()
            return camera



# ----- default start images -----
img1 = cv2.imread("d1.png")
img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
resized1 = cv2.resize(img1, (canvas_width, canvas_height))

img2 = cv2.imread("ipm.png")
img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
resized2 = cv2.resize(img2, (canvas_width, canvas_height))


# ----- Application ------
class App(Tk):
    def __init__(self):
        super().__init__()

        self.title("Image TOP-DOWN Tranformation")
        self.minsize(width=1200, height=700)
        self.config(padx=5, pady=5, bg=ICE)

        im1 = Image.fromarray(resized1)
        im2 = Image.fromarray(resized2)
        self.org_img = ImageTk.PhotoImage(im1)
        self.transf_img = ImageTk.PhotoImage(im2)

        self.canvas1 = Canvas(width=500, height=600, bg=ICE, highlightthickness=2)
        self.org_image_container = self.canvas1.create_image(250, 300, anchor="center", image=self.org_img)
        self.canvas1.grid(padx=10, pady=10, row=1, rowspan=8, column=0)

        self.canvas2 = Canvas(width=500, height=600, bg=ICE, highlightthickness=2)
        self.trf_image_container = self.canvas2.create_image(250, 300, anchor="center", image=self.transf_img)
        self.canvas2.grid(padx=10, pady=10, row=1, rowspan=8, column=1)

        # ----- Labels -----
        self.label1 = Label(text="Original image")
        self.label1.grid(row=0, column=0)

        self.label2 = Label(text="Transformed image")
        self.label2.grid(row=0, column=1)

        self.label3 = Label(text="Functionalities")
        self.label3.grid(row=0, column=2)

        self.move_label = Label(text="Move points")
        self.move_label.grid(row=8, column=2, columnspan=3)

        # ----- Buttons -----

        self.selected_camera = StringVar()
        self.camera_cb = ttk.Combobox(self, textvariable=self.selected_camera)
        self.camera_cb.grid(padx=5, pady=5, row=1, column=2, columnspan=3)
        amount_of_cameras = camera_amount()
        self.camera_cb['values'] = [i for i in range(amount_of_cameras)]
        self.camera_cb.current(0)

        self.change_camera = Button(text="Change Camera", command=self.choose_camera)
        self.change_camera.grid(padx=5, pady=5, row=2, column=2, columnspan=3)

        self.freeze_cam = Button(text="Freeze camera", command=self.freeze_camera)
        self.freeze_cam.grid(padx=5, pady=5, row=3, column=2, columnspan=3)

        self.default_points = Button(text="Set default points", command=self.set_default_points)
        self.default_points.grid(padx=5, pady=5, row=4, column=2, columnspan=3)

        self.transform_button = Button(text="Transform", command=self.transform)
        self.transform_button.grid(padx=5, pady=5, row=5, column=2, columnspan=3)

        self.clear_button = Button(text="CLEAR", command=self.clear)
        self.clear_button.grid(padx=5, pady=5, row=6, column=2, columnspan=3)

        self.save_button = Button(text="SAVE", command=self.save)
        self.save_button.grid(padx=5, pady=5, row=7, column=2, columnspan=3)

        self.up_butt = Button(text="↑", command=self.up)
        self.up_butt.grid(padx=5, pady=5, row=9, column=3)

        self.down_butt = Button(text="↓", command=self.down)
        self.down_butt.grid(padx=5, pady=5, row=10, column=3)

        self.left_butt = Button(text="←", command=self.left)
        self.left_butt.grid(padx=5, pady=5, row=10, column=2, columnspan=2)

        self.right_butt = Button(text="→", command=self.right)
        self.right_butt.grid(padx=5, pady=5, row=10, column=4)

        # ----- fields -----
        self.cap = cv2.VideoCapture(int(self.selected_camera.get()))
        self.imgtk = None
        self.after_id = None
        self.resized11 = None
        self.ipm_matrix = None

        self.input_points = []
        self.output_points = []

        self.ipm_matrixes = [] # list of saved ipm_matrixes which satisfied us

    def choose_camera(self) -> None:
        self.freeze_camera(display=False)
        self.cap = cv2.VideoCapture(int(self.selected_camera.get()))
        self.clear()

    def freeze_camera(self, display: bool = True) -> None:
        self.canvas1.after_cancel(self.after_id)
        if display:
            self.resized11 = cv2.cvtColor(self.resized11, cv2.COLOR_BGR2RGB)
            cv2.namedWindow("image")
            cv2.setMouseCallback("image", self.draw_circle)
            while True:
                cv2.imshow("image", self.resized11)
                if cv2.waitKey(1) & 0xFF == ord("q"):
                    break
            cv2.destroyAllWindows()

    def set_default_points(self) -> None:
        default_points = [[448, 609], [580,609], [580,741], [448,741]]
        for pts in default_points:
            cv2.circle(self.resized11, pts, 5, (0, 0, 255), -2)
        self.canvas1.itemconfig(self.org_image_container, image=self.resized11)

    def transform(self) -> None:
        if len(self.input_points) == 4 and len(self.output_points) == 4:
            ordered_pts = transform.order_points(np.array(self.input_points, dtype=np.float32))
            ordered_out_pts = transform.order_points(np.array(self.output_points, dtype=np.float32))

            self.ipm_matrix = cv2.getPerspectiveTransform(ordered_pts, ordered_out_pts)
            self.transf_img = cv2.warpPerspective(self.resized11, self.ipm_matrix, self.resized11.shape[:2][::-1])

            self.transf_img = cv2.cvtColor(self.transf_img, cv2.COLOR_BGR2RGB)
            self.transf_img = Image.fromarray(self.transf_img)
            warpedtk = ImageTk.PhotoImage(self.transf_img)
            self.canvas2.warpedtk = warpedtk
            self.canvas2.itemconfig(self.trf_image_container, image=warpedtk)
        else:
            tkinter.messagebox.showwarning("Warning", "Choose right points to transformation!")

    def clear(self) -> None:
        self.output_points = []
        self.input_points = []
        self.show_frame()

    def save(self) -> None:
        # add to self.ipm_matrixes tuple (camera number, ipm_matrix)
        self.ipm_matrixes.append((int(self.selected_camera.get()), self.ipm_matrix))
        answer = tkinter.messagebox.askyesno("Save", "Do you want to save your all ipm matrixes to file?")
        if answer:
            self.safe_to_file()
        pass

    def up(self) -> None:
        pass

    def down(self) -> None:
        pass

    def left(self) -> None:
        pass

    def right(self) -> None:
        pass

    def draw_circle(self, event, x, y, flags, param) -> None:
        if event == cv2.EVENT_LBUTTONDBLCLK:
            cv2.circle(self.resized11, (x, y), 5, (255, 0, 0), -2)
            self.input_points.append([x, y])
        if event == cv2.EVENT_RBUTTONDBLCLK:
            cv2.circle(self.resized11, (x, y), 5, (0, 0, 255), -2)
            self.output_points.append([x, y])

    def show_frame(self) -> None:
        """
        https://www.tutorialspoint.com/how-to-show-webcam-in-tkinter-window
        """
        img11 = self.cap.read()[1]
        cv2image = cv2.cvtColor(img11, cv2.COLOR_BGR2RGB)
        self.resized11 = cv2.resize(cv2image, (canvas_width, canvas_height))
        im11 = Image.fromarray(self.resized11)
        self.imgtk = ImageTk.PhotoImage(im11)
        self.canvas1.imgtk = self.imgtk
        self.canvas1.itemconfig(self.org_image_container, image=self.imgtk)
        self.after_id = self.canvas1.after(10, self.show_frame)

    def safe_to_file(self) -> None:
        with open("saved_conf.txt", "a") as file:
            res = ""
            for elem in self.ipm_matrixes:
                res += f"Camera {elem[0]}: {elem[1]}\n"
            res += "\n\n"
            file.write(res)





if __name__ == "__main__":
    app = App()
    app.show_frame()
    app.mainloop()

But if anybody had different idea, how could I do that in one, single tkinter window I would be grateful.

I would appreciate any help.

Michael

Answer 1

In show_frame() you read frame with opencv as Mat then you covert it into PIL image with self.imgtk = ImageTk.PhotoImage(im11). And then when you call set_default_points() to use cv2.circle(self.imgtk, ... ), you need Mat again.

EDIT: You need to do opencv operation with opencv format (Mat) and then when you want to display result, Tkinter needs PIL format.

opencv_image --> opencv_operation --> ImageTk.PhotoImage() --> Tkinter display