Save depth and color streams of Kinect for Windows v2 as image files(e.g. png or jpg)

Question

I am trying to write an application that saves depth and color streams of Kinect for Windows v2 as image files (like png or jpg). So, I used Kinect SDK v2 examples(since I have no prior experience with C# or Kinect API). I modified the ColorBasics-WPF sample code to achieve my goal. Here is the code that only convert color stream to png files(the only part I modified is Reader_ColorFrameArrived function):

//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
//     Copyright (c) Microsoft Corporation.  All rights reserved.
// </copyright>
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.ColorBasics
{
    using System;
    using System.ComponentModel;
    using System.Diagnostics;
    using System.Globalization;
    using System.IO;
    using System.Windows;
    using System.Windows.Media;
    using System.Windows.Media.Imaging;
    using Microsoft.Kinect;
    using System.Collections.Generic;


    /// <summary>
    /// Interaction logic for MainWindow
    /// </summary>
    public partial class MainWindow : Window, INotifyPropertyChanged
    {
        static int count = 0;

        /// <summary>
        /// Active Kinect sensor
        /// </summary>
        private KinectSensor kinectSensor = null;

        /// <summary>
        /// Reader for color frames
        /// </summary>
        private ColorFrameReader colorFrameReader = null;

        /// <summary>
        /// Bitmap to display
        /// </summary>
        private WriteableBitmap colorBitmap = null;

        /// <summary>
        /// Current status text to display
        /// </summary>
        private string statusText = null;

        /// <summary>
        /// Initializes a new instance of the MainWindow class.
        /// </summary>
        public MainWindow()
        {
            // get the kinectSensor object
            this.kinectSensor = KinectSensor.GetDefault();

            // open the reader for the color frames
            this.colorFrameReader = this.kinectSensor.ColorFrameSource.OpenReader();

            // wire handler for frame arrival
            this.colorFrameReader.FrameArrived += this.Reader_ColorFrameArrived;

            // create the colorFrameDescription from the ColorFrameSource using Bgra format
            FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);

            // create the bitmap to display
            this.colorBitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height, 96.0, 96.0, PixelFormats.Bgr32, null);

            // set IsAvailableChanged event notifier
            this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;

            // open the sensor
            this.kinectSensor.Open();

            // set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.NoSensorStatusText;

            // use the window object as the view model in this simple example
            this.DataContext = this;

            // initialize the components (controls) of the window
            this.InitializeComponent();
        }

        /// <summary>
        /// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
        /// </summary>
        public event PropertyChangedEventHandler PropertyChanged;

        /// <summary>
        /// Gets the bitmap to display
        /// </summary>
        public ImageSource ImageSource
        {
            get
            {
                return this.colorBitmap;
            }
        }

        /// <summary>
        /// Gets or sets the current status text to display
        /// </summary>
        public string StatusText
        {
            get
            {
                return this.statusText;
            }

            set
            {
                if (this.statusText != value)
                {
                    this.statusText = value;

                    // notify any bound elements that the text has changed
                    if (this.PropertyChanged != null)
                    {
                        this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
                    }
                }
            }
        }

        /// <summary>
        /// Execute shutdown tasks
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void MainWindow_Closing(object sender, CancelEventArgs e)
        {
            if (this.colorFrameReader != null)
            {
                // ColorFrameReder is IDisposable
                this.colorFrameReader.Dispose();
                this.colorFrameReader = null;
            }

            if (this.kinectSensor != null)
            {
                this.kinectSensor.Close();
                this.kinectSensor = null;
            }
        }

        /// <summary>
        /// Handles the user clicking on the screenshot button
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
        {
            if (this.colorBitmap != null)
            {
                // create a png bitmap encoder which knows how to save a .png file
                BitmapEncoder encoder = new PngBitmapEncoder();

                // create frame from the writable bitmap and add to encoder
                encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
                //bitmaps.Add(BitmapFrame.Create(this.colorBitmap.Clone()));

                string time = count.ToString();
                //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                count++;
                // write the new file to disk
                try
                {
                    // FileStream is IDisposable
                    using (FileStream fs = new FileStream(path, FileMode.Create))
                    {
                        encoder.Save(fs);
                    }

                    this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                }
                catch (IOException)
                {
                    this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                }
            }
        }

        /// <summary>
        /// Handles the color frame data arriving from the sensor
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Reader_ColorFrameArrived(object sender, ColorFrameArrivedEventArgs e)
        {
            // ColorFrame is IDisposable
            using (ColorFrame colorFrame = e.FrameReference.AcquireFrame())
            {
                if (colorFrame != null)
                {
                    FrameDescription colorFrameDescription = colorFrame.FrameDescription;

                    using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
                    {
                        this.colorBitmap.Lock();

                        // verify data and write the new color frame data to the display bitmap
                        if ((colorFrameDescription.Width == this.colorBitmap.PixelWidth) && (colorFrameDescription.Height == this.colorBitmap.PixelHeight))
                        {
                            colorFrame.CopyConvertedFrameDataToIntPtr(
                                this.colorBitmap.BackBuffer,
                                (uint)(colorFrameDescription.Width * colorFrameDescription.Height * 4),
                                ColorImageFormat.Bgra);

                            this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
                        }

                        this.colorBitmap.Unlock();
                    }
                }

                // my modification : save current frame as png file. 
                if (this.colorBitmap != null)
                {
                    // create a png bitmap encoder which knows how to save a .png file
                    BitmapEncoder encoder = new PngBitmapEncoder();

                    // create frame from the writable bitmap and add to encoder
                    encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));

                    string time = count.ToString();
                    //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                    string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                    string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                    count++;
                    // write the new file to disk
                    try
                    {
                        // FileStream is IDisposable
                        using (FileStream fs = new FileStream(path, FileMode.Create))
                        {
                            encoder.Save(fs);
                        }

                        this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                    }
                    catch (IOException)
                    {
                        this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                    }
                }
            }
        }

        /// <summary>
        /// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
        {
            // on failure, set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.SensorNotAvailableStatusText;
        }
    }
}

The problem with this code is that it generates say 200 png files during the recording time but starting from say frame 90 to the end, all the frames are identical to each other(It starts recording as soon as I run it and it stops whenever I close it).

1) Would you help me understand why this happens? Why it doesn't record the rest of the frames and repeats a frame again and again?

2) Do you have any advice or pointer about how I can record depth and color streams as image files simultaneously in an efficient way and with a good frame rate(say 20-30 fps) using Kinect for Windows v2?

Answer 1

Writing simultaneously color and depth streams with the same fps without loosing frames is a little tricky. In order to check your problem with the identical frames I suggest you try to write color and depth images to separate buffers (you can keep its frame timestamp in a buffer too) and write them to the disk after the recording is over. I'll give you an example and you can ajust it to your problem.

Saving color frames to a buffer:

private void myKinectSensor_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
    {
        using (ColorImageFrame color = e.OpenColorImageFrame())
        {
            if (color != null)
            {
                colorbits = new byte[color.PixelDataLength];
                color.CopyPixelDataTo(colorbits);
                image1.Source = BitmapSource.Create(color.Width, color.Height, 96, 96, PixelFormats.Bgr32, null, colorbits, color.Width * color.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveColorTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveColorFrames.AddLast(colorbits);
                }
            }
        }
    }

Saving depth frames to another buffer:

private void myKinectSensor_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
    {
        using (DepthImageFrame depth = e.OpenDepthImageFrame())
        {
            depthPixels = new DepthImagePixel[myKinectSensor.DepthStream.FramePixelDataLength];
            if (depth != null)
            {
                frame = new short[depth.PixelDataLength];
                depth.CopyPixelDataTo(frame);
                for (int i = 0; i < frame.Length; i++)
                {
                    frame[i] = (short)(((ushort)frame[i]) >> 3);
                }
                image3.Source = BitmapSource.Create(depth.Width, depth.Height, 96, 96, PixelFormats.Gray16, null, frame, depth.Width * depth.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveDepthTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveDepthFrames.AddLast(frame);
                }
            }
        }
    }

Finally, loop in each buffer and write frames to disk:

            e = SaveColorTimestamps.GetEnumerator();
            foreach (byte[] node in SaveColorFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Bgr32, null, node, 640*4)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/color/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }
            SaveColorTimestamps.Clear();
            SaveColorFrames.Clear();

            e.Dispose();
            e = SaveDepthTimestamps.GetEnumerator();
            foreach (short[] node in SaveDepthFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Gray16, null, node, 640 * 2)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/depth/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }

It is not the optimal way to do it, but it will help you understand how the frame-writing works and you will not have any drop-frames.