将Kinect for Windows v2的深度和颜色流另存为图像文件（例如png或jpg）

Question

我正在尝试编写一个将Kinect for Windows v2的深度和颜色流另存为图像文件（例如png或jpg）的应用程序。 因此，我使用了Kinect SDK v2示例（因为我以前没有使用C＃或Kinect API的经验）。 我修改了ColorBasics-WPF示例代码以实现我的目标。 这是仅将颜色流转换为png文件的代码（我修改的唯一部分是Reader_ColorFrameArrived函数）：

//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
//     Copyright (c) Microsoft Corporation.  All rights reserved.
// </copyright>
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.ColorBasics
{
    using System;
    using System.ComponentModel;
    using System.Diagnostics;
    using System.Globalization;
    using System.IO;
    using System.Windows;
    using System.Windows.Media;
    using System.Windows.Media.Imaging;
    using Microsoft.Kinect;
    using System.Collections.Generic;


    /// <summary>
    /// Interaction logic for MainWindow
    /// </summary>
    public partial class MainWindow : Window, INotifyPropertyChanged
    {
        static int count = 0;

        /// <summary>
        /// Active Kinect sensor
        /// </summary>
        private KinectSensor kinectSensor = null;

        /// <summary>
        /// Reader for color frames
        /// </summary>
        private ColorFrameReader colorFrameReader = null;

        /// <summary>
        /// Bitmap to display
        /// </summary>
        private WriteableBitmap colorBitmap = null;

        /// <summary>
        /// Current status text to display
        /// </summary>
        private string statusText = null;

        /// <summary>
        /// Initializes a new instance of the MainWindow class.
        /// </summary>
        public MainWindow()
        {
            // get the kinectSensor object
            this.kinectSensor = KinectSensor.GetDefault();

            // open the reader for the color frames
            this.colorFrameReader = this.kinectSensor.ColorFrameSource.OpenReader();

            // wire handler for frame arrival
            this.colorFrameReader.FrameArrived += this.Reader_ColorFrameArrived;

            // create the colorFrameDescription from the ColorFrameSource using Bgra format
            FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);

            // create the bitmap to display
            this.colorBitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height, 96.0, 96.0, PixelFormats.Bgr32, null);

            // set IsAvailableChanged event notifier
            this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;

            // open the sensor
            this.kinectSensor.Open();

            // set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.NoSensorStatusText;

            // use the window object as the view model in this simple example
            this.DataContext = this;

            // initialize the components (controls) of the window
            this.InitializeComponent();
        }

        /// <summary>
        /// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
        /// </summary>
        public event PropertyChangedEventHandler PropertyChanged;

        /// <summary>
        /// Gets the bitmap to display
        /// </summary>
        public ImageSource ImageSource
        {
            get
            {
                return this.colorBitmap;
            }
        }

        /// <summary>
        /// Gets or sets the current status text to display
        /// </summary>
        public string StatusText
        {
            get
            {
                return this.statusText;
            }

            set
            {
                if (this.statusText != value)
                {
                    this.statusText = value;

                    // notify any bound elements that the text has changed
                    if (this.PropertyChanged != null)
                    {
                        this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
                    }
                }
            }
        }

        /// <summary>
        /// Execute shutdown tasks
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void MainWindow_Closing(object sender, CancelEventArgs e)
        {
            if (this.colorFrameReader != null)
            {
                // ColorFrameReder is IDisposable
                this.colorFrameReader.Dispose();
                this.colorFrameReader = null;
            }

            if (this.kinectSensor != null)
            {
                this.kinectSensor.Close();
                this.kinectSensor = null;
            }
        }

        /// <summary>
        /// Handles the user clicking on the screenshot button
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
        {
            if (this.colorBitmap != null)
            {
                // create a png bitmap encoder which knows how to save a .png file
                BitmapEncoder encoder = new PngBitmapEncoder();

                // create frame from the writable bitmap and add to encoder
                encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
                //bitmaps.Add(BitmapFrame.Create(this.colorBitmap.Clone()));

                string time = count.ToString();
                //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                count++;
                // write the new file to disk
                try
                {
                    // FileStream is IDisposable
                    using (FileStream fs = new FileStream(path, FileMode.Create))
                    {
                        encoder.Save(fs);
                    }

                    this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                }
                catch (IOException)
                {
                    this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                }
            }
        }

        /// <summary>
        /// Handles the color frame data arriving from the sensor
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Reader_ColorFrameArrived(object sender, ColorFrameArrivedEventArgs e)
        {
            // ColorFrame is IDisposable
            using (ColorFrame colorFrame = e.FrameReference.AcquireFrame())
            {
                if (colorFrame != null)
                {
                    FrameDescription colorFrameDescription = colorFrame.FrameDescription;

                    using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
                    {
                        this.colorBitmap.Lock();

                        // verify data and write the new color frame data to the display bitmap
                        if ((colorFrameDescription.Width == this.colorBitmap.PixelWidth) && (colorFrameDescription.Height == this.colorBitmap.PixelHeight))
                        {
                            colorFrame.CopyConvertedFrameDataToIntPtr(
                                this.colorBitmap.BackBuffer,
                                (uint)(colorFrameDescription.Width * colorFrameDescription.Height * 4),
                                ColorImageFormat.Bgra);

                            this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
                        }

                        this.colorBitmap.Unlock();
                    }
                }

                // my modification : save current frame as png file. 
                if (this.colorBitmap != null)
                {
                    // create a png bitmap encoder which knows how to save a .png file
                    BitmapEncoder encoder = new PngBitmapEncoder();

                    // create frame from the writable bitmap and add to encoder
                    encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));

                    string time = count.ToString();
                    //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                    string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                    string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                    count++;
                    // write the new file to disk
                    try
                    {
                        // FileStream is IDisposable
                        using (FileStream fs = new FileStream(path, FileMode.Create))
                        {
                            encoder.Save(fs);
                        }

                        this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                    }
                    catch (IOException)
                    {
                        this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                    }
                }
            }
        }

        /// <summary>
        /// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
        {
            // on failure, set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.SensorNotAvailableStatusText;
        }
    }
}

此代码的问题在于，它在录制期间会生成200个png文件，但从第90帧开始到最后一帧，所有帧都是相同的（它在运行时即开始记录，每当我停止时便停止关闭它）。

1）您能帮助我理解为什么会这样吗？ 为什么它不记录其余帧并一次又一次重复一帧？

2）您对使用Kinect for Windows v2如何有效地同时以良好的帧频（例如20-30 fps）将深度和颜色流记录为图像文件有任何建议或指示吗？

Answer 1

在不丢失帧的情况下以相同的fps同时写入颜色和深度流有些棘手。 为了检查相同帧的问题，建议您尝试将彩色图像和深度图像写入单独的缓冲区（也可以将其帧时间戳保存在缓冲区中），并在记录结束后将其写入磁盘。 我举一个例子，您可以根据自己的问题进行调整。

将彩色框保存到缓冲区：

private void myKinectSensor_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
    {
        using (ColorImageFrame color = e.OpenColorImageFrame())
        {
            if (color != null)
            {
                colorbits = new byte[color.PixelDataLength];
                color.CopyPixelDataTo(colorbits);
                image1.Source = BitmapSource.Create(color.Width, color.Height, 96, 96, PixelFormats.Bgr32, null, colorbits, color.Width * color.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveColorTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveColorFrames.AddLast(colorbits);
                }
            }
        }
    }

将深度帧保存到另一个缓冲区：

private void myKinectSensor_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
    {
        using (DepthImageFrame depth = e.OpenDepthImageFrame())
        {
            depthPixels = new DepthImagePixel[myKinectSensor.DepthStream.FramePixelDataLength];
            if (depth != null)
            {
                frame = new short[depth.PixelDataLength];
                depth.CopyPixelDataTo(frame);
                for (int i = 0; i < frame.Length; i++)
                {
                    frame[i] = (short)(((ushort)frame[i]) >> 3);
                }
                image3.Source = BitmapSource.Create(depth.Width, depth.Height, 96, 96, PixelFormats.Gray16, null, frame, depth.Width * depth.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveDepthTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveDepthFrames.AddLast(frame);
                }
            }
        }
    }

最后，循环进入每个缓冲区并将帧写入磁盘：

            e = SaveColorTimestamps.GetEnumerator();
            foreach (byte[] node in SaveColorFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Bgr32, null, node, 640*4)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/color/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }
            SaveColorTimestamps.Clear();
            SaveColorFrames.Clear();

            e.Dispose();
            e = SaveDepthTimestamps.GetEnumerator();
            foreach (short[] node in SaveDepthFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Gray16, null, node, 640 * 2)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/depth/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }

这不是最佳方法，但是它将帮助您了解框架编写的工作原理，并且不会出现任何丢帧现象。