Unity3D & Realsense 2.0 D415 - Depth to Terrain

Question

I am trying to generate a runtime terrain from the depth values of the RealSense camera D415. I got the camera to try and do something for the summer. I never used more than webcams in Unity before so I'm very lost even after going through all the sample scenes.

My terrain is 500 x 500 x 40 using RealSense SDK 2.0 the terrain generator code is below which works with Perlin noise

using UnityEngine;
using Intel.RealSense;

/// <summary>
/// this script is made to test the new terrain in unity with intel realsense depth
/// </summary>
public class TerrainGenerator : MonoBehaviour {

    public int depth = 40;
    public int width = 500;
    public int height = 500;

    public float scale = 20f;

    public float xOffset = 10;
    public float yOffset = 10;
    private DepthFrame depthFrame;
    public bool scroll;
    public float scrollSpeed;

    private Pipeline pipe;
    private void Start()
    {
        ///
        pipe = new Pipeline();
        pipe.Start();


        ///


        xOffset = Random.Range(0f, 9999f);
        yOffset = Random.Range(0f, 9999f);

        scroll = false;
        scrollSpeed = 0;
    }
    void Update()
    {
        Terrain terrain = GetComponent<Terrain>();
        terrain.terrainData = GenerateTerrain(terrain.terrainData);

        if(scroll)
            xOffset += Time.deltaTime * scrollSpeed;

    }
    TerrainData GenerateTerrain(TerrainData tData)
    {
        tData.heightmapResolution = width + 1;
        tData.size = new Vector3(width, depth, height);
        tData.SetHeights(0, 0, GenerateHeights());
        return tData;
    }
    float[,] GenerateHeights()
    {
        float[,] heights = new float[width, height];
        for (int x = 0; x < width; x++)
        {
            for (int y = 0; y < height; y++)
            {

                //heights[x, y] = CalculateHeight(x, y);
                var frames = pipe.WaitForFrames();
                var depth = frames.DepthFrame;
                heights[x, y] = depth.GetDistance(x, y);

            }
        }
        return heights;
    }

    float CalculateHeight(int x, int y)
    {
        //float xCoord = (float)x / width * scale + xOffset;
        //float yCoord = (float)y / height * scale + yOffset;
        //return depthFrame.GetDistance(x,y);
        //return Mathf.PerlinNoise(xCoord, yCoord);

        using (var frames = pipe.WaitForFrames())
        using (var depth = frames.DepthFrame)
            return depth.GetDistance(x, y);
    }

}

I am looking for some guidance on how to init the camera depth properly. I have never used a camera before with Unity.

Answer 1

Bit off-topic but I wanted to ask how well your above code runs? Are you getting errors such as:

ExternalException: rs2_pipeline_wait_for_frames(pipe:000000006402E3C0) Rethrow as Exception: Frame didn't arrived within 1000

I'm trying to do something similar and also having problems. Maybe we can work it out together?

Cheers, Cam