Bulk Insert multiple documents into SQL server 2016

Question

I am trying to bulk insert multiple documents(more than 50,000) from the folder. I have this program works but it is time consuming. Please suggest any alternative ways.

string[] files = Directory.GetFiles("C:\\TEST\\", "*.*", SearchOption.AllDirectories);
using (SqlConnection con= new SqlConnection (conString))
{
    con.Open();
     foreach (string docPath in files)
            {
                   byte[] file;    
                    using (var stream = new FileStream(docPath, FileMode.Open, FileAccess.Read))
                    {
                        using (var reader1 = new BinaryReader(stream))
                        {
                            file = reader1.ReadBytes((int)stream.Length);
                        }
                    }

        string insertSQLString = "INSERT INTO dbo.Test(content, path) VALUES(@File, @path)";
        using (SqlCommand cmd = new SqlCommand (insertSQLString, con))
        {
            cmd.Parameters.Add("@File", SqlDbType.VarBinary, file.Length).Value = file;                             
            cmd.Parameters.Add("@path", SqlDbType.Text).Value = docPath;
            cmd.ExecuteNonQuery();
        }
    }
}

Answer 1

You can use the SqlBulkCopy class from System.Data.SqlClient for that. It is very efficient when inserting large bodies of data.

Here you can find an example how that can be done: http://codinghelmet.com/?path=howto/bulk-insert

Answer 2

Here are a couple of optimizations. There's no need to copy the file contents to a byte array before sending to SQL. You can pass a Stream directly. See SqlClient Streaming Support . And if the files are small, you'll want to batch a few together in a commit. Also it's unnecessary to recreate the SqlCommand each time.

        static void Run()
        { 
            string[] files = Directory.GetFiles("C:\\TEST\\", "*.*", SearchOption.AllDirectories);
            using (SqlConnection con = new SqlConnection("server=localhost;database=testdb;integrated security=true"))
            {
                con.Open();
                string insertSQLString = "INSERT INTO dbo.Test(content, path) VALUES(@File, @path)";
                SqlCommand cmd = new SqlCommand(insertSQLString, con);
                var pFile = cmd.Parameters.Add("@File", SqlDbType.VarBinary, -1);
                var pPath = cmd.Parameters.Add("@path", SqlDbType.Text);

                var tran = con.BeginTransaction();
                var fn = 0;
                foreach (string docPath in files)
                {
                    fn += 1;
                    using (var stream = new FileStream(docPath, FileMode.Open, FileAccess.Read))
                    {
                        pFile.Value = stream;
                        pPath.Value = docPath;
                        cmd.Transaction = tran;
                        cmd.ExecuteNonQuery();
                        if (fn%10==0)
                        {
                            tran.Commit();
                            tran = con.BeginTransaction();
                            Console.Write("|");
                        }
                        Console.Write(".");
                    }
                }
                tran.Commit();
            }
        }

Another approach is to create a File Table and then just robocopy the files.