C# Reading CSV to DataTable and Invoke Rows/Columns

Question

i am currently working on a small Project and i got stuck with a Problem i currently can not manage to solve...

I have multiple ".CSV" Files i want to read, they all have the same Data just with different Values.

Header1;Value1;Info1
Header2;Value2;Info2
Header3;Value3;Info3

While reading the first File i Need to Create the Headers. The Problem is they are not splited in Columns but in rows (as you can see above Header1-Header3).

Then it Needs to read the Value 1 - Value 3 (they are listed in the 2nd Column) and on top of that i Need to create another Header -> Header4 with the data of "Info2" which is always placed in Column 3 and Row 2 (the other values of Column 3 i can ignore).

So the Outcome after the first File should look like this:

Header1;Header2;Header3;Header4;
Value1;Value2;Value3;Info2;

And after multiple files it sohuld be like this:

Header1;Header2;Header3;Header4;
Value1;Value2;Value3;Value4;
Value1b;Value2b;Value3b;Value4b;
Value1c;Value2c;Value3c;Value4c;

I tried it with OleDB but i get the Error "missing ISAM" which i cant mange to fix. The Code i Used is the following:

public DataTable ReadCsv(string fileName)
    {
        DataTable dt = new DataTable("Data");
       /* using (OleDbConnection cn = new OleDbConnection("Provider=Microsoft.Jet.OLEDB.4.0;Data Source=\"" + 
            Path.GetDirectoryName(fileName) + "\";Extendet Properties ='text;HDR=yes;FMT=Delimited(,)';"))
        */
        using (OleDbConnection cn = new OleDbConnection("Provider=Microsoft.Jet.OLEDB.4.0;Data Source=" +
            Path.GetDirectoryName(fileName) + ";Extendet Properties ='text;HDR=yes;FMT=Delimited(,)';"))
        {
            using(OleDbCommand cmd = new OleDbCommand(string.Format("select *from [{0}]", new FileInfo(fileName).Name,cn)))
            {
                cn.Open();
                using(OleDbDataAdapter adapter = new OleDbDataAdapter(cmd))
                {
                    adapter.Fill(dt);
                }
            }
        }


        return dt;
    }

Another attempt i did was using StreamReader. But the Headers are in the wrong place and i dont know how to Change this + do this for every file. the Code i tried is the following:

  public static DataTable ReadCsvFilee(string path)
    {  

        DataTable oDataTable = new DataTable();
        var fileNames = Directory.GetFiles(path);
        foreach (var fileName in fileNames)

        {

            //initialising a StreamReader type variable and will pass the file location
            StreamReader oStreamReader = new StreamReader(fileName);

            // CONTROLS WHETHER WE SKIP A ROW OR NOT
            int RowCount = 0;
            // CONTROLS WHETHER WE CREATE COLUMNS OR NOT
            bool hasColumns = false;
            string[] ColumnNames = null;
            string[] oStreamDataValues = null;
            //using while loop read the stream data till end
            while (!oStreamReader.EndOfStream)
            { 

                String oStreamRowData = oStreamReader.ReadLine().Trim();
                if (oStreamRowData.Length > 0)
                { 

                    oStreamDataValues = oStreamRowData.Split(';');
                    //Bcoz the first row contains column names, we will poluate 
                    //the column name by
                    //reading the first row and RowCount-0 will be true only once
                    // CHANGE TO CHECK FOR COLUMNS CREATED                      
                    if (!hasColumns)
                    {
                        ColumnNames = oStreamRowData.Split(';');

                        //using foreach looping through all the column names
                        foreach (string csvcolumn in ColumnNames)
                        {
                            DataColumn oDataColumn = new DataColumn(csvcolumn.ToUpper(), typeof(string));

                            //setting the default value of empty.string to newly created column
                            oDataColumn.DefaultValue = string.Empty;

                            //adding the newly created column to the table
                            oDataTable.Columns.Add(oDataColumn);
                        }
                        // SET COLUMNS CREATED
                        hasColumns = true;
                        // SET RowCount TO 0 SO WE KNOW TO SKIP COLUMNS LINE
                        RowCount = 0;
                    }
                    else
                    {
                        // IF RowCount IS 0 THEN SKIP COLUMN LINE
                        if (RowCount++ == 0) continue;
                        //creates a new DataRow with the same schema as of the oDataTable            
                        DataRow oDataRow = oDataTable.NewRow();

                        //using foreach looping through all the column names
                        for (int i = 0; i < ColumnNames.Length; i++)
                        {
                            oDataRow[ColumnNames[i]] = oStreamDataValues[i] == null ? string.Empty : oStreamDataValues[i].ToString();
                        }

                        //adding the newly created row with data to the oDataTable       
                        oDataTable.Rows.Add(oDataRow);
                    }

                }
            }
            //close the oStreamReader object
            oStreamReader.Close();
            //release all the resources used by the oStreamReader object
            oStreamReader.Dispose();
        }
            return oDataTable;
        }

I am thankful for everyone who is willing to help. And Thanks for reading this far!

Sincerely yours

Answer 1

If I understood you right, there is a strict parsing there like this:

string OpenAndParse(string filename, bool firstFile=false)
{
    var lines = File.ReadAllLines(filename);

    var parsed = lines.Select(l => l.Split(';')).ToArray();

    var header = $"{parsed[0][0]};{parsed[1][0]};{parsed[2][0]};{parsed[1][0]}\n";
    var data   = $"{parsed[0][1]};{parsed[1][1]};{parsed[2][1]};{parsed[1][2]}\n";

    return firstFile
    ? $"{header}{data}"
    : $"{data}";
}

Where it would return - if first file:

Header1;Header2;Header3;Header2
Value1;Value2;Value3;Value4

if not first file:

Value1;Value2;Value3;Value4

If I am correct, rest is about running this against a list file of files and joining the results in an output file.

EDIT: Against a directory:

void ProcessFiles(string folderName, string outputFileName)
{
    bool firstFile = true;
    foreach (var f in Directory.GetFiles(folderName))
    {
        File.AppendAllText(outputFileName, OpenAndParse(f, firstFile));
        firstFile = false;
    }
}

Note: I missed you want a DataTable and not an output file. Then you could simply create a list and put the results into that list making the list the datasource for your datatable (then why would you use semicolons in there? Probably all you need is to simply attach the array values to a list).

Answer 2

I don't know if this is the best way to do this. But what i would have done in your case, is to rewrite the CSV's the conventionnal way while reading all the files, then create a stream containing the new CSV created.

It would look like something like this :

     var csv = new StringBuilder();
            csv.AppendLine("Header1;Header2;Header3;Header4");
            foreach (var item in file)
            {
                var newLine = string.Format("{0},{1},{2},{3}", item.value1, item.value2, item.value3, item.value4);
                csv.AppendLine(newLine);
            }

            //Create Stream
            MemoryStream stream = new MemoryStream();
            StreamReader reader = new StreamReader(stream);

            //Fill your data table here with your values

Hope this will help.

Answer 3

(Adding as another answer just to make it uncluttered)

void ProcessMyFiles(string folderName)
{
    List<MyData> d = new List<MyData>();
    var files = Directory.GetFiles(folderName);
    foreach (var file in files)
    {
        OpenAndParse(file, d);
    }

    string[] headers = GetHeaders(files[0]);
    DataGridView dgv = new DataGridView {Dock=DockStyle.Fill};
    dgv.DataSource = d;
    dgv.ColumnAdded += (sender, e) => {e.Column.HeaderText = headers[e.Column.Index];};

    Form f = new Form();
    f.Controls.Add(dgv);
    f.Show();
}

string[] GetHeaders(string filename)
{
    var lines = File.ReadAllLines(filename);
    var parsed = lines.Select(l => l.Split(';')).ToArray();
    return new string[] { parsed[0][0], parsed[1][0], parsed[2][0], parsed[1][0] };
}

void OpenAndParse(string filename, List<MyData> d)
{
    var lines = File.ReadAllLines(filename);
    var parsed = lines.Select(l => l.Split(';')).ToArray();
    var data = new MyData
    {
        Col1 = parsed[0][1],
        Col2 = parsed[1][1],
        Col3 = parsed[2][1],
        Col4 = parsed[1][2]
    };
    d.Add(data);
}

public class MyData
{
    public string Col1 { get; set; }
    public string Col2 { get; set; }
    public string Col3 { get; set; }
    public string Col4 { get; set; }
}