SSIS 根據記錄數將平面文件（TXT）拆分為多個

Question

我在 Visual Studio 中的 ssis 項目上有以下 c# 腳本，該腳本根據 sql 表的列值生成文件，我想添加一個部分，當記錄數達到時，它也會拆分文件示例。前 200 個為 nameofile_columnvalue_datetime_1.txt，接下來的 200 個為 nameofile_columnvalue_datetime_2.txt 等等......提前謝謝你！

 public void Main()
        {
            // TODO: Add your code here
            string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
            string datetime = datetime_1.Replace("/", String.Empty);

            try
            {

                //Declare Variables
                string FileNamePart = Dts.Variables["User::FlatFileNamePart"].Value.ToString();
                string DestinationFolder = Dts.Variables["User::DestinationFolder"].Value.ToString();
                string TableName = Dts.Variables["User::TableName"].Value.ToString();
                string FileDelimiter = Dts.Variables["User::FileDelimiter"].Value.ToString();
                string FileExtension = Dts.Variables["User::FileExtension"].Value.ToString();
                string ColumnNameForGrouping = Dts.Variables["User::ColumnNameForGrouping"].Value.ToString();
                string SubFolder = Dts.Variables["User::SubFolder"].Value.ToString();
                Int32 RecordCntPerFile = (Int32)Dts.Variables["User::RecordsPerFile"].Value;
                string RecordCntPerFileDecimal = RecordCntPerFile + ".0";

                //USE ADO.NET Connection from SSIS Package to get data from table
                SqlConnection myADONETConnection = new SqlConnection();
                myADONETConnection = (SqlConnection)(Dts.Connections["AR_GSLO_OLTP"].AcquireConnection(Dts.Transaction) as SqlConnection);

                //Read distinct Group Values for each flat file
                string query = "Select distinct " + ColumnNameForGrouping + " from " + TableName;

                //MessageBox.Show(query.ToString());
                SqlCommand cmd = new SqlCommand(query, myADONETConnection);
                //myADONETConnection.Open();
                DataTable dt = new DataTable();
                dt.Load(cmd.ExecuteReader());
                myADONETConnection.Close();

                //Loop through values for ColumnNameForGroup

                foreach (DataRow dt_row in dt.Rows)
                {
                    string ColumnValue = "";
                    object[] array = dt_row.ItemArray;
                    ColumnValue = array[0].ToString();


                    //Load Data into DataTable from SQL ServerTable
                    string queryString =
                     "SELECT * from " + TableName + " Where " + ColumnNameForGrouping + "='" + ColumnValue + "'";
                    SqlDataAdapter adapter = new SqlDataAdapter(queryString, myADONETConnection);
                    DataSet ds = new DataSet();
                    adapter.Fill(ds);

                    foreach (DataTable d_table in ds.Tables)
                    {
                        string FileFullPath = SubFolder + "\\" + FileNamePart + "_" + ColumnValue + "_" + datetime + FileExtension;

                        StreamWriter sw = null;
                        sw = new StreamWriter(FileFullPath, false);

                        // Write the Header Row to File an thelw na exei kai header ta kanw uncomment apo 152 mexri 160 grammi
                        int ColumnCount = d_table.Columns.Count;
                        // for (int ic = 0; ic < ColumnCount; ic++)
                        //{
                        //    sw.Write(d_table.Columns[ic]);
                        //    if (ic < ColumnCount - 1)
                        //    {
                        //        sw.Write(FileDelimiter);
                        //    }
                        //}
                        // sw.Write(sw.NewLine);

                        // Write All Rows to the File
                        foreach (DataRow dr in d_table.Rows)
                        {
                            for (int ir = 0; ir < ColumnCount; ir++)
                            {
                                if (!Convert.IsDBNull(dr[ir]))
                                {
                                    sw.Write(dr[ir].ToString());
                                }
                                if (ir < ColumnCount - 1)
                                {
                                    sw.Write(FileDelimiter);
                                }
                            }
                            sw.Write(sw.NewLine);
                        }
                        sw.Close();
                    }

                }

            }

            catch (Exception exception)
            {

                // Create Log File for Errors
                using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                    "ErrorLog_" + datetime + ".log"))
                {
                    sw.WriteLine(exception.ToString());
                    Dts.TaskResult = (int)ScriptResults.Failure;


                }
            }

        }

Answer 1

所以假設/理解你有一個腳本任務是你的行的來源？

您可以添加行計數任務來捕獲變量中的行數以供以后驗證。

之后，您可能需要一個新的腳本任務，您可以在其中驗證有多少記錄。 類似於 case 語句/if 語句或每 200 條奇數記錄的循環類型以更改值/文件名。

使用該循環，您可以更改文件名/文件名變量。

所以你需要你的 ConnectionManager 文件目標有一個帶有這個變量文件名的表達式。 因此，一旦達到 200.. 記錄，腳本應該更改文件名，並且 output 應該創建一個新文件。

我不確定它是否會喜歡飛行中的名稱更改。 但認為這可能是一種表達你想法的方式。

Answer 2

我首先使用了一個 foreach 循環容器將 txt 的名稱作為輸入變量，然后在循環內我使用以下 c# 代碼創建了一個腳本

 public void Main()
    {
        // TODO: Add your code here
        string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
        string datetime = datetime_1.Replace("/", String.Empty);
        StreamWriter writer = null;

        try
        {

            //Declare Variables
            string filename= Dts.Variables["User::FileName"].Value.ToString();
            //MessageBox.Show(filename);

            using (StreamReader inputfile = new System.IO.StreamReader(filename))
                {
                    int count = 0;
                    int filecount = 0;
                    string line;
                    while ((line = inputfile.ReadLine()) != null)
                    {

                        if (writer == null || count > 199)
                        {
                            count = 0;
                            if (writer != null)
                            {
                                writer.Close();
                                writer = null;
                            }
                            ++filecount;
                            writer = new System.IO.StreamWriter(filename.Replace(".txt", "_") + filecount.ToString() + ".txt", true);
                        }
                        writer.WriteLine(line);

                        ++count;
                    }

                }
            
        }

        catch (Exception exception)
        {

            // Create Log File for Errors
            using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                "ErrorLog_" + datetime + ".log"))
            {
                sw.WriteLine(exception.ToString());
                Dts.TaskResult = (int)ScriptResults.Failure;


            }
        }
        finally
        {
            if (writer != null)
                writer.Close();
        }

        Dts.TaskResult = (int)ScriptResults.Success;

    }

就是這樣！