SSIS 根据记录数将平面文件（TXT）拆分为多个

Question

我在 Visual Studio 中的 ssis 项目上有以下 c# 脚本，该脚本根据 sql 表的列值生成文件，我想添加一个部分，当记录数达到时，它也会拆分文件示例。前 200 个为 nameofile_columnvalue_datetime_1.txt，接下来的 200 个为 nameofile_columnvalue_datetime_2.txt 等等......提前谢谢你！

 public void Main()
        {
            // TODO: Add your code here
            string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
            string datetime = datetime_1.Replace("/", String.Empty);

            try
            {

                //Declare Variables
                string FileNamePart = Dts.Variables["User::FlatFileNamePart"].Value.ToString();
                string DestinationFolder = Dts.Variables["User::DestinationFolder"].Value.ToString();
                string TableName = Dts.Variables["User::TableName"].Value.ToString();
                string FileDelimiter = Dts.Variables["User::FileDelimiter"].Value.ToString();
                string FileExtension = Dts.Variables["User::FileExtension"].Value.ToString();
                string ColumnNameForGrouping = Dts.Variables["User::ColumnNameForGrouping"].Value.ToString();
                string SubFolder = Dts.Variables["User::SubFolder"].Value.ToString();
                Int32 RecordCntPerFile = (Int32)Dts.Variables["User::RecordsPerFile"].Value;
                string RecordCntPerFileDecimal = RecordCntPerFile + ".0";

                //USE ADO.NET Connection from SSIS Package to get data from table
                SqlConnection myADONETConnection = new SqlConnection();
                myADONETConnection = (SqlConnection)(Dts.Connections["AR_GSLO_OLTP"].AcquireConnection(Dts.Transaction) as SqlConnection);

                //Read distinct Group Values for each flat file
                string query = "Select distinct " + ColumnNameForGrouping + " from " + TableName;

                //MessageBox.Show(query.ToString());
                SqlCommand cmd = new SqlCommand(query, myADONETConnection);
                //myADONETConnection.Open();
                DataTable dt = new DataTable();
                dt.Load(cmd.ExecuteReader());
                myADONETConnection.Close();

                //Loop through values for ColumnNameForGroup

                foreach (DataRow dt_row in dt.Rows)
                {
                    string ColumnValue = "";
                    object[] array = dt_row.ItemArray;
                    ColumnValue = array[0].ToString();


                    //Load Data into DataTable from SQL ServerTable
                    string queryString =
                     "SELECT * from " + TableName + " Where " + ColumnNameForGrouping + "='" + ColumnValue + "'";
                    SqlDataAdapter adapter = new SqlDataAdapter(queryString, myADONETConnection);
                    DataSet ds = new DataSet();
                    adapter.Fill(ds);

                    foreach (DataTable d_table in ds.Tables)
                    {
                        string FileFullPath = SubFolder + "\\" + FileNamePart + "_" + ColumnValue + "_" + datetime + FileExtension;

                        StreamWriter sw = null;
                        sw = new StreamWriter(FileFullPath, false);

                        // Write the Header Row to File an thelw na exei kai header ta kanw uncomment apo 152 mexri 160 grammi
                        int ColumnCount = d_table.Columns.Count;
                        // for (int ic = 0; ic < ColumnCount; ic++)
                        //{
                        //    sw.Write(d_table.Columns[ic]);
                        //    if (ic < ColumnCount - 1)
                        //    {
                        //        sw.Write(FileDelimiter);
                        //    }
                        //}
                        // sw.Write(sw.NewLine);

                        // Write All Rows to the File
                        foreach (DataRow dr in d_table.Rows)
                        {
                            for (int ir = 0; ir < ColumnCount; ir++)
                            {
                                if (!Convert.IsDBNull(dr[ir]))
                                {
                                    sw.Write(dr[ir].ToString());
                                }
                                if (ir < ColumnCount - 1)
                                {
                                    sw.Write(FileDelimiter);
                                }
                            }
                            sw.Write(sw.NewLine);
                        }
                        sw.Close();
                    }

                }

            }

            catch (Exception exception)
            {

                // Create Log File for Errors
                using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                    "ErrorLog_" + datetime + ".log"))
                {
                    sw.WriteLine(exception.ToString());
                    Dts.TaskResult = (int)ScriptResults.Failure;


                }
            }

        }

Answer 1

所以假设/理解你有一个脚本任务是你的行的来源？

您可以添加行计数任务来捕获变量中的行数以供以后验证。

之后，您可能需要一个新的脚本任务，您可以在其中验证有多少记录。 类似于 case 语句/if 语句或每 200 条奇数记录的循环类型以更改值/文件名。

使用该循环，您可以更改文件名/文件名变量。

所以你需要你的 ConnectionManager 文件目标有一个带有这个变量文件名的表达式。 因此，一旦达到 200.. 记录，脚本应该更改文件名，并且 output 应该创建一个新文件。

我不确定它是否会喜欢飞行中的名称更改。 但认为这可能是一种表达你想法的方式。

Answer 2

我首先使用了一个 foreach 循环容器将 txt 的名称作为输入变量，然后在循环内我使用以下 c# 代码创建了一个脚本

 public void Main()
    {
        // TODO: Add your code here
        string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
        string datetime = datetime_1.Replace("/", String.Empty);
        StreamWriter writer = null;

        try
        {

            //Declare Variables
            string filename= Dts.Variables["User::FileName"].Value.ToString();
            //MessageBox.Show(filename);

            using (StreamReader inputfile = new System.IO.StreamReader(filename))
                {
                    int count = 0;
                    int filecount = 0;
                    string line;
                    while ((line = inputfile.ReadLine()) != null)
                    {

                        if (writer == null || count > 199)
                        {
                            count = 0;
                            if (writer != null)
                            {
                                writer.Close();
                                writer = null;
                            }
                            ++filecount;
                            writer = new System.IO.StreamWriter(filename.Replace(".txt", "_") + filecount.ToString() + ".txt", true);
                        }
                        writer.WriteLine(line);

                        ++count;
                    }

                }
            
        }

        catch (Exception exception)
        {

            // Create Log File for Errors
            using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                "ErrorLog_" + datetime + ".log"))
            {
                sw.WriteLine(exception.ToString());
                Dts.TaskResult = (int)ScriptResults.Failure;


            }
        }
        finally
        {
            if (writer != null)
                writer.Close();
        }

        Dts.TaskResult = (int)ScriptResults.Success;

    }

就是这样！