Recognize newly synchronized files (WinSCP)

Question

I am currently working on a parsing application which gets the .csv files first from the remote server and then synchronize it to the local server. After the synchronization, the downloaded files from local path will be parsed and then inserted to SQL database. What if there are new files added from the remote server (which are then synchronized to local server), how would the app know to only parse those particular new files (preventing to re-parse and re-insert the old .csv files which were already parsed)?

My code so far:

public static int Main()
{
    try
    {
        // Setup session options
        SessionOptions sessionOptions = new SessionOptions
        {
            Protocol = Protocol.Scp,
            HostName = hostName,
            UserName = userName,
            Password = passWord,
            SshHostKeyFingerprint = sshHostKey
        };

        using (Session session = new Session())
        {
            // Will continuously report progress of synchronization
            session.FileTransferred += FileTransferred;

            // Connect
            session.Open(sessionOptions);

            // Synchronize files
            SynchronizationResult synchronizationResult;
            synchronizationResult =
                session.SynchronizeDirectories(
                    SynchronizationMode.Local, localPath, remotePath, false);

            // Throw on any error
            synchronizationResult.Check();

            Run();
        }
        return 0;
    }
    catch (Exception e)
    {
        Console.WriteLine("Error: {0}", e);
        Console.ReadLine();
        return 1;
    }
}

This handles the event when synchronizing files:

private static void FileTransferred(object sender, TransferEventArgs e)
{
    if (e.Error == null)
    {
        Console.WriteLine("Upload of {0} from remote to local server succeeded", e.FileName);
    }
    else
    {
        Console.WriteLine("Upload of {0} from remote to local server failed: {1}", e.FileName, e.Error);
    }

    if (e.Chmod != null)
    {
        if (e.Chmod.Error == null)
        {
            Console.WriteLine("Permisions of {0} set to {1}", e.Chmod.FileName, e.Chmod.FilePermissions);
        }
        else
        {
            Console.WriteLine("Setting permissions of {0} failed: {1}", e.Chmod.FileName, e.Chmod.Error);
        }
    }
    else
    {
        Console.WriteLine("Permissions of {0} kept with their defaults", e.Destination);
    }

    if (e.Touch != null)
    {
        if (e.Touch.Error == null)
        {
            Console.WriteLine("Timestamp of {0} set to {1}", e.Touch.FileName, e.Touch.LastWriteTime);
        }
        else
        {
            Console.WriteLine("Setting timestamp of {0} failed: {1}", e.Touch.FileName, e.Touch.Error);
        }
    }
    else
    {
        // This should never happen during "local to remote" synchronization
        Console.WriteLine("Timestamp of {0} kept with its default (current time)", e.Destination);
    }
}

This parses the contents of .csv files. Happens after synchronization.

public static void Run()
{
    dataTable();

    List<string> items = new List<string>();

    foreach (string file in Directory.EnumerateFiles(localPath, "*.csv"))
    {
        if (file.Contains("test"))
        { }
        else
        {
            using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
            {
                using (StreamReader sr = new StreamReader(fs))
                {
                    while (!sr.EndOfStream)
                        items.Add(sr.ReadLine());

                    foreach (string item in items)
                    {
                        var row = dt.NewRow();
                        string[] columnValues = item.Split(',');
                        int column = 3;

                        for (int a = 0; a < columnValues.Length; a++)
                        {
                            string date = columnValues[29] + " " + columnValues[28];
                            row[col[1].ColumnName] = DateTime.Parse(date);
                            string test = file.Split(new[] { splitVal }, StringSplitOptions.None)[1];
                            row[col[2].ColumnName] = test.Split('.')[0];

                            if (a >= 54)
                            { }
                            else
                            {
                                if (string.IsNullOrEmpty(columnValues[a]))
                                {
                                    row[col[column].ColumnName] = DBNull.Value;
                                }
                                else
                                {
                                    try
                                    {
                                        try
                                        {
                                            row[col[column].ColumnName] = columnValues[a].Trim();
                                        }
                                        catch
                                        {
                                            try
                                            {
                                                row[col[column].ColumnName] = Convert.ToDouble(columnValues[a].Trim());
                                            }
                                            catch
                                            {
                                                row[col[column].ColumnName] = int.Parse(columnValues[a].Trim());
                                            }
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        Console.WriteLine("Error [" + col[column].ColumnName + "]:" + ex.ToString());
                                        //row[col[column].ColumnName] = DBNull.Value;
                                    }
                                }
                            }

                            column++;
                        }
                        dt.Rows.Add(row);
                    }

                    using (SqlConnection con = new SqlConnection(consstring))
                    {
                        using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy(con))
                        {
                            //Set the database table name
                            sqlBulkCopy.DestinationTableName = dbTable;
                            con.Open();
                            try
                            {
                                sqlBulkCopy.WriteToServer(dt);
                                Console.WriteLine(file.Substring(file.LastIndexOf('\\') + 1) + " uploaded in the database\n");
                            }
                            catch (Exception ex)
                            {
                                Console.WriteLine(file.Substring(file.LastIndexOf('\\') + 1) + " cannot be uploaded to database. " + ex.ToString());
                            }
                            con.Close();
                        }
                    }
                    sr.Dispose();
                    sr.Close();
                }

                fs.Dispose();
                fs.Close();
            }
        }
    }
}

The above code is based from Session.SynchronizeDirectories Method of WinSCP.

Answer 1

So do not enumerate all *.csv files. Enumerate only those that were synchronized/downloaded:

foreach (TransferEventArgs transfer in synchronizationResult.Downloads)
{
    string file = transfer.Destination;
    ...
}
 

See theSynchronizationResult class .

---

If you need continuous synchronization, you need to run your code in a loop or schedule it to be run in frequent intervals. See WinSCP example Keep local directory up to date (download changed files from remote SFTP/FTP server) – It's in PowerShell, but should give you an idea.