System Process Stops for no reason Windows Server 2016

Question

public void startTraining(bool initial)
    {
        int maxBatches = 100;
        int increment = 100;

        string ioFile = "";
        string ioFilePath = "C:\\pathOfCfg";

        while (maxBatches <= 5000)
        {
            if (maxBatches == increment)
            {

                string serverCmd = "/c HeavyProcessString;
                using (StreamWriter cmdFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\File\\", "cmdCommands_" + maxBatches + ".txt")))
                {
                    cmdFile.WriteLine(serverCmd);
                }
                Process p = new Process();
                p.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal;
                p.StartInfo.UseShellExecute = false;
                p.StartInfo.CreateNoWindow = false;
                p.StartInfo.RedirectStandardOutput = true;
                p.StartInfo.RedirectStandardError = true;
                p.StartInfo.FileName = "C:\\Windows\\system32\\cmd.exe";
                p.StartInfo.Arguments = serverCmd;
                p.wait
                p.Start();

                try
                {
                    string op = p.StandardOutput.ReadToEnd();
                    using (StreamWriter outputFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "output_" + maxBatches + ".txt")))
                    {
                        outputFile.WriteLine(op);
                    }
                }
                catch (Exception ex)
                {
                    string op = ex.ToString();
                    using (StreamWriter outputFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "output_catch" + maxBatches + ".txt")))
                    {
                        outputFile.WriteLine(op);
                    }
                }

                try
                {
                    string ep = p.StandardError.ReadToEnd();
                    using (StreamWriter errorFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "error_" + maxBatches + ".txt")))
                    {
                        errorFile.WriteLine(ep);
                    }
                }
                catch (Exception ex)
                {
                    string ep = ex.ToString();
                    using (StreamWriter errorFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "error_catch" + maxBatches + ".txt")))
                    {
                        errorFile.WriteLine(ep);
                    }
                }

                ioFile =  maxBatches + "_.io";
                ioFile Path= rootPath + "\\" + project.ID + "\\File\\" + ioFile ;
                initial = false;
                p.Close();

            }
            else
            {
                string serverCmd = "/c HeavyProcessString;
                using (StreamWriter cmdFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "cmdCommands_" + maxBatches + ".txt")))
                {
                    cmdFile.WriteLine(serverCmd);
                }

                Process p = new Process();
                p.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal;

                p.StartInfo.UseShellExecute = false;
                p.StartInfo.CreateNoWindow = false;
                p.StartInfo.RedirectStandardOutput = true;
                p.StartInfo.RedirectStandardError = true;
                p.StartInfo.FileName = "C:\\Windows\\system32\\cmd.exe";
                p.StartInfo.Arguments = serverCmd;
                p.Start();

                try
                {
                    string op = p.StandardOutput.ReadToEnd();
                    using (StreamWriter outputFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "output_" + maxBatches + ".txt")))
                    {
                        outputFile.WriteLine(op);
                    }
                }
                catch (Exception ex)
                {
                    string op = ex.ToString();
                    using (StreamWriter outputFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "output_catch" + maxBatches + ".txt")))
                    {
                        outputFile.WriteLine(op);
                    }
                }

                try
                {
                    string ep = p.StandardError.ReadToEnd();
                    using (StreamWriter errorFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "error_" + maxBatches + ".txt")))
                    {
                        errorFile.WriteLine(ep);
                    }
                }
                catch (Exception ex)
                {
                    string ep = ex.ToString();
                    using (StreamWriter errorFile = new StreamWriter(Path.Combine(rootPath + "\\" + project.ID + "\\Train", "error_catch" + maxBatches + ".txt")))
                    {
                        errorFile.WriteLine(ep);
                    }
                }

                ioFile = maxBatches + "_.io";
                ioFilePath = rootPath + "\\" + project.ID + "\\File\\" + ioFile;
                p.Close();
            }
            maxBatches += increment;
        }
    }

I have a function like that which works on server and iteratively takes the output file comes from the process before how ever after 5 times of work it stops giving output. It writes nothing. I thought if the process stops or is there a timeout mechanism or if the process does not work because it is out of memory (but when I run it from cmd it works fine for the 6th iteration) do you have any suggestions or wisdom about that? ps:Files and working directories are working fine

        Task.Run(() => startTraining());

This method is used to call this asynchrounisly is this task being closed after a while ?

Answer 1

Does it stop in 20 minutes? If yes check your ISS pool advantaged settings and make the idle time wait 0 or something longer

Answer 2

You are going to need to read both from standerd output and error at the same time. Likey what is happening is the buffer for standard error is getting full and the progam freezes till it something reads from it (nothing will).

You need to either have your standard error read on another thread or you need to use the event based version of reading standard error to read your errors and put your code to write out to the file in there. See the remarks section of that link for specific instructions on what you need to do to make it work.