Named Pipe Dropping Data in C Program Using fgets in Linux

Question

I'm trying to pull data from a named pipe in a C program (gcc -std=c99) using fgets. I created the pipe at the command line and I can see that it exists in the file system and is parked with ap when using "ls -la" and the rights are set for anyone to use.

prw-rw-rw-  1 www-data www-data    0 Dec 23 00:39 mypipe

I am sending data to the pipe with a simple script:

#script
myPipe="/tmp/mypipe"
    for j in {0..255}
    do
#       echo "C $j"             #when this is enabled slightly more data gets through
        echo "C $j" >> $myPipe
#       sleep 0.00000000000001      #when this is enabled all the data gets through           
    done

The loop my program is trying to read from this pipe is:

void * reader(){

    int messages = 0;                           // keep track of how many messages we process
    char buf[PIPE_BUF];     buf[0] = 0;         //#defined  PIPE_BUF    1024
    int x=100, y=101, z=102, count=103;         //initialized to unlikely initial input results
    char linetag[PIPE_BUF]; linetag[0] = 0;     //space to store the data label
    char * pScan = NULL;
    FILE *myStream ;

    syslog (LOG_INFO, "Reader started\n");

    myStream = fopen(pipeName, "r");
    if ( myStream == NULL ) {
        syslog (LOG_INFO, "Could not open pipe %s as input to Cota\n", pipeName ) ;
        return 0;
    }
    while (1) {

        //>>> This was the problem and removing it seems to fix the issue
        //freopen(pipeName, "r", myStream);  <<<<This was the problem !!!!

        pScan==buf;
        syslog (LOG_INFO, "readp-1:msg:%i >>%s<<>>%s<< pBuf=%p, pScan=%p, count=%i, tag=%s, x=%d, y=%d, z=%d\n", messages, buf, pScan, buf, pScan, count, linetag, x,y,z);

        while ( (pScan = fgets(buf, PIPE_BUF, myStream)) == NULL){};

        if (pScan == NULL){syslog (LOG_WARNING, "ERROR NULL pScan error");}

        syslog (LOG_INFO, "readp-2:msg:%i >>%s<<>>%s<< pBuf=%p, pScan=%p, count=%i, tag=%s, x=%d, y=%d, z=%d\n", messages, buf, pScan, buf, pScan, count, linetag, x,y,z);

        count = sscanf(pScan, "%s %d %d %d", linetag, &x, &y, &z);

        syslog (LOG_INFO, "readp-3:msg:%i >>%s<<>>%s<< pBuf=%p, pScan=%p, count=%i, tag=%s, x=%d, y=%d, z=%d\n", messages, buf, pScan, buf, pScan, count, linetag, x,y,z);

        messages++;
    }
}

My syslog shows the following when I run the script as written above :

Reader started
readp-1:msg:0 >><<>>(null)<< pBuf=0xbe885df4, pScan=(nil), count=103, tag=, x=100, y=101, z=102
readp-2:msg:0 >>C 0#012<<>>C 0#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=103, tag=, x=100, y=101, z=102
readp-3:msg:0 >>C 0#012<<>>C 0#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=0, y=101, z=102
readp-1:msg:1 >>C 0#012<<>>C 0#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=0, y=101, z=102
readp-2:msg:1 >>C 1#012<<>>C 1#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=0, y=101, z=102
readp-3:msg:1 >>C 1#012<<>>C 1#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=1, y=101, z=102
readp-1:msg:2 >>C 1#012<<>>C 1#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=1, y=101, z=102
readp-2:msg:2 >>C 10#012<<>>C 10#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=1, y=101, z=102
readp-3:msg:2 >>C 10#012<<>>C 10#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=10, y=101, z=102
readp-1:msg:3 >>C 10#012<<>>C 10#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=10, y=101, z=102
readp-2:msg:3 >>C 11#012<<>>C 11#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=10, y=101, z=102
readp-3:msg:3 >>C 11#012<<>>C 11#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=11, y=101, z=102
readp-1:msg:4 >>C 11#012<<>>C 11#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=11, y=101, z=102
readp-2:msg:4 >>C 15#012<<>>C 15#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=11, y=101, z=102
readp-3:msg:4 >>C 15#012<<>>C 15#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=15, y=101, z=102
readp-1:msg:5 >>C 15#012<<>>C 15#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=15, y=101, z=102
readp-2:msg:5 >>C 16#012<<>>C 16#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=15, y=101, z=102
readp-3:msg:5 >>C 16#012<<>>C 16#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=16, y=101, z=102
readp-1:msg:6 >>C 16#012<<>>C 16#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=16, y=101, z=102
readp-2:msg:6 >>C 18#012<<>>C 18#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=16, y=101, z=102
readp-3:msg:6 >>C 18#012<<>>C 18#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=18, y=101, z=102
readp-1:msg:7 >>C 18#012<<>>C 18#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=18, y=101, z=102
readp-2:msg:7 >>C 24#012<<>>C 24#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=18, y=101, z=102
readp-3:msg:7 >>C 24#012<<>>C 24#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=24, y=101, z=102
readp-1:msg:8 >>C 24#012<<>>C 24#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=24, y=101, z=102
readp-2:msg:8 >>C 39#012<<>>C 39#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=24, y=101, z=102
readp-3:msg:8 >>C 39#012<<>>C 39#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=39, y=101, z=102
readp-1:msg:9 >>C 39#012<<>>C 39#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=39, y=101, z=102
readp-2:msg:9 >>C 50#012<<>>C 50#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=39, y=101, z=102
readp-3:msg:9 >>C 50#012<<>>C 50#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=50, y=101, z=102
readp-1:msg:10 >>C 50#012<<>>C 50#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=50, y=101, z=102
readp-2:msg:10 >>C 70#012<<>>C 70#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=50, y=101, z=102
readp-3:msg:10 >>C 70#012<<>>C 70#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=70, y=101, z=102
readp-1:msg:11 >>C 70#012<<>>C 70#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=70, y=101, z=102
readp-2:msg:11 >>C 81#012<<>>C 81#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=70, y=101, z=102
readp-3:msg:11 >>C 81#012<<>>C 81#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=81, y=101, z=102
readp-1:msg:12 >>C 81#012<<>>C 81#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=81, y=101, z=102
readp-2:msg:12 >>C 108#012<<>>C 108#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=81, y=101, z=102
readp-3:msg:12 >>C 108#012<<>>C 108#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=108, y=101, z=102
readp-1:msg:13 >>C 108#012<<>>C 108#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=108, y=101, z=102
readp-2:msg:13 >>C 119#012<<>>C 119#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=108, y=101, z=102
readp-3:msg:13 >>C 119#012<<>>C 119#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=119, y=101, z=102
readp-1:msg:14 >>C 119#012<<>>C 119#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=119, y=101, z=102
readp-2:msg:14 >>C 136#012<<>>C 136#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=119, y=101, z=102
readp-3:msg:14 >>C 136#012<<>>C 136#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=136, y=101, z=102
readp-1:msg:15 >>C 136#012<<>>C 136#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=136, y=101, z=102
readp-2:msg:15 >>C 155#012<<>>C 155#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=136, y=101, z=102
readp-3:msg:15 >>C 155#012<<>>C 155#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=155, y=101, z=102
readp-1:msg:16 >>C 155#012<<>>C 155#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=155, y=101, z=102
readp-2:msg:16 >>C 172#012<<>>C 172#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=155, y=101, z=102
readp-3:msg:16 >>C 172#012<<>>C 172#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=172, y=101, z=102
readp-1:msg:17 >>C 172#012<<>>C 172#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=172, y=101, z=102
readp-2:msg:17 >>C 193#012<<>>C 193#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=172, y=101, z=102
readp-3:msg:17 >>C 193#012<<>>C 193#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=193, y=101, z=102
readp-1:msg:18 >>C 193#012<<>>C 193#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=193, y=101, z=102
readp-2:msg:18 >>C 213#012<<>>C 213#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=193, y=101, z=102
readp-3:msg:18 >>C 213#012<<>>C 213#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=213, y=101, z=102
readp-1:msg:19 >>C 213#012<<>>C 213#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=213, y=101, z=102
readp-2:msg:19 >>C 233#012<<>>C 233#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=213, y=101, z=102
readp-3:msg:19 >>C 233#012<<>>C 233#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=233, y=101, z=102
readp-1:msg:20 >>C 233#012<<>>C 233#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=233, y=101, z=102
readp-2:msg:20 >>C 246#012<<>>C 246#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=233, y=101, z=102
readp-3:msg:20 >>C 246#012<<>>C 246#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=246, y=101, z=102
readp-1:msg:21 >>C 246#012<<>>C 246#012<< pBuf=0xbe885df4, pScan=0xbe885df4, count=2, tag=C, x=246, y=101, z=102

The data alyays seems to be properly aligned with a 'C' at the start of the line. If, in my script, I enable local terminal echo to the screen then I drop fewer lines. If I enable the sleep 0.00000000000001 I am able to read all of the data. I really need this to run a full speed and hate to slow things down with a sleep. Am I simply over running a buffer and if so can the buffer size be increased? I'm really stumped. BTW, I'm using fgets rather than fscanf as I will eventually need to add in fastcgi and fsastcgi seems lacks support for fscanf (note in this test I am using the standard io libs, I am not linking in fastcgi and I am not including "fcgi/include/fcgi_stdio.h" in this test so there is no overwriting of the default libs). Any direction to make this be reliable and efficient would be greatly appreciated.

Answer 1

FIFOs are funny (peculiar) files. A reading process blocks on open until there is a writer; a writing process blocks on open until there is a reader. If there are no writers, the reader will get EOF; at the point, it must close the FIFO and reopen it to allow more writers. If there are no readers or writers, the content of the FIFO is discarded. In your shell script, you are repeatedly opening and closing the FIFO. You would do better to move the >> $myPipe onto the done line of the loop, so the FIFO is opened once, then all the messages written, and then the FIFO is closed.

Your code is peculiar. This line is problematic:

while ((pScan = fgets(buf, PIPE_BUF, myStream)) == NULL){};

I see two problems. The less serious is that you have a null statement after the empty loop body. This doesn't do anything harmful except show experienced readers that you're not confident in your C code. The {} would be sufficient; it is an empty loop body. A semicolon on the next line in lieu of the {} would be conventional.

The more serious problem is that if you ever enter the loop, you are going to stay in it indefinitely. Once myStream reports EOF (by returning NULL from fgets() , it will continue to do so indefinitely (unless you're on Linux and the input device is actually a terminal — which is a separate sore point with me).

It seems that the code in the question was hacked after it was asked and the freopen() line at the top of the loop was commented out, and this 'fixed' the problem. This is explicable. When the FIFO was closed for reopening, the content was discarded.

Better code would probably look more like this. It passes the pipe name directly to the function (avoid global variables when you can). The function only returns if it fails to open the FIFO. There's no obvious need or benefit to the function returning NULL (written 0 — it's OK; I do that too), so it is now a void function. The function declaration should be in a header file and not written inline in the code.

On the whole, I'd move most of the variables much closer to their point of first use — only messages would stay outside the infinite loop. There would be a header to declare the various functions

#include <syslog.h>
#include <stdio.h>

#define PIPE_BUF 1024

extern void reader(const char *pipeName);

void reader(const char *pipeName)
{
    int messages = 0;
    char buf[PIPE_BUF];
    int x = 100, y = 101, z = 102, count = 103;
    char linetag[PIPE_BUF];

    syslog(LOG_INFO, "Reader started\n");

    while (1)
    {
        FILE *myStream = fopen(pipeName, "r");
        if (myStream == NULL)
        {
            syslog(LOG_INFO, "Could not open pipe %s as input to Cota\n", pipeName);
            return;
        }
        while (fgets(buf, sizeof(buf), myStream) != 0)
        {
            count = sscanf(buf, "%s %d %d %d", linetag, &x, &y, &z);
            syslog(LOG_INFO, "readp-3:msg:%i >>%s<< count=%i, tag=%s, x=%d, y=%d, z=%d\n",
                   messages, buf, count, linetag, x, y, z);
            messages++;
        }
        fclose(myStream);
    }

    /* NOTREACHED */
    syslog(LOG_INFO, "Reader finished\n");
}