Open file with O_APPEND on FreeBsd and Linux doesn't look like an atomic operation?

Question

#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#define BUFSIZE 1024*1024*100
#define FILE_MODE       (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
int main()
{
    pid_t pid;
    int pfd = open("data.txt", O_CREAT|O_RDWR|O_TRUNC|O_APPEND, FILE_MODE);
    if ((pid = fork()) < 0)
    {
    printf("errr fork\n");
    exit(0);
    } else if (pid == 0)
    {
        int cfd = open("data.txt", O_CREAT|O_RDWR|O_TRUNC|O_APPEND, FILE_MODE);
        for (int i = 0; i < 1000000; ++i)
            if (write(cfd, "child\n", 6) == -1)
        {
        printf("err in child\n");
        exit(0);
        }

        exit(0);
    }
    for (int i = 0; i < 1000000; ++i)
        if (write(pfd, "parent\n", 7) == -1)
    {
        printf("error in parent\n");
        exit(0);
    }
    return 0;
}

Test environment 1:

uname -a
FreeBSD freebsd 10.0-RELEASE-p12 FreeBSD 10.0-RELEASE-p12 #0: Tue Nov 4 05:07:17 UTC 2014 root@amd64-builder.daemonology.net:/usr/obj/usr/src/sys/GENERIC amd64

clang -v
FreeBSD clang version 3.3 (tags/RELEASE_33/final 183502) 20130610 Target: x86_64-unknown-freebsd10.0 Thread model: posix

Test environment 2:

uname -a
Linux debian 3.2.0-4-amd64 #1 SMP Debian 3.2.63-2+deb7u1 x86_64 GNU/Linux

gcc --version
gcc (Debian 4.7.2-5) 4.7.2 Copyright (C) 2012 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

Some lines lost

$ sort data.txt | uniq -c
1000000 child
982937 parent

O_APPEND doesn't seem to work as the man page said it should.

Can anyone give me more information about this?

Answer 1

The trouble is that when the child opens the file, it truncates it, losing the data that the parent has already written to the file, leaving the parent with a short count. It is likely that it will always be the parent with the short count because the child has to open the file a second time, and file opening is an expensive operation. If the first open was moved so it was only in the parent, the timing would be different; it's possible that the child would sometimes have fewer entries.

Modified code:

#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

#define FILE_MODE       (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

int main(void)
{
    pid_t pid;
    int pfd = open("data.txt", O_CREAT | O_RDWR | O_TRUNC | O_APPEND, FILE_MODE);
    if ((pid = fork()) < 0)
    {
        printf("errr fork\n");
        exit(0);
    }
    else if (pid == 0)
    {
        int cfd = open("data.txt", O_CREAT | O_RDWR | O_TRUNC | O_APPEND, FILE_MODE);
        for (int i = 0; i < 1000000; ++i)
        {
            char buffer[32];
            snprintf(buffer, sizeof(buffer), "child%07d\n", i);
            if (write(cfd, buffer, strlen(buffer)) == -1)
            {
                printf("err in child\n");
                exit(0);
            }
        }
    }
    else
    {
        for (int i = 0; i < 1000000; ++i)
        {
            char buffer[32];
            snprintf(buffer, sizeof(buffer), "parent%07d\n", i);
            if (write(pfd, buffer, strlen(buffer)) == -1)
            {
                printf("error in parent\n");
                exit(0);
            }
        }
    }
    return 0;
}

One time when I ran it, the output started:

parent0000052
parent0000053
parent0000054
parent0000055
parent0000056
parent0000057
parent0000058
parent0000059
parent0000060
parent0000061
parent0000062
parent0000063
parent0000064
child0000000
parent0000065
child0000001
parent0000066
child0000002
parent0000067

So, the parent had executed 52 writes before the child truncated the file (0..51). The counts agree:

$ grep -o -E 'child|parent' data.txt | sort | uniq -c
1000000 child
999948 parent
$