Measuring sendfile API on Linux

Question

I wrote a simple file copy application to measure the effectiveness of using sendfile API over normal read-from-file-and-write-to-socket approach for large files. However upon running the application using both the approaches, I found out that the difference in the amount of time taken for the file copy to get completed is very minimal between the two approaches.

I read from multiple sources that "sendfile" API would give tremendous performance improvement over the normal read-from-file-and-write-to-socket approach. But when I tried to benchmark with a single 2GB file following are the numbers I observed (average of 4 iterations):

1. Normal read-from-file-and-write-to-socket approach: 17 secs 444840 usecs
2. sendfile API: 17 secs 431420 usecs

I am running both the server and client pieces of the application on two different machines (Linux kernel version 4.4.162-94.72-default) in an isolated 1Gbps network.

Can someone help me what exactly am doing wrong here or missing here?

Server:

#define _GNU_SOURCE

#include "file_details.h"

void calculate_execution_time(struct timeval start, struct timeval end)
{
    struct timeval  time_diff;


    time_diff.tv_sec = end.tv_sec - start.tv_sec;
    time_diff.tv_usec = end.tv_usec - start.tv_usec;

    // Adjust the time appropriately
    while (time_diff.tv_usec < 0) {
        time_diff.tv_sec--;
        time_diff.tv_usec += 1000000;
    }

    printf("total execution time: = %lds.%ldus\n", time_diff.tv_sec, time_diff.tv_usec);
}


int read_from_file_pread(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
    ssize_t         bytes_read = 0, bytes_sent = 0, total_bytes_sent = 0, bytes_sent_this_itr = 0;
    off_t           offset = 0;
    char            *buffer = NULL;
    struct timeval      start_time, end_time;


    buffer = calloc(chunk_size, sizeof(char));
    if (buffer == NULL) {
        printf("Failed to allocate memory of size: %d bytes\n", chunk_size);
        return -1;
    }

    gettimeofday(&start_time, NULL);

    do {
        bytes_read = pread(fd, buffer, chunk_size, offset);
        switch (bytes_read) {
            case -1:
                printf("Failed to read from file: %s, offset: %lu, error: %d\n", file_name, offset, errno);

                free(buffer);
                return -1;

            case 0:
                printf("Completed reading from file and sending\n");
                break;

            default:
                do {
                    bytes_sent = send(client_sockfd, buffer, (bytes_read - bytes_sent_this_itr), 0);
                    if (bytes_sent == -1) {
                        printf("Failed to send %lu bytes, error: %d\n", (bytes_read - bytes_sent_this_itr), errno);

                        free(buffer);
                        return -1;
                    }

                    bytes_sent_this_itr += bytes_sent;
                } while (bytes_sent_this_itr < bytes_read);

                bytes_sent = 0;
                bytes_sent_this_itr = 0;
                offset += bytes_read;
                total_bytes_sent += bytes_read;
                break;
        }
    } while (total_bytes_sent < file_size_in_bytes);

    gettimeofday(&end_time, NULL);

    printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);

    calculate_execution_time(start_time, end_time);

    free(buffer);
    return 0;
}


int read_from_file_sendfile(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
    ssize_t         bytes_sent = 0, total_bytes_sent = 0;
    off_t           offset = 0;
    struct timeval      start_time, end_time;


    gettimeofday(&start_time, NULL);

    do {
        bytes_sent = sendfile(client_sockfd, fd, &offset, chunk_size);
        if (bytes_sent == -1) {
            printf("Failed to sendfile: %s, offset: %lu, error: %d\n", file_name, offset, errno);
            return -1;
        }

        total_bytes_sent += bytes_sent;
    } while (total_bytes_sent < file_size_in_bytes);

    gettimeofday(&end_time, NULL);

    printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);

    calculate_execution_time(start_time, end_time);

    return 0;
}


int read_from_file(int client_sockfd, char *file_name, char *type, int chunk_size)
{
    int         error_code = 0, fd = 0;
    ssize_t         hdr_length = 0, bytes_sent = 0, file_name_length = strlen(file_name);
    struct stat     file_stat = {0};
    struct file_details *file_details_to_send = NULL;


    fd = open(file_name, O_RDONLY, S_IRUSR);
    if (fd == -1) {
                printf("Failed to open file: %s, error: %d\n", file_name, errno);
                return -1;
    }

    error_code = fstat(fd, &file_stat);
    if (error_code == -1) {
                printf("Failed to get status of file: %s, error: %d\n", file_name, errno);

        close(fd);
        return -1;
    }

    hdr_length = (sizeof(struct file_details) + file_name_length + 1);
    file_details_to_send = calloc(hdr_length, sizeof(char));
    if (file_details_to_send == NULL) {
        perror("Failed to allocate memory");

        close(fd);
        return -1;
    }

    file_details_to_send->file_name_length = file_name_length;
    file_details_to_send->file_size_in_bytes = file_stat.st_size;
    strcpy(file_details_to_send->file_name, file_name);

    printf("File name: %s, size: %lu bytes\n", file_name, file_stat.st_size);

    bytes_sent = send(client_sockfd, file_details_to_send, hdr_length, 0);
    if (bytes_sent == -1) {
        printf("Failed to send header of size: %lu bytes, error: %d\n", hdr_length, errno);

        close(fd);
        return -1;
    }

    if (strcmp(type, "rw") == 0) {
        printf("By pread and send\n");

        read_from_file_pread(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
    } else {
        printf("By sendfile\n");

        read_from_file_sendfile(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
    }

    close(fd);
    return 0;
}


int main(int argc, char *argv[])
{
    ...
    ...

    option_value = 1;
    error_code = setsockopt(client_sockfd, SOL_TCP, TCP_NODELAY, &option_value, sizeof(int));
    if (error_code == -1) {
        printf("Failed to set socket option TCP_NODELAY to socket descriptor: %d, error: %d", client_sockfd, errno);
    }

    read_from_file(client_sockfd, file_name, type, chunk_size);

    ...
}

Answer 1

Your code almost certainly made a big performance improvement. The problem might be that you're measuring wall time. Consider calling getrusage() instead of gettimeofday(). The ru_utime and ru_stime fields represent how much time the kernel and your program spent doing actual work. sendfile() should make those numbers go down. That way you consume less energy, and free up more resources for other programs on your computer. Unfortunately however it can't make the network go faster. Optimal wall time speed to send 2GB on 1GbPS ethernet assuming zero overhead would be ~9s. You're pretty close.