在Linux上測量sendfile API

Question

我編寫了一個簡單的文件復制應用程序，以評估使用sendfile API的有效性，而不是針對大文件的常規“從文件讀取並寫入套接字”方法。 但是，使用這兩種方法運行應用程序時，我發現兩種方法之間完成文件復制所花費的時間差異非常小。

我從多個來源了解到，“發送文件” API與常規的“從文件讀取並寫入套接字”方法相比，將大大提高性能。 但是，當我嘗試使用單個2GB文件進行基准測試時，以下是我觀察到的數字（4次迭代的平均值）：

1. 普通的從文件讀取並寫入套接字的方法：17秒444840 usecs
2. sendfile API：17秒431420個使用情況

我在隔離的1Gbps網絡中的兩台不同機器（Linux內核版本4.4.162-94.72-default）上同時運行應用程序的服務器和客戶端。

有人可以幫我在這里做錯什么或在這里丟失什么嗎？

服務器：

#define _GNU_SOURCE

#include "file_details.h"

void calculate_execution_time(struct timeval start, struct timeval end)
{
    struct timeval  time_diff;


    time_diff.tv_sec = end.tv_sec - start.tv_sec;
    time_diff.tv_usec = end.tv_usec - start.tv_usec;

    // Adjust the time appropriately
    while (time_diff.tv_usec < 0) {
        time_diff.tv_sec--;
        time_diff.tv_usec += 1000000;
    }

    printf("total execution time: = %lds.%ldus\n", time_diff.tv_sec, time_diff.tv_usec);
}


int read_from_file_pread(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
    ssize_t         bytes_read = 0, bytes_sent = 0, total_bytes_sent = 0, bytes_sent_this_itr = 0;
    off_t           offset = 0;
    char            *buffer = NULL;
    struct timeval      start_time, end_time;


    buffer = calloc(chunk_size, sizeof(char));
    if (buffer == NULL) {
        printf("Failed to allocate memory of size: %d bytes\n", chunk_size);
        return -1;
    }

    gettimeofday(&start_time, NULL);

    do {
        bytes_read = pread(fd, buffer, chunk_size, offset);
        switch (bytes_read) {
            case -1:
                printf("Failed to read from file: %s, offset: %lu, error: %d\n", file_name, offset, errno);

                free(buffer);
                return -1;

            case 0:
                printf("Completed reading from file and sending\n");
                break;

            default:
                do {
                    bytes_sent = send(client_sockfd, buffer, (bytes_read - bytes_sent_this_itr), 0);
                    if (bytes_sent == -1) {
                        printf("Failed to send %lu bytes, error: %d\n", (bytes_read - bytes_sent_this_itr), errno);

                        free(buffer);
                        return -1;
                    }

                    bytes_sent_this_itr += bytes_sent;
                } while (bytes_sent_this_itr < bytes_read);

                bytes_sent = 0;
                bytes_sent_this_itr = 0;
                offset += bytes_read;
                total_bytes_sent += bytes_read;
                break;
        }
    } while (total_bytes_sent < file_size_in_bytes);

    gettimeofday(&end_time, NULL);

    printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);

    calculate_execution_time(start_time, end_time);

    free(buffer);
    return 0;
}


int read_from_file_sendfile(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
    ssize_t         bytes_sent = 0, total_bytes_sent = 0;
    off_t           offset = 0;
    struct timeval      start_time, end_time;


    gettimeofday(&start_time, NULL);

    do {
        bytes_sent = sendfile(client_sockfd, fd, &offset, chunk_size);
        if (bytes_sent == -1) {
            printf("Failed to sendfile: %s, offset: %lu, error: %d\n", file_name, offset, errno);
            return -1;
        }

        total_bytes_sent += bytes_sent;
    } while (total_bytes_sent < file_size_in_bytes);

    gettimeofday(&end_time, NULL);

    printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);

    calculate_execution_time(start_time, end_time);

    return 0;
}


int read_from_file(int client_sockfd, char *file_name, char *type, int chunk_size)
{
    int         error_code = 0, fd = 0;
    ssize_t         hdr_length = 0, bytes_sent = 0, file_name_length = strlen(file_name);
    struct stat     file_stat = {0};
    struct file_details *file_details_to_send = NULL;


    fd = open(file_name, O_RDONLY, S_IRUSR);
    if (fd == -1) {
                printf("Failed to open file: %s, error: %d\n", file_name, errno);
                return -1;
    }

    error_code = fstat(fd, &file_stat);
    if (error_code == -1) {
                printf("Failed to get status of file: %s, error: %d\n", file_name, errno);

        close(fd);
        return -1;
    }

    hdr_length = (sizeof(struct file_details) + file_name_length + 1);
    file_details_to_send = calloc(hdr_length, sizeof(char));
    if (file_details_to_send == NULL) {
        perror("Failed to allocate memory");

        close(fd);
        return -1;
    }

    file_details_to_send->file_name_length = file_name_length;
    file_details_to_send->file_size_in_bytes = file_stat.st_size;
    strcpy(file_details_to_send->file_name, file_name);

    printf("File name: %s, size: %lu bytes\n", file_name, file_stat.st_size);

    bytes_sent = send(client_sockfd, file_details_to_send, hdr_length, 0);
    if (bytes_sent == -1) {
        printf("Failed to send header of size: %lu bytes, error: %d\n", hdr_length, errno);

        close(fd);
        return -1;
    }

    if (strcmp(type, "rw") == 0) {
        printf("By pread and send\n");

        read_from_file_pread(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
    } else {
        printf("By sendfile\n");

        read_from_file_sendfile(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
    }

    close(fd);
    return 0;
}


int main(int argc, char *argv[])
{
    ...
    ...

    option_value = 1;
    error_code = setsockopt(client_sockfd, SOL_TCP, TCP_NODELAY, &option_value, sizeof(int));
    if (error_code == -1) {
        printf("Failed to set socket option TCP_NODELAY to socket descriptor: %d, error: %d", client_sockfd, errno);
    }

    read_from_file(client_sockfd, file_name, type, chunk_size);

    ...
}

Answer 1

您的代碼幾乎可以肯定在性能上有了很大的提高。 問題可能是您正在測量掛牆時間。 考慮調用getrusage（）而不是gettimeofday（）。 ru_utime和ru_stime字段表示內核和程序在實際工作中花費了多少時間。 sendfile（）應該使這些數字下降。 這樣，您可以減少能源消耗，並為計算機上的其他程序釋放更多資源。 但是不幸的是，它不能使網絡運行得更快。 假設零開銷約為9秒，則在1GbPS以太網上發送2GB的最佳牆時間速度。 你很親密