In TCP socket program,client send some data, but server need read multiple times. why?

Question

I have a question about socket.I send N-size data from client to server, N-size less than 100 byte.So I think my data should not be split to multiple tcp packet.In my opinion, Client send data should be done at one times and Server can receive data at one time.But The result is not satisfactory.Real situation is the server need call read data.I don't understand it.Follow code:

epoll_server.cpp(only receive data.)

#include <sys/socket.h>
#include <stdio.h>
#include <string.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/epoll.h>
#include <fcntl.h>
#include <netdb.h>

#define BUFSIZE 1024
#define INITSIZE 1024
#define MAXEVENTCOUNT 10240

// add non-blocking to sockfd
int make_socket_non_blocking(int fd)
{
    // get initial flag
    int src_flags;
    src_flags= fcntl(fd, F_GETFL,0);           
    if(src_flags == -1)
    {                    
        perror("fcntl get error.");
        return-1;
    }

    // add non-blocking
    int new_flags = src_flags | O_NONBLOCK;
    int ret_value;
    ret_value = fcntl(fd, F_SETFL, new_flags);
    if(ret_value == -1)
    {
        perror("fcntl set error.");
        return-1;
    }

    return 0;
}


// main function
int main(int argc, char* argv[])
{
    int server_sockfd, client_sockfd;
    int server_len;
    struct sockaddr_in server_address;

    // create server socket fd
    server_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    // init server address struct
    bzero(&server_address, sizeof(server_address));
    server_address.sin_family = AF_INET;
    server_address.sin_port = htons(9567);
    server_address.sin_addr.s_addr = INADDR_ANY;
    server_len = sizeof(server_address);

    // bind server address info for server fd
    if((bind(server_sockfd, (struct sockaddr*)&server_address, server_len)) == -1)
    {
        perror("bind error");
        exit(EXIT_FAILURE);
    }

    // let server is listened state
    listen(server_sockfd, 5);
    printf("server start waiting for connect...\r\n");

    // only suggestion
    int efd = epoll_create(INITSIZE);
    if(-1 == efd)
    {
        printf("epoll_create error happen.\n");
        return -1;
    }

    // set server_sockfd
    struct epoll_event server_event, event;
    server_event.data.fd = server_sockfd;
    server_event.events = EPOLLIN | EPOLLET;
    int ret_epollctl = epoll_ctl(efd, EPOLL_CTL_ADD, server_sockfd, &server_event);
    if(-1 == ret_epollctl)
    {
        printf("epoll_ctl error happen when efd is adding server_sockfd.\n");
        return -1;
    }

    /* event loop */
    struct epoll_event* return_events;
    // set timeout is 3000 ms
    int timeout_msecond = 3000;    
    return_events = (struct epoll_event*)malloc(MAXEVENTCOUNT*sizeof(struct epoll_event));
    int count = 0;
    while(1)
    {
        int ret_epollwait = epoll_wait(efd, return_events, MAXEVENTCOUNT, timeout_msecond);
        // part_1:epoll_wait error happen
        if(-1 == ret_epollwait)
        {
            printf("logged epoll_wait error happen.\n");
            continue;
        }
        // part_2:epoll_wait timeout
        if(0 == ret_epollwait)
        {
            printf("logged epoll_wait timeout.\n");
            continue;
        }
        // part_3:do some other event
        int index = 0;
        for(index = 0; index < MAXEVENTCOUNT; index++)
        {
            // part_3-1:hup ...
            if((return_events[index].events & EPOLLERR)
             || (return_events[index].events & EPOLLHUP)
             || !(return_events[index].events & EPOLLIN) )
            {                
                continue;
            }

            // part_3-2:is connection
            if(return_events[index].data.fd == server_sockfd)
            {
                struct sockaddr_in client_address;
                int client_len = sizeof(client_address);
                // server accept connection from client
                int client_sockfd = accept(server_sockfd, (struct sockaddr*)&client_address, (socklen_t*)&client_len);
                // part_3-2-1:connection error happen
                if(-1 == client_sockfd)
                {
                    if((EAGAIN == errno)
                     || (EWOULDBLOCK == errno) )
                    {                        
                        continue;
                    }
                    else
                    {
                        printf("accept error occured.\n");
                        continue;
                    }
                }
                else // part_3-2-2:normal connection
                {
                    // get clinet some information
                    char hostinfo_buf[BUFSIZE] = {0};
                    char servname_buf[BUFSIZE] = {0};
                    int tmp_ret = getnameinfo((struct sockaddr*)&client_address, client_len, hostinfo_buf, sizeof(hostinfo_buf), servname_buf, sizeof(servname_buf), NI_NUMERICHOST| NI_NUMERICSERV);
                    if(0 == tmp_ret)
                    {
                        printf("Accepted connection on descriptor %d:ip=%s, port=%s.\n", client_sockfd, hostinfo_buf, servname_buf);
                    }
                    // set client_sockfd to non-blocking
                    tmp_ret = make_socket_non_blocking(client_sockfd);
                    if(-1 == tmp_ret)
                    {
                        printf("set client_sockfd=%d to non-blocking error occured.\n", client_sockfd);
                        abort();
                    }

                    // set client_sockfd is EPOLLIN, EPOLLET
                    event.data.fd = client_sockfd;
                    event.events = EPOLLIN | EPOLLET;
                    tmp_ret = epoll_ctl(efd, EPOLL_CTL_ADD, client_sockfd, &event);
                    if(tmp_ret == -1)
                    {
                        printf("efd add %d has a error.\n", client_sockfd);
                        continue;
                    }
                    printf("add descriptor %d:ip=%s, port=%s successfully.\n", client_sockfd, hostinfo_buf, servname_buf);
                }
                continue;
            }

            // part_3-3:read data from client            
            printf("read data start++++\n");
            int temp = 0;

            // get recv_cache size start
            int recvsize = 0;
            socklen_t optlen = sizeof(recvsize);
            int err = getsockopt(return_events[index].data.fd, SOL_SOCKET, SO_RCVBUF, &recvsize, &optlen);
            printf("recv cache size :%d\n", recvsize);
            // get recv_cache size end

            while(1) // start while(1)
            {
                printf("%d times read data\n", ++temp);
                char* recv_buffer = (char*)malloc(1024+1);
                memset(recv_buffer, 0, 1025);
                // int ret_read = read(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer));
                int ret_read = recv(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer), 0);
                // part_3-3-1:read return error
                if(-1 == ret_read)
                {
                    if(EAGAIN != errno)
                    {
                        printf("read data from %d error occured, errno=%d, %s.\n", return_events[index].data.fd, errno, strerror(errno));
                    }
                    break;
                }
                // part_3-3-2:no data
                if(0 == ret_read)
                {
                    continue;
                }
                // part_3-3-3:output data. If data is 'bye', connection will close.
                if(ret_read > 0)
                {
                    printf("%d client's data:size=%dbyte, content=%s\n", return_events[index].data.fd, ret_read, recv_buffer);
                    // part_3-3-3-1:close connection and remove client_sockfd
                    if((recv_buffer[0] == 'b') 
                    && (recv_buffer[1] == 'y')
                    && (recv_buffer[2] == 'e') )
                    {

                        close(return_events[index].data.fd);
                        printf("close %d, ", return_events[index].data.fd);
                        int tmp_ret = epoll_ctl(efd, EPOLL_CTL_DEL, return_events[index].data.fd, NULL);
                        if(tmp_ret == -1)            
                        {     
                            printf("efd del %d has a error.\n", client_sockfd);        
                        }     
                        printf("remove descriptor %d successfully.\n", return_events[index].data.fd);
                    }
                }
            } // end of while(1)

            printf("read data finish------\n");
        }
    }

    free(return_events);
    // close server_sockfd
    shutdown(server_sockfd, 2);

    return 0;
}

epoll_client.cpp(only send data.)

#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>

#define BUFSIZE 1024

int main(int argc, char* argv[])
{
    int sock_clientfd, ret_recvsize, i;
    struct sockaddr_in dest, mine;
    char send_buffer[BUFSIZE + 1];

    // create socket fd
    if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
    {
        perror("Socket");
        exit(EXIT_FAILURE);
    }

    // init server address that client will connetct to.
    bzero(&dest, sizeof(dest));
    dest.sin_family = AF_INET;
    dest.sin_port = htons(9567);
    if(argc != 2)
    {
        printf("Usage: %s <dest ip>\n", argv[0]);
        printf("Usage: %s 127.0.0.1\n", argv[0]);
        return -1;
    }

    printf("-----\n");

    if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
    {
        perror(argv[1]);
        exit(1);
    }

    // connect to server
    printf("will connect!\n");
    if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
    {
        perror("Connect ");
        exit(EXIT_FAILURE);
    }

    while(1)
    {
        bzero(send_buffer, BUFSIZE + 1);
        printf("input message:");
        fgets(send_buffer, BUFSIZE, stdin);
        send_buffer[strlen(send_buffer) - 1] = '\0';
        printf("%d\n", strlen(send_buffer));
        int send_retsize = send(sock_clientfd, send_buffer, strlen(send_buffer), 0);
        if(send_retsize == -1)
        {                                
            perror("send data to client error happen!");
            exit(EXIT_FAILURE);          
        }
        printf("send succ data:%s\n", send_buffer);
        if((send_buffer[0] == 'b')
         && (send_buffer[1] == 'y')
         && (send_buffer[2] == 'e') ) 
        {
            printf("client active close connect.\n");
            break;
        }
    }

    // close sock_clientfd
    close(sock_clientfd);

    return 0;
}

Follow pircture is some run info: epoll_server.png

epoll_client.png

The server read data is only 8 byte, Is the kernel design epoll is this? I guess the reasons are as follows pirture:

Answer 1

The reason you don't receive everything that is available in one read is because you only read 8 bytes at a time.

char* recv_buffer = (char*)malloc(1024+1);
int ret_read = recv(return_events[index].data.fd, recv_buffer, sizeof(recv_buffer), 0);
            // part_3-3-1:read return error

recv_buffer is a char* not an array, so sizeof recv_buffer equals the size of a pointer which in your case is 8 .

Note that you should never rely on data arriving in packages. If your message protocol states that you should be getting 10 bytes never expect all 10 bytes to be available at once. You should always code in a way that can handle data being split up into multiple reads.

If the thread handles a single socket then a simple do { read... } while (total_bytes_received < expected_bytes); will suffice.

If the thread handles multiple connections, then you need to save the bytes you have read and then continue to manage other sockets that are ready before returning to your handling loop that will use select/epoll to wait for more data.