我的 epoll 服务器丢失了一些连接。 为什么？

Question

I'd like to make an epoll server. But my code of the server losses some connections.

My client spawns 100 threads and each sends the same message. Then my server is supposed to receive and print them with counting numbers.

But the server seems like losing connections and I don't know why.

I changed EPOLL_SIZE from 50 to 200, and did backlog argument of listen() from 5 to 1000. But they didn't work.

1.server:

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <memory>
#include <array>

#define BUF_SIZE 100
#define EPOLL_SIZE 200
void error_handling(const char *buf);

int main(int argc, char *argv[])
{
   // Step 1. Initialization
   int server_socket, client_socket;
   struct sockaddr_in client_addr;
   socklen_t addr_size;
   int str_len, i;
   char buf[BUF_SIZE];

   int epfd, event_cnt;

   if (argc != 2) {
      printf("Usage : %s <port>\n", argv[0]);
      exit(1);
   }

   // Step 2. Creating a socket
   server_socket = socket(PF_INET, SOCK_STREAM, 0);

   struct sockaddr_in server_addr;
   memset(&server_addr, 0, sizeof(server_addr));
   server_addr.sin_family = AF_INET;
   server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
   server_addr.sin_port = htons(atoi(argv[1]));

   // Step 3. Binding the server address onto the socket created just right before.
   if (bind(server_socket, (struct sockaddr*) &server_addr, sizeof(server_addr)) == -1)
      error_handling("bind() error");

   // Step 4. Start to listen to the socket.
   if (listen(server_socket, 1000) == -1)
      error_handling("listen() error");

   // Step 5. Create an event poll instance.
   epfd = epoll_create(EPOLL_SIZE);
   auto epoll_events = (struct epoll_event*) malloc(sizeof(struct epoll_event) * EPOLL_SIZE);

   struct epoll_event event;
   event.events = EPOLLIN;
   event.data.fd = server_socket;

   // Step 6. Adding the server socket file descriptor to the event poll's control.
   epoll_ctl(epfd, EPOLL_CTL_ADD, server_socket, &event);
   int recv_cnt = 0;

   while(true)
   {
      // Step 7. Wait until some event happens
      event_cnt = epoll_wait(epfd, epoll_events, EPOLL_SIZE, -1);
      if (event_cnt == -1)
      {
         puts("epoll_wait() error");
         break;
      }

      for (i = 0; i < event_cnt; i++)
      {
         if (epoll_events[i].data.fd == server_socket)
         {
            addr_size = sizeof(client_addr);
            client_socket = accept(server_socket, (struct sockaddr*)&client_addr, &addr_size);
            event.events = EPOLLIN;
            event.data.fd = client_socket;
            epoll_ctl(epfd, EPOLL_CTL_ADD, client_socket, &event);
            //printf("Connected client: %d\n", client_socket);
         }
         else  // client socket?
         {
            str_len = read(epoll_events[i].data.fd, buf, BUF_SIZE);
            if (str_len == 0) // close request!
            {
               epoll_ctl(epfd, EPOLL_CTL_DEL, epoll_events[i].data.fd, nullptr);
               close(epoll_events[i].data.fd);
               printf("%d: %s\n", ++recv_cnt,  buf);
               //printf("closed client: %d \n", epoll_events[i].data.fd);
            }
            else
            {
               write(epoll_events[i].data.fd, buf, str_len);   // echo!
            }
         } // end of else()
      } // end of for()
   }  // end of while()

   close(server_socket);
   close(epfd);
   free(epoll_events);

   return EXIT_SUCCESS;
}

void error_handling(const char *buf)
{
   fputs(buf, stderr);
   fputc('\n', stderr);
   exit(EXIT_FAILURE);
}

2.client:

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <thread>
#include <vector>
#include <algorithm>
#include <mutex>

#define BUF_SIZE 100
#define EPOLL_SIZE 50

void error_handling(const char *buf);

int main(int argc, char *argv[])
{
   // Step 1. Initialization
   int socketfd;

   if (argc != 3) {
      printf("Usage : %s <ip address> <port>\n", argv[0], argv[1]);
      exit(EXIT_FAILURE);
   }
   std::vector<std::thread> cli_threads;
   std::mutex wlock;

   for (int i = 0; i < 100; i++) {
      cli_threads.push_back(std::thread([&](const char* szIpAddr, const char* szPort) {
         // Step 2. Creating a socket
         socketfd = socket(PF_INET, SOCK_STREAM, 0);

         struct sockaddr_in server_addr;
         memset(&server_addr, 0, sizeof(server_addr));
         server_addr.sin_family = AF_INET;
         server_addr.sin_addr.s_addr = inet_addr(szIpAddr);
         server_addr.sin_port = htons(atoi(szPort));

         // Step 3. Connecting to the server
         if(connect(socketfd, (struct sockaddr*)&server_addr, sizeof(server_addr)) == -1)
            error_handling("connect() error"); 

         // Step 4. Writing message to the server
         std::string msg = "Hey I'm a client!";
         wlock.lock();
         auto str_len = write(socketfd, msg.c_str(), msg.size()+1);
         wlock.unlock();

         close(socketfd); 
      }, argv[1], argv[2]));
   }

   std::for_each(cli_threads.begin(), cli_threads.end(), 
      [](std::thread &t)
      {
         t.join();
      }
   );
   return EXIT_SUCCESS;
}

void error_handling(const char *buf)
{
   fputs(buf, stderr);
   fputc('\n', stderr);
   exit(EXIT_FAILURE);
}

expected like...

1: Hey I'm a client!
...
100: Hey I'm a client!

but, the result varies, like...

1: Hey I'm a client!
...
n: Hey I'm a client!

where the n is less than 100.

Answer 1

You had undefined behaviour because of passing socketfd by reference to thread - std::thread([&](... . One instance of socket descriptor was being modified by all threads concurrently - it caused problems. Every thread should store its own descriptor.