AF_UNIX套接字：没有任何要读取的内容时会触发select（）

Question

The program creates n threads for simulating n nodes in a distributed system, each has one socket it listens to and each thread can communicate to n-1 other threads through connect calls.

1. Each thread makes a call to select() to see if anything's available and if so accepts and saves the data.

2. I use ioctl with the flag FIONREAD to check the number of bytes available to be read and do an appropriate read call. Afterwards the new fd (from accept()) is closed.

3. The listening sockets are blocking. O_NONBLOCK is NOT set.

4. All n threads run the same function. All variables declared within the function use thread local storage.

5. There's no explicit synchronization done on my part. More than one thread can try to connect to the same socket at once.

Now, the problem is, once in a while, the select() in a thread on the receiving side notes something new but the amount of bytes available is 0 which it shouldn't be. This happens inconsistently.

Would be great if someone can point where I should look into. Thanks!

creating the sock

  if ( (nptr->sock = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
                perror("socket");
                exit(1);
            }
            fd_max = nptr->sock > fd_max ? nptr->sock : fd_max;

            int ok=1;
            setsockopt(nptr->sock, SOL_SOCKET, SO_REUSEADDR, &ok, sizeof(int));

            nptr->addr.sun_family = AF_UNIX;
            snprintf(nptr->addr.sun_path, 20, "%d", nptr->id);
            //strncpy(nptr->addr.sun_path, sock_path, 20);

            if ( bind(nptr->sock, (struct sockaddr*)&(nptr->addr), sizeof(struct sockaddr_un)) < 0 ) {
                perror("bind");
                exit(1);
            }
            /* socket,  max connections */
            if ( listen(nptr->sock, 2*tot_node) < 0 ) {
                perror("listen");
                exit(1);
            }

sending stuff

for (t=0; t<tot_node; t++) {
            ...

            if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
                perror("socket");
                exit(1);
            }

            printf("Node %d: trying to req node %d... ", self->id, node_catalog[t]->id);
            if ( connect(fd, (struct sockaddr*)&(node_catalog[t]->addr), sizeof(struct sockaddr_un)) == -1 ) {
                perror("connect");
                exit(1);
            }

            buf[0] = TYPE_REQ;
            buf[1] = self->id;
            buf[2] = ts;
            buf[3] = rsc;

            write (fd, buf, 4*sizeof(int));

            //close(fd);

            printf("Node %d: sent req for resource %d to %d\n", self->id, rsc, node_catalog[t]->id);
        }
        usleep(TS_UPDATE_ITV);

receiving stuff

    FD_ZERO(&readset);
            FD_SET(self->sock, &readset);
            t = pselect(self->sock+1, &readset, NULL, NULL, &tout, NULL);
            if (t > 0 && FD_ISSET(self->sock, &readset)) {

                com_fd = accept(self->sock, NULL, NULL);

                ioctl(com_fd, FIONREAD, &t);
    #ifdef DEBUG
                printf("   Node %d: received %d bytes of data\n", self->id, t);
    #endif

                read(com_fd, buf, t);
                close(com_fd);

                dptr = (int *)buf;
                rsc = t / (sizeof(int)); /* var reuse. this is the count of ints to read */

                for (t=0; t<rsc; ) {
                    static __thread int nid, nts, nrsc;
    #ifdef DEBUG
                    printf("   Node %d: data rcvd: %d %d %d %d", self->id, *dptr, *(dptr+1), *(dptr+2), *(dptr+3));
    #endif

                   if (*dptr == TYPE_REQ) {
... } else {...}

Answer 1

Your code doesn't make sense. The reason that select() fired was that there was something to accept . Checking FIONREAD on a socket you've just accepted may or may not result in data being available. It depends entirely on whether the client has sent any. Not on the contract of select() .

If you need to know whether there is something to read, you should add the accepted socket to the read-FD set, and process it in a loop: if the listening socket is readable, call accept() on it, otherwise it is an accepted socket and you should call read() on it.

Checking FIONREAD is really just a waste of time in most circumstances.