[英]SSL_accept with edge triggered non-blocking epoll always returns SSL_ERROR_WANT_READ
There are 3 of us working on adding SSL to epoll based server but after 3 days of pull-your-hair out frustration we decided to ask the world for help. 我们中有3个人致力于将SSL添加到基于epoll的服务器中,但是经过3天的挫败之后,我们决定向世界寻求帮助。
The problem is that SSL_accept()
always returns SSL_ERROR_WANT_READ
, we DO have epoll events for EPOLLIN and EPOLLOUT for the associated socket but even when we get an event and recall the SSL_accept
it just returns the want read.... so frustrating. 问题在于
SSL_accept()
总是返回SSL_ERROR_WANT_READ
,我们确实为关联套接字的EPOLLIN和EPOLLOUT设置了epoll事件,但是即使当我们得到一个事件并回想SSL_accept
它也只返回了想要读取的内容。
There seems to be no good working example of using openssl with epoll that we can find, PLEASE send us to code samples if there are any, we have spent days on the this problem and read about every SE/SO message and spent hours on the openssl mailing list. 似乎没有找到可以在epoll上使用openssl的很好的工作示例,请发送给我们代码样本(如果有的话),我们花了几天时间解决此问题,并阅读了每条SE / SO消息,并花了几个小时在openssl邮件列表。
We have a basic app structure that looks like this for the accept: 我们有一个基本的应用程序结构,如下所示:
if (http_sock == source_fd || https_sock == source_fd) {
// new connection
req_type_t req_type = (http_sock == source_fd) ? HTTP : HTTPS;
int infd;
while (1) {
struct sockaddr_in in_addr;
socklen_t in_addr_len;
ZLOG_DBG(zlog_cat, "New %s request on source_fd: %d http_sock: %d https_sock: %d", req_type == HTTP ? "HTTP" : "HTTPS", source_fd, http_sock, https_sock);
in_addr_len = sizeof in_addr;
infd = accept4(source_fd, &in_addr, &in_addr_len, SOCK_NONBLOCK);
if (infd <= 0)
{
if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
/* We have processed all incoming
connections. */
break;
} else {
ZLOG_ERR(zlog_cat, "ERROR: accept");
break;
}
}
req = new_req_data(kds, req_type, &g_data, infd, &in_addr, in_addr_len, cert_q);
event.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
if (req->type == HTTPS) {
req->c_tls.want_ssl_accept = 1;
event.events = EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLET;
ssl_establish(req);
}
if ( (s = set_socket_blocking(req->client.fd, 1)) < 0)
err(EXIT_FAILURE, "[%s:%d] set_socket_blocking()", __FUNCTION__, __LINE__);
epoll_event_t *epoll_event = new_epoll_event_t(infd, req);
event.data.ptr = epoll_event;
// add new infd to our epoll listener
if ( (s = epoll_ctl(efd, EPOLL_CTL_ADD, req->client.fd, &event)) == -1 )
err(EXIT_FAILURE, "epoll_ctl(http) %s", strerror(errno));
}
ZLOG_DBG(zlog_cat, "New %s connection from %s:%d on FD %d mac_address: %s", req->type ? "HTTPS" : "HTTP",
inet_ntoa(req->client.sock.sin_addr), ntohs(req->client.sock.sin_port), req->client.fd, req->device->mac_address);
g_data.conn_counter++;
continue; // continue to next accept event
} // end new connection
The ssl_establish()
method looks like this: ssl_establish()
方法如下所示:
int ssl_establish(req_data_t *req) {
unsigned long e;
int ssl_err, rc;
// create openssl server context and ssl object
if ((req->s_tls.ctx = create_context(0)) == NULL) {
ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
return -1;
}
if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
return -1;
}
// Establishing SSL/TLS connections with client
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
req->c_tls.servername = req->https_def_domain;
req->c_tls.cert = req->https_def_cert;
req->c_tls.rsa_key = req->https_def_rsa;
req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
req->c_tls.dc_cache = req->dc_cache;
req->c_tls.q_entry = NULL;
if (req->c_tls.ctx == NULL ) {
if ((req->c_tls.ctx = create_context(1)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR create_context");
if (configure_context(&req->c_tls))
ZLOG_ERR(zlog_cat, "ERROR configure_context");
if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR ssl_new");
// SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); // This doesn't seem to do anything
if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");
}
req->client.ssl = req->c_tls.ssl;
}
if ((rc = SSL_accept(req->c_tls.ssl)) < 0) {
ssl_err = SSL_get_error(req->c_tls.ssl, rc);
switch (ssl_err) {
case SSL_ERROR_WANT_READ:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_READ");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_WANT_WRITE:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_WRITE");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_ZERO_RETURN:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_ZERO_RETURN");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_WANT_X509_LOOKUP:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_WANT_X509_LOOKUP");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SYSCALL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SYSCALL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SSL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SSL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
}
}
ZLOG_DBG(zlog_cat, "SSL_accept success");
req->c_tls.want_ssl_accept = 0;
return 1;
}
In the pollin / pollout handlers we do this: 在花粉/花粉处理程序中,我们这样做:
if (req->type == HTTPS && req->c_tls.want_ssl_accept) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN");
if ((rc = ssl_establish(req)) <= 0) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN retry...");
continue;
}
}
Our log looks like this: 我们的日志如下所示:
2016-08-03 23:26:32.224 New HTTPS connection from 192.168.1.195:52659 on FD 15 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 epoll_wait returned 1
2016-08-03 23:26:32.224 epoll event number: 0 on fd: 15
2016-08-03 23:26:32.224 Reading data from Client, source_fd: 15 source_sock->fd: 15 dest_sock->fd: -1 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 ssl_establish EPOLLIN
2016-08-03 23:26:32.484 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.484 ssl_establish EPOLLIN retry...
2016-08-03 23:26:32.954 New HTTPS connection from 192.168.1.195:52660 on FD 16 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.954 epoll_wait returned 1
2016-08-03 23:26:32.954 epoll event number: 0 on fd: 16
2016-08-03 23:26:32.954 Establishing SSL (EPOLLOUT)
2016-08-03 23:26:32.974 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.974 ssl_establish EPOLLOUT retry...
Okay - I finally got it working. 好的-我终于可以使用了。 If anyone stumbles across this, don't use SSL_accept().
如果有人偶然发现此问题,请不要使用SSL_accept()。
I added some state tracking to my main struct and the updated the ssl_establish()
method to look like this: 我在主结构中添加了一些状态跟踪,并更新了
ssl_establish()
方法,如下所示:
int ssl_establish(req_data_t *req, epoll_event_t *epoll_event ) {
unsigned long e;
int ssl_err, rc;
struct epoll_event event;
memset(&event, 0, sizeof(struct epoll_event));
event.events = req->client.events;
event.data.ptr = epoll_event;
if (!req->client.tcp_connected) {
struct pollfd pfd;
pfd.fd = req->client.fd;
pfd.events = POLLOUT | POLLERR;
int r = poll(&pfd, 1, 0);
if (r == 1 && pfd.revents == POLLOUT) {
ZLOG_DBG(zlog_cat, "tcp connected fd %d", req->client.fd);
req->client.tcp_connected = 1;
req->client.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET;
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
} else {
ZLOG_ERR(zlog_cat, "[%d | %d] ERROR poll fd return %d revents %d", req->client.fd, req->state, r, pfd.revents);
return -1;
}
}
// create openssl server context and ssl object
if (!req->s_tls.ctx)
if ((req->s_tls.ctx = create_context(0)) == NULL) {
ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
return -1;
}
if (!req->s_tls.ssl)
if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
return -1;
}
req->orig.ssl = req->s_tls.ssl;
req->proxy.ssl = req->s_tls.ssl;
// Establishing SSL/TLS connections with client
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
req->c_tls.servername = req->https_def_domain;
req->c_tls.cert = req->https_def_cert;
req->c_tls.rsa_key = req->https_def_rsa;
req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
req->c_tls.kudoso = req->kudoso;
req->c_tls.dc_cache = req->dc_cache;
req->c_tls.q_entry = NULL;
if (!req->errBio)
req->errBio = BIO_new_fd(2, BIO_NOCLOSE);
if (!req->c_tls.ctx) {
if ((req->c_tls.ctx = create_context(1)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR create_context");
if (configure_context(&req->c_tls))
ZLOG_ERR(zlog_cat, "ERROR configure_context");
}
if (!req->c_tls.ssl) {
if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR ssl_new");
// SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");
SSL_set_accept_state(req->c_tls.ssl);
req->client.ssl = req->c_tls.ssl;
}
}
int r = SSL_do_handshake(req->c_tls.ssl);
if (r == 1) {
req->client.ssl_connected = 1;
ZLOG_DBG(zlog_cat, "[%d | %d] ssl connected", req->client.fd, req->state);
req->client.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
return 1;
}
int err = SSL_get_error(req->c_tls.ssl, r);
int oldev = req->client.events;
if (err == SSL_ERROR_WANT_WRITE) {
req->client.events |= EPOLLOUT;
req->client.events &= ~EPOLLIN;
ZLOG_DBG(zlog_cat, "do_handshake return want write set events %d", req->client.events);
if (oldev == req->client.events) return 0;
} else if (err == SSL_ERROR_WANT_READ) {
req->client.events |= EPOLLIN;
req->client.events &= ~EPOLLOUT;
ZLOG_DBG(zlog_cat, "do_handshake return want read set events %d", req->client.events);
if (oldev == req->client.events) return 0;
} else {
ZLOG_ERR(zlog_cat, "ERROR SSL_do_handshake return %d error %d errno %d msg %s", r, err, errno, strerror(errno));
ERR_print_errors(req->errBio);
return -1;
}
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "[%d | %d] ERROR: unable to modify epoll_ctl", req->client.fd, req->state);
return 0;
}
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.