[英]SocketServer rfile.read() very very slow
I am building an HTTPS proxy which can forward all SSL traffic. 我正在构建一个可以转发所有SSL流量的HTTPS代理。 It is called a transparent tunneling.
它被称为透明隧道。 Anyway, I have a problem with Python's socketserver.
无论如何,我的Python socketserver有问题。 When I called rfile.read(), it takes a very long time to return.
当我调用rfile.read()时,返回需要很长时间。 Even I used select to make sure the I/O is ready, it still takes a very long time.
即使我使用select来确保I / O准备就绪,它仍然需要很长时间。 Usually 30s and the client's socket is closed because of timeout.
通常30秒,客户端的套接字因超时而关闭。 I cannot make the socket unblocking, because I need to read the data first, and then forward all the data I just read to remote server, it must be returned with data first.
我无法使套接字解除阻塞,因为我需要先读取数据,然后将刚读取的所有数据转发到远程服务器,首先必须返回数据。
My code is following: 我的代码如下:
import SocketServer
import BaseHTTPServer
import socket
import threading
import httplib
import time
import os
import urllib
import ssl
import copy
from history import *
from http import *
from https import *
from logger import Logger
DEFAULT_CERT_FILE = "./cert/ncerts/proxpy.pem"
proxystate = None
class ProxyHandler(SocketServer.StreamRequestHandler):
def __init__(self, request, client_address, server):
self.peer = False
self.keepalive = False
self.target = None
self.tunnel_mode = False
self.forwardSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Just for debugging
self.counter = 0
self._host = None
self._port = 0
SocketServer.StreamRequestHandler.__init__(self, request, client_address, server)
def createConnection(self, host, port):
global proxystate
if self.target and self._host == host:
return self.target
try:
# If a SSL tunnel was established, create a HTTPS connection to the server
if self.peer:
conn = httplib.HTTPSConnection(host, port)
else:
# HTTP Connection
conn = httplib.HTTPConnection(host, port)
except HTTPException as e:
proxystate.log.debug(e.__str__())
# If we need a persistent connection, add the socket to the dictionary
if self.keepalive:
self.target = conn
self._host = host
self._port = port
return conn
def sendResponse(self, res):
self.wfile.write(res)
def finish(self):
if not self.keepalive:
if self.target:
self.target.close()
return SocketServer.StreamRequestHandler.finish(self)
# Otherwise keep-alive is True, then go on and listen on the socket
return self.handle()
def handle(self):
global proxystate
if self.keepalive:
if self.peer:
HTTPSUtil.wait_read(self.request)
else:
HTTPUtil.wait_read(self.request)
# Just debugging
if self.counter > 0:
proxystate.log.debug(str(self.client_address) + ' socket reused: ' + str(self.counter))
self.counter += 1
if self.tunnel_mode:
HTTPUtil.wait_read(self.request)
print "++++++++++++++++++++++++++"
req=self.rfile.read(4096) #This is the line take very lone time!
print "----------------------------------"
data = self.doFORWARD(req)
print "**************************************"
if len(data)!=0:
self.sendResponse(data)
return
try:
req = HTTPRequest.build(self.rfile)
except Exception as e:
proxystate.log.debug(e.__str__() + ": Error on reading request message")
return
if req is None:
return
# Delegate request to plugin
req = ProxyPlugin.delegate(ProxyPlugin.EVENT_MANGLE_REQUEST, req.clone())
# if you need a persistent connection set the flag in order to save the status
if req.isKeepAlive():
self.keepalive = True
else:
self.keepalive = False
# Target server host and port
host, port = ProxyState.getTargetHost(req)
if req.getMethod() == HTTPRequest.METHOD_GET:
res = self.doGET(host, port, req)
self.sendResponse(res)
elif req.getMethod() == HTTPRequest.METHOD_POST:
res = self.doPOST(host, port, req)
self.sendResponse(res)
elif req.getMethod() == HTTPRequest.METHOD_CONNECT:
res = self.doCONNECT(host, port, req)
def _request(self, conn, method, path, params, headers):
global proxystate
conn.putrequest(method, path, skip_host = True, skip_accept_encoding = True)
for header,v in headers.iteritems():
# auto-fix content-length
if header.lower() == 'content-length':
conn.putheader(header, str(len(params)))
else:
for i in v:
conn.putheader(header, i)
conn.endheaders()
if len(params) > 0:
conn.send(params)
def doRequest(self, conn, method, path, params, headers):
global proxystate
try:
self._request(conn, method, path, params, headers)
return True
except IOError as e:
proxystate.log.error("%s: %s:%d" % (e.__str__(), conn.host, conn.port))
return False
def doCONNECT(self, host, port, req):
#global proxystate
self.tunnel_mode = True
self.tunnel_host = host
self.tunnel_port = port
#socket_req = self.request
#certfilename = DEFAULT_CERT_FILE
#socket_ssl = ssl.wrap_socket(socket_req, server_side = True, certfile = certfilename,
#ssl_version = ssl.PROTOCOL_SSLv23, do_handshake_on_connect = False)
HTTPSRequest.sendAck(self.request)
#print "Send ack to the peer %s on port %d for establishing SSL tunnel" % (host, port)
print "into forward mode: %s : %s" % (host, port)
'''
host, port = socket_req.getpeername()
proxystate.log.debug("Send ack to the peer %s on port %d for establishing SSL tunnel" % (host, port))
while True:
try:
socket_ssl.do_handshake()
break
except (ssl.SSLError, IOError) as e:
# proxystate.log.error(e.__str__())
print e.__str__()
return
# Switch to new socket
self.peer = True
self.request = socket_ssl
'''
self.setup()
#self.handle()
def doFORWARD(self, data):
host, port = self.request.getpeername()
#print "client_host", host
#print "client_port", port
try:
print "%s:%s===>data read, now sending..."%(host,port)
self.forwardSocket.connect((self.tunnel_host,self.tunnel_port))
self.forwardSocket.sendall(data)
print data
print "%s:%s===>sent %d bytes to server"%(host,port,len(data))
select.select([self.forwardSocket], [], [])
chunk = self.forwardSocket.recv(4096)
print chunk
print "%s:%s===>receive %d bytes from server"%(host,port,len(chunk))
return chunk
except socket.error as e:
print e.__str__()
return ''
class ThreadedHTTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
allow_reuse_address = True
class ProxyServer():
def __init__(self, init_state):
global proxystate
proxystate = init_state
self.proxyServer_port = proxystate.listenport
self.proxyServer_host = proxystate.listenaddr
def startProxyServer(self):
global proxystate
self.proxyServer = ThreadedHTTPProxyServer((self.proxyServer_host, self.proxyServer_port), ProxyHandler)
# Start a thread with the server (that thread will then spawn a worker
# thread for each request)
server_thread = threading.Thread(target = self.proxyServer.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.setDaemon(True)
proxystate.log.info("Server %s listening on port %d" % (self.proxyServer_host, self.proxyServer_port))
server_thread.start()
while True:
time.sleep(0.1)
This is driving me crazy, because when I forward normal HTTP traffic, the read() returns immediately with the data. 这让我很生气,因为当我转发正常的HTTP流量时,read()会立即返回数据。 But every time when the client sent SSL traffic (binary), the read() takes very long time to return!
但每当客户端发送SSL流量(二进制)时,read()需要很长时间才能返回! And I think there is some mechanics that the read() only returns when the request socket is closed by remote client.
我认为有一些机制,read()仅在远程客户端关闭请求套接字时返回。 Does anyone know how to make the read() fast?
有谁知道如何快速读取()? I tried recv() and readline(), both of them is as slow as read() when handling binary traffic!
我尝试了recv()和readline(),它们在处理二进制流量时和read()一样慢!
I had similar problem on my python server while trying to send image to it via http. 我尝试通过http向它发送图像时,我的python服务器上遇到了类似的问题。
rfile.read took 1700ms for a 800kb image and 4500ms for a 4.5mb image. 对于800kb图像,rfile.read需要1700ms,对于4.5mb图像需要4500ms。 I was also doing this through a limited network speed on a vpn.
我也是通过vpn上有限的网络速度来做到这一点的。
It appears that the rfile.read downloads/uploads the file. 似乎rfile.read下载/上传文件。 By improving connection/network speed between client-server i have reduced 1700ms read to less than 50ms.
通过提高客户端 - 服务器之间的连接/网络速度,我将读取的时间减少了1700ms,小于50ms。
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.