OSError：[Errno 24]在Twisted中使用Reactor.run（）时打开的文件过多

Question

I am having a weird issue: I am running a large amount of utils.getProcessOutputAndValue('cmd', [args]) commands and the result depends on whether I started the reactor using task.react() or reactor.run()

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from progress.bar import IncrementalBar
from twisted.internet import defer
from twisted.internet import task
from twisted.internet import utils
from twisted.python import usage


class Options(usage.Options):
    optFlags = [['reactor', 'r', 'Use reactor.run().'],
                ['task', 't', 'Use task.react().'],
                ['cwr', 'w', 'Use callWhenRunning().']]
    optParameters = [['limit', 'l', 255, 'Number of file descriptors to open.'],
                     ['cmd', 'c', 'echo Testing {i}...', 'Command to run.']]


def run(opt):
    limit = int(opt['limit'])
    cmd, args = opt['cmd'].split(' ', 1)
    bar = IncrementalBar('Running {cmd}'.format(cmd=opt['cmd']), max=limit)
    requests = []
    for i in range(0, limit):
        try:
            _args = args.format(i=i)
            args = _args
        except KeyError:
            pass
        requests.append(utils.getProcessOutputAndValue('echo', [args]))
        bar.next()
    bar.finish()
    return defer.gatherResults(requests)


@defer.inlineCallbacks
def main(reactor, opt):
    d = defer.Deferred()
    limit = int(opt['limit'])
    cmd, args = opt['cmd'].split(' ', 1)
    bar = IncrementalBar('Running {cmd}'.format(cmd=opt['cmd']), max=limit)
    for i in range(0, limit):
        try:
            _args = args.format(i=i)
            args = _args
        except KeyError:
            pass
        yield utils.getProcessOutputAndValue('echo', [args])
        bar.next()
    bar.finish()
    defer.returnValue(d.callback(True))


if __name__ == '__main__':
    opt = Options()
    opt.parseOptions()

    if opt['reactor']:
        from twisted.internet import reactor
        task.deferLater(reactor, 0, run, opt)
        reactor.run()

    elif opt['task']:
        from twisted.internet.task import react
        react(main, [opt])

    elif opt['cwr']:
        from twisted.internet import reactor
        reactor.callWhenRunning(run, opt)
        reactor.run()

When using limit above 400 (in my case) I get the following error:

Upon execvpe echo ['echo', 'Testing 0...'] in environment id 42131264
:Traceback (most recent call last):
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 428, in _fork
    self._setupChild(**kwargs)
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 803, in _setupChild
    for fd in _listOpenFDs():
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 638, in _listOpenFDs
    return detector._listOpenFDs()
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 553, in _listOpenFDs
    self._listOpenFDs = self._getImplementation()
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 576, in _getImplementation
    after = impl()
  File "/home/vagrant/.env/sm/lib/python2.7/site-packages/Twisted-15.5.0-py2.7-linux-x86_64.egg/twisted/internet/process.py", line 606, in _procFDImplementation
    return [int(fd) for fd in self.listdir(dname)]
OSError: [Errno 24] Too many open files: '/proc/23421/fd'
Unhandled error in Deferred:

Which does not occur if I am using task.react()

In resume:

• python pyerr.py -l100 -r : OK
• python pyerr.py -l100 -t : OK
• python pyerr.py -l100 -w : OK
• python pyerr.py -l400 -r : OSERR
• python pyerr.py -l400 -t : OK
• python pyerr.py -l400 -w : OSERR

The problem is that I have a big application that uses reactor, because its an application responding to SMTP connections (so cannot use task.react , I do not want to stop the reactor).

I always thought that task.react was only stopping the reactor once the deferred is done, but I guess is doing more than this...

---

edit : Here a pstree comparaison for task.react vs reactor.run

reactor.run (python pyerr.py -l400 -r) :

init-+-VBoxService---7*[{VBoxService}]
     |-acpid
     |-atd
     |-cron
     |-dbus-daemon
     |-dhclient
     |-6*[getty]
     |-master-+-pickup
     |        `-qmgr
     |-mysqld---18*[{mysqld}]
     |-nginx---4*[nginx]
     |-php5-fpm---2*[php5-fpm]
     |-puppet---{puppet}
     |-rpc.idmapd
     |-rpc.statd
     |-rpcbind
     |-rsyslogd---3*[{rsyslogd}]
     |-ruby---{ruby}
     |-sshd-+-3*[sshd---sshd---sftp-server]
     |      |-sshd---sshd---2*[sftp-server]
     |      |-sshd---sshd---bash---pstree
     |      `-sshd---sshd---bash---python-+-323*[echo]
     |                                    `-5*[python]
     |-systemd-logind
     |-systemd-udevd
     |-upstart-file-br
     |-upstart-socket-
     `-upstart-udev-br

task.react (python pyerr.py -l400 -t) :

init-+-VBoxService---7*[{VBoxService}]
     |-acpid
     |-atd
     |-cron
     |-dbus-daemon
     |-dhclient
     |-6*[getty]
     |-master-+-pickup
     |        `-qmgr
     |-mysqld---18*[{mysqld}]
     |-nginx---4*[nginx]
     |-php5-fpm---2*[php5-fpm]
     |-puppet---{puppet}
     |-rpc.idmapd
     |-rpc.statd
     |-rpcbind
     |-rsyslogd---3*[{rsyslogd}]
     |-ruby---{ruby}
     |-sshd-+-3*[sshd---sshd---sftp-server]
     |      |-sshd---sshd---2*[sftp-server]
     |      |-sshd---sshd---bash---pstree
     |      `-sshd---sshd---bash---python---echo
     |-systemd-logind
     |-systemd-udevd
     |-upstart-file-br
     |-upstart-socket-
     `-upstart-udev-br

Notice the difference between this

 |      `-sshd---sshd---bash---python-+-323*[echo]
 |                                    `-5*[python]

and this

 |      `-sshd---sshd---bash---python---echo

in one cas it seems that processes are not closed as soon as completed.

I have tested this issue on 4 different machines:

• Ubuntu 14.04
• Centos 6
• Centos 7

The issue is exactly the same.

To give a shot, try run watch -n 0.1 "pstree" to see how the processes are evolving.

---

edit: I get it why this is happening thanks to Glyph answer, but how to adapt this to my real life case ?

The application I am developing with Twisted is an SMTP filter based on Milter, here how it works (assume we want to check the email signature):

• connection opens on port 25
• milter protocol get all emails details
• milter calls a remote "module" server that will handle the signature check with /usr/bin/openssl mime call
• the module will return an answer indicating whether or not the signature is valid

In this case, my problem is that is I get 150 simultaneous connections, there will be 150 calls to the module (TCP protocol) and this module will invoke the openssl command once per connection.

The module is completely agnostic, therefore will not know if other calls are running. Where should I put the DeferredSemaphore in your opinion ?

My problem here is that smtp connections are also agnostics and don't know about other possible opens connections.

What is the correct way of handling this parallellism in your opinion?

Answer 1

The problem here has nothing to do with the distinction between task.react and reactor.run , but rather, the subtle but significant difference between the implementation of your run and main functions.

The difference is that run is spawning limit processes in parallel , racking up thousands of simultaneous open file descriptors, easily blowing through your system's limitations. However, main is waiting for every process to completely finish executing before even starting up the next one, which means it never uses more than 4 or 5 at a time.

The reason is that main is decorated by inlineCallbacks and yields every getProcessOutputAndValue Deferred , which suspends execution of main until that Deferred has completed.

In real applications, neither of these approaches is ideal. You want some parallelism, but not unlimited. Twisted comes with some utilities, such as DeferredSemaphore , to facilitate limited parallelism without restricting everything to only run one task at a time. Jean-Paul Calderone wrote an article - 10 years ago! - that explains how to use this, here .

However, just to demonstrate that the issue has nothing to do with task.react , here's a modified version of your example which eliminates the run function and makes an apples-to-apples comparison using main :

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from progress.bar import IncrementalBar
from twisted.internet import defer
from twisted.internet import task
from twisted.internet import utils
from twisted.python import usage


class Options(usage.Options):
    optFlags = [['reactor', 'r', 'Use reactor.run().'],
                ['task', 't', 'Use task.react().'],
                ['cwr', 'w', 'Use callWhenRunning().']]
    optParameters = [['limit', 'l', 255, 'Number of file descriptors to open.'],
                     ['cmd', 'c', 'echo Testing {i}...', 'Command to run.']]


@defer.inlineCallbacks
def main(reactor, opt):
    d = defer.Deferred()
    limit = int(opt['limit'])
    cmd, args = opt['cmd'].split(' ', 1)
    bar = IncrementalBar('Running {cmd}'.format(cmd=opt['cmd']), max=limit)
    for i in range(0, limit):
        try:
            _args = args.format(i=i)
            args = _args
        except KeyError:
            pass
        yield utils.getProcessOutputAndValue('echo', [args])
        bar.next()
    bar.finish()
    defer.returnValue(d.callback(True))


if __name__ == '__main__':
    opt = Options()
    opt.parseOptions()

    if opt['reactor']:
        from twisted.internet import reactor
        task.deferLater(reactor, 0, main, reactor, opt)
        reactor.run()

    elif opt['task']:
        from twisted.internet.task import react
        react(main, [opt])

    elif opt['cwr']:
        from twisted.internet import reactor
        reactor.callWhenRunning(main, reactor, opt)
        reactor.run()

edit, responding to edit in the question:

Since your real problem is with incoming connections, and not just a for loop, rather than using DeferredSemaphore , you might instead need to maintain a counter, and take advantage of the fact that the object returned from listenTCP , or the result of the Deferred that comes back from TCP4ServerEndpoint , implements IPushProducer , and call pauseProducing() on it when too many concurrent connections are doing work, and resumeProducing() when that work is done.