How can I get the response body from pycurl multi curl requests

Question

I am unable to get anything but empty responses when performing curl multi requests. No exceptions are thrown, but the response value has no content (commented in the below snippet)

Here's a simplified version of my code:

from StringIO import StringIO

import pycurl


class CurlStream(object):
    curl_count = 0
    curl_storage = []

    def __init__(self):
        self.curl_multi = pycurl.CurlMulti()

    def add_request(self, request, post_fields=None):
        self.curl_count += 1
        curl = self._create_curl(request, post_fields)
        self.curl_multi.add_handle(curl)

    def perform(self):
        while self.curl_count:
            while True:
                response, self.curl_count = self.curl_multi.perform()
                if response != pycurl.E_CALL_MULTI_PERFORM:
                    break
            self.curl_multi.select(1.0)

    def read_all(self):
        for response in self.curl_storage:
            print response.getvalue() # this does nothing --prints blank lines

    def close(self):
        self.curl_multi.close()

    def _create_curl(self, request, post_fields):
        curl = pycurl.Curl()
        output = StringIO()
        self.curl_storage.append(output)
        curl.setopt(curl.URL, request)
        curl.setopt(curl.WRITEFUNCTION, output.write)
        curl.setopt(curl.TIMEOUT, 20)
        return curl


def main():
    curl_stream = CurlStream()
    curl_stream.add_request('http://www.google.com')
    curl_stream.add_request('http://www.example.com')
    curl_stream.perform()
    curl_stream.read_all()
    curl_stream.close()

if __name__ == '__main__':
    main()

I have made single requests with the same options without using curl multi and it works.

Answer 1

Okay, so I found it worked when I changed the _create_curl method to this (adding write_out for debugging):

def _create_curl(self, request, post_fields):
    curl = pycurl.Curl()
    curl.setopt(curl.URL, request)
    curl.setopt(curl.WRITEFUNCTION, self.write_out)
    curl.setopt(curl.TIMEOUT, 20)

    # Below is the important bit, I am now adding each curl object to a list
    self.curl_storage.append(curl)
    return curl

def write_out(self, data):
        print data
        return len(data)

The issue was when adding the curl object to the multicurl object, I hadn't kept any reference to the single curl object, so it was automagically closed.

According to the pycurl docs for the curl close() method:

Corresponds to curl_easy_cleanup in libcurl. This method is automatically called by pycurl when a Curl object no longer has any references to it, but can also be called explicitly.