使用urllib3进行多部分表单编码和发布

Question

我正在尝试将一个csv文件上传到此站点 。 但是，我遇到了一些问题，我认为这是由于错误的mimetype （也许是）引起的。

我试图通过urllib2手动发布文件，所以我的代码如下所示：

import urllib
import urllib2
import mimetools, mimetypes
import os, stat
from cStringIO import StringIO

#============================
# Note: I found this recipe online. I can't remember where exactly though.. 
#=============================

class Callable:
    def __init__(self, anycallable):
        self.__call__ = anycallable

# Controls how sequences are uncoded. If true, elements may be given multiple values by
#  assigning a sequence.
doseq = 1

class MultipartPostHandler(urllib2.BaseHandler):
    handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first

    def http_request(self, request):
        data = request.get_data()
        if data is not None and type(data) != str:
            v_files = []
            v_vars = []
            try:
                 for(key, value) in data.items():
                     if type(value) == file:
                         v_files.append((key, value))
                     else:
                         v_vars.append((key, value))
            except TypeError:
                systype, value, traceback = sys.exc_info()
                raise TypeError, "not a valid non-string sequence or mapping object", traceback

            if len(v_files) == 0:
                data = urllib.urlencode(v_vars, doseq)
            else:
                boundary, data = self.multipart_encode(v_vars, v_files)

                contenttype = 'multipart/form-data; boundary=%s' % boundary
                if(request.has_header('Content-Type')
                   and request.get_header('Content-Type').find('multipart/form-data') != 0):
                    print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')
                request.add_unredirected_header('Content-Type', contenttype)

            request.add_data(data)

        return request

    def multipart_encode(vars, files, boundary = None, buf = None):
        if boundary is None:
            boundary = mimetools.choose_boundary()
        if buf is None:
            buf = StringIO()
        for(key, value) in vars:
            buf.write('--%s\r\n' % boundary)
            buf.write('Content-Disposition: form-data; name="%s"' % key)
            buf.write('\r\n\r\n' + value + '\r\n')
        for(key, fd) in files:
            file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
            filename = fd.name.split('/')[-1]
            contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
            buf.write('--%s\r\n' % boundary)
            buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
            buf.write('Content-Type: %s\r\n' % contenttype)
            # buffer += 'Content-Length: %s\r\n' % file_size
            fd.seek(0)
            buf.write('\r\n' + fd.read() + '\r\n')
        buf.write('--' + boundary + '--\r\n\r\n')
        buf = buf.getvalue()
        return boundary, buf
    multipart_encode = Callable(multipart_encode)

    https_request = http_request

    import cookielib
    cookies = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
            MultipartPostHandler)

    opener.addheaders = [(
            'User-agent', 
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6'
        )]


    params = {"FILENAME" : open("weather_scrape.csv", 'rb'),
            'CGIREF' : '/calludt.cgi/DDFILE1',
            'USE':'MODEL',
            'MODEL':'CM',
            'CROP':'APPLES',
            'METHOD': 'SS',
            'UNITS' : 'E',
            'LOWTHRESHOLD': '50',
            'UPTHRESHOLD': '88',
            'CUTOFF':'H',
            'COUNTY':'AL',
            'ACTIVE':'Y',
            'FROMMONTH':'3',
            'FROMDAY':'15',
            'FROMYEAR': '2013',
            'THRUMONTH':'5',
            'THRUDAY':'13',
            'THRUYEAR':'2013',
            'DATASOURCE' : 'FILE'
            }

    response = opener.open("http://www.ipm.ucdavis.edu/WEATHER/textupload.cgi", params)

现在，当我发布此内容时，一切似乎都很好，直到我单击第一个POST返回的后续网页上的Submit按钮。 然后，我收到此错误消息：

ERROR (bad data) in file 'weather.csv' at line 135.

Data record = [--192.168.117.2.1.4404.1368589639.796.1--]

Too few values found. Check delimiter specification.

现在，在调查我在浏览器中执行操作时发出的发布请求时，我注意到content-type非常具体，即：

------WebKitFormBoundaryfBp6Jfhv7LlPZLKd
Content-Disposition: form-data; name="FILENAME"; filename="weather.csv"
Content-Type: application/vnd.ms-excel

我不完全确定内容类型是引起错误的原因，但是..这是我目前正在排除的错误（因为我不知道实际出了什么问题。）我看不出有什么办法通过urllib2设置内容类型，因此在进行一些谷歌搜索之后，我偶然发现了urllib3.

Urllib3具有内置的文件发布功能，但是我不确定如何使用它。

Filepost

urllib3.filepost.encode_multipart_formdata(fields, boundary=None)
Encode a dictionary of fields using the multipart/form-data MIME format.

Parameters: 
fields –
Dictionary of fields or list of (key, value) or (key, value, MIME type) field tuples. The key is treated as the field name, and the value as the body of the form-data bytes. If the value is a tuple of two elements, then the first element is treated as the filename of the form-data section and a suitable MIME type is guessed based on the filename. If the value is a tuple of three elements, then the third element is treated as an explicit MIME type of the form-data section.
Field names and filenames must be unicode.
boundary – If not specified, then a random boundary will be generated using mimetools.choose_boundary().
urllib3.filepost.iter_fields(fields)
Iterate over fields.

Supports list of (k, v) tuples and dicts.

使用此库，我尝试将值编码为文档中的注释，但出现错误。

我试图最初，只是为了测试的事情了，作为一个dict 。

params = {"FILENAME" : open("weather.csv", 'rb'),
            'CGIREF' : '/calludt.cgi/DDFILE1',
            'USE':'MODEL',
            'MODEL':'CM',
            'CROP':'APPLES',
            'METHOD': 'SS',
            'UNITS' : 'E',
            'LOWTHRESHOLD': '50',
            'UPTHRESHOLD': '88',
            'CUTOFF':'H',
            'COUNTY':'AL',
            'ACTIVE':'Y',
            'FROMMONTH':'3',
            'FROMDAY':'15',
            'FROMYEAR': '2013',
            'THRUMONTH':'5',
            'THRUDAY':'13',
            'THRUYEAR':'2013',
            'DATASOURCE' : 'FILE'
            }

    values = urllib3.filepost.encode_multipart_formdata(params)

但是，这会引发以下错误：

    values = urllib3.filepost.encode_multipart_formdata(params)
  File "c:\python27\lib\site-packages\urllib3-dev-py2.7.egg\urllib3\filepost.py", line 90, in encode_multipart_formdata
    body.write(data)
TypeError: 'file' does not have the buffer interface

不知道是什么原因造成的，我尝试传递一个元组列表（键，值，mimetype），但这也会引发错误：

params = [
        ("FILENAME" , open("weather_scrape.csv"), 'application/vnd.ms-excel'),
        ('CGIREF' , '/calludt.cgi/DDFILE1'),
        ('USE','MODEL'),
        ('MODEL','CM'),
        ('CROP','APPLES'),
        ('METHOD', 'SS'),
        ('UNITS' , 'E'),
        ('LOWTHRESHOLD', '50'),
        ('UPTHRESHOLD', '88'),
        ('CUTOFF','H'),
        ('COUNTY','AL'),
        ('ACTIVE','Y'),
        ('FROMMONTH','3'),
        ('FROMDAY','15'),
        ('FROMYEAR', '2013'),
        ('THRUMONTH','5'),
        ('THRUDAY','13'),
        ('THRUYEAR','2013'),
        ('DATASOURCE' , 'FILE)')
        ]

    values = urllib3.filepost.encode_multipart_formdata(params)



>>ValueError: too many values to unpack

Answer 1

如果您想为此使用urllib3，它将看起来像这样：

import urllib3

http = urllib3.PoolManager()

headers = urllib3.make_headers(user_agent='Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6')
url = "http://www.ipm.ucdavis.edu/WEATHER/textupload.cgi"
csv_data = open("weather_scrape.csv").read()

params = {
    "FILENAME": csv_data,
    'CGIREF': '/calludt.cgi/DDFILE1',
    'USE': 'MODEL',
    'MODEL': 'CM',
    'CROP': 'APPLES',
    'METHOD': 'SS',
    'UNITS' : 'E',
    'LOWTHRESHOLD': '50',
    'UPTHRESHOLD': '88',
    'CUTOFF': 'H',
    'COUNTY': 'AL',
    'ACTIVE': 'Y',
    'FROMMONTH': '3',
    'FROMDAY': '15',
    'FROMYEAR': '2013',
    'THRUMONTH': '5',
    'THRUDAY': '13',
    'THRUYEAR': '2013',
    'DATASOURCE' : 'FILE',
}

response = http.request('POST', url, params, headers)

我无法使用您的目标网址和csv数据集进行测试，因此其中可能包含一些小错误。 但这是总的想法。