Streaming POST a large file to CherryPy by Python client

Question

I'm want to POST a large file from a python client to cherrypy. I'm using the requests library.

This is my client code:

def upload(fileName=None):
    url = 'http://localhost:8080/upload'
    files = {'myFile': ( fileName, open(fileName, 'rb') )}
    r = requests.post(url, files=files)

#with open(fileName,'rb') as payload:
    #headers = {'content-type': 'multipart/form-data'}
    #r = requests.post('http://127.0.0.1:8080', data=payload,verify=False,headers=headers)

if __name__ == '__main__':
    upload(sys.argv[1])

The problem is that this puts the whole file in the RAM memory. Is there any way to POST the file in pieces?

class FileDemo(object):


@cherrypy.expose
def upload(self, myFile):

    print myFile.filename
    #size = 0
    #decoder = MultipartDecoder(myFile, 'image/jpeg')
    #for part in decoder.parts:
        #print(part.header['content-type'])

    #while True:

        #advances to the content that hasn't been read
        #myFile.file.seek(size, 0)

        #reads 100mb at a time so it doesn't fill up the RAM
        #data = myFile.file.read(10240000)

        #newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
        #newFile.write(data)
        #newFile.close

        #size += len(data)

        #if len(data) < 10240000:
            #break
if __name__ == '__main__':
    cherrypy.quickstart(FileDemo())

This is the code in the server side. It has a lot of comments because I've been trying a lot of stuff. Right now I'm just printing the file name and the client still transfers the whole file to RAM.

I don't know what else to try. Thank you in advance for your help.

Answer 1

If it's CherryPy specific upload you can skip multipart/form-data encoding obstacles and just send streaming POST body of file contents.

client

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import urllib2
import io
import os


class FileLenIO(io.FileIO):

  def __init__(self, name, mode = 'r', closefd = True):
    io.FileIO.__init__(self, name, mode, closefd)

    self.__size = statinfo = os.stat(name).st_size

  def __len__(self):
    return self.__size


f = FileLenIO('/home/user/Videos/video.mp4', 'rb')
request = urllib2.Request('http://127.0.0.1:8080/upload', f)
request.add_header('Content-Type', 'application/octet-stream')
# you can add custom header with filename if you need it
response = urllib2.urlopen(request)

print response.read()

server

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import os
import tempfile
import shutil

import cherrypy


config = {
  'global' : {
    'server.socket_host' : '127.0.0.1',
    'server.socket_port' : 8080,
    'server.thread_pool' : 8,
    # remove any limit on the request body size; cherrypy's default is 100MB
    'server.max_request_body_size' : 0,
    # increase server socket timeout to 60s; cherrypy's defult is 10s
    'server.socket_timeout' : 60
  }
}


class App:

  @cherrypy.config(**{'response.timeout': 3600}) # default is 300s
  @cherrypy.expose()
  def upload(self):
    '''Handle non-multipart upload'''

    destination = os.path.join('/home/user/test-upload')                
    with open(destination, 'wb') as f:
      shutil.copyfileobj(cherrypy.request.body, f)

    return 'Okay'


if __name__ == '__main__':
  cherrypy.quickstart(App(), '/', config)

Tested on 1.3GiB video file. Server-side memory consumption is under 10MiB, client's under 5MiB.

Answer 2

This is how I solved the problem:

client

import poster
def upload(fileName=None):

    register_openers()
    url = 'http://localhost:8080/upload'
    data, headers = multipart_encode({"myFile": open(fileName, "rb")})

    request = urllib2.Request(url, data, headers)
    request.unverifiable = True
    response = urllib2.urlopen(request)
    the_page = response.read()


if __name__ == '__main__':
    upload(sys.argv[1])

server

@cherrypy.expose
def upload(self, myFile):

    cherrypy.response.timeout = 3600
    newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
    newFile.write(myFile.file.read())
    newFile.close