converting curl command to pycurl

Question

So the curl command I'm using is as follows:

cmd = "curl --write-out %{http_code} -X PUT -T " + self.basedir + putfile + " -# -o /dev/null " + self.uri + "/" + self.dist + "/" + putfile

I'd like to change this from invoking a system command to using pycurl. This way I can have more granular control over it and ultimately implement a progress bar for it. However, when I try and convert to python, my resulting script fails. Here is my efforts towards a python script:

f = open(filepath, "rb")
fs = os.path.getsize(filepath)
c = pycurl.Curl()
c.setopt(c.URL, target_url)
c.setopt(c.HTTPHEADER, ["User-Agent: Load Tool (PyCURL Load Tool)"])
c.setopt(c.PUT, 1)
c.setopt(c.READDATA, f)
c.setopt(c.INFILESIZE, int(fs))
c.setopt(c.NOSIGNAL, 1)
c.setopt(c.VERBOSE, 1)
c.body = StringIO()
c.setopt(c.WRITEFUNCTION, c.body.write)
try:
        c.perform()
except:
        import traceback
        traceback.print_exc(file=sys.stderr)
sys.stderr.flush()

f.close()
c.close()

sys.stdout.write(".")
sys.stdout.flush()

Here's what that outputs:

* About to connect() to ************ port 8090 (#0)
*   Trying 16.94.124.53... * connected
> PUT /incoming/ HTTP/1.1
Host: ***********
Accept: */*
User-Agent: Load Tool (PyCURL Load Tool)
Content-Length: 21
Expect: 100-continue

< HTTP/1.1 100 Continue
* We are completely uploaded and fine
< HTTP/1.1 500 Internal Server Error
< Content-type: text/html
* no chunk, no close, no size. Assume close to signal end
< 

Thanks in advance for you help!

Answer 1

I've did uploading working module, you can find your answers looking in code.

• And you can find almost all answers regarding pycurl by digging libcurl examples and Docs.

'''
Created on Oct 22, 2013

@author: me
'''
import pycurl
import os
import wx
import sys
import hashlib
from cStringIO import StringIO

def get_file_hash(full_filename):
    BLOCKSIZE = 65536
    hasher = hashlib.md5()
    with open(full_filename, 'rb') as afile:
        buf = afile.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(BLOCKSIZE)
    return hasher.hexdigest()

class FtpUpload(object):

    def __init__(self, server, username, password, **items):
        self.server = server
        self.username = username
        self.password = password
        self.gauge = items.get("gauge")
        self.sb_speed = items.get("sb_speed")
        self.upload_file_size = items.get("upload_file_size") 
        self.upload_file_speed = items.get("upload_file_speed")
        self.filesize = 0
        self.ftp_filehash = '0'

    def sizeToNiceString(self, byteCount):
        for (cutoff, label) in [(1024*1024*1024, "GB"), (1024*1024, "MB"), (1024, "KB")]:
            if byteCount >= cutoff:
                return "%.2f %s" % (byteCount * 1.0 / cutoff, label)
        if byteCount == 1:
            return "1 byte"
        else:
            return "%d bytes" % byteCount

    def initRange(self, filesize):
        self.filesize = filesize
        self.gauge.SetRange(filesize)

    def updateValue(self, upload_d):
        upload_d_int = int(upload_d)
        self.gauge.SetValue(upload_d_int)

        upload_d_str = self.sizeToNiceString(upload_d)
        upload_percent = int((upload_d*100)/self.filesize)

        upload_d_status = "{0}/{1} ({2}%)".format(upload_d_str, self.sizeToNiceString(self.filesize), upload_percent)
        self.sb_speed.SetStatusText(upload_d_status, 1)
        self.upload_file_size.SetLabel(upload_d_status)
        self.upload_file_speed.SetLabel(upload_d_str)

    def progress(self, download_t, download_d, upload_t, upload_d):
        self.updateValue(upload_d)

    def test(self, debug_type, debug_msg):
        if len(debug_msg) < 300:
            print "debug(%d): %s" % (debug_type, debug_msg.strip())

    def ftp_file_hash(self, buf):
        sys.stderr.write("{0:.<20} : {1}\n".format('FTP RAW ', buf.strip()))
        ftp_filehash = dict()
        item = buf.strip().split('\n')[0]
        ext = item.split('.')[1]
        if len(ext) > 3:
            ftp_filename = item[:-33]
            ftp_filehash = item[-32:]
        self.ftp_filehash = ftp_filehash

    def get_ftp_file_hash(self, filename):
        c = pycurl.Curl()
        list_file_hash = 'LIST -1 ' + filename + "_*"
        sys.stderr.write("{0:.<20} : {1} \n".format('FTP command ', list_file_hash))
        c.setopt(pycurl.URL, self.server)
        c.setopt(pycurl.USERNAME, self.username)
        c.setopt(pycurl.PASSWORD, self.password)
        c.setopt(pycurl.VERBOSE, False)
        c.setopt(pycurl.DEBUGFUNCTION, self.test)
        c.setopt(pycurl.CUSTOMREQUEST, list_file_hash)
        c.setopt(pycurl.WRITEFUNCTION, self.ftp_file_hash)
        c.perform()
        c.close()

    def delete_ftp_hash_file(self, ftp_hash_file_old):
        c = pycurl.Curl()
        delete_hash_file = 'DELE ' + ftp_hash_file_old
        sys.stderr.write("{0:.<20} : {1} \n".format('FTP command ', delete_hash_file))
        c.setopt(pycurl.URL, self.server)
        c.setopt(pycurl.USERNAME, self.username)
        c.setopt(pycurl.PASSWORD, self.password)
        c.setopt(pycurl.VERBOSE, False)
        c.setopt(pycurl.DEBUGFUNCTION, self.test)
        c.setopt(pycurl.CUSTOMREQUEST, delete_hash_file)
        try:
            c.perform()
        except Exception as e:
            print e
        c.close()

    def upload(self, full_filename, filesize):
        self.initRange(filesize)
        filename = os.path.basename(full_filename)
        sys.stderr.write("filename: %s\n" % full_filename)

        c = pycurl.Curl()
        c.setopt(pycurl.USERNAME, self.username)
        c.setopt(pycurl.PASSWORD, self.password)

        c.setopt(pycurl.VERBOSE, False)
        c.setopt(pycurl.DEBUGFUNCTION, self.test)

        c.setopt(pycurl.NOBODY, True)
        c.setopt(pycurl.HEADER, False)

        ftp_file_path = os.path.join(self.server, os.path.basename(full_filename))
        file_hash = get_file_hash(full_filename)
        ftp_hash_file = ftp_file_path + "_%s" % file_hash

        # Getting filesize if exist on server.
        try:
            c.setopt(pycurl.URL, ftp_file_path)
            c.perform()
            filesize_offset = int(c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
        except Exception as error_msg:
            print error_msg
            wx.MessageBox(str(error_msg), 'Connection error!', 
                wx.OK | wx.ICON_ERROR)
            # Exit upload function.
            return True

        ftp_file_append = True
        # Get ftp file hash.
        self.get_ftp_file_hash(filename)

        offset = filesize_offset == -1 and '0' or filesize_offset
        sys.stderr.write("L_file hash : {0:.<60}: {1:<40}\n".format(filename, file_hash))
        sys.stderr.write("F_file hash : {0:.<60}: {1:<40}\n".format(filename, self.ftp_filehash))
        sys.stderr.write("{0:15} : {1:.>15}\n".format('filesize:', filesize))
        sys.stderr.write("{0:15} : {1:.>15}\n".format('ftp_filesize', offset))
        sys.stderr.write("{0:15} : {1:.>15}\n".format('to upload:', filesize - int(offset)))
        # File not exist on FTP server.
        if filesize_offset == -1:
            # file not exist: uploading from offset zero.
            ftp_file_append = False
            filesize_offset = 0
        # Local and FTP file size and files MD5 are the same.
        elif filesize_offset == self.filesize and file_hash == self.ftp_filehash:
            sys.stderr.write("--- File exist on server! ---\n\n")
            self.upload_file_speed.SetLabel("File exist on server!")
            self.sb_speed.SetStatusText("File exist on server!", 1)
            # Check next filename.
            return False
        # Ftp file and local file different data.
        elif file_hash != self.ftp_filehash:
            ftp_file_append = False
            filesize_offset = 0
            ftp_hash_file_old = filename + "_" + self.ftp_filehash
            # delete old hash file.
            self.delete_ftp_hash_file(ftp_hash_file_old)

        c.setopt(pycurl.FTPAPPEND, ftp_file_append)
        c.setopt(pycurl.UPLOAD, True)
        c.setopt(pycurl.PROGRESSFUNCTION, self.progress)

        with open('filehash.txt', 'w') as f:
            f.write(file_hash)

        for item in ("filehash.txt", full_filename):
            # dont show progress by default.
            noprogress = True
            # upload ftp_hash_file first.
            ftp_url = ftp_hash_file

            with open(item, "rb") as f:
                # chages ftp_url and show progress values, add filesize_offset.
                if item != "filehash.txt":
                    f.seek(filesize_offset)
                    noprogress = False
                    ftp_url = ftp_file_path

                c.setopt(pycurl.URL, ftp_url)
                c.setopt(pycurl.NOPROGRESS, noprogress)
                c.setopt(pycurl.READFUNCTION, f.read)
                try:
                    c.perform()
                    if item != "filehash.txt":
                        sys.stderr.write("{0:15} : {1:.>15}\n\n".format("size uploaded", int(c.getinfo(pycurl.SIZE_UPLOAD))))
                except Exception as error_msg:
                    print error_msg
                    wx.MessageBox(str(error_msg), 'Connection error!', 
                        wx.OK | wx.ICON_ERROR)
                    # Exit upload function.
                    return True
        self.ftp_filehash = '0'
        c.close()

if __name__ == '__main__':
    pass