I am uploading a large file using the Python requests package, and I can't find any way to give data back about the progress of the upload. I have seen a number of progress meters for downloading a file, but these will not work for a file upload.

The ideal solution would be some sort of callback method such as:

def progress(percent):
  print percent
r = requests.post(URL, files={'f':hugeFileHandle}, callback=progress)

Thanks in advance for your help :)

share|improve this question
2  
You'd have to implement the progress in hugeFileHandle. I'm not sure why requests doesn't provide a clean way of doing this. – Blender Dec 17 '12 at 7:22
up vote 12 down vote accepted

requests doesn't support upload streaming e.g.:

import os
import sys
import requests  # pip install requests

class upload_in_chunks(object):
    def __init__(self, filename, chunksize=1 << 13):
        self.filename = filename
        self.chunksize = chunksize
        self.totalsize = os.path.getsize(filename)
        self.readsofar = 0

    def __iter__(self):
        with open(self.filename, 'rb') as file:
            while True:
                data = file.read(self.chunksize)
                if not data:
                    sys.stderr.write("\n")
                    break
                self.readsofar += len(data)
                percent = self.readsofar * 1e2 / self.totalsize
                sys.stderr.write("\r{percent:3.0f}%".format(percent=percent))
                yield data

    def __len__(self):
        return self.totalsize

# XXX fails
r = requests.post("http://httpbin.org/post",
                  data=upload_in_chunks(__file__, chunksize=10))

btw, if you don't need to report progress; you could use memory-mapped file to upload large file.

To workaround it, you could create a file adaptor similar to the one from urllib2 POST progress monitoring:

class IterableToFileAdapter(object):
    def __init__(self, iterable):
        self.iterator = iter(iterable)
        self.length = len(iterable)

    def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
        return next(self.iterator, b'')

    def __len__(self):
        return self.length

Example

it = upload_in_chunks(__file__, 10)
r = requests.post("http://httpbin.org/post", data=IterableToFileAdapter(it))

# pretty print
import json
json.dump(r.json, sys.stdout, indent=4, ensure_ascii=False)
share|improve this answer
    
This worked for the most part, but I found that this just uploaded the contents of the file. Really, what I need is to use requests.post(url, files={'file',fileobj}), and doing this only gives the first chunk of the file using your method – Robin Begbie Dec 18 '12 at 1:44
    
@Robin: the above is a hack that can easily fail. You could try poster instead. It supports progress callbacks and streaming (with known content-length) of multipart/form-data. btw, remove the tick if the answer is not acceptable for your question. – jfs Dec 18 '12 at 18:29
    
Time to update this answer – Dima Tisnek Oct 29 '14 at 8:10
    
@qarma: If you know a better answer; post it. – jfs Oct 29 '14 at 8:16
    
kennethreitz commented on 10 Jan 2013: Done. github.com/kennethreitz/requests/issues/952 – Dima Tisnek Oct 29 '14 at 13:51

I got it working with the code from here: Simple file upload progressbar in PyQt. I changed it a bit, to use BytesIO instead of StringIO.

class CancelledError(Exception):
    def __init__(self, msg):
        self.msg = msg
        Exception.__init__(self, msg)

    def __str__(self):
        return self.msg

    __repr__ = __str__

class BufferReader(BytesIO):
    def __init__(self, buf=b'',
                 callback=None,
                 cb_args=(),
                 cb_kwargs={}):
        self._callback = callback
        self._cb_args = cb_args
        self._cb_kwargs = cb_kwargs
        self._progress = 0
        self._len = len(buf)
        BytesIO.__init__(self, buf)

    def __len__(self):
        return self._len

    def read(self, n=-1):
        chunk = BytesIO.read(self, n)
        self._progress += int(len(chunk))
        self._cb_kwargs.update({
            'size'    : self._len,
            'progress': self._progress
        })
        if self._callback:
            try:
                self._callback(*self._cb_args, **self._cb_kwargs)
            except: # catches exception from the callback
                raise CancelledError('The upload was cancelled.')
        return chunk


def progress(size=None, progress=None):
    print("{0} / {1}".format(size, progress))


files = {"upfile": ("file.bin", open("file.bin", 'rb').read())}

(data, ctype) = requests.packages.urllib3.filepost.encode_multipart_formdata(files)

headers = {
    "Content-Type": ctype
}

body = BufferReader(data, progress)
requests.post(url, data=body, headers=headers)

The trick is, to generate data and header from the files list manually, using encode_multipart_formdata() from urllib3

share|improve this answer

I recommend to use a tool package named requests-toolbelt, which make monitoring upload bytes very easy, like

from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
import requests

def my_callback(monitor):
    # Your callback function
    print monitor.bytes_read

e = MultipartEncoder(
    fields={'field0': 'value', 'field1': 'value',
            'field2': ('filename', open('file.py', 'rb'), 'text/plain')}
    )
m = MultipartEncoderMonitor(e, my_callback)

r = requests.post('http://httpbin.org/post', data=m,
                  headers={'Content-Type': m.content_type})

And you may want to read this to show a progress bar.

share|improve this answer
    
This would basically be what I need... BUT... is there a way to upload the contents of f.e. file.py in chunks now? – Georg Oct 13 '15 at 12:58

falloutboy's solution is nice. And I found another project named requests-toolbelt in requests official website: http://docs.python-requests.org/en/latest/user/quickstart/#post-a-multipart-encoded-file

here is the requests-toolbelt's doc:https://toolbelt.rtfd.org/

share|improve this answer

Usually you would build a streaming datasource (a generator) that reads the file chunked and reports its progress on the way (see kennethreitz/requests#663. This does not work with requests file-api, because requests doesn’t support streaming uploads (see kennethreitz/requests#295) – a file to upload needs to be complete in memory before it starts getting processed.

but requests can stream content from a generator as J.F. Sebastian has proven before, but this generator needs to generate the complete datastream including the multipart encoding and boundaries. This is where poster comes to play.

poster is originally written to be used with pythons urllib2 and supports streaming generation of multipart requests, providing progress indication as it goes along. Posters Homepage provides examples of using it together with urllib2 but you really don’t want to use urllib2. Check out this example-code on how to to HTTP Basic Authentication with urllib2. Horrrrrrrrible.

So we really want to use poster together with requests to do file uploads with tracked progress. And here is how:

# load requests-module, a streamlined http-client lib
import requests

# load posters encode-function
from poster.encode import multipart_encode



# an adapter which makes the multipart-generator issued by poster accessable to requests
# based upon code from http://stackoverflow.com/a/13911048/1659732
class IterableToFileAdapter(object):
    def __init__(self, iterable):
        self.iterator = iter(iterable)
        self.length = iterable.total

    def read(self, size=-1):
        return next(self.iterator, b'')

    def __len__(self):
        return self.length

# define a helper function simulating the interface of posters multipart_encode()-function
# but wrapping its generator with the file-like adapter
def multipart_encode_for_requests(params, boundary=None, cb=None):
    datagen, headers = multipart_encode(params, boundary, cb)
    return IterableToFileAdapter(datagen), headers



# this is your progress callback
def progress(param, current, total):
    if not param:
        return

    # check out http://tcd.netinf.eu/doc/classnilib_1_1encode_1_1MultipartParam.html
    # for a complete list of the properties param provides to you
    print "{0} ({1}) - {2:d}/{3:d} - {4:.2f}%".format(param.name, param.filename, current, total, float(current)/float(total)*100)

# generate headers and gata-generator an a requests-compatible format
# and provide our progress-callback
datagen, headers = multipart_encode_for_requests({
    "input_file": open('recordings/really-large.mp4', "rb"),
    "another_input_file": open('recordings/even-larger.mp4', "rb"),

    "field": "value",
    "another_field": "another_value",
}, cb=progress)

# use the requests-lib to issue a post-request with out data attached
r = requests.post(
    'https://httpbin.org/post',
    auth=('user', 'password'),
    data=datagen,
    headers=headers
)

# show response-code and -body
print r, r.text
share|improve this answer

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Not the answer you're looking for? Browse other questions tagged or ask your own question.