Image migration method not working in production, but works in localhost

Question

I have written following method using Files API for GAE for migrating images from my SQL server to GAE Blobstore.

import urllib2,csv
from abc.model import *
from google.appengine.api import files
from google.appengine.ext import ndb, blobstore
from urllib2 import HTTPError


def foo():
    i = []
    j = []
    csv_reader = csv.reader(open('tbl_property_images.csv','r'))
    csv_prop = csv.reader(open('property.csv','r'))
    ids = []
    for ele in csv_prop:
        ids.append(ele[0])
    for row in csv_reader:
        i.append(row[2])
        j.append(row[3])
    i = iter(i)
    j = iter(j)
    k = list(zip(i, j))
    d = {}
    for x, y in k:
        if x in d:
            d[x] = d[x] + [y]
    else:
            d[x] = [y]
d.pop('fld_property_id')
to_put = []
for ab in d.iterkeys():
    if ab in ids:
        for b in d[ab]:
            url = 'abc###.com/%s' % b
            try:
                file_name = files.blobstore.create(mime_type='image/jpeg')

                image = urllib2.urlopen(url)
                with files.open(file_name, 'a') as f:
                    f.write(image.read())
                files.finalize(file_name)
                blob_key = files.blobstore.get_blob_key(file_name)
                blob_info = blobstore.BlobInfo.get(blob_key)
                kwargs = {}
                kwargs['id'] = File.allocate_ids(1)[0]
                kwargs['identifier'] = '%s-%s' % (kwargs['id'], blob_info.filename)
                file = File(filename=blob_info.filename, content_type=blob_info.content_type, size=blob_info.size, blob=blob_info.key(), **kwargs)
                prop = Property.query(Property.id == ab).get()
                image1 = Image.build(file=file, property=prop.key)
                prop.image_url = image1.image_url
                to_put.append(prop)
                to_put.append(file)
                # if prop.key not in to_put properties key
                to_put.append(image1)

            except HTTPError:
                print url
                continue

ndb.put_multi(to_put)

It works correctly if I run it from Interactive Console of my localhost. But breaks when put in production website.

I am getting following error in my log.

Traceback (most recent call last): File "", line 1, in

File "temp_part2.py", line 39, in foo

f.write(image.read())

File "/home/rohit/workspace/google_appengine/google/appengine/api/files/file.py", line 300, in exit

self.close()

File "/home/rohit/workspace/google_appengine/google/appengine/api/files/file.py", line 294, in close

self._make_rpc_call_with_retry('Close', request, response)

File "/home/rohit/workspace/google_appengine/google/appengine/api/files/file.py", line 430, in _make_rpc_call_with_retry

_make_call(method, request, response)

File "/home/rohit/workspace/google_appengine/google/appengine/api/files/file.py", line 255, in _make_call _raise_app_error(e)

File "/home/rohit/workspace/google_appengine/google/appengine/api/files/file.py", line 198, in _raise_app_error

raise FileNotOpenedError(e)

FileNotOpenedError: ApplicationError: 10

I am stuck with this error for a while, any help will be highly appreciated.

Answer 1

See another similar question, https://stackoverflow.com/a/18308034/1686094

Holding the file open for more than 30 seconds will also result in this error. Try to break the writes into multiple writes.