blobstore inside a taskqueue/deferred

Question

I am trying to import a large User Information list from a json file to the datastore using taskqueue and deferred.

A User contains the user's information including an image url from a different app. During the importing process, the image should be grabbed and uploaded to the blob (which works just fine when tested).

I got stuck with getting the blob_key of the uploaded image. And I think it only occurs inside a taskqueue/deferred because I tried it inside a 'normal' GET request handler, it works just fine.

This is my handler:

class MigrationTask(BaseHandler):
    def post(self):
        if not self.request.get('file'):
            return
        json_data = open(self.request.get('file'))
        data = json.load(json_data)
        json_data.close()

        for datum in data['results']:
            deferred.defer(push_user_to_db, datum)

this are my functions:

@ndb.transactional(xg=True)
def _push_user_to_db(profilePicture=None, ...):
    if profilePicture:
        if 'url' in profilePicture:
            con = urlfetch.fetch(image_url)
            if con.status_code == 200:
                file_name = files.blobstore.create(mime_type='application/octet-stream')
                with files.open(file_name, 'a') as f:
                    f.write(con.content)
                files.finalize(file_name)
                blob_key = files.blobstore.get_blob_key(file_name)  # this part is where it errs
                image_url = images.get_serving_url(file_name)

    # some codes here...

def push_user_to_db(kwargs):
    _push_user_to_db(**kwargs)

part of the traceback:

    blob_key = files.blobstore.get_blob_key(file_name)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\files\blobstore.py", line 132, in get_blob_key

    namespace='')])[0]

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\datastore.py", line 654, in Get

    return GetAsync(keys, **kwargs).get_result()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\datastore.py", line 629, in GetAsync

    return _GetConnection().async_get(config, keys, local_extra_hook)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\datastore\datastore_rpc.py", line 1574, in async_get

    pbs = [key_to_pb(key) for key in keys]

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 653, in key_to_pb

    return key.reference()

AttributeError: 'Key' object has no attribute 'reference'

PS: I've also tried taskqueue instead of deferred.

EDIT(1):

This is the traceback:

ERROR    2015-03-03 06:32:44,720 webapp2.py:1552] 'Key' object has no attribute 'reference'

Traceback (most recent call last):

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 1535, in __call__

    rv = self.handle_exception(request, response, e)

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 1529, in __call__

    rv = self.router.dispatch(request, response)

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 1278, in default_dispatcher

    return route.handler_adapter(request, response)

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 1102, in __call__

    return handler.dispatch()

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 572, in dispatch

    return self.handle_exception(e, self.app.debug)

  File "C:\Program Files (x86)\Google\google_appengine\lib\webapp2-2.5.2\webapp2.py", line 570, in dispatch

    return method(*args, **kwargs)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\deferred\deferred.py", line 310, in post

    self.run_from_request()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\deferred\deferred.py", line 305, in run_from_request

    run(self.request.body)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\deferred\deferred.py", line 147, in run

    return func(*args, **kwds)

  File "C:\project directory\migration.py", line 141, in push_user_to_db

    _push_user_to_db(**kwargs)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\utils.py", line 179, in inner_wrapper

    return wrapped_decorator(func, args, kwds, **options)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 3759, in transactional

    func, args, kwds, **options).get_result()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\tasklets.py", line 325, in get_result

    self.check_success()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\tasklets.py", line 371, in _help_tasklet_along

    value = gen.send(val)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\context.py", line 999, in transaction

    result = callback()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 3767, in <lambda>

    return transaction_async(lambda: func(*args, **kwds), **options)

  File "C:\project directory\migration.py", line 56, in _push_user_to_db

    blob_key = files.blobstore.get_blob_key(file_name)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\files\blobstore.py", line 132, in get_blob_key

    namespace='')])[0]

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\datastore.py", line 654, in Get

    return GetAsync(keys, **kwargs).get_result()

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\api\datastore.py", line 629, in GetAsync

    return _GetConnection().async_get(config, keys, local_extra_hook)

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\datastore\datastore_rpc.py", line 1574, in async_get

    pbs = [key_to_pb(key) for key in keys]

  File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 653, in key_to_pb

    return key.reference()

AttributeError: 'Key' object has no attribute 'reference'

Answer 1

Heads up! Writing files to the Blobstore using the files api has been deprecated. I had this issue before. My codes run perfectly fine in development server (localhost) but erred on App Engine server. The solution is to write the files in the Google Cloud Storage via Blobstore API.