将多个CSV文件从Google Cloud Bucket导入到Datalab

Question

I'm trying to get the below code to import multiple csv files from a Google Cloud Bucket to Datalab in python 3.x:

import google.datalab.storage as storage
import pandas as pd
from io import BytesIO

myBucket = storage.Bucket('some-bucket')
object_list = myBucket.objects(prefix='some-prefix')
df_list = []

for obj in object_list:
  %gcs read --object $obj.uri --variable data  
  df_list.append(pd.read_csv(BytesIO(data), compression='gzip'))

df = pd.concat(df_list, ignore_index=True)
df.head()

I'm getting the following error right at the beginning of the for loop:

TypeError: a bytes-like object is required, not 'str'

I have spent some time resolving this issue but no luck! Any help would be greatly appreciated!

Below is the entire traceback just in case:

---------------------------------------------------------------------------
JSONDecodeError                           Traceback (most recent call last)
/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/utils/_http.py in __init__(self, status, content)
     49       else:
---> 50         error = json.loads(str(content, encoding='UTF-8'))['error']
     51       if 'errors' in error:

/usr/local/envs/py3env/lib/python3.5/json/__init__.py in loads(s, 
encoding, cls, object_hook, parse_float, parse_int, parse_constant, 
object_pairs_hook, **kw)
    318             parse_constant is None and object_pairs_hook is None and not kw):
--> 319         return _default_decoder.decode(s)
    320     if cls is None:

/usr/local/envs/py3env/lib/python3.5/json/decoder.py in decode(self, s, _w)
    338         """
--> 339         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
    340         end = _w(s, end).end()

/usr/local/envs/py3env/lib/python3.5/json/decoder.py in raw_decode(self, s, idx)
    356         except StopIteration as err:
--> 357             raise JSONDecodeError("Expecting value", s, err.value) from None
    358         return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
<ipython-input-9-6d51e52b6c6f> in <module>()
      7 df_list = []
      8 
----> 9 for obj in object_list:
     10   get_ipython().run_line_magic('gcs', 'read --object $obj.uri --variable data')
     11   df_list.append(pd.read_csv(BytesIO(data), compression='gzip'))

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/utils/_iterator.py in __iter__(self)
     34     """Provides iterator functionality."""
     35     while self._first_page or (self._page_token is not None):
---> 36       items, next_page_token = self._retriever(self._page_token, self._count)
     37 
     38       self._page_token = next_page_token

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/storage/_object.py in _retrieve_objects(self, page_token, _)
    319                                          page_token=page_token)
    320     except Exception as e:
--> 321       raise e
    322 
    323     objects = list_info.get('items', [])

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/storage/_object.py in _retrieve_objects(self, page_token, _)
    317       list_info = self._api.objects_list(self._bucket,
    318                                          prefix=self._prefix, 
delimiter=self._delimiter,
--> 319                                          page_token=page_token)
    320     except Exception as e:
    321       raise e

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/storage/_api.py in objects_list(self, bucket, prefix, delimiter, projection, versions, max_results, page_token)
    246 
    247     url = Api._ENDPOINT + (Api._OBJECT_PATH % (bucket, ''))
--> 248     return google.datalab.utils.Http.request(url, args=args, credentials=self._credentials)
    249 
    250   def objects_patch(self, bucket, key, info):

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/utils/_http.py in request(url, args, data, headers, method, credentials, raw_response, stats)
    156           return json.loads(str(content, encoding='UTF-8'))
    157       else:
--> 158         raise RequestException(response.status, content)
    159     except ValueError:
    160       raise Exception('Failed to process HTTP response.')

/usr/local/envs/py3env/lib/python3.5/site-packages/google/datalab/utils/_http.py in __init__(self, status, content)
     53       self.message += ': ' + error['message']
     54     except Exception:
---> 55       lines = content.split('\n') if isinstance(content, basestring) else []
     56       if lines:
     57         self.message += ': ' + lines[0]

TypeError: a bytes-like object is required, not 'str'

Answer 1

The BytesIO class requires the argument to be bytes, not a string:

>>> from io import BytesIO
>>> BytesIO(b'hi')
<_io.BytesIO object at 0x1088aedb0>
>>> BytesIO('hi')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: a bytes-like object is required, not 'str'

If your data is a string and not bytes, you should use:

from io import StringIO
pd.read_csv(StringIO(data))