使用 python 将具有不同架构的项目附加到 Avro 中的现有文件

Question

I have just started with Avro (and python). I wanted to check the schema evolution. I prepared 2 schemas, first saved data with the first one, and next append new data and save with the schema 2. I do not get any error by writing, but I am not able to deserialise the data. I guess my syntax is wrong. How shall I proceed to add the items with a new schema to existing file?

schema = avro.schema.Parse(open('user.avsc', "r").read())

writer = DataFileWriter(open("users.avro", "wb"), DatumWriter(), schema)
writer.append({"name": "Anna", "favorite_number": 1})
writer.append({"name": "Jan", "favorite_number": 13, "favorite_color": "blue"})
writer.close()

reader = DataFileReader(open("users.avro", "rb"), DatumReader())
for user in reader:
    print (user)
reader.close()

{'name': 'Anna', 'favorite_number': 1, 'favorite_color': None}
{'name': 'Jan', 'favorite_number': 13, 'favorite_color': 'blue'}

schema2 = avro.schema.Parse(open('user2.avsc', "r").read())

writer = DataFileWriter(open("users.avro", "ab"), DatumWriter(), schema2)
writer.append({"name": "Eva", "favorite_number": 5, "favorite_food":"raclette"})
writer.append({"name": "Adam", "favorite_number": 122, "favorite_color": "black", "favorite_film": "Gone with the wind"})
writer.close()

reader = DataFileReader(open("users.avro", "rb"), DatumReader())
for user in reader:
    print (user)
reader.close()

Invalid UTF-8 input bytes: b'\x01\x04\x14avro.codec\x08null\x16avro.schema\xf0\x05{"type": "record", "n'


{'name': 'Anna', 'favorite_number': 1, 'favorite_color': None}
{'name': 'Jan', 'favorite_number': 13, 'favorite_color': 'blue'}

---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-128-cbc8ab11fe9f> in <module>
      1 reader = DataFileReader(open("users.avro", "rb"), DatumReader())
----> 2 for user in reader:
      3     print (user)
      4 reader.close()

~\Anaconda3\lib\site-packages\avro\datafile.py in __next__(self)
    524         self._read_block_header()
    525 
--> 526     datum = self.datum_reader.read(self.datum_decoder)
    527     self._block_count -= 1
    528     return datum

~\Anaconda3\lib\site-packages\avro\io.py in read(self, decoder)
    487     if self.reader_schema is None:
    488       self.reader_schema = self.writer_schema
--> 489     return self.read_data(self.writer_schema, self.reader_schema, decoder)
    490 
    491   def read_data(self, writer_schema, reader_schema, decoder):

~\Anaconda3\lib\site-packages\avro\io.py in read_data(self, writer_schema, reader_schema, decoder)
    532       return self.read_union(writer_schema, reader_schema, decoder)
    533     elif writer_schema.type in ['record', 'error', 'request']:
--> 534       return self.read_record(writer_schema, reader_schema, decoder)
    535     else:
    536       fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type

~\Anaconda3\lib\site-packages\avro\io.py in read_record(self, writer_schema, reader_schema, decoder)
    732       readers_field = readers_fields_dict.get(field.name)
    733       if readers_field is not None:
--> 734         field_val = self.read_data(field.type, readers_field.type, decoder)
    735         read_record[field.name] = field_val
    736       else:

~\Anaconda3\lib\site-packages\avro\io.py in read_data(self, writer_schema, reader_schema, decoder)
    510       return decoder.read_boolean()
    511     elif writer_schema.type == 'string':
--> 512       return decoder.read_utf8()
    513     elif writer_schema.type == 'int':
    514       return decoder.read_int()

~\Anaconda3\lib\site-packages\avro\io.py in read_utf8(self)
    260     except UnicodeDecodeError as exn:
    261       logger.error('Invalid UTF-8 input bytes: %r', input_bytes)
--> 262       raise exn
    263 
    264   def check_crc32(self, bytes):

~\Anaconda3\lib\site-packages\avro\io.py in read_utf8(self)
    257     input_bytes = self.read_bytes()
    258     try:
--> 259       return input_bytes.decode('utf-8')
    260     except UnicodeDecodeError as exn:
    261       logger.error('Invalid UTF-8 input bytes: %r', input_bytes)

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf0 in position 30: invalid continuation byte

Answer 1

An Avro object file can only contain a single schema, according to the spec.

The process of evolution is defined as having a reader schema that is different than the writer schema, but still being able to read old data.

For example, you could read a file that has no favorite film, but the reader schema defined a default favorite film of "none"