How to create a deterministic Jackson ObjectMapper?

Question

I would like to be able to generate an MD5 checksum of any Java POJO across JVMs. The approach would be to serialize the object to JSON then MD5 the JSON.

The issue is JSON serialization with Jackson is not deterministic mainly because many collections are not deterministic.

ObjectMapper mapper = new ObjectMapper()                                               
    .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)                                           
    .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true)
    ... // all other custom modules / features
;

These two features solve two of the problems of keeping fields sorted on POJOs as well as Maps.

The next challenge is to modify any collection on the fly and sort it. This requires every element in every collection to be sortable but let's assume that is ok for now.

Is there a way to intercept every collection and sort it before it is serialized?

Answer 1

I kind of achieved this with the following code. Read more on Creating a somewhat deterministic Jackson ObjectMapper

public class DeterministicObjectMapper {

    private DeterministicObjectMapper() { }

    public static ObjectMapper create(ObjectMapper original, CustomComparators customComparators) {
        ObjectMapper mapper = original.copy()
            .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
            .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);

        /*
         *  Get the original instance of the SerializerProvider before we add our custom module.
         *  Our Collection Delegating code does not call itself.
         */
        SerializerProvider serializers = mapper.getSerializerProviderInstance();

        // This module is reponsible for replacing non-deterministic objects
        // with deterministic ones. Example convert Set to a sorted List.
        SimpleModule module = new SimpleModule();
        module.addSerializer(Collection.class,
             new CustomDelegatingSerializerProvider(serializers, new CollectionToSortedListConverter(customComparators))
        );
        mapper.registerModule(module);
        return mapper;
    }

    /*
     * We need this class to delegate to the original SerializerProvider
     * before we added our module to it. If we have a Collection -> Collection converter
     * it delegates to itself and infinite loops until the stack overflows.
     */
    private static class CustomDelegatingSerializerProvider extends StdDelegatingSerializer
    {
        private final SerializerProvider serializerProvider;

        private CustomDelegatingSerializerProvider(SerializerProvider serializerProvider,
                                                   Converter<?, ?> converter)
        {
            super(converter);
            this.serializerProvider = serializerProvider;
        }

        @Override
        protected StdDelegatingSerializer withDelegate(Converter<Object,?> converter,
                                                       JavaType delegateType, JsonSerializer<?> delegateSerializer)
        {
            return new StdDelegatingSerializer(converter, delegateType, delegateSerializer);
        }

        /*
         *  If we do not override this method to delegate to the original
         *  serializerProvider we get a stack overflow exception because it recursively
         *  calls itself. Basically we are hijacking the Collection serializer to first
         *  sort the list then delegate it back to the original serializer.
         */
        @Override
        public JsonSerializer<?> createContextual(SerializerProvider provider, BeanProperty property)
                throws JsonMappingException
        {
            return super.createContextual(serializerProvider, property);
        }
    }

    private static class CollectionToSortedListConverter extends StdConverter<Collection<?>, Collection<?>>
    {
        private final CustomComparators customComparators;

        public CollectionToSortedListConverter(CustomComparators customComparators) {
            this.customComparators = customComparators;
        }
        @Override
        public Collection<? extends Object> convert(Collection<?> value)
        {
            if (value == null || value.isEmpty())
            {
                return Collections.emptyList();
            }

            /**
             * Sort all elements by class first, then by our custom comparator.
             * If the collection is heterogeneous or has anonymous classes its useful
             * to first sort by the class name then by the comparator. We don't care
             * about that actual sort order, just that it is deterministic.
             */
            Comparator<Object> comparator = Comparator.comparing(x -> x.getClass().getName())
                                                      .thenComparing(customComparators::compare);
            Collection<? extends Object> filtered = Seq.seq(value)
                                                       .filter(Objects::nonNull)
                                                       .sorted(comparator)
                                                       .toList();
            if (filtered.isEmpty())
            {
                return Collections.emptyList();
            }

            return filtered;
        }
    }

    public static class CustomComparators {
        private final LinkedHashMap<Class<?>, Comparator<? extends Object>> customComparators;

        public CustomComparators() {
            customComparators = new LinkedHashMap<>();
        }

        public <T> void addConverter(Class<T> clazz, Comparator<?> comparator) {
            customComparators.put(clazz, comparator);
        }

        @SuppressWarnings({ "unchecked", "rawtypes" })
        public int compare(Object first, Object second) {
            // If the object is comparable use its comparator
            if (first instanceof Comparable) {
                return ((Comparable) first).compareTo(second);
            }

            // If the object is not comparable try a custom supplied comparator
            for (Entry<Class<?>, Comparator<?>> entry : customComparators.entrySet()) {
                Class<?> clazz = entry.getKey();
                if (first.getClass().isAssignableFrom(clazz)) {
                    Comparator<Object> comparator = (Comparator<Object>) entry.getValue();
                    return comparator.compare(first, second);
                }
            }

            // we have no way to order the collection so fail hard
            String message = String.format("Cannot compare object of type %s without a custom comparator", first.getClass().getName());
            throw new UnsupportedOperationException(message);
        }
    }
}

Answer 2

I made a utility class to normalize json. It sorts properties by key and values, to sort values converts they as json string. The performance is not the best, but it works.

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

public final class JsonNormalized {

    private static final Logger LOGGER = LoggerFactory.getLogger(JsonNormalized.class);

    private JsonNormalized() {
    }

    public static String normalize(String json) {
        return serialize(deserialize(json));
    }

    private static String serialize(Object object) {
        try {
            return getObjectMapper().writeValueAsString(object);
        } catch (JsonProcessingException e) {
            LOGGER.error("Error serializing json", e);
            throw new RuntimeException(e);
        }
    }

    private static Object deserialize(String json) {
        try {
            JsonObject jsonObject = getObjectMapper().readValue(json, JsonObject.class);
            return jsonObject.getData();
        } catch (IOException e) {
            LOGGER.error("Error deserializing json", e);
            throw new RuntimeException(e);
        }
    }

    private static ObjectMapper getObjectMapper() {
        ObjectMapper mapper = new ObjectMapper();
        mapper.setVisibility(mapper.getSerializationConfig().getDefaultVisibilityChecker()
                .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
                .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
                .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
                .withCreatorVisibility(JsonAutoDetect.Visibility.NONE));
        mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
        mapper.configure(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS, false);
        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        mapper.setSerializationInclusion(Include.NON_NULL);
        mapper.disable(SerializationFeature.INDENT_OUTPUT);
        mapper.enableDefaultTyping(ObjectMapper.DefaultTyping.OBJECT_AND_NON_CONCRETE);
        mapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);

        mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
        mapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);

        SimpleModule module = new SimpleModule();
        module.addDeserializer(JsonObject.class, new JSONCustomDeserializer());
        mapper.registerModule(module);
        return mapper;
    }

}

class JsonObject {

    final Object data;

    JsonObject(Object data) {
        this.data = data;
    }

    public Object getData() {
        return data;
    }
}

class JSONCustomDeserializer extends JsonDeserializer<JsonObject> {

    @Override
    public JsonObject deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
        ObjectCodec oc = jp.getCodec();
        JsonNode node = oc.readTree(jp);
        return new JsonObject(toObject(node));
    }

    public Object toObject(JsonNode node) {

        if (node.fields().hasNext()) {
            Map<String, Object> mapResult = new TreeMap<>();
            for (Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
                Map.Entry<String, JsonNode> entryChildren = it.next();
                String childrenKey = entryChildren.getKey();
                JsonNode children = entryChildren.getValue();
                mapResult.put(childrenKey, children);
            }
            return new JsonObject(mapResult);
        } else if (node.elements().hasNext()) {
            List<Object> listResult = new ArrayList<Object>();
            for (Iterator<JsonNode> it = node.elements(); it.hasNext(); ) {
                JsonNode children = it.next();
                listResult.add(children);
            }
            Collections.sort(listResult, (lhs, rhs) -> {
                String lJson = JsonNormalized.serialize(lhs);
                String rJson = JsonNormalized.serialize(rhs);
                return lJson.compareTo(rJson);
            });

            return listResult;
        } else {
            return node.asText();
        }
    }
}