Swiping loses the touched element

Question

For an app I'm building, I use Animated and PanResponder in React Native, and the below code works, however, when swiping, one needs a very steady hand.

Would you know how I can make it more forgiving?

Here's a screencast: https://giphy.com/gifs/2wGSCsPGJwYcsWsRc4

import React, { useRef, useEffect } from 'react';
import PropTypes from 'prop-types';
import { useNavigation } from '@react-navigation/native';
import { Animated, PanResponder } from 'react-native';
import Emoji from './emoji';

const animStyle = {
  zIndex: 1,
  elevation: 1,
};

const animConfig = {
  useNativeDriver: false,
};

function Swipe({ emoji, swipePast, clubId }) {
  const { navigate } = useNavigation();
  const pan = useRef(new Animated.ValueXY()).current;
  const onPanResponderMove = useRef(Animated.event([null, { dx: pan.x }], animConfig)).current;
  useEffect(() => {
    function onPan({ x }) {
      if (x > swipePast) {
        pan.resetAnimation();
        navigate('Done', { clubId });
      }
    }
    pan.addListener(onPan);
    return () => pan.removeAllListeners();
  }, [clubId, pan, swipePast, navigate]);
  const { panHandlers } = useRef(
    PanResponder.create({
      onMoveShouldSetPanResponder: () => true,
      onPanResponderMove,
      onPanResponderRelease: () => {
        Animated.spring(pan, {
          ...animConfig,
          toValue: { x: 0, y: 0 },
        }).start();
      },
    }),
  ).current;
  return (
    <Animated.View
      // eslint-disable-next-line react/jsx-props-no-spreading
      {...panHandlers}
      style={{ ...animStyle, transform: [{ translateX: pan.x }] }}
    >
      <Emoji emoji={emoji} />
    </Animated.View>
  );
}

Swipe.propTypes = {
  emoji: PropTypes.string.isRequired,
  swipePast: PropTypes.number.isRequired,
  clubId: PropTypes.string.isRequired,
};

export default Swipe;