堆栈内存溢出
  繁体   English   中英

删除不适用于 debezium 源连接器/jdbc-sink-connector

[英]delete is not working for debezium source connector/jdbc-sink-connector

 原文  2022-11-12 10:43:39  4   1    mysql/ jdbc/ apache-kafka/ apache-kafka-connect/ debezium

问题描述

我有这个 debezium 源连接器和一个 jdbc-sink-connector，如下所示：

{
  "name": "smartdevsignupconnector111",  
  "config": {  
    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
    "tasks.max": "1",
    "key.converter": "org.apache.kafka.connect.json.JsonConverter",
    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
    "key.converter.schemas.enable": "true",
    "value.converter.schemas.enable": "true",  
    "database.hostname": "mysql1",  
    "database.port": "3306",
    "database.user": "clusterAdmin",
    "database.password": "RUNSman001",
    "database.server.id": "184055",  
    "database.server.name": "smartdevdbserver1",  
    "database.include.list": "signup_db",
    "schema.history.internal.kafka.topic": "schema-changes.signup_db",
    "schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
    "table.include.list": "signup_db.users",
    "column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
    "snapshot.mode": "when_needed",
    "topic.creation.enable": "true",
    "topic.prefix": "smartdevdbserver1",
    "topic.creation.default.replication.factor": "1",
    "topic.creation.default.partitions": "1",
    "transforms": "unwrap,dropTopicPrefix",
    "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
    "transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
    "transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
    "transforms.dropTopicPrefix.replacement": "$1",
    "include.schema.changes": "true"
  }
}

{
    "name": "resetpassword-sink-connector",
    "config": {
        "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
        "tasks.max": "1",
        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
        "value.converter": "org.apache.kafka.connect.json.JsonConverter",
        "key.converter.schemas.enable": "true",
        "value.converter.schemas.enable": "true", 
        "topics": "users",
        "connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
        "connection.user": "rpwd_user",
        "connection.password": "*RUNSman001*",
        "table.name.format": "users",
        "fields.whitelist": "id,email,password,User_status,auth_token",
        "transforms": "unwrap",
        "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
        "transforms.unwrap.drop.tombstones": "false",
        "auto.create": "true",
        "insert.mode": "upsert",
        "delete.enabled": "true",
        "pk.fields": "id",
        "pk.mode": "record_key"
    }
}

两者都适用于从 mysql 数据库表到 kafka 主题的 cdc 以及从 kafka 主题到另一个 mysql 数据库表的下沉数据，但问题是删除数据不起作用。

从源数据中删除数据不会删除 kafka 主题中的记录，也不会删除目标表中的记录。

请有人能帮我看看我错过了什么。

编辑：

源表中的示例记录根本没有改变，看起来仍然一样。 请看下面的记录：

{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}

我从源表中删除的记录是最后一条记录 email == testing14@firstclicklimited.com

mysql> delete from users where email = 'testing14@firstclicklimited.com';
Query OK, 1 row affected (0.11 sec)

使用 --property print.key=true 来自源表的示例记录：

PS C:\Users\EBITI\Desktop\Sign-up> docker exec kafka1 /kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic users --from-beginning --property print.key=true
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":1}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":2}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":4}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":5}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":6}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":7}}      {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":29}}     {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":36}}     {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":50}}     {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}

1 个解决方案

解决方案1
 1 已采纳  2022-11-14 07:56:44

好吧，我想通了这个问题。 我在源连接器和接收器连接器上都有 SMT“ExtractNewRecordState”，它应该只在其中一个连接器上（在我的例子中，我选择在源连接器上执行 SMT“ExtractNewRecordState”）。 然后 transforms.unwrap.drop.tombstones 应设置为 false 或 delete.handling.mode 应设置为 none（您可以同时设置两者，没问题）这些是源连接器和接收器连接器现在的样子：

{
  "name": "smartdevsignupconnector111",  
  "config": {  
    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
    "tasks.max": "1",
    "key.converter": "org.apache.kafka.connect.json.JsonConverter",
    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
    "key.converter.schemas.enable": "true",
    "value.converter.schemas.enable": "true",  
    "database.hostname": "mysql1",  
    "database.port": "3306",
    "database.user": "clusterAdmin",
    "database.password": "RUNSman001",
    "database.server.id": "184055",  
    "database.server.name": "smartdevdbserver1",  
    "database.include.list": "signup_db",
    "schema.history.internal.kafka.topic": "schema-changes.signup_db",
    "schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
    "table.include.list": "signup_db.users",
    "column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
    "snapshot.mode": "when_needed",
    "topic.creation.enable": "true",
    "topic.prefix": "smartdevdbserver1",
    "topic.creation.default.replication.factor": "1",
    "topic.creation.default.partitions": "1",
    "transforms": "unwrap,dropTopicPrefix",
    "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
    "transforms.unwrap.drop.tombstones": "false",
    "delete.handling​.mode": "none",
    "transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
    "transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
    "transforms.dropTopicPrefix.replacement": "$1",
    "include.schema.changes": "true"
  }
}

{
    "name": "resetpassword-sink-connector",
    "config": {
        "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
        "tasks.max": "1",
        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
        "value.converter": "org.apache.kafka.connect.json.JsonConverter",
        "key.converter.schemas.enable": "true",
        "value.converter.schemas.enable": "true", 
        "topics": "users",
        "connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
        "connection.user": "rpwd_user",
        "connection.password": "*RUNSman001*",
        "table.name.format": "users",
        "fields.whitelist": "id,email,password,User_status,auth_token",
        "auto.create": "true",
        "insert.mode": "upsert",
        "delete.enabled": "true",
        "pk.fields": "id",
        "pk.mode": "record_key"
    }
}

暂无
暂无

声明:本站的技术帖子网页，遵循CC BY-SA 4.0协议，如果您需要转载，请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.

相关问题 如何使用 postgres JDBC sink 连接器流式传输到 mysql 连接器 尝试加载JDBC接收器连接器时出错 Confluent JDBC Source 连接器的问题 汇合 jdbc-source-connector 无法从 Java 适用于MySQL的JDBC连接器在JMeter中不起作用 Debezium MySQL Source Connector 是否支持 mysql ldap 可插拔身份验证 提取和转换 jdbc 接收器连接器的 kafka 消息特定字段 Kafka Connect JDBC Sink Connector 给 WorkerSinkTask ERROR Debezium - 自定义负载 - MySQL 连接器 无法在 debezium 连接器上生成过滤器
相关标签
 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM