[英]delete is not working for debezium source connector/jdbc-sink-connector
I have this debezium source connector and a jdbc-sink-connector as shown below:我有这个 debezium 源连接器和一个 jdbc-sink-connector,如下所示:
{
"name": "smartdevsignupconnector111",
"config": {
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"database.hostname": "mysql1",
"database.port": "3306",
"database.user": "clusterAdmin",
"database.password": "RUNSman001",
"database.server.id": "184055",
"database.server.name": "smartdevdbserver1",
"database.include.list": "signup_db",
"schema.history.internal.kafka.topic": "schema-changes.signup_db",
"schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
"table.include.list": "signup_db.users",
"column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
"snapshot.mode": "when_needed",
"topic.creation.enable": "true",
"topic.prefix": "smartdevdbserver1",
"topic.creation.default.replication.factor": "1",
"topic.creation.default.partitions": "1",
"transforms": "unwrap,dropTopicPrefix",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
"transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
"transforms.dropTopicPrefix.replacement": "$1",
"include.schema.changes": "true"
}
}
{
"name": "resetpassword-sink-connector",
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"topics": "users",
"connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
"connection.user": "rpwd_user",
"connection.password": "*RUNSman001*",
"table.name.format": "users",
"fields.whitelist": "id,email,password,User_status,auth_token",
"transforms": "unwrap",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.unwrap.drop.tombstones": "false",
"auto.create": "true",
"insert.mode": "upsert",
"delete.enabled": "true",
"pk.fields": "id",
"pk.mode": "record_key"
}
}
Both are working well for cdc from mysql database table to kafka topic and sinking data from kafka topic to another mysql database table, but the issue is that deleting data is not working.两者都适用于从 mysql 数据库表到 kafka 主题的 cdc 以及从 kafka 主题到另一个 mysql 数据库表的下沉数据,但问题是删除数据不起作用。
Deleting data from the source data doesn't get to delete the record in the kafka topic and also doesn't delete the record in the destination table.从源数据中删除数据不会删除 kafka 主题中的记录,也不会删除目标表中的记录。
Please can someone help show me what i am missing.请有人能帮我看看我错过了什么。
Edit:编辑:
Examples record from source table doesn't change at all, still looks same.源表中的示例记录根本没有改变,看起来仍然一样。 Please see record below:
请看下面的记录:
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}
The record i deleted from the source table is the very last record with email == testing14@firstclicklimited.com我从源表中删除的记录是最后一条记录 email == testing14@firstclicklimited.com
mysql> delete from users where email = 'testing14@firstclicklimited.com';
Query OK, 1 row affected (0.11 sec)
Examples record from source table with --property print.key=true:使用 --property print.key=true 来自源表的示例记录:
PS C:\Users\EBITI\Desktop\Sign-up> docker exec kafka1 /kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic users --from-beginning --property print.key=true
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":1}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":2}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":4}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":5}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":6}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":7}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":29}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":36}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":50}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}
Okay so i figured out the problem.好吧,我想通了这个问题。 I had SMT "ExtractNewRecordState" on both source and sink connector, it should have been on only either of the connectors (in my case i choose to do SMT "ExtractNewRecordState" on the source connector).
我在源连接器和接收器连接器上都有 SMT“ExtractNewRecordState”,它应该只在其中一个连接器上(在我的例子中,我选择在源连接器上执行 SMT“ExtractNewRecordState”)。 And then either transforms.unwrap.drop.tombstones should be set to false or delete.handling.mode should be set to none (you can set both at the same time, no issues) These is how the source and sink connectors look now:
然后 transforms.unwrap.drop.tombstones 应设置为 false 或 delete.handling.mode 应设置为 none(您可以同时设置两者,没问题)这些是源连接器和接收器连接器现在的样子:
{
"name": "smartdevsignupconnector111",
"config": {
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"database.hostname": "mysql1",
"database.port": "3306",
"database.user": "clusterAdmin",
"database.password": "RUNSman001",
"database.server.id": "184055",
"database.server.name": "smartdevdbserver1",
"database.include.list": "signup_db",
"schema.history.internal.kafka.topic": "schema-changes.signup_db",
"schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
"table.include.list": "signup_db.users",
"column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
"snapshot.mode": "when_needed",
"topic.creation.enable": "true",
"topic.prefix": "smartdevdbserver1",
"topic.creation.default.replication.factor": "1",
"topic.creation.default.partitions": "1",
"transforms": "unwrap,dropTopicPrefix",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.unwrap.drop.tombstones": "false",
"delete.handling.mode": "none",
"transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
"transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
"transforms.dropTopicPrefix.replacement": "$1",
"include.schema.changes": "true"
}
}
{
"name": "resetpassword-sink-connector",
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"topics": "users",
"connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
"connection.user": "rpwd_user",
"connection.password": "*RUNSman001*",
"table.name.format": "users",
"fields.whitelist": "id,email,password,User_status,auth_token",
"auto.create": "true",
"insert.mode": "upsert",
"delete.enabled": "true",
"pk.fields": "id",
"pk.mode": "record_key"
}
}
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.