I have this debezium source connector and a jdbc-sink-connector as shown below:
{
"name": "smartdevsignupconnector111",
"config": {
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"database.hostname": "mysql1",
"database.port": "3306",
"database.user": "clusterAdmin",
"database.password": "RUNSman001",
"database.server.id": "184055",
"database.server.name": "smartdevdbserver1",
"database.include.list": "signup_db",
"schema.history.internal.kafka.topic": "schema-changes.signup_db",
"schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
"table.include.list": "signup_db.users",
"column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
"snapshot.mode": "when_needed",
"topic.creation.enable": "true",
"topic.prefix": "smartdevdbserver1",
"topic.creation.default.replication.factor": "1",
"topic.creation.default.partitions": "1",
"transforms": "unwrap,dropTopicPrefix",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
"transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
"transforms.dropTopicPrefix.replacement": "$1",
"include.schema.changes": "true"
}
}
{
"name": "resetpassword-sink-connector",
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"topics": "users",
"connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
"connection.user": "rpwd_user",
"connection.password": "*RUNSman001*",
"table.name.format": "users",
"fields.whitelist": "id,email,password,User_status,auth_token",
"transforms": "unwrap",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.unwrap.drop.tombstones": "false",
"auto.create": "true",
"insert.mode": "upsert",
"delete.enabled": "true",
"pk.fields": "id",
"pk.mode": "record_key"
}
}
Both are working well for cdc from mysql database table to kafka topic and sinking data from kafka topic to another mysql database table, but the issue is that deleting data is not working.
Deleting data from the source data doesn't get to delete the record in the kafka topic and also doesn't delete the record in the destination table.
Please can someone help show me what i am missing.
Edit:
Examples record from source table doesn't change at all, still looks same. Please see record below:
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}
The record i deleted from the source table is the very last record with email == testing14@firstclicklimited.com
mysql> delete from users where email = 'testing14@firstclicklimited.com';
Query OK, 1 row affected (0.11 sec)
Examples record from source table with --property print.key=true:
PS C:\Users\EBITI\Desktop\Sign-up> docker exec kafka1 /kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic users --from-beginning --property print.key=true
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":1}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":1,"email":"testing1@firstclicklimited.com","password":"$2a$10$Vah8yMoB3jopzwreKwHRKuH59UVFGXwxCSP0hQs99wcWEnbqLp7cO","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":2}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":2,"email":"testing2@firstclicklimited.com","password":"$2a$10$mqT6BtiLybFxuBpWcuiFt.M2IuL5O3bq6pB1CMUxqdyncMeVjKLNC","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":4}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":4,"email":"testing4@firstclicklimited.com","password":"$2a$10$9dsA1QCGNb31cloPeu3uq.w25rRzepS3mb04GcKZjIEOrl.ImcqDO","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":5}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":5,"email":"testing5@firstclicklimited.com","password":"$2a$10$52nsmqYYIit4.Ztmu6h4geAlvH1VkeauWIDu83i8FmOQohzFost7C","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":6}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":6,"email":"testing6@firstclicklimited.com","password":"$2a$10$fy4OT0W7pmV2pwOcRb4m8eHSWs8tA8ZvWOTrdK85SpmcHJBLy9lm6","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":7}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":7,"email":"testing7@firstclicklimited.com","password":"$2a$10$IvhBY9iVZkpRvg6M.LnQ4OO3c2cKUUjZbYMnII7ZWe.t0iYCK2L5u","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":29}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":29,"email":"testing11@firstclicklimited.com","password":"$2a$10$0KBAnV9AHhDeAe8jg4wkWeqrIE1hDvqFMjvsl9IR/6zmWPtb1C3M2","User_status":"INACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":36}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":36,"email":"testing12@firstclicklimited.com","password":"$2a$10$1IiE/vWLz6YwtJLpXR/InewQbEfsMc6VimiOzI6yR2WOmhzxURMsm","User_status":"ACTIVE","auth_token":null}}
{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Key"},"payload":{"id":50}} {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"email"},{"type":"string","optional":false,"field":"password"},{"type":"string","optional":false,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"ACTIVE,INACTIVE"},"default":"INACTIVE","field":"User_status"},{"type":"string","optional":true,"field":"auth_token"}],"optional":false,"name":"smartdevdbserver1.signup_db.users.Value"},"payload":{"id":50,"email":"testing14@firstclicklimited.com","password":"$2a$10$/6x9dP7MUKzWIbHluQG/0ebdA7tYO1qm5ky9X5YIVRSvDqQKuWdp.","User_status":"ACTIVE","auth_token":null}}
Okay so i figured out the problem. I had SMT "ExtractNewRecordState" on both source and sink connector, it should have been on only either of the connectors (in my case i choose to do SMT "ExtractNewRecordState" on the source connector). And then either transforms.unwrap.drop.tombstones should be set to false or delete.handling.mode should be set to none (you can set both at the same time, no issues) These is how the source and sink connectors look now:
{
"name": "smartdevsignupconnector111",
"config": {
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"database.hostname": "mysql1",
"database.port": "3306",
"database.user": "clusterAdmin",
"database.password": "RUNSman001",
"database.server.id": "184055",
"database.server.name": "smartdevdbserver1",
"database.include.list": "signup_db",
"schema.history.internal.kafka.topic": "schema-changes.signup_db",
"schema.history.internal.kafka.bootstrap.servers": "kafka1:9092",
"table.include.list": "signup_db.users",
"column.exclude.list": "signup_db.users.fullName, signup_db.users.address, signup_db.users.phoneNo, signup_db.users.gender, signup_db.users.userRole, signup_db.users.reason_for_inactive, signup_db.users.firstvisit, signup_db.users.last_changed_PW, signup_db.users.regDate",
"snapshot.mode": "when_needed",
"topic.creation.enable": "true",
"topic.prefix": "smartdevdbserver1",
"topic.creation.default.replication.factor": "1",
"topic.creation.default.partitions": "1",
"transforms": "unwrap,dropTopicPrefix",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.unwrap.drop.tombstones": "false",
"delete.handling.mode": "none",
"transforms.dropTopicPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
"transforms.dropTopicPrefix.regex": "smartdevdbserver1.signup_db.(.*)",
"transforms.dropTopicPrefix.replacement": "$1",
"include.schema.changes": "true"
}
}
{
"name": "resetpassword-sink-connector",
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
"tasks.max": "1",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "true",
"value.converter.schemas.enable": "true",
"topics": "users",
"connection.url": "jdbc:mysql://rpwd_mysql:3306/rpwd_db",
"connection.user": "rpwd_user",
"connection.password": "*RUNSman001*",
"table.name.format": "users",
"fields.whitelist": "id,email,password,User_status,auth_token",
"auto.create": "true",
"insert.mode": "upsert",
"delete.enabled": "true",
"pk.fields": "id",
"pk.mode": "record_key"
}
}
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.