从另一个 docker 容器连接到 Kafka

Question

This is my compose setup:

version: "3"

services:
  zoo:
    image: confluentinc/cp-zookeeper:7.2.1
    hostname: zoo
    container_name: zoo
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo:2888:3888

  kafka:
    image: confluentinc/cp-kafka:7.2.1
    hostname: kafka
    container_name: kafka
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    depends_on:
      - zoo

  fastapi:
    build: ./apiservice
    image: entryservice
    ports:
      - "8000:8000"
    restart: always
    depends_on:
      - zoo
      - kafka

My FastAPI Service looks like this:

from typing import Union
from confluent_kafka.admin import AdminClient, NewTopic
from confluent_kafka import Producer
from fastapi import FastAPI
from pydantic import BaseModel

admin = AdminClient({'bootstrap.servers': 'kafka:9092'})
producer = Producer({'bootstrap.servers': 'kafka:9092'})

new_topic = NewTopic("topic1", num_partitions=3, replication_factor=1)
new_topic2 = NewTopic("topic2", num_partitions=3, replication_factor=1)

fs = admin.create_topics([new_topic, new_topic2])

class Message(BaseModel):
    description: str

app = FastAPI()

@app.on_event("startup")
async def startup_event():
    for topic, f in fs.items():
        try:
            f.result()
            print(f"Topic {topic} wurde erstellt")
        except Exception as e:
            print(f"Error: Topic {topic} konnte nicht erstellt werden: {e}")

@app.post("/")
def create_message(message: Message):
    producer.produce('topic1', message.description.encode('utf-8'))
    producer.flush()
    return {"message": message.description}

When I start my service I get the following:

fastapi_1  | %3|1667288143.941|FAIL|rdkafka#producer-1| [thrd:127.0.0.1:9092/1]: 127.0.0.1:9092/1: Connect to ipv4#127.0.0.1:9092 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)

Running the same code (only change kafka:9092 to localhost:9092 ) locally works totally fine. I assume that something in my kafka configuration does not allow a connection other than from localhost. I took the setup from a github repo, so to be honest I don´t really too much about it. Does anyone know what to change to allow my other service connect to it?

Answer 1

It seems you are missing the.network in your docker-compose. Try to create a.network in this way. You can learn more about.networks from docker doc

edit

I tried your code and I found that the problem is just the kafka port number you are trying to connect to. You have to use the INTERNAL port (19092) and not the EXTERNAL one (9092). These ports are configured in KAFKA_ADVERTISED_LISTENERS env variable. If you want to reach your dockerized kafka from pycharm, and so from outside docker (external), you have to use 9092, but if you want to reach it from inside another container (internal) you have to use the internal one

version: "3"

services:
  zoo:
    image: confluentinc/cp-zookeeper:7.2.1
    hostname: zoo
    container_name: zoo
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo:2888:3888
    networks:
      - kafka_default

  kafka:
    image: confluentinc/cp-kafka:7.2.1
    hostname: kafka
    container_name: kafka
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    depends_on:
      - zoo
    networks:
      - kafka_default

  fastapi:
    build: ./apiservice
    image: entryservice
    ports:
      - "8000:8000"
    restart: always
    depends_on:
      - zoo
      - kafka
    networks:
      - kafka_default

networks:
  kafka_default:
    driver: bridge