Technobabble

Two node Kafka development cluster using docker

Omkar Patil's photo
Omkar Patil
·

1 min read

Here is a docker-compose yaml file to start a two Kafka cluster and kafdrop ui for a quick local development setup.

# STARTING UP: docker-compose up --remove-orphans -d
# STOP: docker-compose down
# CHECK LOGS: docker-compose logs -f -t
version: '3'

networks:
  kafka-net:
    driver: bridge

services:
  zookeeper:
    image: bitnami/zookeeper
    networks:
      - kafka-net
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    # volumes:
    #   - ./data/zookeeper/data:/data
    #   - ./data/zookeeper/datalog:/datalog
  kafka1:
    image: bitnami/kafka
    networks:
      - kafka-net
    ports:
      - '9091:9091'
    environment:
      - KAFKA_CFG_BROKER_ID=1
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_LISTENERS=INTERNAL://:19091,EXTERNAL://:9091
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka1:19091,EXTERNAL://localhost:9091
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
    # volumes:
    #   - ./data/kafka1/data:/var/lib/kafka/data
    depends_on:
      - zookeeper
    restart: always
  kafka2:
    image: bitnami/kafka
    networks:
      - kafka-net
    ports:
      - '9092:9092'
    environment:
      - KAFKA_CFG_BROKER_ID=2
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_LISTENERS=INTERNAL://:19092,EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka2:19092,EXTERNAL://localhost:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
    # volumes:
    #   - ./data/kafka2/data:/var/lib/kafka/data
    depends_on:
      - zookeeper
    restart: always
  kafdrop:
    image: obsidiandynamics/kafdrop
    networks:
      - kafka-net
    restart: 'no'
    ports:
      - '9000:9000'
    environment:
      - KAFKA_BROKERCONNECT=kafka1:19091,kafka2:19092
    depends_on:
      - kafka1
      - kafka2
kafkaDocker