Kaushik's Notes

Kafka Implementation

1 min read

1. Kafka with KRaft docker-compose

version: "3.8"
 
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    container_name: kafka
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      # Unique node ID for this broker
      KAFKA_NODE_ID: 1
      KAFKA_CLUSTER_ID: TswZGSRTSJS3glx7pIljtg
      CLUSTER_ID: TswZGSRTSJS3glx7pIljtg
 
      # Run as both broker and controller (KRaft mode)
      KAFKA_PROCESS_ROLES: broker,controller
 
      # Listeners for client (broker) and controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
 
      # Quorum voters (for multi-node clusters you add more IDs)
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
 
      # Log directories (must be inside the mounted volume below, otherwise nothing persists)
      KAFKA_LOG_DIRS: /var/lib/kafka/data
 
      # Single-node replication configs
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
 
      # Auto-create topics for dev
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
 
    volumes:
      - ./data:/var/lib/kafka/data

2. Create a unique cluster_id

docker run --rm confluentinc/cp-kafka:7.5.0 kafka-storage random-uuid

Output

TswZGSRTSJS3glx7pIljtg

3. Format the storage directory before starting Kafka

docker run --rm \
  -v $(pwd)/data:/var/lib/kafka/data \
  confluentinc/cp-kafka:7.5.0 \
  kafka-storage format \
    --ignore-formatted \
    --cluster-id=TswZGSRTSJS3glx7pIljtg \
    --config /etc/kafka/kafka.properties

4. Start Container

docker compose up -d
  • Check logs
docker logs -f kafka

Graph View