Установка Kafka

Установка на сервер локально

Устанавливаем JDK:

# yum install java-11-openjdk.x86_64

Создаем пользователя и даем ему права sudo:

# adduser kafka
# yum install sudo
# echo "kafka ALL=(ALL)   NOPASSWD: ALL" >> /etc/sudoers
# su -l kafka

Скачиваем и распаковываем Kafka:

$ mkdir ~/Downloads
$ curl "https://downloads.apache.org/kafka/3.3.2/kafka_2.12-3.3.2.tgz" -o ~/Downloads/kafka.tgz
$ mkdir -p ~/kafka &&cd ~/kafka
$ tar -xvzf ~/Downloads/kafka.tgz --strip 1

Редактируем конфигурационный файл ~/kafka/config/server.properties

Добавляем:

delete.topic.enable = true

Изменяем:

log.dirs=/home/kafka/logs

Создаем сервис для автозапуска zookeeper /etc/systemd/system/zookeeper.service

[Unit]
Requires=network.target remote-fs.target
After=network.target remote-fs.target

[Service]
Type=simple
User=kafka
ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties
ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh
Restart=on-abnormal

[Install]
WantedBy=multi-user.target

Создаем сервис для запуска kafka /etc/systemd/system/kafka.service:

[Unit]
Requires=zookeeper.service
After=zookeeper.service

[Service]
Type=simple
User=kafka
ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties > /home/kafka/kafka/kafka.log 2>&1'
ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh
Restart=on-abnormal

[Install]
WantedBy=multi-user.target

Запускаем:

# systemctl enable zookeeper.service
# systemctl start zookeeper.service
# systemctl enable kafka.service
# systemctl start kafka.service

Проверка работоспособности

Проверяем, что сервисы запущены и работают:

# systemctl status kafka.service
# systemctl status zookeeper.service

Создаем topic:

$ ~/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic TutorialTopic
Created topic TutorialTopic.

 Публикуем сообщение в topic:

$ echo"Hello, World" | ~/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic TutorialTopic

Проверяем, что есть сообщение:

$ ~/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic TutorialTopic --from-beginning

Запрашиваем все topics:

$ ~/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092 localhost:2181

Адаптируем под SWARM

version: "3.9"
services:

  kafka:
    image: "confluentinc/cp-kafka:5.5.0"
    environment:
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
      KAFKA_LOG_RETENTION_HOURS: "24"
      KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust
      KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too
      CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
      KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN"
      KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
      KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
    ports:
      - "9092:9092"
    volumes:
      - "./sentry-kafka:/var/lib/kafka/data"
      - "./sentry-kafka-log:/var/lib/kafka/log"
      - "./sentry-secrets:/etc/kafka/secrets"
      - /etc/hosts:/etc/hosts:ro
    healthcheck:
      test: ["CMD-SHELL", "nc -z localhost 9092"]
    depends_on:
      - zookeeper
    deploy:
      resources:
        limits:
          cpus: '2'
          memory: 4096M
        reservations:
          cpus: '0.5'
          memory: 2048M

  zookeeper:
    image: "confluentinc/cp-zookeeper:5.5.0"
    environment:
      ZOOKEEPER_CLIENT_PORT: "2181"
      CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
      ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
      KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
    ports:
      - "2181:2181"
    volumes:
      - "./sentry-zookeeper:/var/lib/zookeeper/data"
      - "./sentry-zookeeper-log:/var/lib/zookeeper/log"
      - "./sentry-secrets:/etc/zookeeper/secrets"
      - /etc/hosts:/etc/hosts:ro
    healthcheck:
      test:
        ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok']
    deploy:
      resources:
        limits:
          cpus: '2'
          memory: 2048M
        reservations:
          cpus: '0.5'
          memory: 1024M