docker-compose部署kafka集群_docker-compose kafka

背景

用于测试、开发环境部署kafka集群，实际上也可以用于生产环境，但是需要调整kafka集群参数、镜像系统参数，以达到kafka运行的最佳环境。

本例因为资源有限，在一台服务器上进行集群模拟安装。

安装zookeeper集群

创建工作目录

mkdir /data/zk
1

编写docker-compose文件

version: '3'

services:
  zoo1:
    image: zookeeper
    restart: always
    hostname: zoo1
    container_name: zoo1
    # 每台zk对外的端口
    ports:
      - 12181:2181
    volumes:
    # 数据目录及快照目录
      - /data/zk/zk1/data:/data
      - /data/zk/zk1/datalog:/datalog
    environment:
      # zk服务器唯一Id
      ZOO_MY_ID: 1
      # zk集群的服务器
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

  zoo2:
    image: zookeeper
    restart: always
    hostname: zoo2
    container_name: zoo2
    ports:
      - 22181:2181
    volumes:
      - /data/zk/zk2/data:/data
      - /data/zk/zk2/datalog:/datalog
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

  zoo3:
    image: zookeeper
    restart: always
    hostname: zoo3
    container_name: zoo3
    ports:
      - 32181:2181
    volumes:
      - /data/zk/zk3/data:/data
      - /data/zk/zk3/datalog:/datalog
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

启动zk集群

cd /data/zk
docker-compose up -d
1
2

安装kafka集群

创建工作目录

mkdir /data/kafka
1

创建yaml文件

version: '3'
services:
  kafka1:
    image: wurstmeister/kafka
    container_name: kafka1
    ports:
      - "19092:9092"
      - "19999:19999"
    environment:
	  #镜像的参数规则参考：https://github.com/wurstmeister/kafka-docker
      KAFKA_BROKER_ID: 0
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:12181,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:19092
      JMX_PORT: 19999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=19999 -Dcom.sun.management.jmxremote.port=19999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
    volumes:
      - /data/kafka/broker1/logs:/opt/kafka/logs
    restart: always
 
  kafka2:
    image: wurstmeister/kafka
    container_name: kafka2
    ports:
      - "29092:9092"
      - "29999:29999"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:12181,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:29092
      JMX_PORT: 29999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=29999 -Dcom.sun.management.jmxremote.port=29999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
    volumes:
      - /data/kafka/broker2/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
  
  kafka3:
    image: wurstmeister/kafka
    container_name: kafka3
    ports:
      - "39092:9092"
      - "39999:39999"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:2182,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:39092
      JMX_PORT: 39999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=39999 -Dcom.sun.management.jmxremote.port=39999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
    volumes:
      - /data/kafka/broker3/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

启动集群

cd /data/kafka
docker-compose up -d
1
2