便捷高效的搭建Kafka集群和批量启动脚本

Posted 2023-02-20 Peter-OK

便捷高效的搭建Kafka集群和批量启动脚本

# kafka集群，伪集群
cd /usr/local/src
wget http://archive.apache.org/dist/kafka/2.3.0/kafka_2.11-2.3.0.tgz
tar -zxvf kafka_2.12-2.3.0.tgz -C /usr/local/
cd /usr/local/kafka_2.12-2.3.0/config
mkdir -p /kafkadata/kafka1,kafka2,kafka3
cp server.properties server1.properties
vim server1.properties
    broker.id=1
    delete.topic.enable=true
    listeners=PLAINTEXT://:9092
    advertised.listeners=PLAINTEXT://localhost:9092
    log.dirs=/kafkadata/kafka1
    zookeeper.connect=localhost:2181,localhost:2182,localhost:2183

cp server1.properties server2.properties
vim server2.properties
    broker.id=2
    delete.topic.enable=true
    listeners=PLAINTEXT://:9093
    log.dirs=/kafkadata/kafka2
    zookeeper.connect=localhost:2181,localhost:2182,localhost:2183

cp server1.properties server3.properties
vim server3.properties
    broker.id=3
    delete.topic.enable=true
    listeners=PLAINTEXT://:9094
    log.dirs=/kafkadata/kafka3
    zookeeper.connect=localhost:2181,localhost:2182,localhost:2183

# 启动集群
vim start.sh
    #!/bin/bash

    bash bin/kafka-server-start.sh -daemon config/server1.properties
    bash bin/kafka-server-start.sh -daemon config/server2.properties
    bash bin/kafka-server-start.sh -daemon config/server3.properties

# 停止集群
vim stop.sh 
    #!/bin/bash

    bash bin/kafka-server-stop.sh -daemon config/server1.properties
    bash bin/kafka-server-stop.sh -daemon config/server2.properties
    bash bin/kafka-server-stop.sh -daemon config/server3.properties



# 到此，有一个问题，需要在kafka-server-start.sh文件中配置端口，有如下三种办法:

# 第一种：在启动脚本start.sh中添加指定端口
# 第二种：多复制几个kafka-server-start.sh文件，然后进行修改，最后在start.sh中修改一下
# 第三种：复制并修改kafka目录，比如kafka_1,kafka_2,kafka_3,然后再每个目录下修改kafka-server-start.sh文件

# 以下三种方法任选其一即可

# 第一种方法，相应行修改成如下形式，注意端口号不同
# 使用的用一个目录下的同一个文件，只是在每个命令前指定端口号
vim start.sh
    #!/bin/bash

    JMX_PORT=9997 bash bin/kafka-server-start.sh -daemon config/server1.properties
    JMX_PORT=9998 bash bin/kafka-server-start.sh -daemon config/server2.properties
    JMX_PORT=9999 bash bin/kafka-server-start.sh -daemon config/server3.properties



# 第二种方法
# 使用的是同一个目录下的不同kafka-server-start文件
# start.sh文件也需要做相应的修改
cp kafka-server-start.sh kafka-server-start-1.sh
cp kafka-server-start.sh kafka-server-start-2.sh
cp kafka-server-start.sh kafka-server-start-3.sh

vim kafka-server-start-1.sh
    if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
       # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
        export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
        export JMX_PORT="9997"
    fi
vim kafka-server-start-2.sh
    if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
       # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
        export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
        export JMX_PORT="9998"
    fi
vim kafka-server-start-3.sh
    if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
       # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
        export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
        export JMX_PORT="9999"
    fi

vim start.sh 
    #!/bin/bash

    bash bin/kafka-server-start-1.sh -daemon config/server1.properties
    bash bin/kafka-server-start-2.sh -daemon config/server2.properties
    bash bin/kafka-server-start-3.sh -daemon config/server3.properties



# 第三种方法
# 使用的是不同目录下的不同kafka-server-start.sh文件
# start.sh文件也需要做相应的修改
# kafka_1/bin/kafka-server-start.sh
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
   # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
    export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
    export JMX_PORT="9997"
fi
# kafka_2/bin/kafka-server-start.sh
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
   # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
    export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
    export JMX_PORT="9998"
fi
# kafka_3/bin/kafka-server-start.sh
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
   # export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G"
    export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
    export JMX_PORT="9999"
fi

# start.sh
    #!/bin/bash
    bash kafka_1/bin/kafka-server-start.sh -daemon config/server.properties
    bash kafka_2/bin/kafka-server-start.sh -daemon config/server.properties
    bash kafka_3/bin/kafka-server-start.sh -daemon config/server.properties

 

附：zookeeper集群的按照请参考：

便捷高效的搭建zookeeper集群和批量启动脚本