docker-compose es + kibana 集群/单机部署

Posted tang_jian_dong

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了docker-compose es + kibana 集群/单机部署相关的知识，希望对你有一定的参考价值。

配置参考官网

1： 集群部署

.env

# elastic账号的密码 (至少六个字符),自定义
ELASTIC_PASSWORD=

# kibana_system账号的密码 (至少六个字符)，该账号仅用于一些kibana的内部设置，不能用来查询es ,自定义
KIBANA_PASSWORD=

# es和kibana的版本
STACK_VERSION=8.5.3

# 集群名字,自定义
CLUSTER_NAME=es-cluster

# x-pack安全设置，这里选择basic，基础设置，如果选择了trail，则会在30天后到期
LICENSE=basic
#LICENSE=trial

# es映射到宿主机的的端口,自定义
ES_PORT=

# kibana映射到宿主机的的端口,自定义
KIBANA_PORT=

# es容器的内存大小，请根据自己硬件情况调整
MEM_LIMIT=1073741824

# 命名空间，会体现在容器名的前缀上,自定义
COMPOSE_PROJECT_NAME=es

新建目录

mkdir -p /docker/es
chmod g+rwx /docker/es
chgrp 0 /docker/es

docker-compose

version: '3.2'


networks:
  glo_es_bridge:
    external: true
    name: glo_es_bridge


services:
   setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
    volumes:
      - /docker/es/certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x$ELASTIC_PASSWORD == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x$KIBANA_PASSWORD == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \\
          "instances:\\n"\\
          "  - name: es01\\n"\\
          "    dns:\\n"\\
          "      - es01\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          "  - name: es02\\n"\\
          "    dns:\\n"\\
          "      - es02\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          "  - name: es03\\n"\\
          "    dns:\\n"\\
          "      - es03\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \\\\ \\;;
        find . -type f -exec chmod 640 \\\\ \\;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:$ELASTIC_PASSWORD" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "\\"password\\":\\"$KIBANA_PASSWORD\\"" | grep -q "^"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

   es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
    container_name: es01
    restart: always
    deploy:
      resources:
         limits:
            memory: 1G
    volumes:
      - /docker/es/certs:/usr/share/elasticsearch/config/certs
      - /docker/es/data01:/usr/share/elasticsearch/data
      - /docker/es/masterConfig/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /docker/es/masterConfig/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties

    ports:
      - $ES_PORT:9200
    environment:
      - node.name=es01
      - cluster.name=$CLUSTER_NAME
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es02,es03
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=$LICENSE
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    mem_limit: $MEM_LIMIT
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

   es02:
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
    container_name: es02
    restart: always
    deploy:
      resources:
         limits:
            memory: 1G
    volumes:
      - /docker/es/certs:/usr/share/elasticsearch/config/certs
      - /docker/es/data02:/usr/share/elasticsearch/data
      - /docker/es/followConfig/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /docker/es/followConfig/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
    environment:
      - node.name=es02
      - cluster.name=$CLUSTER_NAME
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=$LICENSE
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    mem_limit: $MEM_LIMIT
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120


   kibana:
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      #es03:
        #condition: service_healthy
    image: docker.elastic.co/kibana/kibana:$STACK_VERSION
    container_name: kb
    restart: always
    deploy:
      resources:
         limits:
            memory: 1G
    volumes:
      - docker/es/certs:/usr/share/kibana/config/certs
      - docker/kibana/data:/usr/share/kibana/data
      - docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - $KIBANA_PORT:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: $MEM_LIMIT
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
 

2：单机部署, 在集群的配置上修改而来

docker-compose

version: '3.2'


networks:
  glo_es_bridge:
    external: true
    name: glo_es_bridge


services:
   setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
    volumes:
      - /docker/es/certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x$ELASTIC_PASSWORD == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x$KIBANA_PASSWORD == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \\
          "instances:\\n"\\
          "  - name: es01\\n"\\
          "    dns:\\n"\\
          "      - es01\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          "  - name: es02\\n"\\
          "    dns:\\n"\\
          "      - es02\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          "  - name: es03\\n"\\
          "    dns:\\n"\\
          "      - es03\\n"\\
          "      - localhost\\n"\\
          "    ip:\\n"\\
          "      - 127.0.0.1\\n"\\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \\\\ \\;;
        find . -type f -exec chmod 640 \\\\ \\;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:$ELASTIC_PASSWORD" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "\\"password\\":\\"$KIBANA_PASSWORD\\"" | grep -q "^"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

   es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
    container_name: es01
    restart: always
    deploy:
      resources:
         limits:
            memory: 1G
    volumes:
      - /docker/es/certs:/usr/share/elasticsearch/config/certs
      - /docker/es/data01:/usr/share/elasticsearch/data
      - /docker/es/masterConfig/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /docker/es/masterConfig/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties

    ports:
      - $ES_PORT:9200
    environment:
      - node.name=es01
      - discovery.type=single-node
      - TZ=Asia/Shanghai
      #- cluster.name=$CLUSTER_NAME
      #- cluster.initial_master_nodes=es01,es02,es03
      #- discovery.seed_hosts=es02,es03
      - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=$LICENSE
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    mem_limit: $MEM_LIMIT
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

 #  es02:
 #   depends_on:
 #     - es01
 #   image: docker.elastic.co/elasticsearch/elasticsearch:$STACK_VERSION
 #   container_name: es02
 #   restart: always
 #   deploy:
 #     resources:
 #        limits:
 #           memory: 1G
 #   volumes:
 #     - /docker/es/certs:/usr/share/elasticsearch/config/certs
 #     - /docker/es/data02:/usr/share/elasticsearch/data
 #     - /docker/es/followConfig/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 #     - /docker/es/followConfig/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
 #   environment:
 #     - node.name=es02
 #     - cluster.name=$CLUSTER_NAME
 #     - cluster.initial_master_nodes=es01,es02,es03
 #     - discovery.seed_hosts=es01,es03
 #     - bootstrap.memory_lock=true
 #     - xpack.security.enabled=true
 #     - xpack.security.http.ssl.enabled=true
 #     - xpack.security.http.ssl.key=certs/es02/es02.key
 #     - xpack.security.http.ssl.certificate=certs/es02/es02.crt
 #     - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
 #     - xpack.security.transport.ssl.enabled=true
 #     - xpack.security.transport.ssl.key=certs/es02/es02.key
 #     - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
 #     - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
 #     - xpack.security.transport.ssl.verification_mode=certificate
 #     - xpack.license.self_generated.type=$LICENSE
 #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
 #   mem_limit: $MEM_LIMIT
 #   ulimits:
 #     memlock:
 #       soft: -1
 #       hard: -1
 #   healthcheck:
 #     test:
 #       [
 #         "CMD-SHELL",
 #         "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
 #       ]
 #     interval: 10s
 #     timeout: 10s
 #     retries: 120


   kibana:
    depends_on:
      es01:
        condition: service_healthy
     # es02:
     #   condition: service_healthy
      #es03:
        #condition: service_healthy
    image: docker.elastic.co/kibana/kibana:$STACK_VERSION
    container_name: kb
    restart: always
    deploy:
      resources:
         limits:
            memory: 1G
    volumes:
      - /docker/es/certs:/usr/share/kibana/config/certs
      - /docker/kibana/data:/usr/share/kibana/data
      - /docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - $KIBANA_PORT:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: $MEM_LIMIT
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
 

3： kibana nginx 代理

kibana.yml 配置：

server.basePath: "/kibana"

nginx：

   location ^~/kibana

            proxy_pass          http://127.0.0.1:端口/;
            rewrite ^/kibana/(.*)$ /$1 break;
            proxy_set_header    Host             $host:$server_port;
            proxy_set_header    REMOTE-HOST      $remote_addr;
            proxy_set_header    X-Real-IP        $remote_addr;
            proxy_set_header    X-Forwarded-For  $proxy_add_x_forwarded_for;

            #auth_basic "RestrictedDlAccess";
            #auth_basic_user_file /usr/local/nginx/conf/dlpassmm.txt;
       
 

 

以上是关于docker-compose es + kibana 集群/单机部署的主要内容，如果未能解决你的问题，请参考以下文章

docker-compose部署6.8.23版本elasticsearch+es-head+kibana多节点集群及部分排错处理

docker-compose安装elasticsearch集群

Docker搭建es集群（超简单）

批量搞机：分布式ELK平台Elasticsearch介绍Elasticsearch集群安装ES 插件的安装与使用

拆分版Docker-compose构建Elasticsearch 7.1.0集群

如何在Elasticsearch中删除中文索引