k8s通过sidecar模式收集pod的容器日志至ELK

Posted 2023-02-28

架构：

已完成的部署

1、ES集群及kibana部署

​​https://blog.51cto.com/yht1990/6080981​​

2、kafaka+zookeeper集群

​​https://blog.51cto.com/yht1990/6081518​​

准备sidecar镜像(filebeat)

找一台服务器打镜像

[root@yw-test filebeat]# cat Dockerfile 
FROM docker.elastic.co/beats/filebeat:7.9.0
COPY filebeat.yml /usr/share/filebeat/filebeat.yml
USER root
RUN chown root:filebeat /usr/share/filebeat/filebeat.yml
USER filebeat
[root@yw-test filebeat]# 
[root@yw-test filebeat]# 
[root@yw-test filebeat]# cat filebeat.yml 
filebeat.inputs:
- input_type: log
  paths:
    - /data/logic/log/*.log
  fields:
    log_topic: "$TOPIC_ID"
  tail_files: true
  clean_inactive: 48h
  ignore_older: 24h
  close_inactive: 1m

output.kafka:
  hosts: ["10.0.7.53:9092", "10.0.7.54:9092", "10.0.7.55:9092"]
  topic: %[fields.log_topic]
  partition.round_robin:
    reachable_only: true
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

logging.level: error
[root@yw-test filebeat]# docker build . -t 10.0.7.12/k8s/filebeat/sidecar:7.9.0
[root@yw-test filebeat]# docker push 10.0.7.12/k8s/filebeat/sidecar:7.9.0

创建deployment

[root@k8s-test-manager src]# cat test-clavaplus-consume-sidecar-filebeat.yml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: test-clavaplus-consume-deployment-label
  name: test-clavaplus-consume-deployment
  namespace: test-app
spec:
  replicas: 2
  selector:
    matchLabels:
      app: test-clavaplus-consume-selector
  template:
    metadata:
      labels:
        app: test-clavaplus-consume-selector
    spec:
      imagePullSecrets:
      - name: myregistrykey
      containers:
      - name: test-clavaplus-consume
        image: registry.cn-shenzhen.aliyuncs.com/test/swoole:test-clavaplus-consume-2023-0227-1833
        imagePullPolicy: IfNotPresent
        #imagePullPolicy: Always
        ports:
        - containerPort: 9501
          protocol: TCP
          name: http
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 200m
            memory: "512Mi"
        volumeMounts:
        - name: applogs
          mountPath: /data/logic/log
        startupProbe:
          httpGet:
            path: /lavaHealthCheck
            port: 9501
          initialDelaySeconds: 5 #首次检测延迟5s
          failureThreshold: 3  #从成功转为失败的次数
          periodSeconds: 3 #探测间隔周期
        readinessProbe:
          httpGet:
            path: /lavaHealthCheck
            port: 9501
          initialDelaySeconds: 5
          periodSeconds: 3
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
        livenessProbe:
          httpGet:
            #path: /monitor/monitor.html
            path: /lavaHealthCheck
            port: 9501
          initialDelaySeconds: 5
          periodSeconds: 3
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
      - name: sidecar-filebeat
        image: 10.0.7.12/k8s/filebeat/sidecar:7.9.0
        imagePullPolicy: IfNotPresent
        #imagePullPolicy: Always
        env:
        - name: "TOPIC_ID"
          value: "test-clavaplus-consume"
        - name: "CODEC"
          value: "json"
        volumeMounts:
        - name: applogs
          mountPath: /data/logic/log
      volumes:
      - name: applogs #定义通过emptyDir实现业务容器与sidecar容器的日志共享，以让sidecar收集业务容器中的日志
        emptyDir: 

创建deployement

kubectl apply -f test-clavaplus-consume-sidecar-filebeat.yml

kafka工具检查有无数据

可以看到目前已有数据

部署logstash

yum -y install java-1.8.0-openjdk
yum -y install https://mirror.tuna.tsinghua.edu.cn/elasticstack/7.x/yum/7.9.0/logstash-7.9.0.rpm
systemctl start logstash
systemctl enable logstash

创建logstash配置文件

vim /etc/logstash/conf.d/clavaplus-consume.conf
input 
  kafka 
    bootstrap_servers => "10.0.7.53:9092,10.0.7.54:9092,10.0.7.55:9092"
    topics => ["test-clavaplus-consume"]
    codec => json 
        charset => "UTF-8"
    
  


output 
  if [fields][log_topic] == "test-clavaplus-consume"  
    elasticsearch 
      hosts => ["10.0.7.46:9200","10.0.7.47:9200","10.0.7.48:9200"]
      index => "test-clavaplus-consume-%+YYYY.MM.dd"
    

启动logstash

systemctl restart logstash && tail -f /var/log/logstash/logstash-plain.log

登陆kibana创建索引并查看数据