基于k8s构建企业jenkins CICD

Posted 2023-03-03

介绍及准备工作

1.概述

• 持续集成(Continuous Integration,CI)：代码合并、构建、部署、测试都在一起，不断地执行这个过程，并对结果反馈。
• 持续部署(Continuous Deployment,CD)：部署到测试环境、预生产环境、生产环境。
• 持续交付(Continuous Delivery,CD)：将最终产品发布到生产环境给用户使用。

闭环流程

引入k8s

部署到k8s平台流程

• 制作镜像
• 容器放到pod
• 控制器管理pod
• 暴露应用
• 对外发布应用
• 日志管理/监控

2.准备工作

部署harbor和git

1、下载安装包
wget https://storage.googleapis.com/harbor-releases/release-1.8.0/harbor-offline-installer-v1.8.1.tgz
2、解压
tar -zxvf harbor-offline-installer-v1.8.1.tgz 
3、编辑配置文件
cd harbor
vim harbor.yaml
hostname = x.x.x.x (ip)
4、准备配置
./prepare
5、导入镜像并启动
./install.sh
6、查看容器状态
docker-compose ps

# 安装docker
1、安装依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
2、添加源
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
3、安装
yum install docker-ce -y
4、配置加速器
curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://f1361db2.m.daocloud.io
systemctl restart docker
systemctl enable docker
5、安装docker-compose
curl -L https://get.daocloud.io/docker/compose/releases/download/1.22.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose

# 安装git
1、安装git
yum install git -y
2、创建git用户并设置密码
useradd git
passwd git
3、创建仓库
su - git
mkdir app-git
cd app-git
git --bare init
4、配置客户端与git服务器SSH免密交互
ssh-keygen  # 一路回车
ssh-copy-id git@192.168.100.2
5、测试
git clone git@192.168.100.2:/home/
git add .
git commit -m "test"
git push origin master

k8s中部署jenkins

# nfs安装及挂载
1、安装nfs
yum install nfs-utils -y
2、设置共享暴露目录
vim /etc/exports
/nfs *(rw,no_root_squash)
3、启动服务
systemctl start nfs
4、客户端挂载
mount -t nfs 192.168.100.2:/nfs /mnt

构建jenkins-slave镜像

• 构建镜像

Dockerfile：构建jenkins-slave镜像
jenkins-slave：shell脚本（需要加入可执行权限 chmod +x jenkins-slave），在镜像构建时需要用到。
slave.jar: 启动脚本

# Dockerfile
FROM ubuntu
ENV JAVA_HOME /usr/local/jdk
ENV MAVEN_HOME /usr/local/maven
ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH

RUN apt-get update && \\
    apt-get install -y curl git libltdl-dev && \\ 
    apt-get clean all && \\ 
    mkdir -p /usr/share/jenkins

COPY slave.jar /usr/share/jenkins/slave.jar  
COPY jenkins-slave /usr/bin/jenkins-slave
RUN  chmod +x /usr/bin/jenkins-slave

ENTRYPOINT ["jenkins-slave"]

# centos版的
FROM centos:7

RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \\
    yum clean all && \\
    rm -rf /var/cache/yum/* && \\
    mkdir -p /usr/share/jenkins

COPY slave.jar /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/bin/jenkins-slave
COPY settings.xml /etc/maven/settings.xml
RUN chmod +x /usr/bin/jenkins-slave

ENTRYPOINT ["jenkins-slave"]

• 配置java的基础环境

配置 JDK和maven，把下载的二进制包解压放到如下目录
apache-maven-3.6.2-bin.tar.gz解压到的地址为 /usr/local/maven里面。
jdk-8u231-linux-x64.tar.gz解压到的地址为 /usr/local/jdk里面。
# 配置maven源，copy出maven的conf下的settings.xml文件
    <mirror>
      <id>central</id>
      <mirrorOf>central</mirrorOf>
      <name>aliyun maven</name>
      <url>https://maven.aliyun.com/repository/public</url>
    </mirror>

• 构建镜像

docker build -t 192.168.100.2/ops/jenkins-slave .

• 推送镜像

1、docker中添加harbor仓库ip，否则可能无法正常登录
vim /etc/docker/daemon.json

    "registry-mirrors": ["https://registry.docker-cn.com"],
    "insecure-registries": ["192.168.100.2"]

修改完重启下docker
2、登录harbor
docker login 192.168.100.2
docker push 192.168.100.2/ops/jenkins-slave

部署jenkins

jenkins-service-account.yml: jenkins的服务账号创建
jenkins.yml: 创建容器和service 服务得。进行可以访问。
Dockerfile: 主要用来生成 jenkins 镜像的。
registry-pull-secret.yaml：主要是用来部署时可以直接登录harbor仓库来拉取镜像（在部署jenkins需要用到）

# 构建镜像,并推送到harbor
docker build -t 192.168.100.2/ops/jenkins:lts-alpine .
docker push 192.168.100.2/ops/jenkins:lts-alpine

# 添加harbor登录信息
生成方式为: 可以在有登录的harbor 的node服务器上面执行以下命令生成认证信息
cat ~/.docker/config.json |base64 -w0 即可生成信息,然后替换registry-pull-secret.yaml里的.dockerconfigjson内容

# 依次执行yml文件
kubectl create -f jenkins-service-account.yml
# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins
  
# registry-pull-secret.yaml
kubectl create -f registry-pull-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: registry-pull-secret
  namespace: default
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjEwMC4yIjogewoJCQkiYXV0aCI6ICJZV1J0YVc0NmVIaDZlRUEzT0RrPSIKCQl9Cgl9LAoJIkh0dHBIZWFkZXJzIjogewoJCSJVc2VyLUFnZW50IjogIkRvY2tlci1DbGllbnQvMTkuMDMuMyAobGludXgpIgoJfQp9 
type: kubernetes.io/dockerconfigjson

# jenkins.yml
kubectl create -f jenkins.yml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-jenkins-pv
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /nfs/jenkins_home
    server: 192.168.100.2

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-jenkins-pvc
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi

---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  serviceName: jenkins
  replicas: 1
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccountName: jenkins
#      imagePullSecrets:
#        - name: registry-pull-secret 
      containers:
        - name: jenkins
          image: 192.168.100.2/ops/jenkins:lts-alpine
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
            - containerPort: 50000
          resources:
            limits:
              cpu: 1
              memory: 1Gi
            requests:
              cpu: 0.5
              memory: 500Mi
          env:
            - name: LIMITS_MEMORY
              valueFrom:
                resourceFieldRef:
                  resource: limits.memory
                  divisor: 1Mi
            - name: JAVA_OPTS
              # value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
              value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          volumeMounts:
            - name: jenkins-home
              mountPath: /var/jenkins_home
          livenessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
      #securityContext:
      #  fsGroup: 1000
      volumes:
        - name: jenkins-home
          persistentVolumeClaim:
            claimName: nfs-jenkins-pvc

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  annotations:
    # ensure the client ip is propagated to avoid the invalid crumb issue (k8s <1.7)
    # service.beta.kubernetes.io/external-traffic: OnlyLocal
spec:
  #type: LoadBalancer
  type: NodePort
  selector:
    name: jenkins
  # k8s 1.7+
  # externalTrafficPolicy: Local
  ports:
    -
      name: http
      port: 80
      targetPort: 8080
      protocol: TCP
      nodePort: 30003
    -
      name: agent
      port: 50000
      protocol: TCP

• 访问

kubectl log jenkins-0   # 查看管理员密码
# 或者进入jenkins容器里查看

#私藏项目实操分享#