基于K8s插件版的Jenkins动态节点实践内含最佳实践

Posted 2022-07-29 zeyangcom

手动在Jenkins Server上面创建一个Jenkins Agent节点，选择JNLP 类型。然后获取Jenkins Agent与Master连接所需要的​​JENKINS_AGENT_NAME​​​ 和​​JENKINS_AGENT_SECRET​​信息。

静态的AGENT节点，需要编写一个​​Deployment​​​部署文件并传递上面的连接信息，然后​​kubectl apply​​；

动态的AGENT节点，需要使用​​kubernetes​​插件， 首先配置CLOUD，添加Kubernetes集群的配置信息。然后编写Pipeline Agent部分的YAML和Stages。

1. 创建节点

Jenkins > 节点管理 > 添加节点

配置节点信息

创建成功（后面需要用到节点名称和secret信息）

2. 静态节点

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jenkinsagent
  namespace: devops
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: jenkinsagent
  template:
    metadata:
      labels:
        k8s-app: jenkinsagent
      namespace: devops
      name: jenkinsagent
    spec:
      containers:
        - name: jenkinsagent
          image: jenkins/inbound-agent:4.10-3-jdk8
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 1000m
              memory: 2Gi
            requests:
              cpu: 500m
              memory: 512Mi
          env:
            - name: JENKINS_URL
              value: http://192.168.1.200:8080
            - name: JENKINS_SECRET
              value: 2f3f27ee602171c5b51bc69c2bda501d703ac65f42de16600b3e3007dfdec422
            - name: JENKINS_AGENT_NAME
              value: buildserver1
            - name: JENKINS_AGENT_WORKDIR
              value: /home/jenkins/workspace

3. 动态节点

4. 最佳实践

@Library("mylib@feature-k8s") _     //加载共享库


podYaml = """
kind: Pod
apiVersion: v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jnlp
  namespace: devops
spec:
  securityContext:
    runAsUser: 0
  containers:
  - name: jnlp
    image: jenkins/inbound-agent:4.10-3-jdk8
    imagePullPolicy: IfNotPresent
  - name: maven
    image: maven:3.8.4-jdk-8
    imagePullPolicy: IfNotPresent
    command:
    - cat
    tty: true
  - name: dind
    image: docker:stable-dind
    command:
    - dockerd
    - --host=unix:///var/run/docker.sock
    - --host=tcp://0.0.0.0:8000
    - --insecure-registry=192.168.1.200:8088
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "docker"
    tty: true
    image: docker:19.03.15-git
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "curl"
    tty: true
    image: curlimages/curl:7.84.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  - name: "helm"
    tty: true
    image: dtzar/helm-kubectl:3.9.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  volumes:
    - name: docker-dir
      emptyDir: 
"""


pipeline
    agent
        kubernetes
            label "test01"
            cloud kubernetes
            yaml podYaml

        
    
    
    options 
        skipDefaultCheckout()
    

    stages
        stage("Checkout")
            steps
                script 
                    println("GetCode")
                    checkout([$class: GitSCM, 
                        branches: [[name: "$env.branchName"]], 
                        extensions: [], 
                        userRemoteConfigs: [[credentialsId: 62a3150f-38f8-4c9d-9334-79072b1d75cc, 
                                            url: "$env.srcUrl"]]])
                    
                    env.commitID = GetCommitID()
                    env.buName = "$JOB_NAME".split("-")[0]
                    env.serviceName = "$JOB_NAME".split("_")[0]
                    currentBuild.description = """ branchName: $env.branchName \\n"""
                    currentBuild.displayName = "$env.commitID"
                
            
        

        stage("Build")
            steps
                script
                    settingsContent = libraryResource config/settings.xml
                    writeFile file: settings.xml, text: settingsContent
                
                container(maven) 
                    sh "$env.buildShell  -s settings.xml"
                
            
        

        stage("PushArtifact")
            steps
                container("curl")
                    script 
                        //Dir /buName/serviceName/version/serviceName-version.xxx
                        version = "$env.branchName-$env.commitID"
                        // 重命名
                        JarName = sh returnStdout: true, script: """ls target | grep -E "jar\\$" """
                        fileName = JarName -"\\n"
                        fileType = fileName.split(\\\\.)[-1]
                        env.newFileName = "$env.serviceName-$version.$fileType"

                        sh "cd target ; mv $fileName  $newFileName "
                        PushArtifact("$env.buName/$env.serviceName/$version", "target", "$newFileName")
                    
                
            
        


        stage("Docker")
            steps
                script
                    env.registry = "192.168.1.200:8088"
                    // tag: branch-commid
                    env.version = "$env.branchName-$env.commitID"
                    env.imageName = "$env.registry/$env.buName/$env.serviceName:$env.version"
                
                container(docker)
                     withCredentials([usernamePassword(credentialsId: fad4e7c7-1f8b-45b2-83be-2b914bf08edf, passwordVariable: DOCKER_PASSWD, usernameVariable: DOCKER_USER)]) 

                        sh """
                            # 构建镜像
                            docker build -t $env.imageName .  --build-arg pkgname=target/$env.newFileName

                            #登录镜像仓库
                            docker login -u $DOCKER_USER -p $DOCKER_PASSWD $env.registry
                            
                            # 上传镜像
                            docker push $env.imageName

                            # 删除镜像
                            sleep 2
                            docker rmi $env.imageName
                        """
                    
                
            
        

        stage("ReleaseFile")
            steps
                container("curl")
                    script

                        // 下载模板文件
                        projectId = 18
                        fileData = GetRepoFile(projectId,"values.yaml", "master" )
                        //println(fileData)

                        yamlData = readYaml text: fileData  

                        // 替换模板文件内容
                        yamlData.image.tag  = "$env.version"
                        yamlData.image.repository = "$env.registry/$env.buName/$env.serviceName"
                        
                        sh "rm -fr values.yaml"
                        writeYaml  charset: UTF-8, file: values.yaml, data: yamlData
                        sh "cat values.yaml"

                         //上传替换后的版本文件（新建文件或者更新文件）
                        newYaml = sh returnStdout: true, script: cat  values.yaml
                        //println(newYaml)
                        //更新gitlab文件内容
                        base64Content = newYaml.bytes.encodeBase64().toString()

                        try 
                            CreateRepoFile(projectId,"values.yaml",base64Content, "master")
                        catch(e)
                            UpdateRepoFile(projectId,"values.yaml",base64Content, "master")
                        
                    
                
            
        
    


//获取CommitID
def GetCommitID()
    ID = sh returnStdout: true, script:"git rev-parse HEAD"
    ID = ID -"\\n"
    return ID[0..7] 


//上传制品
def PushArtifact(targetDir, filePath, fileName)
    sh """
        curl -X POST "http://192.168.1.200:8081/service/rest/v1/components?repository=devops4-local" \\
        -H "accept: application/json" \\
        -H "Content-Type: multipart/form-data" \\
        -F "raw.directory=$targetDir" \\
        -F "raw.asset1=@$filePath/$fileName;type=application/java-archive" \\
        -F "raw.asset1.filename=$fileName" \\
        -u admin:admin123
    """


//更新文件内容
def UpdateRepoFile(projectId,filePath,fileContent, branchName)
    apiUrl = "projects/$projectId/repository/files/$filePath"
    reqBody = """"branch": "$branchName","encoding":"base64", "content": "$fileContent", "commit_message": "update a new file""""
    response = HttpReqByHttpRequest(PUT,apiUrl,reqBody)
    println(response)




//创建文件
def CreateRepoFile(projectId,filePath,fileContent, branchName)
    apiUrl = "projects/$projectId/repository/files/$filePath"
    reqBody = """"branch": "$branchName","encoding":"base64", "content": "$fileContent", "commit_message": "update a new file""""
    response = HttpReqByHttpRequest(POST,apiUrl,reqBody)
    println(response)



//获取文件内容
def GetRepoFile(projectId,filePath, branchName )
   //GET /projects/:id/repository/files/:file_path/raw
   apiUrl = "/projects/$projectId/repository/files/$filePath/raw?ref=$branchName"
   response = HttpReq(GET, apiUrl)
   return response



// 封装HTTP
def HttpReqByHttpRequest(reqType, reqUrl,reqBody )
    def gitServer = "http://192.168.1.200:8077/api/v4"
    withCredentials([string(credentialsId: 553fcb38-81e1-4cdd-aa60-a8886a2eb323, variable: GITLABTOKEN)]) 
        response = httpRequest acceptType: APPLICATION_JSON_UTF8, 
                          consoleLogResponseBody: true, 
                          contentType: APPLICATION_JSON_UTF8, 
                          customHeaders: [[maskValue: false, name: PRIVATE-TOKEN, value: "$GITLABTOKEN"]], 
                          httpMode: "$reqType", 
                          url: "$gitServer/$reqUrl", 
                          wrapAsMultipart: false,
                          requestBody: "$reqBody"

    
    return response


def HttpReq(method, apiUrl)
    response = sh  returnStdout: true, 
        script: """ 
            curl --location --request $method \\
            http://192.168.1.200:8077/api/v4/$apiUrl \\
            --header PRIVATE-TOKEN: N9mvJV4hq-z7yCcYEsC- 
        """
    return response

@Library("mylib@feature-k8s") _     //加载共享库


podYaml = """
kind: Pod
apiVersion: v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jnlp
  namespace: devops
spec:
  securityContext:
    runAsUser: 0
  containers:
  - name: jnlp
    image: jenkins/inbound-agent:4.10-3-jdk8
    imagePullPolicy: IfNotPresent
  - name: maven
    image: maven:3.8.4-jdk-8
    imagePullPolicy: IfNotPresent
    command:
    - cat
    tty: true
  - name: dind
    image: docker:stable-dind
    command:
    - dockerd
    - --host=unix:///var/run/docker.sock
    - --host=tcp://0.0.0.0:8000
    - --insecure-registry=192.168.1.200:8088
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "docker"
    tty: true
    image: docker:19.03.15-git
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "curl"
    tty: true
    image: curlimages/curl:7.84.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  - name: "helm"
    tty: true
    image: dtzar/helm-kubectl:3.9.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  volumes:
    - name: docker-dir
      emptyDir: 
"""

env.projectName = "$JOB_NAME".split("_")[0]
env.buName = "$JOB_NAME".split("-")[0]
env.serviceName = "$JOB_NAME".split("_")[0]

pipeline
    agent
        kubernetes
            label "test01"
            cloud kubernetes
            yaml podYaml

        
    
    
    options 
        skipDefaultCheckout()
    

    stages
        stage("Checkout")
            steps
                script             
                    
                    println("GetCode")
                    sh "[ -d $env.serviceName ] || mkdir $env.serviceName"
                    ws("$WORKSPACE/$env.serviceName")
                        checkout([$class: GitSCM, 
                                branches: [[name: "$env.branchName"]], 
                                extensions: [], 
                                userRemoteConfigs: [[credentialsId: 62a3150f-38f8-4c9d-9334-79072b1d75cc, 
                                                    url: "$env.srcUrl"]]])
                        env.commitID = GetCommitID()
                        currentBuild.description = """ branchName: $env.branchName \\n"""
                        currentBuild.displayName = "$env.commitID"
                    
                    
                
            
        
        

        stage("HelmDeploy")
            steps
                container("helm")
                    script

                        withCredentials([file(credentialsId: cdcb0d0b-7420-4c7a-84dd-a47b57878999, variable: KUBE_CONFIG)]) 
                            content = sh returnStdout: true, script: "cat $KUBE_CONFIG"
                            writeFile file: ./config, text: content
                        

                        sh " mkdir -p /root/.kube ; mv config /root/.kube/"
                       
                        env.namespace = "$env.buName"
                        env.appName = "$env.serviceName"
                        currentBuild.description = "NS: $env.namespace"
                      
                        //Helm 发布
                        sh """
                            helm package "$env.appName/" 
                            helm upgrade --install --create-namespace  "$env.appName" ./"$env.appName"-*.tgz -n $env.namespace 
                            helm history "$env.appName"  -n $env.namespace 
                        """ 

                        //获取release的历史版本   
                        env.revision = sh returnStdout: true, script: """helm history $env.appName -n $env.namespace  | grep -v REVISION | awk print \\$1 """
                        println("$env.revision")
                        println("$env.revision.split(\\n).toString()")
                        env.REVISION = "$env.revision.split(\\n).toString()"
                        println("$env.REVISION")

                        // 获取应用状态
                        5.times
                            sh "sleep 2; kubectl -n $env.namespace get pod  | grep $env.appName"
                        

                    
                
            
        

        stage("CheckHealth")
            steps
                container("curl")
                    script
                        sh """
                           echo "192.168.1.200 devops4.ops.service" >>/etc/hosts
                           """
                        result = sh returnStdout: true, script: """ curl "http://$env.domainName/health" """
                        if (result == "ok\\n")
                            println("success!")
                        
                    
                
            
        

        stage("RollOut")
            input 
                message "是否进行回滚"
                ok "提交"
                submitter "zeyang,aa"
                parameters 
                    choice(choices: [yes, no], name: opts)
                
            

            steps

                container("helm")
                    script
                        switch("$opts") 
                            case "yes":
                                def result = input  message: "选择回滚版本?", 
                                parameters: [choice(choices: env.REVISION, name: rversion)]

                                println("$result")
                                sh "helm rollback $env.appName $result -n $env.namespace "
                                break

                            case "no":
                            break
                        
                    
                
            
        
    


//获取CommitID
def GetCommitID()
    ID = sh returnStdout: true, script:"git rev-parse HEAD"
    ID = ID -"\\n"
    return ID[0..7]