云原生(三十八) | Kubernetes篇之Jenkins入门和安装

​Jenkins入门和安装

一、helm安装

代码语言:javascript
复制
#可定制镜像
FROM jenkins/jenkins:lts
RUN jenkins-plugin-cli --plugins kubernetes workflow-aggregator git configuration-as-code
代码语言:javascript
复制
#helm 安装
helm repo add jenkinsci https://charts.jenkins.io/
helm pull jenkinsci/jenkins --version 3.3.18
代码语言:javascript
复制
controller:
  componentName: "jenkins-controller"
  image: "jenkinsci/blueocean"
  tag: "1.24.7"
  imagePullPolicy: "Always"
  adminSecret: true

adminUser: "admin"
adminPassword: "admin"

servicePort: 8080
targetPort: 8080
serviceType: ClusterIP

healthProbes: true
probes:
startupProbe:
httpGet:
path: '{{ default "" .Values.controller.jenkinsUriPrefix }}/login'
port: http
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 12
livenessProbe:
failureThreshold: 5
httpGet:
path: '{{ default "" .Values.controller.jenkinsUriPrefix }}/login'
port: http
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
failureThreshold: 3
httpGet:
path: '{{ default "" .Values.controller.jenkinsUriPrefix }}/login'
port: http
periodSeconds: 10
timeoutSeconds: 5
agentListenerServiceType: "ClusterIP"

installPlugins:
- kubernetes:1.29.4
- workflow-aggregator:2.6
- git:4.7.1
- configuration-as-code:1.51

initializeOnce: true

ingress:
enabled: true
paths:
- backend:
service:
name: jenkins
port:
number: 8080
path: "/"
pathType: Prefix
apiVersion: "networking.k8s.io/v1s"
kubernetes.io/ingress.class: nginx

hostName: jenkins.itlansonli.com
tls:
- secretName: itlansonli.com
  hosts:
    - jenkins.itlansonli.com

prometheus:
enabled: true
scrapeInterval: 60s
scrapeEndpoint: /prometheus

agent:
enabled: true
workspaceVolume:
type: PVC
claimName: jenkins-workspace-pvc
readOnly: false

additionalAgents:
maven:
podName: maven
customJenkinsLabels: maven
image: jenkins/jnlp-agent-maven

persistence:
enabled: true
storageClass: "rook-ceph-block"
accessMode: "ReadWriteOnce"
size: "8Gi"

代码语言:javascript
复制
 openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.cert -subj "/CN=.itlansonli.com/O=.itlansonli.com"

kubectl create ns devops
kubectl create secret tls itlansonli.com --key tls.key --cert tls.cert -n devops

代码语言:javascript
复制
helm install -f values.yaml -f override.yaml jenkins ./ -n devops
代码语言:javascript
复制
1. Get your 'admin' user password by running:
kubectl exec --namespace devops -it svc/jenkins -c jenkins -- /bin/cat /run/secrets/chart-admin-password && echo

  1. Visit http://jenkins.itlansonli.com

  2. Login with the password from step 1 and the username: admin

  3. Configure security realm and authorization strategy

  4. Use Jenkins Configuration as Code by specifying configScripts in your values.yaml file, see documentation: http://jenkins.itlansonli.com/configuration-as-code and examples: https://github.com/jenkinsci/configuration-as-code-plugin/tree/master/demos

For more information on running Jenkins on Kubernetes, visit:
https://cloud.google.com/solutions/jenkins-on-container-engine

For more information about Jenkins Configuration as Code, visit:
https://jenkins.io/projects/jcasc/

NOTE: Consider using a custom image with pre-installed plugins

二、手动安装

代码语言:javascript
复制
#创建证书,或者使用以前的证书
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=.itlansonli.com/O=.itlansonli.com"

1、编写Jenkins配置文件

代码语言:javascript
复制
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jenkins
namespace: devops
spec:
selector:
matchLabels:
app: jenkins # has to match .spec.template.metadata.labels
serviceName: "jenkins"
replicas: 1
template:
metadata:
labels:
app: jenkins # has to match .spec.selector.matchLabels
spec:
serviceAccountName: "jenkins"
terminationGracePeriodSeconds: 10
containers:
- name: jenkins
image: jenkinsci/blueocean:1.24.7
securityContext:
runAsUser: 0 #设置以ROOT用户运行容器
privileged: true #拥有特权
ports:
- containerPort: 8080
name: web
- name: jnlp #jenkins slave与集群的通信口
containerPort: 50000
resources:
limits:
memory: 2Gi
cpu: "2000m"
requests:
memory: 700Mi
cpu: "500m"
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: "JAVA_OPTS" #设置变量,指定时区和 jenkins slave 执行者设置
value: "
-Xmx$(LIMITS_MEMORY)m
-XshowSettings:vm
-Dhudson.slaves.NodeProvisioner.initialDelay=0
-Dhudson.slaves.NodeProvisioner.MARGIN=50
-Dhudson.slaves.NodeProvisioner.MARGIN0=0.75
-Duser.timezone=Asia/Shanghai
"
volumeMounts:
- name: home
mountPath: /var/jenkins_home
volumeClaimTemplates:

  • metadata:
    name: home
    spec:
    accessModes: [ "ReadWriteOnce" ]
    storageClassName: "rook-ceph-block"
    resources:
    requests:
    storage: 5Gi

apiVersion: v1
kind: Service
metadata:
name: jenkins
namespace: devops
spec:
selector:
app: jenkins
type: ClusterIP
ports:

  • name: web
    port: 8080
    targetPort: 8080
    protocol: TCP
  • name: jnlp
    port: 50000
    targetPort: 50000
    protocol: TCP

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: jenkins
namespace: devops
spec:
tls:

  • hosts:
    • jenkins.itlansonli.com
      secretName: itlansonli.com
      rules:
  • host: jenkins.itlansonli.com
    http:
    paths:
    • path: /
      pathType: Prefix
      backend:
      service:
      name: jenkins
      port:
      number: 8080

apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
namespace: devops


apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: jenkins
rules:

  • apiGroups: ["extensions", "apps"]
    resources: ["deployments"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  • apiGroups: [""]
    resources: ["services"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  • apiGroups: [""]
    resources: ["pods"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  • apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  • apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get","list","watch"]
  • apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: jenkins
roleRef:
kind: ClusterRole
name: jenkins
apiGroup: rbac.authorization.k8s.io
subjects:

  • kind: ServiceAccount
    name: jenkins
    namespace: devops

2、安装核心插件

代码语言:javascript
复制
- kubernetes
  • docker
    #以上插件可能无法下载,可以手动去jenkins-plugins下载并上传.
    kubectl cp /root/other/kubernetes-client-api.hpi devops/jenkins-0:/var/jenkins_home/plugins/
  • #配置镜像源
    https://updates.jenkins.io/update-center.json 默认的
    http://updates.jenkins-ci.org/update-center.json
    #或者国内源
    https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json

    3、配置集群整合

    3.1、动态slave架构

    3.2、配置整合

    点击《系统管理》—>《Configure System》—>《配置一个云》—>《kubernetes》,如下:

    3.3、配置kubernetes集群信息

    ​编辑

    3.4、配置slave的pod模板

    Slave就是动态运行起来的容器环境.

    jenkins的所有构建命令会在这个容器里面运行

    • 注意配置以下内容

      • 名称: 自定义
      • 命名空间 : devops
      • 标签列表: 自定义
      • 容器名称、镜像: jenkins/inbound-agent:4.7-1-alpine
      • serviceAccount挂载项: jenkins
      • 运行命令: 改为 jenkins-slave

    注意:

    • jenkins-url如果是一个域名,测试环境下可能不能访问,此时需要给各个主机配置域名转发到vpc网络的ip
    • 修改各个主机的 /etc/hosts文件即可
    • 也可以直接设置jenkins-url为公网ip地址

    4、测试动态slave

    4.1、自由风格

    4.2、流水线写法

    代码语言:javascript
    复制
    pipeline {
        agent {
          label 'maven-jnlp-slave'
        }
    
    stages {
        stage('Hello') {
            steps {
                echo 'Hello World'
            }
        }
    }
    

    }

    5、配置更多的slave

    slave-label

    镜像

    集成工具

    maven

    registry.cn-hangzhou.aliyuncs.com/lanson_k8s_images/jnlp-maven:3.6.3

    jq、curl、maven

    nodejs

    registry.cn-hangzhou.aliyuncs.com/lanson_k8s_images/jnlp-nodejs:14.16.1

    jq、curl、nodejs、npm(已经设置全局目录在 /root/npm下)

    kubectl

    registry.cn-hangzhou.aliyuncs.com/lanson_k8s_images/jnlp-kubectl:1.21.1

    kubectl、helm、helm-push、jq、curl、

    allin

    registry.cn-hangzhou.aliyuncs.com/lanson_k8s_images/jnlp-all:v1.0

    kubectl、helm、maven、nodejs、jq、curl

    docker

    registry.cn-hangzhou.aliyuncs.com/lanson_k8s_images/jnlp-docker:20.10.2

    jq、curl、docker

    根据自己环境可以有很多定制的slave

    三、slave参考配置

    下面的截图忘了选中 user,group 都应该为 0,0 也就是root用户

    每个打包机都应该hostPath模式挂载/etc/hosts文件。方便统一域名管理。或者全系统内部都不用域名,都使用ip进行交互也可以【但是推荐域名,域名可以统一修改,ip变化所有引用的地方都来修改很麻烦】

    1、maven配置

    使用要求

    • 1、提前创建好maven的settings.xml,并且以configMap的形式保存到k8s集群的devops名称空间。configmap名叫maven-conf,里面有一个键名settings.xml,值为 maven配置文件的值
    • 2、准备名为maven-jar-pvc 的pvc 在 devops名称空间下。为RWX模式
    • 例如:
    • kubectl create configmap maven-conf --from-file=settings.xml=/root/settings.xml -n devops
    代码语言:javascript
    复制
    apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
    name: maven-jar-pvc
    namespace: devops
    labels:
      app: maven-jar-pvc
    spec:
    storageClassName: rook-cephfs
    accessModes:
    - ReadWriteMany
    resources:
    requests:
    storage: 5Gi
    代码语言:javascript
    复制
    <?xml version="1.0" encoding="UTF-8"?>
    

    <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
    <!-- 这个目录是被maven打包机使用pvc挂载出去的
    -->
    <localRepository>/root/maven/.m2</localRepository>

    <pluginGroups>

    </pluginGroups>

    <proxies>

    </proxies>

    <servers>

    </servers>

    <mirrors>
    <mirror>
    <id>nexus-aliyun</id>
    <mirrorOf>central</mirrorOf>
    <name>Nexus aliyun</name>
    <url>http://maven.aliyun.com/nexus/content/groups/public</url>
    </mirror>
    </mirrors>
    <profiles>
    <profile>
    <id>jdk-1.8</id>
    <activation>
    <activeByDefault>true</activeByDefault>
    <jdk>1.8</jdk>
    </activation>
    <properties>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
    <maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion>
    </properties>
    </profile>
    </profiles>
    </settings>

    代码语言:javascript
    复制
    apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
    name: maven-jar-pvc
    namespace: devops
    labels:
    app: maven-jar-pvc
    spec:
    storageClassName: rook-cephfs
    accessModes:

    • ReadWriteMany
      resources:
      requests:
      storage: 5Gi

    2、kubectl配置

    在容器内 /root/.kube/config,config文件的内容是我们集群之前的admin.conf的内容

    使用要求

    • 必须提前给集群创建一个ConfigMap,名叫 kubectl-admin.conf,里面有一个键名叫config,键值可以是master节点 /root/.kube/config的内容
  • 例如
  • kubectl create configmap kubectl-admin.conf --from-file=config=/root/.kube/config -n devops
  • 3、nodejs配置

    使用说明

    • 准备名为npm-modules-pvc 的pvc 在 devops名称空间下。为RWX模式
    代码语言:javascript
    复制
    apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
        name: npm-modules-pvc
        namespace: devops
        labels:
           app: npm-modules-pvc
    spec:
        storageClassName: rook-cephfs
        accessModes:
        - ReadWriteMany
        resources:
           requests:
             storage: 5Gi

    4、docker配置

    使用注意:

    • docker访问harbor之类的私有仓库且是https,要注意配置证书受信任。提前各个机器配置好

    四、示例流水线

    1、打包机检查

    代码语言:javascript
    复制
    pipeline {
        //无代理,各阶段声明自己的代理
        agent none
        stages {
            stage('检查nodejs打包机') {
                //使用nodejs代理
                agent {
                    label 'nodejs'
                }
                steps {
                    echo "nodejs版本:"
                    sh 'node -v'
                    echo "npm modules目录位置"
                    sh 'npm config ls -l | grep prefix'
                    echo "检查完成..."
                }
            }
    
        stage(&#39;检查maven打包机&#39;) {
            //使用nodejs代理
            agent {
                label &#39;maven&#39;
            }
            steps {
                echo &#34;maven版本:&#34;
                sh &#39;mvn -v&#39;
                echo &#34;maven配置文件&#34;
                sh &#39;cat /app/maven/settings.xml&#39;
    
                echo &#34;maven目录位置信息&#34;
                sh &#39;ls -al /app/maven/&#39;
            }
        }
        stage(&#39;检查docker打包机&#39;) {
            //使用nodejs代理
            agent {
                label &#39;docker&#39;
            }
            steps {
                echo &#34;docker版本:&#34;
                sh &#39;docker version&#39;
                sh &#39;docker images&#39;
            }
        }
    
        stage(&#39;检查kubectl打包机&#39;) {
            //使用nodejs代理
            agent {
                label &#39;kubectl&#39;
            }
            steps {
                echo &#34;kubectl版本:&#34;
                sh &#39; kubectl version&#39;
                echo &#34;kubectl操作集群: 所有Pod&#34;
                sh &#39;kubectl get pods&#39;
    
                echo &#34;kubectl操作集群: 所有nodes&#34;
                sh &#39;kubectl get nodes&#39;
            }
        }
    }
    

    }

    2、复杂流水线

    2.1、java-Dockerfile模板

    代码语言:javascript
    复制
    #这个也得有
    FROM openjdk:8-jre-alpine
    LABEL maintainer="xxxxx@qq.com"
    #复制打好的jar包
    COPY target/*.jar /app.jar
    RUN apk add -U tzdata;
    ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime;
    echo 'Asia/Shanghai' >/etc/timezone;
    touch /app.jar;

    ENV JAVA_OPTS=""
    ENV PARAMS=""

    EXPOSE 8080

    ENTRYPOINT [ "sh", "-c", "java -Djava.security.egd=file:/dev/./urandom JAVA_OPTS -jar /app.jar PARAMS" ]

    2.1、准备数据库镜像

    代码语言:javascript
    复制
    FROM mysql:5.7

    所有在 /docker-entrypoint-initdb.d 下的sql,数据库会自己初始化

    COPY mall.sql /docker-entrypoint-initdb.d

    2.2、准备示例项目

    mall: mall项目是一套电商系统,包括前台商城系统及后台管理系统,基于SpringBoot+MyBatis实现,采用Docker容器化部署。 前台商城系统包含首页门户、商品推荐、商品搜索、商品展示、购物车、订单流程、会员中心、客户服务、帮助中心等模块。 后台管理系统包含商品管理、订单管理、会员管理、促销管理、运营管理、内容管理、统计报表、财务管理、权限管理、设置等模块。