目录结构如下所示

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

.
├── default.yaml
├── example_certs2
│   ├── choral.io.crt
│   ├── choral.io.key
│   ├── local.choral.io.crt
│   ├── local.choral.io.csr
│   └── local.choral.io.key
├── gitlab-minio.config
├── gitlab-minio.yaml
├── helm-gitlab.yaml
├── helm-traefik.yaml
├── hxf.md
├── ingressroute1-gitlab.yaml
├── ingressroute2-mini.yaml
├── ingressroute-dashboard.yaml
└── ingressroute-gitlab.yaml

2 directories, 15 files

用 helm 部署 gitlab

1
2
3
4

安装gitla报错
helm upgrade --install gitlab --namespace apps-gitlab \
  --values helm-gitlab.yaml \
  gitlab/gitlab

1
2
3

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
 helm install postgres bitnami/postgresql --namespace dev --create-namespace --set global.storageClass=local-path

helm install postgres bitnami/postgresql -–namespace dev –create-namespace –-set global.storageClass=local-storage

helm-gitlab.yml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

registry:
  enabled: false
certmanager-issuer:
#   # The email address to register certificates requested from Let's Encrypt.
#   # Required if using Let's Encrypt.
   email: email@example.com
postgresql:
  install: false
certmanager:
  install: false
prometheus:
  install: false
nginx-ingress:
  enabled: false
global:
  edition: ce
  time_zone: Asia/Shanghai
  kas:
    enabled: false
  minio:
    enabled: false
  hosts:
    https: true
    domain: local.choral.io
    gitlab:
      name: code.local.choral.io
  ingress:
    enabled: false
  psql:
    host: postgres-postgresql.dev.svc.cluster.local
    database: gitlab_test
    username: gitlab_test
    password:
      secret: gitlab-postgresql-secret
      key: postgresql-password
  appConfig:
    lfs:
      bucket: gitlab-gitlfs
      connection:
        secret: gitlab-minio-secret
        key: connection
    artifacts:
      bucket: gitlab-artifacts
      connection:
        secret: gitlab-minio-secret
        key: connection
    uploads:
      bucket: gitlab-uploads
      connection:
        secret: gitlab-minio-secret
        key: connection
    packages:
      bucket: gitlab-packages
      connection:
        secret: gitlab-minio-secret
        key: connection
    dependencyProxy:
      enabled: true
      bucket: gitlab-dependency-proxy
      connection:
        secret: gitlab-minio-secret
        key: connection
    backups:
      bucket: gitlab-backups
      tmpBucket: gitlab-backups-tmp
    defaultProjectsFeatures:
      issues: true
      mergeRequests: true
      wiki: true
      snippets: true
      builds: true
      containerRegistry: false
gitlab:
  webservice:
    registry:
      enabled: false
    resources:
      requests:
        cpu: 150m
  sidekiq:
    registry:
      enabled: false
    resources:
      requests:
        cpu: 200m
  toolbox:
    backups:
      objectStorage:
        config:
          secret: gitlab-minio-secret
          key: config
  gitaly:
    persistence:
      size: 20Gi
gitlab-runner:
  runners:
    privileged: true
    config: |
      [[runners]]
        pre_get_sources_script = "echo '10.7.20.242 code.local.choral.io' >> /etc/hosts \n  git config --global http.sslVerify false\n"
        [runners.kubernetes]
          image = "debian:bullseye"
          privileged = true
          image_pull_secrets = []
        [runners.cache]
          Type = "s3"
          Path = "runners"
          Shared = true
          [runners.cache.s3]
            ServerAddress = "http://minio.minio.svc.cluster.local:9000"
            AccessKey = "admin"
            SecretKey = "minio123"
            BucketName = "gitlab-caches"
            BucketLocation = "cn-north-1"
            Insecure = true
  gitlabUrl: http://gitlab-webservice-default:8181
  rbac:
   create: true
   rules:
    - apiGroups: [''] #"" indicates the core API group
      resources: ['*']
      verbs: ['*']
    - apiGroups: ['networking.k8s.io']
      resources: ['ingresses']
      verbs: ['*']
    - apiGroups: ['apps']
      resources: ['deployments']
      verbs: ['*']
   clusterWideAccess: true
   serviceAccountName: gitlab-runner

添加 helm chart

1
2

helm repo add gitlab https://charts.gitlab.io/
helm repo update

导出默认配置到文件

1

helm show values gitlab/gitlab > values.yaml

前期已经部署完成了 postgres 和 minio

配置 IngressGateway（这个没部署成功）

因为我使用的是 Istio 的 IngressGateway, 所以无法直接使用 Helm 的 Ingress 生成. 参考配置如下:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

apiVersion: networking.istio.io/v1beta1
kind: Gateway
metadata:
  name: gitlab-gateway
spec:
  selector:
    istio: ingressgateway # use Istio default gateway implementation
  servers:
  - port:
      number: 80
      name: http-gitlab
      protocol: HTTP
    hosts:
      - gitlab.xiaolanglang.net
  - port:
      number: 443
      name: https-gitlab
      protocol: HTTPS
    tls:
      mode: SIMPLE
      credentialName: xiaolanglang-net-wildcard-certificate # This should match the Certificate secretName
    hosts:
      - gitlab.xiaolanglang.net
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: gitlab-vs
spec:
  hosts:
    - gitlab.xiaolanglang.net
    - gitlab-webservice-default.gitlab.svc.cluster.local
  gateways:
  - gitlab-gateway
  - mesh
  http:
  - route:
    - destination:
        port:
          number: 8181
        host: gitlab-webservice-default.gitlab.svc.cluster.local
---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: gitlab-dr
spec:
  host: gitlab-webservice-default.gitlab.svc.cluster.local

用 ingressRoute 配置

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: gitlab-http-web
  namespace: apps-gitlab
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`code.local.choral.io`)
      kind: Rule
      services:
        - name: gitlab-webservice-default
          kind: Service
          port: 8181
  tls:
    secretName: local-choral-io-tls

• 创建命名空间

  本次演练中将 GitLab 安装至apps-gitlab命名空间，可根据需要替换。

  1	1kubectl create namespace apps-gitlab

• 创建 TLS 证书Secret

  从已准备好的证书key文件和crt文件创建Secret。

  1 2	1kubectl create secret tls local-choral-io-tls -n apps-gitlab \ 2 --key=local.choral.io.key --cert=local.choral.io.crt

• 创建Traefik EntryPoint

  创建一个新的Traefik EntryPoint，用于提供对 GitLab Shell SSH 协议的访问。

  首先，导出当前Traefik部署的配置文件。

  1	1helm get values --output yaml --namespace kube-system traefik > helm-traefik.yaml

  添加新的参数，更新 Traefik 部署。

  1 2 3 4 5 6 7 8 9

  1# ports.git-ssh.expose=false 禁用公开访问 稍后会手动创建用于访问该端口的负载均衡器 2# ports.git-ssh.port=8022 指定绑定端口 3# ports.git-ssh.protocol=TCP 指定绑定协议 4helm upgrade --install --namespace kube-system \ 5 --values helm-traefik.yaml \ 6 --set ports.git-ssh.expose=false \ 7 --set ports.git-ssh.port=8022 \ 8 --set ports.git-ssh.protocol=TCP \ 9 traefik traefik/traefik

• 创建负载均衡器

  创建一个新的LoadBalancer类型的Service，用于提供对22、80和443端口的访问。

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28

  1cat <<EOF | kubectl apply -f - > /dev/null 2apiVersion: v1 3kind: Service 4metadata: 5 name: traefik-git 6 namespace: kube-system 7 labels: 8 app.kubernetes.io/instance: traefik-kube-system 9 app.kubernetes.io/name: traefik 10spec: 11 type: LoadBalancer 12 selector: 13 app.kubernetes.io/instance: traefik-kube-system 14 app.kubernetes.io/name: traefik 15 ports: 16 - name: ssh 17 port: 22 18 protocol: TCP 19 targetPort: 8022 20 - name: web 21 port: 80 22 protocol: TCP 23 targetPort: web 24 - name: websecure 25 port: 443 26 protocol: TCP 27 targetPort: websecure 28EOF

安装 GitLab

• 创建PostgreSQL密码Secret

  1 2	1kubectl create secret -n apps-gitlab generic gitlab-postgresql-secret \ 2 --from-literal=postgresql-password=37Z8FeRZlkYuBtMWKtLsiLPz

• 创建MinIO认证凭据Secret

  创建gitlab-minio.yaml。

  1 2 3 4 5 6 7 8

  1provider: AWS 2region: cn-north-1 3aws_access_key_id: TL6JVVW85A9L4MFI4985 4aws_secret_access_key: ILnRAe8cuEJUmbCxTAPOodM3Rhu5gvD4ulZJskEL 5aws_signature_version: 4 6host: minio.local.choral.io 7endpoint: 'https://minio.local.choral.io' 8path_style: true

  创建gitlab-minio.config。

  1 2 3 4 5 6 7 8 9

  1[default] 2host_base = minio.local.choral.io 3host_bucket = minio.local.choral.io 4use_https = True 5signature_v2 = False 6access_key = TL6JVVW85A9L4MFI4985 7secret_key = ILnRAe8cuEJUmbCxTAPOodM3Rhu5gvD4ulZJskEL 8bucket_location = cn-north-1 9multipart_chunk_size_mb = 128

  创建包含上述两个文件的Secret对象。

  1 2 3

  1kubectl create secret -n apps-gitlab generic gitlab-minio-secret \ 2 --from-file=connection=gitlab-minio.yaml \ 3 --from-file=config=gitlab-minio.config

• 创建所需的 Buckets

  1	1mc mb choral-local/gitlab-{artifacts,backups,backups-tmp,caches,dependency-proxy,gitlfs,packages,uploads}

  1 2 3 4 5 6 7 8

  1Bucket created successfully `choral-local/gitlab-artifacts`. 2Bucket created successfully `choral-local/gitlab-backups`. 3Bucket created successfully `choral-local/gitlab-backups-tmp`. 4Bucket created successfully `choral-local/gitlab-caches`. 5Bucket created successfully `choral-local/gitlab-dependency-proxy`. 6Bucket created successfully `choral-local/gitlab-gitlfs`. 7Bucket created successfully `choral-local/gitlab-packages`. 8Bucket created successfully `choral-local/gitlab-uploads`.

• 创建helm-gitlab.yaml

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111

  registry: enabled: false postgresql: install: false certmanager: install: false prometheus: install: false nginx-ingress: enabled: false global: edition: ce time_zone: Asia/Shanghai kas: enabled: false minio: enabled: false hosts: https: true domain: local.choral.io gitlab: name: code.local.choral.io ingress: enabled: false psql: host: postgresql.data-postgresql database: gitlab username: gitlab password: secret: gitlab-postgresql-secret key: postgresql-password appConfig: lfs: bucket: gitlab-gitlfs connection: secret: gitlab-minio-secret key: connection artifacts: bucket: gitlab-artifacts connection: secret: gitlab-minio-secret key: connection uploads: bucket: gitlab-uploads connection: secret: gitlab-minio-secret key: connection packages: bucket: gitlab-packages connection: secret: gitlab-minio-secret key: connection dependencyProxy: enabled: true bucket: gitlab-dependency-proxy connection: secret: gitlab-minio-secret key: connection backups: bucket: gitlab-backups tmpBucket: gitlab-backups-tmp defaultProjectsFeatures: issues: true mergeRequests: true wiki: true snippets: true builds: true containerRegistry: false gitlab: webservice: registry: enabled: false resources: requests: cpu: 150m sidekiq: registry: enabled: false resources: requests: cpu: 200m toolbox: backups: objectStorage: config: secret: gitlab-minio-secret key: config gitaly: persistence: size: 20Gi gitlab-runner: runners: privileged: true config: | [[runners]] [runners.kubernetes] image = "debian:bullseye" privileged = true image_pull_secrets = [] [runners.cache] Type = "s3" Path = "runners" Shared = true [runners.cache.s3] ServerAddress = "minio.local.choral.io" AccessKey = "TL6JVVW85A9L4MFI4985" SecretKey = "ILnRAe8cuEJUmbCxTAPOodM3Rhu5gvD4ulZJskEL" BucketName = "gitlab-caches" BucketLocation = "cn-north-1" Insecure = false gitlabUrl: http://gitlab-webservice-default:8181

• 安装 GitLab

  使用上一步中创建的配置文件安装 GitLab。

  1 2 3

  helm upgrade --install gitlab --namespace apps-gitlab \ --values helm-gitlab.yaml \ gitlab/gitlab

• 配置 Ingress 入口规则

  创建用于提供 http 和 https 协议访问 Ingress Route 对象。

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19

  cat <<EOF | kubectl apply -f - > /dev/null apiVersion: traefik.io/v1alpha1 kind: IngressRoute metadata: name: gitlab-http-web namespace: apps-gitlab spec: entryPoints: - websecure routes: - match: Host(\`code.local.choral.io\`) kind: Rule services: - name: gitlab-webservice-default kind: Service port: 8181 tls: secretName: local-choral-io-tls EOF

  创建用于提供 SSH 协议访问的 Ingress Route TCP 对象。

  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

  cat <<EOF | kubectl apply -f - > /dev/null apiVersion: traefik.io/v1alpha1 kind: IngressRouteTCP metadata: name: gitlab-ssh-shell namespace: apps-gitlab spec: entryPoints: - git-ssh routes: - match: HostSNI(\`*\`) services: - name: gitlab-gitlab-shell port: 22 EOF

• 获取root用户初始密码

  1 2	kubectl get secret --namespace apps-gitlab gitlab-gitlab-initial-root-password \ -o jsonpath="{.data.password}" | base64 -d

  获取密码之后可以使用root用户名和密码登录 GitLab（本次演练中访问地址为https://code.local.choral.io/）。

参考资料

• https://docs.gitlab.com/charts/

• https://tiscs.choral.io/notes/k8s-gitlab/

• https://xiaolanglang.net/2022/07/03/install-gitlab/#more