[PKOS] Thanos를 통한 고가용성 모니터링(프로메테우스) 시스템 구축하기

1
2

Production Kubernetes Online Study (=PKOS)는 쿠버네티스 실무 실습 스터디입니다.
CloudNet@ Gasida(가시다)님이 진행하시며, 책 "24단계 실습으로 정복하는 쿠버네티스"을 기반으로 진행하고 있습니다.

1
2
3

helm repo add minio https://charts.bitnami.com/bitnami
helm repo update
helm fetch minio/minio --untar --version 12.2.1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# values-minio.yaml
mode: distributed

auth:
  rootUser: admin
  rootPassword: "admin1234"

statefulset:
  replicaCount: 4
  zones: 1
  drivesPerNode: 1

provisioning:
  config: 
    - name: region
      options:
        name: ap-northeast-2

ingress:
  enabled: true 
  hostname: minio.hanhorang.link
  path: /*
  annotations:
    kubernetes.io/ingress.class: alb
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}, {"HTTPS":9090}]'
    alb.ingress.kubernetes.io/certificate-arn: "$ACM arn "

persistence:
  storageClass: "kops-csi-1-21"

1
2

kubectl create ns minio
helm install minio minio/minio -f values-minio.yaml -n minio --version 12.2.1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "admin:*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "kms:*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:*"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}

1
2
3
4
5
6
7

#minio-key.yaml
type: s3
config:
  bucket: thanos
  endpoint: minio.hanhorang.link
  access_key: aajl91wFPCRVmfWR
  secret_key: SfP4woqjY3fcyh1cuwF1CNQFEe6hs4X6

1
2

kubectl create ns monitoring
kubectl create secret generic thanos-minio-secret -n monitoring --from-file=minio-key.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm fetch prometheus-community/kube-prometheus-stack --untar --version 45.7.1

# 사용 리전의 인증서 ARN 확인
CERT_ARN=`aws acm list-certificates --query 'CertificateSummaryList[].CertificateArn[]' --output text`
echo "alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN"

KOPS_CLUSTER_NAME="hanhorang.link"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

# values-kube-prometheus-stack.yaml

cat <<EOT > ./values-kube-prometheus-stack.yaml
alertmanager:
  enabled: false 

grafana:
  defaultDashboardsTimezone: Asia/Seoul
  adminPassword: admin1234

  ingress:
    enabled: true
    ingressClassName: alb

    annotations:
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
      alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN
      alb.ingress.kubernetes.io/success-codes: 200-399
      alb.ingress.kubernetes.io/group.name: "monitoring"

    hosts:
      - grafana.$KOPS_CLUSTER_NAME

    paths:
      - /*

prometheus:
   # 사이드카 노출 서비스 설정
  thanosService:
    enabled: true

  ingress:
    enabled: true
    ingressClassName: alb

    annotations:
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
      alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN
      alb.ingress.kubernetes.io/success-codes: 200-399
      alb.ingress.kubernetes.io/group.name: "monitoring"

    hosts:
      - prometheus.$KOPS_CLUSTER_NAME

    paths:
      - /*

  prometheusSpec:
    podMonitorSelectorNilUsesHelmValues: false
    serviceMonitorSelectorNilUsesHelmValues: false
    retention: 5d
    retentionSize: "10GiB"
    scrapeInterval: "15s"
    # alert 관련 설정으로 주석 처리 
    # evaluationInterval: 15s
    # 가용성 설정
    replicas: 3 
    # 타노스 설정
    thanos: 
      image: "quay.io/thanos/thanos:v0.27.0"
      objectStorageConfig:
        key: minio-key.yaml
        name: thanos-minio-secret
      version: v0.27.0
    # 볼륨 설정 
    storageSpec: {}
    ## Using PersistentVolumeClaim
    ##
    #  volumeClaimTemplate:
    #    spec:
    #      storageClassName: gluster
    #      accessModes: ["ReadWriteOnce"]
    #      resources:
    #        requests:
    #          storage: 50Gi
    #    selector: {}
EOT

1
2
3
4

kubectl create ns monitoring

helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack --version 45.7.1 \
-f values-kube-prometheus-stack.yaml --namespace monitoring

1

kubectl describe pods prometheus-kube-prometheus-stack-prometheus-0  -n monitoring

1
2
3

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm fetch bitnami/thanos --untar --version 12.3.2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

objstoreConfig: |-
  type: s3
  config:
    bucket: monitoring
    endpoint: minio.minio.svc.cluster.local:9000
    access_key: aajl91wFPCRVmfWR
    secret_key: SfP4woqjY3fcyh1cuwF1CNQFEe6hs4X6
    insecure: true   
querier:
  stores:
    - kube-prometheus-stack-thanos-discovery.monitoring.svc.cluster.local:10901
    - thanos-storegateway.monitoring.svc.cluster.local:10901
  replicaCount: 2
  ingress:
    enabled: true 
    hostname: thanos.hanhorang.link
    ingressClassName: "alb"
    annotations:
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]'
      alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:ap-northeast-2:955963799952:certificate/7569648c-bfd5-4860-b2c1-16ef02acbb58
      alb.ingress.kubernetes.io/success-codes: 200-399
      alb.ingress.kubernetes.io/group.name: "monitoring" 

    path : /*
bucketweb:
  enabled: true
compactor:
  enabled: true
storegateway:
  enabled: true
ruler:
  enabled: false

1
2
3

helm install thanos bitnami/thanos --version 12.3.2 \
-f values-thanos.yaml --namespace monitoring