部署日志收集系统

说明loki 日志收集系统分三个部分

Loki

主要功能存储日志，存储方式有多种，我们这里选择minio 对象存储，集中管理，后面的链路追踪也放在minio中

Promtail

主要功能是收集日志，发送给loki存储, 再本系统中Promtail 负责收集k8s中所有带有 loki.logs/enable: "true" 注解的pod的标准输出

Grafana

可视化前端,查询loki存储的日志,这个没有技术难点，可以官网上找下文档，部署上面即可

loki-value.yaml 文件编写

loki:
  schemaConfig:
    configs:
      - from: "2024-04-01"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  querier:
    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
    max_concurrent: 4
  pattern_ingester:
    enabled: true
  limits_config:
    allow_structured_metadata: true
    volume_enabled: true
# Should authentication be enabled
auth_enabled: false

deploymentMode: SimpleScalable
backend:
  replicas: 2
  persistence:
    # -- Enable volume claims in pod spec
    volumeClaimsEnabled: true
    # -- Parameters used for the `data` volume when volumeClaimEnabled if false
    dataVolumeParameters:
      emptyDir: {}
    # -- Enable StatefulSetAutoDeletePVC feature
    enableStatefulSetAutoDeletePVC: true
    # -- Size of persistent disk
    size: 50Gi
    # -- Storage class to be used.
    # If defined, storageClassName: <storageClass>.
    # If set to "-", storageClassName: "", which disables dynamic provisioning.
    # If empty or set to null, no storageClassName spec is
    # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
    storageClass: "nfs-client"
read:
  replicas: 2
write:
  replicas: 3 # To ensure data durability with replication
  persistence:
        # -- Enable volume claims in pod spec
        volumeClaimsEnabled: true
        # -- Parameters used for the `data` volume when volumeClaimEnabled if false
        dataVolumeParameters:
          emptyDir: {}
        # -- Enable StatefulSetAutoDeletePVC feature
        enableStatefulSetAutoDeletePVC: true
        # -- Size of persistent disk
        size: 50Gi
        # -- Storage class to be used.
        # If defined, storageClassName: <storageClass>.
        # If set to "-", storageClassName: "", which disables dynamic provisioning.
        # If empty or set to null, no storageClassName spec is
        # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
        # 存储类名，默认为 "nfs-client",前面部署过
        storageClass: "nfs-client"
# Enable minio for storage
minio:
  # minio 访问密码，后面部署tempo需要用到，可以修改
  rootUser: root-user
  rootPassword: supersecretpassword 
  enabled: true
  buckets:
  - name: loki-chunks
    policy: none
    purge: false
  - name: loki-ruler
    policy: none
    purge: false
  - name: loki-admin
    policy: none
    purge: false
  persistence:
    size: 10Ti
    accessMode: ReadWriteOnce
    storageClass: "nfs-client"
gateway:
  service:
    type: ClusterIP

promtail-values.yaml

daemonset:
  # -- Deploys Promtail as a DaemonSet
  enabled: true
  autoscaling:
    # -- Creates a VerticalPodAutoscaler for the daemonset
    enabled: false

    # Recommender responsible for generating recommendation for the object.
    # List should be empty (then the default recommender will generate the recommendation)
    # or contain exactly one recommender.
    # recommenders:
    # - name: custom-recommender-performance

    # -- List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory
    controlledResources: []

    # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits.
    # controlledValues: RequestsAndLimits

    # -- Defines the max allowed resources for the pod
    maxAllowed: {}
    # cpu: 200m
    # memory: 100Mi
    # -- Defines the min allowed resources for the pod
    minAllowed: {}
    # cpu: 200m
    # memory: 100Mi

    # updatePolicy:
      # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction
      # minReplicas: 1
      # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates
      # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto".
      # updateMode: Auto

configmap:
  # -- If enabled, promtail config will be created as a ConfigMap instead of a secret
  enabled: true

image:
  # -- The Docker registry
  registry: docker.io
  # -- Docker image repository
  repository: grafana/promtail
  # -- Overrides the image tag whose default is the chart's appVersion
  tag: ""
  # -- Docker image pull policy
  pullPolicy: IfNotPresent

# -- Image pull secrets for Docker images
imagePullSecrets: []

# -- Pod labels
podLabels: {}

# -- Pod annotations
podAnnotations: {}
#  prometheus.io/scrape: "true"
#  prometheus.io/port: "http-metrics"

# -- The name of the Namespace to deploy
# If not set, `.Release.Namespace` is used
namespace: "logging"

# -- Default volumes that are mounted into pods. In most cases, these should not be changed.
# Use `extraVolumes`/`extraVolumeMounts` for additional custom volumes.
# @default -- See `values.yaml`
defaultVolumes:
  - name: run
    hostPath:
      path: /run/promtail
  - name: containers
    hostPath:
      path: /data/docker/containers
  - name: pods
    hostPath:
      path: /var/log/pods

# -- Default volume mounts. Corresponds to `volumes`.
# @default -- See `values.yaml`
defaultVolumeMounts:
  - name: run
    mountPath: /run/promtail
  - name: containers
    mountPath: /data/docker/containers
    readOnly: true
  - name: pods
    mountPath: /var/log/pods
    readOnly: true

# Extra volumes to be added in addition to those specified under `defaultVolumes`.
extraVolumes: []

# Extra volume mounts together. Corresponds to `extraVolumes`.
extraVolumeMounts: []

# Extra args for the Promtail container.
extraArgs: []
# -- Example:
# -- extraArgs:
# --   - -client.external-labels=hostname=$(HOSTNAME)

# -- Extra environment variables. Set up tracing enviroment variables here if .Values.config.enableTracing is true.
# Tracing currently only support configure via environment variables. See:
# https://grafana.com/docs/loki/latest/clients/promtail/configuration/#tracing_config
# https://www.jaegertracing.io/docs/1.16/client-features/
extraEnv: []

# -- Extra environment variables from secrets or configmaps
extraEnvFrom: []

# -- Configure enableServiceLinks in pod
enableServiceLinks: true

# -- Section for crafting Promtails config file. The only directly relevant value is `config.file`
# which is a templated string that references the other values and snippets below this key.
# @default -- See `values.yaml`
config:
  # -- Enable Promtail config from Helm chart
  # Set `configmap.enabled: true` and this to `false` to manage your own Promtail config
  # See default config in `values.yaml`
  enabled: true
  # -- The log level of the Promtail server
  # Must be reference in `config.file` to configure `server.log_level`
  # See default config in `values.yaml`
  logLevel: info
  # -- The log format of the Promtail server
  # Must be reference in `config.file` to configure `server.log_format`
  # Valid formats: `logfmt, json`
  # See default config in `values.yaml`
  logFormat: logfmt
  # -- The port of the Promtail server
  # Must be reference in `config.file` to configure `server.http_listen_port`
  # See default config in `values.yaml`
  serverPort: 3101
  # -- The config of clients of the Promtail server
  # Must be reference in `config.file` to configure `clients`
  # @default -- See `values.yaml`
  clients:
    - url: http://loki-gateway.logging.svc.cluster.local/loki/api/v1/push
      headers:
       # 指定租户ID
       X-Scope-OrgID: "fsst"  
  # -- Configures where Promtail will save it's positions file, to resume reading after restarts.
  # Must be referenced in `config.file` to configure `positions`
  positions:
    filename: /run/promtail/positions.yaml
  # -- The config to enable tracing
  enableTracing: false
  # -- A section of reusable snippets that can be reference in `config.file`.
  # Custom snippets may be added in order to reduce redundancy.
  # This is especially helpful when multiple `kubernetes_sd_configs` are use which usually have large parts in common.
  # @default -- See `values.yaml`
  snippets:
    pipelineStages:
      - cri: {}
    common:
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
          - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
          - namespace
          - app
        target_label: job
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        regex: true/(.*)
        separator: /
        source_labels:
          - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
          - __meta_kubernetes_pod_container_name
        target_label: __path__

      # 删除所有 __meta_kubernetes 开头的标签（关键步骤）
      - action: labeldrop
        regex: "__meta_kubernetes.*"

    # If set to true, adds an additional label for the scrape job.
    # This helps debug the Promtail config.
    addScrapeJobLabel: false

    # -- You can put here any keys that will be directly added to the config file's 'limits_config' block.
    # @default -- empty
    extraLimitsConfig: ""

    # -- You can put here any keys that will be directly added to the config file's 'server' block.
    # @default -- empty
    extraServerConfigs: ""

    # -- You can put here any additional scrape configs you want to add to the config file.
    # @default -- empty
    extraScrapeConfigs: ""

    # -- You can put here any additional relabel_configs to "kubernetes-pods" job
    extraRelabelConfigs: []

    # 日志收集配置,注意只收集有logi_logs_enable=true的pod
    scrapeConfigs: |
      # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
      - job_name: kubernetes-pods
        pipeline_stages:
          {{- toYaml .Values.config.snippets.pipelineStages | nindent 4 }}
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: 
              - __meta_kubernetes_pod_annotation_loki_logs_enable
            action: keep
            regex: "true"

          - source_labels:
              - __meta_kubernetes_pod_label_k8s_cmstop_com_name
            action: replace
            target_label: app

          {{- if .Values.config.snippets.addScrapeJobLabel }}
          - replacement: kubernetes-pods
            target_label: scrape_job
          {{- end }}
          {{- toYaml .Values.config.snippets.common | nindent 4 }}
          {{- with .Values.config.snippets.extraRelabelConfigs }}
          {{- toYaml . | nindent 4 }}
          {{- end }}

  # -- Config file contents for Promtail.
  # Must be configured as string.
  # It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
  # @default -- See `values.yaml`
  file: |
    server:
      log_level: {{ .Values.config.logLevel }}
      log_format: {{ .Values.config.logFormat }}
      http_listen_port: {{ .Values.config.serverPort }}
      {{- with .Values.httpPathPrefix }}
      http_path_prefix: {{ . }}
      {{- end }}
      {{- tpl .Values.config.snippets.extraServerConfigs . | nindent 2 }}

    clients:
      {{- tpl (toYaml .Values.config.clients) . | nindent 2 }}

    positions:
      {{- tpl (toYaml .Values.config.positions) . | nindent 2 }}

    scrape_configs:
      {{- tpl .Values.config.snippets.scrapeConfigs . | nindent 2 }}
      {{- tpl .Values.config.snippets.extraScrapeConfigs . | nindent 2 }}

    limits_config:
      {{- tpl .Values.config.snippets.extraLimitsConfig . | nindent 2 }}

    tracing:
      enabled: {{ .Values.config.enableTracing }}

sidecar:
  configReloader:
    enabled: true
    image:
      # -- The Docker registry for sidecar config-reloader
      registry: ghcr.io
      # -- Docker image repository for sidecar config-reloader
      repository: jimmidyson/configmap-reload
      # -- Docker image tag for sidecar config-reloader
      tag: v0.12.0
      # -- Docker image pull policy for sidecar config-reloader
      pullPolicy: IfNotPresent
    # Extra args for the config-reloader container.
    extraArgs: []
    # -- Extra environment variables for sidecar config-reloader
    extraEnv: []
    # -- Extra environment variables from secrets or configmaps for sidecar config-reloader
    extraEnvFrom: []
    # -- The security context for containers for sidecar config-reloader
    containerSecurityContext:
      readOnlyRootFilesystem: true
      capabilities:
        drop:
          - ALL
      allowPrivilegeEscalation: false
    # -- Readiness probe for sidecar config-reloader
    readinessProbe: {}
    # -- Liveness probe for sidecar config-reloader
    livenessProbe: {}
    # -- Resource requests and limits for sidecar config-reloader
    resources: {}
    #  limits:
    #    cpu: 200m
    #    memory: 128Mi
    #  requests:
    #    cpu: 100m
    #    memory: 128Mi
    config:
      # -- The port of the config-reloader server
      serverPort: 9533
    serviceMonitor:
      enabled: true

heml 命令的

# 添加grafana仓库
helm repo add grafana https://grafana.github.io/helm-charts

# 更新仓库
helm repo update

# 部署 loki
helm install --values loki-value.yaml --namespace logging   loki    grafana/loki

# 升级
helm upgrade loki grafana/loki -n logging -f loki-value.yaml

#grafana
#http://loki-gateway.logging.svc.cluster.local/

# 部署
helm install --values promtail-values.yaml --namespace logging  promtail  grafana/promtail