Fluentd 설정 방법

fluentd.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: oscmp-dev
imagePullSecrets: 
- name: redii

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: oscmp-dev
---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: fluentd
  namespace: oscmp-dev
  labels:
    k8s-app: fluentd-logging
    version: v1
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: fluentd-logging
      version: v1
  template:
    metadata:
      labels:
        k8s-app: fluentd-logging
        version: v1
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      containers:
      - name: fluentd
        image: sds.redii.net/oscmp-devops/fluentd:v1.14.6-debian-opensearch-1.1-cmp.2
        env:
          - name:  FLUENT_OPENSEARCH_HOST
            value: opensearch-cluster-master-headless
          - name:  FLUENT_OPENSEARCH_PORT
            value: "9200"
          - name: FLUENT_OPENSEARCH_SCHEME
            value: "https"
          - name: FLUENT_OPENSEARCH_SSL_VERIFY
            value: "true"
          - name: FLUENT_OPENSEARCH_SSL_VERSION
            value: "TLSv1_2"
          - name: FLUENTD_SYSTEMD_CONF
            value: disable
          - name: FLUENT_OPENSEARCH_USER
            value: admin
          - name: FLUENT_OPENSEARCH_PASS
            value: admin
          - name: RABBITMQ_HOST
            value: oscmp-rabbitmq-headless
          - name: RABBITMQ_PASS
            value: guest
          - name: RABBITMQ_USER
            value: guest
          - name: RABBITMQ_VHOST
            value: /
        resources:
          limits:
            memory: 1000Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: config
          mountPath: /fluentd/etc/fluent.conf
          subPath: fluent.conf
      terminationGracePeriodSeconds: 30
      volumes:
      - name: config
        configMap:
          name: fluentd-rmq-config
      nodeSelector:
        oscmp-ctl-node: "true"

fluentd_configmap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-rmq-config
  namespace: oscmp-dev
data:
  tail_container_parse.conf: |-
    <parse>
      @type cri
    </parse>
  fluent.conf: |-
    <source>
      @type rabbitmq
      @label @rabbitmq_noti
      tag "notification"
      host "#{ENV['RABBITMQ_HOST']}"
      user "#{ENV['RABBITMQ_USER']}"
      pass "#{ENV['RABBITMQ_PASS']}"
      vhost "#{ENV['RABBITMQ_VHOST']}"
      exchange "openstack"
      queue "notifications.info"
      routing_key "notifications.info"
      heartbeat 10
      <parse>
        @type json
      </parse>
    </source>

    <source>
      @type rabbitmq
      @label @rabbitmq_noti
      tag "notification"
      host "#{ENV['RABBITMQ_HOST']}"
      user "#{ENV['RABBITMQ_USER']}"
      pass "#{ENV['RABBITMQ_PASS']}"
      vhost "#{ENV['RABBITMQ_VHOST']}"
      exchange "openstack"
      queue "notifications.warn"
      routing_key "notifications.warn"
      heartbeat 10
      <parse>
        @type json
      </parse>
    </source>

    <source>
      @type rabbitmq
      @label @rabbitmq_noti
      tag "notification"
      host "#{ENV['RABBITMQ_HOST']}"
      user "#{ENV['RABBITMQ_USER']}"
      pass "#{ENV['RABBITMQ_PASS']}"
      vhost "#{ENV['RABBITMQ_VHOST']}"
      exchange "openstack"
      queue "notifications.debug"
      routing_key "notifications.debug"
      heartbeat 10
      <parse>
        @type json
      </parse>
    </source>

    <source>
      @type rabbitmq
      @label @rabbitmq_noti
      tag "notification"
      host "#{ENV['RABBITMQ_HOST']}"
      user "#{ENV['RABBITMQ_USER']}"
      pass "#{ENV['RABBITMQ_PASS']}"
      vhost "#{ENV['RABBITMQ_VHOST']}"
      exchange "openstack"
      queue "notifications.error"
      routing_key "notifications.error"
      heartbeat 10
      <parse>
        @type json
      </parse>
    </source>

    <label @rabbitmq_noti>
      <filter **>
        @type parser
        key_name oslo.message
        reserve_data true
        <parse>
          @type json
        </parse>
      </filter>

      <filter **>
        @type record_transformer
        enable_ruby
        <record>
          Payload ${record["payload"]}
          product_type ${record.dig("payload", "product_type")}
        </record>
        remove_keys ["oslo.message", "payload"]
      </filter>

      <match **>
        @type opensearch
        include_tag_key true
        host "#{ENV['FLUENT_OPENSEARCH_HOST']}"
        port "#{ENV['FLUENT_OPENSEARCH_PORT']}"
        path "#{ENV['FLUENT_OPENSEARCH_PATH']}"
        scheme "#{ENV['FLUENT_OPENSEARCH_SCHEME'] || 'https'}"
        ssl_verify "#{ENV['FLUENT_OPENSEARCH_SSL_VERIFY'] || 'false'}"
        ssl_version "#{ENV['FLUENT_OPENSEARCH_SSL_VERSION'] || 'TLSv1_2'}"
        user "#{ENV['FLUENT_OPENSEARCH_USER']}"
        password "#{ENV['FLUENT_OPENSEARCH_PASS']}"
        logstash_format true
        logstash_prefix notification
        logstash_dateformat %Y.%m.%d
      </match>
    </label>

 

oslo.message라는 필드 안에 있는 문자열 json을 꺼낼 때 reserve_data를 true로 두면 해제 가능

 

“oslo.message” : “{ “a” : “a” }”

→

“oslo.message” : “{ “a” : “a” }”

“a” : “a”

 

특정 필드를 지우거나 생성할 때

      <filter **>
        @type record_transformer
        enable_ruby
        <record>
          Payload ${record["payload"]}
          product_type ${record.dig("payload", "product_type")}
        </record>
        remove_keys ["oslo.message", "payload"]
      </filter>

<record> 안에 있는 데이터는 필드 생성

remove_keys는 필드 삭제

 

Payload ${record["payload"]}는 현재 있는 필드 중 payload라는 필드의 데이터를 받는 다는 의미

“payload” : { “a” : “a” }

→

“payload” : { “a” : “a” }

“Payload” : { “a” : “a” }

 

product_type ${record.dig("payload", "product_type")}는 더 안에 있는 데이터를 수집할 때 사용, [][]의 형태도 가능하지만 없을 경우 오류가 나고 dig는 nil 반환

“payload” : { “product_type” : { “a” : “a” } }

→

“payload” : { “product_type” : { “a” : “a” } }

“product_type” : { “a” : “a” }

 

remove_keys ["oslo.message", "payload"]는 삭제

“payload” : { “a” : “a” }

“Payload” : { “a” : “a” }

→

“Payload” : { “a” : “a” }