kubernetesに展開するfluentd-daemonset-elasticsearchのconfigMapを設定する

はじめに

AKSなどkubernetesで、fluentdを利用してElastic Searchに転送する場合、公式をデプロイすると様々ログを取得します。
しかし、Elastic Search側のディスクの制限などにより必要となるログのみ取得したい事があったので作成しました。
https://github.com/fluent/fluentd-kubernetes-daemonset

そこでconfigMapで必要なLogのみ取得する方法を解説します。
https://github.com/jkudo/fluentd-kubernetes-daemonset

configMap

fluentd-daemonset-elasticsearch-rbac-configmap.yaml

deployment.apps/nginx-deploy の場合にnginxのログを取得したい場合。

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-config
  namespace: kube-system
  labels:
    k8s-app: fluentd-logging
    version: v1
    kubernetes.io/cluster-service: "true"
data:
  fluent.conf: |
    #@include systemd.conf
    @include kubernetes.conf
    <match **>
       @type elasticsearch
       @id out_es
       @log_level info
       include_tag_key true
       host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
       port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
       scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
       ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
       user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
       password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
       reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
       logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
       logstash_format true
       type_name fluentd
       <buffer>
         flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
         flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
         chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
         queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
         retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
         retry_forever true
       </buffer>
    </match>
  kubernetes.conf: |
    <match fluent.**>
      @type null
    </match>
    #Target Logs (ex:nginx)
    <source>
      @type tail
      @id in_tail_container_logs
      path "/var/log/containers/nginx*.log"
      pos_file "/var/log/fluentd-nginx.log.pos"
      tag "kubernetes.*"
      read_from_head true
      <parse>
        @type "json"
        time_format "%Y-%m-%dT%H:%M:%S.%NZ"
        time_type string
      </parse>
    </source>
    #<source>
    #  @type tail
    #  @id in_tail_container_logs
    #  path "/var/log/containers/*.log"
    #  pos_file "/var/log/fluentd-{log name}.log.pos"
    #  tag "kubernetes.*"
    #  read_from_head true
    #  <parse>
    #    @type "json"
    #    time_format "%Y-%m-%dT%H:%M:%S.%NZ"
    #    time_type string
    #  </parse>
    #</source>
  #systemd.conf: |
  #  <source>
  #    @type systemd
  #    @id in_systemd_kubelet
  #    matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
  #    <storage>
  #      @type local
  #      persistent true
  #      path /var/log/fluentd-journald-kubelet-cursor.json
  #    </storage>
  #    <entry>
  #      fields_strip_underscores true
  #    </entry>
  #    read_from_head true
  #    tag kubelet
  #  </source>
  #  <source>
  #    @type systemd
  #    @id in_systemd_docker
  #    matches [{ "_SYSTEMD_UNIT": "docker.service" }]
  #    <storage>
  #      @type local
  #      persistent true
  #      path /var/log/fluentd-journald-docker-cursor.json
  #    </storage>
  #    <entry>
  #      fields_strip_underscores true
  #    </entry>
  #    read_from_head true
  #    tag docker.systemd
  #  </source>
  #  <source>
  #    @type systemd
  #    @id in_systemd_bootkube
  #    matches [{ "_SYSTEMD_UNIT": "bootkube.service" }]
  #    <storage>
  #      @type local
  #      persistent true
  #      path /var/log/fluentd-journald-bootkube-cursor.json
  #    </storage>
  #    <entry>
  #      fields_strip_underscores true
  #    </entry>
  #    read_from_head true
  #    tag bootkube
  #  </source>

システム系のLogを取得したい場合はsystemd.confを有効にします。

Pod

fluentd-daemonset-elasticsearch-rbac.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: fluentd
  namespace: kube-system
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: kube-system
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-system
  labels:
    k8s-app: fluentd-logging
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  template:
    metadata:
      labels:
        k8s-app: fluentd-logging
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      initContainers:
      - name: config-fluentd
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh","-c"]
        args:
        - cp /fluentd/etc2/fluent.conf /fluentd/etc/fluent.conf;
          cp /fluentd/etc2/kubernetes.conf /fluentd/etc/kubernetes.conf;
          #cp /fluentd/etc2/systemd.conf /fluentd/etc/systemd.conf;
        volumeMounts:
        - name: config-path
          mountPath: /fluentd/etc
        - name: config-source
          mountPath: /fluentd/etc2
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1.3.0-debian-elasticsearch
        env:
          - name:  FLUENT_ELASTICSEARCH_HOST
            value: "10.240.0.4"
          - name:  FLUENT_ELASTICSEARCH_PORT
            value: "9200"
          - name: FLUENT_ELASTICSEARCH_SCHEME
            value: "http"
          # X-Pack Authentication
          # =====================
          - name: FLUENT_ELASTICSEARCH_USER
            value: "elastic"
          - name: FLUENT_ELASTICSEARCH_PASSWORD
            value: "changeme"
          - name: FLUENT_UID
            value: "0"
        resources:
          limits:
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: config-path
          mountPath: /fluentd/etc
      terminationGracePeriodSeconds: 30
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: config-source
        configMap:
          name: fluentd-config
      - name: config-path
        emptyDir: {}

システム系のLogを取得したい場合はsystemd.confを有効にします。

log

確認してみると、対象のlogがtailされていることがわかります。

2019-01-02 03:38:22 +0000 [info]: parsing config file is succeeded path="/fluentd/etc/fluent.conf"
2019-01-02 03:38:22 +0000 [info]: [out_es] Connection opened to Elasticsearch cluster => {:host=>"10.240.0.4", :port=>9200, :scheme=>"http", :user=>"elastic", :password=>"obfuscated"}
2019-01-02 03:38:22 +0000 [info]: [out_es] Detected ES 6.x: ES 7.x will only accept `_doc` in type_name.
2019-01-02 03:38:22 +0000 [info]: using configuration file: <ROOT>
  <match fluent.**>
    @type null
  </match>
  <source>
    @type tail
    @id in_tail_container_logs
    path "/var/log/containers/nginx*.log"
    pos_file "/var/log/fluentd-nginx.log.pos"
    tag "kubernetes.*"
    read_from_head true
    <parse>
      @type "json"
      time_format "%Y-%m-%dT%H:%M:%S.%NZ"
      time_type string
    </parse>
  </source>
  <match **>
    @type elasticsearch
    @id out_es
    @log_level "info"
    include_tag_key true
    host "10.240.0.4"
    port 9200
    scheme http
    ssl_verify true
    user "elastic"
    password xxxxxx
    reload_connections true
    logstash_prefix "logstash"
    logstash_format true
    type_name "fluentd"
    <buffer>
      flush_thread_count 8
      flush_interval 5s
      chunk_limit_size 2M
      queue_limit_length 32
      retry_max_interval 30
      retry_forever true
    </buffer>
  </match>
</ROOT>
2019-01-02 03:38:22 +0000 [info]: starting fluentd-1.3.0 pid=5 ruby="2.3.3"
2019-01-02 03:38:22 +0000 [info]: spawn command to main:  cmdline=["/usr/bin/ruby2.3", "-Eascii-8bit:ascii-8bit", "/fluentd/vendor/bundle/ruby/2.3.0/bin/fluentd", "-c", "/fluentd/etc/fluent.conf", "-p", "/fluentd/plugins", "--gemfile", "/fluentd/Gemfile", "--under-supervisor"]
2019-01-02 03:38:23 +0000 [info]: gem 'fluent-plugin-elasticsearch' version '2.11.11'
2019-01-02 03:38:23 +0000 [info]: gem 'fluent-plugin-kubernetes_metadata_filter' version '2.1.4'
2019-01-02 03:38:23 +0000 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '2.1.0'
2019-01-02 03:38:23 +0000 [info]: gem 'fluent-plugin-systemd' version '1.0.1'
2019-01-02 03:38:23 +0000 [info]: gem 'fluentd' version '1.3.0'
2019-01-02 03:38:23 +0000 [info]: adding match pattern="fluent.**" type="null"
2019-01-02 03:38:23 +0000 [info]: adding match pattern="**" type="elasticsearch"
2019-01-02 03:38:23 +0000 [info]: #0 [out_es] Connection opened to Elasticsearch cluster => {:host=>"10.240.0.4", :port=>9200, :scheme=>"http", :user=>"elastic", :password=>"obfuscated"}
2019-01-02 03:38:23 +0000 [info]: #0 [out_es] Detected ES 6.x: ES 7.x will only accept `_doc` in type_name.
2019-01-02 03:38:23 +0000 [info]: adding source type="tail"
2019-01-02 03:38:23 +0000 [info]: #0 starting fluentd worker pid=9 ppid=5 worker=0
2019-01-02 03:38:23 +0000 [info]: #0 [in_tail_container_logs] following tail of /var/log/containers/nginx-deploy-54d77946c9-cmjj7_default_nginx-9b6f8b66d16d9967d7e2fc019918ab36300ddb42b7d1b741bec4436003ca3212.log
2019-01-02 03:38:23 +0000 [info]: #0 [in_tail_container_logs] following tail of /var/log/containers/nginx-deploy-54d77946c9-cmjj7_default_nginx-c4bb67ea7576473de695a9cb0add2c63a1d6ffac604ae5556d5a3c360836c710.log
2019-01-02 03:38:23 +0000 [info]: #0 fluentd worker is now running worker=0

これらはIssueなどを参考にyamlを構成しています。

まとめ

configMapを設定することでkubernetes でもfluentdを柔軟に利用することが可能です。
今回はElastic Searchを対象にしましたが、他のDeamonSetでも使えると思います。