使用 fluentd 进行 kubernetes 审计日志过滤并转发到 Splunk

Question

经过一番努力，我熟练地将 Openshift 审计日志文件转发到 Splunk。 然而，这导致了大量事件，所以我应用了一个过滤器来排除“get”和“watch”。 我想包括获取机密。

我的问题是，如何更改过滤器以排除“get”但包含“get secret”？

apiVersion: v1
kind: ConfigMap
metadata:
  name: splunk-kubernetes-audit
  namespace: splunk-logging
  labels:
    app: splunk-kubernetes-audit
data:
  fluent.conf: |-
    <system>
      log_level info
    </system>
    @include source.audit.conf
    @include output.conf
  output.conf: |-
    <label @SPLUNK>
      # = filters for non-container log files =
      # extract sourcetype
       <filter tail.file.**>
          @type grep
          <exclude>
             key verb
             pattern /watch/
          </exclude>
          <and>
            <exclude>
               key verb
               pattern /get/
            </exclude>
          </and>
  </filter>
      <filter tail.file.**>
        @type jq_transformer
        jq '.record.sourcetype = (.tag | ltrimstr("tail.file.")) | .record.cluster_name = "opcdev" | .record.splunk_index = "openshift_audit_n" | .record'
      </filter>
      # = custom filters specified by users =

      # = output =
      <match **>
        @type splunk_hec
        protocol https
        hec_host "splunk-heavyforwarder.linux.rabobank.nl"
        hec_port 8088
        hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
        index_key splunk_index
        insecure_ssl false
        ca_file /fluentd/etc/splunk/hec_ca_file
        host "#{ENV['K8S_NODE_NAME']}"
        source_key source
        sourcetype_key sourcetype
        <fields>
          # currently CRI does not produce log paths with all the necessary
          # metadata to parse out pod, namespace, container_name, container_id.
          # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
          pod
          namespace
          container_name
          cluster_name
          container_id
        </fields>
        app_name splunk-kubernetes-audit
        app_version 1.4.7
        <buffer>
          @type memory
          chunk_limit_records 100000
          chunk_limit_size 10m
          flush_interval 10s
          flush_thread_count 1
          overflow_action block
          retry_max_times 5
          retry_type exponential_backoff
          retry_wait 2
          retry_max_interval 300
          total_limit_size 600m
        </buffer>
        <format>
          @type "json"
        </format>
      </match>
    </label>
  source.audit.conf: |-
    # This fluentd conf file contains sources for log files other than container logs.
    <source>
      @id tail.file.kube-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.kube-api-audit
      path /var/log/kube-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-kube-api-audit.pos
      read_from_head true
      path_key source
      <parse>
        @type json
      </parse>
    </source>
    <source>
      @id tail.file.oauth-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.oauth-api-audit
      path /var/log/oauth-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-oauth-api-audit.pos
      read_from_head true
      path_key source
      <parse>
        @type json
      </parse>
    </source>
    <source>
      @id tail.file.openshift-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.openshift-api-audit
      path /var/log/openshift-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-openshift-api-audit.pos
      read_from_head true
      path_key source
      <parse>
        @type json
      </parse>
    </source>

秘密

apiVersion: v1
kind: Secret
metadata:
  labels:
    app: splunk-kubernetes-audit
  name: splunk-kubernetes-audit
  namespace: splunk-logging
type: Opaque
data:
  hec_ca_file: {{ base64 encoded CA certificate }}
  splunk_hec_token: {{ base64 encoded Get_token_for_index }}

和守护进程

apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    configmap.update: "1"
    deprecated.daemonset.template.generation: "34"
  generation: 34
  labels:
    app: splunk-kubernetes-audit
    engine: fluentd
  name: splunk-kubernetes-audit
  namespace: splunk-logging
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: splunk-kubernetes-audit
      release: rabo-splunk
  template:
    metadata:
      annotations:
        checksum/config: 0574cfe32baa34dcb02d7e3293f7c5ac0379ffb45cf4b7e455eb6975e6102320
        configmap.update.trigger: "1"
        prometheus.io/port: "24231"
        prometheus.io/scrape: "true"
      creationTimestamp: null
      labels:
        app: splunk-kubernetes-audit
        release: rabo-splunk
    spec:
      containers:
      - env:
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: MY_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: SPLUNK_HEC_TOKEN
          valueFrom:
            secretKeyRef:
              key: splunk_hec_token
              name: splunk-kubernetes-audit
        - name: SSL_CERT_FILE
          value: /fluentd/etc/splunk/hec_ca_file
        image: docker.io/splunk/fluentd-hec:1.2.6
        imagePullPolicy: Always
        name: splunk-fluentd-k8s-audit
        ports:
        - containerPort: 24231
          name: metrics
          protocol: TCP
        resources:
          requests:
            cpu: 500m
            memory: 600Mi
        securityContext:
          privileged: true
          runAsUser: 0
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/log
          name: varlog
        - mountPath: /var/log/kube-apiserver
          name: varlogkube
          readOnly: true
        - mountPath: /var/log/oauth-apiserver
          name: varlogoauth
          readOnly: true
        - mountPath: /var/log/openshift-apiserver
          name: varlogopenshift
          readOnly: true
        - mountPath: /fluentd/etc
          name: conf-configmap
        - mountPath: /fluentd/etc/splunk
          name: secrets
          readOnly: true
      dnsPolicy: ClusterFirst
      imagePullSecrets:
      - name: acr-secret
      nodeSelector:
        node-role.kubernetes.io/master: ''
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: splunk-logging
      serviceAccountName: splunk-logging
      terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
      - hostPath:
          path: /var/log
          type: ""
        name: varlog
      - hostPath:
          path: /var/log/kube-apiserver
          type: ""
        name: varlogkube
      - hostPath:
          path: /var/log/oauth-apiserver
          type: ""
        name: varlogoauth
      - hostPath:
          path: /var/log/openshift-apiserver
          type: ""
        name: varlogopenshift
      - configMap:
          defaultMode: 420
          name: splunk-kubernetes-audit
        name: conf-configmap
      - name: secrets
        secret:
          defaultMode: 420
          secretName: splunk-kubernetes-audit
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate

Answer 1

我没有排除除秘密之外的所有 get、list 和 watch 操作，而是选择排除导致最多事件的对象，例如命名空间、pod 和 configmap。 这导致了下面的额外过滤器。 这使 Splunk 事件减少了约 65%。 休息中的 Openshift 集群每天生成约 12 GB 的审计日志，无需过滤。

      # reduce the number of events by removing get, watch and list api calls
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /list/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /namespaces/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /list/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /pods/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /watch/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /namespaces/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /watch/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /pods/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /watch/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /configmaps/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /get/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /configmaps/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /get/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /namespaces/
          </exclude>
        </and>
      </filter>
      <filter tail.file.**>
        @type grep
        <and>
          <exclude>
             key verb
             pattern /get/
          </exclude>

          <exclude>
             key $.objectRef.resource
             pattern /clusterrolebindings/
          </exclude>
         </and>
      </filter>