使用Kindling 观测 Kubernetes 应用网络连接状态

kindling介绍

Kindling 解决的是在不入侵应用的前提下如何观测网络的问题其功能主要是通过暴露内核事件来实现观测。如果主机内核版本高于 4.14可以使用 eBPF 模块如果主机内核是低版本采用的是 Sysdig 实现相关观测。

[root@master yaml]# uname -a
Linux master 3.10.0-1160.el7.x86_64 #1 SMP Mon Oct 19 16:18:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux

我的内核小于4.14

1. yaml安装kindlking

git clone https://github.com/shaowenchen/demo
cd kindling/yaml
kubectl apply -f ./

2. 修改kindling-daemonset的资源限制

apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    k8s-app: kindling-agent
  name: kindling-agent
  namespace: kindling
spec:
  selector:
    matchLabels:
      k8s-app: kindling-agent
  template:
    metadata:
      labels:
        k8s-app: kindling-agent
    spec:
      serviceAccount: kindling-agent
      containers:
      - name: kindling-agent
        image: shaowenchen/kindling-agent:ubuntu-20.04
        imagePullPolicy: IfNotPresent
        securityContext:
          privileged: true
        resources:
          limits:
            memory: 1Gi
            cpu: 500m
          requests:
            cpu: 10m
            memory: 300Mi
        env:
        - name: HOST_PROC
          value: /host/proc
        - name: PL_HOST_PATH
          value: /host
        - name: SYSDIG_HOST_ROOT
          value: /host
        - name: GOGC
          value: "400"
        - name: MY_NODE_IP
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: MY_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        readinessProbe:
          exec:
            command:
            - cat
            - /opt/kernel-support
          initialDelaySeconds: 5
          periodSeconds: 5
        volumeMounts:
        - mountPath: /etc/kindling/config
          name: kindlingcfg
        - mountPath: /sys
          name: sys-vol
        - mountPath: /etc/modprobe.d
          name: modprobe-d
          readOnly: true
        - mountPath: /host/dev
          name: dev-vol
        - mountPath: /host/proc
          name: proc-vol
          readOnly: true
        - mountPath: /host/etc
          name: etc-vol
          readOnly: true
        - mountPath: /host/boot
          name: boot-vol
          readOnly: true
        - mountPath: /host/lib/modules
          name: modules-vol
          readOnly: true
        - mountPath: /host/usr
          name: usr-vol
          readOnly: true
        - mountPath: /host/run
          name: run-vol
        - mountPath: /host/var/run
          name: varrun-vol
        - mountPath: /dev/shm
          name: dshm
      dnsPolicy: ClusterFirstWithHostNet
      hostNetwork: true
      hostPID: true
      restartPolicy: Always
      schedulerName: default-scheduler
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: kindlingcfg
        name: kindlingcfg
      - hostPath:
          path: /etc/modprobe.d
        name: modprobe-d
      - emptyDir:
          medium: Memory
        name: dshm
      - hostPath:
          path: /dev
        name: dev-vol
      - hostPath:
          path: /proc
        name: proc-vol
      - hostPath:
          path: /etc
        name: etc-vol
      - hostPath:
          path: /boot
        name: boot-vol
      - hostPath:
          path: /lib/modules
        name: modules-vol
      - hostPath:
          path: /usr
        name: usr-vol
      - hostPath:
          path: /run
        name: run-vol
      - hostPath:
          path: /var/run
        name: varrun-vol
      - hostPath:
          path: /sys
        name: sys-vol

4. 手动修改svc当然也可以使用svc的地址

[root@master yaml]# cat kindling-service.yml
apiVersion: v1
kind: Service
metadata:
  namespace: kindling
  name: kindling-agent
  labels:
    k8s-app: kindling-agent
  annotations:
    prometheus.io/path: /metrics
    prometheus.io/port: "9500"
    prometheus.io/scrape: "true"
spec:
  selector:
    k8s-app: kindling-agent
  type: NodePort
  ports:
  - name: http
    port: 9500
    nodePort: 31792
    targetPort: 9500
    protocol: TCP

3. 访问是否有数据返回http://master_ip:31792/metrics
   
4. prometheus安装不说了配置如下
   k8s prometheus一键安装包https://download.csdn.net/download/weixin_43606975/85824711?spm=1001.2014.3001.5503
   
5. Grafana 安装忽略granfan为8.3.1安装插件如下
   下载插件

git clone https://github.com/shaowenchen/demo
cd kindling/dashboard
kubectl cp topo-plugin.tar.gz grafana-7559f9456f-l5dqr:/var/lib/grafana/plugins/
kubectl exec -it grafana-7559f9456f-l5dqr -n prometheus bash
#cd var/lib/grafana/plugins/
#mkdir kindlingproject-topology-panel
#mv topo-plugin.tar.gz kindlingproject-topology-panel/
#cd kindlingproject-topology-panel/
#tar xvf topo-plugin.tar.gz

7. granfan配置文件修改

[root@master yaml]# cat /home/sd-prometheus/grafana/grafana-conf.yaml
apiVersion: v1
data:
  grafana.ini: |
        [paths]
        [server]
        [database]
        [remote_cache]
        [dataproxy]
        [analytics]
        [security]
        [snapshots]
        [dashboards]
        [users]
        [auth]
        [auth.anonymous]
        [auth.github]
        [auth.gitlab]
        [auth.google]
        [auth.grafana_com]
        [auth.azuread]
        [auth.okta]
        [auth.generic_oauth]
        [auth.basic]
        [auth.proxy]
        [auth.ldap]
        enabled = true
        config_file = /etc/grafana/ldap.toml
        allow_sign_up = true
        [smtp]
        enabled = true
        host = smtp.exmail.qq.com:465
        user = gitlab@movie.cn
        password = password
        from_address = gitlab@movie.cn
        from_name = Grafana
        [emails]
        [log]
        [log.console]
        [log.file]
        [log.syslog]
        [quota]
        [alerting]
        [annotations.dashboard]
        [annotations.api]
        [explore]
        [metrics]
        [metrics.environment_info]
        [metrics.graphite]
        [grafana_com]
        [tracing.jaeger]
        [external_image_storage]
        [external_image_storage.s3]
        [external_image_storage.webdav]
        [external_image_storage.gcs]
        [external_image_storage.azure_blob]
        [external_image_storage.local]
        [rendering]
        [panels]
        [plugins]
        enable_alpha = true
        plugin_admin_enabled = true
        plugin_admin_external_mange_enabled = true
        allow_loading_unsigned_plugins = kindlingproject-topology-panel
        [plugin.grafana-image-renderer]
        [enterprise]
        [feature_toggles]
        [date_formats]

7. 重启granfan
8. 访问granfan并导入json

https://github.com/shaowenchen/demo/tree/master/kindling/dashboard

9. 查看