diff --git a/res/collector-pipeline-config.json b/res/collector-pipeline-config.json new file mode 100644 index 00000000..4d72990a --- /dev/null +++ b/res/collector-pipeline-config.json @@ -0,0 +1,63 @@ +{ + "log-level": "trace", + "metricsSettings": { + "disableGlobalServer": true + }, + "parameters": [ + { + "name": "enrich", + "transform": { + "type": "network", + "network": { + "rules": [ + { + "type": "add_kubernetes", + "kubernetes": { + "add_zone": true, + "ipField": "SrcAddr", + "output": "SrcK8S" + } + }, + { + "type": "add_kubernetes", + "kubernetes": { + "add_zone": true, + "ipField": "DstAddr", + "output": "DstK8S" + } + }, + { + "type": "reinterpret_direction" + } + ], + "directionInfo": { + "reporterIPField": "AgentIP", + "srcHostField": "SrcK8S_HostIP", + "dstHostField": "DstK8S_HostIP", + "flowDirectionField": "FlowDirection" + } + } + } + }, + { + "name": "send", + "write": { + "type": "grpc", + "grpc": { + "targetHost": "{{TARGET_HOST}}", + "targetPort": 9999 + } + } + } + ], + "pipeline": [ + { + "name": "enrich", + "follows": "preset-ingester" + }, + { + "name": "send", + "follows": "enrich" + } + ] +} \ No newline at end of file diff --git a/res/flow-capture.yml b/res/flow-capture.yml index 0e2564c7..cbf1e9f2 100644 --- a/res/flow-capture.yml +++ b/res/flow-capture.yml @@ -46,81 +46,13 @@ spec: - name: ENABLE_FLOW_FILTER value: "false" - name: FLOW_FILTER_RULES - value: > - [ { "direction": "", "ip_cidr": "0.0.0.0/0", "protocol": "", "source_port": 0, - "destination_port": 0, "port": 0, "source_port_range": "", - "source_ports": "", "destination_port_range": "", - "destination_ports": "", "port_range": "", "ports": "", - "icmp_type": 0, "icmp_code": 0, "peer_ip": "", "action": "Accept", - "tcp_flags": "", "drops": false } - ] + value: >- + [] - name: EXPORT value: "direct-flp" - name: FLP_CONFIG - value: > - { - "log-level": "trace", - "metricsSettings":{ - "disableGlobalServer": true - }, - "parameters":[ - { - "name":"enrich", - "transform":{ - "type":"network", - "network":{ - "rules":[ - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"SrcAddr", - "output":"SrcK8S" - } - }, - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"DstAddr", - "output":"DstK8S" - } - }, - { - "type":"reinterpret_direction" - } - ], - "directionInfo":{ - "reporterIPField":"AgentIP", - "srcHostField":"SrcK8S_HostIP", - "dstHostField":"DstK8S_HostIP", - "flowDirectionField":"FlowDirection" - } - } - } - }, - { - "name":"send", - "write":{ - "type":"grpc", - "grpc":{ - "targetHost":"{{TARGET_HOST}}", - "targetPort":9999 - } - } - } - ], - "pipeline":[ - { - "name":"enrich", - "follows":"preset-ingester" - }, - { - "name":"send", - "follows":"enrich" - } - ] - } + value: | + {} volumeMounts: - name: bpf-kernel-debug mountPath: /sys/kernel/debug diff --git a/res/flow-filter.json b/res/flow-filter.json new file mode 100644 index 00000000..092ab4fc --- /dev/null +++ b/res/flow-filter.json @@ -0,0 +1,20 @@ +{ + "direction": "", + "ip_cidr": "0.0.0.0/0", + "protocol": "", + "source_port": 0, + "destination_port": 0, + "port": 0, + "source_port_range": "", + "source_ports": "", + "destination_port_range": "", + "destination_ports": "", + "port_range": "", + "ports": "", + "icmp_type": 0, + "icmp_code": 0, + "peer_ip": "", + "action": "Accept", + "tcp_flags": "", + "drops": false +} \ No newline at end of file diff --git a/res/metric-capture.yml b/res/metric-capture.yml index 12f61828..485268f7 100644 --- a/res/metric-capture.yml +++ b/res/metric-capture.yml @@ -31,461 +31,14 @@ spec: value: "false" - name: LOG_LEVEL value: info - - name: INTERFACES - value: "" - - name: EXCLUDE_INTERFACES - value: "lo" - - name: SAMPLING - value: "1" - - name: ENABLE_RTT - value: "false" - - name: ENABLE_PKT_DROPS - value: "false" - - name: ENABLE_DNS_TRACKING - value: "false" - - name: ENABLE_NETWORK_EVENTS_MONITORING - value: "false" - - name: ENABLE_FLOW_FILTER - value: "false" - - name: FILTER_DIRECTION - value: "" - - name: FILTER_IP_CIDR - value: "0.0.0.0/0" - - name: FILTER_PROTOCOL - value: "" - - name: FILTER_SOURCE_PORT - value: "" - - name: FILTER_DESTINATION_PORT - value: "" - - name: FILTER_PORT - value: "" - - name: FILTER_SOURCE_PORT_RANGE - value: "" - - name: FILTER_DESTINATION_PORT_RANGE - value: "" - - name: FILTER_PORT_RANGE - value: "" - - name: FILTER_SOURCE_PORTS - value: "" - - name: FILTER_DESTINATION_PORTS - value: "" - - name: FILTER_PORTS - value: "" - - name: FILTER_ICMP_TYPE - value: "" - - name: FILTER_ICMP_CODE - value: "" - - name: FILTER_PEER_IP - value: "" - - name: FILTER_TCP_FLAGS - value: "" - - name: FILTER_DROPS - value: "false" - - name: FILTER_ACTION - value: "Accept" + - name: FLOW_FILTER_RULES + value: >- + [] - name: EXPORT value: "direct-flp" - name: FLP_CONFIG - value: > - { - "log-level": "trace", - "metricsSettings":{ - "port":9401, - "prefix":"on_demand_netobserv_", - "noPanic":true - }, - "parameters":[ - { - "name":"enrich", - "transform":{ - "type":"network", - "network":{ - "rules":[ - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"SrcAddr", - "output":"SrcK8S" - } - }, - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"DstAddr", - "output":"DstK8S" - } - }, - { - "type":"reinterpret_direction" - }, - { - "type":"add_kubernetes_infra", - "kubernetes_infra":{ - "namespaceNameFields":[ - { - "name":"SrcK8S_Name", - "namespace":"SrcK8S_Namespace" - }, - { - "name":"DstK8S_Name", - "namespace":"DstK8S_Namespace" - } - ], - "output":"K8S_FlowLayer", - "infra_prefixes":[ - "netobserv", - "openshift" - ], - "infra_refs":[ - { - "name":"kubernetes", - "namespace":"default" - }, - { - "name":"openshift", - "namespace":"default" - } - ] - } - }, - ], - "directionInfo":{ - "reporterIPField":"AgentIP", - "srcHostField":"SrcK8S_HostIP", - "dstHostField":"DstK8S_HostIP", - "flowDirectionField":"FlowDirection" - } - } - } - }, - { - "name":"prometheus", - "encode":{ - "type":"prom", - "prom":{ - "metrics":[ - { - "name":"node_egress_bytes_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"1|2", - "type":"match_regex" - } - ], - "valueKey":"Bytes", - "labels":[ - "SrcK8S_HostName", - "DstK8S_HostName" - ], - "remap":null, - "buckets":null - }, - { - "name":"node_ingress_bytes_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"0|2", - "type":"match_regex" - } - ], - "valueKey":"Bytes", - "labels":[ - "SrcK8S_HostName", - "DstK8S_HostName" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_egress_bytes_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"1|2", - "type":"match_regex" - } - ], - "valueKey":"Bytes", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_ingress_bytes_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"0|2", - "type":"match_regex" - } - ], - "valueKey":"Bytes", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_egress_packets_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"1|2", - "type":"match_regex" - } - ], - "valueKey":"Packets", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_ingress_packets_total", - "type":"counter", - "filters":[ - { - "key":"FlowDirection", - "value":"0|2", - "type":"match_regex" - } - ], - "valueKey":"Packets", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_flows_total", - "type":"counter", - "filters":[], - "valueKey":"", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_rtt_seconds", - "type":"histogram", - "filters":[ - { - "key":"TimeFlowRttNs", - "value":"", - "type":"presence" - } - ], - "valueKey":"TimeFlowRttNs", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type" - ], - "remap":null, - "buckets":[ - 0.005, - 0.01, - 0.02, - 0.03, - 0.04, - 0.05, - 0.075, - 0.1, - 0.25, - 1 - ], - "valueScale":1000000000 - }, - { - "name":"workload_drop_packets_total", - "type":"counter", - "filters":[ - { - "key":"PktDropPackets", - "value":"", - "type":"presence" - } - ], - "valueKey":"PktDropPackets", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type", - "PktDropLatestState", - "PktDropLatestDropCause" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_drop_bytes_total", - "type":"counter", - "filters":[ - { - "key":"PktDropBytes", - "value":"", - "type":"presence" - } - ], - "valueKey":"PktDropBytes", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type", - "PktDropLatestState", - "PktDropLatestDropCause" - ], - "remap":null, - "buckets":null - }, - { - "name":"workload_dns_latency_seconds", - "type":"histogram", - "filters":[ - { - "key":"DnsId", - "value":"", - "type":"presence" - } - ], - "valueKey":"DnsLatencyMs", - "labels":[ - "SrcK8S_Namespace", - "DstK8S_Namespace", - "K8S_FlowLayer", - "SrcSubnetLabel", - "DstSubnetLabel", - "SrcK8S_OwnerName", - "DstK8S_OwnerName", - "SrcK8S_OwnerType", - "DstK8S_OwnerType", - "SrcK8S_Type", - "DstK8S_Type", - "DnsFlagsResponseCode" - ], - "remap":null, - "buckets":[ - 0.005, - 0.01, - 0.02, - 0.03, - 0.04, - 0.05, - 0.075, - 0.1, - 0.25, - 1 - ], - "valueScale":1000 - } - ], - "prefix":"on_demand_netobserv_", - "expiryTime":"0s" - } - } - } - ], - "pipeline":[ - { - "name":"enrich", - "follows":"preset-ingester" - }, - { - "name":"prometheus", - "follows":"enrich" - } - ], - } + value: | + {} ports: - name: prometheus containerPort: 9401 diff --git a/res/metrics-pipeline-config.json b/res/metrics-pipeline-config.json new file mode 100644 index 00000000..3e9409fe --- /dev/null +++ b/res/metrics-pipeline-config.json @@ -0,0 +1,399 @@ +{ + "log-level": "trace", + "metricsSettings": { + "port": 9401, + "prefix": "on_demand_netobserv_", + "noPanic": true + }, + "parameters": [ + { + "name": "enrich", + "transform": { + "type": "network", + "network": { + "rules": [ + { + "type": "add_kubernetes", + "kubernetes": { + "add_zone": true, + "ipField": "SrcAddr", + "output": "SrcK8S" + } + }, + { + "type": "add_kubernetes", + "kubernetes": { + "add_zone": true, + "ipField": "DstAddr", + "output": "DstK8S" + } + }, + { + "type": "reinterpret_direction" + }, + { + "type": "add_kubernetes_infra", + "kubernetes_infra": { + "namespaceNameFields": [ + { + "name": "SrcK8S_Name", + "namespace": "SrcK8S_Namespace" + }, + { + "name": "DstK8S_Name", + "namespace": "DstK8S_Namespace" + } + ], + "output": "K8S_FlowLayer", + "infra_prefixes": [ + "netobserv", + "openshift" + ], + "infra_refs": [ + { + "name": "kubernetes", + "namespace": "default" + }, + { + "name": "openshift", + "namespace": "default" + } + ] + } + } + ], + "directionInfo": { + "reporterIPField": "AgentIP", + "srcHostField": "SrcK8S_HostIP", + "dstHostField": "DstK8S_HostIP", + "flowDirectionField": "FlowDirection" + } + } + } + }, + { + "name": "prometheus", + "encode": { + "type": "prom", + "prom": { + "metrics": [ + { + "name": "node_egress_bytes_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "1|2", + "type": "match_regex" + } + ], + "valueKey": "Bytes", + "labels": [ + "SrcK8S_HostName", + "DstK8S_HostName" + ], + "remap": null, + "buckets": null + }, + { + "name": "node_ingress_bytes_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "0|2", + "type": "match_regex" + } + ], + "valueKey": "Bytes", + "labels": [ + "SrcK8S_HostName", + "DstK8S_HostName" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_egress_bytes_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "1|2", + "type": "match_regex" + } + ], + "valueKey": "Bytes", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_ingress_bytes_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "0|2", + "type": "match_regex" + } + ], + "valueKey": "Bytes", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_egress_packets_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "1|2", + "type": "match_regex" + } + ], + "valueKey": "Packets", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_ingress_packets_total", + "type": "counter", + "filters": [ + { + "key": "FlowDirection", + "value": "0|2", + "type": "match_regex" + } + ], + "valueKey": "Packets", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_flows_total", + "type": "counter", + "filters": [], + "valueKey": "", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_rtt_seconds", + "type": "histogram", + "filters": [ + { + "key": "TimeFlowRttNs", + "value": "", + "type": "presence" + } + ], + "valueKey": "TimeFlowRttNs", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type" + ], + "remap": null, + "buckets": [ + 0.005, + 0.01, + 0.02, + 0.03, + 0.04, + 0.05, + 0.075, + 0.1, + 0.25, + 1 + ], + "valueScale": 1000000000 + }, + { + "name": "workload_drop_packets_total", + "type": "counter", + "filters": [ + { + "key": "PktDropPackets", + "value": "", + "type": "presence" + } + ], + "valueKey": "PktDropPackets", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type", + "PktDropLatestState", + "PktDropLatestDropCause" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_drop_bytes_total", + "type": "counter", + "filters": [ + { + "key": "PktDropBytes", + "value": "", + "type": "presence" + } + ], + "valueKey": "PktDropBytes", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type", + "PktDropLatestState", + "PktDropLatestDropCause" + ], + "remap": null, + "buckets": null + }, + { + "name": "workload_dns_latency_seconds", + "type": "histogram", + "filters": [ + { + "key": "DnsId", + "value": "", + "type": "presence" + } + ], + "valueKey": "DnsLatencyMs", + "labels": [ + "SrcK8S_Namespace", + "DstK8S_Namespace", + "K8S_FlowLayer", + "SrcSubnetLabel", + "DstSubnetLabel", + "SrcK8S_OwnerName", + "DstK8S_OwnerName", + "SrcK8S_OwnerType", + "DstK8S_OwnerType", + "SrcK8S_Type", + "DstK8S_Type", + "DnsFlagsResponseCode" + ], + "remap": null, + "buckets": [ + 0.005, + 0.01, + 0.02, + 0.03, + 0.04, + 0.05, + 0.075, + 0.1, + 0.25, + 1 + ], + "valueScale": 1000 + } + ], + "prefix": "on_demand_netobserv_", + "expiryTime": "0s" + } + } + } + ], + "pipeline": [ + { + "name": "enrich", + "follows": "preset-ingester" + }, + { + "name": "prometheus", + "follows": "enrich" + } + ] +} \ No newline at end of file diff --git a/res/packet-capture.yml b/res/packet-capture.yml index eda93208..3fa81e38 100644 --- a/res/packet-capture.yml +++ b/res/packet-capture.yml @@ -32,81 +32,13 @@ spec: - name: LOG_LEVEL value: info - name: FLOW_FILTER_RULES - value: > - [ { "direction": "", "ip_cidr": "0.0.0.0/0", "protocol": "", "source_port": 0, - "destination_port": 0, "port": 0, "source_port_range": "", - "source_ports": "", "destination_port_range": "", - "destination_ports": "", "port_range": "", "ports": "", - "icmp_type": 0, "icmp_code": 0, "peer_ip": "", "action": "Accept", - "tcp_flags": "", "drops": false } - ] + value: >- + [] - name: EXPORT value: "direct-flp" - name: FLP_CONFIG - value: > - { - "log-level": "trace", - "metricsSettings":{ - "disableGlobalServer": true - }, - "parameters":[ - { - "name":"enrich", - "transform":{ - "type":"network", - "network":{ - "rules":[ - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"SrcAddr", - "output":"SrcK8S" - } - }, - { - "type":"add_kubernetes", - "kubernetes":{ - "add_zone": true, - "ipField":"DstAddr", - "output":"DstK8S" - } - }, - { - "type":"reinterpret_direction" - } - ], - "directionInfo":{ - "reporterIPField":"AgentIP", - "srcHostField":"SrcK8S_HostIP", - "dstHostField":"DstK8S_HostIP", - "flowDirectionField":"FlowDirection" - } - } - } - }, - { - "name":"send", - "write":{ - "type":"grpc", - "grpc":{ - "targetHost":"{{TARGET_HOST}}", - "targetPort":9999 - } - } - } - ], - "pipeline":[ - { - "name":"enrich", - "follows":"preset-ingester" - }, - { - "name":"send", - "follows":"enrich" - } - ] - } + value: | + {} volumeMounts: - name: bpf-kernel-debug mountPath: /sys/kernel/debug diff --git a/scripts/functions.sh b/scripts/functions.sh index f775a02c..8bb09990 100755 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -40,6 +40,14 @@ if [ -n "$NETOBSERV_AGENT_IMAGE" ]; then agentImg="$NETOBSERV_AGENT_IMAGE" fi +FLOWS_MANIFEST_FILE="flow-capture.yml" +PACKETS_MANIFEST_FILE="packet-capture.yml" +METRICS_MANIFEST_FILE="metric-capture.yml" +CONFIG_JSON_TEMP="config.json" +CLUSTER_CONFIG="cluster-config-v1.yaml" +NETWORK_CONFIG="cluster-network.yaml" +MANIFEST_OUTPUT_PATH="tmp" + function loadYAMLs() { namespaceYAML=' namespaceYAMLContent @@ -64,7 +72,6 @@ function loadYAMLs() { flowAgentYAML="$(cat ./res/flow-capture.yml)" fi flowAgentYAML="${flowAgentYAML/"{{NAMESPACE}}"/${namespace}}" - flowAgentYAML="${flowAgentYAML/"{{TARGET_HOST}}"/${targetHost}}" flowAgentYAML="${flowAgentYAML/"{{AGENT_IMAGE_URL}}"/${agentImg}}" packetAgentYAML=' @@ -74,7 +81,6 @@ function loadYAMLs() { packetAgentYAML="$(cat ./res/packet-capture.yml)" fi packetAgentYAML="${packetAgentYAML/"{{NAMESPACE}}"/${namespace}}" - packetAgentYAML="${packetAgentYAML/"{{TARGET_HOST}}"/${targetHost}}" packetAgentYAML="${packetAgentYAML/"{{AGENT_IMAGE_URL}}"/${agentImg}}" metricAgentYAML=' @@ -103,6 +109,37 @@ function loadYAMLs() { smYAML="${smYAML//"{{NAMESPACE}}"/${namespace}}" } +# set pipeline for flows & packets using collector +function setCollectorPipelineConfig() { + # load pipeline json + collectorPipelineConfigJSON=' + collectorPipelineConfigJSONContent + ' + if [ -f ./res/collector-pipeline-config.json ]; then + collectorPipelineConfigJSON="$(< ./res/collector-pipeline-config.json tr '\n' ' ')" + fi + + # replace target host + collectorPipelineConfigJSON="${collectorPipelineConfigJSON/"{{TARGET_HOST}}"/${targetHost}}" + + # append json to yaml file + "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLP_CONFIG\").value |= ($collectorPipelineConfigJSON | tojson)" "$1" +} + +# set pipeline for metrics +function setMetricsPipelineConfig() { + # load pipeline json + metricsPipelineConfigJSON=' + metricsPipelineConfigJSONContent + ' + if [ -f ./res/metrics-pipeline-config.json ]; then + metricsPipelineConfigJSON="$(< ./res/metrics-pipeline-config.json tr '\n' ' ')" + fi + + # append json to yaml file + "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLP_CONFIG\").value |= ($metricsPipelineConfigJSON | tojson)" "$1" +} + function clusterIsReady() { # use oc whoami as connectivity check by default and fallback to kubectl get all if needed K8S_CLI_CONNECTIVITY="${K8S_CLI_BIN} whoami" @@ -125,14 +162,6 @@ function namespaceFound() { fi } -FLOWS_MANIFEST_FILE="flow-capture.yml" -PACKETS_MANIFEST_FILE="packet-capture.yml" -CONFIG_JSON_TEMP="config.json" -CLUSTER_CONFIG="cluster-config-v1.yaml" -NETWORK_CONFIG="cluster-network.yaml" -METRICS_MANIFEST_FILE="metric-capture.yml" -MANIFEST_OUTPUT_PATH="tmp" - function getSubnets() { declare -n sn="$1" @@ -215,6 +244,7 @@ function setup { fi manifest="${MANIFEST_OUTPUT_PATH}/${FLOWS_MANIFEST_FILE}" echo "${flowAgentYAML}" >${manifest} + setCollectorPipelineConfig "$manifest" options="$*" check_args_and_apply "$options" "$manifest" "flows" elif [ "$1" = "packets" ]; then @@ -227,6 +257,7 @@ function setup { fi manifest="${MANIFEST_OUTPUT_PATH}/${PACKETS_MANIFEST_FILE}" echo "${packetAgentYAML}" >${manifest} + setCollectorPipelineConfig "$manifest" options="$*" check_args_and_apply "$options" "$manifest" "packets" elif [ "$1" = "metrics" ]; then @@ -239,6 +270,7 @@ function setup { fi manifest="${MANIFEST_OUTPUT_PATH}/${METRICS_MANIFEST_FILE}" echo "${metricAgentYAML}" >${manifest} + setMetricsPipelineConfig "$manifest" options="$*" check_args_and_apply "$options" "$manifest" "metrics" fi @@ -404,12 +436,39 @@ function updateFLPConfig { "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"FLP_CONFIG\").value|=\"$jsonContent\"" "$2" } +# append a new flow filter rule to array +function addFlowFilter() { + flowFilterJSON=' + flowFilterJSONContent + ' + if [ -f ./res/flow-filter.json ]; then + flowFilterJSON="$(cat ./res/flow-filter.json)" + fi + + "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | . += $flowFilterJSON | tojson)" "$1" +} + +# update last flow filter of the array +function setLastFlowFilter() { + "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | .[-1].$1 = $2 | tostring)" "$3" +} + +# replace the configuration in the manifest file function edit_manifest() { - ## replace the configuration in the manifest file - echo "opt: $1, evalue: $2" + if [ -z "${2}" ]; then + echo "opt: $1" + else + echo "opt: $1, value: $2" + fi if [[ $1 == "filter_"* ]]; then "$YQ_BIN" e --inplace ".spec.template.spec.containers[0].env[] |= select(.name==\"ENABLE_FLOW_FILTER\").value|=\"true\"" "$3" + + # add first filter in the array + currentFilters=$( "$YQ_BIN" -r ".spec.template.spec.containers[0].env[] | select(.name == \"FLOW_FILTER_RULES\").value" "$3" ) + if [[ $currentFilters == "[]" ]]; then + addFlowFilter "$3" + fi fi case "$1" in @@ -460,63 +519,66 @@ function edit_manifest() { fi fi ;; + "add_filter") + addFlowFilter "$3" + ;; "filter_direction") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.direction = \"$2\")| tostring)" "$3" + setLastFlowFilter "direction" "\"$2\"" "$3" ;; "filter_cidr") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.ip_cidr = \"$2\")| tostring)" "$3" + setLastFlowFilter "ip_cidr" "\"$2\"" "$3" ;; "filter_protocol") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.protocol = \"$2\")| tostring)" "$3" + setLastFlowFilter "protocol" "\"$2\"" "$3" ;; "filter_sport") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.source_port = $2)| tostring)" "$3" + setLastFlowFilter "source_port" = "$2" "$3" ;; "filter_dport") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.destination_port = $2)| tostring)" "$3" + setLastFlowFilter "destination_port" "$2" "$3" ;; "filter_port") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.port = $2)| tostring)" "$3" + setLastFlowFilter "port" "$2" "$3" ;; "filter_sport_range") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.source_port_range = \"$2\")| tostring)" "$3" + setLastFlowFilter "source_port_range" "\"$2\"" "$3" ;; "filter_dport_range") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.destination_port_range = \"$2\")| tostring)" "$3" + setLastFlowFilter "destination_port_range" "\"$2\"" "$3" ;; "filter_port_range") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.port_range = \"$2\")| tostring)" "$3" + setLastFlowFilter "port_range" "\"$2\"" "$3" ;; "filter_sports") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.source_ports = \"$2\")| tostring)" "$3" + setLastFlowFilter "source_ports" "\"$2\"" "$3" ;; "filter_dports") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.destination_ports = \"$2\")| tostring)" "$3" + setLastFlowFilter "destination_ports" "\"$2\"" "$3" ;; "filter_ports") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.ports = \"$2\")| tostring)" "$3" + setLastFlowFilter "ports" "\"$2\"" "$3" ;; "filter_icmp_type") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.icmp_type = $2)| tostring)" "$3" + setLastFlowFilter "icmp_type" "$2" "$3" ;; "filter_icmp_code") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.icmp_code = $2)| tostring)" "$3" + setLastFlowFilter "icmp_code" "$2" "$3" ;; "filter_peer_ip") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.peer_ip = \"$2\")| tostring)" "$3" + setLastFlowFilter "peer_ip" "\"$2\"" "$3" ;; "filter_action") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.action = \"$2\")| tostring)" "$3" + setLastFlowFilter "action" "\"$2\"" "$3" ;; "filter_tcp_flags") - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.tcp_flags = \"$2\")| tostring)" "$3" + setLastFlowFilter "tcp_flags" "\"$2\"" "$3" ;; "filter_pkt_drops") if [[ "$2" == "true" ]]; then # force enable drops before setting filter edit_manifest "pktdrop_enable" "$2" "$3" fi - "$YQ_BIN" e --inplace " .spec.template.spec.containers[0].env[] |= select(.name == \"FLOW_FILTER_RULES\").value |=(fromjson | map(.drops = $2)| tostring)" "$3" + setLastFlowFilter "drops" "$2" "$3" ;; "filter_regexes") copyFLPConfig "$3" @@ -581,6 +643,9 @@ function check_args_and_apply() { key="${option%%=*}" value="${option#*=}" case "$key" in + or) # Increment flow filter array + edit_manifest "add_filter" "" "$2" + ;; --background) # Run command in background defaultValue "true" if [[ "$value" == "true" || "$value" == "false" ]]; then diff --git a/scripts/inject.sh b/scripts/inject.sh index e8828b60..ee4e62ee 100755 --- a/scripts/inject.sh +++ b/scripts/inject.sh @@ -65,6 +65,15 @@ if [ -z "$KREW_PLUGIN" ] || [ "$KREW_PLUGIN" = "false" ]; then fi # inject YAML files to functions.sh +sed -i.bak '/collectorPipelineConfigJSONContent/{r ./res/collector-pipeline-config.json +d +}' ./tmp/functions.sh +sed -i.bak '/metricsPipelineConfigJSONContent/{r ./res/metrics-pipeline-config.json +d +}' ./tmp/functions.sh +sed -i.bak '/flowFilterJSONContent/{r ./res/flow-filter.json +d +}' ./tmp/functions.sh sed -i.bak '/namespaceYAMLContent/{r ./res/namespace.yml d }' ./tmp/functions.sh