From 829aec91572592fdf51f6fdedb0a86714cd7fab7 Mon Sep 17 00:00:00 2001 From: DerInti Date: Sat, 22 Jun 2024 16:04:18 +0200 Subject: [PATCH] Initial Commit --- .gitignore | 4 + cluster/podman/kafka-configmap.yml | 97 +++++++++++++++++++ cluster/podman/kafka-ctl-config.yml | 8 ++ cluster/podman/kafka-kube.yml | 129 ++++++++++++++++++++++++++ cluster/terraform/.terraform.lock.hcl | 23 +++++ cluster/terraform/main.tf | 19 ++++ single/kafka-configmap.yml | 22 +++++ single/kafka-kube.yml | 16 ++++ 8 files changed, 318 insertions(+) create mode 100644 .gitignore create mode 100644 cluster/podman/kafka-configmap.yml create mode 100644 cluster/podman/kafka-ctl-config.yml create mode 100644 cluster/podman/kafka-kube.yml create mode 100644 cluster/terraform/.terraform.lock.hcl create mode 100644 cluster/terraform/main.tf create mode 100644 single/kafka-configmap.yml create mode 100644 single/kafka-kube.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a363bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.tfstate* +.terraform/ + + diff --git a/cluster/podman/kafka-configmap.yml b/cluster/podman/kafka-configmap.yml new file mode 100644 index 0000000..c0f61cb --- /dev/null +++ b/cluster/podman/kafka-configmap.yml @@ -0,0 +1,97 @@ +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: controller-env + namespace: default +data: + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: broker-env + namespace: default +data: + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093' + KAFKA_LISTENERS: 'PLAINTEXT://:49092,PLAINTEXT_HOST://:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: controller-1 + namespace: default +data: + KAFKA_NODE_ID: 1 +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: controller-2 + namespace: default +data: + KAFKA_NODE_ID: 2 +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: controller-3 + namespace: default +data: + KAFKA_NODE_ID: 3 +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: broker-1 + namespace: default +data: + KAFKA_NODE_ID: 4 + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k1:49092,PLAINTEXT_HOST://192.168.178.61:19092' +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: broker-2 + namespace: default +data: + KAFKA_NODE_ID: 5 + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k2:49092,PLAINTEXT_HOST://192.168.178.61:29092' +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: broker-3 + namespace: default +data: + KAFKA_NODE_ID: 6 + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k3:49092,PLAINTEXT_HOST://192.168.178.61:39092' + diff --git a/cluster/podman/kafka-ctl-config.yml b/cluster/podman/kafka-ctl-config.yml new file mode 100644 index 0000000..b3a24ba --- /dev/null +++ b/cluster/podman/kafka-ctl-config.yml @@ -0,0 +1,8 @@ +--- +contexts: + default: + brokers: + - k1:9092 + - k2:9092 + - k3:9092 +current-context: default diff --git a/cluster/podman/kafka-kube.yml b/cluster/podman/kafka-kube.yml new file mode 100644 index 0000000..757942a --- /dev/null +++ b/cluster/podman/kafka-kube.yml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: k1 + +spec: + containers: + - name: controller + image: apache/kafka:latest + envFrom: + - configMapRef: + name: controller-env + - configMapRef: + name: controller-1 + securityContext: + capabilities: + add: + - NET_RAW + - name: broker + image: apache/kafka:latest + ports: + - containerPort: 9092 + hostPort: 19092 + protocol: TCP + envFrom: + - configMapRef: + name: broker-env + - configMapRef: + name: broker-1 + securityContext: + capabilities: + add: + - NET_RAW +--- +apiVersion: v1 +kind: Pod +metadata: + name: k2 +spec: + containers: + - name: controller + image: apache/kafka:latest + envFrom: + - configMapRef: + name: controller-env + - configMapRef: + name: controller-2 + securityContext: + capabilities: + add: + - NET_RAW + - name: broker + image: apache/kafka:latest + envFrom: + - configMapRef: + name: broker-env + - configMapRef: + name: broker-2 + ports: + - containerPort: 9092 + hostPort: 29092 + protocol: TCP + securityContext: + capabilities: + add: + - NET_RAW +--- +apiVersion: v1 +kind: Pod +metadata: + name: k3 +spec: + containers: + - name: controller + image: apache/kafka:latest + envFrom: + - configMapRef: + name: controller-env + - configMapRef: + name: controller-3 + securityContext: + capabilities: + add: + - NET_RAW + - name: broker + image: apache/kafka:latest + envFrom: + - configMapRef: + name: broker-env + - configMapRef: + name: broker-3 + ports: + - containerPort: 9092 + hostPort: 39092 + protocol: TCP + securityContext: + capabilities: + add: + - NET_RAW +--- +# https://kubernetes.io/docs/concepts/workloads/pods/ +apiVersion: v1 +kind: Pod +metadata: + name: "goflow" +spec: + containers: + - name: collector + image: "docker.io/netsampler/goflow2:latest" + # args: ['-transport=file', '-transport.file.sep=', '-format=json'] + args: # ['-transport=kafka', '-transport.file.sep=', '-format=json'] + - "-transport=kafka" + - "-transport.kafka.brokers=k1:9092,k2:9092,k3:9092" + - "-format=json" + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + ports: + - containerPort: 2025 + hostPort: 2025 + protocol: UDP + restartPolicy: Always +--- + diff --git a/cluster/terraform/.terraform.lock.hcl b/cluster/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..cf6bfd7 --- /dev/null +++ b/cluster/terraform/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/mongey/kafka" { + version = "0.7.1" + constraints = "0.7.1" + hashes = [ + "h1:Mg9O6C8oN+FlRdnJTf7Rg5C8/go07nqspXfWugWi++U=", + "zh:2032d7625d7ce6f00d89a479db715468dc127a8347160ebdad8bcf433235254b", + "zh:4e7bbe0759ebb79337b26399d062dcece727503a738f63af255509a159079da3", + "zh:525af3406910de7cb989676eb87552ce729eaedbd771ae8d04d619f7a930fc63", + "zh:596625aadc2103f27f89be31f06d237740ca713c0a93a0c1144f6fd026f5c084", + "zh:60bd31f525e039b64c4666ea9a428e2f3c0e06fc41cb371bb3a3f33b5fbfdad1", + "zh:63bee7b312208b36698025112af72172cc420f90ff884d860602b0eca3e9e9a0", + "zh:671cec0d716b8fb5092046575657d8fedac6bf9015f18b5f6b03c2ae26226862", + "zh:7b203eabeadf1c9459d8c86e7a7bdee5a5d8d1b52a1059cb7f768b39e129cfb2", + "zh:7d9211c4dbfa003fe583a662b2ad143c3550d23c8ad85024ec1e5e2bc8e1485c", + "zh:a60c2266b448dd7d4303eabdefdf97a86b2a32a3f5a45b63a91edff2c1caf659", + "zh:c995180e7b7e6ef048c41609cd03abcf9691da0d22aa5b7c4a032d588ca67c6c", + "zh:e73314cda592cc5dafde17d3e88a2270502f2c43d32fd3f74e61820af3efa1fa", + "zh:ff39bd48e880d609c43cf766b928b57e1c1acc359c9f6837ddfc893e0b87a103", + ] +} diff --git a/cluster/terraform/main.tf b/cluster/terraform/main.tf new file mode 100644 index 0000000..4c47a98 --- /dev/null +++ b/cluster/terraform/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + kafka = { + source = "Mongey/kafka" + version = "0.7.1" + } + } +} + +provider "kafka" { + bootstrap_servers = ["localhost:19092", "localhost:29092", "localhost:39092"] + tls_enabled = false +} + +resource "kafka_topic" "logs" { + name = "flowdata" + replication_factor = 3 + partitions = 10 +} diff --git a/single/kafka-configmap.yml b/single/kafka-configmap.yml new file mode 100644 index 0000000..72d2044 --- /dev/null +++ b/single/kafka-configmap.yml @@ -0,0 +1,22 @@ +--- +# https://kubernetes.io/docs/concepts/configuration/configmap/ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-env + namespace: default +data: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://broker:19092' + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' diff --git a/single/kafka-kube.yml b/single/kafka-kube.yml new file mode 100644 index 0000000..f722306 --- /dev/null +++ b/single/kafka-kube.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: goflow + +spec: + containers: + - name: broker + image: apache/kafka:latest + ports: + - containerPort: 9092 + hostPort: 9092 + protocol: TCP + envFrom: + - configMapRef: + name: kafka-env