labs/kafka
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
DerInti
2024-06-22 16:04:18 +02:00
commit 829aec9157
8 changed files with 318 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
*.tfstate*
.terraform/

97
cluster/podman/kafka-configmap.yml Normal file
View File

@@ -0,0 +1,97 @@
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-env
namespace: default
data:
KAFKA_PROCESS_ROLES: 'controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:9093'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-env
namespace: default
data:
KAFKA_PROCESS_ROLES: 'broker'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093'
KAFKA_LISTENERS: 'PLAINTEXT://:49092,PLAINTEXT_HOST://:9092'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-1
namespace: default
data:
KAFKA_NODE_ID: 1
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-2
namespace: default
data:
KAFKA_NODE_ID: 2
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-3
namespace: default
data:
KAFKA_NODE_ID: 3
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-1
namespace: default
data:
KAFKA_NODE_ID: 4
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k1:49092,PLAINTEXT_HOST://192.168.178.61:19092'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-2
namespace: default
data:
KAFKA_NODE_ID: 5
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k2:49092,PLAINTEXT_HOST://192.168.178.61:29092'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-3
namespace: default
data:
KAFKA_NODE_ID: 6
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k3:49092,PLAINTEXT_HOST://192.168.178.61:39092'

8
cluster/podman/kafka-ctl-config.yml Normal file
View File

@@ -0,0 +1,8 @@
---
contexts:
default:
brokers:
- k1:9092
- k2:9092
- k3:9092
current-context: default

129
cluster/podman/kafka-kube.yml Normal file
View File

@@ -0,0 +1,129 @@
---
apiVersion: v1
kind: Pod
metadata:
name: k1
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-1
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
ports:
- containerPort: 9092
hostPort: 19092
protocol: TCP
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-1
securityContext:
capabilities:
add:
- NET_RAW
---
apiVersion: v1
kind: Pod
metadata:
name: k2
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-2
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-2
ports:
- containerPort: 9092
hostPort: 29092
protocol: TCP
securityContext:
capabilities:
add:
- NET_RAW
---
apiVersion: v1
kind: Pod
metadata:
name: k3
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-3
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-3
ports:
- containerPort: 9092
hostPort: 39092
protocol: TCP
securityContext:
capabilities:
add:
- NET_RAW
---
# https://kubernetes.io/docs/concepts/workloads/pods/
apiVersion: v1
kind: Pod
metadata:
name: "goflow"
spec:
containers:
- name: collector
image: "docker.io/netsampler/goflow2:latest"
# args: ['-transport=file', '-transport.file.sep=', '-format=json']
args: # ['-transport=kafka', '-transport.file.sep=', '-format=json']
- "-transport=kafka"
- "-transport.kafka.brokers=k1:9092,k2:9092,k3:9092"
- "-format=json"
resources:
limits:
cpu: 200m
memory: 500Mi
requests:
cpu: 100m
memory: 200Mi
ports:
- containerPort: 2025
hostPort: 2025
protocol: UDP
restartPolicy: Always
---

23
cluster/terraform/.terraform.lock.hcl generated Normal file
View File

@@ -0,0 +1,23 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/mongey/kafka" {
version = "0.7.1"
constraints = "0.7.1"
hashes = [
"h1:Mg9O6C8oN+FlRdnJTf7Rg5C8/go07nqspXfWugWi++U=",
"zh:2032d7625d7ce6f00d89a479db715468dc127a8347160ebdad8bcf433235254b",
"zh:4e7bbe0759ebb79337b26399d062dcece727503a738f63af255509a159079da3",
"zh:525af3406910de7cb989676eb87552ce729eaedbd771ae8d04d619f7a930fc63",
"zh:596625aadc2103f27f89be31f06d237740ca713c0a93a0c1144f6fd026f5c084",
"zh:60bd31f525e039b64c4666ea9a428e2f3c0e06fc41cb371bb3a3f33b5fbfdad1",
"zh:63bee7b312208b36698025112af72172cc420f90ff884d860602b0eca3e9e9a0",
"zh:671cec0d716b8fb5092046575657d8fedac6bf9015f18b5f6b03c2ae26226862",
"zh:7b203eabeadf1c9459d8c86e7a7bdee5a5d8d1b52a1059cb7f768b39e129cfb2",
"zh:7d9211c4dbfa003fe583a662b2ad143c3550d23c8ad85024ec1e5e2bc8e1485c",
"zh:a60c2266b448dd7d4303eabdefdf97a86b2a32a3f5a45b63a91edff2c1caf659",
"zh:c995180e7b7e6ef048c41609cd03abcf9691da0d22aa5b7c4a032d588ca67c6c",
"zh:e73314cda592cc5dafde17d3e88a2270502f2c43d32fd3f74e61820af3efa1fa",
"zh:ff39bd48e880d609c43cf766b928b57e1c1acc359c9f6837ddfc893e0b87a103",
]
}

19
cluster/terraform/main.tf Normal file
View File

@@ -0,0 +1,19 @@
terraform {
required_providers {
kafka = {
source = "Mongey/kafka"
version = "0.7.1"
}
}
}
provider "kafka" {
bootstrap_servers = ["localhost:19092", "localhost:29092", "localhost:39092"]
tls_enabled = false
}
resource "kafka_topic" "logs" {
name = "flowdata"
replication_factor = 3
partitions = 10
}

22
single/kafka-configmap.yml Normal file
View File

@@ -0,0 +1,22 @@
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: kafka-env
namespace: default
data:
KAFKA_NODE_ID: 1
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://broker:19092'
KAFKA_PROCESS_ROLES: 'broker,controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'

16
single/kafka-kube.yml Normal file
View File

@@ -0,0 +1,16 @@
apiVersion: v1
kind: Pod
metadata:
name: goflow
spec:
containers:
- name: broker
image: apache/kafka:latest
ports:
- containerPort: 9092
hostPort: 9092
protocol: TCP
envFrom:
- configMapRef:
name: kafka-env