labs/kafka
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
DerInti
2024-06-22 16:04:18 +02:00
commit 829aec9157
8 changed files with 318 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
cluster/podman/kafka-configmap.yml Normal file
View File

@@ -0,0 +1,97 @@
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-env
namespace: default
data:
KAFKA_PROCESS_ROLES: 'controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:9093'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-env
namespace: default
data:
KAFKA_PROCESS_ROLES: 'broker'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@k1:9093,2@k2:9093,3@k3:9093'
KAFKA_LISTENERS: 'PLAINTEXT://:49092,PLAINTEXT_HOST://:9092'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-1
namespace: default
data:
KAFKA_NODE_ID: 1
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-2
namespace: default
data:
KAFKA_NODE_ID: 2
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: controller-3
namespace: default
data:
KAFKA_NODE_ID: 3
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-1
namespace: default
data:
KAFKA_NODE_ID: 4
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k1:49092,PLAINTEXT_HOST://192.168.178.61:19092'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-2
namespace: default
data:
KAFKA_NODE_ID: 5
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k2:49092,PLAINTEXT_HOST://192.168.178.61:29092'
---
# https://kubernetes.io/docs/concepts/configuration/configmap/
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-3
namespace: default
data:
KAFKA_NODE_ID: 6
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://k3:49092,PLAINTEXT_HOST://192.168.178.61:39092'

8
cluster/podman/kafka-ctl-config.yml Normal file
View File

@@ -0,0 +1,8 @@
---
contexts:
default:
brokers:
- k1:9092
- k2:9092
- k3:9092
current-context: default

129
cluster/podman/kafka-kube.yml Normal file
View File

@@ -0,0 +1,129 @@
---
apiVersion: v1
kind: Pod
metadata:
name: k1
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-1
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
ports:
- containerPort: 9092
hostPort: 19092
protocol: TCP
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-1
securityContext:
capabilities:
add:
- NET_RAW
---
apiVersion: v1
kind: Pod
metadata:
name: k2
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-2
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-2
ports:
- containerPort: 9092
hostPort: 29092
protocol: TCP
securityContext:
capabilities:
add:
- NET_RAW
---
apiVersion: v1
kind: Pod
metadata:
name: k3
spec:
containers:
- name: controller
image: apache/kafka:latest
envFrom:
- configMapRef:
name: controller-env
- configMapRef:
name: controller-3
securityContext:
capabilities:
add:
- NET_RAW
- name: broker
image: apache/kafka:latest
envFrom:
- configMapRef:
name: broker-env
- configMapRef:
name: broker-3
ports:
- containerPort: 9092
hostPort: 39092
protocol: TCP
securityContext:
capabilities:
add:
- NET_RAW
---
# https://kubernetes.io/docs/concepts/workloads/pods/
apiVersion: v1
kind: Pod
metadata:
name: "goflow"
spec:
containers:
- name: collector
image: "docker.io/netsampler/goflow2:latest"
# args: ['-transport=file', '-transport.file.sep=', '-format=json']
args: # ['-transport=kafka', '-transport.file.sep=', '-format=json']
- "-transport=kafka"
- "-transport.kafka.brokers=k1:9092,k2:9092,k3:9092"
- "-format=json"
resources:
limits:
cpu: 200m
memory: 500Mi
requests:
cpu: 100m
memory: 200Mi
ports:
- containerPort: 2025
hostPort: 2025
protocol: UDP
restartPolicy: Always
---