From 93e06e58184674a69cb7ea9576339c656f8041d2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 11 Dec 2018 06:38:14 +0100 Subject: [PATCH 1/5] An attempt to set up minimal Kafka using deployments --- minikafka/minikafka.yaml | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 minikafka/minikafka.yaml diff --git a/minikafka/minikafka.yaml b/minikafka/minikafka.yaml new file mode 100644 index 00000000..5da6c813 --- /dev/null +++ b/minikafka/minikafka.yaml @@ -0,0 +1,98 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: zookeeper +spec: + selector: + app: zookeeper + ports: + - protocol: TCP + port: 2181 + targetPort: 2181 +--- +kind: Service +apiVersion: v1 +metadata: + name: kafka +spec: + selector: + app: kafka + ports: + - protocol: TCP + port: 9092 + targetPort: 9092 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zk + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 + command: + - ./bin/zookeeper-server-start.sh + args: + - ./config/zookeeper.properties +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + spec: + containers: + - name: broker + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 + command: + - ./bin/kafka-server-start.sh + args: + - ./config/server.properties + - --override + - zookeeper.connect=zookeeper:2181 + - --override + - background.threads=1 + - --override + - num.io.threads=1 + - --override + - num.network.threads=1 + - --override + - log.cleaner.enable=false + - --override + - group.initial.rebalance.delay.ms=0 + - --override + - advertised.listeners=PLAINTEXT://kafka:9092 + ports: + - containerPort: 9092 + resources: + limits: + cpu: 100m + memory: 400Mi From 568b4301a6fa5ed1c32027dc97cb3fd1183a7272 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 11 Dec 2018 06:59:07 +0100 Subject: [PATCH 2/5] Statefulset works --- minikafka/minikafka.yaml | 64 ++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/minikafka/minikafka.yaml b/minikafka/minikafka.yaml index 5da6c813..bd349c91 100644 --- a/minikafka/minikafka.yaml +++ b/minikafka/minikafka.yaml @@ -1,47 +1,43 @@ --- -kind: Service apiVersion: v1 +kind: Service metadata: - name: zookeeper + name: minizoo spec: selector: - app: zookeeper + app: minizoo ports: - - protocol: TCP - port: 2181 - targetPort: 2181 + - port: 9092 + clusterIP: None --- -kind: Service apiVersion: v1 +kind: Service metadata: - name: kafka + name: minikafka spec: selector: - app: kafka + app: minikafka ports: - - protocol: TCP - port: 9092 - targetPort: 9092 + - port: 9092 + clusterIP: None --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: zookeeper + name: minizoo spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 selector: matchLabels: - app: zookeeper + app: minizoo + serviceName: "minizoo" + replicas: 1 template: metadata: labels: - app: zookeeper + app: minizoo + annotations: spec: + terminationGracePeriodSeconds: 5 containers: - name: zk image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 @@ -51,24 +47,22 @@ spec: - ./config/zookeeper.properties --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: kafka + name: minikafka spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 selector: matchLabels: - app: kafka + app: minikafka + serviceName: "minikafka" + replicas: 1 template: metadata: labels: - app: kafka + app: minikafka + annotations: spec: + terminationGracePeriodSeconds: 5 containers: - name: broker image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 @@ -77,7 +71,7 @@ spec: args: - ./config/server.properties - --override - - zookeeper.connect=zookeeper:2181 + - zookeeper.connect=minizoo:2181 - --override - background.threads=1 - --override @@ -89,10 +83,10 @@ spec: - --override - group.initial.rebalance.delay.ms=0 - --override - - advertised.listeners=PLAINTEXT://kafka:9092 + - listeners=PLAINTEXT://minikafka-0:9092 ports: - containerPort: 9092 resources: limits: - cpu: 100m + cpu: 200m memory: 400Mi From 7382b34aa28381c7157f02426290dd7ae4887616 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 11 Dec 2018 07:14:23 +0100 Subject: [PATCH 3/5] Can we somehow support bootstrap from other namespaces? My test case is kubectl run --rm -t -i --image=solsson/kafkacat --restart=Never kafkaprobe -- -b minikafka.test123 -L -d broker --- minikafka/minikafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minikafka/minikafka.yaml b/minikafka/minikafka.yaml index bd349c91..7db874aa 100644 --- a/minikafka/minikafka.yaml +++ b/minikafka/minikafka.yaml @@ -83,7 +83,7 @@ spec: - --override - group.initial.rebalance.delay.ms=0 - --override - - listeners=PLAINTEXT://minikafka-0:9092 + - listeners=PLAINTEXT://minikafka:9092 ports: - containerPort: 9092 resources: From 09a97cf42241865db112c25eca08982b4de19380 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 11 Dec 2018 07:17:03 +0100 Subject: [PATCH 4/5] This setup should replace the ever-rebased branches --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ebefa45..7da75e24 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,10 @@ Available for: * [Brokers](./outside-services/) -## Fewer than three nodes? +## Minikafka -For [minikube](https://github.com/kubernetes/minikube/), [youkube](https://github.com/Yolean/youkube) etc: - - * [Scale 1](https://github.com/Yolean/kubernetes-kafka/pull/44) - * [Scale 2](https://github.com/Yolean/kubernetes-kafka/pull/118) +There's a single pod setup in [minikafka](./minikafka/). +Contributions for making it as lightweight as possible are very welcome. ## Stream... From 4b9f070ee3a9aac2803666ccd758f28472c9cb69 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 11 Dec 2018 07:17:32 +0100 Subject: [PATCH 5/5] Keep using the same image as main --- update-kafka-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update-kafka-image.sh b/update-kafka-image.sh index 87c2f504..064b8754 100755 --- a/update-kafka-image.sh +++ b/update-kafka-image.sh @@ -5,6 +5,6 @@ IMAGE=$1 [[ $IMAGE != solsson/kafka:* ]] && echo "Should be the full image identifier" && exit 1 -for F in kafka/ kafka/test/ zookeeper/ avro-tools/test/ maintenance/; do +for F in kafka/ kafka/test/ zookeeper/ avro-tools/test/ maintenance/ minikafka/; do sed -i "s|image: solsson/kafka:.*|image: $IMAGE|" $F*.yml done