GOAL: Setup a Kafka cluster

Using the Kafka version distributed by Confluent Inc.

Download docker images (Open Source version of Confluent Inc.)

$ docker pull confluentinc/cp-zookeeper:5.0.0
$ docker pull confluentinc/cp-kafka:5.0.0
$ docker pull confluentinc/cp-kafka-connect:5.0.0
$ docker pull confluentinc/cp-schema-registry:5.0.0
$ docker pull confluentinc/cp-kafka-rest:5.0.0

Docker network

$ docker network create \
  --driver=bridge \
  --subnet=10.33.0.0/24 \
  --gateway=10.33.0.1 \
  knet

$ docker network list

1. zookeeper

docker run -d \
  --name zookeeper \
  --hostname zookeeper \
  --env ZOOKEEPER_CLIENT_PORT=2181 \
  --env ZOOKEEPER_TICK_TIME=2000 \
  -p 2181:2181 \
  --network=knet \
  confluentinc/cp-zookeeper:5.0.0

2. broker

docker run -d \
  --name broker \
  --hostname broker \
  -p 9092:9092 \
  -p 29092:29092 \
  --network=knet \
  --env KAFKA_BROKER_ID=1 \
  --env KAFKA_ZOOKEEPER_CONNECT='zookeeper:2181' \
  --env KAFKA_LISTENER_SECURITY_PROTOCOL_MAP='PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' \
  --env KAFKA_ADVERTISED_LISTENERS='PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092' \
  --env KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
  --env KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \
  --env CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS='broker:9092' \
  --env CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT='zookeeper:2181' \
  --env CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 \
  --env CONFLUENT_METRICS_ENABLE='true' \
  --env CONFLUENT_SUPPORT_CUSTOMER_ID='anonymous' \
  confluentinc/cp-kafka:5.0.0

3. Schema Registry

docker run -d \
  --name schema-registry \
  --hostname schema-registry \
  -p 8081:8081 \
  --network=knet \
  --env SCHEMA_REGISTRY_HOST_NAME='schema-registry' \
  --env SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL='zookeeper:2181' \
  confluentinc/cp-schema-registry:5.0.0

4. Kafka Connect

docker run -d \
  --name connect \
  --hostname connect \
  -p 8083:8083 \
  --network=knet \
  -v /Users/larrysu/repos/kafka-test/con-spooldir/usr/share/kafka-connect/kafka-connect-spooldir\
:/usr/share/java/kafka-connect/kafka-connect/spooldir \
  --env CONNECT_BOOTSTRAP_SERVERS='broker:9092' \
  --env CONNECT_REST_ADVERTISED_HOST_NAME='connect' \
  --env CONNECT_REST_PORT='8083' \
  --env CONNECT_GROUP_ID='compose-connect-group' \
  --env CONNECT_CONFIG_STORAGE_TOPIC='docker-connect-configs' \
  --env CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR='1' \
  --env CONNECT_OFFSET_FLUSH_INTERVAL_MS='10000' \
  --env CONNECT_OFFSET_STORAGE_TOPIC='docker-connect-offsets' \
  --env CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR='1' \
  --env CONNECT_STATUS_STORAGE_TOPIC='docker-connect-status' \
  --env CONNECT_STATUS_STORAGE_REPLICATION_FACTOR='1' \
  --env CONNECT_KEY_CONVERTER='io.confluent.connect.avro.AvroConverter' \
  --env CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL='http://schema-registry:8081' \
  --env CONNECT_VALUE_CONVERTER='io.confluent.connect.avro.AvroConverter' \
  --env CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL='http://schema-registry:8081' \
  --env CONNECT_INTERNAL_KEY_CONVERTER='org.apache.kafka.connect.json.JsonConverter' \
  --env CONNECT_INTERNAL_VALUE_CONVERTER='org.apache.kafka.connect.json.JsonConverter' \
  --env CONNECT_ZOOKEEPER_CONNECT='zookeeper:2181' \
  --env CONNECT_PLUGIN_PATH='/usr/share/java,/usr/share/java/kafka-connect/kafka-connect/spooldir' \
  --env CONNECT_LOG4J_LOGGERS='org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR' \
  confluentinc/cp-kafka-connect:5.0.0

Test

# create topics
$ docker exec broker kafka-topics --create --zookeeper zookeeper:2181 \
  --replication-factor 1 --partitions 1 --topic connect-testroot

# list topics
$ docker exec broker kafka-topics --list --zookeeper zookeeper:2181

# create a basic source-connector
$ docker exec connect curl -X POST \
  -H "Content-Type: application/json" \
  --data '{ "name": "helloworld-fs-source", "config": { "connector.class": "FileStreamSource", "tasks.max": 1, "file": "/test.txt", "topic": "connect-testroot" } }' \
  http://localhost:8083/connectors

# check running connectors
$ docker exec connect curl http://localhost:8083/connectors

$ docker exec connect kafka-console-consumer --bootstrap-server broker:9092 --topic connect-testroot --from-beginning

$ docker exec connect curl http://localhost:8083/connectors/helloworld-fs-source/status
$ docker exec connect curl -X POST http://localhost:8083/connectors/helloworld-fs-source/restart

$ docker exec connect curl http://localhost:8083/connector-plugins


$ docker exec broker kafka-topics --create --zookeeper zookeeper:2181 \
  --replication-factor 1 --partitions 1 --topic g1-says

$ docker exec connect curl -X POST \
    -H "Content-Type: application/json" \
    --data '{ "name": "helloworld-g1", "config": { "connector.class": "FileStreamSource", "tasks.max": 1, "file": "/g1.txt", "topic": "g1-says" } }' \
    http://localhost:8083/connectors

$ docker exec connect kafka-console-consumer --bootstrap-server broker:9092 --topic g1-says --from-beginning

$ docker exec connect curl -X DELETE http://localhost:8083/connectors/helloworld-g1

ref

docker run -d \
  --name jenkins-rnd \
  --env JENKINS_OPTS="--prefix=/jenkins" \
  --env JENKINS_SLAVE_AGENT_PORT=50010 \
  --env TZ=Asia/Taipei \
  --hostname jenkins-rnd \
  -v jenkins-rnd-vol:/var/jenkins_home \
  --network=br0 \
  --ip=10.99.0.100 \
  -p 9080:8080 -p 50010:50010 \
  migocorp/jenkins-rnd:latest