Kafka

#start zookeep
zookeeper-server-start /usr/local/etc/kafka/zookeeper.properties 

#start kafka
kafka-server-start /usr/local/etc/kafka/server.properties

#start a kafka consumer
kafka-console-consumer --zookeeper localhost --topic my-topic

#produce data for kafka
python3 /Users/user/Desktop/kaf.py



#required packages for python
pip3 install kafka-python
pip3 install msgpack-python



zkserver start

#kafka home
/usr/local/opt/kafka/bin/

# change retention time
bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --entity-type topics --entity-name github --add-config retention.ms=1000 

# check current config
bin/kafka-configs.sh --zookeeper zookeeper:2181 --describe --entity-type topics --entity-name github



/data/kafka/bin/kafka-console-producer.sh --broker-list localhost:8087 --topic my-topic

/data/kafka/bin/kafka-console-consumer.sh --zookeeper localhost --topic my-topic

/data/kafka/bin/kafka-topics.sh --zookeeper localhost --list

/data/kafka/bin/kafka-consumer-groups.sh --zookeeper localhost --list

/data/kafka/bin/kafka-topics.sh --create --topic hdfs_clean --zookeeper localhost:2181 --partitions 1 --replication-factor 1

/data/kafka/bin/kafka-topics.sh --zookeeper localhost --describe --topic swagger_clean

/data/kafka/bin/kafka-topics.sh --zookeeper localhost --delete --topic test

/data/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic my-topic --from-beginning