python kafka 调试常用

kafkaProducer.py

# encoding=utf-8
import json
from kafka import KafkaProducer
import time
from backports.configparser.helpers import str


def sendMsg(topic,msg_dict):
    '''send json  String to kafka  '''
    producer = KafkaProducer(bootstrap_servers=['192.168.18.129:9092'],value_serializer=lambda v: json.dumps(v).encode('utf-8'))
#     producer = KafkaProducer(bootstrap_servers='192.168.18.129:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8'))
    producer.send(topic, msg_dict)
    producer.close()
    

if __name__ == '__main__':
    topic = input("pls input topic：");
    while 0<1:
        msg = input("pls input msg：");    
        sendMsg(topic,json.loads(msg))
        print("send over !!!")

kafkaConsumer.py

#encoding=utf-8
from kafka.consumer.group import KafkaConsumer
from kafka.structs import TopicPartition
from backports.configparser.helpers import str

topic = input("pls input topic：");
bootstrap_servers = input("pls input bootstrap_servers  eg : '10.160.3.142:9212','10.160.3.163:9211','10.160.6.162:9210' :");
group_id = input("pls input group_id：");

consumer = KafkaConsumer(topic,bootstrap_servers = [bootstrap_servers],
group_id =group_id,
auto_offset_reset ='earliest',
enable_auto_commit =False)

for msg in consumer:
    offset = msg.offset
    print("offset"+str(offset))
    print("============= consumering 2 +++=========== ")
    recv = "%s:%d:%d: key=%s value=%s" %(msg.topic,msg.partition,msg.offset,msg.key,msg.value)
    print(recv)
    consumer.commit()
    print("============= start consumer +++=========== ")