python生产消费Kafka

Posted 2020-12-16 Mars.wang

python生产消费Kafka主要是跟KafkaConsumer和KafkaProducer两个类打交道.

from kafka import KafkaProducer, KafkaConsumer


class PythonKafka(object):
    def __init__(self, topic=None, username=None, password=None):
        self.topic = topic
        self.username = username
        self.password = password
        self.count = 0
        self.error_count = 0

    def on_send_success(self, *args, **kwargs):
        self.count = self.count + 1

    def on_send_error(self, *args, **kwargs):
        self.error_count = self.error_count + 1

    def get_kafka_producer(self):
        kafka_producer = KafkaProducer(bootstrap_servers=[‘ip1:port‘, ‘ip2:port‘, ‘ip3:port‘],
                                       security_protocol=‘SASL_PLAINTEXT‘,
                                       sasl_mechanism=‘PLAIN‘,
                                       sasl_plain_username=‘username‘,
                                       sasl_plain_password=‘password‘,
                                       request_timeout_ms=40000,
                                       api_version=(0, 10, 1))
        return kafka_producer

    def get_kafka_consumer(self):
        # 指定超时时间，不然会一直阻塞
        consumer_params = {
            ‘security_protocol‘: ‘SASL_PLAINTEXT‘,
            ‘sasl_mechanism‘: ‘PLAIN‘,
            ‘group_id‘: ‘test_group2‘,
            ‘sasl_plain_username‘: ‘username‘,
            ‘sasl_plain_password‘: ‘password‘,
            ‘auto_offset_reset‘: ‘earliest‘,
            ‘api_version‘: (0, 10),
            ‘consumer_timeout_ms‘: 10000
        }

        return KafkaConsumer(self.topic,
                             bootstrap_servers=[‘ip1:port‘, ‘ip2:port‘, ‘ip3:port‘],
                             **consumer_params)

    # 异步发送
    def send_data(self, json_str):
        json_bytes = str.encode(json_str)
        producer = self.get_kafka_producer()
        producer.send(self.topic, value=json_bytes).add_callback(self.on_send_success).add_errback(
            self.on_send_error)
        producer.flush()
        producer.close()

    # 接收数据
    def receive_data(self):
        consumer = self.get_kafka_consumer()
        for msg in consumer:
            msg_str = str(msg.value, encoding=‘utf8‘)
            print(msg_str)
        consumer.close()