一、整合Flume和Kafka完成实时数据采集 流程
二、修改flume配置文件,从控制台输出到kafka
/**
*之前的配置控制台文件
*
*/
[peng@bogon conf]$ cat avro-memory-logger.conf
vro-memory-logger.sources = avro-source
avro-memory-logger.sinks = logger-sink
avro-memory-logger.channels = memory-channel
avro-memory-logger.sources.avro-source.type = avro
avro-memory-logger.sources.avro-source.bind = localhost
avro-memory-logger.sources.avro-source.port = 44444
avro-memory-logger.sinks.logger-sink.type = logger
avro-memory-logger.channels.memory-channel.type = memory
avro-memory-logger.sources.avro-source.channels=memory-channel
avro-memory-logger.sinks.logger-sink.channel=memory-channel
[peng@bogon conf]$
/**
*配置到kafka
*/
avro-memory-kafka.sources = avro-source
avro-memory-kafka.sinks = logger-sink
avro-memory-kafka.channels = memory-channel
avro-memory-kafka.sources.avro-source.type = avro
avro-memory-kafka.sources.avro-source.bind = localhost
avro-memory-kafka.sources.avro-source.port = 44444
avro-memory-kafka.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
avro-memory-kafka.sinks.kafka-sink.brokerList = localhost:9092
avro-memory-kafka.sinks.kafka-sink.topic = peng_topic
avro-memory-kafka.sinks.kafka-sink.batchSize = 5
avro-memory-kafka.sinks.kafka-sink.requiredAcks = 1
avro-memory-kafka.channels.memory-channel.type = memory
avro-memory-kafka.sources.avro-source.channels=memory-channel
avro-memory-kafka.sinks.logger-sink.channel=memory-channel