Flume 监听log文件

Flume 监听log文件

vim exec-memory-avro.conf

exec-memory-avro.sources = exec-memory-source
exec-memory-avro.sinks = exec-memory-sink
exec-memory-avro.channels = exec-memory-channel

### 监听的文件
exec-memory-avro.sources.exec-memory-source.type = exec
exec-memory-avro.sources.exec-memory-source.command = tail -F /home/houzhenglan/app/nginx/logs/access.log
exec-memory-avro.sources.exec-memory-source.shell = /bin/sh -c

exec-memory-avro.channels.exec-memory-channel.type = memory

exec-memory-avro.sinks.exec-memory-sink.type = avro
exec-memory-avro.sinks.exec-memory-sink.hostname = localhost
exec-memory-avro.sinks.exec-memory-sink.port = 44444


exec-memory-avro.sources.exec-memory-source.channels = exec-memory-channel
exec-memory-avro.sinks.exec-memory-sink.channel = exec-memory-channel

vim avro-memory-kafka.conf

avro-memory-kafka.sources = avro-source
avro-memory-kafka.sinks = kafka-sink
avro-memory-kafka.channels = memory-channel

avro-memory-kafka.sources.avro-source.type = avro
avro-memory-kafka.sources.avro-source.bind = 0.0.0.0
avro-memory-kafka.sources.avro-source.port = 44444

#avro-memory-kafka.sinks.kafka-sink.type = logger

avro-memory-kafka.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
avro-memory-kafka.sinks.kafka-sink.kafka.bootstrap.servers = localhost:9092
avro-memory-kafka.sinks.kafka-sink.kafka.topic = spark-test
avro-memory-kafka.sinks.kafka-sink.kafka.flumeBatchSize = 5
avro-memory-kafka.sinks.kafka-sink.kafka.producer.acks = 1

avro-memory-kafka.channels.memory-channel.type = memory

avro-memory-kafka.sources.avro-source.channels = memory-channel
avro-memory-kafka.sinks.kafka-sink.channel = memory-channel

./bin/flume-ng agent --conf conf --conf-file conf/exec-memory-avro.conf --name exec-memory-avro -Dflume.root.logger=INFO,console

vim tail-log-kafka.conf

tail-memory-kafka.sources = tail-source
tail-memory-kafka.sinks = kafka-sink
tail-memory-kafka.channels = memory-channel

tail-memory-kafka.sources.tail-source.type = exec
tail-memory-kafka.sources.tail-source.command = tail -F /home/houzhenglan/app/nginx/logs/access.log
exec-memory-avro.sources.tail-source.shell = /bin/sh -c

tail-memory-kafka.channels.memory-channel.type = memory

tail-memory-kafka.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
tail-memory-kafka.sinks.kafka-sink.kafka.bootstrap.servers = localhost:9092
tail-memory-kafka.sinks.kafka-sink.kafka.topic = spark-test
tail-memory-kafka.sinks.kafka-sink.kafka.flumeBatchSize = 5
tail-memory-kafka.sinks.kafka-sink.kafka.producer.acks = 1


tail-memory-kafka.sources.tail-source.channels = memory-channel
tail-memory-kafka.sinks.kafka-sink.channel = memory-channel