生成环境 经常使用 taildir : 支持断点续传

配置文件 flume-taildir-logger.conf：

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = TAILDIR
a1.sources.r1.positionFile = /opt/module/flume/tail_dir.json
a1.sources.r1.filegroups = f1 f2
a1.sources.r1.filegroups.f1 = /opt/module/flume/files/file1.txt
a1.sources.r1.filegroups.f2 = /opt/module/flume/files/file2.txt


# Describe the sink
a1.sinks.k1.type = logger

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

运行命令:

bin/flume-ng  agent - c conf/ -f job/flume-taildir-logger.conf  -n a1 -Dflume.root.logger=INFO,console

写入数据:

[root@hadoop202 files]# echo hello >> file1.txt
[root@hadoop202 files]# echo hello >> file2.txt

运行结果:注意查看位置记录文件:tail_dir.json文件中记录的位置信息.