will not continue rolling files under this path due to under-replication
a1.sinks.k1.hdfs.minBlockReplicas=1
让程序感知不到写的文件所在块正在复制
# Name the components on this agent a1.sources = r1 a1.sinks = k1 a1.channels = c1 # Describe/configure the source a1.sources.r1.type = TAILDIR a1.sources.r1.positionFile = /weblog/flume/taildir_position.json a1.sources.r1.filegroups = f1 f2 a1.sources.r1.filegroups.f1 = /weblog/test1/example.log a1.sources.r1.filegroups.f2 = /weblog/test2/.*log.* # Describe the sink a1.sinks.k1.type = hdfs a1.sinks.k1.hdfs.path = /weblog/%y-%m-%d/%H-%M/ a1.sinks.k1.hdfs.filePrefix = itcast- a1.sinks.k1.hdfs.minBlockReplicas=1 a1.sinks.k1.hdfs.round = true a1.sinks.k1.hdfs.roundValue = 10 a1.sinks.k1.hdfs.roundUnit = minute a1.sinks.k1.hdfs.rollInterval = 0 a1.sinks.k1.hdfs.rollSize = 134217728 a1.sinks.k1.hdfs.rollCount = 0 a1.sinks.k1.hdfs.idleTimeout = 20 a1.sinks.k1.hdfs.batchSize = 1 a1.sinks.k1.hdfs.useLocalTimeStamp = true #生成的文件类型,默认是Sequencefile,可用DataStream,则为普通文本 a1.sinks.k1.hdfs.fileType = DataStream # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1
配置java环境
chmod a+x flume-env.sh
bin/flume-ng agent -c conf -f conf/taildirsource_hdfs.conf -n a1 -Dflume.root.logger=INFO,console