zoukankan      html  css  js  c++  java
  • centos7安装flume

    解压安装包

    tar -zvxf apache-flume-1.6.0-bin.tar.gz

    添加配置文件

    #定义三大组件的名称
    ag1.sources = source1
    ag1.sinks = sink1
    ag1.channels = channel1
    
    # 配置source组件
    ag1.sources.source1.type = spooldir
    ag1.sources.source1.spoolDir = /root/log/
    ag1.sources.source1.fileSuffix=.FINISHED
    ag1.sources.source1.deserializer.maxLineLength=5120
    
    # 配置sink组件
    ag1.sinks.sink1.type = hdfs
    ag1.sinks.sink1.hdfs.path =hdfs://hdp-01:9000/access_log/%y-%m-%d/%H-%M
    ag1.sinks.sink1.hdfs.filePrefix = app_log
    ag1.sinks.sink1.hdfs.fileSuffix = .log
    ag1.sinks.sink1.hdfs.batchSize= 100
    ag1.sinks.sink1.hdfs.fileType = DataStream
    ag1.sinks.sink1.hdfs.writeFormat =Text
    
    ## roll:滚动切换:控制写文件的切换规则
    ## 按文件体积(字节)来切 
    ag1.sinks.sink1.hdfs.rollSize = 512000
    ## 按event条数切      
    ag1.sinks.sink1.hdfs.rollCount = 1000000
     ## 按时间间隔切换文件
    ag1.sinks.sink1.hdfs.rollInterval = 60
    
    ## 控制生成目录的规则
    ag1.sinks.sink1.hdfs.round = true
    ag1.sinks.sink1.hdfs.roundValue = 10
    ag1.sinks.sink1.hdfs.roundUnit = minute
    
    ag1.sinks.sink1.hdfs.useLocalTimeStamp = true
    
    # channel组件配置
    ag1.channels.channel1.type = memory
    ## event条数
    ag1.channels.channel1.capacity = 500000
     ##flume事务控制所需要的缓存容量600条event  
    ag1.channels.channel1.transactionCapacity = 600
    
    # 绑定source、channel和sink之间的连接
    ag1.sources.source1.channels = channel1
    ag1.sinks.sink1.channel = channel1

    启动flume

    # -c conf:指定自带配置文件-f conf/tail-hdfs.conf:自定义配置文件  -n ag1:agent名称
    bin/flume-ng agent -c conf -f conf/tail-hdfs.conf -n ag1
  • 相关阅读:
    笔记-归并排序
    Repeated Substring Pattern
    Assign Cookies
    Number of Boomerangs
    Paint Fence
    Path Sum III
    Valid Word Square
    Sum of Two Integers
    Find All Numbers Disappeared in an Array
    First Unique Character in a String
  • 原文地址:https://www.cnblogs.com/mufeng07/p/12728519.html
Copyright © 2011-2022 走看看