Flume NG + Hadoop Log Collector Setting :: 2013/09/10 11:37

[Flume Agent]


vi flume.conf

#########################################################
#### Configure for Flume Agent (Gateway Log Agent)
#########################################################

agentA.sources = gwSource
agentA.channels = gwChannel
agentA.sinks = gwSink

##########################################
#### Configure for Source
##########################################

# For each one of the sources, the type is defined
agentA.sources.gwSource.type = exec
agentA.sources.gwSource.command = tail -Fs 180 /home/yusbha/nginx/logs/access.log
agentA.sources.gwSource.restart = true
agentA.sources.gwSource.restartThrottle=1000
agentA.sources.gwSource.interceptors = i1
agentA.sources.gwSource.interceptors.i1.type = timestamp
agentA.sources.gwSource.channels = gwChannel


##########################################
#### Configure for Channel
##########################################

# Each channel's type is defined.
agentA.channels.gwChannel.type = file
agentA.channels.gwChannel.dataDirs = /home/yusbha/flume/file-channel/data/01
agentA.channels.gwChannel.checkpointDir = /home/yusbha/flume/file-channel/checkpoint/01
agentA.channels.gwChannel.maxFileSize=524288000
agentA.channels.gwChannel.checkpointInterval=10000
agentA.channels.gwChannel.transactionCapacity=1000

##########################################
#### Configure for Sink
##########################################
# Each sink's type must be defined
agentA.sinks.gwSink.type = avro
agentA.sinks.gwSink.hostname = 172.27.106.48
agentA.sinks.gwSink.port = 35853
agentA.sinks.gwSink.channel = gwChannel

Flume Agent 실행 Command

bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name agentA -Dflume.root.logger=INFO,console

[Flume Collector]

vi flume.conf

#########################################################
#### Configure for Flume Agent (Gateway Log Collector)
#########################################################

collector.sources = collectorSource
collector.channels = collectorChannel
collector.sinks = HDFS

##########################################
#### Configure for Source : 172.27.106.48
##########################################

# For each one of the sources, the type is defined.
collector.sources.collectorSource.type = avro
collector.sources.collectorSource.bind = 0.0.0.0
collector.sources.collectorSource.port = 35853
collector.sources.collectorSource.channels = collectorChannel

##########################################
#### Configure for Channel
##########################################
collector.channels.collectorChannel.type = memory

#collector.channels.collectorChannel.type = file
#collector.channels.collectorChannel.dataDirs = /home/hadoop/flume/file-channel/data/01
#collector.channels.collectorChannel.checkpointDir = /home/hadoop/flume/file-channel/checkpoint/01
#collector.channels.collectorChannel.transactionCapacity = 1000
#collector.channels.collectorChannel.checkpointInterval = 30000
#collector.channels.collectorChannel.maxFileSize = 2146435071
#collector.channels.collectorChannel.minimumRequiredSpace = 524288000
#collector.channels.collectorChannel.keep-alive = 5
#collector.channels.collectorChannel.write-timeout = 10
#collector.channels.collectorChannel.checkpoint-timeout = 600
#collector.channels.collectorChannel.capacity = 500000

##########################################
#### Configure for Sink
##########################################
# Each sink's type must be defined
collector.sinks.HDFS.type = hdfs
collector.sinks.HDFS.hdfs.path = hdfs://name.odp.kt.com/logs
collector.sinks.HDFS.hdfs.filePrefix = %Y%m%d%H%M%S
collector.sinks.HDFS.hdfs.fileType = DataStream
collector.sinks.HDFS.hdfs.fileSuffix = .log
collector.sinks.HDFS.hdfs.inUseSuffix = .work

collector.sinks.HDFS.hdfs.maxOpenFiles = 200
collector.sinks.HDFS.hdfs.rollSize = 0
collector.sinks.HDFS.hdfs.rollInterval = 60
collector.sinks.HDFS.hdfs.rollCount = 0
collector.sinks.HDFS.hdfs.rollTimerPoolSize = 1
collector.sinks.HDFS.hdfs.batchSize = 100
collector.sinks.HDFS.hdfs.threadsPoolSize = 1
collector.sinks.HDFS.hdfs.callTimeout = 60000

collector.sinks.HDFS.hdfs.writeFormat = TEXT
collector.sinks.HDFS.serializer = text
collector.sinks.HDFS.serializer.appendNewline = true
collector.sinks.HDFS.channel = collectorChannel

Flume Agent 실행 Command

bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name collector -Dflume.root.logger=INFO,console

2013/09/10 11:37 2013/09/10 11:37
Trackback Address :: http://www.yongbi.net/trackback/580
[로그인][오픈아이디란?]
오픈아이디로만 댓글을 남길 수 있습니다
Name
Password
Homepage

Secret
< PREV |  1  |  ...  210  |  211  |  212  |  213  |  214  |  215  |  216  |  217  |  218  |  ...  566  |  NEXT >