#########################################################

#### Configure for Flume Agent (Gateway Log Agent)

#########################################################

agentA.sources = gwSource

agentA.channels = gwChannel

agentA.sinks = gwSink

##########################################

#### Configure for Source

##########################################

# For each one of the sources, the type is defined

agentA.sources.gwSource.type = exec

agentA.sources.gwSource.command = tail -Fs 180 /home/yusbha/nginx/logs/access.log

agentA.sources.gwSource.restart = true

agentA.sources.gwSource.restartThrottle=1000

agentA.sources.gwSource.interceptors = i1

agentA.sources.gwSource.interceptors.i1.type = timestamp

agentA.sources.gwSource.channels = gwChannel

##########################################

#### Configure for Channel

##########################################

# Each channel's type is defined.

agentA.channels.gwChannel.type = file

agentA.channels.gwChannel.dataDirs = /home/yusbha/flume/file-channel/data/01

agentA.channels.gwChannel.checkpointDir = /home/yusbha/flume/file-channel/checkpoint/01

agentA.channels.gwChannel.maxFileSize=524288000

agentA.channels.gwChannel.checkpointInterval=10000

agentA.channels.gwChannel.transactionCapacity=1000

##########################################

#### Configure for Sink

##########################################

# Each sink's type must be defined

agentA.sinks.gwSink.type = avro

agentA.sinks.gwSink.hostname = 172.27.106.48

agentA.sinks.gwSink.port = 35853

agentA.sinks.gwSink.channel = gwChannel

bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name agentA -Dflume.root.logger=INFO,console

#########################################################

#### Configure for Flume Agent (Gateway Log Collector)

#########################################################

collector.sources = collectorSource

collector.channels = collectorChannel

collector.sinks = HDFS

##########################################

#### Configure for Source : 172.27.106.48

##########################################

# For each one of the sources, the type is defined.

collector.sources.collectorSource.type = avro

collector.sources.collectorSource.bind = 0.0.0.0

collector.sources.collectorSource.port = 35853

collector.sources.collectorSource.channels = collectorChannel

##########################################

#### Configure for Channel

##########################################

collector.channels.collectorChannel.type = memory

#collector.channels.collectorChannel.type = file

#collector.channels.collectorChannel.dataDirs = /home/hadoop/flume/file-channel/data/01

#collector.channels.collectorChannel.checkpointDir = /home/hadoop/flume/file-channel/checkpoint/01

#collector.channels.collectorChannel.transactionCapacity = 1000

#collector.channels.collectorChannel.checkpointInterval = 30000

#collector.channels.collectorChannel.maxFileSize = 2146435071

#collector.channels.collectorChannel.minimumRequiredSpace = 524288000

#collector.channels.collectorChannel.keep-alive = 5

#collector.channels.collectorChannel.write-timeout = 10

#collector.channels.collectorChannel.checkpoint-timeout = 600

#collector.channels.collectorChannel.capacity = 500000

##########################################

#### Configure for Sink

##########################################

# Each sink's type must be defined

collector.sinks.HDFS.type = hdfs

collector.sinks.HDFS.hdfs.path = hdfs://name.odp.kt.com/logs

collector.sinks.HDFS.hdfs.filePrefix = %Y%m%d%H%M%S

collector.sinks.HDFS.hdfs.fileType = DataStream

collector.sinks.HDFS.hdfs.fileSuffix = .log

collector.sinks.HDFS.hdfs.inUseSuffix = .work

collector.sinks.HDFS.hdfs.maxOpenFiles = 200

collector.sinks.HDFS.hdfs.rollSize = 0

collector.sinks.HDFS.hdfs.rollInterval = 60

collector.sinks.HDFS.hdfs.rollCount = 0

collector.sinks.HDFS.hdfs.rollTimerPoolSize = 1

collector.sinks.HDFS.hdfs.batchSize = 100

collector.sinks.HDFS.hdfs.threadsPoolSize = 1

collector.sinks.HDFS.hdfs.callTimeout = 60000

collector.sinks.HDFS.hdfs.writeFormat = TEXT

collector.sinks.HDFS.serializer = text

collector.sinks.HDFS.serializer.appendNewline = true

collector.sinks.HDFS.channel = collectorChannel

bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name collector -Dflume.root.logger=INFO,console