ini 水槽

Posted 2021-05-24

bin/flume-ng agent --conf conf --conf-file conf/flume2.conf --name a1 -Dflume.root.logger=DEBUG, console &

a1.sources=r1
a1.sinks=k1
a1.channels=c1

# Describe/configure the source
a1.sources.r1.type=avro
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=8089
#a1.sources.r1.interceptors=t1
#a1.sources.r1.interceptors.t1.type=timestamp

# Describe/configure the sink
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://master:9000/logs/%{flume.client.environment}/%{flume.client.source}/%Y-%m-%d
a1.sinks.k1.hdfs.batchSize=100
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.writeFormat=Text
a1.sinks.k1.hdfs.rollSize=102400000
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.rollInterval=86400
a1.sinks.k1.hdfs.minBlockReplicas=1
a1.sinks.k1.hdfs.idleTimeout=1800
#a1.sinks.k1.hdfs.useLocalTimeStamp=true

# Use a channel which buffers events in memory
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=1000

# Bind the source and sink to the channel
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

a1.sources=r1
a1.sinks=k1
a1.channels=c1

# Describe/configure the source
a1.sources.r1.type=avro
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=8089

# Describe/configure the sink
a1.sinks.k1.type=avro
a1.sinks.k1.hostname=192.168.11.177 
a1.sinks.k1.port=9520 
a1.sinks.k1.channel=c1

# Use a channel which buffers events in memory
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=1000

# Bind the source and sink to the channel
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

a1.sources=r1
a1.sinks=k1
a1.channels=c1

# Describe/configure the source
a1.sources.r1.type=avro
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=8089

# Describe/configure the sink
a1.sinks.k1.type=file_roll
a1.sinks.k1.sink.directory=/home/huhy/log/flume/tmp
a1.sinks.k1.sink.rollInterval=86400
a1.sinks.k1.sink.batchSize=100
a1.sinks.k1.sink.serializer=text
a1.sinks.k1.sink.serializer.appendNewline=false

# Use a channel which buffers events in memory
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=1000

# Bind the source and sink to the channel
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1