Flume伪分布式配置

Posted 2022-10-27 studynotessl

配置Flume

tar -zxvf apache-flume-1.8.0-bin.tar.gz

mkdir /opt/flume/
mv apache-flume-1.8.0-bin.tar.gz /opt/flume/flume1.8.0/

配置环境变量
vim /etc/profile

export FLUME_HOME=/opt/flume/flume1.8.0
export FLUME_CONF_DIR=$FLUME_HOME/conf
export PATH=.:$JAVA_HOME/bin:$SCALA_HOME/bin:$SPARK_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin:$KAFKA_HOME/bin:$SQOOP_HOME/bin:$FLUME_HOME/bin:$PATH

source /etc/profile

修改flume-env.sh

cd /opt/flume/flume1.8.0/conf/

mv flume-env.sh.template flume-env.sh
vim flume-env.sh

修改Hbase的hbase-env.sh（未出现异常可以忽略）

防止测试会报找不到主类的异常需要配置hbase的hbase-env.sh文件
cd /opt/hbase/hbase1.2/conf
vim hbase-env.sh

注释掉 HBASE_CLASSPATH

export JAVA_CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

查看Flume配置版本

cd /opt/flume/flume1.8.0/bin

./flume-ng version

测试Flume（avro source）

修改avro.conf

vim /opt/flume/flume1.8.0/conf/avro.conf

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.channels = c1
a1.sources.r1.bind = 0.0.0.0
a1.sources.r1.port = 4141
 
# Describe the sink
a1.sinks.k1.type = logger
 
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
 
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

启动flume agent a1

cd /opt/flume/flume1.8.0/bin

#启动日志控制台
flume-ng agent -c . -f /opt/flume/flume1.8.0/conf/avro.conf -n a1 -Dflume.root.logger=INFO,console

向指定文件写入日志

echo "hello world" > /root/log.00

启动avro-client

cd /opt/flume/flume1.8.0/bin

flume-ng avro-client --host localhost -p 4141 -F /root/log.00

在启动的服务控制台查看结果