CODE

Posted 2020-12-03 lerio

#!/bin/bash
db_database=inforcenter
jdbc_inforcenter=jdbc:sqlserver://10.0.92.149;username=sa;password=123;database=$db_database
db_table=operatelog
cur_dtime=$(date -d "-1 day" +"%Y-%m-%d %H:%M:%S")
top_table=top_table
hdfs_uri=hdfs://10.0.92.222

#时间转化为时间戳
cur_dtimestamp=$cur_dtime

#检查数据库是否存在
hadoop fs -test -e /apps/hive/warehouse/$db_database
if [ $? -eq 0 ] ;then
	echo ‘Database Exist‘
else
	echo ‘Database Not Exist‘
	echo ‘Create Database‘
	
	#创建inforcenter数据库
	hive -e "create database "${db_database}
fi

#检查表是否存在
hadoop fs -test -e /apps/hive/warehouse/$db_database/$db_table
if [ $? -eq 0 ] ;then
	echo ‘Table Exist‘
else
	echo ‘Table Not Exist‘
	echo ‘Create Table‘
	#创建operatelog表
	sqoop create-hive-table --connect $jdbc_inforcenter --table $db_table --hive-database $jdbc_inforcenter --fields-terminated-by ‘01‘
fi

#将昨天的数据导入到hive中
sqoop import --connect $jdbc_inforcenter -table $db_table -hive-database $db_database -hive-table $db_table -hive-import -m -1 --incremental append --check-column CREATETIME --last-value $cur_dtimestamp --hive-drop-import-delims --split-by createtime

#创建分析表
hadoop fs -test -e /apps/hive/warehouse/$db_database/$top_table
if [ $? -eq 0 ] ;then
	echo ‘Table Exist‘
else
	echo ‘Table Not Exist‘
	echo ‘Create Table‘
	hive -e "create table "${top_table}"top_table (datainfo string,count int,optype string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ‘01‘"
fi

#清空历史数据
hive -e "truncate table "${top_table}

#统计ObjType数据
hive -e "insert into "${top_table}" select objtype,count(1) ct,‘objtype‘ from "${db_table}" where objtype!=‘null‘ group by objtype order by ct desc limit 10"
#统计objId数据
hive -e "insert into "${top_table}" select objid,count(1) ct,‘objid‘ from "${db_table}" where objid!=‘null‘ group by objid order by ct desc limit 10"
#operate数据
hive -e "insert into "${top_table}" select operate,count(1) ct,‘operate‘ from "${db_table}" where operate!=‘null‘ group by operate order by ct desc limit 10"
#operate id数据
hive -e "insert into "${top_table}" select concat(objid,‘&‘,operate),count(1) ct,‘objid&operate‘ from "${db_table}" where objid!=‘null‘ group by objid,operate order by ct desc limit 10"
#ip数据
hive -e "insert into "${top_table}" select ip,count(1) ct,‘ip‘ from "${db_table}" where ip!=‘null‘ group by ip order by ct desc limit 10"

#清空sqlserver表数据
sqoop eval --connect $jdbc_inforcenter --query "truncate table "${top_table}

#导出到sqlserver中
sqoop export --connect $jdbc_inforcenter --table $top_table --export-dir $hdfs_uri/apps/hive/warehouse/$db_database/$top_table