股票数据自动入库hive到oracle

Posted 2022-02-07 燃烧的岁月_

------import_stock_d.py-----------------------------------

#!/usr/bin/python

import tushare as ts
import os
import re
stocklistpath = '/home/cloudera/data/list/stocklist.txt'
savepath='/home/cloudera/data/data/'
openstock = open(stocklistpath,'r+')
for i in openstock:
    r = r"S[ZH]\\d\\d\\d\\d\\d\\d"
    stocklist = re.findall(r,i)


for i in stocklist:
    stocknum = i[2:8]
    df = ts.get_hist_data(stocknum)
    df.to_csv(savepath + i +'.txt')
    print i


for path,d,filelist in os.walk('/home/cloudera/data/data/'):
    for filename in filelist:
        filepath = os.path.join(path,filename)
        print filepath
        file = open(filepath,'r+')
        file.seek(0,0)
        filename1 = filename[0:8]+','
        print filename1
        for line in file.readlines():
           
            print file.writelines(filename1 + line)

        file.close()

--------------------------------StockRun.sh------------------------------------------------------

python /home/cloudera/python/import_stock_d.py


hadoop fs -put /home/cloudera/data/data /stock


hive -e "LOAD DATA INPATH '/stock/data/*' OVERWRITE INTO TABLE import_stock_d";


hive -e "insert overwrite table import_stock_d select * from import_stock_d where turnover is not null"


sqoop  export --table import_stock_d  -connect jdbc:oracle:thin:@192.168.1.10:1521:orcl  --username stock --password stock     --export-dir '/user/hive/warehouse/import_stock_d/*'  --input-fields-terminated-by ',' --input-lines-terminated-by '\\n' --columns 'code,T_DATE,OPEN,HIGH,CLOSE,LOW,VOLUME,PRICE_CHANGE,P_CHANGE,MA5,MA10,MA20,V_MA5,V_MA10,V_MA20,TURNOVER'