用wget+awk处理腾讯股票js数据

Posted 2020-08-17

    因深交所原始dbf数据源停止更新，所以需要寻找临时深交所数据，并延迟15分钟展示。

现在需要获取深圳股票数据的代码：000002，000423，000810，000999，399001，399107，399108

一、获取腾讯股票js数据

参考连接：http://hunkz.blog.51cto.com/6157447/1869450

如：

wget http://qt.gtimg.cn/q=sz000002  -O szhq_js/000002_20161104080001.txt

cat szhq_js/000002_20161104080001.txt
v_sz000002="51~β  >000002~24.87~24.90~24.77~519229~226426~292802~24.87~7945~24.86~1880~24.85~3300~24.84~562~24.83~1208~24.88~2260~24.89~1561~24.90~1353~24.91~1177~24.92~403~15:00:03/24.87/12452/S/30968148/15151|14:57:00/24.87/195/S/485070/15059|14:56:57/24.87/47/S/116918/15055|14:56:54/24.88/234/B/582007/15051|14:56:51/24.88/232/B/576987/15048|14:56:48/24.88/492/B/1224164/15045~20161103150130~-0.03~-0.12~25.19~24.71~24.87/506777/1264439662~519229~129541~0.53~14.06~~25.19~24.71~1.93~2414.41~2745.45~2.73~27.39~22.41~0.65";

二、通过awk处理下载后的js数据

/bin/awk -f sz000002 szhq_js/000002_20161104080001.txt > szhq_js/sz000002_20161104080001.txt

cat  sz000002
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"万科A\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"}

cat szhq_js/sz000002_20161104080001.txt
,{id:"sz000002",code:"000002",name:"万科A",price:"24.87",lastCP:"24.90",diff:"-0.03",percent:"-0.12",hp:"25.19",lp:"24.71",op:"24.77",bpl:"24.87",spl:"24.88",ta:"519229",tm:"129541",hqtime:"2016-11-03 15:01:30"}

三、每分钟将15分钟前的复制出来使用

假设现在时间是201611040815

/bin/cp szhq_js/sz000002_20161104080001.txt  000002.txt

000002.txt就是我们需要的文件，放置线上。

四，下面是我线上的 主要脚本

1，获取股票就是数据，并做处理

# cat szhq_wget.sh 
#!/bin/bash
DIR=/usr/local/scripts/szhq
Datetime=`date  +"%Y%m%d%H%M%S"`
#Delaytime=`date -d -15min +%Y%m%d%H%M` 
for i in `cat ${DIR}/szcode.txt`
do
wget http://qt.gtimg.cn/q=sz${i}  -O ${DIR}/szhq_js/${i}_${Datetime}.txt >/dev/null 2>&1
#/bin/cp  ${DIR}/szhq_js/${i}_${Delaytime}*.txt ${DIR}/szhq_js/${i}.txt
/bin/awk -f ${DIR}/sz${i} ${DIR}/szhq_js/${i}_${Datetime}.txt > ${DIR}/szhq_js/sz${i}_${Datetime}.txt 
done

2，对线上hs.txt文件处理

# cat szhq.sh
#!/bin/bash
#Hunk
#2016-11-01
LOCAL_DIR=/usr/local/scripts/szhq
DIR=/usr/local/scripts/szhq
Delaytime=`date -d -15min +%Y%m%d%H%M` 
for i in `cat ${DIR}/szcode.txt`
do
#/bin/cp ${DIR}/szhq_js/sz${i}_${Delaytime}*.txt  /tmp/${i}.txt
/bin/cp ${DIR}/szhq_js/sz${i}_${Delaytime}*.txt  /${LOCAL_DIR}/${i}.txt
done
/bin/cp /usr/local/tomcat/webapps/TodayInvestment/WEB-INF/hs.txt ${LOCAL_DIR}
#cd ${LOCAL_DIR}
/bin/awk -f ${LOCAL_DIR}/sz.txt ${LOCAL_DIR}/hs.txt  > ${LOCAL_DIR}/now/hs.txt
/bin/chown tomcat.tomcat ${LOCAL_DIR}/now/hs.txt
/bin/chmod -x  ${LOCAL_DIR}/now/hs.txt
#/bin/cp -ar ${LOCAL_DIR}/now/hs.txt /tmp
/bin/cp -ar ${LOCAL_DIR}/now/hs.txt  /usr/local/tomcat/webapps/TodayInvestment/WEB-INF/

3，awk的指定配置文件

# cat sz000002 
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"万科A\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz000423 
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"东阿阿胶\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz000810 
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"创维数字\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz000999 
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"华润三九\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz399001
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证成指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz399107 
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证A指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz399108
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证B指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}

# cat sz.txt 
BEGIN {FS=","}
{if(NR==1){print "["}
if($2 ~ "sz399001"){cmd ="cat /usr/local/scripts/szhq/399001.txt";cmd|getline;print $0}
if($2 ~ "sz399107"){cmd ="cat /usr/local/scripts/szhq/399107.txt";cmd|getline;print $0}
if($2 ~ "sz399108"){cmd ="cat /usr/local/scripts/szhq/399108.txt";cmd|getline;print $0}
if($2 ~ "sz000002"){cmd ="cat /usr/local/scripts/szhq/000002.txt";cmd|getline;print $0}
if($2 ~ "sz000999"){cmd ="cat /usr/local/scripts/szhq/000999.txt";cmd|getline;print $0}
if($2 ~ "sz000810"){cmd ="cat /usr/local/scripts/szhq/000810.txt";cmd|getline;print $0}
if($2 ~ "sz000423"){cmd ="cat /usr/local/scripts/szhq/000423.txt";cmd|getline;print $0}
#if($2 !~ "sz399001"&&$2 !~ "sz399107"&&$2 !~ "sz399108"&&$2 !~ "sz000002" &&$2 !~ "sz000810"&&$2 !~ "sz000423" ){print $0}
if( NR != 1 && $2 !~ "sz399001"&&$2 !~ "sz399107"&&$2 !~ "sz399108"&&$2 !~ "sz000002"&&$2 !~ "sz000999" &&$2 !~ "sz000810"&&$2 !~ "sz000423" ){print $0}
}

# cat hs.txt 
[
{id:"sh000001",code:"000001",name:"上证指数",price:"3125.317",lastCP:"3128.936",diff:"-3.619",percent:"-0.116",hp:"3141.334",lp:"3119.535",op:"3126.349",bpl:"0.000",spl:"0.000",ta:"197683197",tm:"23904325.728",hqtime:"2016-11-04 15:01:14"}
,{id:"sz000002",code:"000002",name:"万科A",price:"24.59",lastCP:"24.87",diff:"-0.28",percent:"-1.13",hp:"25.00",lp:"24.46",op:"24.84",bpl:"24.59",spl:"24.60",ta:"582099",tm:"143475",hqtime:"2016-11-04 15:01:36"}
,{id:"sz200002",code:"200002",name:"万科B",price:"12.410",lastCP:"12.200",diff:"+0.210",percent:"+1.721",hp:"12.440",lp:"12.250",op:"12.290",bpl:"12.410",spl:"12.420",ta:"83971",tm:"10401.116",hqtime:"2014-06-03 15:33:17"}
,{id:"sz000999",code:"000999",name:"华润三九",price:"27.08",lastCP:"27.48",diff:"-0.40",percent:"-1.46",hp:"27.66",lp:"27.06",op:"27.50",bpl:"27.08",spl:"27.09",ta:"45135",tm:"12314",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000810",code:"000810",name:"创维数字",price:"16.91",lastCP:"17.00",diff:"-0.09",percent:"-0.53",hp:"17.15",lp:"16.86",op:"17.01",bpl:"16.91",spl:"16.92",ta:"57331",tm:"9736",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000423",code:"000423",name:"东阿阿胶",price:"58.20",lastCP:"57.14",diff:"1.06",percent:"1.86",hp:"59.47",lp:"57.50",op:"57.50",bpl:"58.19",spl:"58.20",ta:"104522",tm:"61229",hqtime:"2016-11-04 15:01:36"}
,{id:"sh600062",code:"600062",name:"双鹤药业",price:"23.580",lastCP:"23.900",diff:"-0.320",percent:"-1.339",hp:"23.970",lp:"23.400",op:"23.900",bpl:"23.560",spl:"23.580",ta:"47197",tm:"11177.127",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600055",code:"600055",name:"万东医疗",price:"19.520",lastCP:"19.280",diff:"+0.240",percent:"+1.245",hp:"19.540",lp:"19.200",op:"19.260",bpl:"19.510",spl:"19.520",ta:"36881",tm:"7153.988",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600893",code:"600893",name:"航空动力",price:"35.000",lastCP:"35.000",diff:"0.000",percent:"0.000",hp:"0.000",lp:"0.000",op:"0.000",bpl:"0.000",spl:"0.000",ta:"0",tm:"0.000",hqtime:"2016-11-04 15:34:02"}
,

# cat szcode.txt
000002
000423
000810
000999
399001
399107
399108

5，最终目标文件

# cat now/hs.txt 
[
{id:"sh000001",code:"000001",name:"上证指数",price:"3125.317",lastCP:"3128.936",diff:"-3.619",percent:"-0.116",hp:"3141.334",lp:"3119.535",op:"3126.349",bpl:"0.000",spl:"0.000",ta:"197683197",tm:"23904325.728",hqtime:"2016-11-04 15:01:14"}
,{id:"sz000002",code:"000002",name:"万科A",price:"24.59",lastCP:"24.87",diff:"-0.28",percent:"-1.13",hp:"25.00",lp:"24.46",op:"24.84",bpl:"24.59",spl:"24.60",ta:"582099",tm:"143475",hqtime:"2016-11-04 15:01:36"}
,{id:"sz200002",code:"200002",name:"万科B",price:"12.410",lastCP:"12.200",diff:"+0.210",percent:"+1.721",hp:"12.440",lp:"12.250",op:"12.290",bpl:"12.410",spl:"12.420",ta:"83971",tm:"10401.116",hqtime:"2014-06-03 15:33:17"}
,{id:"sz000999",code:"000999",name:"华润三九",price:"27.08",lastCP:"27.48",diff:"-0.40",percent:"-1.46",hp:"27.66",lp:"27.06",op:"27.50",bpl:"27.08",spl:"27.09",ta:"45135",tm:"12314",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000810",code:"000810",name:"创维数字",price:"16.91",lastCP:"17.00",diff:"-0.09",percent:"-0.53",hp:"17.15",lp:"16.86",op:"17.01",bpl:"16.91",spl:"16.92",ta:"57331",tm:"9736",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000423",code:"000423",name:"东阿阿胶",price:"58.20",lastCP:"57.14",diff:"1.06",percent:"1.86",hp:"59.47",lp:"57.50",op:"57.50",bpl:"58.19",spl:"58.20",ta:"104522",tm:"61229",hqtime:"2016-11-04 15:01:36"}
,{id:"sh600062",code:"600062",name:"双鹤药业",price:"23.580",lastCP:"23.900",diff:"-0.320",percent:"-1.339",hp:"23.970",lp:"23.400",op:"23.900",bpl:"23.560",spl:"23.580",ta:"47197",tm:"11177.127",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600055",code:"600055",name:"万东医疗",price:"19.520",lastCP:"19.280",diff:"+0.240",percent:"+1.245",hp:"19.540",lp:"19.200",op:"19.260",bpl:"19.510",spl:"19.520",ta:"36881",tm:"7153.988",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600893",code:"600893",name:"航空动力",price:"35.000",lastCP:"35.000",diff:"0.000",percent:"0.000",hp:"0.000",lp:"0.000",op:"0.000",bpl:"0.000",spl:"0.000",ta:"0",tm:"0.000",hqtime:"2016-11-04 15:34:02"}
,

6，linux 任务计划

# crontab -l
##szhq 
*/1 09-17 * * 1-5 /bin/bash /usr/local/scripts/szhq/szhq.sh > /dev/null 2>&1
*/1 08-17 * * 1-5 /bin/bash /usr/local/scripts/szhq/szhq_wget.sh >  /dev/null 2>&1

五，小结

重要的命令：

awk中可以通过getline读取执行linux命令返回的结果，并赋值给$0

if($2 ~ "sz399001"){cmd ="cat /usr/local/scripts/szhq/399001.txt";cmd|getline;print $0}