Redis文件交集并集

Posted 2020-12-03 DoubleW

package com.huawei.iop.timetask.task;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;

import com.huawei.iop.bean.UserDefinedGroup;
import com.huawei.iop.bean.WorldCupTask;
import com.huawei.iop.manager.worldCUP.userCluster.UserDefinedGroupManager;
import com.huawei.iop.manager.worldCUP.userCluster.WorldCUPManager;
import com.huawei.iop.util.ConfigPropertiesUtil;

public class AggregationTask
{
    @Autowired
    private UserDefinedGroupManager userDefinedGroupManager;

    @Autowired
    private WorldCUPManager worldCUPManager;

    @Autowired
    private RedisTemplate<String, String> redisTemplate;

    protected final static Logger logger = Logger.getLogger("runtime");

    public void aggregation()
    {

        try
        {
            logger.info("Spring Task Start, System.currentTimeMillis=" + System.currentTimeMillis());

            // 读历史记录表，取出最早的未完成任务
            UserDefinedGroup userDefinedGroup = userDefinedGroupManager.getOneUnfinished();
            if (null == userDefinedGroup)
            {
                // 没有未完成任务
                logger.info("execute once with no task to execute. System.currentTimeMillis=" + System.currentTimeMillis());
                return;
            }
            else
            {
                logger.info("execute once, System.currentTimeMillis=" + System.currentTimeMillis());
            }

            // 獲取當前時間的年月
            Date nowDate = new Date();
            SimpleDateFormat df = new SimpleDateFormat("yyyyMM");
            String dateTime = df.format(nowDate);

            String resultPathFromConf = ConfigPropertiesUtil.readValue("definedresultpath") + dateTime;// download/resultgroups
            logger.info("resultPathFromConf read from definedresultpath = " + resultPathFromConf);
            if (StringUtils.isEmpty(resultPathFromConf))
            {
                String tempPath = "/home/iop/download/definedGroupUpload/" + dateTime;
                File tmpFile = new File(tempPath);
                if (!tmpFile.exists())
                {
                    boolean b = tmpFile.mkdirs();
                    if (b)
                    {
                        logger.info("tmpFile.mkdirs():" + b);
                    }
                }
                resultPathFromConf = tempPath;
            }
            String resultPath = resultPathFromConf;
            // userHome + File.separator +
            // resultPathFromConf+File.separator +"taskresults"+ File.separator
            // + month;///opt/iop510/download/resultgroups
            String resultName = "Upload" + userDefinedGroup.getGroupID() + ".csv";
            File resultFile = new File(resultPath, resultName);// /opt/iop510/download/groupid.csv
            logger.info("resultFile=" + resultFile);
            String sourceFile = userDefinedGroup.getSourceFile() + "";
            logger.info("sourceFile:" + sourceFile);

            String[] fileNames = sourceFile.split(",");
            if (sourceFile != null && sourceFile.length() != 0)
            {
                logger.info("sourceFile OK, not null and length>0");
            }
            else
            {
                // 历史记录表没有要计算的文件
                logger.info("AggregationTask.aggregation() : no value of filed ‘sourceFile‘ from DIM_WORLDCUP_DEFINED_TASK");
                throw new Exception("AggregationTask.aggregation() : no value of filed ‘sourceFile‘ from DIM_WORLDCUP_DEFINED_TASK");
            }

            String setName1 = "tempSet1";
            String setName2 = "tempSet2";
            String setName3 = "tempSet3";
            String setName4 = "tempSet4";
            redisTemplate.boundSetOps(setName1);
            redisTemplate.boundSetOps(setName2);
            redisTemplate.boundSetOps(setName3);
            redisTemplate.boundSetOps(setName4);
            // xxx.csv,i,xxx.csv,u,xxx.csv
            Set<String> result = new HashSet<String>(1024);

            // 单个文件任务，直接读文件输出结果;多文件
            if (fileNames.length == 1)// /单文件
            {
                String firstFile = fileNames[0];
                File f = new File(resultPath, firstFile);// 文件
                if (f.isFile() && f.canRead())
                {
                    FileReader fileReader = null;// 文件流
                    BufferedReader bufferedReader = null;// 缓冲文件流
                    try
                    {
                        fileReader = new FileReader(f);
                        bufferedReader = new BufferedReader(fileReader);
                        logger.info("bufferedReader read success");
                        String oneLine = null;
                        String readLine = bufferedReader.readLine();
                        logger.info("readLine:" + readLine);
                        String[] msisdn = null;
                        while ((oneLine = readLine) != null)
                        {
                            msisdn = oneLine.split(",");
                            /* logger.info("msisdn:" + msisdn); */
                            // redisTemplate.boundSetOps(setName).add(msisdn[0]);
                            result.add(msisdn[0]);
                            readLine = bufferedReader.readLine();
                        }

                    }
                    catch (IOException e)
                    {
                        logger.error("fileReader read failed" + e.getLocalizedMessage());
                    }
                    finally
                    {
                        if (null != bufferedReader)
                        {
                            try
                            {
                                bufferedReader.close();
                            }
                            catch (IOException e)
                            {
                                logger.info("bufferedReader close failed");
                                e.printStackTrace();
                            }
                        }
                        if (null != fileReader)
                        {
                            try
                            {
                                fileReader.close();
                            }
                            catch (IOException e)
                            {
                                logger.info("fileReader close failed");
                                e.printStackTrace();
                            }
                        }

                    }
                    logger.info("Only one file to deal with, task ok");
                }
                else
                {
                    logger.error("File read failed");
                    throw new Exception(resultPathFromConf + "file read failed");
                }
            }
            else
            {
                logger.info("Multi files to deal with");
                logger.info("fileNames length" + fileNames.length);
                // 多文件 ,index对应的都是 i/u
                for (int index = 1; index < fileNames.length - 1; index = index + 2)
                {

                    // index + 2 , 13579都是符号

                    if (index == 1)
                    {// 第一次进，所有集合都是空
                        String firstFile = fileNames[index - 1];
                        File f = new File(resultPath, firstFile);// 第一个文件
                        this.importFile(f, setName1, redisTemplate);// 第一个文件导入第一个集合
                        String file = fileNames[index + 1];// 第2个文件
                        f = new File(resultPath, file);
                        this.importFile(f, setName2, redisTemplate);// 第2个文件导入第2个集合
                        if ("i".equals(fileNames[index] + ""))
                        {// 第一次做交集
                            result = redisTemplate.opsForSet().intersect(setName1, setName2);// result接收交集结果
                            logger.info("1result.size()=" + result.size());
                            logger.info("1Spring Task: first intersect.currentTimeMillis=" + System.currentTimeMillis());
                        }
                        else if ("u".equals(fileNames[index] + ""))
                        {// 第一次做并集
                            result = redisTemplate.opsForSet().union(setName1, setName2);
                            logger.info("2result.size() = " + result.size());
                            logger.info("2Spring Task: first union. currentTimeMillis=" + System.currentTimeMillis());
                        }
                        // 清空tempSet1
                        // redisTemplate.opsForSet().remove(setName1, 0,
                        // redisTemplate.boundSetOps(setName1).size() - 1);
                        redisTemplate.opsForSet().unionAndStore(setName3, setName4, setName1);
                        logger.info("3redisTemplate.boundSetOps(setName1).size()=" + redisTemplate.boundSetOps(setName1).size());
                        logger.info("3Empty setName1. System.currentTimeMillis=" + System.currentTimeMillis());

                        // 清空tempSet2
                        // redisTemplate.opsForSet().remove(setName2, 0,
                        // redisTemplate.boundSetOps(setName2).size() - 1);
                        redisTemplate.opsForSet().unionAndStore(setName3, setName4, setName2);
                        logger.info("4redisTemplate.boundSetOps(setName2).size()=" + redisTemplate.boundSetOps(setName2).size());
                        logger.info("4Spring Task: first redisTemplate.opsForSet().remove().currentTimeMillis="
                                + System.currentTimeMillis());
                    }
                    else
                    {// 不是第一次进，所有集合都不空
                     // 前一次result -> tempSet1
                        Iterator<String> it = result.iterator();
                        while (it.hasNext())
                        {
                            redisTemplate.boundSetOps(setName1).add(it.next());
                            logger.info("redisTemplate.boundSetOps success");
                        }
                        // 新文件
                        String file = fileNames[index + 1];
                        File f = new File(resultPath, file);
                        importFile(f, setName2, redisTemplate);
                        if ("i".equals(fileNames[index] + ""))
                        {// 第一次做交集
                            result = redisTemplate.opsForSet().intersect(setName1, setName2);// result接收交集结果
                            logger.info("5redisTemplate.opsForSet().intersect(setName1, setName2) .size() = " + result.size());
                            logger.info("5Spring Task: first union. currentTimeMillis=" + System.currentTimeMillis());
                        }
                        else if ("u".equals(fileNames[index] + ""))
                        {// 第一次做并集
                            result = redisTemplate.opsForSet().union(setName1, setName2);
                            logger.info("6redisTemplate.opsForSet().union(setName1, setName2) .size() = " + result.size());
                            logger.info("6Spring Task: first union. currentTimeMillis=" + System.currentTimeMillis());
                        }
                        // 清空tempSet1
                        // redisTemplate.opsForSet().remove(setName1, 0,
                        // redisTemplate.boundSetOps(setName1).size() - 1);
                        redisTemplate.opsForSet().unionAndStore(setName3, setName4, setName1);
                        logger.info("7redisTemplate.boundSetOps(setName1).size()=" + redisTemplate.boundSetOps(setName1).size());
                        logger.info("7Empty setName1. System.currentTimeMillis=" + System.currentTimeMillis());
                        // 清空tempSet2
                        // redisTemplate.opsForSet().remove(setName2, 0,
                        // redisTemplate.boundSetOps(setName2).size() - 1);
                        redisTemplate.opsForSet().unionAndStore(setName3, setName4, setName2);
                        logger.info("8redisTemplate.boundSetOps(setName2).size()=" + redisTemplate.boundSetOps(setName2).size());
                        logger.info("8Spring Task: first redisTemplate.opsForSet().remove().currentTimeMillis="
                                + System.currentTimeMillis());
                    }

                }
            }

            int resultSize = result.size();

            // 插入數據
            if (resultSize >= 1)
            {
                List<WorldCupTask> worldCupTasklist = new ArrayList<WorldCupTask>();
                for (String str : result)
                {
                    WorldCupTask worldCupTask = new WorldCupTask();
                    worldCupTask.setGroupID(userDefinedGroup.getGroupID());
                    worldCupTask.setUserID(str);
                    worldCupTasklist.add(worldCupTask);
                }
                if (null != worldCupTasklist && worldCupTasklist.size() > 0)
                {
                    try
                    {
                        worldCUPManager.addUserInfo(worldCupTasklist);
                        logger.info("Insert data success");
                    }
                    catch (Exception e)
                    {
                        logger.error("Insert data failed");
                        e.printStackTrace();
                    }
                }
            }

            // 文件写出去

            logger.info("resultSize=" + resultSize);
            String msisdn = "";
            BufferedWriter bufferedWriter = null;
            if (resultSize >= 1)
            {
                try
                {
                    bufferedWriter = new BufferedWriter(new FileWriter(resultFile));
                    Iterator<String> it = result.iterator();
                    while (it.hasNext())
                    {
                        msisdn = it.next() + "";
                        bufferedWriter.write(msisdn + "," + msisdn + "," + "0");
                        bufferedWriter.newLine();
                    }
                }
                catch (Exception e)
                {
                    logger.error("bufferedWriter write is failed");
                    e.printStackTrace();
                }
                finally
                {
                    bufferedWriter.close();
                    bufferedWriter = null;
                }
            }
            else
            {
                try
                {
                    bufferedWriter = new BufferedWriter(new FileWriter(resultFile));
                    bufferedWriter.write("" + "," + "" + "," + "");
                    bufferedWriter.newLine();
                }
                catch (Exception e)
                {
                    logger.error("bufferedWriter write is failed");
                    e.printStackTrace();
                }
                finally
                {
                    bufferedWriter.close();
                    bufferedWriter = null;
                }
            }

            userDefinedGroup.setStatus("1");// 0-executing, 1-finished
            /*
             * userDefinedGroup.setUserNum(result.size() + "");
             * logger.info("setUserNum" + result.size());
             */
            userDefinedGroup.setUserNum(resultSize + "");
            logger.info("setUserNum = " + resultSize);

            userDefinedGroup.setFileName(resultName);
            logger.info("setFileName = " + resultName);

            userDefinedGroup.setFilePath(resultPath);
            logger.info("setFilePath = " + resultPath);

            userDefinedGroup.setDownLoadPathByString("/definedGroups");

            String finishTime = null;
            try
            {

                logger.info("setDownLoadPathByString = " + resultPath);
                Calendar c = Calendar.getInstance();
                c.setTimeInMillis(System.currentTimeMillis());
                logger.info("c.setTimeInMillis");

                Date date = c.getTime();
                SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddhhmmssSSS");
                finishTime = sdf.format(date);
                userDefinedGroup.setFinishTime(finishTime);
                logger.info("setFinishTime = " + finishTime);
            }
            catch (Exception e)
            {
                logger.error("userDefinedGroup set is failed");
                e.printStackTrace();
            }

            /*
             * logger.info("redisTemplate.boundSetOps.size=[" +
             * redisTemplate.boundSetOps(setName1).size() + "," +
             * redisTemplate.boundSetOps(setName2).size() + "," +
             * redisTemplate.boundSetOps(setName3).size() + "," +
             * redisTemplate.boundSetOps(setName4).size() + "]");
             */
            logger.info("Sping Task update-start, System. finishTime=" + finishTime);
            userDefinedGroupManager.updateToFinished(userDefinedGroup);
            logger.info("Sping Task update-end, System.currentTimeMillis=" + System.currentTimeMillis());

        }
        catch (FileNotFoundException e)
        {
            logger.error("file is not found");
            e.printStackTrace();
        }
        catch (IOException e)
        {
            logger.error("IOException:" + e.getLocalizedMessage());
            e.printStackTrace();
        }
        catch (Exception e)
        {
            logger.error("Exception:" + e.getLocalizedMessage());
            e.printStackTrace();
        }
        finally
        {
            logger.info("Sping Task End, System.currentTimeMillis=" + System.currentTimeMillis());
        }
    }

    /**
     * read all records from a source file to Redis set
     * 
     * @param f
     *            source file to read to Redis set
     * @param setName
     *            Redis set
     * @throws FileNotFoundException
     */
    private void importFile(File f, String setName, RedisTemplate<String, String> redisTemplate) throws FileNotFoundException, IOException
    {
        if (f.isFile() && f.canRead())
        {
            FileReader fileReader = null;// 文件流
            // FileInputStream fileInputStream = new FileInputStream(file);
            BufferedReader bufferedReader = null;// 缓冲文件流
            try
            {
                fileReader = new FileReader(f);
                bufferedReader = new BufferedReader(fileReader);
                String oneLine = "";
                while ((oneLine = bufferedReader.readLine()) != null)
                {
                    String msisdn[] = oneLine.split(",");
                    redisTemplate.boundSetOps(setName).add(msisdn[0]);
                }
            }
            catch (Exception e)
            {
                e.printStackTrace();
            }
            finally
            {
                if (null != bufferedReader)
                {
                    try
                    {
                        bufferedReader.close();
                    }
                    catch (IOException e)
                    {
                        logger.error("bufferedReader close failed");
                    }
                }
                if (null != fileReader)
                {
                    try
                    {

                        fileReader.close();
                    }
                    catch (IOException e)
                    {
                        logger.error("fileReader close failed");
                    }
                }

            }
        }
    }

}