最新去重法

Posted 2020-12-02 liangxi

                logger.info("fetchData end, eids.length={}, eids={}", eids.length, eids);
                //复杂的去重处理,先把相同的数据统一起来,最后在取相同数据中最大的EID数据
                List<HashMap<String,Object>> list = new ArrayList<HashMap<String,Object>>();
                
                Set<String> companyCodeSet = new HashSet<String>();
                Set<String> noticeDateSet = new HashSet<String>();
                //添加COMPANYCODE
                for (long eid : eids) {
                    DataRow zbyzInfo = dao.queryZBYZInfo(eid); 
                    if(zbyzInfo!=null && zbyzInfo.size()>0) {
                        Date noticeDate = (Date) zbyzInfo.get("NOTICEDATE");//公告日期
                        String COMPANYCODE= zbyzInfo.getString("COMPANYCODE");//公司代码
                        String DESCRIPTION = zbyzInfo.getString("DESCRIPTION").toString();//内容
                        //先保存所有的数据
                        HashMap<String,Object> map = new HashMap<String,Object>();
                        map.put("eid", eid);
                        map.put("noticeDate", sdf.format(noticeDate));
                        map.put("COMPANYCODE", COMPANYCODE);
                        map.put("DESCRIPTION", DESCRIPTION);
                        companyCodeSet.add(COMPANYCODE);
                        noticeDateSet.add(sdf.format(noticeDate));
                        list.add(map);
                    }
                }
                if(companyCodeSet.size() > 0 && list.size() > 0){
                    Map<String,ArrayList<HashMap<String, Object>>> map6 = new HashMap<String,ArrayList<HashMap<String,Object>>>();
                    for (String companyCode : companyCodeSet) {
                        ArrayList<HashMap<String, Object>> list2 = new ArrayList<HashMap<String,Object>>();
                        //根据公司代码遍历所有的数据来分类
                        for (HashMap<String, Object> map : list) {
                            if(map.containsValue(companyCode)){
                                list2.add(map);
                            }
                        }
                        map6.put(companyCode, list2);
                    }
                    //处理同一公司code的数据  最复杂的地方
                    //1.遍历map
                    Map<String,ArrayList<HashMap<String, Object>>> map7 = new HashMap<String,ArrayList<HashMap<String,Object>>>();
                    for (String date : noticeDateSet) {
                        Iterator<Entry<String, ArrayList<HashMap<String, Object>>>> iterator = map6.entrySet().iterator(); 
                        while(iterator.hasNext()){
                            ArrayList<HashMap<String, Object>> list3 = new ArrayList<HashMap<String,Object>>();
                            Entry<String, ArrayList<HashMap<String, Object>>> entry = iterator.next();
                            ArrayList<HashMap<String,Object>> value = entry.getValue();
                            String dateF = "";
                            String codeF = "";
                            for (HashMap<String, Object> hashMap : value) {
                                //同一公司  日期相同的筛选
                                if(hashMap.containsValue(date)){
                                    dateF = String.valueOf(hashMap.get("noticeDate"));
                                    codeF = String.valueOf(hashMap.get("COMPANYCODE"));
                                    list3.add(hashMap);
                                }
                            }
                            if(list3 != null && list3.size() != 0){
                                map7.put(dateF+"&"+codeF, list3);
                            }
                        }
                    }
                    
                    //最后比较内容，取最大的EID
                    Iterator<Entry<String, ArrayList<HashMap<String, Object>>>> iterator = map7.entrySet().iterator();
                    List<HashMap<String,Long>> listss = new ArrayList<HashMap<String,Long>>();
                    while(iterator.hasNext()){
                        Entry<String, ArrayList<HashMap<String, Object>>> entry = iterator.next();
                        //取出相同code，相同日期的，现在要判断内容是否相同
                        ArrayList<HashMap<String, Object>> value = entry.getValue();
                        //标记map
                        HashMap<String,Long> map = new HashMap<String,Long>();
                        for (int i = 0; i < value.size(); i++) {
                            String cont = String.valueOf(value.get(i).get("DESCRIPTION"));
                            Long eid = Long.valueOf(String.valueOf(value.get(i).get("eid")));
                            if(map.containsKey(cont)){
                                Long leid = map.get(cont);
                                if(eid > leid){
                                    map.put(cont, eid);
                                }
                            }else{
                                map.put(cont, eid);
                            }
                        }
                        listss.add(map);
                    }
                    for (HashMap<String, Long> hashMap : listss) {
                        Collection<Long> values = hashMap.values();
                        if(hashMap != null && hashMap.size() > 0){
                            for (Long long1 : values) {
                                eidList.add(long1);
                            }
                        }
                    }
                }
                return eidList;