require:
/** * before: * file A1.csv {1,2,3,4,5} * file A2.csv {2,3,9,10,11} * file B1.csv {5,12,13,14,15} * file B2.csv {16,14,15,4,9,20,30} * A1.csv A2.csv A3.csv A4.csv cant not repeat * * after: * file A1.csv {1,4} * file A2.csv {2,3,10,11} * file B1.csv {12,13} * file B2.csv {16,9,20,30} */
[email protected]:~/csvrepeat$ ls A1.csv A2.csv B1.csv B2.csv
CSVUtilVersion2.java
import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.LineIterator; import org.springframework.util.StringUtils; import java.io.*; import java.lang.reflect.Array; import java.util.*; /** * before: * file A1.csv {1,2,3,4,5} * file A2.csv {2,3,9,10,11} * file B1.csv {5,12,13,14,15} * file B2.csv {16,14,15,4,9,20,30} * A1.csv A2.csv A3.csv A4.csv cant not repeat * * after: * file A1.csv {1,4} * file A2.csv {2,3,10,11} * file B1.csv {12,13} * file B2.csv {16,9,20,30} */ @Slf4j public class CSVUtilVersion2 { private static final String CSV_PATH = "/home/tangxin/csvrepeat/"; private static final boolean CREATE_SWITCH = true; /** * read single column data list * @param path * @return */ public static List<String> ids(String path) { List<String> result = new ArrayList<>(); File csv = new File(path); // CSV文件路径 LineIterator it = null; try { it = FileUtils.lineIterator(csv); while (it.hasNext()) { String line = it.nextLine(); if (line.trim().contains("ID")) { continue; } String[] arr = line.split(","); String ID = arr[0]; ID = ID.replaceAll("\"", "").trim(); if (!StringUtils.isEmpty(ID)) { result.add(ID); } } } catch (Exception e) { log.error("读取ID csv文件失败:{}", e.getMessage()); } finally { LineIterator.closeQuietly(it); } return result; } /** * from src delete oth * @param src * @param oth * @return */ public static List removeAll(List src, List oth) { LinkedList result = new LinkedList(src); HashSet othHash = new HashSet(oth); Iterator iter = result.iterator(); while (iter.hasNext()) { if (othHash.contains(iter.next())) { iter.remove(); } } return result; } /** * -Xms1g -Xmx1g -XX:PermSize=128m -XX:SurvivorRatio=2 -XX:+UseParallelGC * @param args * @throws Exception */ public static void main(String[] args) throws Exception { //∑=1+2+3+...+(n-1) group LinkedList<String> fileList = new LinkedList<>(); fileList.add("A1.csv"); fileList.add("A2.csv"); fileList.add("B1.csv"); fileList.add("B2.csv"); // fileList.add("C1.csv"); DescartesRepeat(fileList); ded(fileList); } private static void DescartesRepeat(LinkedList<String> fileList) { Set<String> repeatList = new HashSet<>(); Set<String> groupSet = new HashSet<>(); Set<String> goONList = new HashSet<>(); //A1->A2,B1,B2 for (int i = 0; i < fileList.size(); i++) { String itemI = fileList.get(i); for (int j = 0; j < fileList.size(); j++) { String itemJ = fileList.get(j); if (!itemI.equals(itemJ)) { String groupR1 = itemI + "->" + itemJ; String groupR2 = itemJ + "->" + itemI; if (groupSet.contains(groupR1) || groupSet.contains(groupR2)){ continue; } groupSet.add(groupR1); String repeatT = repeat(CSV_PATH + itemI, CSV_PATH + itemJ); if(!StringUtils.isEmpty(repeatT)){ repeatList.add(repeatT); //System.out.println(groupR1+"->"+repeatT); } } } } if (CollectionUtils.isNotEmpty(repeatList)) { // System.out.println(repeatList); for (String repeatItem : repeatList) { Iterator<String> iterator = fileList.iterator(); while (iterator.hasNext()) { String oldItem = iterator.next(); String oldS = oldItem.replace(".csv", "").replace("-new",""); String repeatS = repeatItem.replace(".csv","").replace("-new",""); if (repeatS.contains(oldS)) { iterator.remove(); goONList.add(repeatItem); } } } fileList.addAll(goONList); System.out.println(fileList); DescartesRepeat(fileList); } } public static void ded(List<String> args) { //保证指定csv列表每组都不能有重复数据 for (int i = 0; i < args.size(); i++) { // if(i>0){ // continue; // } String source = CSV_PATH + args.get(i); for (int j = 0; j < args.size(); j++) { if (i == j) { continue; } String target = CSV_PATH + args.get(j); intersection(source, target); } } } public static void intersection(String sourcePath, String targetPath) { List<String> ids1 = ids(sourcePath); List<String> ids2 = ids(targetPath); List<String> inter = (List<String>) CollectionUtils.intersection(ids1, ids2); System.out.println(sourcePath + "和" + targetPath + "的重复数据大小" + inter.size()); } public static String repeat(String source, String target){ //cdd fund xyd List<String> ids1 = ids(source); List<String> ids2 = ids(target); // System.out.println(source + "集合大小" + ids1.size()); // System.out.println(target + "集合大小" + ids2.size()); List<String> inter = (List<String>) CollectionUtils.intersection(ids1, ids2); // System.out.println("去重数据大小:" + inter.size()); if (inter != null && inter.size() > 0) { if (ids1.size() > ids2.size()) { return repeatInner(source, ids1, inter); } else if (ids2.size() > ids1.size()) { return repeatInner(target, ids2, inter); } else { return repeatInner(source, ids1, inter); } } return ""; } private static String repeatInner(String source, List<String> ids, List<String> inter) { String newPath = source.replace(".csv", "-new.csv"); List<String> ids1new = removeAll(ids, inter); createCSV(ids1new, newPath); return newPath.replace(CSV_PATH,""); } /** * 创建CSV文件 */ public static void createCSV(List<String> list, String fileName) { if(!CREATE_SWITCH){ // System.out.println("创建csv开关关闭"); return; }else{ // System.out.println("创建csv开关开启"); } // 表格头 Object[] head = {"ID"}; List<Object> headList = Arrays.asList(head); //数据 List<List<Object>> dataList = new ArrayList<>(); List<Object> rowList = null; for (int i = 0; i < list.size(); i++) { rowList = new ArrayList<>(); rowList.add(list.get(i)); dataList.add(rowList); } File csvFile; BufferedWriter csvWtriter = null; try { csvFile = new File(fileName); File parent = csvFile.getParentFile(); if (parent != null && !parent.exists()) { parent.mkdirs(); } csvFile.createNewFile(); // GB2312使正确读取分隔符"," csvWtriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), "GB2312"), 1024); // 写入文件头部 writeRow(headList, csvWtriter); // 写入文件内容 for (List<Object> row : dataList) { writeRow(row, csvWtriter); } csvWtriter.flush(); } catch (Exception e) { e.printStackTrace(); } finally { try { csvWtriter.close(); } catch (IOException e) { e.printStackTrace(); } } } /** * 写一行数据 * * @param row 数据列表 * @param csvWriter * @throws IOException */ private static void writeRow(List<Object> row, BufferedWriter csvWriter) throws IOException { for (Object data : row) { StringBuffer sb = new StringBuffer(); String rowStr = sb.append("\"").append(data).append("\",").toString(); csvWriter.write(rowStr); } csvWriter.newLine(); } }
[email protected]:~/csvrepeat$ ls A1.csv A1-new.csv A1-new-new.csv A2.csv A2-new.csv B1.csv B2.csv B2-new.csv B2-new-new.csv