spark之从外部文件获取广播变量

Posted 2020-12-17 wangbin2188

public static Map<String,String> loadDataFromFile() {
        Map<String,String> map = new ConcurrentHashMap<>();
        try {
            File file = new File("filename.txt");
            if (file.isFile() && file.exists()) {
                InputStreamReader read = new InputStreamReader(new FileInputStream(file), "UTF-8");
                BufferedReader bufferedReader = new BufferedReader(read);
                String line = null;
                while ((line = bufferedReader.readLine()) != null) {
                    String[] split = line.split(",");
                    if (split.length==2) {
                        map.put(split[0],split[1]);
                    }
                }
                bufferedReader.close();
                read.close();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return map;
    }

//获取文件内容并广播
Map<String,String> pidMap = loadDataFromFile();
        Broadcast<Map<String,String>> broadcast = sc.broadcast(pidMap);

//在worker中使用
broadcast.getValue();