hadoop之MapReduce的案例（多表关联）

Posted 2022-03-23 月疯

 

 order_detail.txt

 item_id    item_type
sp001    type001
sp002    type002
sp003    type002

 iteminfo.txt

item_id    item_type
sp001    type001
sp002    type002
sp003    type002

代码部分：

package squencefile;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

//多表关联
public class ReducerJoin 
    public static class MyMapper extends Mapper<LongWritable,Text,Text,Text>
        //map处理逻辑
        //1、判断是哪个表
        //2、针对不同的表输出不同的数据
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
            //判断是哪个表文件
            String fileName=((FileSplit)context.getInputSplit()).getPath().getName();
            //切分每行数据
            String line = value.toString();
            String[] lineArr = line.split("\\t");

            if("order_detail.txt".equals(fileName))
                //订单明细<item_id,"1:order_id:amout>"
                context.write(new Text(lineArr[1]),new Text("1:"+lineArr[0]+":"+lineArr[2]));

            else if("iteminfo.txt".equals(fileName))
                //商品表<item_id,"2:item_type">
                context.write(new Text(lineArr[0]),new Text("2:"+lineArr[1]));

            
        
    
    public static class MyReducer extends Reducer<Text,Text,Text,Text>
        //1、将相同商品id的订单信息明细和商品信息进行拆分，拆分后存到响应的订单明细表和商品明细表中
        //2、将订单明细列表和商品列表进行嵌套遍历
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException 
            //0、定义订单明细列表和商品信息列表
            List<String> orderDetailList=new ArrayList<>();
            List<String> itemInfoList=new ArrayList<>();
            for (Text tempVal:values)
                String tempValStr = tempVal.toString();
                String[] tempValArr=tempValStr.split(":");
                System.out.print(tempValStr);
                if ("1".equals((tempValArr[0])))
                    orderDetailList.add(tempValStr.substring(2));
                else
                    itemInfoList.add(tempValArr[1]);
                
            
            for(String itemInfo:itemInfoList)
                for(String orderDetail:orderDetailList)
                    context.write(key,new Text(itemInfo+":"+orderDetail));
                
            
        
    
    public static void main (String[] args ) throws IOException, ClassNotFoundException, InterruptedException 
        //创建一个job，也就是一个运行环境
        Configuration conf=new Configuration();
        //集群运行
//        conf.set("fs.defaultFS","hdfs://hadoop:8088");
        //本地运行
        Job job=Job.getInstance(conf,"reduce-join");
        //程序入口（打jar包）
        job.setJarByClass(ReducerJoin.class);

        //需要输入俩个文件：输入文件
        FileInputFormat.addInputPath(job,new Path("F:\\\\filnk_package\\\\hadoop-2.10.1\\\\data\\\\test5\\\\order_detail.txt"));
        FileInputFormat.addInputPath(job,new Path("F:\\\\filnk_package\\\\hadoop-2.10.1\\\\data\\\\test5\\\\iteminfo.txt"));
        //编写mapper处理逻辑
        job.setMapperClass(ReducerJoin.MyMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        //shuffle流程

        //编写reduce处理逻辑
        job.setReducerClass(ReducerJoin.MyReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        //输出文件
        FileOutputFormat.setOutputPath(job,new Path("F:\\\\filnk_package\\\\hadoop-2.10.1\\\\data\\\\test5\\\\out"));

        //运行job，需要放到Yarn上运行
        boolean result =job.waitForCompletion(true);
        System.out.print(result?1:0);

    

数据倾斜如何处理：

注意：reduceJoin会产生数据倾斜，比如俩个task1和task2，task1处理的任务比task2处理的比较多，这样会导致性能很低，如何使得俩个task处理任务比较均衡。
方案：map输出的key添加随机数后缀，将生成的新的key分发到不同的reduce task上
sp001_8888
sp002_999
商品表中map输出需要扩容10000条，输出到各个reduce task上
reduce：将俩个表输出数据进行合并，将后缀删除