最全wordcount

Posted 2022-12-10 醉卧千山下，诗酒趁年华。

World Count

一，hadoop API

1,在HDFS上创建/wordcount/input目录 ，并把word.txt文件上传到该目录下。

hadoop fs -mkdir -p /wordcount/input
hadoop fs -put /root/word.txt /wordcount/input

2,进入$HADOOP_HOME/share/hadoop/mapreduce/目录下，查看文件
cd $HADOOP_HOME/share/hadoop/mapreduce/

3,直接使用hadoop-mapreduce-examples-2.7.7.jar 包进行wordcount计算
hadoop jar hadoop-mapreduce-examples-2.7.7.jar wordcount \\
/wordcount/input/word.txt /wordcount/output

4,查看结果数据

hadoop fs -cat /wordcount/output/part-r-00000

二，spark scala

1,首先进入命令行输入spark-shell
spark-shell
2,单词统计,注意路径是hadoop上的
val rdd1 = sc.textFile("/wordcount/input/word.txt")
val rdd2 = rdd1.flatMap(line => line.split(" "))
val rdd3 = rdd2.map(word => (word,1))
val rdd4 = rdd3.reduceByKey(_ + _)
rdd4.collect

三，word count MR 程序

1 Driver 代码

package com.hdfs.mr;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;


public class WordDriver 
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException 

        String[] args1 = new String[2];
        args1[0]= "d:/hello.txt";
        args1[1]="d:/output10";
        Configuration conf = new Configuration();

        //1, 获取job对象
        Job job = Job.getInstance(conf);

        //2, 设置JAR存储位置
        job.setJarByClass(WordDriver.class);

        //3, 关联Map和Reducer类
        job.setMapperClass(WordMapper.class);
        job.setReducerClass(WordReducer.class);

        //4, 设置Maper阶段输出数据的Key和Value类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        //5, 设置最终输出阶段的Key和Value类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //6, 设置输入和输出路径
        FileInputFormat.setInputPaths(job,new Path(args1[0]));
        FileOutputFormat.setOutputPath(job,new Path(args1[1]));

        //7, 提交JOB
        job.waitForCompletion(true);
    

2 Mapper 代码

package com.hdfs.mr;



import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordMapper extends Mapper<LongWritable, Text,Text, IntWritable> 

    Text k = new Text();
    IntWritable v = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
        String line = value.toString();
        String[] words = line.split(" ");


        for (String word:words) 
            k.set(word);
            context.write(k,v);
        
    

3 Reducer代码

package com.hdfs.mr;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.Iterator;

public class WordReducer extends Reducer <Text, IntWritable ,Text, IntWritable>

    IntWritable v = new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException 
        int sum = 0;

        Iterator it = values.iterator();

        while (it.hasNext())
            sum += Integer.parseInt(it.next().toString());
        
        v.set(sum);

        context.write(key,v);