hadoop 天气案例

Posted Andy

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了hadoop 天气案例相关的知识,希望对你有一定的参考价值。

对下面一组气温数据进行处理,得到每个月份最高的两个气温值

2018-12-12 14:30 25c
2018-12-12 15:30 26c
2017-12-12 12:30 36c
2019-01-01 14:30 22c
2018-05-05 15:30 26c
2018-05-26 15:30 37c
2018-05-06 15:30 36c
2018-07-05 15:30 36c
2018-07-05 12:30 40c
2017-12-15 12:30 16c

 

输出格式如下:

2019-1 22
2018-12 26
2018-12 25
2018-7 40
2018-7 36
2018-5 37
2018-5 36
2017-12 36
2017-12 16

 

public class App {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration(true);
        conf.set("fs.defaultFS","hdfs://hadoop01:9000");
     //windows下面运行添加一下两个配置 conf.
set("mapreduce.app-submission.cross-platform","true"); conf.set("mapreduce.framework.name","local"); Job job = Job.getInstance(conf); //设置jobName job.setJobName("myJob"); job.setJarByClass(App.class); //配置map //mapper类 job.setMapperClass(MyMapperClass.class); //输出的key类型 job.setMapOutputKeyClass(TQ.class); //输出的value类型 job.setMapOutputValueClass(IntWritable.class); //将输出的(K,V)=>(K,V,P) //job.setPartitionerClass(MyPartitioner.class); //数据在内存spill(溢写)之前先排序,注:继承WritableComparator job.setSortComparatorClass(MySortComparator.class); //配置reduce //根据需求确定分组的维度,继承自WritableComparator job.setGroupingComparatorClass(MyGrouping.class); //如map阶段根据年、月、温度三个维度排序,而reduce只根据年、月两个维度 job.setReducerClass(MyReduce.class); Path input=new Path("/input/weather.txt"); Path out=new Path("/output/weather"); if(out.getFileSystem(conf).exists(out)){ out.getFileSystem(conf).delete(out,true); } //数据来源 HDFS路径 FileInputFormat.addInputPath(job,input); //计算结果的输出目录 FileOutputFormat.setOutputPath(job,out); //job.setNumReduceTasks(2); job.waitForCompletion(true); } }

 

public class TQ implements WritableComparable<TQ> {

    private int year;

    public int getYear() {
        return year;
    }

    public void setYear(int year) {
        this.year = year;
    }

    public int getMonth() {
        return month;
    }

    public void setMonth(int month) {
        this.month = month;
    }

    public int getDay() {
        return day;
    }

    public void setDay(int day) {
        this.day = day;
    }

    public int getTemp() {
        return temp;
    }

    public void setTemp(int temp) {
        this.temp = temp;
    }

    private int month;
    private int day;
    /**
        温度
     */
    private int temp;

    @Override
    public int compareTo(TQ other) {

        int c1= Integer.compare(this.getYear(),other.getYear());
        if(c1==0){
           return Integer.compare(this.getMonth(),other.getMonth());
        }
        return c1;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeInt(this.year);
        out.writeInt(this.month);
        out.writeInt(this.day);
        out.writeInt(this.temp);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        this.year=in.readInt();
        this.month=in.readInt();
        this.day=in.readInt();
        this.temp=in.readInt();
    }
}

 

 

 

/**
 * 根据年-月对map输出进行分组
 */
public class MyGrouping extends WritableComparator {
    public MyGrouping(){
        super(TQ.class,true);
    }
    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        TQ tq1 = (TQ) a;
        TQ tq2 = (TQ) b;
        if (tq1.getYear() == tq2.getYear() && tq1.getMonth() == tq2.getMonth()) {
            return 0;
        }
        return 1;
    }
}
public class MyMapperClass extends Mapper<LongWritable,Text,TQ, IntWritable> {
    TQ tq=new TQ();
    IntWritable outVal=new IntWritable();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[]splits= value.toString().split(" ");
        String[]date=splits[0].split("-");
        tq.setYear(Integer.parseInt(date[0]));
        tq.setMonth(Integer.parseInt(date[1]));
        tq.setDay(Integer.parseInt(date[2]));

        tq.setTemp(Integer.parseInt(splits[2].replace("c","")));
        outVal.set(tq.getTemp());
        context.write(tq,outVal);

    }
}
public class MyReduce extends Reducer<TQ, IntWritable, Text,IntWritable> {
    Text txtKey=new Text();

    @Override
    protected void reduce(TQ key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        Iterator<IntWritable> iterator = values.iterator();
        int flag=0;
        while (iterator.hasNext()) {
            if (flag == 2) {
                break;
            }
            txtKey.set(String.format("%s-%s",key.getYear(),key.getMonth()));

            IntWritable next = iterator.next();
            context.write(txtKey,next);
            flag++;
        }
    }
}
/**
    数据在内存spill(溢写)之前先排序,根据年月温度
 */
public class MySortComparator extends WritableComparator {
    public MySortComparator(){
        super(TQ.class,true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        TQ tq1=(TQ)a;
        TQ tq2=(TQ)b;

        int c1= Integer.compare(tq1.getYear(),tq2.getYear());
            if(c1==0){
                int c2=Integer.compare(tq1.getMonth(),tq2.getMonth());
                if (c2 == 0) {
                    return -Integer.compare(tq1.getTemp(),tq2.getTemp());
                }
                return -c2;
            }
        return -c1;
    }
}

 

以上是关于hadoop 天气案例的主要内容,如果未能解决你的问题,请参考以下文章

Kotlin实现《第一行代码》案例“酷欧天气”

Scrapy 案例一:通过 API 抓取天气预报数据

Scrapy 案例一:通过 API 抓取天气预报数据

案例篇:Python天气播报系统,这才是未雨绸缪

基于jdk调用天气服务案例及问题

axios 查询天气的案例