devbox
喵喵爱编程
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

MapReduce--13--学生成绩（增强版）--需求2_com.ghgj.mr.exercise.pojo.coursescore;

作者：喵喵爱编程 | 2024-06-22 23:23:49

赞

踩

com.ghgj.mr.exercise.pojo.coursescore;

题目描述

关于对于学生成绩相关的练习题，之前是一个入门级别的需求，现在对这些需求进行增强，首先看数据的改变：

  1. computer,huangxiaoming,85,86,41,75,93,42,85
  2. computer,xuzheng,54,52,86,91,42
  3. computer,huangbo,85,42,96,38
  4. english,zhaobenshan,54,52,86,91,42,85,75
  5. english,liuyifei,85,41,75,21,85,96,14
  6. algorithm,liuyifei,75,85,62,48,54,96,15
  7. computer,huangjiaju,85,75,86,85,85
  8. english,liuyifei,76,95,86,74,68,74,48
  9. english,huangdatou,48,58,67,86,15,33,85
  10. algorithm,huanglei,76,95,86,74,68,74,48
  11. algorithm,huangjiaju,85,75,86,85,85,74,86
  12. computer,huangdatou,48,58,67,86,15,33,85
  13. english,zhouqi,85,86,41,75,93,42,85,75,55,47,22
  14. english,huangbo,85,42,96,38,55,47,22
  15. algorithm,liutao,85,75,85,99,66
  16. computer,huangzitao,85,86,41,75,93,42,85
  17. math,wangbaoqiang,85,86,41,75,93,42,85
  18. computer,liujialing,85,41,75,21,85,96,14,74,86
  19. computer,liuyifei,75,85,62,48,54,96,15
  20. computer,liutao,85,75,85,99,66,88,75,91
  21. computer,huanglei,76,95,86,74,68,74,48
  22. english,liujialing,75,85,62,48,54,96,15
  23. math,huanglei,76,95,86,74,68,74,48
  24. math,huangjiaju,85,75,86,85,85,74,86
  25. math,liutao,48,58,67,86,15,33,85
  26. english,huanglei,85,75,85,99,66,88,75,91
  27. math,xuzheng,54,52,86,91,42,85,75
  28. math,huangxiaoming,85,75,85,99,66,88,75,91
  29. math,liujialing,85,86,41,75,93,42,85,75
  30. english,huangxiaoming,85,86,41,75,93,42,85
  31. algorithm,huangdatou,48,58,67,86,15,33,85
  32. algorithm,huangzitao,85,86,41,75,93,42,85,75

一、数据解释

数据字段个数不固定：
第一个是课程名称，总共四个课程，computer，math，english，algorithm，
第二个是学生姓名，后面是每次考试的分数

 

二、统计需求：

1、统计每门课程的参考人数和课程平均分

2、统计每门课程参考学生的平均分，并且按课程存入不同的结果文件，要求一门课程一个结果文件，并且按平均分从高到低排序，分数保留一位小数

3、求出每门课程参考学生成绩最高的学生的信息：课程，姓名和平均分

 

三、解题思路

mapper阶段的输出：

key： CourseScore

value： NullWritable

reducer阶段的输出：

key：CourseScore

value：NullWritable

 

四、代码实现

  1. package com.ghgj.mazh.mapreduce.exercise.coursescore3;
  2.  
  3. import org.apache.hadoop.conf.Configuration;
  4. import org.apache.hadoop.fs.FileSystem;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.hadoop.hive.ql.metadata.Partition;
  7. import org.apache.hadoop.io.LongWritable;
  8. import org.apache.hadoop.io.NullWritable;
  9. import org.apache.hadoop.io.Text;
  10. import org.apache.hadoop.mapreduce.Job;
  11. import org.apache.hadoop.mapreduce.Mapper;
  12. import org.apache.hadoop.mapreduce.Partitioner;
  13. import org.apache.hadoop.mapreduce.Reducer;
  14. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  15. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  16.  
  17. import java.io.IOException;
  18. import java.util.ArrayList;
  19. import java.util.Collections;
  20. import java.util.List;
  21.  
  22. public class CourseScoreMR_Pro_02 {
  23.  
  24.     public static void main(String[] args) throws Exception {
  25.         /**
  26.          * 一些参数的初始化
  27.          */
  28.         String inputPath = "D:\\bigdata\\coursescore2\\input";
  29.         String outputPath = "D:\\bigdata\\coursescore2\\output2";
  30.  
  31.         /**
  32.          * 初始化一个Job对象
  33.          */
  34.         Configuration conf = new Configuration();
  35.         Job job = Job.getInstance(conf);
  36.  
  37.         /**
  38.          * 设置jar包所在路径
  39.          */
  40.         job.setJarByClass(CourseScoreMR_Pro_02.class);
  41.  
  42.         /**
  43.          * 指定mapper类和reducer类 等各种其他业务逻辑组件
  44.          */
  45.         job.setMapperClass(Mapper_CS.class);
  46.         job.setReducerClass(Reducer.class);
  47.         // 指定maptask的输出类型
  48.         job.setMapOutputKeyClass(CourseScore.class);
  49.         job.setMapOutputValueClass(NullWritable.class);
  50.         // 指定reducetask的输出类型
  51.         job.setOutputKeyClass(CourseScore.class);
  52.         job.setOutputValueClass(NullWritable.class);
  53.  
  54.         /**
  55.          * 设置reduceTask数量和分区器
  56.          */
  57.         job.setNumReduceTasks(4);
  58.         job.setPartitionerClass(MyPartitioner.class);
  59.  
  60.         /**
  61.          * 指定该mapreduce程序数据的输入和输出路径
  62.          */
  63.         Path input = new Path(inputPath);
  64.         Path output = new Path(outputPath);
  65.         FileSystem fs = FileSystem.get(conf);
  66.         if (fs.exists(output)) {
  67.             fs.delete(output, true);
  68.         }
  69.         FileInputFormat.setInputPaths(job, input);
  70.         FileOutputFormat.setOutputPath(job, output);
  71.  
  72.         /**
  73.          * 最后提交任务
  74.          */
  75.         boolean waitForCompletion = job.waitForCompletion(true);
  76.         System.exit(waitForCompletion ? 0 : 1);
  77.     }
  78.  
  79.     /**
  80.      * Mapper组件：
  81.      * <p>
  82.      * 输入的key:
  83.      * 输入的value:
  84.      * <p>
  85.      * 输出的key:
  86.      * 输入的value:
  87.      */
  88.     private static class Mapper_CS extends Mapper<LongWritable, Text, CourseScore, NullWritable> {
  89.  
  90.         CourseScore keyOut = new CourseScore();
  91.  
  92.         @Override
  93.         protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
  94.  
  95.             String[] splits = value.toString().split(",");
  96.             String course = splits[0];
  97.             String name = splits[1];
  98.  
  99.             int sum = 0;
  100.             int num = 0;
  101.             for(int i=2; i<splits.length; i++){
  102.                 sum += Integer.valueOf(splits[i]);
  103.                 num ++;
  104.             }
  105.             double avgScore = Math.round(sum * 1D / num * 10) / 10D;
  106.  
  107.             keyOut.setCourse(course);
  108.             keyOut.setName(name);
  109.             keyOut.setScore(avgScore);
  110.  
  111.             context.write(keyOut, NullWritable.get());
  112.         }
  113.     }
  114.  
  115.     /**
  116.      * 自定义分区组件
  117.      */
  118.     public static class MyPartitioner extends Partitioner<CourseScore, NullWritable>{
  119.  
  120.         @Override
  121.         public int getPartition(CourseScore courseScore, NullWritable nullWritable, int numPartitions) {
  122.  
  123.             String course = courseScore.getCourse();
  124.             if(course.equals("computer")){
  125.                 return 0;
  126.             }else if(course.equals("english")){
  127.                 return 1;
  128.             }else if(course.equals("algorithm")){
  129.                 return 2;
  130.             }else{
  131.                 return 3;
  132.             }
  133.         }
  134.     }
  135.  
  136. }

其中 CourseScore类的实现：

  1. package com.ghgj.mazh.mapreduce.exercise.coursescore3;
  2.  
  3. import org.apache.hadoop.io.WritableComparable;
  4.  
  5. import java.io.DataInput;
  6. import java.io.DataOutput;
  7. import java.io.IOException;
  8.  
  9. public class CourseScore implements WritableComparable<CourseScore> {
  10.  
  11.     private String course;
  12.     private String name;
  13.     private double score;
  14.  
  15.     public CourseScore(String course, String name, double score) {
  16.         super();
  17.         this.course = course;
  18.         this.name = name;
  19.         this.score = score;
  20.     }
  21.  
  22.     public CourseScore() {
  23.     }
  24.  
  25.     public String getCourse() {
  26.         return course;
  27.     }
  28.  
  29.     public void setCourse(String course) {
  30.         this.course = course;
  31.     }
  32.  
  33.     public String getName() {
  34.         return name;
  35.     }
  36.  
  37.     public void setName(String name) {
  38.         this.name = name;
  39.     }
  40.  
  41.     public double getScore() {
  42.         return score;
  43.     }
  44.  
  45.     public void setScore(double score) {
  46.         this.score = score;
  47.     }
  48.  
  49.     @Override
  50.     public void write(DataOutput out) throws IOException {
  51.         // TODO Auto-generated method stub
  52.         out.writeUTF(course);
  53.         out.writeUTF(name);
  54.         out.writeDouble(score);
  55.     }
  56.  
  57.     @Override
  58.     public void readFields(DataInput in) throws IOException {
  59.         // TODO Auto-generated method stub
  60.         this.course = in.readUTF();
  61.         this.name = in.readUTF();
  62.         this.score = in.readDouble();
  63.     }
  64.  
  65.     /**
  66.      * 排序规则
  67.      * compareTo方法既充当排序用，用充当分组规则
  68.      */
  69.     @Override
  70.     public int compareTo(CourseScore cs) {
  71.  
  72.         int compareTo = this.course.compareTo(cs.getCourse());
  73.  
  74.         if (compareTo == 0) {
  75.             double diff = cs.getScore() - this.score;
  76.             if (diff >
  77.                     0) {
  78.                 return 1;
  79.             } else if (diff < 0) {
  80.                 return -1;
  81.             } else {
  82.                 return 0;
  83.             }
  84.         } else {
  85.             return compareTo;
  86.         }
  87.     }
  88.  
  89.  
  90.     @Override
  91.     public String toString() {
  92.         return course + "\t" + name + "\t" + score;
  93.     }
  94. }

 

五、执行结果

 

 

声明：本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：【wpsshop博客】
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号