MR - MRUnit 샘플, EclEmma Java Code Coverage

Posted 01 17, 2014 19:02, Filed under: BigData/MapReduce


MRUnit 설정및 WordCount 테스트


# MRUnit 다운로드
http://mrunit.apache.org/general/downloads.html

# 참고 사이트
https://cwiki.apache.org/confluence/display/MRUNIT/Testing+Word+Count

# 필요 라이브러리
> apache-mrunit-1.0.0-hadoop1/lib/
commons-logging-1.1.1.jar
hamcrest-core-1.1.jar
junit-4.10.jar
mrunit-1.0.0-hadoop1.jar
mockito-all-1.8.5.jar

> $HADOOP_HOME/
hadoop-core-1.1.2.jar














# MRUnit - WordCount Sample


# WordCountMapper.java
package org.apache.hadoop.mr;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    private final static IntWritable one = new IntWritable(1);

    @Override
    public void map(LongWritable key, Text value, Context context) 
                              throws IOException, InterruptedException {

        StringTokenizer itr = new StringTokenizer(value.toString());

        while(itr.hasMoreTokens()) {
            context.write(new Text(itr.nextToken()), one);
        }
    }
}



# WordCountReducer.java
package org.apache.hadoop.mr;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values, Context context) 
                                         throws IOException, InterruptedException {
        int sum = 0;
        for (IntWritable value : values) {
            sum += value.get();
        }
        result.set(sum);
        context.write(key, result);
    }
}


# WordCountTest.java
package test.org.apache.hadoop.mr;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mr.WordCountMapper;
import org.apache.hadoop.mr.WordCountReducer;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

/**
 * # Word Count Test
 */
public class WordCountTest {
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;

    @Before
    public void setUp() {
        WordCountMapper mapper = new WordCountMapper();
        WordCountReducer reducer = new WordCountReducer();
        
        mapDriver = new MapDriver<LongWritable, Text, Text, IntWritable>();
        mapDriver.setMapper(mapper);
        reduceDriver = new ReduceDriver<Text, IntWritable, Text, IntWritable>();
        reduceDriver.setReducer(reducer);
        mapReduceDriver = new MapReduceDriver
          <LongWritable, Text, Text, IntWritable, Text, IntWritable>();
        mapReduceDriver.setMapper(mapper);
        mapReduceDriver.setReducer(reducer);
    }

    @Test
    public void testMapper() throws IOException {
        mapDriver.withInput(new LongWritable(1), new Text("cat cat dog rabit"));
        mapDriver.withOutput(new Text("cat"), new IntWritable(1));
        mapDriver.withOutput(new Text("cat"), new IntWritable(1));
        mapDriver.withOutput(new Text("dog"), new IntWritable(1));
        mapDriver.withOutput(new Text("rabit"), new IntWritable(1));
        mapDriver.runTest();
    }

    @Test
    public void testReducer() throws IOException {
        List<IntWritable> values = new ArrayList<IntWritable>();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("cat"), values);
        reduceDriver.withOutput(new Text("cat"), new IntWritable(2));
        reduceDriver.runTest();
    }

    @Test
    public void testMapReduce() throws IOException {
        mapReduceDriver.withInput(new LongWritable(1), new Text("cat cat dog rabit"));
        mapReduceDriver.addOutput(new Text("cat"), new IntWritable(2));
        mapReduceDriver.addOutput(new Text("dog"), new IntWritable(1));
        mapReduceDriver.addOutput(new Text("rabit"), new IntWritable(1));
        mapReduceDriver.runTest();
    }
}




# EclEmma Java Code Coverage 플러그인 설치


> TDD(Test-Driven Development) 에서 테스트된 범위를 측정.

> 플러그인 설치
   Eclipse메뉴 > Help > Eclipse Marketplace...


> 색상별 상태
   녹색 : 테스트 검증됨
   빨강색 : 테스트 필요
   노랑색 : 부분적으로 테스트됨

사용자 삽입 이미지





Code Coverage, 참고 사이트 : http://dsmoon.tistory.com/107


※ 위 내용은, 여러 자료를 참고하거나 제가 주관적으로 정리한 것입니다.
   잘못된 정보나 보완이 필요한 부분을, 댓글 또는 메일로 보내주시면 많은 도움이 되겠습니다.
01 17, 2014 19:02 01 17, 2014 19:02


Trackback URL : http://develop.sunshiny.co.kr/trackback/989

Leave a comment

« Previous : 1 : ... 53 : 54 : 55 : 56 : 57 : 58 : 59 : 60 : 61 : ... 648 : Next »

Recent Posts

  1. HDFS - Python Encoding 오류 처리
  2. HP - Vertica ROS Container 관련 오류...
  3. HDFS - Hive 실행시 System Time 오류
  4. HP - Vertica 사용자 쿼리 이력 테이블...
  5. Client에서 HDFS 환경의 데이터 처리시...

Recent Comments

  1. Right away I am ready to do my bre... 골목게임 11 17,
  2. Terrific article! That is the type... 선릉야구장 11 16,
  3. Yes! Finally someone writes about /. / 11 16,
  4. Круто, круто! Некот... карточные игры на... 11 13,
  5. 안녕하세요^^ 배그핵

Recent Trackbacks

  1. master djs bozeman master djs bozeman %M
  2. wedding dj bozeman mt wedding dj bozeman mt 17 11
  3. joes dj bozeman joes dj bozeman 17 11
  4. Mysql - mysql 설치후 Character set... 멀고 가까움이 다르기 때문 %M

Calendar

«   11 2019   »
          1 2
3 4 5 6 7 8 9
10 11 12 13 14 15 16
17 18 19 20 21 22 23
24 25 26 27 28 29 30

Bookmarks

  1. 위키피디아
  2. MysqlKorea
  3. 오라클 클럽
  4. API - Java
  5. Apache Hadoop API
  6. Apache Software Foundation
  7. HDFS 생태계 솔루션
  8. DNSBL - Spam Database Lookup
  9. Ready System
  10. Solaris Freeware
  11. Linux-Site
  12. 윈디하나의 솔라나라

Site Stats

TOTAL 2755890 HIT
TODAY 132 HIT
YESTERDAY 638 HIT