1

Below are my code snippet for using WritableComparator, but it does not work

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class MovieComparator extends WritableComparator{

    public MovieComparator(){
        super(Movie.class);
    }
    @Override
    public int compare(WritableComparable o,WritableComparable o2){
        System.out.println("in compare");
        Movie m = (Movie)o;
        Movie m2 = (Movie)o2;
        System.out.println(m.compareTo(m2));
        return m.movieId.compareTo(m2.movieId);
    }

}


public class Movie implements WritableComparable {

    Text movieId;
    Text movieTitle;

    public Movie(Text movieId, Text movieTitle) {

        this.movieId = movieId;
        this.movieTitle = movieTitle;
    }

    public Movie(){

    }

    public String getMovieId() {
        return movieId.toString();
    }

    public void setMovieId(String movieId) {
        this.movieId = new Text(movieId);
    }

    public String getMovieTitle() {
        return movieTitle.toString();
    }

    public void setMovieTitle(String movieTitle) {
        this.movieTitle = new Text(movieTitle);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        //movieId = in.read;
        movieId.readFields(in);
        movieTitle.readFields(in);
    }

    @Override
    public void write(DataOutput out) throws IOException {
        //out.writeUTF(movieId);
        //out.writeUTF(movieTitle);
        movieId.write(out);
        movieTitle.write(out);

    }

    @Override
    public int compareTo(Movie o) {

    //  System.out.println("in compareTo");
        int res=movieTitle.compareTo(o.movieTitle);
        return res;
    }



    @Override
    public int hashCode(){
        return movieId.hashCode();
    }

    @Override
    public boolean equals(Object o){
        Movie m=(Movie)o; 
        return movieId.equals(m.movieId);
    }

    @Override
    public String toString(){
        return movieTitle.toString();
    }

}

In driver class I am setting the comparator by below line
job.setSortComparatorClass(MovieComparator.class);
Can any body tell me where I am wrong in this at it gives exception below
14/09/08 14:17:03 WARN mapred.LocalJobRunner: job_local_0001
java.io.IOException: Spill failed
    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1029)
    at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:691)
    at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
    at com.impetus.MovieMapper.map(MovieMapper.java:44)
    at com.impetus.MovieMapper.map(MovieMapper.java:1)
    at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:144)
    at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
    at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:212)

1 Answer 1

1

I found the issue that Instead of using super(Movie.class), I will have to use super(Movie.class,true). As by sending true, WritableComparator will instantiate the object other wise it will pass null in compare method

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.