1

trying to run a Reduce Side join on 3 datasets. Unfortunalyy, i keep getting an ArrayIndex exception. I have tried to handle it with a try & Catch, but to no avail. Can someone please recommend a solution?

package Joins;

import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Reducer;

public class JoinReducer extends Reducer<Text, Text, Text, Text> 
{
Map<String, String> divStkJoin = new HashMap<String, String>();
Map<String, String> divStkMetaJoin = new HashMap<String, String>();
Map<String, String> stockData = new HashMap<String, String>();
Map<String, String> metaData = new HashMap<String, String>();
Map<String, String> divData = new HashMap<String, String>();
Text k = new Text();
Text v = new Text();

public void setup(Context context)
{   
    metaData.clear();
    divData.clear();
    stockData.clear();
    divStkJoin.clear();
    divStkMetaJoin.clear();
}

public void reduce(Text keys, Iterable<Text> values, Context context)
{

    Iterator it = values.iterator();

    while(it.hasNext()){
    String [] keyLine = keys.toString().split(","); 
    String valueLine = values.toString();

    try {

        if(keyLine[4].equals("_s"))
        {
            String keyLineStock = keyLine[0] + "," + keyLine[1] +"," + keyLine[2] + "," +   keyLine[3];
            stockData.put(keyLineStock, valueLine);

        }
        else if(keyLine[4].equals("_d"))
        {
            String keyLineDiv = keyLine[0] + "," + keyLine[1] +"," + keyLine[2] + "," + keyLine[3];
             divData.put(keyLineDiv, valueLine);
        }

        else if (keyLine[1].equals("_m"))
        {
            String keyLineMeta = keyLine[0];
            metaData.put(keyLineMeta, valueLine);
        }
        else
            return;

    } catch (ArrayIndexOutOfBoundsException e){return;}
    }   



        //JOINS

        for(Entry<String, String> entryStock: stockData.entrySet())
            for(Entry<String, String> entryDiv: divData.entrySet())
                {
                    if(entryStock.getKey().equals(entryDiv.getKey()))
                    {
                        divStkJoin.put(entryStock.getKey(), entryStock.getValue()+ ","+ entryDiv.getValue());
                    }
                }
        for(Entry<String, String> entrydivStkJoin: divStkJoin.entrySet())
        {
            String [] entrydivStkJoinKeyArr = entrydivStkJoin.getKey().toString().split(",");
            for(Entry<String, String> meta: metaData.entrySet())    
            {
                String [] metaArr = meta.getKey().split(",");

                if(metaArr[0].equals(entrydivStkJoinKeyArr[1]))
                {
                    divStkMetaJoin.put(entrydivStkJoin.toString(), meta.getValue());
                }
            }
        }
    }   



public void cleanup(Context context) throws IOException, InterruptedException
{

        String keyJ;
        String valJ;
    for(Map.Entry<String, String> entry : divStkMetaJoin.entrySet())
    {
        keyJ=entry.getKey();
        valJ=entry.getValue();
        Text k = new Text(keyJ);
        Text v = new Text(valJ);

        context.write(k, v);
    }
}

}

8
  • which is the statement that produces the index out of bounds?
    – gkrls
    Sep 14, 2014 at 17:49
  • It is in the reduce method.
    – Don E
    Sep 14, 2014 at 17:50
  • 1
    Looks like keyLine doesn't always have 5 elements. You can print it and debug it yourself. Sep 14, 2014 at 17:54
  • 1
    You need to check for keyLine length before accessing an index if you suspect it may have fewer elements. Sep 14, 2014 at 18:00
  • 1
    Thanks BobTheBuilder. So perhaps something like "keyLine.length==5 && keyLine[4].equals("_s")". I'd run it it first, but it is a Mapreduce job in one node and takes 40 minutes, so i have to make sure.
    – Don E
    Sep 14, 2014 at 18:06

1 Answer 1

1

I think the error comes from this line:

if(keyLine[4].equals("_s")),

My solucion would be ckecking if keyLine is null or if keyLine < 4:

 if(ss == null || ss.length()<4){
     return;
 }

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.