Announcement Announcement Module
Collapse
No announcement yet.
Issue in running job with Spring-data hadoop Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Issue in running job with Spring-data hadoop

    I have created following Mapper and Reducer using Mahout

    Code:
     package mypackage.ItemSimilarity;
        
        import java.io.IOException;
        import java.util.regex.Matcher;
        import java.util.regex.Pattern;
        
        import org.apache.hadoop.io.LongWritable;
        import org.apache.hadoop.io.Text;
        import org.apache.hadoop.mapreduce.Mapper;
        import org.apache.mahout.math.VarLongWritable;
        
        public class ItemPrefMapper extends
                Mapper<LongWritable, Text, VarLongWritable, VarLongWritable> {
        
            private static final Pattern NUMBERS = Pattern.compile("(\\d+)");
        
            @Override
            public void map(LongWritable key, Text value, Context context)
                    throws IOException, InterruptedException {
                String line = value.toString();
                Matcher m = NUMBERS.matcher(line);
                m.find();
                VarLongWritable userID = new VarLongWritable(Long.parseLong(m.group()));
                VarLongWritable itemID = new VarLongWritable();
                while (m.find()) {
                    itemID.set(Long.parseLong(m.group()));
                    context.write(userID, itemID);
                }
            }
        }
    Reduces class

    Code:
    package mypackage.ItemSimilarity;
        
        import java.io.IOException;
        
        import org.apache.hadoop.mapreduce.Reducer;
        import org.apache.mahout.math.RandomAccessSparseVector;
        import org.apache.mahout.math.VarLongWritable;
        import org.apache.mahout.math.Vector;
        import org.apache.mahout.math.VectorWritable;
        
        public class UserVectorReducer
                extends
                Reducer<VarLongWritable, VarLongWritable, VarLongWritable, VectorWritable> {
            @Override
            public void reduce(VarLongWritable userID,
                    Iterable<VarLongWritable> itemPrefs, Context context)
                    throws IOException, InterruptedException {
                Vector userVector = new RandomAccessSparseVector(Integer.MAX_VALUE, 100);
                for (VarLongWritable itemPref : itemPrefs) {
                    userVector.set((int) itemPref.get(), 1.0f);
                }
                context.write(userID, new VectorWritable(userVector));
            }
        
        }
    Spring configuration to run this

    Code:
    <job id="mahoutJob" input-path="/home/ubuntu/input/data.txt" output-path="/home/ubuntu/output"
        mapper="mypackage.ItemSimilarity.ItemPrefMapper"
        reducer="mypackage.ItemSimilarity.UserVectorReducer"
        jar-by-class="mypackage.ItemSimilarity.ItemPrefMapper"/>
            
        <job-runner id="myjob-runner" pre-action="setupScript"  job-ref="mahoutJob"
        run-at-startup="true"/>
    When I run this I got the following error. I have extended the Hadoop mapper class but spring said it is not a mapper class.

    java.lang.RuntimeException: class mypackage.ItemSimilarity.ItemPrefMapper not org.apache.hadoop.mapreduce.Mapper
    at org.apache.hadoop.conf.Configuration.setClass(Conf iguration.java:931)
    at org.apache.hadoop.mapreduce.Job.setMapperClass(Job .java:175)
    at org.springframework.data.hadoop.mapreduce.JobFacto ryBean.afterPropertiesSet(JobFactoryBean.java:153)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1571)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1509)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:521)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:458)


    Can anyone please tell me the reason of getting this error?
Working...
X