Announcement Announcement Module
Collapse
No announcement yet.
Remote Hadoop Configuration Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Remote Hadoop Configuration

    I'm developing a Webapp (that includes Spring-data-hadoop) on a windows Box.
    I installed Hadoop on a remote Ubuntu instance (in Amazon Ec2 cloud).
    How Can I set the connection to Hadoop from my local Windows box?

    I tried the following configuration, but I keep getting Connection refused exception (note that ${elasticIp} is the Amazon instance IP).

    <hdp:configuration >
    fs.default.name=hdfs://${elasticIp}:54310
    hadoop.tmp.dir=/tmp
    </hdp:configuration>

    <hdp:job-runner id="myjob-runner" job-ref="myjob" run-at-startup="true" />

    <hdp:job id="myjob" input-path="/tmp/input/" output-path="/tmp/output/"
    mapper="org.apache.hadoop.examples.WordCount.Token izerMapper"
    reducer="org.apache.hadoop.examples.WordCount.IntS umReducer" />


    Stacktrace:

    2012-11-11 10:44:37,442 ERROR - Context initialization failed [localhost-startStop-1] (org.springframework.web.context.ContextLoader:227 )
    org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob-runner': Invocation of init method failed; nested exception is org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
    at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
    at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
    at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
    at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:192)
    at org.springframework.beans.factory.support.DefaultL istableBeanFactory.preInstantiateSingletons(Defaul tListableBeanFactory.java:585)
    at org.springframework.context.support.AbstractApplic ationContext.finishBeanFactoryInitialization(Abstr actApplicationContext.java:895)
    at org.springframework.context.support.AbstractApplic ationContext.refresh(AbstractApplicationContext.ja va:425)
    at org.springframework.web.context.ContextLoader.crea teWebApplicationContext(ContextLoader.java:282)
    at org.springframework.web.context.ContextLoader.init WebApplicationContext(ContextLoader.java:204)
    at org.springframework.web.context.ContextLoaderListe ner.contextInitialized(ContextLoaderListener.java: 47)
    at org.apache.catalina.core.StandardContext.listenerS tart(StandardContext.java:4791)
    at org.apache.catalina.core.StandardContext.startInte rnal(StandardContext.java:5285)
    at org.apache.catalina.util.LifecycleBase.start(Lifec ycleBase.java:150)
    at org.apache.catalina.core.ContainerBase.addChildInt ernal(ContainerBase.java:901)
    at org.apache.catalina.core.ContainerBase.addChild(Co ntainerBase.java:877)
    at org.apache.catalina.core.StandardHost.addChild(Sta ndardHost.java:618)
    at org.apache.catalina.startup.HostConfig.deployDescr iptor(HostConfig.java:650)
    at org.apache.catalina.startup.HostConfig$DeployDescr iptor.run(HostConfig.java:1582)
    at java.util.concurrent.Executors$RunnableAdapter.cal l(Executors.java:441)
    at java.util.concurrent.FutureTask$Sync.innerRun(Futu reTask.java:303)
    at java.util.concurrent.FutureTask.run(FutureTask.jav a:138)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run Task(ThreadPoolExecutor.java:886)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:908)
    at java.lang.Thread.run(Thread.java:662)
    Caused by: org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
    at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
    at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
    at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
    at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:196)
    at org.springframework.data.hadoop.mapreduce.JobExecu tor.findJobs(JobExecutor.java:91)
    at org.springframework.data.hadoop.mapreduce.JobExecu tor.executeJobs(JobExecutor.java:64)
    at org.springframework.data.hadoop.mapreduce.JobRunne r.call(JobRunner.java:52)
    at org.springframework.data.hadoop.mapreduce.JobRunne r.afterPropertiesSet(JobRunner.java:44)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
    ... 26 more
    Caused by: java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
    at org.apache.hadoop.ipc.Client.wrapException(Client. java:1099)
    at org.apache.hadoop.ipc.Client.call(Client.java:1075 )
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java: 225)
    at $Proxy59.getProtocolVersion(Unknown Source)
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:396)
    at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:379)
    at org.apache.hadoop.hdfs.DFSClient.createRPCNamenode (DFSClient.java:119)
    at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:238)
    at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:203)
    at org.apache.hadoop.hdfs.DistributedFileSystem.initi alize(DistributedFileSystem.java:89)
    at org.apache.hadoop.fs.FileSystem.createFileSystem(F ileSystem.java:1386)
    at org.apache.hadoop.fs.FileSystem.access$200(FileSys tem.java:66)
    at org.apache.hadoop.fs.FileSystem$Cache.get(FileSyst em.java:1404)
    at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:254)
    at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:123)
    at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:238)
    at org.apache.hadoop.fs.Path.getFileSystem(Path.java: 187)
    at org.apache.hadoop.mapreduce.lib.input.FileInputFor mat.addInputPath(FileInputFormat.java:372)
    at org.springframework.data.hadoop.mapreduce.JobFacto ryBean.afterPropertiesSet(JobFactoryBean.java:208)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
    at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
    ... 38 more
    Caused by: java.net.ConnectException: Connection refused: no further information
    at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
    at sun.nio.ch.SocketChannelImpl.finishConnect(SocketC hannelImpl.java:567)
    at org.apache.hadoop.net.SocketIOWithTimeout.connect( SocketIOWithTimeout.java:206)
    at org.apache.hadoop.net.NetUtils.connect(NetUtils.ja va:489)
    at org.apache.hadoop.ipc.Client$Connection.setupConne ction(Client.java:434)
    at org.apache.hadoop.ipc.Client$Connection.setupIOstr eams(Client.java:560)
    at org.apache.hadoop.ipc.Client$Connection.access$200 0(Client.java:184)
    at org.apache.hadoop.ipc.Client.getConnection(Client. java:1206)
    at org.apache.hadoop.ipc.Client.call(Client.java:1050 )
    ... 57 more
Working...
X