Announcement Announcement Module
Collapse
No announcement yet.
Write method is calling again and again Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Write method is calling again and again

    We are trying the following scenario:-
    Reading data from a flatfile and writing data into a database by keeping the chunk level as 10. So the application will read 10 records line by line and then pass the records into the write method in a list which contains 10 items.

    Suppose we are getting an error at the write level then the spring batch will throw an item level error and continue with the next records. But we are experiencing a strange behaviour after the error here like spring batch is calling the write method by passing one by one record in the list. Thus the write method will get called 11 times and there will be duplicate data in the destination database. The following is the configuration we are using in our application:-
    <?xml version="1.0" encoding="UTF-8"?>

    <beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.springframework.org/schema/p"
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:batch="http://www.springframework.org/schema/batch"
    xsi:schemaLocation="http://www.springframework.org/schema/beans
    http://www.springframework.org/schem...ring-beans.xsd
    http://www.springframework.org/schema/context
    http://www.springframework.org/schem...ng-context.xsd
    http://www.springframework.org/schema/aop http://www.springframework.org/schem...ng-aop-2.0.xsd
    http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-2.1.xsd">

    <bean id="mydataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
    <property name="driverClassName" value="oracle.jdbc.driver.OracleDriver" />
    <property name="url" value="jdbc:oracle:thin:@//localhost:1521/orcl" />
    <property name="username" value="orcl" />
    <property name="password" value="orcl" />
    </bean>

    <bean id="mydataSource" class="org.apache.commons.dbcp.BasicDataSource"
    destroy-method="close">
    <property name="driverClassName" value="com.mysql.jdbc.Driver" />
    <property name="url" value="jdbc:mysql://localhost:3306/springbatch" />
    <property name="username" value="root" />
    <property name="password" value="" />
    </bean>
    <bean id="jobdataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
    <property name="driverClassName" value="com.mysql.jdbc.Driver" />
    <property name="url" value="jdbc:mysql://localhost:3306/jobrepository" />
    <property name="username" value="root" />
    <property name="password" value="" />
    </bean>

    <bean id="jobRepository-transactionManager" class="org.springframework.jdbc.datasource.DataSou rceTransactionManager" lazy-init="true">
    <property name="dataSource" ref="jobdataSource" />

    <bean id="testDB-transactionManager" class="org.springframework.jdbc.datasource.DataSou rceTransactionManager">
    <property name="dataSource" ref="mydataSource" />
    </bean>

    <bean id="jobRepository-transactionManager2" class="org.springframework.batch.support.transacti on.ResourcelessTransactionManager"/>



    <bean id="jobRepository" class="org.springframework.batch.core.repository.s upport.MapJobRepositoryFactoryBean">
    <property name="transactionManager" ref="jobRepository-transactionManager"></property>
    </bean>

    <bean id="jobLauncher" class="org.springframework.batch.core.launch.suppo rt.SimpleJobLauncher">
    <property name="jobRepository" ref="jobRepository"/>
    </bean>

    <bean id="careerProcessor"
    class="com.spring.batch.rollback.CareerProcessor" />

    <bean id="playerFileItemReader" class="com.spring.batch.rollback.CustomerFileReade r">
    <property name="resource" value="file:\Data\Input\player.csv" />
    <property name="lineMapper">
    <bean class="org.springframework.batch.item.file.mapping .DefaultLineMapper">
    <property name="lineTokenizer">
    <bean class="org.springframework.batch.item.file.transfo rm.DelimitedLineTokenizer">
    <property name="delimiter" value=","/>
    <property name="names" value="ID,lastName,firstName,position,debutYear,fi nalYear" />
    </bean>
    </property>
    <property name="fieldSetMapper">
    <bean class="com.spring.batch.rollback.PlayerFieldSetMap per" />
    </property>
    </bean>
    </property>
    </bean>

    <bean id="playerDBWriter" class="com.spring.batch.rollback.CustomerDBWriter" >

    <property name="itemSqlParameterSourceProvider">
    <bean name="sqlParameterSourceProvider" id="sqlParameterSourceProvider"
    class="org.springframework.batch.item.database.Bea nPropertyItemSqlParameterSourceProvider" />
    </property>
    <property name="sql" value="insert into PLAYER (PLAYER_ID, FIRST_NAME, LAST_NAME, POSITION,
    DEBUT_YEAR, FINAL_YEAR) values (:id, :lastName , :firstName , osition , :debutYear ,:finalYear)"/>
    <property name="dataSource" ref="mydataSource" />
    </bean>

    <bean id="dynamicJobParameters" class="com.spring.batch.rollback.DynamicJobParamet ers" />

    <batch:job id="file2DBJob" job-repository="jobRepository" incrementer="dynamicJobParameters">
    <batch:step id="step1">
    <batch:tasklet transaction-manager="jobRepository-transactionManager" >
    <batch:chunk reader="playerFileItemReader" processor="careerProcessor" writer="playerDBWriter"
    commit-interval="10" retry-limit="2">
    </batch:chunk>
    </batch:tasklet>
    </batch:step>
    </batch:job>
    </beans>

  • #2
    The framework has no way to identify the failed item from a call to ItemWriter.write() when there is more than one item. So it has to loop back and try again with a failed chunk.

    Your problems are connected with the transaction manager and the JobReposository - you should try to use only one transaction manager, so that there is only one transaction active in the step. The easiest way to do that is use a JDBC JobRepository in the same database as your business data.

    If you can forgo restartability and/or skippability you can use a more relaxed approach to transactions, and then the failed chunk doesn't need to retry itself.

    Comment


    • #3
      Thanks Dave!!!!

      It seems to be working fine now. I will update you later.

      Comment

      Working...
      X