Announcement Announcement Module
Collapse
No announcement yet.
Chunk Commit Problem Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Chunk Commit Problem

    Hello Guys,

    I have a problem with chunk commit. I'm using MySQL in my tests. My Job consists in a read of a thousand of registers from XML File and write to DB. Each file contais about 100.000 registers and the commit interval that i configured is 1000. The proccess of extraction and write occurs normally. Altought, i realized that the commit occurs only at the end of job/step. I have monitored the proccess through queries directly on DB and when Spring Batch commits the chunk, nothing occurs at DB, only at the final of proccess. It's very strange. I will post my application context for your analisys.

    Thanks from Brazil!

    <?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:aop="http://www.springframework.org/schema/aop"
    xmlns:tx="http://www.springframework.org/schema/tx"
    xmlns="http://www.springframework.org/schema/p"
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:util="http://www.springframework.org/schema/util"
    xmlns:batch="http://www.springframework.org/schema/batch"
    xsi:schemaLocation="
    http://www.springframework.org/schema/beans http://www.springframework.org/schem...-beans-2.0.xsd
    http://www.springframework.org/schema/tx http://www.springframework.org/schem...ing-tx-2.0.xsd
    http://www.springframework.org/schema/aop http://www.springframework.org/schem...ng-aop-2.0.xsd
    http://www.springframework.org/schema/util http://www.springframework.org/schem...g-util-2.0.xsd
    http://www.springframework.org/schema/batch http://www.springframework.org/schem...-batch-2.0.xsd
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd">

    <context:component-scan base-package="br.gov.ans.sibxml" />

    <bean id="dataSource" class="org.springframework.jdbc.datasource.DriverM anagerDataSource">
    <property name="driverClassName" value="com.mysql.jdbc.Driver" />
    <property name="url" value="jdbc:mysql://localhost/seamdb" />
    <property name="username" value="root" />
    <property name="password" value="root" />
    </bean>

    <bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSou rceTransactionManager">
    <property name="dataSource" ref="dataSource" />
    </bean>

    <tx:annotation-driven transaction-manager="transactionManager" />

    <bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate" >
    <property name="dataSource" ref="dataSource" />
    </bean>

    <bean id="jobRepository" class="org.springframework.batch.core.repository.s upport.MapJobRepositoryFactoryBean">
    <property name="transactionManager" ref="transactionManager" />
    </bean>

    <!-- <bean id="jobRepository" class="org.springframework.batch.core.repository.s upport.JobRepositoryFactoryBean" p:dataSource-ref="dataSource" p:transactionManager-ref="transactionManager" />-->

    <bean id="jobLauncher"
    class="org.springframework.batch.core.launch.suppo rt.SimpleJobLauncher">
    <property name="jobRepository" ref="jobRepository" />
    </bean>

    <batch:job id="simpleJob" restartable="true">
    <batch:listeners>
    <batch:listener ref="appJobExecutionListener" />
    </batch:listeners>
    <batch:step id="step1">
    <batch:tasklet transaction-manager="transactionManager">
    <batch:listeners>
    <batch:listener ref="itemFailureLoggerListener" />
    <batch:listener ref="itemProcessListener" />
    </batch:listeners>
    <batch:chunk reader="xmlMultipleItemReader" processor="beneficiarioProcessor" writer="beneficiarioCompositeItemWriter" commit-interval="2000" />
    </batch:tasklet>
    </batch:step>
    </batch:job>

    <bean id="xmlMultipleItemReader" class="br.gov.ans.sibxml.reader.MultiFragmentStaxE ventItemReader">
    <property name="fragmentRootElementNames">
    <list>
    <value>inclusao</value>
    <value>alteracao</value>
    <value>exclusao</value>
    </list>
    </property>
    <property name="resource" ref="inputResource" />
    <property name="unmarshaller" ref="annotatedMarshaller" />
    </bean>

    <bean id="beneficiarioCompositeItemWriter" class="org.springframework.batch.item.support.Clas sifierCompositeItemWriter">
    <property name="classifier" ref="beneficiarioClassifier" />
    </bean>

    <bean id="beneficiarioClassifier" class="br.gov.ans.sibxml.writer.BeneficiarioClassi fier">
    <property name="inclusaoBeneficiarioWriter" ref="inclusaoBeneficiarioWriter" />
    <property name="alteracaoBeneficiarioWriter" ref="alteracaoBeneficiarioWriter" />
    <property name="exclusaoBeneficiarioWriter" ref="exclusaoBeneficiarioWriter" />
    <property name="erroBeneficiarioWriter" ref="erroBeneficiarioWriter" />
    </bean>

    <bean id="annotatedMarshaller" class="org.springframework.oxm.xstream.AnnotationX StreamMarshaller">
    <property name="annotatedClasses">
    <list>
    <value>br.gov.ans.sibxml.dominio.xml.InclusaoBenef iciario</value>
    <value>br.gov.ans.sibxml.dominio.xml.AlteracaoBene ficiario</value>
    <value>br.gov.ans.sibxml.dominio.xml.ExclusaoBenef iciario</value>
    </list>
    </property>

    <property name="converters">
    <list>
    <bean class="com.thoughtworks.xstream.converters.basic.D ateConverter">
    <constructor-arg value="yyyy-MM-dd" />
    <constructor-arg value="dd/MM/yyyy" />
    </bean>
    </list>
    </property>
    </bean>

    <bean id="inputResource" scope="step" class="org.springframework.core.io.FileSystemResou rce">
    <constructor-arg value="C://Temp/#{jobParameters['nomeArquivo']}" />
    </bean>

    </beans>

  • #2
    Spring Batch commits at the end of every chunk (2000 items in the example you gave - which is rather high for most use cases). If you don't see anything before the end of the job, maybe you are launching the job with an enclosing transaction (bad idea)?

    N.B. I wouldn't recommend using MapJobRepositoryFactoryBean if you care about restartability (so your step should be marked as restartable=false).

    Comment


    • #3
      Thank you for reply. I just solved the problem which was related about isolation level configuration on MySQL DB and some transaction attributes of JUnit class.

      Comment

      Working...
      X