Announcement Announcement Module
Collapse
No announcement yet.
8 Thread max with JDBC JobRepository Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • 8 Thread max with JDBC JobRepository

    Hello,

    I have a probleme with my multi threaded batch.
    I made a small hello world batch to test it on a very small project.
    And it appear that when I use a MapJobRepositoryFactoryBean there is no issue but is I use a JDBC jobRepository I have 8 thread max

    Here is the exemple with MapJobRepositoryFactoryBean :

    Spring application context
    Code:
    <?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xmlns:util="http://www.springframework.org/schema/util"
        xmlns:context="http://www.springframework.org/schema/context"
        xmlns:task="http://www.springframework.org/schema/task"
        xmlns:tx="http://www.springframework.org/schema/tx"
        xmlns:batch="http://www.springframework.org/schema/batch"
        xsi:schemaLocation="http://www.springframework.org/schema/beans
            http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
            http://www.springframework.org/schema/context
            http://www.springframework.org/schema/context/spring-context-3.1.xsd
            http://www.springframework.org/schema/util
            http://www.springframework.org/schema/util/spring-util.xsd
            http://www.springframework.org/schema/task
            http://www.springframework.org/schema/task/spring-task-3.1.xsd
            http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.1.xsd
            http://www.springframework.org/schema/batch
            http://www.springframework.org/schema/batch/spring-batch-2.1.xsd">
        
    
        
        <bean name="testReader" class="com.priceminister.rgp.batch.reader.TestReader" />
        <bean name="testWritter" class="com.priceminister.rgp.batch.writer.TestWritter" />
        
        <bean id="jobRepository"
            class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
            <property name="transactionManager" ref="transactionManager" />
        </bean>
        
        <bean id="transactionManager"
            class="org.springframework.batch.support.transaction.ResourcelessTransactionManager" />
        
        <bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
            <property name="jobRepository" ref="jobRepository" />
        </bean>
        
             <bean id="taskExecutorStep1" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
           <property name="corePoolSize" value="100" />
           <property name="maxPoolSize" value="100" />
        </bean>
        
        <batch:job id="testJob">
            <batch:step id="step1">
                <batch:tasklet task-executor="taskExecutorStep1" throttle-limit="100">
                    <batch:chunk reader="testReader" writer="testWritter" commit-interval="1"/>
                </batch:tasklet>
            </batch:step>
        </batch:job>
    </beans>
    Reader Class
    Code:
    public class TestReader  implements ItemReader<String> {
    
        public final static Queue<String> queue = new  ConcurrentLinkedQueue<String>();
        
        TestReader() {
            for (int i = 0; i < 100; i++) {
                queue.add(String.valueOf(i)) ;
            }
        }
    
        public String read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
            return queue.poll();
        }
    }

    Writer Class
    Code:
    public class TestWritter  implements ItemWriter<String> {
    
        public void write(List<? extends String> items) throws Exception {
            System.out.println(items);
            Thread.sleep(5000l);
        }
    
    }
    Launcher Class
    Code:
    public class TestLauncher {
        
    private static ApplicationContext applicationContext;
        
        public static void main(String[] args) throws UnknownHostException {
            
            //Getting application Context
            applicationContext = new ClassPathXmlApplicationContext("testBatchThread.xml");
            //Initializing beans
            Job job = (Job) applicationContext.getBean("testJob");
            JobLauncher launcher = (JobLauncher) applicationContext.getBean("jobLauncher");
            JobParameters jobParameters = new JobParametersBuilder().addString("instanceName", String.valueOf(new Date().getTime())).toJobParameters();
    
            // Launch the job
            try {
                launcher.run(job, jobParameters);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    
    }
    Il you launch the test class you will have 100 thread each display their number at the same time

    But if you change for a jdbc jobRepository :

    New Spring application context :

    Code:
    <?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xmlns:util="http://www.springframework.org/schema/util"
        xmlns:context="http://www.springframework.org/schema/context"
        xmlns:task="http://www.springframework.org/schema/task"
        xmlns:tx="http://www.springframework.org/schema/tx"
        xmlns:batch="http://www.springframework.org/schema/batch"
        xsi:schemaLocation="http://www.springframework.org/schema/beans
            http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
            http://www.springframework.org/schema/context
            http://www.springframework.org/schema/context/spring-context-3.1.xsd
            http://www.springframework.org/schema/util
            http://www.springframework.org/schema/util/spring-util.xsd
            http://www.springframework.org/schema/task
            http://www.springframework.org/schema/task/spring-task-3.1.xsd
            http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.1.xsd
            http://www.springframework.org/schema/batch
            http://www.springframework.org/schema/batch/spring-batch-2.1.xsd">
    
        <bean name="testReader" class="com.priceminister.rgp.batch.reader.TestReader" />
        <bean name="testWritter" class="com.priceminister.rgp.batch.writer.TestWritter" />
        
        <!-- Common configuration to every job -->
        <bean id="jobRepository" class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
            <property name="transactionManager" ref="transactionManager" />
            <property name="dataSource" ref="dataSource" />
        </bean>
        
        <bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
            <property name="driverClassName" value="oracle.jdbc.driver.OracleDriver"/>
            <property name="url" value="jdbc:oracle:thin:@jouet:1521:DEV9_SAT"/>
            <property name="username" value="rgp"/>
            <property name="password" value="rgp"/>
        </bean>
    
        <bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
            <property name="dataSource" ref="dataSource" />
        </bean>
        
        <bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
            <property name="jobRepository" ref="jobRepository" />
        </bean>
        
             <bean id="taskExecutorStep1" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
           <property name="corePoolSize" value="100" />
           <property name="maxPoolSize" value="100" />
        </bean>
        
        <batch:job id="testJob">
            <batch:step id="step1">
                <batch:tasklet task-executor="taskExecutorStep1" throttle-limit="100">
                    <batch:chunk reader="testReader" writer="testWritter" commit-interval="1"/>
                </batch:tasklet>
            </batch:step>
        </batch:job>
    </beans>

    Only 8 threads are launched at the same time.

    I tried also with a SimpleAsyncTaskExecutor but it's the same.

    I don't understand why the JDBC jobRepository change my Step thread behavior because there is no database connection in my step.

    I check my max database connections but it's 300 and there is no exception.

    Do you have an idea where it could come from?

    Thank's a lot.

    --
    Bastien

  • #2
    Sorry for not getting to this sooner. We are in the process of moving to StackOverflow for our forums.

    This question is probably a better candidate for StackOverflow, perhaps against the #spring-batch tag. If you do post it there, please reply here with the link. Thanks!

    Comment

    Working...
    X