Announcement Announcement Module
Collapse
No announcement yet.
Problems migrating to 2.0.3 Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Problems migrating to 2.0.3

    We've migrating from Spring batch 1.1.1 to 2.0.3 and we are experiencing a problem behavior.
    We have some jobs that must run during the day, so they use a similar scheduling configuration.

    NOTE:All jobs are working fine if they don't share the scheduling windows.

    We are dealing with the folowing problem.
    the first Job (ValidadorJob) starts, and then starts procesing files, then if the Second jobs starts when the 1st step of the 1st job is working, the item reader (validadorStepItemProvider) gets into an infinite loop.

    This behavior is repeated if we change the start delay in the scheduler congifuration of the Second Job.
    For example, if we modify the start delay of the 2nd job calculating that the job is going to start when the other step of the first job is running (transformarStep), the same problem appears, TransformacionFaseItemProvider starts looping.


    Thanks in advance....
    Mario

    This is our jobs config.

    ********* SCHEDULE CONFIGURATION JOB ************************
    Code:
    <bean id="validadorTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
    	<property name="jobDetail" ref="validadorLauncher" />
    	<property name="startDelay" value="2000" />
    	<property name="repeatInterval" value="600000" />
    	<property name="repeatCount" value="-1" />
    </bean>
    
    <bean id="importadorEstadoTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
    	<property name="jobDetail" ref="importadorEstadoLauncher" />
    	<property name="startDelay" value="14000" />
    	<property name="repeatInterval" value="600000" />
    	<property name="repeatCount" value="-1" />
    </bean>
    
    <bean id="validadorLauncher" class="org.springframework.scheduling.quartz.JobDetailBean">
    	<property name="jobClass" value="com.swissmedical.rem.batch.launcher.FileScanQuartzBatchJobLauncher" />
    	<property name="jobDataAsMap">
    		<map>		
    			<entry key="batchContainer" 	value="batchContainer" 			/>
    			<entry key="hostName" 			value="localhost" 				/>
    			<entry key="jobName" 			value="validadorJob"      		/>
    			<entry key="jobStream" 			value="validatorStream" 		/>
    			<entry key="userName" 			value="${user.name}" 			/>
    			<entry key="jobRun" 			value="1" 						/>
    			<entry key="directoryName" 		value="${validadorJob.validadorStep.workingDirectory}"	/>
    		</map>
    	</property>
    </bean>
    
    <bean id="importadorEstadoLauncher" class="org.springframework.scheduling.quartz.JobDetailBean">
    	<property name="jobClass" value="com.swissmedical.rem.batch.launcher.FileScanQuartzBatchJobLauncher" />
    	<property name="jobDataAsMap">
    		<map>
    			<entry key="batchContainer" 	value="batchContainer" 			/>
    			<entry key="hostName" 			value="localhost" 				/>
    			<entry key="jobName" 			value="importadorEstadoJob"   	/>
    			<entry key="jobStream" 			value="importadorEstadoStream"  />
    			<entry key="userName" 			value="${user.name}" 			/>
    			<entry key="jobRun" 			value="1" 						/>
    			<entry key="directoryName" 		value="${importadorEstadoJob.importadorStep.workingDirectory}"	/>
    		</map>
    	</property>
    </bean>
    ************** FIRST JOB ***********************************
    Code:
    <job id="validadorJob" job-repository="simpleJobRepository" >
    	<step id="validarStep" next="moveFilesValidadorStep">
    		<tasklet transaction-manager="transactionManager">
    			<chunk reader="validadorStepItemProvider" processor="validadorStepItemProcess" writer="validadorStepItemWriter" commit-interval="1" skip-limit="10">
    				<skippable-exception-classes>
    					java.lang.Exception
    				</skippable-exception-classes>
    			</chunk>
    			<listeners>
    				<listener ref="stepListenerStep"/>
    				<listener ref="errorCreateFileListenerStep"/>
    			</listeners>
    		</tasklet>
    	</step>	
    
    ......
    OMITING other steps config (see validadorJob.xml)
    ......
    	<step id="transformarStep" next="moveFilesTransformadosStep">
    				<tasklet transaction-manager="transactionManager">
    					<chunk reader="transformadorStepItemProvider" processor="transformadorStepItemProcess" writer="transformadorStepItemWriter" commit-interval="1" skip-limit="10">
    						<skippable-exception-classes>
    							java.lang.Exception
    						</skippable-exception-classes>
    					</chunk>
    					<listeners>
    						<listener ref="stepListenerStep"/>
    						<listener ref="errorCreateFileTransfromacionListenerStep"/>
    					</listeners>
    				</tasklet>
    	</step>	
    		
    </job>
    
    <beans:bean id="validadorStepItemProvider" class="com.swissmedical.rem.batch.item.provider.DirectoryItemProvider">
    	<beans:property name="fileLister" ref="filesToValidateFileLister" />
    	<beans:property name="fileProcessor" ref="fileToValidateFileProcessor" />
    </beans:bean>
    
    <beans:bean id="validadorStepItemProcess" class="com.swissmedical.rem.batch.item.processor.ValidadorItemProcessor">
    	<beans:property name="validadorService" ref="validadorService" /> 
    	<beans:property name="erroresMostrar" ref="codeErrorToShowFileProcessor" />
    	<beans:property name="streamConverter" ref="ingresoFileToValidateStreamConverter" />
    	<beans:property name="errorFileProcessor" ref="afterErroneousProcessingFileRenamer" />	
    </beans:bean>
    
    <beans:bean id="validadorStepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorItemWriter">
    	<beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" />
    </beans:bean>
    ********************* SECOND JOB **************************
    Code:
    <job id="importadorEstadoJob" job-repository="simpleJobRepository" >
    	<step id="importadorEstadoStep" next="moveFilesEstadoStep">
    		<tasklet transaction-manager="transactionManager">
    			<chunk reader="importadorEstadoStepItemProvider" processor="importadorEstadoItemProcessor" writer="importadorEstadoItemWriter" commit-interval="1" skip-limit="10">
    				<skippable-exception-classes>
    					java.lang.Exception
    				</skippable-exception-classes>
    			</chunk>
    			<listeners>
    				<listener ref="stepListenerStep"/>
    			</listeners>
    		</tasklet>
    	</step>	
    
    	<step id="moveFilesEstadoStep" parent="taskletStep">
    		<tasklet ref="moveFilesImportadorEstadoBean" transaction-manager="transactionManager"/>
    	</step>	
    	
    </job>
    
    <beans:bean id="stepListenerStep" class="com.swissmedical.rem.batch.interceptor.CleanUpFileAfterProcessingInterceptor">
    	<beans:property name="service" ref="errorImportacionService" />
    </beans:bean>
    
    <beans:bean id="importadorEstadoItemProcessor" class="com.swissmedical.rem.batch.item.processor.ImportadorEstadoItemProcessor">
    	<beans:property name="importadorEstadoService" ref="importadorEstadoService" />
    	<beans:property name="streamConverter" ref="estadoFileToValidateStreamConverter" />
    </beans:bean>
    
    <beans:bean id="importadorEstadoItemWriter" class="com.swissmedical.rem.batch.item.writer.ImportadorEstadoItemWriter">
    	<beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" />
    </beans:bean>

  • #2
    It's tough to say, since it I think you didn't include the second jobs reader, but given the first one is listed as a DirectoryItemProvider, which I'm assuming you wrote, my guess would be that it's something about your readers that can't be running at the same time. I'm not sure why this would just show up when upgrading though. Are you sure it's an infinite loop and not just hanging on the read? We really need some more information about exactly what's happening, and what the readers are doing to know whether it's framework related, or just your custom implementation.

    Comment


    • #3
      Thanks for replying quickly
      Here I'm attaching some configuration files and our custom iplementation of the items readers. I think that this would be enough to diagnosticate.
      Let me now if you need something else.

      Thanks again.
      Mario

      Comment


      • #4
        Jira

        Hi, Lucas,
        Is there any posibility that my problem could be related with this recent Jira ticket?

        http://forum.springsource.org/newrep...ote=1&p=260664

        Regards,
        Mario

        Comment


        • #5
          Keep trying on this

          Hi lucas,
          I developed another Job. And i'm facing with the same problem.
          I'm not using a custom reader this time.
          All that I could find out is that the job gets into an infinite loop when I try to skip an exception. So, if I avoid to use skipable exceptions or the job simply don't throw one. All works fine.
          Here is my Job configurartion.

          Code:
          
          	<bean id="headerRecordTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
          		<property name="delimiter" value=">"/>
          		<property name="names" value="nombreArchivo,md5Code" />
          	</bean>
          		
          	<beans:bean id="customRecordTokenizer" class="org.springframework.batch.item.file.transform.FixedLengthTokenizer">
          		<beans:property name="names" value="nombreArchivo,separador,md5Code" />
          		<beans:property name="columns" value="1-48,49-58,59-90" />
          	</beans:bean>
          
          	<beans:bean id="fieldSetMapper"  class="com.swissmedical.rem.batch.item.mapper.TxtRecordMD5Mapper" />
          Code:
          <job id="validadorMD5Job" job-repository="simpleJobRepository" >
          		<step id="validarMD5Step">
          			<tasklet transaction-manager="transactionManager" >
          				<chunk reader="validadorMD5StepItemReader" 
          					   processor="validadorMD5StepItemProcessor" 
          					   writer="validadorMD5StepItemWriter" 
          					   commit-interval="10"
          					   skip-limit="100">				
          					<streams>
          							<stream ref="validadorMD5StepItemReader"/>
          							<stream ref="validadorMD5StepItemWriter"/>
          					</streams>
          					<skippable-exception-classes>
          						java.io.FileNotFoundException
          					</skippable-exception-classes>
          				</chunk>
          				<listeners>
          					<listener ref="MD5HeaderAndFooterCallback"/>
          					<!--<listener ref="fileNameListener" />
          				--></listeners>					
          			</tasklet>
          			<!--<next on="COMPLETED WITH SKIPS" to="comunicarErrores" />
          			--><fail on="FAILED" exit-code="FAILED"/>
          		</step>
          
          		<!--<step id="comunicarErrores">
          			<tasklet ref="comunicarErroresRem"/>
          		</step>	
          	--></job>
          	
          	<beans:bean id="comunicarErroresRem" class="com.swissmedical.rem.batch.common.ErrorLogTasklet">
          		<!-- <beans:property name="dataSource" ref="dataSource"/> -->
          	</beans:bean>
          
          	<beans:bean id="itemReaderParent" class="org.springframework.batch.item.file.MultiResourceItemReader" abstract="true">
          		<beans:property name="delegate">
          			<beans:bean class="org.springframework.batch.item.file.FlatFileItemReader">
          				<!-- <beans:property name="resource" value="#{stepExecutionContext[fileName]}/*"/>  -->
          				<beans:property name="lineMapper">
          					<beans:bean class="org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper">
          						<beans:property name="tokenizers">
          							<beans:map>
          								<beans:entry key="rem*" value-ref="headerRecordTokenizer" />
          								<beans:entry key="documentos*" value-ref="customRecordTokenizer" />
          							</beans:map>
          						</beans:property>
          						<beans:property name="fieldSetMappers">
          							<beans:map>
          								<beans:entry key="rem*" value-ref="fieldSetMapper" />
          								<beans:entry key="documentos*" value-ref="fieldSetMapper" />
          							</beans:map>
          						</beans:property>
          					</beans:bean>
          				</beans:property>
          			</beans:bean>
          		</beans:property>
          	</beans:bean>
          
          	<beans:bean id="MD5HeaderAndFooterCallback" class="com.swissmedical.rem.batch.support.MD5StepSupport" />
          
          	<beans:bean id="validadorMD5StepItemReader" parent="itemReaderParent" scope="step" autowire-candidate="false" > 
          		 <beans:property name="resources" value="${validadorMD5Job.workingDirectory}/*.md5" />
          	</beans:bean>
          	
          	<beans:bean id="fileOutputLocator" class="org.springframework.core.io.FileSystemResource">
          		<beans:constructor-arg type="java.lang.String" value="${outPutMd5}" />
          	</beans:bean>
          	
          	<beans:bean id="validadorMD5StepItemProcessor" class="com.swissmedical.rem.batch.item.processor.ValidadorMD5ItemProcessor">
          	</beans:bean>
          	<!--
          	<beans:bean id="validadorMD5StepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorMD5ItemWriter">
          	  <beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" /> 
          	</beans:bean>
          	
          
          	<beans:bean id="fileNameListener" class="com.swissmedical.rem.batch.common.OutputFileListener" />-->	
          	
          	<beans:bean id="validadorMD5StepItemWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
          	    <beans:property name="resource" value="${outPutMd5}" />
          		<beans:property name="lineAggregator">
          			<beans:bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
          				<beans:property name="delimiter" value=">" />
          				<beans:property name="fieldExtractor">
          					<beans:bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
          						<beans:property name="names" value="fileName,fileMD5,estado" />
          					</beans:bean>
          				</beans:property>
          			</beans:bean>
          		</beans:property>
          	</beans:bean>
          	<!--	
          	<beans:bean id="outputResource" class="org.springframework.core.io.FileSystemResource">
          		<beans:constructor-arg value="${outPutMd5}" />	
          	</beans:bean>	
          	-->

          Comment


          • #6
            I'm sorry I put a wrong link
            This is the correct one.
            Is related with an ItemWriter, however, i'm experiencing the same with a ItemReader

            http://jira.springframework.org/browse/BATCH-1354

            I'm using SB 2.0.3 were this problem (of the ItemWriter) was solved.

            Comment


            • #7
              Why does your reader/writer throw an Error? If it does then change that anyway. But maybe that's not the problem. There isn't enough detail yet. Probably you should post your reader / writer implementations.

              Comment


              • #8
                I missed the .zip attachment you provided earlier, sorry, that was useful. Isn't your business logic in a loop (I didn't follow it all exactly)? There is a file rename piece, and if your item craps out before it gets there then the file filter picks up that file again, and again, and again?

                Comment


                • #9
                  Avoiding custom reader

                  Hi Dave, thanks for trying too help me.
                  At the begining I thought the same, for that reason I post other job without a custom reader implementation.
                  Here is a job development with 2.1.0.M1. Without custom readers.

                  The classes defined by me in this job are just, the Processor (Which thows the exception) , a Listener a FieldSetMapper a Pojo item to be process (File record).

                  Here is the main job definition, I also attached the whole job and container definition, a couple of files, and a log file in case that you need them.

                  Code:
                  <job id="validadorMD5Job" job-repository="simpleJobRepository" >
                  		<step id="validarMD5Step" >
                  			<tasklet >
                  				<chunk 	reader="validadorMD5StepItemReader" 
                  						processor="validadorMD5StepItemProcessor" 
                  						writer="validadorMD5StepItemWriter"						
                  					   	commit-interval="50" 
                  					   	skip-limit="10"
                  					   >				
                  					<streams>
                  						<stream ref="validadorMD5StepItemReader"/>
                  					</streams>
                  					<skippable-exception-classes merge="true">
                  						<include class="java.io.FileNotFoundException" />
                  					</skippable-exception-classes>
                  				</chunk>
                  					<listeners>
                  						<listener ref="comunicarErroresRem" />
                  					</listeners>				
                  			</tasklet>
                  		</step>	
                  	</job>
                  Reader , writer processor and listener
                  Code:
                  <beans:bean id="comunicarErroresRem" class="com.swissmedical.rem.batch.item.listeners.ErrorLogListener">
                  	</beans:bean>
                  
                  	<beans:bean id="itemReaderParent" class="org.springframework.batch.item.file.MultiResourceItemReader" abstract="true">
                  		<beans:property name="delegate">
                  			<beans:bean class="org.springframework.batch.item.file.FlatFileItemReader">
                  				<beans:property name="lineMapper">
                  					<beans:bean class="org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper">
                  						<beans:property name="tokenizers">
                  							<beans:map>
                  								<beans:entry key="rem*" value-ref="headerRecordTokenizer" />
                  								<beans:entry key="documentos*" value-ref="customRecordTokenizer" />
                  							</beans:map>
                  						</beans:property>
                  						<beans:property name="fieldSetMappers">
                  							<beans:map>
                  								<beans:entry key="rem*" value-ref="fieldSetMapper" />
                  								<beans:entry key="documentos*" value-ref="fieldSetMapper" />
                  							</beans:map>
                  						</beans:property>
                  					</beans:bean>
                  				</beans:property>
                  			</beans:bean>
                  		</beans:property>
                  	</beans:bean>
                  
                  	<beans:bean id="validadorMD5StepItemReader" parent="itemReaderParent" scope="step" autowire-candidate="false" > 
                  		 <beans:property name="resources" value="${validadorMD5Job.workingDirectory}/*.md5" />
                  	</beans:bean>
                  		
                  	<beans:bean id="validadorMD5StepItemProcessor" class="com.swissmedical.rem.batch.item.processor.ValidadorMD5ItemProcessor">
                  	</beans:bean>
                  	
                  	<beans:bean id="validadorMD5StepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorMD5ItemWriter">
                  	</beans:bean>
                  Mappers and tokenizers

                  Code:
                  	<beans:bean id="headerRecordTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
                  		<beans:property name="delimiter" value=">"/>
                  		<beans:property name="names" value="nombreArchivo,md5Code" />
                  	</beans:bean>
                  		
                  	<beans:bean id="customRecordTokenizer" class="org.springframework.batch.item.file.transform.FixedLengthTokenizer">
                  		<beans:property name="names" value="nombreArchivo,separador,md5Code" />
                  		<beans:property name="columns" value="1-48,49-58,59-90" />
                  	</beans:bean>
                  	
                  	<beans:bean id="fieldSetMapper"  class="com.swissmedical.rem.batch.item.mapper.md5.TxtRecordMD5Mapper" />
                  If you check the log file you will see that the process always ends when the skip limit is reached. (It doesn't matter if it's 1 or 1000...)

                  Note: Nowadays I'm using a FileNotFoundException just to try to solve this, but the idea is to use a ProccesingException (Cutom Exception implementation) for all the jobs to log the errors against a service

                  Comment


                  • #10
                    adding zip file

                    I'm sorry here is the zip file

                    Comment


                    • #11
                      I can't see anything obviously wrong. You had more than 10 errors so the skip limit was busted. Those errors weren't consecutive or anything, so it looks like there was just a problem with the data?

                      Comment


                      • #12
                        Hi Dave,
                        Thank for answering again,
                        That's not the problem because, if I set the skip limit in 1 , 10 or 1000 the the process coninues until the skip limit is reached.
                        In this process, we check that the files showed on each record (at *.md5 files) exist.
                        If all files are in the path , everything goes ok, but if I just remove only one, the process start looping until the skip limit is reached

                        Regards,
                        Mario

                        Comment


                        • #13
                          That wasn't what I saw in the logs you provided. It seemed like there there were many items processed in between failures, and a different number in each case. If the behaviour is as simple as you describe you should be able to create a proper test case. Then we can understand what is happening more easily.

                          Comment


                          • #14
                            I found some thing

                            Ok, I'm going to prepare the test case.

                            However, I found that if I use just one tokenizer the skip functionality works fine. (leaving the file to process just with the "rem" prefix for each record)


                            Code:
                            <beans:bean id="itemReaderParent" class="org.springframework.batch.item.file.MultiResourceItemReader" abstract="true">
                            		<beans:property name="delegate">
                            			<beans:bean class="org.springframework.batch.item.file.FlatFileItemReader">
                            				<beans:property name="lineMapper">
                            					<beans:bean class="org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper">
                            						<beans:property name="tokenizers">
                            							<beans:map>
                            								<beans:entry key="rem*" value-ref="headerRecordTokenizer" />
                            								<!--<beans:entry key="documentos*" value-ref="customRecordTokenizer" />
                            							--></beans:map>
                            						</beans:property>
                            						<beans:property name="fieldSetMappers">
                            							<beans:map>
                            								<beans:entry key="rem*" value-ref="fieldSetMapper" />
                            								<!--<beans:entry key="documentos*" value-ref="fieldSetMapper" />
                            							--></beans:map>
                            						</beans:property>
                            					</beans:bean>
                            				</beans:property>
                            			</beans:bean>
                            		</beans:property>
                            	</beans:bean>

                            Comment

                            Working...
                            X