2020-02 spring batch example added

2026-05-30 10:50:42 +00:00 · 2020-06-06 01:30:13 +04:00
parent 148a56e31c
commit 5d2e27cdf8
16 changed files with 630 additions and 0 deletions
@@ -0,0 +1,6 @@
+.idea/
+*.iml
+
+target/
+
+output.csv
@@ -0,0 +1,16 @@
+Ivan,23
+John,24
+Ivan,23
+John,24
+Ivan,23
+Mary,24
+Ivan,23
+John,24
+Sunny,23
+John,24
+Ivan,23
+John,24
+Ivan,23
+John,24
+Ivan,23
+John,24
@@ -0,0 +1,16 @@
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
+Person(name=Ivan, age=24)
+Person(name=Mary, age=25)
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
+Person(name=Sunny, age=24)
+Person(name=John, age=25)
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
+Person(name=Ivan, age=24)
+Person(name=John, age=25)
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>ru.otus.example</groupId>
+    <artifactId>spring-batch-demo</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.2.4.RELEASE</version>
+    </parent>
+
+    <properties>
+        <java.version>13</java.version>
+        <maven.compiler.sourcre>13</maven.compiler.sourcre>
+        <maven.compiler.target>13</maven.compiler.target>
+        <mongock.version>2.0.2</mongock.version>
+    </properties>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-batch</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>de.flapdoodle.embed</groupId>
+            <artifactId>de.flapdoodle.embed.mongo</artifactId>
+            <version>LATEST</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.github.cloudyrock.mongock</groupId>
+            <artifactId>mongock-spring</artifactId>
+            <version>${mongock.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <version>2.1.8.RELEASE</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.shell</groupId>
+            <artifactId>spring-shell-starter</artifactId>
+            <version>2.0.1.RELEASE</version>
+        </dependency>
+
+        <!--Тестирование-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.batch</groupId>
+            <artifactId>spring-batch-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit-jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit-jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>spring-batch-demo</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
@@ -0,0 +1,17 @@
+package ru.otus.example.springbatch;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.PropertySource;
+
+import java.io.IOException;
+
+@SpringBootApplication
+public class Main {
+
+    public static void main(String[] args) throws IOException {
+        SpringApplication.run(Main.class, args);
+    }
+}
+
+
@@ -0,0 +1,45 @@
+package ru.otus.example.springbatch.chandgelogs;
+
+import com.github.cloudyrock.mongock.ChangeLog;
+import com.github.cloudyrock.mongock.ChangeSet;
+import com.mongodb.client.MongoDatabase;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import ru.otus.example.springbatch.model.Person;
+
+@ChangeLog(order = "001")
+public class InitMongoDBDataChangeLog {
+
+    @ChangeSet(order = "000", id = "dropDB", author = "stvort", runAlways = true)
+    public void dropDB(MongoDatabase database){
+        database.drop();
+    }
+
+    @ChangeSet(order = "001", id = "initPersons", author = "stvort", runAlways = true)
+    public void initPersons(MongoTemplate template){
+        template.save(new Person("John", 21));
+        template.save(new Person("Ivan", 32));
+        template.save(new Person("Dima", 52));
+        template.save(new Person("Mike", 22));
+        template.save(new Person("Duke", 33));
+        template.save(new Person("John", 21));
+        template.save(new Person("Ivan", 32));
+        template.save(new Person("Dima", 52));
+        template.save(new Person("Mike", 22));
+        template.save(new Person("Duke", 33));
+        template.save(new Person("John", 21));
+        template.save(new Person("Ivan", 32));
+        template.save(new Person("Dima", 52));
+        template.save(new Person("Mike", 22));
+        template.save(new Person("Duke", 33));
+        template.save(new Person("John", 21));
+        template.save(new Person("Ivan", 32));
+        template.save(new Person("Dima", 52));
+        template.save(new Person("Mike", 22));
+        template.save(new Person("Duke", 33));
+        template.save(new Person("John", 21));
+        template.save(new Person("Ivan", 32));
+        template.save(new Person("Dima", 52));
+        template.save(new Person("Mike", 22));
+        template.save(new Person("Duke", 33));
+    }
+}
@@ -0,0 +1,21 @@
+package ru.otus.example.springbatch.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Component;
+
+@Data
+@Component
+public class AppProps {
+
+    @Value("${spring.data.mongodb.database}")
+    private String mongoDBName;
+
+    @Value("${app.input-file:''}")
+    private String inputFileName;
+
+    @Value("${app.output-file:''}")
+    private String outputFileName;
+
+}
@@ -0,0 +1,43 @@
+package ru.otus.example.springbatch.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.*;
+import org.springframework.batch.core.configuration.JobRegistry;
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.batch.core.job.flow.FlowJob;
+import org.springframework.batch.core.launch.support.RunIdIncrementer;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.file.FlatFileItemReader;
+import org.springframework.batch.item.file.FlatFileItemWriter;
+import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
+import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder;
+import org.springframework.batch.item.file.transform.DelimitedLineAggregator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.transaction.PlatformTransactionManager;
+import ru.otus.example.springbatch.model.Person;
+import ru.otus.example.springbatch.service.HappyBirthdayService;
+
+import java.util.List;
+
+@EnableBatchProcessing
+@Configuration
+public class BatchConfig {
+    @Bean
+    public JobRegistryBeanPostProcessor postProcessor(JobRegistry jobRegistry) {
+        var processor = new JobRegistryBeanPostProcessor();
+        processor.setJobRegistry(jobRegistry);
+        return processor;
+    }
+
+}
@@ -0,0 +1,138 @@
+package ru.otus.example.springbatch.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.*;
+import org.springframework.batch.core.configuration.JobRegistry;
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.batch.core.launch.support.RunIdIncrementer;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.file.FlatFileItemReader;
+import org.springframework.batch.item.file.FlatFileItemWriter;
+import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
+import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder;
+import org.springframework.batch.item.file.transform.DelimitedLineAggregator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.FileSystemResource;
+import ru.otus.example.springbatch.model.Person;
+import ru.otus.example.springbatch.service.HappyBirthdayService;
+
+import java.util.List;
+
+@SuppressWarnings("all")
+@Configuration
+public class JobConfig {
+    private static final int CHUNK_SIZE = 5;
+    private final Logger logger = LoggerFactory.getLogger("Batch");
+
+    public static final String OUTPUT_FILE_NAME = "outputFileName";
+    public static final String INPUT_FILE_NAME = "inputFileName";
+    public static final String IMPORT_USER_JOB_NAME = "importUserJob";
+
+    @Autowired
+    private JobBuilderFactory jobBuilderFactory;
+
+    @Autowired
+    private StepBuilderFactory stepBuilderFactory;
+
+    @StepScope
+    @Bean
+    public FlatFileItemReader<Person> reader(@Value("#{jobParameters['" + INPUT_FILE_NAME + "']}") String inputFileName) {
+        return new FlatFileItemReaderBuilder<Person>()
+                .name("personItemReader")
+                .resource(new FileSystemResource(inputFileName))
+
+                // Работа через lineMapper
+                .lineMapper((s, i) -> {
+                    String[] fieldsValues = s.split(",");
+                    return new Person(fieldsValues[0], Integer.parseInt(fieldsValues[1]));
+                })
+/*
+                // Работа через fieldSetMapper
+                .delimited()
+                .names("name", "age")
+                .fieldSetMapper(new BeanWrapperFieldSetMapper<>() {{
+                    setTargetType(Person.class);
+                }})
+*/
+                .build();
+    }
+
+
+
+    @StepScope
+    @Bean
+    public ItemProcessor processor(HappyBirthdayService happyBirthdayService) {
+        return (ItemProcessor<Person, Person>) happyBirthdayService::doHappyBirthday;
+    }
+
+    @StepScope
+    @Bean
+    public FlatFileItemWriter writer(@Value("#{jobParameters['" + OUTPUT_FILE_NAME + "']}") String outputFileName) {
+        return new FlatFileItemWriterBuilder<>()
+                .name("personItemWriter")
+                .resource(new FileSystemResource(outputFileName))
+                .lineAggregator(new DelimitedLineAggregator<>())
+                .build();
+    }
+
+    @Bean
+    public Job importUserJob(Step step1) {
+        return jobBuilderFactory.get(IMPORT_USER_JOB_NAME)
+                .incrementer(new RunIdIncrementer())
+                .flow(step1)
+                .end()
+                .listener(new JobExecutionListener() {
+                    @Override
+                    public void beforeJob(JobExecution jobExecution) {
+                        logger.info("Начало job");
+                    }
+
+                    @Override
+                    public void afterJob(JobExecution jobExecution) {
+                        logger.info("Конец job");
+                    }
+                })
+                .build();
+    }
+
+    @Bean
+    public Step step1(FlatFileItemWriter writer, ItemReader reader, ItemProcessor itemProcessor) {
+        return stepBuilderFactory.get("step1")
+                .chunk(CHUNK_SIZE)
+                .reader(reader)
+                .processor(itemProcessor)
+                .writer(writer)
+                .listener(new ItemReadListener() {
+                    public void beforeRead() { logger.info("Начало чтения"); }
+                    public void afterRead(Object o) { logger.info("Конец чтения"); }
+                    public void onReadError(Exception e) { logger.info("Ошибка чтения"); }
+                })
+                .listener(new ItemWriteListener() {
+                    public void beforeWrite(List list) { logger.info("Начало записи"); }
+                    public void afterWrite(List list) { logger.info("Конец записи"); }
+                    public void onWriteError(Exception e, List list) { logger.info("Ошибка записи"); }
+                })
+                .listener(new ItemProcessListener() {
+                    public void beforeProcess(Object o) {logger.info("Начало обработки");}
+                    public void afterProcess(Object o, Object o2) {logger.info("Конец обработки");}
+                    public void onProcessError(Object o, Exception e) {logger.info("Ошбка обработки");}
+                })
+                .listener(new ChunkListener() {
+                    public void beforeChunk(ChunkContext chunkContext) {logger.info("Начало пачки");}
+                    public void afterChunk(ChunkContext chunkContext) {logger.info("Конец пачки");}
+                    public void afterChunkError(ChunkContext chunkContext) {logger.info("Ошибка пачки");}
+                })
+//                .taskExecutor(new SimpleAsyncTaskExecutor())
+                .build();
+    }
+}
@@ -0,0 +1,24 @@
+package ru.otus.example.springbatch.config;
+
+import com.github.cloudyrock.mongock.Mongock;
+import com.github.cloudyrock.mongock.SpringMongockBuilder;
+import com.mongodb.MongoClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import ru.otus.example.springbatch.chandgelogs.InitMongoDBDataChangeLog;
+
+@Configuration
+public class MongoConfig {
+
+    @Autowired
+    private AppProps appProps;
+
+    @Bean
+    public Mongock mongock(MongoClient mongoClient) {
+        return new SpringMongockBuilder(mongoClient, appProps.getMongoDBName(),
+                InitMongoDBDataChangeLog.class.getPackageName())
+                .build();
+    }
+}
@@ -0,0 +1,13 @@
+package ru.otus.example.springbatch.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
+public class Person {
+    private String name;
+    private int age;
+}
@@ -0,0 +1,13 @@
+package ru.otus.example.springbatch.service;
+
+import org.springframework.stereotype.Service;
+import ru.otus.example.springbatch.model.Person;
+
+@Service
+public class HappyBirthdayService {
+
+    public Person doHappyBirthday(Person person){
+        person.setAge(person.getAge() + 1);
+        return person;
+    }
+}
@@ -0,0 +1,56 @@
+package ru.otus.example.springbatch.shell;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.explore.JobExplorer;
+import org.springframework.batch.core.launch.JobLauncher;
+import org.springframework.batch.core.launch.JobOperator;
+import org.springframework.shell.standard.ShellComponent;
+import org.springframework.shell.standard.ShellMethod;
+import ru.otus.example.springbatch.config.AppProps;
+
+
+import static ru.otus.example.springbatch.config.JobConfig.*;
+
+@RequiredArgsConstructor
+@ShellComponent
+public class BatchCommands {
+
+    private final AppProps appProps;
+    private final Job importUserJob;
+
+    private final JobLauncher jobLauncher;
+    private final JobOperator jobOperator;
+    private final JobExplorer jobExplorer;
+
+    //http://localhost:8080/h2-console/
+
+    @SneakyThrows
+    @ShellMethod(value = "startMigrationJobWithJobLauncher", key = "sm-jl")
+    public void startMigrationJobWithJobLauncher() {
+        JobExecution execution = jobLauncher.run(importUserJob, new JobParametersBuilder()
+                .addString(INPUT_FILE_NAME, appProps.getInputFileName())
+                .addString(OUTPUT_FILE_NAME, appProps.getOutputFileName())
+                .toJobParameters());
+        System.out.println(execution);
+    }
+
+    @SneakyThrows
+    @ShellMethod(value = "startMigrationJobWithJobOperator", key = "sm-jo")
+    public void startMigrationJobWithJobOperator() {
+        Long executionId = jobOperator.start(IMPORT_USER_JOB_NAME,
+                INPUT_FILE_NAME + "=" + appProps.getInputFileName() + "\n" +
+                        OUTPUT_FILE_NAME + "=" + appProps.getOutputFileName()
+        );
+        System.out.println(jobOperator.getSummary(executionId));
+    }
+
+    @ShellMethod(value = "showInfo", key = "i")
+    public void showInfo() {
+        System.out.println(jobExplorer.getJobNames());
+        System.out.println(jobExplorer.getLastJobInstance(IMPORT_USER_JOB_NAME));
+    }
+}
@@ -0,0 +1,31 @@
+spring:
+  batch:
+    job:
+      enabled: false
+  shell:
+    interactive:
+      enabled: true
+
+  datasource:
+    url: jdbc:h2:mem:testdb
+    driverClassName: org.h2.Driver
+    username: sa
+    password:
+
+  h2:
+    console:
+      enabled: true
+      path: /h2-console
+
+  data:
+    mongodb:
+      uri: mongodb://localhost
+      port: 0 #27017
+      database: SpringBatchExampleDB
+
+app:
+  ages-count-to-add: 1
+  input-file: entries.csv
+  output-file: output.dat
+
+#debug: true
@@ -0,0 +1,59 @@
+package ru.otus.example.springbatch.config;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.test.AssertFile;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.batch.test.JobRepositoryTestUtils;
+import org.springframework.batch.test.context.SpringBatchTest;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.FileSystemResource;
+
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static ru.otus.example.springbatch.config.JobConfig.*;
+
+@SpringBootTest
+@SpringBatchTest
+class ImportUserJobTest {
+
+    private static final String TEST_INPUT_FILE_NAME = "entries.csv";
+    private static final String EXPECTED_OUTPUT_FILE_NAME = "expected_output.dat";
+    private static final String TEST_OUTPUT_FILE_NAME = "test_output.dat";
+
+    @Autowired
+    private JobLauncherTestUtils jobLauncherTestUtils;
+
+    @Autowired
+    private JobRepositoryTestUtils jobRepositoryTestUtils;
+
+    @BeforeEach
+    void clearMetaData() {
+        jobRepositoryTestUtils.removeJobExecutions();
+    }
+
+    @Test
+    void testJob() throws Exception {
+        FileSystemResource expectedResult = new FileSystemResource(EXPECTED_OUTPUT_FILE_NAME);
+        FileSystemResource actualResult = new FileSystemResource(TEST_OUTPUT_FILE_NAME);
+
+        Job job = jobLauncherTestUtils.getJob();
+        assertThat(job).isNotNull()
+                .extracting(Job::getName)
+                .isEqualTo(IMPORT_USER_JOB_NAME);
+
+        JobParameters parameters = new JobParametersBuilder()
+                .addString(INPUT_FILE_NAME, TEST_INPUT_FILE_NAME)
+                .addString(OUTPUT_FILE_NAME, TEST_OUTPUT_FILE_NAME)
+                .toJobParameters();
+        JobExecution jobExecution = jobLauncherTestUtils.launchJob(parameters);
+
+        assertThat(jobExecution.getExitStatus().getExitCode()).isEqualTo("COMPLETED");
+        AssertFile.assertFileEquals(expectedResult, actualResult);
+    }
+}
@@ -0,0 +1,19 @@
+spring:
+  batch:
+    job:
+      enabled: false
+  shell:
+    interactive:
+      enabled: false
+
+  datasource:
+    url: jdbc:h2:mem:testdb
+    driverClassName: org.h2.Driver
+    username: sa
+    password:
+
+  data:
+    mongodb:
+      uri: mongodb://localhost
+      port: 0
+      database: SpringBatchTestExampleDB