feat: 实现睿行车辆报废月数据入库定时任务

readme.md

@@ -6,7 +6,11 @@
 
 ## 定时任务api
 
-### 睿行车辆基本信息日数据入库定时任务
+### 车辆
+
+#### 睿行
+
+##### 睿行车辆基本信息日数据入库定时任务
 
 ```shell
 curl --location 'http://localhost:39110/jobs/runJob' \
@@ -15,3 +19,23 @@ curl --location 'http://localhost:39110/jobs/runJob' \
     "jobName": "CLJBXX_JOB"
 }'
 ```
+
+##### 睿行车辆越界报警日数据入库定时任务
+
+```shell
+curl --location 'http://localhost:39110/jobs/runJob' \
+--header 'Content-Type: application/json' \
+--data '{
+    "jobName": "YJBJRTJ_JOB"
+}'
+```
+
+##### 睿行车辆报废月数据入库定时任务
+
+```shell
+curl --location 'http://localhost:39110/jobs/runJob' \
+--header 'Content-Type: application/json' \
+--data '{
+    "jobName": "CLBF_JOB"
+}'
+```

src/main/java/com/nokia/finance/tasks/config/JobConfig.java

@@ -48,4 +48,12 @@ public class JobConfig {
      * 睿行车辆越界报警日数据归档路径
      */
     private String yjbjrtjHistoryPath;
+    /**
+     * 睿行车辆报废月数据路径
+     */
+    private String clbfSourcePath;
+    /**
+     * 睿行车辆报废月数据归档路径
+     */
+    private String clbfHistoryPath;
 }

src/main/java/com/nokia/finance/tasks/enums/JobEnum.java

@@ -9,5 +9,9 @@ public enum JobEnum {
      * 睿行车辆越界报警日数据入库定时任务
      */
     YJBJRTJ_JOB,
+    /**
+     * 睿行车辆报废月数据入库定时任务
+     */
+    CLBF_JOB,
     ;
 }

src/main/java/com/nokia/finance/tasks/jobs/car/ruixing/ClbfJob.java

@@ -0,0 +1,302 @@
+package com.nokia.finance.tasks.jobs.car.ruixing;
+
+import com.nokia.finance.tasks.common.exception.MyRuntimeException;
+import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
+import com.nokia.finance.tasks.config.JobConfig;
+import com.nokia.finance.tasks.pojo.po.common.AreaPo;
+import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
+import com.nokia.finance.tasks.service.car.CarService;
+import com.nokia.finance.tasks.service.common.AreaService;
+import com.nokia.finance.tasks.service.common.OrganizationService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.DateUtil;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+/**
+ * 睿行车辆报废月数据入库定时任务
+ */
+@Slf4j
+@Service
+public class ClbfJob {
+    private final JobConfig jobConfig;
+    private final CarService carService;
+    private final OrganizationService organizationService;
+    private final AreaService areaService;
+
+    public ClbfJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
+                   AreaService areaService) {
+        this.jobConfig = jobConfig;
+        this.carService = carService;
+        this.organizationService = organizationService;
+        this.areaService = areaService;
+    }
+
+    /**
+     * 执行任务
+     */
+    @Scheduled(cron = "0 0 23 1 * ?")
+    public void runJob() {
+        // 数据目录
+        Path dir = Paths.get(jobConfig.getClbfSourcePath());
+        try (Stream<Path> stream = Files.list(dir)) {
+            // 获取数据目录下的文件列表
+            List<Path> pathList = stream.filter(t -> t.toString().endsWith(".xlsx")).sorted().toList();
+            log.info("睿行车辆报废月数据文件列表: {}", pathList);
+            if (CollectionUtils.isEmpty(pathList)) {
+                throw new MyRuntimeException("睿行车辆报废月数据没有文件");
+            }
+            for (Path path : pathList) {
+                singleJob(path);
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 处理单个文件
+     *
+     * @param path 文件路径
+     */
+    public void singleJob(Path path) throws Exception {
+        List<Map<String, String>> list = readFile(path);
+        List<Map<String, String>> distinctList = dataProcessing(path, list);
+        Path csvPath = toCsv(path, distinctList);
+        copyCsv(csvPath);
+        move(path);
+    }
+
+    /**
+     * 读取文件
+     *
+     * @param path 文件路径
+     */
+    public List<Map<String, String>> readFile(Path path) throws Exception {
+        log.info("读取: {}", path);
+        List<String> rawHeaders = Stream.of("车牌号码", "车辆所属单位", "报废类型", "报废标准", "报废日期",
+                "行驶公里数(Km)", "备注", "资产编号", "资产名称", "规格型号", "实际使用年限", "资产原值(万元)",
+                "累计折旧值(万元)").toList();
+        List<String> headers = Stream.of("che_pai_hao", "che_liang_suo_shu_dan_wei", "bai_fei_lei_xing",
+                "bao_fei_biao_zhun", "bao_fei_ri_qi", "xing_shi_gong_li_shu", "bei_zhu", "zi_chan_bian_hao",
+                "zi_chan_ming_cheng", "gui_ge_xing_hao", "shi_ji_shi_yong_nian_xian", "zi_chan_yuan_zhi_wan_yuan",
+                "lei_ji_zhe_jiu_zhi_wan_yuan").toList();
+        try (InputStream inputStream = Files.newInputStream(path);
+             Workbook workbook = new XSSFWorkbook(inputStream)
+        ) {
+            List<Map<String, String>> resultList = new ArrayList<>();
+            // 读取第一个工作表
+            Sheet sheet = workbook.getSheetAt(0);
+            // 表头行
+            Row headerRow = sheet.getRow(0);
+            // 列数
+            int columnCount = headerRow.getPhysicalNumberOfCells();
+            log.info("columnCount: {}", columnCount);
+            // 检查表头
+            if (headers.size() != columnCount) {
+                throw new MyRuntimeException(path.getFileName() + "列数错误");
+            }
+            for (int i = 0; i < columnCount; i++) {
+                Cell cell = headerRow.getCell(i);
+                if (cell == null || !rawHeaders.get(i).equals(cell.getStringCellValue())) {
+                    throw new MyRuntimeException(path.getFileName() + " 表头错误");
+                }
+            }
+            // 最后行数
+            int lastRowNum = sheet.getLastRowNum();
+            log.info("lastRowNum: {}", lastRowNum);
+            if (lastRowNum == 0) {
+                throw new MyRuntimeException(path.getFileName() + " 为空");
+            }
+            // 遍历行
+            for (int i = 1; i <= lastRowNum; i++) {
+                Row row = sheet.getRow(i);
+                if (row == null) {
+                    continue;
+                }
+                Map<String, String> rowMap = new LinkedHashMap<>();
+                // 遍历列
+                for (int j = 0; j < columnCount; j++) {
+                    String cellValue = "";
+                    rowMap.put(headers.get(j), cellValue);
+                    Cell cell = row.getCell(j);
+                    if (cell == null) {
+                        continue;
+                    }
+                    switch (cell.getCellType()) {
+                        case STRING:
+                            // 删除字符串空白字符
+                            cellValue = StringUtils.trimAllWhitespace(cell.getStringCellValue());
+                            break;
+                        case NUMERIC:
+                            if (DateUtil.isCellDateFormatted(cell)) {
+                                cellValue = DateUtil.getLocalDateTime(cell.getNumericCellValue())
+                                        .format(DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss"));
+                                break;
+                            }
+                            cellValue = String.valueOf(cell.getNumericCellValue());
+                            break;
+                        case BOOLEAN:
+                            cellValue = String.valueOf(cell.getBooleanCellValue());
+                            break;
+                        default:
+                            break;
+                    }
+                    rowMap.put(headers.get(j), cellValue);
+                }
+                resultList.add(rowMap);
+            }
+            return resultList;
+        }
+    }
+
+    /**
+     * 数据加工
+     *
+     * @param path 文件路径
+     * @param list 数据
+     */
+    public List<Map<String, String>> dataProcessing(Path path, List<Map<String, String>> list) {
+        List<OrganizationPo> secondOrgs = organizationService.getSecondOrgs();
+        List<OrganizationPo> thirdOrgs = organizationService.getThirdOrgs();
+        Map<String, OrganizationPo> orgMap = organizationService.getOrgMap(secondOrgs, thirdOrgs);
+        Map<String, List<OrganizationPo>> thirdOrganizationListMap =
+                organizationService.getThirdOrganizationListMap(secondOrgs, thirdOrgs);
+        List<AreaPo> cities = areaService.getCities();
+        List<AreaPo> districts = areaService.getDistricts();
+        Map<String, AreaPo> areaMap = areaService.getAreaMap(cities, districts);
+        Map<String, List<AreaPo>> districtListMap = areaService.getDistrictListMap(cities, districts);
+        for (Map<String, String> map : list) {
+            String baoFeiRiQi = map.get("bao_fei_ri_qi");
+            LocalDate localDate = LocalDate.parse(baoFeiRiQi, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+            String yearMonth = localDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
+            String year = String.valueOf(localDate.getYear());
+            String month = String.valueOf(localDate.getMonthValue());
+            map.put("year_month", yearMonth);
+            map.put("year_no", year);
+            map.put("month_no", month);
+            String rawChePaiHao = map.get("che_pai_hao");
+            map.put("raw_che_pai_hao", rawChePaiHao);
+            String chePaiHao = carService.getChePai(rawChePaiHao);
+            map.put("che_pai_hao", chePaiHao);
+            String chePaiFail = carService.chePaiFail(rawChePaiHao);
+            map.put("che_pai_fail", chePaiFail);
+            String cheLiangSuoShuDanWei = map.get("che_liang_suo_shu_dan_wei");
+            String firstUnit = carService.getFirstUnit(cheLiangSuoShuDanWei);
+            map.put("first_unit", firstUnit);
+            String secondUnit = carService.getSecondUnit(cheLiangSuoShuDanWei, firstUnit);
+            map.put("second_unit", secondUnit);
+            String thirdUnit = carService.getThirdUnit(cheLiangSuoShuDanWei, secondUnit);
+            map.put("third_unit", thirdUnit);
+            String areaNo = carService.getAreaNo(secondOrgs, cheLiangSuoShuDanWei);
+            map.put("area_no", areaNo);
+            String areaName = carService.getOrgName(orgMap, areaNo);
+            map.put("area_name", areaName);
+            String cityNo = carService.getCityNo(thirdOrganizationListMap, areaNo, areaName, cheLiangSuoShuDanWei);
+            map.put("city_no", cityNo);
+            String cityName = carService.getOrgName(orgMap, cityNo);
+            map.put("city_name", cityName);
+            String areaNo2 = carService.getAreaNo2(areaName, cityName);
+            map.put("area_no2", areaNo2);
+            String areaName2 = carService.getOrgName(orgMap, areaNo2);
+            map.put("area_name2", areaName2);
+            String cityId = carService.getCityId(cities, cheLiangSuoShuDanWei);
+            map.put("city_id", cityId);
+            String city = carService.getAreaName(areaMap, cityId);
+            map.put("city", city);
+            String districtId = carService.getDistrictId(districtListMap, cityId, cityName, cheLiangSuoShuDanWei);
+            map.put("district_id", districtId);
+            String district = carService.getAreaName(areaMap, districtId);
+            map.put("district", district);
+            map.put("source", path.getFileName().toString());
+        }
+        // 去重
+        return list.stream().filter(distinctByKey(map -> map.get("che_pai_hao") + map.get("bao_fei_ri_qi"))).toList();
+    }
+
+    /**
+     * 去重
+     */
+    private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
+        Set<Object> set = ConcurrentHashMap.newKeySet();
+        return t -> set.add(keyExtractor.apply(t));
+    }
+
+    /**
+     * 生成csv
+     *
+     * @param path 源文件路径
+     * @param list 数据
+     */
+    public Path toCsv(Path path, List<Map<String, String>> list) throws Exception {
+        log.info("去重后条数：{}", list.size());
+        Files.createDirectories(Paths.get(jobConfig.getClbfHistoryPath()));
+        Path csvPath = Paths.get(jobConfig.getClbfHistoryPath() + path.getFileName() + ".csv");
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(csvPath),
+                StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            Map<String, String> header = list.get(0);
+            // 表头
+            printer.printRecord(header.keySet());
+            for (Map<String, String> map : list) {
+                printer.printRecord(map.values());
+            }
+        }
+        return csvPath;
+    }
+
+    /**
+     * 导入数据库
+     *
+     * @param path 文件路径
+     */
+    public void copyCsv(Path path) {
+        String dbTable = "car.car_bao_fei";
+        String csv = path.toString();
+        String columns = "(che_pai_hao,che_liang_suo_shu_dan_wei,bai_fei_lei_xing,bao_fei_biao_zhun,bao_fei_ri_qi,xing_shi_gong_li_shu,bei_zhu,zi_chan_bian_hao,zi_chan_ming_cheng,gui_ge_xing_hao,shi_ji_shi_yong_nian_xian,zi_chan_yuan_zhi_wan_yuan,lei_ji_zhe_jiu_zhi_wan_yuan,year_month,year_no,month_no,raw_che_pai_hao,che_pai_fail,first_unit,second_unit,third_unit,area_no,area_name,city_no,city_name,area_no2,area_name2,city_id,city,district_id,district,source)";
+        Long timeout = 60000L;
+        PsqlUtil.copyCsv(jobConfig.getCopyScriptPath(), jobConfig.getDbHost(), jobConfig.getDbPort(),
+                jobConfig.getDbUsername(), jobConfig.getDbPassword(), jobConfig.getDbName(), dbTable, csv, columns,
+                timeout, null);
+    }
+
+    /**
+     * 移动源文件到历史文件夹
+     *
+     * @param path 源文件路径
+     */
+    public void move(Path path) throws Exception {
+        Path targetPath = Paths.get(jobConfig.getClbfHistoryPath(), path.getFileName().toString());
+        Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
+    }
+}

src/main/java/com/nokia/finance/tasks/service/JobService.java

@@ -1,6 +1,7 @@
 package com.nokia.finance.tasks.service;
 
 import com.nokia.finance.tasks.common.R;
+import com.nokia.finance.tasks.jobs.car.ruixing.ClbfJob;
 import com.nokia.finance.tasks.jobs.car.ruixing.CljbxxJob;
 import com.nokia.finance.tasks.jobs.car.ruixing.YjbjrtjJob;
 import com.nokia.finance.tasks.pojo.RunJobDto;
@@ -10,10 +11,12 @@ import org.springframework.stereotype.Service;
 public class JobService {
     private final CljbxxJob cljbxxJob;
     private final YjbjrtjJob yjbjrtjJob;
+    private final ClbfJob clbfJob;
 
-    public JobService(CljbxxJob cljbxxJob, YjbjrtjJob yjbjrtjJob) {
+    public JobService(CljbxxJob cljbxxJob, YjbjrtjJob yjbjrtjJob, ClbfJob clbfJob) {
         this.cljbxxJob = cljbxxJob;
         this.yjbjrtjJob = yjbjrtjJob;
+        this.clbfJob = clbfJob;
     }
 
     public R<Object> runJob(RunJobDto dto) {
@@ -24,6 +27,9 @@ public class JobService {
             case YJBJRTJ_JOB:
                 yjbjrtjJob.runJob();
                 break;
+            case CLBF_JOB:
+                clbfJob.runJob();
+                break;
             default:
                 break;
         }

src/main/resources/application-dev.yml

@@ -27,3 +27,7 @@ job:
     yjbjrtj-source-path: data/rxftp/yjbjrtj/
     # 睿行车辆越界报警日数据归档路径
     yjbjrtj-history-path: data/history/rxftp/yjbjrtj/
+    # 睿行车辆报废月数据路径
+    clbf-source-path: data/rxftp/clbf/
+    # 睿行车辆报废月数据归档路径
+    clbf-history-path: data/history/rxftp/clbf/

src/main/resources/application-prod.yml

@@ -27,3 +27,7 @@ job:
     yjbjrtj-source-path: /data/rxftp/yjbjrtj/
     # 睿行车辆越界报警日数据归档路径
     yjbjrtj-history-path: /data/history/rxftp/yjbjrtj/
+    # 睿行车辆报废月数据路径
+    clbf-source-path: /data/rxftp/clbf/
+    # 睿行车辆报废月数据归档路径
+    clbf-history-path: /data/history/rxftp/clbf/

src/test/java/com/nokia/finance/tasks/car/ruixing/ClbfJobTests.java

@@ -0,0 +1,22 @@
+package com.nokia.finance.tasks.car.ruixing;
+
+import com.nokia.finance.tasks.jobs.car.ruixing.ClbfJob;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+@Slf4j
+@SpringBootTest
+@ActiveProfiles("dev")
+class ClbfJobTests {
+    @Autowired
+    ClbfJob clbfJob;
+
+    @Test
+    void runJobTest() {
+        clbfJob.runJob();
+    }
+
+}