|
@@ -1,379 +1,379 @@
|
|
|
-package com.nokia.finance.tasks.jobs.car.ruixing;
|
|
|
-
|
|
|
-import com.nokia.finance.tasks.common.exception.MyRuntimeException;
|
|
|
-import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
|
|
|
-import com.nokia.finance.tasks.config.JobConfig;
|
|
|
-import com.nokia.finance.tasks.enums.DataStageEnum;
|
|
|
-import com.nokia.finance.tasks.pojo.bo.car.CarUnitBo;
|
|
|
-import com.nokia.finance.tasks.pojo.po.common.AreaPo;
|
|
|
-import com.nokia.finance.tasks.pojo.po.common.DataLogPo;
|
|
|
-import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
|
|
|
-import com.nokia.finance.tasks.service.car.CarService;
|
|
|
-import com.nokia.finance.tasks.service.common.AreaService;
|
|
|
-import com.nokia.finance.tasks.service.common.DataLogServiceImpl;
|
|
|
-import com.nokia.finance.tasks.service.common.OrganizationService;
|
|
|
-import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.csv.CSVFormat;
|
|
|
-import org.apache.commons.csv.CSVPrinter;
|
|
|
-import org.apache.poi.ss.usermodel.Cell;
|
|
|
-import org.apache.poi.ss.usermodel.DateUtil;
|
|
|
-import org.apache.poi.ss.usermodel.Row;
|
|
|
-import org.apache.poi.ss.usermodel.Sheet;
|
|
|
-import org.apache.poi.ss.usermodel.Workbook;
|
|
|
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
|
-import org.springframework.scheduling.annotation.Scheduled;
|
|
|
-import org.springframework.stereotype.Service;
|
|
|
-import org.springframework.util.CollectionUtils;
|
|
|
-import org.springframework.util.StringUtils;
|
|
|
-
|
|
|
-import java.io.IOException;
|
|
|
-import java.io.InputStream;
|
|
|
-import java.io.OutputStreamWriter;
|
|
|
-import java.math.BigDecimal;
|
|
|
-import java.math.RoundingMode;
|
|
|
-import java.nio.charset.StandardCharsets;
|
|
|
-import java.nio.file.Files;
|
|
|
-import java.nio.file.Path;
|
|
|
-import java.nio.file.Paths;
|
|
|
-import java.nio.file.StandardCopyOption;
|
|
|
-import java.time.LocalDate;
|
|
|
-import java.time.format.DateTimeFormatter;
|
|
|
-import java.time.temporal.ChronoUnit;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.LinkedHashMap;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
-import java.util.Set;
|
|
|
-import java.util.concurrent.CompletableFuture;
|
|
|
-import java.util.concurrent.ConcurrentHashMap;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-import java.util.function.Function;
|
|
|
-import java.util.function.Predicate;
|
|
|
-import java.util.regex.Matcher;
|
|
|
-import java.util.regex.Pattern;
|
|
|
-import java.util.stream.Stream;
|
|
|
-
|
|
|
-/**
|
|
|
- * 睿行车辆基本信息日数据入库定时任务
|
|
|
- */
|
|
|
-@Slf4j
|
|
|
-@Service
|
|
|
-public class CarBaseDataDayJob {
|
|
|
- private final JobConfig jobConfig;
|
|
|
- private final CarService carService;
|
|
|
- private final OrganizationService organizationService;
|
|
|
- private final AreaService areaService;
|
|
|
- private final DataLogServiceImpl dataLogService;
|
|
|
-
|
|
|
- public CarBaseDataDayJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
|
|
|
- AreaService areaService, DataLogServiceImpl dataLogService) {
|
|
|
- this.jobConfig = jobConfig;
|
|
|
- this.carService = carService;
|
|
|
- this.organizationService = organizationService;
|
|
|
- this.areaService = areaService;
|
|
|
- this.dataLogService = dataLogService;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 执行任务
|
|
|
- */
|
|
|
- @Scheduled(cron = "0 56 23 * * ?")
|
|
|
- public void runJob() {
|
|
|
- List<DataLogPo> dataLogPoList = new ArrayList<>();
|
|
|
- // 数据目录
|
|
|
- Path dir = Paths.get(jobConfig.getCarBaseDataDaySourcePath());
|
|
|
- try (Stream<Path> stream = Files.list(dir)) {
|
|
|
- // 获取数据目录下的文件列表
|
|
|
- List<Path> pathList = stream.filter(t -> t.toString().endsWith(".xlsx")).sorted().toList();
|
|
|
- log.info("睿行车辆基本信息日数据文件列表: {}", pathList);
|
|
|
- if (CollectionUtils.isEmpty(pathList)) {
|
|
|
- throw new MyRuntimeException("睿行车辆基本信息日数据没有文件");
|
|
|
- }
|
|
|
- for (Path path : pathList) {
|
|
|
- CompletableFuture.runAsync(() -> {
|
|
|
- try {
|
|
|
- singleJob(path, dataLogPoList);
|
|
|
- } catch (Exception e) {
|
|
|
- throw new MyRuntimeException(e);
|
|
|
- }
|
|
|
- }).get(1, TimeUnit.MINUTES);
|
|
|
- }
|
|
|
- } catch (InterruptedException e) {
|
|
|
- log.error("线程中断: {}", e, e);
|
|
|
- Thread.currentThread().interrupt();
|
|
|
- DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆基本信息日数据", DataStageEnum.CC, "入库");
|
|
|
- } catch (Exception e) {
|
|
|
- log.error(e.toString(), e);
|
|
|
- DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆基本信息日数据", DataStageEnum.CC, "入库");
|
|
|
- } finally {
|
|
|
- if (!CollectionUtils.isEmpty(dataLogPoList)) {
|
|
|
- dataLogService.saveBatch(dataLogPoList);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 处理单个文件
|
|
|
- *
|
|
|
- * @param path 文件路径
|
|
|
- * @param dataLogPoList 数据处理日志列表
|
|
|
- */
|
|
|
- public void singleJob(Path path, List<DataLogPo> dataLogPoList) throws IOException {
|
|
|
- DataLogPo dataLogPo = new DataLogPo();
|
|
|
- dataLogPo.setDataName("睿行车辆基本信息日数据");
|
|
|
- dataLogPo.setObject(path.getFileName().toString());
|
|
|
- dataLogPo.setStage(DataStageEnum.CC.value);
|
|
|
- dataLogPo.setOperationName("入库");
|
|
|
- long fileSize = Files.size(path);
|
|
|
- dataLogPo.setDataSize(fileSize);
|
|
|
- dataLogPoList.add(dataLogPo);
|
|
|
- if (fileSize == 0) {
|
|
|
- move(path);
|
|
|
- throw new MyRuntimeException(path.getFileName() + " 空文件");
|
|
|
- }
|
|
|
- List<Map<String, String>> list = readFile(path);
|
|
|
- List<Map<String, String>> distinctList = dataProcessing(path, list);
|
|
|
- dataLogPo.setDataCount(distinctList.size());
|
|
|
- Path csvPath = toCsv(path, distinctList);
|
|
|
- copyCsv(csvPath);
|
|
|
- move(path);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 读取文件
|
|
|
- *
|
|
|
- * @param path 文件路径
|
|
|
- */
|
|
|
- public List<Map<String, String>> readFile(Path path) throws IOException {
|
|
|
- log.info("读取: {}", path);
|
|
|
- List<String> rawHeaders = Stream.of("车牌号", "车辆品牌", "排量", "排量类型", "燃油类型", "燃油标号",
|
|
|
- "登记日期", "车辆来源", "车辆类型", "车辆使用性质", "环保等级", "用车模式", "发动机号", "车架号", "初始里程",
|
|
|
- "车辆所属单位", "行驶证号", "备注", "审核状态", "车辆状态", "资产编号", "绑定驾驶员姓名", "购置价格(万元)",
|
|
|
- "车龄(年)", "标准百公里油耗", "设备工作模式").toList();
|
|
|
- List<String> headers = Stream.of("che_pai_hao", "che_liang_pin_pai", "pai_liang", "pai_liang_lei_xing",
|
|
|
- "ran_you_lei_xing", "ran_you_biao_hao", "deng_ji_ri_qi", "che_liang_lai_yuan", "che_liang_lei_xing",
|
|
|
- "che_liang_shi_yong_xing_zhi", "huan_bao_deng_ji", "yong_che_mo_shi", "fa_dong_ji_hao", "che_jia_hao",
|
|
|
- "chu_shi_li_cheng", "che_liang_suo_shu_dan_wei", "xing_shi_zheng_hao", "bei_zhu", "shen_he_zhuang_tai",
|
|
|
- "che_liang_zhuang_tai", "zi_chan_bian_hao", "bang_ding_jia_shi_yuan_xing_ming", "gou_zhi_jia_ge",
|
|
|
- "che_ling_nian", "biao_zhun_bai_gong_li_you_hao", "she_bei_gong_zuo_mo_shi").toList();
|
|
|
- try (InputStream inputStream = Files.newInputStream(path);
|
|
|
- Workbook workbook = new XSSFWorkbook(inputStream)
|
|
|
- ) {
|
|
|
- List<Map<String, String>> resultList = new ArrayList<>();
|
|
|
- // 读取第一个工作表
|
|
|
- Sheet sheet = workbook.getSheetAt(0);
|
|
|
- // 表头行
|
|
|
- Row headerRow = sheet.getRow(0);
|
|
|
- // 列数
|
|
|
- int columnCount = headerRow.getPhysicalNumberOfCells();
|
|
|
- log.info("columnCount: {}", columnCount);
|
|
|
- // 检查表头
|
|
|
- if (headers.size() != columnCount) {
|
|
|
- throw new MyRuntimeException(path.getFileName() + "列数错误");
|
|
|
- }
|
|
|
- for (int i = 0; i < columnCount; i++) {
|
|
|
- Cell cell = headerRow.getCell(i);
|
|
|
- if (cell == null || !rawHeaders.get(i).equals(cell.getStringCellValue())) {
|
|
|
- throw new MyRuntimeException(path.getFileName() + " 表头错误");
|
|
|
- }
|
|
|
- }
|
|
|
- // 最后行数
|
|
|
- int lastRowNum = sheet.getLastRowNum();
|
|
|
- log.info("lastRowNum: {}", lastRowNum);
|
|
|
- if (lastRowNum == 0) {
|
|
|
- move(path);
|
|
|
- throw new MyRuntimeException(path.getFileName() + " 数据0条");
|
|
|
- }
|
|
|
- // 遍历行
|
|
|
- for (int i = 1; i <= lastRowNum; i++) {
|
|
|
- log.debug("row: {}", i);
|
|
|
- Row row = sheet.getRow(i);
|
|
|
- if (row == null || row.getCell(0) == null) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- Map<String, String> rowMap = new LinkedHashMap<>();
|
|
|
- // 遍历列
|
|
|
- for (int j = 0; j < columnCount; j++) {
|
|
|
- log.debug("column: {}", j);
|
|
|
- String cellValue = "";
|
|
|
- rowMap.put(headers.get(j), cellValue);
|
|
|
- Cell cell = row.getCell(j);
|
|
|
- if (cell == null) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- switch (cell.getCellType()) {
|
|
|
- case STRING:
|
|
|
- // 删除字符串空白字符
|
|
|
- cellValue = StringUtils.trimAllWhitespace(cell.getStringCellValue());
|
|
|
- break;
|
|
|
- case NUMERIC:
|
|
|
- if (DateUtil.isCellDateFormatted(cell)) {
|
|
|
- cellValue = DateUtil.getLocalDateTime(cell.getNumericCellValue())
|
|
|
- .format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
- break;
|
|
|
- }
|
|
|
- cellValue = String.valueOf(cell.getNumericCellValue());
|
|
|
- break;
|
|
|
- case BOOLEAN:
|
|
|
- cellValue = String.valueOf(cell.getBooleanCellValue());
|
|
|
- break;
|
|
|
- default:
|
|
|
- break;
|
|
|
- }
|
|
|
- rowMap.put(headers.get(j), cellValue);
|
|
|
- }
|
|
|
- resultList.add(rowMap);
|
|
|
- }
|
|
|
- return resultList;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 数据加工
|
|
|
- *
|
|
|
- * @param path 文件路径
|
|
|
- * @param list 数据
|
|
|
- */
|
|
|
- public List<Map<String, String>> dataProcessing(Path path, List<Map<String, String>> list) {
|
|
|
- // 从文件名提取日期
|
|
|
- String regex = "\\d{8}";
|
|
|
- Pattern pattern = Pattern.compile(regex);
|
|
|
- Matcher matcher = pattern.matcher(path.getFileName().toString());
|
|
|
- String dateString;
|
|
|
- if (matcher.find()) {
|
|
|
- dateString = matcher.group();
|
|
|
- } else {
|
|
|
- throw new MyRuntimeException(path.getFileName() + " 提取日期失败");
|
|
|
- }
|
|
|
- List<OrganizationPo> secondOrgs = organizationService.getSecondOrgs();
|
|
|
- List<OrganizationPo> thirdOrgs = organizationService.getThirdOrgs();
|
|
|
- Map<String, OrganizationPo> orgMap = organizationService.getOrgMap(secondOrgs, thirdOrgs);
|
|
|
- Map<String, List<OrganizationPo>> thirdOrganizationListMap =
|
|
|
- organizationService.getThirdOrganizationListMap(secondOrgs, thirdOrgs);
|
|
|
- List<AreaPo> cities = areaService.getCities();
|
|
|
- List<AreaPo> districts = areaService.getDistricts();
|
|
|
- Map<String, AreaPo> areaMap = areaService.getAreaMap(cities, districts);
|
|
|
- Map<String, List<AreaPo>> districtListMap = areaService.getDistrictListMap(cities, districts);
|
|
|
- LocalDate localDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
- LocalDate lastMonthDate = localDate.minusMonths(1);
|
|
|
- String dataDate = localDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
- String yearMonth = localDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
- String year = String.valueOf(localDate.getYear());
|
|
|
- String month = String.valueOf(localDate.getMonthValue());
|
|
|
- String lastMonth = lastMonthDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
- for (Map<String, String> map : list) {
|
|
|
- map.put("data_date", dataDate);
|
|
|
- map.put("year_month", yearMonth);
|
|
|
- map.put("year_no", year);
|
|
|
- map.put("month_no", month);
|
|
|
- map.put("last_month", lastMonth);
|
|
|
- String rawChePaiHao = map.get("che_pai_hao");
|
|
|
- map.put("raw_che_pai_hao", rawChePaiHao);
|
|
|
- String chePaiHao = carService.getChePai(rawChePaiHao);
|
|
|
- map.put("che_pai_hao", chePaiHao);
|
|
|
- String chePaiFail = carService.chePaiFail(rawChePaiHao);
|
|
|
- map.put("che_pai_fail", chePaiFail);
|
|
|
- String cheLiangSuoShuDanWei = map.get("che_liang_suo_shu_dan_wei");
|
|
|
- CarUnitBo carUnitBo = carService.getCarUnitBo(cheLiangSuoShuDanWei);
|
|
|
- String firstUnit = carService.getFirstUnit(carUnitBo);
|
|
|
- if (!StringUtils.hasText(firstUnit)) {
|
|
|
- throw new MyRuntimeException("车辆单位错误:" + rawChePaiHao + " " + cheLiangSuoShuDanWei);
|
|
|
- }
|
|
|
- map.put("first_unit", firstUnit);
|
|
|
- String secondUnit = carService.getSecondUnit(carUnitBo, firstUnit);
|
|
|
- map.put("second_unit", secondUnit);
|
|
|
- String thirdUnit = carService.getThirdUnit(carUnitBo, secondUnit);
|
|
|
- map.put("third_unit", thirdUnit);
|
|
|
- String areaNo = carService.getAreaNo(secondOrgs, firstUnit);
|
|
|
- map.put("area_no", areaNo);
|
|
|
- String areaName = carService.getOrgName(orgMap, areaNo);
|
|
|
- map.put("area_name", areaName);
|
|
|
- String cityNo = carService.getCityNo(thirdOrganizationListMap, areaNo, areaName, carUnitBo);
|
|
|
- map.put("city_no", cityNo);
|
|
|
- String cityName = carService.getOrgName(orgMap, cityNo);
|
|
|
- map.put("city_name", cityName);
|
|
|
- String areaNo2 = carService.getAreaNo2(areaName, cityName);
|
|
|
- map.put("area_no2", areaNo2);
|
|
|
- String areaName2 = carService.getOrgName(orgMap, areaNo2);
|
|
|
- map.put("area_name2", areaName2);
|
|
|
- String cityId = carService.getCityId(cities, cheLiangSuoShuDanWei);
|
|
|
- map.put("city_id", cityId);
|
|
|
- String city = carService.getAreaName(areaMap, cityId);
|
|
|
- map.put("city", city);
|
|
|
- String districtId = carService.getDistrictId(districtListMap, cityId, cityName, cheLiangSuoShuDanWei);
|
|
|
- map.put("district_id", districtId);
|
|
|
- String district = carService.getAreaName(areaMap, districtId);
|
|
|
- map.put("district", district);
|
|
|
- String cheLing = "0";
|
|
|
- String dengJiRiQi = map.get("deng_ji_ri_qi");
|
|
|
- if (StringUtils.hasText(dengJiRiQi)) {
|
|
|
- LocalDate dengJiRiQiDate = LocalDate.parse(dengJiRiQi, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
- long days = ChronoUnit.DAYS.between(dengJiRiQiDate, localDate);
|
|
|
- if (days > 0) {
|
|
|
- BigDecimal n = BigDecimal.valueOf(days).divide(BigDecimal.valueOf(365), 1, RoundingMode.HALF_UP);
|
|
|
- cheLing = n.toString();
|
|
|
- }
|
|
|
- }
|
|
|
- map.put("che_ling", cheLing);
|
|
|
- map.put("source", path.getFileName().toString());
|
|
|
- }
|
|
|
- // 去重
|
|
|
- return list.stream().filter(distinctByKey(map -> map.get("che_pai_hao"))).toList();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 去重
|
|
|
- */
|
|
|
- private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
|
|
|
- Set<Object> set = ConcurrentHashMap.newKeySet();
|
|
|
- return t -> set.add(keyExtractor.apply(t));
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 生成csv
|
|
|
- *
|
|
|
- * @param path 源文件路径
|
|
|
- * @param list 数据
|
|
|
- */
|
|
|
- public Path toCsv(Path path, List<Map<String, String>> list) throws IOException {
|
|
|
- log.info("去重后条数:{}", list.size());
|
|
|
- Files.createDirectories(Paths.get(jobConfig.getCarBaseDataDayHistoryPath()));
|
|
|
- Path csvPath = Paths.get(jobConfig.getCarBaseDataDayHistoryPath() + path.getFileName() + ".csv");
|
|
|
- try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(csvPath),
|
|
|
- StandardCharsets.UTF_8);
|
|
|
- CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
|
|
|
- // 添加bom头避免excel乱码
|
|
|
- osw.write('\ufeff');
|
|
|
- Map<String, String> header = list.get(0);
|
|
|
- // 表头
|
|
|
- printer.printRecord(header.keySet());
|
|
|
- for (Map<String, String> map : list) {
|
|
|
- printer.printRecord(map.values());
|
|
|
- }
|
|
|
- }
|
|
|
- return csvPath;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 导入数据库
|
|
|
- * @param path 文件路径
|
|
|
- */
|
|
|
- public void copyCsv(Path path) {
|
|
|
- String dbTable = "car.car_base_data_day";
|
|
|
- String csv = path.toString();
|
|
|
- String columns = "(che_pai_hao,che_liang_pin_pai,pai_liang,pai_liang_lei_xing,ran_you_lei_xing,ran_you_biao_hao,deng_ji_ri_qi,che_liang_lai_yuan,che_liang_lei_xing,che_liang_shi_yong_xing_zhi,huan_bao_deng_ji,yong_che_mo_shi,fa_dong_ji_hao,che_jia_hao,chu_shi_li_cheng,che_liang_suo_shu_dan_wei,xing_shi_zheng_hao,bei_zhu,shen_he_zhuang_tai,che_liang_zhuang_tai,zi_chan_bian_hao,bang_ding_jia_shi_yuan_xing_ming,gou_zhi_jia_ge,che_ling_nian,biao_zhun_bai_gong_li_you_hao,she_bei_gong_zuo_mo_shi,data_date,year_month,year_no,month_no,last_month,raw_che_pai_hao,che_pai_fail,first_unit,second_unit,third_unit,area_no,area_name,city_no,city_name,area_no2,area_name2,city_id,city,district_id,district,che_ling,source)";
|
|
|
- Long timeout = 60000L;
|
|
|
- PsqlUtil.copyCsv(jobConfig.getCopyScriptPath(), jobConfig.getDbHost(), jobConfig.getDbPort(),
|
|
|
- jobConfig.getDbUsername(), jobConfig.getDbPassword(), jobConfig.getDbName(), dbTable, csv, columns,
|
|
|
- timeout, null);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 移动源文件到历史文件夹
|
|
|
- * @param path 源文件路径
|
|
|
- */
|
|
|
- public void move(Path path) throws IOException {
|
|
|
- Path targetPath = Paths.get(jobConfig.getCarBaseDataDayHistoryPath(), path.getFileName().toString());
|
|
|
- Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
|
|
- }
|
|
|
-}
|
|
|
+//package com.nokia.finance.tasks.jobs.car.ruixing;
|
|
|
+//
|
|
|
+//import com.nokia.finance.tasks.common.exception.MyRuntimeException;
|
|
|
+//import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
|
|
|
+//import com.nokia.finance.tasks.config.JobConfig;
|
|
|
+//import com.nokia.finance.tasks.enums.DataStageEnum;
|
|
|
+//import com.nokia.finance.tasks.pojo.bo.car.CarUnitBo;
|
|
|
+//import com.nokia.finance.tasks.pojo.po.common.AreaPo;
|
|
|
+//import com.nokia.finance.tasks.pojo.po.common.DataLogPo;
|
|
|
+//import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
|
|
|
+//import com.nokia.finance.tasks.service.car.CarService;
|
|
|
+//import com.nokia.finance.tasks.service.common.AreaService;
|
|
|
+//import com.nokia.finance.tasks.service.common.DataLogServiceImpl;
|
|
|
+//import com.nokia.finance.tasks.service.common.OrganizationService;
|
|
|
+//import lombok.extern.slf4j.Slf4j;
|
|
|
+//import org.apache.commons.csv.CSVFormat;
|
|
|
+//import org.apache.commons.csv.CSVPrinter;
|
|
|
+//import org.apache.poi.ss.usermodel.Cell;
|
|
|
+//import org.apache.poi.ss.usermodel.DateUtil;
|
|
|
+//import org.apache.poi.ss.usermodel.Row;
|
|
|
+//import org.apache.poi.ss.usermodel.Sheet;
|
|
|
+//import org.apache.poi.ss.usermodel.Workbook;
|
|
|
+//import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
|
+//import org.springframework.scheduling.annotation.Scheduled;
|
|
|
+//import org.springframework.stereotype.Service;
|
|
|
+//import org.springframework.util.CollectionUtils;
|
|
|
+//import org.springframework.util.StringUtils;
|
|
|
+//
|
|
|
+//import java.io.IOException;
|
|
|
+//import java.io.InputStream;
|
|
|
+//import java.io.OutputStreamWriter;
|
|
|
+//import java.math.BigDecimal;
|
|
|
+//import java.math.RoundingMode;
|
|
|
+//import java.nio.charset.StandardCharsets;
|
|
|
+//import java.nio.file.Files;
|
|
|
+//import java.nio.file.Path;
|
|
|
+//import java.nio.file.Paths;
|
|
|
+//import java.nio.file.StandardCopyOption;
|
|
|
+//import java.time.LocalDate;
|
|
|
+//import java.time.format.DateTimeFormatter;
|
|
|
+//import java.time.temporal.ChronoUnit;
|
|
|
+//import java.util.ArrayList;
|
|
|
+//import java.util.LinkedHashMap;
|
|
|
+//import java.util.List;
|
|
|
+//import java.util.Map;
|
|
|
+//import java.util.Set;
|
|
|
+//import java.util.concurrent.CompletableFuture;
|
|
|
+//import java.util.concurrent.ConcurrentHashMap;
|
|
|
+//import java.util.concurrent.TimeUnit;
|
|
|
+//import java.util.function.Function;
|
|
|
+//import java.util.function.Predicate;
|
|
|
+//import java.util.regex.Matcher;
|
|
|
+//import java.util.regex.Pattern;
|
|
|
+//import java.util.stream.Stream;
|
|
|
+//
|
|
|
+///**
|
|
|
+// * 睿行车辆基本信息日数据入库定时任务
|
|
|
+// */
|
|
|
+//@Slf4j
|
|
|
+//@Service
|
|
|
+//public class CarBaseDataDayJob {
|
|
|
+// private final JobConfig jobConfig;
|
|
|
+// private final CarService carService;
|
|
|
+// private final OrganizationService organizationService;
|
|
|
+// private final AreaService areaService;
|
|
|
+// private final DataLogServiceImpl dataLogService;
|
|
|
+//
|
|
|
+// public CarBaseDataDayJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
|
|
|
+// AreaService areaService, DataLogServiceImpl dataLogService) {
|
|
|
+// this.jobConfig = jobConfig;
|
|
|
+// this.carService = carService;
|
|
|
+// this.organizationService = organizationService;
|
|
|
+// this.areaService = areaService;
|
|
|
+// this.dataLogService = dataLogService;
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 执行任务
|
|
|
+// */
|
|
|
+// @Scheduled(cron = "0 56 23 * * ?")
|
|
|
+// public void runJob() {
|
|
|
+// List<DataLogPo> dataLogPoList = new ArrayList<>();
|
|
|
+// // 数据目录
|
|
|
+// Path dir = Paths.get(jobConfig.getCarBaseDataDaySourcePath());
|
|
|
+// try (Stream<Path> stream = Files.list(dir)) {
|
|
|
+// // 获取数据目录下的文件列表
|
|
|
+// List<Path> pathList = stream.filter(t -> t.toString().endsWith(".xlsx")).sorted().toList();
|
|
|
+// log.info("睿行车辆基本信息日数据文件列表: {}", pathList);
|
|
|
+// if (CollectionUtils.isEmpty(pathList)) {
|
|
|
+// throw new MyRuntimeException("睿行车辆基本信息日数据没有文件");
|
|
|
+// }
|
|
|
+// for (Path path : pathList) {
|
|
|
+// CompletableFuture.runAsync(() -> {
|
|
|
+// try {
|
|
|
+// singleJob(path, dataLogPoList);
|
|
|
+// } catch (Exception e) {
|
|
|
+// throw new MyRuntimeException(e);
|
|
|
+// }
|
|
|
+// }).get(1, TimeUnit.MINUTES);
|
|
|
+// }
|
|
|
+// } catch (InterruptedException e) {
|
|
|
+// log.error("线程中断: {}", e, e);
|
|
|
+// Thread.currentThread().interrupt();
|
|
|
+// DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆基本信息日数据", DataStageEnum.CC, "入库");
|
|
|
+// } catch (Exception e) {
|
|
|
+// log.error(e.toString(), e);
|
|
|
+// DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆基本信息日数据", DataStageEnum.CC, "入库");
|
|
|
+// } finally {
|
|
|
+// if (!CollectionUtils.isEmpty(dataLogPoList)) {
|
|
|
+// dataLogService.saveBatch(dataLogPoList);
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 处理单个文件
|
|
|
+// *
|
|
|
+// * @param path 文件路径
|
|
|
+// * @param dataLogPoList 数据处理日志列表
|
|
|
+// */
|
|
|
+// public void singleJob(Path path, List<DataLogPo> dataLogPoList) throws IOException {
|
|
|
+// DataLogPo dataLogPo = new DataLogPo();
|
|
|
+// dataLogPo.setDataName("睿行车辆基本信息日数据");
|
|
|
+// dataLogPo.setObject(path.getFileName().toString());
|
|
|
+// dataLogPo.setStage(DataStageEnum.CC.value);
|
|
|
+// dataLogPo.setOperationName("入库");
|
|
|
+// long fileSize = Files.size(path);
|
|
|
+// dataLogPo.setDataSize(fileSize);
|
|
|
+// dataLogPoList.add(dataLogPo);
|
|
|
+// if (fileSize == 0) {
|
|
|
+// move(path);
|
|
|
+// throw new MyRuntimeException(path.getFileName() + " 空文件");
|
|
|
+// }
|
|
|
+// List<Map<String, String>> list = readFile(path);
|
|
|
+// List<Map<String, String>> distinctList = dataProcessing(path, list);
|
|
|
+// dataLogPo.setDataCount(distinctList.size());
|
|
|
+// Path csvPath = toCsv(path, distinctList);
|
|
|
+// copyCsv(csvPath);
|
|
|
+// move(path);
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 读取文件
|
|
|
+// *
|
|
|
+// * @param path 文件路径
|
|
|
+// */
|
|
|
+// public List<Map<String, String>> readFile(Path path) throws IOException {
|
|
|
+// log.info("读取: {}", path);
|
|
|
+// List<String> rawHeaders = Stream.of("车牌号", "车辆品牌", "排量", "排量类型", "燃油类型", "燃油标号",
|
|
|
+// "登记日期", "车辆来源", "车辆类型", "车辆使用性质", "环保等级", "用车模式", "发动机号", "车架号", "初始里程",
|
|
|
+// "车辆所属单位", "行驶证号", "备注", "审核状态", "车辆状态", "资产编号", "绑定驾驶员姓名", "购置价格(万元)",
|
|
|
+// "车龄(年)", "标准百公里油耗", "设备工作模式").toList();
|
|
|
+// List<String> headers = Stream.of("che_pai_hao", "che_liang_pin_pai", "pai_liang", "pai_liang_lei_xing",
|
|
|
+// "ran_you_lei_xing", "ran_you_biao_hao", "deng_ji_ri_qi", "che_liang_lai_yuan", "che_liang_lei_xing",
|
|
|
+// "che_liang_shi_yong_xing_zhi", "huan_bao_deng_ji", "yong_che_mo_shi", "fa_dong_ji_hao", "che_jia_hao",
|
|
|
+// "chu_shi_li_cheng", "che_liang_suo_shu_dan_wei", "xing_shi_zheng_hao", "bei_zhu", "shen_he_zhuang_tai",
|
|
|
+// "che_liang_zhuang_tai", "zi_chan_bian_hao", "bang_ding_jia_shi_yuan_xing_ming", "gou_zhi_jia_ge",
|
|
|
+// "che_ling_nian", "biao_zhun_bai_gong_li_you_hao", "she_bei_gong_zuo_mo_shi").toList();
|
|
|
+// try (InputStream inputStream = Files.newInputStream(path);
|
|
|
+// Workbook workbook = new XSSFWorkbook(inputStream)
|
|
|
+// ) {
|
|
|
+// List<Map<String, String>> resultList = new ArrayList<>();
|
|
|
+// // 读取第一个工作表
|
|
|
+// Sheet sheet = workbook.getSheetAt(0);
|
|
|
+// // 表头行
|
|
|
+// Row headerRow = sheet.getRow(0);
|
|
|
+// // 列数
|
|
|
+// int columnCount = headerRow.getPhysicalNumberOfCells();
|
|
|
+// log.info("columnCount: {}", columnCount);
|
|
|
+// // 检查表头
|
|
|
+// if (headers.size() != columnCount) {
|
|
|
+// throw new MyRuntimeException(path.getFileName() + "列数错误");
|
|
|
+// }
|
|
|
+// for (int i = 0; i < columnCount; i++) {
|
|
|
+// Cell cell = headerRow.getCell(i);
|
|
|
+// if (cell == null || !rawHeaders.get(i).equals(cell.getStringCellValue())) {
|
|
|
+// throw new MyRuntimeException(path.getFileName() + " 表头错误");
|
|
|
+// }
|
|
|
+// }
|
|
|
+// // 最后行数
|
|
|
+// int lastRowNum = sheet.getLastRowNum();
|
|
|
+// log.info("lastRowNum: {}", lastRowNum);
|
|
|
+// if (lastRowNum == 0) {
|
|
|
+// move(path);
|
|
|
+// throw new MyRuntimeException(path.getFileName() + " 数据0条");
|
|
|
+// }
|
|
|
+// // 遍历行
|
|
|
+// for (int i = 1; i <= lastRowNum; i++) {
|
|
|
+// log.debug("row: {}", i);
|
|
|
+// Row row = sheet.getRow(i);
|
|
|
+// if (row == null || row.getCell(0) == null) {
|
|
|
+// continue;
|
|
|
+// }
|
|
|
+// Map<String, String> rowMap = new LinkedHashMap<>();
|
|
|
+// // 遍历列
|
|
|
+// for (int j = 0; j < columnCount; j++) {
|
|
|
+// log.debug("column: {}", j);
|
|
|
+// String cellValue = "";
|
|
|
+// rowMap.put(headers.get(j), cellValue);
|
|
|
+// Cell cell = row.getCell(j);
|
|
|
+// if (cell == null) {
|
|
|
+// continue;
|
|
|
+// }
|
|
|
+// switch (cell.getCellType()) {
|
|
|
+// case STRING:
|
|
|
+// // 删除字符串空白字符
|
|
|
+// cellValue = StringUtils.trimAllWhitespace(cell.getStringCellValue());
|
|
|
+// break;
|
|
|
+// case NUMERIC:
|
|
|
+// if (DateUtil.isCellDateFormatted(cell)) {
|
|
|
+// cellValue = DateUtil.getLocalDateTime(cell.getNumericCellValue())
|
|
|
+// .format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
+// break;
|
|
|
+// }
|
|
|
+// cellValue = String.valueOf(cell.getNumericCellValue());
|
|
|
+// break;
|
|
|
+// case BOOLEAN:
|
|
|
+// cellValue = String.valueOf(cell.getBooleanCellValue());
|
|
|
+// break;
|
|
|
+// default:
|
|
|
+// break;
|
|
|
+// }
|
|
|
+// rowMap.put(headers.get(j), cellValue);
|
|
|
+// }
|
|
|
+// resultList.add(rowMap);
|
|
|
+// }
|
|
|
+// return resultList;
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 数据加工
|
|
|
+// *
|
|
|
+// * @param path 文件路径
|
|
|
+// * @param list 数据
|
|
|
+// */
|
|
|
+// public List<Map<String, String>> dataProcessing(Path path, List<Map<String, String>> list) {
|
|
|
+// // 从文件名提取日期
|
|
|
+// String regex = "\\d{8}";
|
|
|
+// Pattern pattern = Pattern.compile(regex);
|
|
|
+// Matcher matcher = pattern.matcher(path.getFileName().toString());
|
|
|
+// String dateString;
|
|
|
+// if (matcher.find()) {
|
|
|
+// dateString = matcher.group();
|
|
|
+// } else {
|
|
|
+// throw new MyRuntimeException(path.getFileName() + " 提取日期失败");
|
|
|
+// }
|
|
|
+// List<OrganizationPo> secondOrgs = organizationService.getSecondOrgs();
|
|
|
+// List<OrganizationPo> thirdOrgs = organizationService.getThirdOrgs();
|
|
|
+// Map<String, OrganizationPo> orgMap = organizationService.getOrgMap(secondOrgs, thirdOrgs);
|
|
|
+// Map<String, List<OrganizationPo>> thirdOrganizationListMap =
|
|
|
+// organizationService.getThirdOrganizationListMap(secondOrgs, thirdOrgs);
|
|
|
+// List<AreaPo> cities = areaService.getCities();
|
|
|
+// List<AreaPo> districts = areaService.getDistricts();
|
|
|
+// Map<String, AreaPo> areaMap = areaService.getAreaMap(cities, districts);
|
|
|
+// Map<String, List<AreaPo>> districtListMap = areaService.getDistrictListMap(cities, districts);
|
|
|
+// LocalDate localDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyyMMdd"));
|
|
|
+// LocalDate lastMonthDate = localDate.minusMonths(1);
|
|
|
+// String dataDate = localDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
+// String yearMonth = localDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
+// String year = String.valueOf(localDate.getYear());
|
|
|
+// String month = String.valueOf(localDate.getMonthValue());
|
|
|
+// String lastMonth = lastMonthDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
+// for (Map<String, String> map : list) {
|
|
|
+// map.put("data_date", dataDate);
|
|
|
+// map.put("year_month", yearMonth);
|
|
|
+// map.put("year_no", year);
|
|
|
+// map.put("month_no", month);
|
|
|
+// map.put("last_month", lastMonth);
|
|
|
+// String rawChePaiHao = map.get("che_pai_hao");
|
|
|
+// map.put("raw_che_pai_hao", rawChePaiHao);
|
|
|
+// String chePaiHao = carService.getChePai(rawChePaiHao);
|
|
|
+// map.put("che_pai_hao", chePaiHao);
|
|
|
+// String chePaiFail = carService.chePaiFail(rawChePaiHao);
|
|
|
+// map.put("che_pai_fail", chePaiFail);
|
|
|
+// String cheLiangSuoShuDanWei = map.get("che_liang_suo_shu_dan_wei");
|
|
|
+// CarUnitBo carUnitBo = carService.getCarUnitBo(cheLiangSuoShuDanWei);
|
|
|
+// String firstUnit = carService.getFirstUnit(carUnitBo);
|
|
|
+// if (!StringUtils.hasText(firstUnit)) {
|
|
|
+// throw new MyRuntimeException("车辆单位错误:" + rawChePaiHao + " " + cheLiangSuoShuDanWei);
|
|
|
+// }
|
|
|
+// map.put("first_unit", firstUnit);
|
|
|
+// String secondUnit = carService.getSecondUnit(carUnitBo, firstUnit);
|
|
|
+// map.put("second_unit", secondUnit);
|
|
|
+// String thirdUnit = carService.getThirdUnit(carUnitBo, secondUnit);
|
|
|
+// map.put("third_unit", thirdUnit);
|
|
|
+// String areaNo = carService.getAreaNo(secondOrgs, firstUnit);
|
|
|
+// map.put("area_no", areaNo);
|
|
|
+// String areaName = carService.getOrgName(orgMap, areaNo);
|
|
|
+// map.put("area_name", areaName);
|
|
|
+// String cityNo = carService.getCityNo(thirdOrganizationListMap, areaNo, areaName, carUnitBo);
|
|
|
+// map.put("city_no", cityNo);
|
|
|
+// String cityName = carService.getOrgName(orgMap, cityNo);
|
|
|
+// map.put("city_name", cityName);
|
|
|
+// String areaNo2 = carService.getAreaNo2(areaName, cityName);
|
|
|
+// map.put("area_no2", areaNo2);
|
|
|
+// String areaName2 = carService.getOrgName(orgMap, areaNo2);
|
|
|
+// map.put("area_name2", areaName2);
|
|
|
+// String cityId = carService.getCityId(cities, cheLiangSuoShuDanWei);
|
|
|
+// map.put("city_id", cityId);
|
|
|
+// String city = carService.getAreaName(areaMap, cityId);
|
|
|
+// map.put("city", city);
|
|
|
+// String districtId = carService.getDistrictId(districtListMap, cityId, cityName, cheLiangSuoShuDanWei);
|
|
|
+// map.put("district_id", districtId);
|
|
|
+// String district = carService.getAreaName(areaMap, districtId);
|
|
|
+// map.put("district", district);
|
|
|
+// String cheLing = "0";
|
|
|
+// String dengJiRiQi = map.get("deng_ji_ri_qi");
|
|
|
+// if (StringUtils.hasText(dengJiRiQi)) {
|
|
|
+// LocalDate dengJiRiQiDate = LocalDate.parse(dengJiRiQi, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
+// long days = ChronoUnit.DAYS.between(dengJiRiQiDate, localDate);
|
|
|
+// if (days > 0) {
|
|
|
+// BigDecimal n = BigDecimal.valueOf(days).divide(BigDecimal.valueOf(365), 1, RoundingMode.HALF_UP);
|
|
|
+// cheLing = n.toString();
|
|
|
+// }
|
|
|
+// }
|
|
|
+// map.put("che_ling", cheLing);
|
|
|
+// map.put("source", path.getFileName().toString());
|
|
|
+// }
|
|
|
+// // 去重
|
|
|
+// return list.stream().filter(distinctByKey(map -> map.get("che_pai_hao"))).toList();
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 去重
|
|
|
+// */
|
|
|
+// private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
|
|
|
+// Set<Object> set = ConcurrentHashMap.newKeySet();
|
|
|
+// return t -> set.add(keyExtractor.apply(t));
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 生成csv
|
|
|
+// *
|
|
|
+// * @param path 源文件路径
|
|
|
+// * @param list 数据
|
|
|
+// */
|
|
|
+// public Path toCsv(Path path, List<Map<String, String>> list) throws IOException {
|
|
|
+// log.info("去重后条数:{}", list.size());
|
|
|
+// Files.createDirectories(Paths.get(jobConfig.getCarBaseDataDayHistoryPath()));
|
|
|
+// Path csvPath = Paths.get(jobConfig.getCarBaseDataDayHistoryPath() + path.getFileName() + ".csv");
|
|
|
+// try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(csvPath),
|
|
|
+// StandardCharsets.UTF_8);
|
|
|
+// CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
|
|
|
+// // 添加bom头避免excel乱码
|
|
|
+// osw.write('\ufeff');
|
|
|
+// Map<String, String> header = list.get(0);
|
|
|
+// // 表头
|
|
|
+// printer.printRecord(header.keySet());
|
|
|
+// for (Map<String, String> map : list) {
|
|
|
+// printer.printRecord(map.values());
|
|
|
+// }
|
|
|
+// }
|
|
|
+// return csvPath;
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 导入数据库
|
|
|
+// * @param path 文件路径
|
|
|
+// */
|
|
|
+// public void copyCsv(Path path) {
|
|
|
+// String dbTable = "car.car_base_data_day";
|
|
|
+// String csv = path.toString();
|
|
|
+// String columns = "(che_pai_hao,che_liang_pin_pai,pai_liang,pai_liang_lei_xing,ran_you_lei_xing,ran_you_biao_hao,deng_ji_ri_qi,che_liang_lai_yuan,che_liang_lei_xing,che_liang_shi_yong_xing_zhi,huan_bao_deng_ji,yong_che_mo_shi,fa_dong_ji_hao,che_jia_hao,chu_shi_li_cheng,che_liang_suo_shu_dan_wei,xing_shi_zheng_hao,bei_zhu,shen_he_zhuang_tai,che_liang_zhuang_tai,zi_chan_bian_hao,bang_ding_jia_shi_yuan_xing_ming,gou_zhi_jia_ge,che_ling_nian,biao_zhun_bai_gong_li_you_hao,she_bei_gong_zuo_mo_shi,data_date,year_month,year_no,month_no,last_month,raw_che_pai_hao,che_pai_fail,first_unit,second_unit,third_unit,area_no,area_name,city_no,city_name,area_no2,area_name2,city_id,city,district_id,district,che_ling,source)";
|
|
|
+// Long timeout = 60000L;
|
|
|
+// PsqlUtil.copyCsv(jobConfig.getCopyScriptPath(), jobConfig.getDbHost(), jobConfig.getDbPort(),
|
|
|
+// jobConfig.getDbUsername(), jobConfig.getDbPassword(), jobConfig.getDbName(), dbTable, csv, columns,
|
|
|
+// timeout, null);
|
|
|
+// }
|
|
|
+//
|
|
|
+// /**
|
|
|
+// * 移动源文件到历史文件夹
|
|
|
+// * @param path 源文件路径
|
|
|
+// */
|
|
|
+// public void move(Path path) throws IOException {
|
|
|
+// Path targetPath = Paths.get(jobConfig.getCarBaseDataDayHistoryPath(), path.getFileName().toString());
|
|
|
+// Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
|
|
+// }
|
|
|
+//}
|