|
|
@@ -0,0 +1,321 @@
|
|
|
+package com.nokia.finance.tasks.jobs.car.ruixing;
|
|
|
+
|
|
|
+import com.nokia.finance.tasks.common.exception.MyRuntimeException;
|
|
|
+import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
|
|
|
+import com.nokia.finance.tasks.config.JobConfig;
|
|
|
+import com.nokia.finance.tasks.pojo.po.common.AreaPo;
|
|
|
+import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
|
|
|
+import com.nokia.finance.tasks.service.car.CarService;
|
|
|
+import com.nokia.finance.tasks.service.common.AreaService;
|
|
|
+import com.nokia.finance.tasks.service.common.OrganizationService;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.csv.CSVFormat;
|
|
|
+import org.apache.commons.csv.CSVPrinter;
|
|
|
+import org.apache.poi.ss.usermodel.Cell;
|
|
|
+import org.apache.poi.ss.usermodel.DateUtil;
|
|
|
+import org.apache.poi.ss.usermodel.Row;
|
|
|
+import org.apache.poi.ss.usermodel.Sheet;
|
|
|
+import org.apache.poi.ss.usermodel.Workbook;
|
|
|
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
|
+import org.springframework.scheduling.annotation.Scheduled;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+import org.springframework.util.CollectionUtils;
|
|
|
+import org.springframework.util.StringUtils;
|
|
|
+
|
|
|
+import java.io.InputStream;
|
|
|
+import java.io.OutputStreamWriter;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.nio.file.Files;
|
|
|
+import java.nio.file.Path;
|
|
|
+import java.nio.file.Paths;
|
|
|
+import java.nio.file.StandardCopyOption;
|
|
|
+import java.time.LocalDate;
|
|
|
+import java.time.format.DateTimeFormatter;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.LinkedHashMap;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.Set;
|
|
|
+import java.util.concurrent.ConcurrentHashMap;
|
|
|
+import java.util.function.Function;
|
|
|
+import java.util.function.Predicate;
|
|
|
+import java.util.regex.Matcher;
|
|
|
+import java.util.regex.Pattern;
|
|
|
+import java.util.stream.Stream;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 睿行车辆行驶里程月数据入库定时任务
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+@Service
|
|
|
+public class XslcytjJob {
|
|
|
+ private final JobConfig jobConfig;
|
|
|
+ private final CarService carService;
|
|
|
+ private final OrganizationService organizationService;
|
|
|
+ private final AreaService areaService;
|
|
|
+
|
|
|
+ public XslcytjJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
|
|
|
+ AreaService areaService) {
|
|
|
+ this.jobConfig = jobConfig;
|
|
|
+ this.carService = carService;
|
|
|
+ this.organizationService = organizationService;
|
|
|
+ this.areaService = areaService;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 执行任务
|
|
|
+ */
|
|
|
+ @Scheduled(cron = "0 35 23 * * ?")
|
|
|
+ public void runJob() {
|
|
|
+ // 数据目录
|
|
|
+ Path dir = Paths.get(jobConfig.getXslcytjSourcePath());
|
|
|
+ try (Stream<Path> stream = Files.list(dir)) {
|
|
|
+ // 获取数据目录下的文件列表
|
|
|
+ List<Path> pathList = stream.filter(t -> t.toString().endsWith(".xlsx")).sorted().toList();
|
|
|
+ log.info("睿行车辆行驶里程月数据文件列表: {}", pathList);
|
|
|
+ if (CollectionUtils.isEmpty(pathList)) {
|
|
|
+ throw new MyRuntimeException("睿行车辆行驶里程月数据没有文件");
|
|
|
+ }
|
|
|
+ for (Path path : pathList) {
|
|
|
+ singleJob(path);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error(e.getMessage(), e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 处理单个文件
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public void singleJob(Path path) throws Exception {
|
|
|
+ List<Map<String, String>> list = readFile(path);
|
|
|
+ List<Map<String, String>> distinctList = dataProcessing(path, list);
|
|
|
+ Path csvPath = toCsv(path, distinctList);
|
|
|
+ copyCsv(csvPath);
|
|
|
+ move(path);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 读取文件
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public List<Map<String, String>> readFile(Path path) throws Exception {
|
|
|
+ log.info("读取: {}", path);
|
|
|
+ List<String> rawHeaders = Stream.of("车牌号码", "车辆所属单位", "车辆类型", "车辆来源", "车辆使用性质",
|
|
|
+ "行驶时长(小时)", "行驶天数", "总里程(公里)", "OBD总油耗(升)", "百公里油耗(升)").toList();
|
|
|
+ List<String> headers = Stream.of("che_pai_hao", "che_liang_suo_shu_dan_wei", "che_liang_lei_xing",
|
|
|
+ "che_liang_lai_yuan", "che_liang_shi_yong_xing_zhi", "xing_shi_shi_chang", "xing_shi_tian_shu",
|
|
|
+ "zong_li_cheng", "obd_you_hao", "bai_gong_li_you_hao").toList();
|
|
|
+ try (InputStream inputStream = Files.newInputStream(path);
|
|
|
+ Workbook workbook = new XSSFWorkbook(inputStream)
|
|
|
+ ) {
|
|
|
+ List<Map<String, String>> resultList = new ArrayList<>();
|
|
|
+ // 读取第一个工作表
|
|
|
+ Sheet sheet = workbook.getSheetAt(0);
|
|
|
+ // 表头行
|
|
|
+ Row headerRow = sheet.getRow(0);
|
|
|
+ // 列数
|
|
|
+ int columnCount = headerRow.getPhysicalNumberOfCells();
|
|
|
+ log.info("columnCount: {}", columnCount);
|
|
|
+ // 检查表头
|
|
|
+ if (headers.size() != columnCount) {
|
|
|
+ throw new MyRuntimeException(path.getFileName() + "列数错误");
|
|
|
+ }
|
|
|
+ for (int i = 0; i < columnCount; i++) {
|
|
|
+ Cell cell = headerRow.getCell(i);
|
|
|
+ if (cell == null || !rawHeaders.get(i).equals(cell.getStringCellValue())) {
|
|
|
+ throw new MyRuntimeException(path.getFileName() + " 表头错误");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 最后行数
|
|
|
+ int lastRowNum = sheet.getLastRowNum();
|
|
|
+ log.info("lastRowNum: {}", lastRowNum);
|
|
|
+ if (lastRowNum == 0) {
|
|
|
+ throw new MyRuntimeException(path.getFileName() + " 为空");
|
|
|
+ }
|
|
|
+ // 遍历行
|
|
|
+ for (int i = 1; i <= lastRowNum; i++) {
|
|
|
+ Row row = sheet.getRow(i);
|
|
|
+ if (row == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Map<String, String> rowMap = new LinkedHashMap<>();
|
|
|
+ // 遍历列
|
|
|
+ for (int j = 0; j < columnCount; j++) {
|
|
|
+ String cellValue = "";
|
|
|
+ rowMap.put(headers.get(j), cellValue);
|
|
|
+ Cell cell = row.getCell(j);
|
|
|
+ if (cell == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ switch (cell.getCellType()) {
|
|
|
+ case STRING:
|
|
|
+ // 删除字符串空白字符
|
|
|
+ cellValue = StringUtils.trimAllWhitespace(cell.getStringCellValue());
|
|
|
+ break;
|
|
|
+ case NUMERIC:
|
|
|
+ if (DateUtil.isCellDateFormatted(cell)) {
|
|
|
+ cellValue = DateUtil.getLocalDateTime(cell.getNumericCellValue())
|
|
|
+ .format(DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss"));
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ cellValue = String.valueOf(cell.getNumericCellValue());
|
|
|
+ break;
|
|
|
+ case BOOLEAN:
|
|
|
+ cellValue = String.valueOf(cell.getBooleanCellValue());
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ rowMap.put(headers.get(j), cellValue);
|
|
|
+ }
|
|
|
+ resultList.add(rowMap);
|
|
|
+ }
|
|
|
+ return resultList;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 数据加工
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ * @param list 数据
|
|
|
+ */
|
|
|
+ public List<Map<String, String>> dataProcessing(Path path, List<Map<String, String>> list) {
|
|
|
+ // 从文件名提取日期
|
|
|
+ String regex = "\\d{6}";
|
|
|
+ Pattern pattern = Pattern.compile(regex);
|
|
|
+ Matcher matcher = pattern.matcher(path.getFileName().toString());
|
|
|
+ String dateString;
|
|
|
+ if (matcher.find()) {
|
|
|
+ dateString = matcher.group() + "01";
|
|
|
+ } else {
|
|
|
+ throw new MyRuntimeException(path.getFileName() + " 提取日期失败");
|
|
|
+ }
|
|
|
+ List<OrganizationPo> secondOrgs = organizationService.getSecondOrgs();
|
|
|
+ List<OrganizationPo> thirdOrgs = organizationService.getThirdOrgs();
|
|
|
+ Map<String, OrganizationPo> orgMap = organizationService.getOrgMap(secondOrgs, thirdOrgs);
|
|
|
+ Map<String, List<OrganizationPo>> thirdOrganizationListMap =
|
|
|
+ organizationService.getThirdOrganizationListMap(secondOrgs, thirdOrgs);
|
|
|
+ List<AreaPo> cities = areaService.getCities();
|
|
|
+ List<AreaPo> districts = areaService.getDistricts();
|
|
|
+ Map<String, AreaPo> areaMap = areaService.getAreaMap(cities, districts);
|
|
|
+ Map<String, List<AreaPo>> districtListMap = areaService.getDistrictListMap(cities, districts);
|
|
|
+ LocalDate localDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyyMMdd")).minusMonths(1);
|
|
|
+ LocalDate lastMonthDate = localDate.minusMonths(1);
|
|
|
+ String dataDate = localDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
|
|
+ String yearMonth = localDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
+ String year = String.valueOf(localDate.getYear());
|
|
|
+ String month = String.valueOf(localDate.getMonthValue());
|
|
|
+ String lastMonth = lastMonthDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
+ for (Map<String, String> map : list) {
|
|
|
+ map.put("data_date", dataDate);
|
|
|
+ map.put("year_month", yearMonth);
|
|
|
+ map.put("year_no", year);
|
|
|
+ map.put("month_no", month);
|
|
|
+ map.put("last_month", lastMonth);
|
|
|
+ String rawChePaiHao = map.get("che_pai_hao");
|
|
|
+ map.put("raw_che_pai_hao", rawChePaiHao);
|
|
|
+ String chePaiHao = carService.getChePai(rawChePaiHao);
|
|
|
+ map.put("che_pai_hao", chePaiHao);
|
|
|
+ String chePaiFail = carService.chePaiFail(rawChePaiHao);
|
|
|
+ map.put("che_pai_fail", chePaiFail);
|
|
|
+ String cheLiangSuoShuDanWei = map.get("che_liang_suo_shu_dan_wei");
|
|
|
+ String firstUnit = carService.getFirstUnit(cheLiangSuoShuDanWei);
|
|
|
+ map.put("first_unit", firstUnit);
|
|
|
+ String secondUnit = carService.getSecondUnit(cheLiangSuoShuDanWei, firstUnit);
|
|
|
+ map.put("second_unit", secondUnit);
|
|
|
+ String thirdUnit = carService.getThirdUnit(cheLiangSuoShuDanWei, secondUnit);
|
|
|
+ map.put("third_unit", thirdUnit);
|
|
|
+ String areaNo = carService.getAreaNo(secondOrgs, cheLiangSuoShuDanWei);
|
|
|
+ map.put("area_no", areaNo);
|
|
|
+ String areaName = carService.getOrgName(orgMap, areaNo);
|
|
|
+ map.put("area_name", areaName);
|
|
|
+ String cityNo = carService.getCityNo(thirdOrganizationListMap, areaNo, areaName, cheLiangSuoShuDanWei);
|
|
|
+ map.put("city_no", cityNo);
|
|
|
+ String cityName = carService.getOrgName(orgMap, cityNo);
|
|
|
+ map.put("city_name", cityName);
|
|
|
+ String areaNo2 = carService.getAreaNo2(areaName, cityName);
|
|
|
+ map.put("area_no2", areaNo2);
|
|
|
+ String areaName2 = carService.getOrgName(orgMap, areaNo2);
|
|
|
+ map.put("area_name2", areaName2);
|
|
|
+ String cityId = carService.getCityId(cities, cheLiangSuoShuDanWei);
|
|
|
+ map.put("city_id", cityId);
|
|
|
+ String city = carService.getAreaName(areaMap, cityId);
|
|
|
+ map.put("city", city);
|
|
|
+ String districtId = carService.getDistrictId(districtListMap, cityId, cityName, cheLiangSuoShuDanWei);
|
|
|
+ map.put("district_id", districtId);
|
|
|
+ String district = carService.getAreaName(areaMap, districtId);
|
|
|
+ map.put("district", district);
|
|
|
+ String baoFei = carService.baoFei(rawChePaiHao);
|
|
|
+ map.put("bao_fei", baoFei);
|
|
|
+ if ("1".equals(baoFei)) {
|
|
|
+ map.put("che_pai_hao", rawChePaiHao);
|
|
|
+ }
|
|
|
+ map.put("source", path.getFileName().toString());
|
|
|
+ }
|
|
|
+ // 去重
|
|
|
+ return list.stream().filter(distinctByKey(map -> map.get("che_pai_hao"))).toList();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 去重
|
|
|
+ */
|
|
|
+ private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
|
|
|
+ Set<Object> set = ConcurrentHashMap.newKeySet();
|
|
|
+ return t -> set.add(keyExtractor.apply(t));
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 生成csv
|
|
|
+ *
|
|
|
+ * @param path 源文件路径
|
|
|
+ * @param list 数据
|
|
|
+ */
|
|
|
+ public Path toCsv(Path path, List<Map<String, String>> list) throws Exception {
|
|
|
+ log.info("去重后条数：{}", list.size());
|
|
|
+ Files.createDirectories(Paths.get(jobConfig.getXslcytjHistoryPath()));
|
|
|
+ Path csvPath = Paths.get(jobConfig.getXslcytjHistoryPath() + path.getFileName() + ".csv");
|
|
|
+ try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(csvPath),
|
|
|
+ StandardCharsets.UTF_8);
|
|
|
+ CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
|
|
|
+ // 添加bom头避免excel乱码
|
|
|
+ osw.write('\ufeff');
|
|
|
+ Map<String, String> header = list.get(0);
|
|
|
+ // 表头
|
|
|
+ printer.printRecord(header.keySet());
|
|
|
+ for (Map<String, String> map : list) {
|
|
|
+ printer.printRecord(map.values());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return csvPath;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 导入数据库
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public void copyCsv(Path path) {
|
|
|
+ String dbTable = "car.car_li_cheng_month";
|
|
|
+ String csv = path.toString();
|
|
|
+ String columns = "(che_pai_hao,che_liang_suo_shu_dan_wei,che_liang_lei_xing,che_liang_lai_yuan,che_liang_shi_yong_xing_zhi,xing_shi_shi_chang,xing_shi_tian_shu,zong_li_cheng,obd_you_hao,bai_gong_li_you_hao,data_date,year_month,year_no,month_no,last_month,raw_che_pai_hao,che_pai_fail,first_unit,second_unit,third_unit,area_no,area_name,city_no,city_name,area_no2,area_name2,city_id,city,district_id,district,bao_fei,source)";
|
|
|
+ Long timeout = 60000L;
|
|
|
+ PsqlUtil.copyCsv(jobConfig.getCopyScriptPath(), jobConfig.getDbHost(), jobConfig.getDbPort(),
|
|
|
+ jobConfig.getDbUsername(), jobConfig.getDbPassword(), jobConfig.getDbName(), dbTable, csv, columns,
|
|
|
+ timeout, null);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 移动源文件到历史文件夹
|
|
|
+ *
|
|
|
+ * @param path 源文件路径
|
|
|
+ */
|
|
|
+ public void move(Path path) throws Exception {
|
|
|
+ Path targetPath = Paths.get(jobConfig.getXslcytjHistoryPath(), path.getFileName().toString());
|
|
|
+ Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
|
|
+ }
|
|
|
+}
|
weijianghai