|
@@ -3,10 +3,13 @@ package com.nokia.finance.tasks.jobs.car.ruixing;
|
|
|
import com.nokia.finance.tasks.common.exception.MyRuntimeException;
|
|
|
import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
|
|
|
import com.nokia.finance.tasks.config.JobConfig;
|
|
|
+import com.nokia.finance.tasks.enums.DataStageEnum;
|
|
|
import com.nokia.finance.tasks.pojo.po.common.AreaPo;
|
|
|
+import com.nokia.finance.tasks.pojo.po.common.DataLogPo;
|
|
|
import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
|
|
|
import com.nokia.finance.tasks.service.car.CarService;
|
|
|
import com.nokia.finance.tasks.service.common.AreaService;
|
|
|
+import com.nokia.finance.tasks.service.common.DataLogServiceImpl;
|
|
|
import com.nokia.finance.tasks.service.common.OrganizationService;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.csv.CSVFormat;
|
|
@@ -22,6 +25,7 @@ import org.springframework.stereotype.Service;
|
|
|
import org.springframework.util.CollectionUtils;
|
|
|
import org.springframework.util.StringUtils;
|
|
|
|
|
|
+import java.io.IOException;
|
|
|
import java.io.InputStream;
|
|
|
import java.io.OutputStreamWriter;
|
|
|
import java.nio.charset.StandardCharsets;
|
|
@@ -52,13 +56,15 @@ public class CarYongCheJob {
|
|
|
private final CarService carService;
|
|
|
private final OrganizationService organizationService;
|
|
|
private final AreaService areaService;
|
|
|
+ private final DataLogServiceImpl dataLogService;
|
|
|
|
|
|
public CarYongCheJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
|
|
|
- AreaService areaService) {
|
|
|
+ AreaService areaService, DataLogServiceImpl dataLogService) {
|
|
|
this.jobConfig = jobConfig;
|
|
|
this.carService = carService;
|
|
|
this.organizationService = organizationService;
|
|
|
this.areaService = areaService;
|
|
|
+ this.dataLogService = dataLogService;
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -66,6 +72,7 @@ public class CarYongCheJob {
|
|
|
*/
|
|
|
@Scheduled(cron = "0 58 5 1 * ?")
|
|
|
public void runJob() {
|
|
|
+ List<DataLogPo> dataLogPoList = new ArrayList<>();
|
|
|
// 数据目录
|
|
|
Path dir = Paths.get(jobConfig.getCarYongCheSourcePath());
|
|
|
try (Stream<Path> stream = Files.list(dir)) {
|
|
@@ -78,26 +85,45 @@ public class CarYongCheJob {
|
|
|
for (Path path : pathList) {
|
|
|
CompletableFuture.runAsync(() -> {
|
|
|
try {
|
|
|
- singleJob(path);
|
|
|
+ singleJob(path, dataLogPoList);
|
|
|
} catch (Exception e) {
|
|
|
throw new MyRuntimeException(e);
|
|
|
}
|
|
|
}).get(1, TimeUnit.MINUTES);
|
|
|
}
|
|
|
} catch (InterruptedException e) {
|
|
|
- log.error("线程中断: {}", e.getMessage(), e);
|
|
|
+ log.error("线程中断: {}", e, e);
|
|
|
Thread.currentThread().interrupt();
|
|
|
+ DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆用车月统计数据", DataStageEnum.CC, "入库");
|
|
|
} catch (Exception e) {
|
|
|
- log.error(e.getMessage(), e);
|
|
|
+ log.error(e.toString(), e);
|
|
|
+ DataLogPo.addExceptionInfo(dataLogPoList, e, "睿行车辆用车月统计数据", DataStageEnum.CC, "入库");
|
|
|
+ } finally {
|
|
|
+ if (!CollectionUtils.isEmpty(dataLogPoList)) {
|
|
|
+ dataLogService.saveBatch(dataLogPoList);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 处理单个文件
|
|
|
*
|
|
|
- * @param path 文件路径
|
|
|
+ * @param path 文件路径
|
|
|
+ * @param dataLogPoList 数据处理日志列表
|
|
|
*/
|
|
|
- public void singleJob(Path path) throws Exception {
|
|
|
+ public void singleJob(Path path, List<DataLogPo> dataLogPoList) throws IOException {
|
|
|
+ DataLogPo dataLogPo = new DataLogPo();
|
|
|
+ dataLogPo.setDataName("睿行车辆用车月统计数据");
|
|
|
+ dataLogPo.setObject(path.getFileName().toString());
|
|
|
+ dataLogPo.setStage(DataStageEnum.CC.value);
|
|
|
+ dataLogPo.setOperationName("入库");
|
|
|
+ long fileSize = Files.size(path);
|
|
|
+ dataLogPo.setDataSize(fileSize);
|
|
|
+ dataLogPoList.add(dataLogPo);
|
|
|
+ if (fileSize == 0) {
|
|
|
+ move(path);
|
|
|
+ throw new MyRuntimeException(path.getFileName() + " 空文件");
|
|
|
+ }
|
|
|
// 从文件名提取日期
|
|
|
String regex = "\\d{6}";
|
|
|
Pattern pattern = Pattern.compile(regex);
|
|
@@ -108,12 +134,9 @@ public class CarYongCheJob {
|
|
|
} else {
|
|
|
throw new MyRuntimeException(path.getFileName() + " 提取日期失败");
|
|
|
}
|
|
|
- if (Files.size(path) == 0) {
|
|
|
- move(path);
|
|
|
- throw new MyRuntimeException(path.getFileName() + " 空文件");
|
|
|
- }
|
|
|
List<Map<String, String>> list = readFile(path);
|
|
|
List<Map<String, String>> distinctList = dataProcessing(path, list, dateString);
|
|
|
+ dataLogPo.setDataCount(distinctList.size());
|
|
|
Path csvPath = toCsv(path, distinctList);
|
|
|
copyCsv(csvPath);
|
|
|
move(path);
|
|
@@ -124,7 +147,7 @@ public class CarYongCheJob {
|
|
|
*
|
|
|
* @param path 文件路径
|
|
|
*/
|
|
|
- public List<Map<String, String>> readFile(Path path) throws Exception {
|
|
|
+ public List<Map<String, String>> readFile(Path path) throws IOException {
|
|
|
log.info("读取: {}", path);
|
|
|
List<String> rawHeaders = Stream.of("申请单号", "派遣单号", "申请类型", "车牌号", "车辆类型", "车辆级别",
|
|
|
"司机名称", "用车人", "用车部门", "用车事由", "出发地", "目的地", "出发时间", "归队时间", "行驶里程", "出车时长")
|
|
@@ -206,7 +229,7 @@ public class CarYongCheJob {
|
|
|
case NUMERIC:
|
|
|
if (DateUtil.isCellDateFormatted(cell)) {
|
|
|
cellValue = DateUtil.getLocalDateTime(cell.getNumericCellValue())
|
|
|
- .format(DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss"));
|
|
|
+ .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
|
|
break;
|
|
|
}
|
|
|
cellValue = String.valueOf(cell.getNumericCellValue());
|
|
@@ -307,7 +330,7 @@ public class CarYongCheJob {
|
|
|
* @param path 源文件路径
|
|
|
* @param list 数据
|
|
|
*/
|
|
|
- public Path toCsv(Path path, List<Map<String, String>> list) throws Exception {
|
|
|
+ public Path toCsv(Path path, List<Map<String, String>> list) throws IOException {
|
|
|
log.info("去重后条数:{}", list.size());
|
|
|
Files.createDirectories(Paths.get(jobConfig.getCarYongCheHistoryPath()));
|
|
|
Path csvPath = Paths.get(jobConfig.getCarYongCheHistoryPath() + path.getFileName() + ".csv");
|
|
@@ -346,7 +369,7 @@ public class CarYongCheJob {
|
|
|
*
|
|
|
* @param path 源文件路径
|
|
|
*/
|
|
|
- public void move(Path path) throws Exception {
|
|
|
+ public void move(Path path) throws IOException {
|
|
|
Path targetPath = Paths.get(jobConfig.getCarYongCheHistoryPath(), path.getFileName().toString());
|
|
|
Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
|
|
}
|