|
@@ -0,0 +1,244 @@
|
|
|
+package com.nokia.finance.tasks.jobs.car.chengben;
|
|
|
+
|
|
|
+import com.nokia.finance.tasks.common.exception.MyRuntimeException;
|
|
|
+import com.nokia.finance.tasks.common.utils.psql.PsqlUtil;
|
|
|
+import com.nokia.finance.tasks.config.JobConfig;
|
|
|
+import com.nokia.finance.tasks.pojo.po.common.AreaPo;
|
|
|
+import com.nokia.finance.tasks.pojo.po.common.OrganizationPo;
|
|
|
+import com.nokia.finance.tasks.service.car.CarService;
|
|
|
+import com.nokia.finance.tasks.service.common.AreaService;
|
|
|
+import com.nokia.finance.tasks.service.common.OrganizationService;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.csv.CSVFormat;
|
|
|
+import org.apache.commons.csv.CSVParser;
|
|
|
+import org.apache.commons.csv.CSVPrinter;
|
|
|
+import org.apache.commons.csv.CSVRecord;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+import org.springframework.util.CollectionUtils;
|
|
|
+import org.springframework.util.StringUtils;
|
|
|
+
|
|
|
+import java.io.InputStreamReader;
|
|
|
+import java.io.OutputStreamWriter;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.nio.file.Files;
|
|
|
+import java.nio.file.Path;
|
|
|
+import java.nio.file.Paths;
|
|
|
+import java.nio.file.StandardCopyOption;
|
|
|
+import java.time.LocalDateTime;
|
|
|
+import java.time.format.DateTimeFormatter;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.LinkedHashMap;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.stream.Stream;
|
|
|
+
|
|
|
+/**
|
|
|
+ * 河北成本管理系统车辆保险数据入库定时任务
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+@Service
|
|
|
+public class CarBaoXianJob {
|
|
|
+ private final JobConfig jobConfig;
|
|
|
+ private final CarService carService;
|
|
|
+ private final OrganizationService organizationService;
|
|
|
+ private final AreaService areaService;
|
|
|
+
|
|
|
+ public CarBaoXianJob(JobConfig jobConfig, CarService carService, OrganizationService organizationService,
|
|
|
+ AreaService areaService) {
|
|
|
+ this.jobConfig = jobConfig;
|
|
|
+ this.carService = carService;
|
|
|
+ this.organizationService = organizationService;
|
|
|
+ this.areaService = areaService;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 执行任务
|
|
|
+ */
|
|
|
+// @Scheduled(cron = "0 35 23 1 * ?")
|
|
|
+ public void runJob() {
|
|
|
+ // 数据目录
|
|
|
+ Path dir = Paths.get(jobConfig.getCarBaoXianSourcePath());
|
|
|
+ try (Stream<Path> stream = Files.list(dir)) {
|
|
|
+ // 获取数据目录下的文件列表
|
|
|
+ List<Path> pathList = stream.filter(t -> !t.toString().endsWith(".MD5")).sorted().toList();
|
|
|
+ log.info("河北成本管理系统车辆保险数据文件列表: {}", pathList);
|
|
|
+ if (CollectionUtils.isEmpty(pathList)) {
|
|
|
+ throw new MyRuntimeException("河北成本管理系统车辆保险数据没有文件");
|
|
|
+ }
|
|
|
+ for (Path path : pathList) {
|
|
|
+ singleJob(path);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error(e.getMessage(), e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 处理单个文件
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public void singleJob(Path path) throws Exception {
|
|
|
+ List<Map<String, String>> list = readFile(path);
|
|
|
+ List<Map<String, String>> distinctList = dataProcessing(path, list);
|
|
|
+ Path csvPath = toCsv(path, distinctList);
|
|
|
+ copyCsv(csvPath);
|
|
|
+ move(path);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 读取文件
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public List<Map<String, String>> readFile(Path path) throws Exception {
|
|
|
+ log.info("读取: {}", path);
|
|
|
+ List<String> headers = Stream.of(
|
|
|
+ "data_time_nlsd",
|
|
|
+ "data_time", "form_type", "submitter", "submitter_dept_id", "form_no", "reimbursement_no",
|
|
|
+ "start_time", "form_id", "license_plate_number", "motorcycle_type", "dept_id", "rent_start_time",
|
|
|
+ "rent_end_time", "supplier", "invoice_type", "jq_amount_excluding_tax", "cc_tax_amount",
|
|
|
+ "jq_total_amount", "sy_amount_excluding_tax", "sy_total_amount", "total_amount",
|
|
|
+ "commercial_insurance").toList();
|
|
|
+ int headerSize = headers.size();
|
|
|
+ char delimiter = 1;
|
|
|
+ try (CSVParser parser = CSVFormat.DEFAULT.builder().setDelimiter(delimiter).build()
|
|
|
+ .parse(new InputStreamReader(Files.newInputStream(path), StandardCharsets.UTF_8))
|
|
|
+ ) {
|
|
|
+ List<Map<String, String>> resultList = new ArrayList<>();
|
|
|
+ for (CSVRecord csvRecord : parser) {
|
|
|
+ Map<String, String> rowMap = new LinkedHashMap<>();
|
|
|
+ for (int i = 0; i < headerSize; i++) {
|
|
|
+ String header = headers.get(i);
|
|
|
+ String value = csvRecord.get(i);
|
|
|
+ if (!"start_time".equals(header)
|
|
|
+ && !"rent_start_time".equals(header)
|
|
|
+ && !"rent_end_time".equals(header)) {
|
|
|
+ // 删除空白字符
|
|
|
+ value = StringUtils.trimAllWhitespace(value);
|
|
|
+ }
|
|
|
+ rowMap.put(header, value);
|
|
|
+ }
|
|
|
+ resultList.add(rowMap);
|
|
|
+ }
|
|
|
+ return resultList;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 数据加工
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ * @param list 数据
|
|
|
+ */
|
|
|
+ public List<Map<String, String>> dataProcessing(Path path, List<Map<String, String>> list) {
|
|
|
+ List<OrganizationPo> secondOrgs = organizationService.getSecondOrgs();
|
|
|
+ List<OrganizationPo> thirdOrgs = organizationService.getThirdOrgs();
|
|
|
+ Map<String, OrganizationPo> orgMap = organizationService.getOrgMap(secondOrgs, thirdOrgs);
|
|
|
+ Map<String, List<OrganizationPo>> thirdOrganizationListMap =
|
|
|
+ organizationService.getThirdOrganizationListMap(secondOrgs, thirdOrgs);
|
|
|
+ List<AreaPo> cities = areaService.getCities();
|
|
|
+ List<AreaPo> districts = areaService.getDistricts();
|
|
|
+ Map<String, AreaPo> areaMap = areaService.getAreaMap(cities, districts);
|
|
|
+ Map<String, List<AreaPo>> districtListMap = areaService.getDistrictListMap(cities, districts);
|
|
|
+ for (Map<String, String> map : list) {
|
|
|
+ String startTime = map.get("start_time");
|
|
|
+ LocalDateTime localDateTime = LocalDateTime.parse(startTime,
|
|
|
+ DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
|
|
+ String yearMonth = localDateTime.format(DateTimeFormatter.ofPattern("yyyyMM"));
|
|
|
+ String year = String.valueOf(localDateTime.getYear());
|
|
|
+ String month = String.valueOf(localDateTime.getMonthValue());
|
|
|
+ map.put("year_month", yearMonth);
|
|
|
+ map.put("year_no", year);
|
|
|
+ map.put("month_no", month);
|
|
|
+ String rawChePaiHao = map.get("license_plate_number");
|
|
|
+ map.put("raw_license_plate_number", rawChePaiHao);
|
|
|
+ String chePaiHao = carService.getChePai(rawChePaiHao);
|
|
|
+ map.put("license_plate_number", chePaiHao);
|
|
|
+ String chePaiFail = carService.chePaiFail(rawChePaiHao);
|
|
|
+ map.put("che_pai_fail", chePaiFail);
|
|
|
+ String cheLiangSuoShuDanWei = map.get("dept_id");
|
|
|
+ String firstUnit = carService.getFirstUnit(cheLiangSuoShuDanWei);
|
|
|
+ map.put("first_unit", firstUnit);
|
|
|
+ String secondUnit = carService.getSecondUnit(cheLiangSuoShuDanWei, firstUnit);
|
|
|
+ map.put("second_unit", secondUnit);
|
|
|
+ String thirdUnit = carService.getThirdUnit(cheLiangSuoShuDanWei, secondUnit);
|
|
|
+ map.put("third_unit", thirdUnit);
|
|
|
+ String areaNo = carService.getAreaNo(secondOrgs, cheLiangSuoShuDanWei);
|
|
|
+ map.put("area_no", areaNo);
|
|
|
+ String areaName = carService.getOrgName(orgMap, areaNo);
|
|
|
+ map.put("area_name", areaName);
|
|
|
+ String cityNo = carService.getCityNo(thirdOrganizationListMap, areaNo, areaName, cheLiangSuoShuDanWei);
|
|
|
+ map.put("city_no", cityNo);
|
|
|
+ String cityName = carService.getOrgName(orgMap, cityNo);
|
|
|
+ map.put("city_name", cityName);
|
|
|
+ String areaNo2 = carService.getAreaNo2(areaName, cityName);
|
|
|
+ map.put("area_no2", areaNo2);
|
|
|
+ String areaName2 = carService.getOrgName(orgMap, areaNo2);
|
|
|
+ map.put("area_name2", areaName2);
|
|
|
+ String cityId = carService.getCityId(cities, cheLiangSuoShuDanWei);
|
|
|
+ map.put("city_id", cityId);
|
|
|
+ String city = carService.getAreaName(areaMap, cityId);
|
|
|
+ map.put("city", city);
|
|
|
+ String districtId = carService.getDistrictId(districtListMap, cityId, cityName, cheLiangSuoShuDanWei);
|
|
|
+ map.put("district_id", districtId);
|
|
|
+ String district = carService.getAreaName(areaMap, districtId);
|
|
|
+ map.put("district", district);
|
|
|
+ map.put("source", path.getFileName().toString());
|
|
|
+ }
|
|
|
+ return list;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 生成csv
|
|
|
+ *
|
|
|
+ * @param path 源文件路径
|
|
|
+ * @param list 数据
|
|
|
+ */
|
|
|
+ public Path toCsv(Path path, List<Map<String, String>> list) throws Exception {
|
|
|
+ log.info("条数:{}", list.size());
|
|
|
+ Files.createDirectories(Paths.get(jobConfig.getCarBaoXianHistoryPath()));
|
|
|
+ Path csvPath = Paths.get(jobConfig.getCarBaoXianHistoryPath() + path.getFileName() + ".csv");
|
|
|
+ try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(csvPath),
|
|
|
+ StandardCharsets.UTF_8);
|
|
|
+ CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
|
|
|
+ // 添加bom头避免excel乱码
|
|
|
+ osw.write('\ufeff');
|
|
|
+ Map<String, String> header = list.get(0);
|
|
|
+ // 表头
|
|
|
+ printer.printRecord(header.keySet());
|
|
|
+ for (Map<String, String> map : list) {
|
|
|
+ printer.printRecord(map.values());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return csvPath;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 导入数据库
|
|
|
+ *
|
|
|
+ * @param path 文件路径
|
|
|
+ */
|
|
|
+ public void copyCsv(Path path) {
|
|
|
+ String dbTable = "car.car_bao_xian";
|
|
|
+ String csv = path.toString();
|
|
|
+ String columns = "(data_time_nlsd,data_time,form_type,submitter,submitter_dept_id,form_no,reimbursement_no,start_time,form_id,license_plate_number,motorcycle_type,dept_id,rent_start_time,rent_end_time,supplier,invoice_type,jq_amount_excluding_tax,cc_tax_amount,jq_total_amount,sy_amount_excluding_tax,sy_total_amount,total_amount,commercial_insurance,year_month,year_no,month_no,raw_license_plate_number,che_pai_fail,first_unit,second_unit,third_unit,area_no,area_name,city_no,city_name,area_no2,area_name2,city_id,city,district_id,district,source)";
|
|
|
+ Long timeout = 60000L;
|
|
|
+ PsqlUtil.copyCsv(jobConfig.getCopyScriptPath(), jobConfig.getDbHost(), jobConfig.getDbPort(),
|
|
|
+ jobConfig.getDbUsername(), jobConfig.getDbPassword(), jobConfig.getDbName(), dbTable, csv, columns,
|
|
|
+ timeout, null);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 移动源文件到历史文件夹
|
|
|
+ *
|
|
|
+ * @param path 源文件路径
|
|
|
+ */
|
|
|
+ public void move(Path path) throws Exception {
|
|
|
+ Files.move(Paths.get(path + ".MD5"),
|
|
|
+ Paths.get(jobConfig.getCarBaoXianHistoryPath(), path.getFileName().toString() + ".MD5"),
|
|
|
+ StandardCopyOption.REPLACE_EXISTING);
|
|
|
+ Files.move(path, Paths.get(jobConfig.getCarBaoXianHistoryPath(), path.getFileName().toString()),
|
|
|
+ StandardCopyOption.REPLACE_EXISTING);
|
|
|
+ }
|
|
|
+}
|