feat: 实现车辆php页面请求日志入库定时任务

src/main/java/com/nokia/finance/tasks/dao/common/RequestLogDao.java

@@ -0,0 +1,63 @@
+package com.nokia.finance.tasks.dao.common;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.nokia.finance.tasks.pojo.po.common.CarPhpRequestLogPo;
+import com.nokia.finance.tasks.pojo.po.common.RequestLogPo;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.apache.ibatis.annotations.Update;
+
+import java.util.List;
+
+@Mapper
+public interface RequestLogDao extends BaseMapper<RequestLogPo> {
+    /**
+     * 批量插入
+     */
+    @Update("""
+<script>
+insert into common.request_log
+(
+request_time,
+login_id,
+page_url,
+api,
+request_parameters,
+headers,
+app_id,
+time_stamp,
+token
+)
+values
+<foreach collection="list" item="item" index="index" separator=",">
+(
+#{item.requestTime},
+#{item.loginId},
+#{item.pageUrl},
+#{item.api},
+#{item.requestParameters},
+#{item.headers},
+#{item.appId},
+#{item.timeStamp},
+#{item.token}
+)
+</foreach>
+</script>
+""")
+    int insertBatch(List<RequestLogPo> list);
+
+    @Select("""
+select
+    to_timestamp(create_time)::timestamp as request_time,
+    query_string
+from
+    car_theme.wz_adminlog
+where create_time >= #{startTime}
+and create_time < #{endTime}
+order by
+    create_time
+""")
+    List<CarPhpRequestLogPo> listCarPhpRequestLog(@Param("startTime") Long startTime,
+                                                  @Param("endTime") Long endTime);
+}

src/main/java/com/nokia/finance/tasks/dao/house/HouseProcedureDao.java

@@ -0,0 +1,28 @@
+package com.nokia.finance.tasks.dao.house;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Update;
+
+@Mapper
+public interface HouseProcedureDao {
+    /**
+     * 更新自有房产维修
+     */
+    @Update("""
+with
+t101 as (
+select * from house.building_month where year_month = (select max(year_month) from house.building_month)
+)
+update house.building_repair_month a set
+second_unit = b.second_unit,
+third_unit = b.third_unit,
+city = b.city,
+district = b.district,
+area_no = b.area_no,
+area_name = b.area_name,
+city_no = b.city_no,
+city_name = b.city_name
+from t101 b where a.jz_jzid = b.building_id and a.year_month = (select max(year_month) from house.building_repair_month)
+""")
+    int updateBuildingRepairMonth();
+}

src/main/java/com/nokia/finance/tasks/jobs/car/procedure/CarPhpRequestLogJob.java

@@ -0,0 +1,124 @@
+package com.nokia.finance.tasks.jobs.car.procedure;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.nokia.finance.tasks.dao.common.RequestLogDao;
+import com.nokia.finance.tasks.pojo.po.common.CarPhpRequestLogPo;
+import com.nokia.finance.tasks.pojo.po.common.RequestLogPo;
+import com.nokia.finance.tasks.utils.AESUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.MultiValueMap;
+import org.springframework.util.StringUtils;
+import org.springframework.web.util.UriComponents;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * 车辆php页面请求日志入库定时任务
+ */
+@Slf4j
+@Service
+public class CarPhpRequestLogJob {
+    private final RequestLogDao requestLogDao;
+
+    public CarPhpRequestLogJob(RequestLogDao requestLogDao) {
+        this.requestLogDao = requestLogDao;
+    }
+
+    /**
+     * 执行任务
+     */
+    @Scheduled(cron = "0 30 * * * ?")
+    public void runJob() {
+        log.info("执行车辆php页面请求日志入库定时任务");
+        try {
+            CompletableFuture.runAsync(() -> {
+                Gson gson = new Gson();
+//                LocalDateTime startLocalDateTime = LocalDateTime.of(2024, 3, 23, 0, 0);
+//                LocalDateTime endLocalDateTime = LocalDateTime.of(2024, 3, 25, 18, 0);
+                LocalDateTime now = LocalDateTime.now();
+                LocalDateTime startLocalDateTime = now.minusHours(1).withMinute(0).withSecond(0).withNano(0);
+                LocalDateTime endLocalDateTime = startLocalDateTime.plusHours(1);
+                Long startTime = startLocalDateTime.toEpochSecond(ZoneOffset.ofHours(8));
+                Long endTime = endLocalDateTime.toEpochSecond(ZoneOffset.ofHours(8));
+                List<CarPhpRequestLogPo> carPhpRequestLogPoList = requestLogDao.listCarPhpRequestLog(startTime, endTime);
+                List<RequestLogPo> list = new ArrayList<>();
+                for (CarPhpRequestLogPo carPhpRequestLogPo : carPhpRequestLogPoList) {
+                    LocalDateTime requestTime = carPhpRequestLogPo.getRequestTime();
+                    String queryString = carPhpRequestLogPo.getQueryString();
+                    if (requestTime == null || !StringUtils.hasText(queryString)) {
+                        continue;
+                    }
+                    UriComponents uriComponents = UriComponentsBuilder.fromUriString("?" + queryString).build();
+                    MultiValueMap<String, String> queryParams = uriComponents.getQueryParams();
+                    Map<String, String> singleValueMap = queryParams.toSingleValueMap();
+                    String data = singleValueMap.get("data");
+                    if (!StringUtils.hasText(data)) {
+                        continue;
+                    }
+                    String urlDecodeToken = URLDecoder.decode(data, StandardCharsets.UTF_8);
+                    String decodeToken;
+                    try {
+                        decodeToken = AESUtil.decrypt(urlDecodeToken);
+                    } catch (Exception e) {
+                        log.error("token解密失败: {} -> {} -> {}", urlDecodeToken, queryString, e.getMessage(), e);
+                        continue;
+                    }
+                    Map<String, String> tokenMap = gson.fromJson(decodeToken, new TypeToken<Map<String, String>>() {
+                    }.getType());
+                    String loginId = tokenMap.get("LOGIN_ID");
+                    String pageUrl = tokenMap.get("REQUEST_URL");
+                    String api = StringUtils.hasText(singleValueMap.get("type")) ? pageUrl + "?type=" + singleValueMap.get("type") : pageUrl;
+                    Map<String, String> requestParameters = new HashMap<>();
+                    requestParameters.put("request_parameters", gson.toJson(singleValueMap));
+                    String appId = tokenMap.get("APP_ID");
+                    RequestLogPo requestLogPo = new RequestLogPo();
+                    requestLogPo.setRequestTime(carPhpRequestLogPo.getRequestTime());
+                    requestLogPo.setLoginId(loginId);
+                    requestLogPo.setPageUrl(pageUrl);
+                    requestLogPo.setApi(api);
+                    requestLogPo.setRequestParameters(gson.toJson(requestParameters));
+                    requestLogPo.setAppId(appId);
+                    setTimeStamp(tokenMap, requestLogPo);
+                    requestLogPo.setToken(urlDecodeToken);
+                    list.add(requestLogPo);
+                }
+                if (CollectionUtils.isEmpty(list)) {
+                    return;
+                }
+                requestLogDao.insertBatch(list);
+            }).get(1, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            log.error("线程中断: {}", e.getMessage(), e);
+            Thread.currentThread().interrupt();
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+
+    private void setTimeStamp(Map<String, String> map, RequestLogPo requestLogPo) {
+        String timeStamp = map.get("TIME_STAMP");
+        if (StringUtils.hasText(timeStamp)) {
+            try {
+                requestLogPo.setTimeStamp(LocalDateTime.parse(map.get("TIME_STAMP"),
+                        DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
+            } catch (Exception e) {
+                log.warn("时间戳解析失败: {}", timeStamp);
+            }
+        }
+    }
+}

src/main/java/com/nokia/finance/tasks/pojo/po/common/CarPhpRequestLogPo.java

@@ -0,0 +1,20 @@
+package com.nokia.finance.tasks.pojo.po.common;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.LocalDateTime;
+
+@NoArgsConstructor
+@Data
+public class CarPhpRequestLogPo {
+    /**
+     * 请求时间
+     */
+    private LocalDateTime requestTime;
+
+    /**
+     * 请求参数
+     */
+    private String queryString;
+}

src/main/java/com/nokia/finance/tasks/pojo/po/common/RequestLogPo.java

@@ -0,0 +1,57 @@
+package com.nokia.finance.tasks.pojo.po.common;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.LocalDateTime;
+
+@NoArgsConstructor
+@Data
+@TableName("common.request_log")
+public class RequestLogPo {
+    /**
+     * 请求时间
+     */
+    private LocalDateTime requestTime;
+
+    /**
+     * 账号，从token解密
+     */
+    private String loginId;
+
+    /**
+     * 页面url，从token解密
+     */
+    private String pageUrl;
+
+    /**
+     * 接口url
+     */
+    private String api;
+
+    /**
+     * 请求参数
+     */
+    private String requestParameters;
+
+    /**
+     * 请求头
+     */
+    private String headers;
+
+    /**
+     * 应用id，从token解密
+     */
+    private String appId;
+
+    /**
+     * 时间戳，从token解密
+     */
+    private LocalDateTime timeStamp;
+
+    /**
+     * 访问令牌
+     */
+    private String token;
+}

src/main/java/com/nokia/finance/tasks/utils/AESUtil.java

@@ -0,0 +1,93 @@
+package com.nokia.finance.tasks.utils;
+
+import cn.hutool.core.codec.Base64;
+import com.google.gson.Gson;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.SecretKeySpec;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+
+/**
+ * aes加密解密
+ */
+public class AESUtil {
+    static final String KEY = "2na$$PdV9AW8b#CS";
+    // 固定替换
+    private static final String ADD = "/add/";
+
+    // 加密
+    public static String encrypt(String str, String key) throws Exception {
+        // 加密
+        if (str == null || key == null) {
+            // 加密
+            return null;
+        }
+        // 加密
+        Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+        // 加密
+        cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key.getBytes("utf-8"), "AES"));
+        // 加密
+        byte[] bytes = cipher.doFinal(str.getBytes("utf-8"));
+        // 加密
+        String result = Base64.encode(bytes);
+//        String result = new BASE64Encoder().encode(bytes);
+        // 加密
+        result = result.replaceAll("\r\n", "");
+        // 加密
+        result = result.replaceAll("\\+", ADD);
+        // 加密
+        return result;
+    }
+
+    // 解密
+    public static String decrypt(String str, String key) throws Exception {
+        // 解密
+        if (str == null || key == null) {
+            // 解密
+            return null;
+        }
+        // 解密
+        str = str.replaceAll("\\\\n", "");
+        // 解密
+        str = str.replaceAll(ADD, "+");
+        // 解密
+        Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+        // 解密
+        cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key.getBytes("utf-8"), "AES"));
+        // 解密
+        byte[] bytes = Base64.decode(str);
+//        byte[] bytes = new BASE64Decoder().decodeBuffer(str);
+        // 解密
+        bytes = cipher.doFinal(bytes);
+        // 解密
+        return new String(bytes);
+    }
+
+    public static String decrypt(String str) throws Exception {
+        return decrypt(str, KEY);
+    }
+
+    public static void main(String[] args) throws Exception {
+// 参数
+        HashMap<String, String> map = new HashMap<>();
+        // 财务系统id, 固定值
+        map.put("APP_ID", "FINANCE");
+        // 时间戳
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        map.put("TIME_STAMP", sdf.format(new Date()));
+        // 当前登录账号
+        map.put("LOGIN_ID", "test");
+        //请求的地址
+        map.put("REQUEST_URL","");
+
+        // 加密串
+        String encrypt = AESUtil.encrypt(new Gson().toJson(map), "2na$$PdV9AW8b#CS");
+        // 返回
+        System.out.println(encrypt);
+        //解密
+        System.out.println(AESUtil.decrypt(encrypt,"2na$$PdV9AW8b#CS"));
+        System.out.println(AESUtil.decrypt("zoQtYlmhk/add/mBUBZD5mFJB1IXEwaLRS97Uf9z9Hlqdh8eF/add/c2gDUmD3pwGPfA8FQKBNTCB8LPlCPF/vHcU/2tL7Ps/add/6rn6w5rDS98R4GJueGgU01n12YZVS/FNx6pSI/add/BuEy1XyrSg8QBfx7igSozPxmdZ6a235/add/fqZPPSjoQReI","2na$$PdV9AW8b#CS"));
+    }
+}

src/test/java/com/nokia/finance/tasks/car/procedure/CarPhpRequestLogJobTests.java

@@ -0,0 +1,22 @@
+package com.nokia.finance.tasks.car.procedure;
+
+import com.nokia.finance.tasks.jobs.car.procedure.CarPhpRequestLogJob;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+@Slf4j
+@SpringBootTest
+@ActiveProfiles("prod")
+class CarPhpRequestLogJobTests {
+    @Autowired
+    CarPhpRequestLogJob carPhpRequestLogJob;
+
+    @Test
+    void runJobTest() {
+        carPhpRequestLogJob.runJob();
+    }
+
+}

src/test/java/com/nokia/finance/tasks/requestlogs/JavaRequestLogTest.java

@@ -0,0 +1,86 @@
+package com.nokia.finance.tasks.requestlogs;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.QuoteMode;
+import org.junit.jupiter.api.Test;
+
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+@Slf4j
+class JavaRequestLogTest {
+    @Test
+    void test() throws Exception {
+        Gson gson = new Gson();
+        Pattern pattern = Pattern.compile("调用记录: \\s*(.*)");
+        Files.createDirectories(Paths.get("test/output"));
+        try (Stream<String> stream = Files.lines(Paths.get("test/data/java-request-log.txt"));
+             OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get("test/output/java-request-log.csv")),
+                     StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT.builder().setQuoteMode(QuoteMode.ALL).build())
+        ) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            printer.printRecord("request_time", "login_id", "page_url", "api", "request_parameters", "headers", "app_id", "time_stamp", "token");
+            stream.forEachOrdered(line -> {
+                Matcher matcher = pattern.matcher(line);
+                if (matcher.find()) {
+                    String s = matcher.group(1);
+                    log.info(s);
+                    RequestLogBo bo = gson.fromJson(s, new TypeToken<>() {});
+                    Map<String, String> requestParameters = new HashMap<>();
+                    requestParameters.put("request_parameters", bo.getParameters());
+                    requestParameters.put("request_body", bo.getBody());
+                    try {
+                        printer.printRecord(
+                                LocalDateTime.parse(bo.getTime()).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")),
+                                bo.getLOGIN_ID(),
+                                bo.getREQUEST_URL(),
+                                bo.getApi(),
+                                gson.toJson(requestParameters),
+                                bo.getHeaders(),
+                                bo.getAPP_ID(),
+                                bo.getTIME_STAMP(),
+                                bo.getToken()
+                        );
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            });
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+
+    @NoArgsConstructor
+    @Data
+    public static class RequestLogBo {
+        private String APP_ID;
+        private String LOGIN_ID;
+        private String TIME_STAMP;
+        private String REQUEST_URL;
+        private String api;
+        private String body;
+        private String time;
+        private String headers;
+        private String parameters;
+        private String token;
+    }
+
+}

src/test/java/com/nokia/finance/tasks/requestlogs/PhpRequestLogTest.java

@@ -0,0 +1,97 @@
+package com.nokia.finance.tasks.requestlogs;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.nokia.finance.tasks.utils.AESUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.QuoteMode;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.util.MultiValueMap;
+import org.springframework.util.StringUtils;
+import org.springframework.web.util.UriComponents;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.io.OutputStreamWriter;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+@SpringBootTest
+@ActiveProfiles("prod")
+class PhpRequestLogTest {
+    @Autowired
+    JdbcTemplate jdbcTemplate;
+
+    @Test
+    void test() throws Exception {
+        Gson gson = new Gson();
+        Files.createDirectories(Paths.get("test/output"));
+        String sql = """
+select to_char(to_timestamp(create_time), 'YYYY-MM-DD HH24:MI:SS') as request_time, query_string from car_theme.wz_adminlog order by create_time
+                """;
+        List<Map<String, Object>> list = jdbcTemplate.queryForList(sql);
+        try (OutputStreamWriter osw = new OutputStreamWriter(
+                Files.newOutputStream(Paths.get("test/output/php-request-log.csv")), StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT.builder().setQuoteMode(QuoteMode.ALL).build())
+        ) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            printer.printRecord("request_time", "login_id", "page_url", "api", "request_parameters", "headers", "app_id", "time_stamp", "token");
+            for (Map<String, Object> t : list) {
+                String requestTime = (String) t.get("request_time");
+                String queryString = (String) t.get("query_string");
+                if (!StringUtils.hasText(requestTime) || !StringUtils.hasText(queryString)) {
+                    continue;
+                }
+                UriComponents uriComponents = UriComponentsBuilder.fromUriString("?" + queryString).build();
+                MultiValueMap<String, String> queryParams = uriComponents.getQueryParams();
+                Map<String, String> singleValueMap = queryParams.toSingleValueMap();
+                String data = singleValueMap.get("data");
+                if (!StringUtils.hasText(data)) {
+                    continue;
+                }
+                String urlDecodeToken = URLDecoder.decode(data, StandardCharsets.UTF_8);
+                String decodeToken;
+                try {
+                    decodeToken = AESUtil.decrypt(urlDecodeToken);
+                } catch (Exception e) {
+                    log.error("token解密失败: {} -> {} -> {}", urlDecodeToken, queryString, e.getMessage(), e);
+                    continue;
+                }
+                Map<String, String> tokenMap = gson.fromJson(decodeToken, new TypeToken<Map<String, String>>() {
+                }.getType());
+                String loginId = tokenMap.get("LOGIN_ID");
+                String pageUrl = tokenMap.get("REQUEST_URL");
+                String api = StringUtils.hasText(singleValueMap.get("type")) ? pageUrl + "?type=" + singleValueMap.get("type") : pageUrl;
+                Map<String, String> requestParameters = new HashMap<>();
+                requestParameters.put("request_parameters", gson.toJson(singleValueMap));
+                String appId = tokenMap.get("APP_ID");
+                String timeStamp = tokenMap.get("TIME_STAMP");
+                printer.printRecord(
+                        requestTime,
+                        loginId,
+                        pageUrl,
+                        api,
+                        gson.toJson(requestParameters),
+                        null,
+                        appId,
+                        timeStamp,
+                        urlDecodeToken
+                );
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+}