feat: 导入数据由gpload改成copy，userlabel字段截取前100个字符

README.md

@@ -1,4 +1,47 @@
-# 工程简介
+# 5gpm数据导入
 
-# 延伸阅读
+## 正式环境部署位置
 
+192.168.10.7/data/pm_5g
+
+## 测试环境部署位置
+
+192.168.50.3/data1/pm_5g
+
+## pm数据接口
+
+当前lte pm数据入库程序部署位置 133.96.94.108 /data1/pm
+
+5Gpm小时数据导出：10.17.180.55:/data/out2/pm_5g_hour/pm_5g_hour_2022072910.csv
+
+数据源 10.17.180.55 /data/out2/pm_4g_hour
+pm_4g_hour_2022053005.csv
+
+133.96.94.19
+esbftp Esb2019ftp!
+nokia   Nokia*123
+
+[xxl-job-server访问地址](http://192.168.10.7:8087/xxl-job-admin)
+
+admin
+@XXL-job133413
+
+## 统计各小时数据
+
+```sql
+select rpt_time,
+       count(1)
+from tsfx.dw_sa_omc_ci_h
+group by rpt_time
+order by rpt_time desc;
+```
+
+## 检查重复数据
+```sql
+select nci,
+       count(1)
+from tsfx.dw_sa_omc_ci_h
+where rpt_time = '2022-12-25 19:00:00'
+group by nci
+having count(1) > 1;
+```

build-dir/import.sh

@@ -0,0 +1,15 @@
+#!/bin/expect
+
+set timeout 300
+set host [lindex $argv 0]
+set port [lindex $argv 1]
+set username [lindex $argv 2]
+set password [lindex $argv 3]
+set dbname [lindex $argv 4]
+set table [lindex $argv 5]
+set filename [lindex $argv 6]
+spawn psql -h ${host} -p ${port} -U ${username} -d ${dbname} -c "\\copy ${table} from ${filename} with csv"
+expect "Password" {
+  send "${password}\r"
+}
+expect eof

build-dir/run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-nohup java -jar /data1/pm_5g/pmInterface5g.jar > /dev/null 2>&1 &
+nohup java -jar pmInterface5g.jar > /dev/null 2>&1 &

build-dir/stop.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-for i in $(ps -ef | grep pmInterface5g.jar | grep -v grep | awk '{print $2}'); do
+for i in $(pgrep -f pmInterface5g.jar); do
   kill -9 "$i"
 done

doc/readme.md

@@ -1,37 +0,0 @@
-# pm数据接口
-
-当前lte pm数据入库程序部署位置 133.96.94.108 /data1/pm
-
-5Gpm小时数据导出：10.17.180.55:/data/out2/pm_5g_hour/pm_5g_hour_2022072910.csv
-
-数据源 10.17.180.55 /data/out2/pm_4g_hour
-pm_4g_hour_2022053005.csv
-
-133.96.94.19
-esbftp Esb2019ftp!
-nokia   Nokia*123
-
-[xxl-job-server访问地址](http://133.96.94.105:8087/xxl-job-admin/joblog)
-
-admin
-Richr00t#
-
-## 统计各小时数据
-
-```sql
-select rpt_time,
-       count(1)
-from tsfx.dw_sa_omc_ci_h
-group by rpt_time
-order by rpt_time desc;
-```
-
-## 检查重复数据
-```sql
-select nci,
-       count(1)
-from tsfx.dw_sa_omc_ci_h
-where rpt_time = '2022-12-25 19:00:00'
-group by nci
-having count(1) > 1;
-```

pom.xml

@@ -2,12 +2,12 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <groupId>com.nokia</groupId>
-        <artifactId>hb_springboot_parent</artifactId>
-        <version>1.0</version>
-        <relativePath/>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.6.14</version>
+        <relativePath/> <!-- lookup parent from repository -->
     </parent>
-
+    <groupId>com.nokia</groupId>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>pm_interface_5g</artifactId>
     <version>0.0.1-SNAPSHOT</version>
@@ -16,8 +16,10 @@
 
     <properties>
         <java.version>8</java.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <project.reporting.outputEncoding>${project.build.sourceEncoding}</project.reporting.outputEncoding>
         <skipTests>true</skipTests>
     </properties>
 
@@ -37,6 +39,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-configuration-processor</artifactId>
+            <optional>true</optional>
+        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
@@ -47,15 +54,30 @@
             <artifactId>xxl-job-core</artifactId>
             <version>2.3.0</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-csv -->
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
+            <version>1.9.0</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <version>2.0.22</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.jcraft/jsch -->
         <dependency>
             <groupId>com.jcraft</groupId>
             <artifactId>jsch</artifactId>
             <version>0.1.55</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/com/nokia/common/command/CommandUtil.java

@@ -0,0 +1,54 @@
+package com.nokia.common.command;
+
+import com.xxl.job.core.context.XxlJobHelper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 执行命令工具
+ */
+@Slf4j
+public class CommandUtil {
+    public static List<String> exec(String command) throws IOException, InterruptedException {
+        List<String> error = new ArrayList<>();
+        List<String> input = new ArrayList<>();
+        Process process = null;
+        try {
+            process = Runtime.getRuntime().exec(command);
+            try (BufferedReader inputReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+                 BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
+                String line;
+                // 读取错误流
+                while ((line = errorReader.readLine()) != null) {
+                    error.add(line);
+                }
+                // 读取标准流
+                while ((line = inputReader.readLine()) != null) {
+                    input.add(line);
+                }
+            }
+
+            if (!CollectionUtils.isEmpty(input)) {
+                log.info("inputStream: " + input);
+                XxlJobHelper.log("inputStream: " + input);
+            }
+            int code = process.waitFor();
+            // 非正常退出或错误流不为空
+            if (code != 0 || !CollectionUtils.isEmpty(error)) {
+                throw new RuntimeException(error.toString());
+            }
+            return input;
+        } finally {
+            if (process != null) {
+                log.info("销毁进程");
+                process.destroy();
+            }
+        }
+    }
+}

src/main/java/com/nokia/common/psql/PsqlUtil.java

@@ -0,0 +1,66 @@
+package com.nokia.common.psql;
+
+import com.nokia.common.command.CommandUtil;
+import com.xxl.job.core.context.XxlJobHelper;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * psql命令工具
+ */
+@Slf4j
+public class PsqlUtil {
+    /**
+     * 匹配psql copy成功结果
+     */
+    private static final Pattern PATTERN = Pattern.compile("^(COPY) (\\d+)$");
+
+    /**
+     * 导入csv
+     *
+     * @param script         脚本
+     * @param dbHost         数据库主机
+     * @param dbPort         数据库端口
+     * @param dbUsername     数据库用户名
+     * @param dbPassword     数据库密码
+     * @param dbName         数据库名字
+     * @param dbTable        数据库表
+     * @param csv            csv
+     * @param minInsertCount 最小值插入数
+     * @throws IOException          ioexception
+     * @throws InterruptedException 中断异常
+     */
+    public static void copyCsv(String script, String dbHost, Integer dbPort, String dbUsername, String dbPassword,
+                               String dbName, String dbTable, String csv, Long minInsertCount)
+            throws IOException, InterruptedException {
+        String command = "expect -f"
+                + " " + script
+                + " " + dbHost
+                + " " + dbPort
+                + " " + dbUsername
+                + " " + dbPassword
+                + " " + dbName
+                + " " + dbTable
+                + " " + csv;
+        List<String> list = CommandUtil.exec(command);
+        Long count = null;
+        for (String s : list) {
+            Matcher matcher = PATTERN.matcher(s);
+            while (matcher.find()) {
+                count = Long.parseLong(matcher.group(2));
+            }
+        }
+        if (count == null) {
+            throw new RuntimeException("导入数据失败: " + list);
+        }
+        log.info("插入 {} 条数据", count);
+        XxlJobHelper.log("插入 {} 条数据", count);
+        if (count < minInsertCount) {
+            throw new RuntimeException(csv + " 数据异常，少于 " + minInsertCount);
+        }
+    }
+}

src/main/java/com/nokia/pm_interface_5g/task/FiveGPmTask.java

@@ -4,6 +4,7 @@ import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.SftpException;
 import com.nokia.common.gpload.GploadUtil;
 import com.nokia.common.gpload.entity.GploadResult;
+import com.nokia.common.psql.PsqlUtil;
 import com.nokia.common.ssh.SSHUtil;
 import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
@@ -13,7 +14,7 @@ import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVRecord;
-import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
@@ -27,41 +28,99 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.rmi.RemoteException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+@ConfigurationProperties("task")
 @Data
 @Component
 @Slf4j
 public class FiveGPmTask {
-
-    @Value("${fiveg.pm.download.host:10.17.180.55}")
-    private String host;
-    @Value("${fiveg.pm.download.port:22}")
-    private Integer port;
-    @Value("${fiveg.pm.download.username:nokia}")
-    private String username;
-    @Value("${fiveg.pm.download.password:Nokia*123}")
-    private String password;
-    @Value("${fiveg.pm.download.sourceDir:/data/out2/pm_5g_hour/}")
+    /**
+     * 无文件次数告警值
+     */
+    private static final int ALERT_COUNT = 12;
+    /**
+     * 无文件次数
+     */
+    private static final AtomicInteger NO_FILE_COUNT = new AtomicInteger(0);
+    /**
+     * ssh ip
+     */
+    private String sshHost;
+    /**
+     * ssh端口
+     */
+    private Integer sshPort;
+    /**
+     * ssh账号
+     */
+    private String sshUsername;
+    /**
+     * ssh密码
+     */
+    private String sshPassword;
+    /**
+     * 5gpm数据文件目录
+     */
     private String sourceDir;
-    @Value("${fiveg.pm.download.targetDir:download/}")
-    private String downloadTargetDir;
-    @Value("${fiveg.pm.filePrefix:pm_5g_hour_}")
+    /**
+     * 5gpm数据文件下载目录
+     */
+    private String downloadDir;
+    /**
+     * 5gpm数据文件前缀
+     */
     private String filePrefix;
-    @Value("${fiveg.pm.distinct.targetDir:distinct/}")
-    private String distinctTargetDir;
+    /**
+     * 5gpm数据文件去重目录
+     */
+    private String distinctDir;
+    /**
+     * 导入数据脚本路径
+     */
+    private String importScript;
+    /**
+     * 导入的数据库ip
+     */
+    private String dbHost;
+    /**
+     * 数据库端口
+     */
+    private Integer dbPort;
+    /**
+     * 数据库账号
+     */
+    private String dbUsername;
+    /**
+     * 数据库密码
+     */
+    private String dbPassword;
+    /**
+     * 数据库名称
+     */
+    private String dbName;
+    /**
+     * 数据库表
+     */
+    private String dbTable;
     /**
      * 最小插入数据
      */
-    @Value("${fiveg.pm.minInsertCount:10000}")
-    private long minInsertCount;
+    private Long minInsertCount;
+    /**
+     * 超时分钟
+     */
+    private Long timeout;
     private final DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHH");
     private SSHUtil sshUtil;
 
@@ -74,22 +133,52 @@ public class FiveGPmTask {
     public void pm5gJobHandler() {
         try {
             // 创建文件夹
-            Files.createDirectories(Paths.get(downloadTargetDir));
-            Files.createDirectories(Paths.get(distinctTargetDir));
-            sshUtil = new SSHUtil(host, port, username, password);
+            Files.createDirectories(Paths.get(downloadDir));
+            Files.createDirectories(Paths.get(distinctDir));
+            sshUtil = new SSHUtil(sshHost, sshPort, sshUsername, sshPassword);
             // 获取文件列表
             List<String> list = sshUtil.ls(sourceDir);
             if (CollectionUtils.isEmpty(list)) {
-                log.info("没有文件");
-                XxlJobHelper.log("没有文件");
+                int count = NO_FILE_COUNT.incrementAndGet();
+                log.info("{}次没有文件", count);
+                XxlJobHelper.log("{}次没有文件", count);
+                // 无文件次数告警
+                if (count > ALERT_COUNT) {
+                    log.error("超过{}次没有文件", ALERT_COUNT);
+                    XxlJobHelper.log("超过{}次没有文件", ALERT_COUNT);
+                    // 重置无文件计数
+                    NO_FILE_COUNT.set(0);
+                }
                 return;
             }
 
+            // 重置无文件计数
+            NO_FILE_COUNT.set(0);
             log.info("扫描到的文件: {}", list);
             XxlJobHelper.log("扫描到的文件: {}", list);
             for (String t : list) {
-                singleTask(t);
+                CompletableFuture.runAsync(() -> {
+                    try {
+                        singleTask(t);
+                    } catch (InterruptedException e) {
+                        log.error("线程中断: {}", e.getMessage(), e);
+                        XxlJobHelper.log("线程中断: {}", e.getMessage(), e);
+                        Thread.currentThread().interrupt();
+                        throw new RuntimeException(e);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }).get(timeout, TimeUnit.MINUTES);
             }
+        } catch (InterruptedException e) {
+            log.error("线程中断: {}", e.getMessage(), e);
+            XxlJobHelper.log("线程中断: {}", e.getMessage(), e);
+            Thread.currentThread().interrupt();
+            XxlJobHelper.handleFail(e.getMessage());
+        } catch (TimeoutException e) {
+            log.error("执行超时: {}", e.getMessage(), e);
+            XxlJobHelper.log("执行超时: {}", e.getMessage(), e);
+            XxlJobHelper.handleFail(e.getMessage());
         } catch (Exception e) {
             log.error("发生异常了: {}", e.getMessage(), e);
             XxlJobHelper.log("发生异常了: {}", e.getMessage(), e);
@@ -98,8 +187,8 @@ public class FiveGPmTask {
             try {
                 sshUtil.disconnect();
             } catch (IOException e) {
-                log.error("发生异常了: {}", e.getMessage(), e);
-                XxlJobHelper.log("发生异常了: {}", e.getMessage(), e);
+                log.error("ssh关闭异常: {}", e.getMessage(), e);
+                XxlJobHelper.log("ssh关闭异常: {}", e.getMessage(), e);
                 XxlJobHelper.handleFail(e.getMessage());
             }
         }
@@ -108,7 +197,7 @@ public class FiveGPmTask {
     @XxlJob("pm5gRestHandler")
     public void pm5gRestHandler() {
         // 读取目标文件夹下的文件
-        try (Stream<Path> stream = Files.list(Paths.get(distinctTargetDir))) {
+        try (Stream<Path> stream = Files.list(Paths.get(distinctDir))) {
             List<Path> pathList = stream.collect(Collectors.toList());
             log.info("pathList: {}", pathList);
             XxlJobHelper.log("pathList: {}", pathList);
@@ -116,8 +205,28 @@ public class FiveGPmTask {
                 String filename = t.getFileName().toString();
                 log.info("file: {}", filename);
                 XxlJobHelper.log("file: {}", filename);
-                gpload(filename);
+                CompletableFuture.runAsync(() -> {
+                    try {
+                        importData(filename);
+                    } catch (InterruptedException e) {
+                        log.error("线程中断: {}", e.getMessage(), e);
+                        XxlJobHelper.log("线程中断: {}", e.getMessage(), e);
+                        Thread.currentThread().interrupt();
+                        throw new RuntimeException(e);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }).get(timeout, TimeUnit.MINUTES);
             }
+        } catch (InterruptedException e) {
+            log.error("线程中断: {}", e.getMessage(), e);
+            XxlJobHelper.log("线程中断: {}", e.getMessage(), e);
+            Thread.currentThread().interrupt();
+            XxlJobHelper.handleFail(e.getMessage());
+        } catch (TimeoutException e) {
+            log.error("执行超时: {}", e.getMessage(), e);
+            XxlJobHelper.log("执行超时: {}", e.getMessage(), e);
+            XxlJobHelper.handleFail(e.getMessage());
         } catch (Exception e) {
             log.error("发生异常了: {}", e.getMessage(), e);
             XxlJobHelper.log("发生异常了: {}", e.getMessage(), e);
@@ -130,10 +239,10 @@ public class FiveGPmTask {
      *
      * @param filename 文件名
      */
-    public void singleTask(String filename) throws JSchException, IOException, SftpException {
+    public void singleTask(String filename) throws JSchException, IOException, SftpException, InterruptedException {
         download(filename);
         distinct(filename);
-        gpload(filename);
+        importData(filename);
     }
 
     /**
@@ -145,7 +254,7 @@ public class FiveGPmTask {
         log.info("下载文件: {}", filename);
         XxlJobHelper.log("下载文件: {}", filename);
         String src = sourceDir + filename;
-        String dst = downloadTargetDir + filename;
+        String dst = downloadDir + filename;
         sshUtil.get(src, dst);
         // 删除远程文件
         log.info("删除远程文件: {}", filename);
@@ -161,8 +270,8 @@ public class FiveGPmTask {
     public void distinct(String filename) throws IOException {
         log.info("文件 {} 去重...", filename);
         XxlJobHelper.log("文件 {} 去重...", filename);
-        String inputFilePath = downloadTargetDir + filename;
-        String outputFilePath = distinctTargetDir + filename;
+        String inputFilePath = downloadDir + filename;
+        String outputFilePath = distinctDir + filename;
         Path inputPath = Paths.get(inputFilePath);
         try (CSVParser parser = CSVFormat.DEFAULT.builder().build()
                 .parse(new InputStreamReader(Files.newInputStream(inputPath), StandardCharsets.UTF_8));
@@ -186,6 +295,19 @@ public class FiveGPmTask {
 //        XxlJobHelper.log("删除本地源文件 {}...", filename);
     }
 
+    /**
+     * 导入数据
+     *
+     * @param filename 文件名
+     * @throws IOException ioexception
+     */
+    public void importData(String filename) throws IOException, InterruptedException {
+        String csv = distinctDir + filename;
+        PsqlUtil.copyCsv(importScript, dbHost, dbPort, dbUsername, dbPassword, dbName, dbTable, csv, minInsertCount);
+        // 删除重排文件
+        Files.deleteIfExists(Paths.get(distinctDir + filename));
+    }
+
     public void gpload(String filename) throws IOException {
         String gploadCommand = "sh /data1/pm_5g/gpload/pm_nr_gpload.sh " + filename;
         GploadResult gpload = GploadUtil.gpload(gploadCommand);
@@ -193,10 +315,10 @@ public class FiveGPmTask {
             log.info("gpload完成: {}", gpload);
             XxlJobHelper.log("gpload完成: {}", gpload);
             if (gpload.getInsertedCount() < minInsertCount) {
-                throw new RemoteException(filename + " 数据异常，少于 " + minInsertCount);
+                throw new RuntimeException(filename + " 数据异常，少于 " + minInsertCount);
             }
             // 删除重排文件
-            Files.deleteIfExists(Paths.get(distinctTargetDir + filename));
+            Files.deleteIfExists(Paths.get(distinctDir + filename));
         } else {
             throw new RuntimeException("gpload失败: " + gpload.getMessage());
         }
@@ -221,6 +343,8 @@ public class FiveGPmTask {
             String nciStr = t.get(2);
             String enbid = "";
             String cellId = "";
+            // 数据库字段限制100个字符
+            String userLabel = org.apache.commons.lang3.StringUtils.substring(t.get(3), 0, 100);
             if (!StringUtils.hasText(nciStr)) {
                 continue;
             }
@@ -242,7 +366,7 @@ public class FiveGPmTask {
             int nci = Integer.parseInt(enbid) * 4096 + Integer.parseInt(cellId);
             printer.printRecord(
                     t.get(1),
-                    t.get(3),
+                    userLabel,
                     nci,
                     enbid,
                     cellId,

src/main/resources/application-product.properties

@@ -0,0 +1,35 @@
+server.port=12092
+spring.application.name=pm_interface_5g
+logging.level.com.nokia=info
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
+### xxl-job, access token
+xxl.job.accessToken=
+### xxl-job executor appname
+xxl.job.executor.appname=pm5g-prod
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
+xxl.job.executor.address=
+### xxl-job executor server-info
+xxl.job.executor.ip=
+xxl.job.executor.port=10001
+### xxl-job executor log-path
+xxl.job.executor.logpath=./log/xxl/
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays=30
+task.ssh-host=10.17.180.55
+task.ssh-port=22
+task.ssh-username=nokia
+task.ssh-password=Nokia*123
+task.source-dir=/data/out2/pm_5g_hour/
+task.download-dir=download/
+task.file-prefix=pm_5g_hour_
+task.distinct-dir=distinct/
+task.min-insert-count=10000
+task.import-script=import.sh
+task.db-host=192.168.70.109
+task.db-port=5432
+task.db-username=gpadmin
+task.db-password=Richr00t#
+task.db-name=sqmmt
+task.db-table=tsfx.dw_sa_omc_ci_h
+task.timeout=5

src/main/resources/application-test.properties

@@ -0,0 +1,36 @@
+server.port=12092
+spring.application.name=pm_interface_5g
+# log配置
+logging.level.com.nokia=debug
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
+### xxl-job, access token
+xxl.job.accessToken=
+### xxl-job executor appname
+xxl.job.executor.appname=pm5g-test
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
+xxl.job.executor.address=
+### xxl-job executor server-info
+xxl.job.executor.ip=
+xxl.job.executor.port=10001
+### xxl-job executor log-path
+xxl.job.executor.logpath=./log/xxl/
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays=30
+task.ssh-host=10.17.180.55
+task.ssh-port=22
+task.ssh-username=nokia
+task.ssh-password=Nokia*123
+task.source-dir=/data/out2/pm_5g_hour/
+task.download-dir=download/
+task.file-prefix=pm_5g_hour_
+task.distinct-dir=distinct/
+task.min-insert-count=10000
+task.import-script=import.sh
+task.db-host=192.168.50.5
+task.db-port=5432
+task.db-username=sqmdb
+task.db-password=sqmdb_1QAZ
+task.db-name=sqmmt
+task.db-table=tsfx.dw_sa_omc_ci_h
+task.timeout=5

src/main/resources/application.properties

@@ -1,20 +1,2 @@
-server.port=12092
-spring.application.name=pm_interface_5g
-logging.level.com.nokia=info
-### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
-xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
-### xxl-job, access token
-xxl.job.accessToken=
-
-### xxl-job executor appname
-xxl.job.executor.appname=xxl-job-executor-sample2
-### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
-xxl.job.executor.address=
-### xxl-job executor server-info
-xxl.job.executor.ip=
-xxl.job.executor.port=9999
-### xxl-job executor log-path
-xxl.job.executor.logpath=/data1/pm_5g/xxl/
-### xxl-job executor log-retention-days
-xxl.job.executor.logretentiondays=30
-fiveg.pm.minInsertCount=10000
+# 启用的配置
+spring.profiles.active=product

src/main/resources/logback-spring.xml

@@ -3,7 +3,7 @@
     <property name="PATH" value="./log"/>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <Pattern>%d{HH:mm:ss.SSS} %highlight(%-5level) %yellow(%X{traceId}) %cyan(%logger:%line) %msg%n</Pattern>
+            <Pattern>%d{HH:mm:ss.SSS} %highlight(%-5level) %yellow(%X{traceId}) %magenta([%thread]) %cyan(%logger:%line) %msg%n</Pattern>
         </encoder>
     </appender>
     <appender name="TRACE_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -17,7 +17,7 @@
             <totalSizeCap>20GB</totalSizeCap>
         </rollingPolicy>
         <encoder>
-            <Pattern>%d %highlight(%-5level) %yellow(%X{traceId}) %cyan(%logger:%line) %msg%n</Pattern>
+            <Pattern>%d %highlight(%-5level) %yellow(%X{traceId}) %magenta([%thread]) %cyan(%logger:%line) %msg%n</Pattern>
         </encoder>
     </appender>
     <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -31,7 +31,7 @@
             <totalSizeCap>20GB</totalSizeCap>
         </rollingPolicy>
         <encoder>
-            <Pattern>%d %highlight(%-5level) %yellow(%X{traceId}) %cyan(%logger:%line) %msg%n</Pattern>
+            <Pattern>%d %-5level %X{traceId} [%thread] %logger:%line %msg%n</Pattern>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.LevelFilter">
             <level>ERROR</level>

src/test/java/com/nokia/pm_interface_5g/PmInterface5gApplicationTests.java

@@ -9,6 +9,7 @@ import org.apache.commons.csv.CSVRecord;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
 import org.springframework.util.StringUtils;
 
 import java.io.IOException;
@@ -20,6 +21,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
+@ActiveProfiles("test")
 @Slf4j
 @SpringBootTest
 class PmInterface5gApplicationTests {