feat: gpload改成copy，同步新数据库

build-dir/run.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-nohup java -jar site_info.jar >/dev/null 2>&1 &

pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>2.6.14</version>
+        <version>2.7.18</version>
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
     <artifactId>siteinfo</artifactId>
@@ -55,7 +55,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
-            <version>1.9.0</version>
+            <version>1.10.0</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -65,7 +65,12 @@
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
-            <version>42.4.1</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-exec -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-exec</artifactId>
+            <version>1.4.0</version>
         </dependency>
     </dependencies>
 
@@ -75,6 +80,14 @@
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <excludes>
+                        <exclude>
+                            <groupId>org.projectlombok</groupId>
+                            <artifactId>lombok</artifactId>
+                        </exclude>
+                    </excludes>
+                </configuration>
             </plugin>
         </plugins>
     </build>

scripts/copy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+export PGPASSWORD=$4
+host=$1
+port=$2
+username=$3
+dbname=$5
+table=$6
+filename=$7
+columns=$8
+psql -h "${host}" -p "${port}" -U "${username}" -d "${dbname}" -c "\\copy ${table} ${columns} from ${filename} with csv header;"

scripts/prod/rollback.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+sh stop.sh
+rm -rf site_info.jar
+mv site_info.jar.bak site_info.jar
+sh run.sh

scripts/prod/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+nohup java -Dspring.profiles.active=prod -jar site_info.jar >/dev/null 2>&1 &

scripts/prod/update.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+rm -rf site_info.jar.bak
+sh stop.sh
+mv site_info.jar site_info.jar.bak
+mv site_info.jar.new site_info.jar
+sh run.sh

scripts/test/rollback.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+sh stop.sh
+rm -rf site_info.jar
+mv site_info.jar.bak site_info.jar
+sh run.sh

scripts/test/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+nohup java -Dspring.profiles.active=test -jar site_info.jar >/dev/null 2>&1 &

scripts/test/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+for i in $(pgrep -f site_info.jar); do
+  kill -9 "$i"
+done

scripts/test/update.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+rm -rf site_info.jar.bak
+sh stop.sh
+mv site_info.jar site_info.jar.bak
+mv site_info.jar.new site_info.jar
+sh run.sh

src/main/java/com/nokia/common/exception/MyRuntimeException.java

@@ -0,0 +1,27 @@
+package com.nokia.common.exception;
+
+public class MyRuntimeException extends RuntimeException{
+    public MyRuntimeException() {
+    }
+
+    public MyRuntimeException(String message) {
+        super(message);
+    }
+
+    public MyRuntimeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public MyRuntimeException(Throwable cause) {
+        super(cause);
+    }
+
+    public MyRuntimeException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    @Override
+    public synchronized Throwable fillInStackTrace() {
+        return this;
+    }
+}

src/main/java/com/nokia/common/psql/PsqlUtil.java

@@ -0,0 +1,92 @@
+package com.nokia.common.psql;
+
+import com.nokia.common.exception.MyRuntimeException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.springframework.util.StringUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.time.Duration;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * psql命令工具
+ */
+@Slf4j
+public class PsqlUtil {
+    /**
+     * 匹配psql copy成功结果
+     */
+    private static final Pattern PATTERN = Pattern.compile("^(COPY) (\\d+)$");
+
+    /**
+     * 导入csv
+     *
+     * @param script         脚本
+     * @param dbHost         数据库主机
+     * @param dbPort         数据库端口
+     * @param dbUsername     数据库用户名
+     * @param dbPassword     数据库密码
+     * @param dbName         数据库名字
+     * @param dbTable        数据库表
+     * @param csv            csv
+     * @param columns        字段
+     * @param timeout        超时分钟
+     * @param minInsertCount 最小值插入数
+     */
+    public static void copyCsv(String script, String dbHost, String dbPort, String dbUsername, String dbPassword,
+                               String dbName, String dbTable, String csv, String columns, Long timeout,
+                               Long minInsertCount) {
+        String command = "sh " + script;
+        CommandLine commandLine = CommandLine.parse(command);
+        commandLine.addArgument(dbHost);
+        commandLine.addArgument(dbPort);
+        commandLine.addArgument(dbUsername);
+        commandLine.addArgument(dbPassword);
+        commandLine.addArgument(dbName);
+        commandLine.addArgument(dbTable);
+        commandLine.addArgument(csv);
+        commandLine.addArgument(columns);
+        log.info("command: {}", commandLine);
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        ByteArrayOutputStream err = new ByteArrayOutputStream();
+        DefaultExecutor executor = DefaultExecutor.builder().get();
+        ExecuteWatchdog watchdog = ExecuteWatchdog.builder().setTimeout(Duration.ofMinutes(timeout)).get();
+        executor.setWatchdog(watchdog);
+        PumpStreamHandler streamHandler = new PumpStreamHandler(out, err);
+        executor.setStreamHandler(streamHandler);
+        try {
+            int exitValue = executor.execute(commandLine);
+            log.info("exitValue: {}", exitValue);
+            String outString = out.toString();
+            Long count = null;
+            Matcher matcher = PATTERN.matcher(outString);
+            if (matcher.find()) {
+                count = Long.parseLong(matcher.group(2));
+            }
+            if (count == null) {
+                throw new MyRuntimeException("导入数据失败");
+            }
+            log.info("插入 {} 条数据", count);
+            if (minInsertCount != null && count < minInsertCount) {
+                throw new MyRuntimeException(csv + " 数据异常，少于 " + minInsertCount);
+            }
+        } catch (Exception e) {
+            if (watchdog.killedProcess()) {
+                throw new MyRuntimeException("执行超时", e);
+            }
+            throw new MyRuntimeException(e);
+        } finally {
+            String outString = out.toString();
+            String errString = err.toString();
+            log.info("out: {}", outString);
+            if (StringUtils.hasText(errString)) {
+                log.error("err: {}", errString);
+            }
+        }
+    }
+}

src/main/java/com/nokia/siteinfo/config/DataSourceConfig.java

@@ -1,25 +1,65 @@
 package com.nokia.siteinfo.config;
 
-import org.springframework.boot.context.properties.ConfigurationProperties;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.jdbc.DataSourceBuilder;
 import org.springframework.context.annotation.Bean;
-import org.springframework.stereotype.Component;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import javax.sql.DataSource;
 
-@Component
+/**
+ * 数据源配置
+ */
+@Slf4j
+@Configuration
 public class DataSourceConfig {
+
     @Bean
-    @ConfigurationProperties(prefix = "spring.datasource.input")
-    public DataSource inputDataSource()
+    public DataSource dataSource1(TaskConfig taskConfig)
     {
-        return DataSourceBuilder.create().build();
+        return DataSourceBuilder.create()
+                .driverClassName(taskConfig.getDriverClassName1())
+                .url(taskConfig.getJdbcUrl1())
+                .username(taskConfig.getDbUsername1())
+                .password(taskConfig.getDbPassword1())
+                .build();
     }
 
     @Bean
-    @ConfigurationProperties(prefix = "spring.datasource.output")
-    public DataSource outputDataSource()
+    public DataSource dataSource2(TaskConfig taskConfig)
     {
-        return DataSourceBuilder.create().build();
+        return DataSourceBuilder.create()
+                .driverClassName(taskConfig.getDriverClassName2())
+                .url(taskConfig.getJdbcUrl2())
+                .username(taskConfig.getDbUsername2())
+                .password(taskConfig.getDbPassword2())
+                .build();
+    }
+
+    @Bean
+    public DataSource dataSource3(TaskConfig taskConfig)
+    {
+        return DataSourceBuilder.create()
+                .driverClassName(taskConfig.getDriverClassName3())
+                .url(taskConfig.getJdbcUrl3())
+                .username(taskConfig.getDbUsername3())
+                .password(taskConfig.getDbPassword3())
+                .build();
+    }
+
+    @Bean
+    public JdbcTemplate jdbcTemplate1(DataSource dataSource1) {
+        return new JdbcTemplate(dataSource1);
+    }
+
+    @Bean
+    public JdbcTemplate jdbcTemplate2(DataSource dataSource2) {
+        return new JdbcTemplate(dataSource2);
+    }
+
+    @Bean
+    public JdbcTemplate jdbcTemplate3(DataSource dataSource3) {
+        return new JdbcTemplate(dataSource3);
     }
 }

src/main/java/com/nokia/siteinfo/config/JdbcTemplateConfig.java

@@ -1,20 +0,0 @@
-package com.nokia.siteinfo.config;
-
-import org.springframework.context.annotation.Bean;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.stereotype.Component;
-
-import javax.sql.DataSource;
-
-@Component
-public class JdbcTemplateConfig {
-    @Bean
-    public JdbcTemplate inputJdbcTemplate(DataSource inputDataSource) {
-        return new JdbcTemplate(inputDataSource);
-    }
-
-    @Bean
-    public JdbcTemplate outputJdbcTemplate(DataSource outputDataSource) {
-        return new JdbcTemplate(outputDataSource);
-    }
-}

src/main/java/com/nokia/siteinfo/config/TaskConfig.java

@@ -0,0 +1,107 @@
+package com.nokia.siteinfo.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties("task.config")
+public class TaskConfig {
+    /**
+     * 导入数据脚本路径
+     */
+    private String importScript;
+    /**
+     * 数据库ip
+     */
+    private String dbHost1;
+    /**
+     * 数据库端口
+     */
+    private String dbPort1;
+    /**
+     * 数据库账号
+     */
+    private String dbUsername1;
+    /**
+     * 数据库密码
+     */
+    private String dbPassword1;
+    /**
+     * 数据库名称
+     */
+    private String dbName1;
+    /**
+     * 数据库驱动
+     */
+    private String driverClassName1;
+    /**
+     * jdbc链接
+     */
+    private String jdbcUrl1;
+    /**
+     * 数据库ip
+     */
+    private String dbHost2;
+    /**
+     * 数据库端口
+     */
+    private String dbPort2;
+    /**
+     * 数据库账号
+     */
+    private String dbUsername2;
+    /**
+     * 数据库密码
+     */
+    private String dbPassword2;
+    /**
+     * 数据库名称
+     */
+    private String dbName2;
+    /**
+     * 数据库驱动
+     */
+    private String driverClassName2;
+    /**
+     * jdbc链接
+     */
+    private String jdbcUrl2;
+    /**
+     * 数据库ip
+     */
+    private String dbHost3;
+    /**
+     * 数据库端口
+     */
+    private String dbPort3;
+    /**
+     * 数据库账号
+     */
+    private String dbUsername3;
+    /**
+     * 数据库密码
+     */
+    private String dbPassword3;
+    /**
+     * 数据库名称
+     */
+    private String dbName3;
+    /**
+     * 数据库驱动
+     */
+    private String driverClassName3;
+    /**
+     * jdbc链接
+     */
+    private String jdbcUrl3;
+    /**
+     * 最小插入数据
+     */
+    private Long minInsertCount;
+    /**
+     * 超时分钟
+     */
+    private Long timeout;
+}

src/main/java/com/nokia/siteinfo/task/UpdateTask.java

@@ -1,14 +1,14 @@
 package com.nokia.siteinfo.task;
 
-import com.nokia.common.gpload.GploadUtil;
-import com.nokia.common.gpload.entity.GploadResult;
+import com.nokia.common.exception.MyRuntimeException;
+import com.nokia.common.psql.PsqlUtil;
+import com.nokia.siteinfo.config.TaskConfig;
 import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
@@ -27,23 +27,21 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-@ConfigurationProperties("task")
 @Data
 @Component
 @Slf4j
 public class UpdateTask {
-    private String gploadShellPath;
-    /**
-     * 超时分钟
-     */
-    private Long timeout;
+    private final TaskConfig taskConfig;
+    private final JdbcTemplate jdbcTemplate1;
+    private final JdbcTemplate jdbcTemplate2;
+    private final JdbcTemplate jdbcTemplate3;
 
-    private final JdbcTemplate inputJdbcTemplate;
-    private final JdbcTemplate outputJdbcTemplate;
-
-    public UpdateTask(JdbcTemplate inputJdbcTemplate, JdbcTemplate outputJdbcTemplate) {
-        this.inputJdbcTemplate = inputJdbcTemplate;
-        this.outputJdbcTemplate = outputJdbcTemplate;
+    public UpdateTask(TaskConfig taskConfig, JdbcTemplate jdbcTemplate1, JdbcTemplate jdbcTemplate2,
+                      JdbcTemplate jdbcTemplate3) {
+        this.taskConfig = taskConfig;
+        this.jdbcTemplate1 = jdbcTemplate1;
+        this.jdbcTemplate2 = jdbcTemplate2;
+        this.jdbcTemplate3 = jdbcTemplate3;
     }
 
     //    0 0 1 ? * 2
@@ -54,14 +52,14 @@ public class UpdateTask {
                 try {
                     singleTask();
                 } catch (Exception e) {
-                    throw new RuntimeException(e);
+                    throw new MyRuntimeException(e);
                 }
-            }).get(timeout, TimeUnit.MINUTES);
+            }).get(29, TimeUnit.MINUTES);
         } catch (InterruptedException e) {
             log.error("线程中断: {}", e.getMessage(), e);
             XxlJobHelper.log("线程中断: {}", e.getMessage(), e);
             Thread.currentThread().interrupt();
-            throw new RuntimeException(e);
+            throw new MyRuntimeException(e);
         } catch (TimeoutException e) {
             log.error("执行超时: {}", e.getMessage(), e);
             XxlJobHelper.log("执行超时: {}", e.getMessage(), e);
@@ -79,21 +77,19 @@ public class UpdateTask {
         Map<String, Map<String, Object>> five = new HashMap<>();
         update4g(four);
         update5g(five);
-        writeCsvs(four, five, siteLevelMap);
-        backup("customer_service.cfg_p_netconf_std");
-        gpload("customer_service.cfg_p_netconf_std");
-        backup("customer_service.cfg_cell_info");
-        gpload("customer_service.cfg_cell_info");
+        Map<Object, Map<String, Object>> map = siteInfoToCfgPNetconfStd(four, five, siteLevelMap);
+        updateCfgPNetconfStd(map);
+        updateCfgCellInfo(map);
     }
 
     private Map<Object, Object> getSiteLevelMap() {
         Map<Object, Object> siteLevelMap = new HashMap<>();
         String sql = "select * from cfm.cfg_0_4g_sitelevel";
-        List<Map<String, Object>> l1 = outputJdbcTemplate.queryForList(sql);
+        List<Map<String, Object>> l1 = jdbcTemplate2.queryForList(sql);
         log.info("cfg_0_4g_sitelevel: {}", l1.size());
         l1.forEach(t -> siteLevelMap.put(t.get("site_id"), t.get("area3")));
         sql = "select * from cfm.cfg_0_5g_sitelevel";
-        List<Map<String, Object>> l2 = outputJdbcTemplate.queryForList(sql);
+        List<Map<String, Object>> l2 = jdbcTemplate2.queryForList(sql);
         log.info("cfg_0_5g_sitelevel: {}", l2.size());
         l2.forEach(t -> siteLevelMap.put(t.get("site_id"), t.get("area3")));
         log.info("siteLevelMap: {}", siteLevelMap.size());
@@ -103,19 +99,38 @@ public class UpdateTask {
     private void update4g(Map<String, Map<String, Object>> map) throws IOException {
         // 查询o2p.cfg_0_4g_siteinfo
         String sql = "select * from o2p.cfg_0_4g_siteinfo";
-        List<Map<String, Object>> list = inputJdbcTemplate.queryForList(sql);
+        List<Map<String, Object>> list = jdbcTemplate1.queryForList(sql);
         // 去重
         list.forEach(t -> map.put((String) t.get("cgi"), t));
         log.info("cfg_0_4g_siteinfo: {} -> {}", list.size(), map.size());
         XxlJobHelper.log("cfg_0_4g_siteinfo: {} -> {}", list.size(), map.size());
         String table = "cfm.cfg_0_4g_siteinfo";
-        writeCsv(map, table);
+        String csv = table + ".csv";
+        log.info("writeCsv: {}", table);
+        XxlJobHelper.log("writeCsv: {}", table);
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(csv)),
+                StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            // 表头
+            printer.printRecord("sdate","city_code","city_name","district_code","district_name","network_name","phystation_address","bbu_name","enbid","cell_name","cell_id","cgi","lon","lat","direction","height","m_downtilt","e_downtilt","station_type","isdigitalindoor","down_freq","vender","own_schoolyard","toweraddress_code","property","scene","is_scenesite","marketing_network","terminalamount_5g","sector_incoming","is_busy","is_alive","is_alive_update_time","construction","is_same_address","same_address_sites","is_same_address_ct");
+            for (Map<String, Object> t : map.values()) {
+                printer.printRecord(t.values());
+            }
+        }
         backup(table);
         // o2p.cfg_0_4g_siteinfo -> cfm.cfg_0_4g_siteinfo
-        gpload(table);
+        String columns = "(sdate,city_code,city_name,district_code,district_name,network_name,phystation_address,bbu_name,enbid,cell_name,cell_id,cgi,lon,lat,direction,height,m_downtilt,e_downtilt,station_type,isdigitalindoor,down_freq,vender,own_schoolyard,toweraddress_code,property,scene,is_scenesite,marketing_network,terminalamount_5g,sector_incoming,is_busy,is_alive,is_alive_update_time,construction,is_same_address,same_address_sites,is_same_address_ct)";
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost2(), taskConfig.getDbPort2(),
+                taskConfig.getDbUsername2(), taskConfig.getDbPassword2(), taskConfig.getDbName2(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost3(), taskConfig.getDbPort3(),
+                taskConfig.getDbUsername3(), taskConfig.getDbPassword3(), taskConfig.getDbName3(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
         // 查询cfm.cfg_0_4g_siteinfo_gx
         sql = "select * from cfm.cfg_0_4g_siteinfo_gx";
-        list = outputJdbcTemplate.queryForList(sql);
+        list = jdbcTemplate2.queryForList(sql);
         // 添加新元素
         list.forEach(t -> map.putIfAbsent((String) t.get("cgi"), t));
         log.info("cfg_0_4g_siteinfo_gx: {} -> {}", list.size(), map.size());
@@ -125,19 +140,38 @@ public class UpdateTask {
     private void update5g(Map<String, Map<String, Object>> map) throws IOException {
         // 查询o2p.cfg_0_5g_siteinfo
         String sql = "select * from o2p.cfg_0_5g_siteinfo";
-        List<Map<String, Object>> list = inputJdbcTemplate.queryForList(sql);
+        List<Map<String, Object>> list = jdbcTemplate1.queryForList(sql);
         // 去重
         list.forEach(t -> map.put((String) t.get("cgisai"), t));
         log.info("cfg_0_5g_siteinfo: {} -> {}", list.size(), map.size());
         XxlJobHelper.log("cfg_0_5g_siteinfo: {} -> {}", list.size(), map.size());
         String table = "cfm.cfg_0_5g_siteinfo";
-        writeCsv(map, table);
+        String csv = table + ".csv";
+        log.info("writeCsv: {}", table);
+        XxlJobHelper.log("writeCsv: {}", table);
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(csv)),
+                StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            // 表头
+            printer.printRecord("sdate","city_code","city_name","district_code","district_name","network_name","phystation_address","station_name","gnbid","cell_name","cell_id","cgisai","lon","lat","direction","height","m_downtilt","e_downtilt","station_type","isdigitalindoor","down_freq","vender","own_schoolyard","toweraddress_code","property","scene","is_scenesite","marketing_network","terminalamount_5g","sector_incoming","is_busy","is_alive","is_alive_update_time","construction","is_same_address","same_address_sites","is_same_address_ct");
+            for (Map<String, Object> t : map.values()) {
+                printer.printRecord(t.values());
+            }
+        }
         backup(table);
         // o2p.cfg_0_5g_siteinfo -> cfm.cfg_0_5g_siteinfo
-        gpload(table);
+        String columns = "(sdate,city_code,city_name,district_code,district_name,network_name,phystation_address,station_name,gnbid,cell_name,cell_id,cgisai,lon,lat,direction,height,m_downtilt,e_downtilt,station_type,isdigitalindoor,down_freq,vender,own_schoolyard,toweraddress_code,property,scene,is_scenesite,marketing_network,terminalamount_5g,sector_incoming,is_busy,is_alive,is_alive_update_time,construction,is_same_address,same_address_sites,is_same_address_ct)";
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost2(), taskConfig.getDbPort2(),
+                taskConfig.getDbUsername2(), taskConfig.getDbPassword2(), taskConfig.getDbName2(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost3(), taskConfig.getDbPort3(),
+                taskConfig.getDbUsername3(), taskConfig.getDbPassword3(), taskConfig.getDbName3(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
         // 查询cfm.cfg_0_5g_siteinfo_gx
         sql = "select * from cfm.cfg_0_5g_siteinfo_gx";
-        list = outputJdbcTemplate.queryForList(sql);
+        list = jdbcTemplate2.queryForList(sql);
         // 添加新元素
         list.forEach(t -> map.putIfAbsent((String) t.get("cgisai"), t));
         log.info("cfg_0_5g_siteinfo_gx: {} -> {}", list.size(), map.size());
@@ -145,26 +179,67 @@ public class UpdateTask {
     }
 
     /**
-     * 将cfg_0_4g_siteinfo、cfg_0_5g_siteinfo转成cfg_p_netconf_std、cfg_cell_info写入csv文件
+     * 将cfg_0_4g_siteinfo、cfg_0_5g_siteinfo转成cfg_p_netconf_std
+     */
+    private Map<Object, Map<String, Object>> siteInfoToCfgPNetconfStd(Map<String, Map<String, Object>> four,
+                                                                      Map<String, Map<String, Object>> five,
+                                                                      Map<Object, Object> siteLevelMap) {
+        Map<Object, Map<String, Object>> map = new HashMap<>();
+        fourGCfgPNetconfStd(four, map, siteLevelMap);
+        fiveGCfgPNetconfStd(five, map, siteLevelMap);
+        log.info("cfg_p_netconf_std: {}", map.size());
+        XxlJobHelper.log("cfg_p_netconf_std: {}", map.size());
+        return map;
+    }
+
+    /**
+     * 导入customer_service.cfg_p_netconf_std
+     */
+    private void updateCfgPNetconfStd(Map<Object, Map<String, Object>> map) throws IOException {
+        String table = "customer_service.cfg_p_netconf_std";
+        String csv = table + ".csv";
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(csv)),
+                StandardCharsets.UTF_8);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            // 表头
+            printer.printRecord("id","city_id","city_name","area_id","core_ne_name","radio_ne_name","site_name","site_id","cell_no","cell_name","cell_id","ecgi","gnb_cu_id","gnb_du_id","gsm_cell_id","network_type","type_5g","tac","cover_type","nodeb_type","rac","lac","grid","areatype","sub_areatype","area3","longitude","latitude","vendor","height","downtilt","azimuth","fr","scramber","province_id","bsbusip","bsbusip_1","isanchor","operator","isshare","slice_type");
+            printCfgPNetconfStd(printer, map);
+        }
+        backup(table);
+        String columns = "(id,city_id,city_name,area_id,core_ne_name,radio_ne_name,site_name,site_id,cell_no,cell_name,cell_id,ecgi,gnb_cu_id,gnb_du_id,gsm_cell_id,network_type,type_5g,tac,cover_type,nodeb_type,rac,lac,grid,areatype,sub_areatype,area3,longitude,latitude,vendor,height,downtilt,azimuth,fr,scramber,province_id,bsbusip,bsbusip_1,isanchor,operator,isshare,slice_type)";
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost2(), taskConfig.getDbPort2(),
+                taskConfig.getDbUsername2(), taskConfig.getDbPassword2(), taskConfig.getDbName2(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost3(), taskConfig.getDbPort3(),
+                taskConfig.getDbUsername3(), taskConfig.getDbPassword3(), taskConfig.getDbName3(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
+    }
+
+    /**
+     * 导入customer_service.cfg_cell_info
      */
-    private void writeCsvs(Map<String, Map<String, Object>> four, Map<String, Map<String, Object>> five,
-                           Map<Object, Object> siteLevelMap) throws IOException {
-        try (OutputStreamWriter osw1 = new OutputStreamWriter(
-                Files.newOutputStream(Paths.get("customer_service.cfg_p_netconf_std.csv")),
+    private void updateCfgCellInfo(Map<Object, Map<String, Object>> map) throws IOException {
+        String table = "customer_service.cfg_cell_info";
+        String csv = table + ".csv";
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(csv)),
                 StandardCharsets.UTF_8);
-             CSVPrinter printer1 = new CSVPrinter(osw1, CSVFormat.DEFAULT);
-             OutputStreamWriter osw2 = new OutputStreamWriter(
-                     Files.newOutputStream(Paths.get("customer_service.cfg_cell_info.csv")),
-                     StandardCharsets.UTF_8);
-             CSVPrinter printer2 = new CSVPrinter(osw2, CSVFormat.DEFAULT);) {
-            Map<Object, Map<String, Object>> map = new HashMap<>();
-            fourGCfgPNetconfStd(four, map, siteLevelMap);
-            fiveGCfgPNetconfStd(five, map, siteLevelMap);
-            log.info("cfg_p_netconf_std: {}", map.size());
-            XxlJobHelper.log("cfg_p_netconf_std: {}", map.size());
-            printCfgPNetconfStd(printer1, map);
-            printCfgCellInfo(printer2, map);
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
+            // 表头
+            printer.printRecord("cell_id","site_name","city_name","longitude","latitude","tac");
+            printCfgCellInfo(printer, map);
         }
+        backup(table);
+        String columns = "(cell_id,site_name,city_name,longitude,latitude,tac)";
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost2(), taskConfig.getDbPort2(),
+                taskConfig.getDbUsername2(), taskConfig.getDbPassword2(), taskConfig.getDbName2(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
+        PsqlUtil.copyCsv(taskConfig.getImportScript(), taskConfig.getDbHost3(), taskConfig.getDbPort3(),
+                taskConfig.getDbUsername3(), taskConfig.getDbPassword3(), taskConfig.getDbName3(), table, csv,
+                columns, taskConfig.getTimeout(), taskConfig.getMinInsertCount());
     }
 
     /**
@@ -286,7 +361,9 @@ public class UpdateTask {
         XxlJobHelper.log("writeCsv: {}", table);
         try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(table + ".csv")),
                 StandardCharsets.UTF_8);
-             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT);) {
+             CSVPrinter printer = new CSVPrinter(osw, CSVFormat.DEFAULT)) {
+            // 添加bom头避免excel乱码
+            osw.write('\ufeff');
             for (Map<String, Object> t : map.values()) {
                 printer.printRecord(t.values());
             }
@@ -305,7 +382,8 @@ public class UpdateTask {
         String nowString = dateFormat(now);
         // 备份
         String sql = "create table " + table + "_bak_" + nowString + " as select * from " + table;
-        outputJdbcTemplate.execute(sql);
+        jdbcTemplate2.execute(sql);
+        jdbcTemplate3.execute(sql);
         // 删除之前的备份
         sql = "DO $$\n" +
                 "DECLARE t varchar(255);\n" +
@@ -317,10 +395,12 @@ public class UpdateTask {
                 "END LOOP;\n" +
                 "END;\n" +
                 "$$";
-        outputJdbcTemplate.execute(sql);
+        jdbcTemplate2.execute(sql);
+        jdbcTemplate3.execute(sql);
         // 清空表
         sql = "truncate table " + table;
-        outputJdbcTemplate.execute(sql);
+        jdbcTemplate2.execute(sql);
+        jdbcTemplate3.execute(sql);
     }
 
     /**
@@ -333,19 +413,6 @@ public class UpdateTask {
         return localDate.format(DateTimeFormatter.ofPattern("MMdd"));
     }
 
-    private void gpload(String table) throws IOException {
-        String gploadCommand = "sh " + gploadShellPath + " " + table;
-        GploadResult gpload = GploadUtil.gpload(gploadCommand);
-        if (Boolean.TRUE.equals(gpload.getTaskStatus())) {
-            log.info("gpload {} 完成: {}", table, gpload);
-            XxlJobHelper.log("gpload {} 完成: {}", table, gpload);
-            // 删除文件
-            Files.deleteIfExists(Paths.get(table + ".csv"));
-        } else {
-            throw new RuntimeException("gpload " + table + " 失败: " + gpload.getMessage());
-        }
-    }
-
     /**
      * 将cfg_p_netconf_std写入csv文件
      */

src/main/resources/application-prod.properties

@@ -1,15 +1,7 @@
 server.port=12093
 spring.application.name=siteinfo_update
-logging.level.root=info
-spring.datasource.input.jdbc-url=jdbc:postgresql://192.168.10.9:5432/sqmmt
-spring.datasource.input.username=pmparse
-spring.datasource.input.password=Richr00t#
-spring.datasource.input.driverClassName=org.postgresql.Driver
-
-spring.datasource.output.jdbc-url=jdbc:postgresql://192.168.70.109:5432/sqmmt
-spring.datasource.output.username=sqmdb
-spring.datasource.output.password=sqmdb_1QAZ
-spring.datasource.output.driverClassName=org.postgresql.Driver
+logging.level.com.nokia=debug
+logging.level.org.springframework.jdbc.core.JdbcTemplate=debug
 ### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
 xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
 ### xxl-job, access token
@@ -26,5 +18,27 @@ xxl.job.executor.logpath=./log/xxl/
 ### xxl-job executor log-retention-days
 xxl.job.executor.logretentiondays=30
 
-task.gpload-shell-path=/data/site_info/gpload/gpload.sh
-task.timeout=25
+task.config.min-insert-count=10000
+task.config.import-script=copy.sh
+task.config.timeout=3
+task.config.jdbc-url1=jdbc:postgresql://192.168.10.9:5432/sqmmt
+task.config.driver-class-name1=org.postgresql.Driver
+task.config.db-host1=192.168.10.9
+task.config.db-port1=5432
+task.config.db-username1=pmparse
+task.config.db-password1=Richr00t#
+task.config.db-name1=sqmmt
+task.config.jdbc-url2=jdbc:postgresql://192.168.70.109:5432/sqmmt
+task.config.driver-class-name2=org.postgresql.Driver
+task.config.db-host2=192.168.70.109
+task.config.db-port2=5432
+task.config.db-username2=sqmdb
+task.config.db-password2=sqmdb_1QAZ
+task.config.db-name2=sqmmt
+task.config.jdbc-url3=jdbc:postgresql://192.168.70.172:5432/sqmmt
+task.config.driver-class-name3=org.postgresql.Driver
+task.config.db-host3=192.168.70.172
+task.config.db-port3=5432
+task.config.db-username3=sqmdb
+task.config.db-password3=sqmdb_1QAZ
+task.config.db-name3=sqmmt

src/main/resources/application-test.properties

@@ -1,15 +1,7 @@
 server.port=12093
 spring.application.name=siteinfo_update
 logging.level.com.nokia=debug
-spring.datasource.input.jdbc-url=jdbc:postgresql://192.168.10.9:5432/sqmmt
-spring.datasource.input.username=pmparse
-spring.datasource.input.password=Richr00t#
-spring.datasource.input.driverClassName=org.postgresql.Driver
-
-spring.datasource.output.jdbc-url=jdbc:postgresql://192.168.50.5:5432/sqmmt
-spring.datasource.output.username=sqmdb
-spring.datasource.output.password=sqmdb_1QAZ
-spring.datasource.output.driverClassName=org.postgresql.Driver
+logging.level.org.springframework.jdbc.core.JdbcTemplate=debug
 ### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
 xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
 ### xxl-job, access token
@@ -26,5 +18,27 @@ xxl.job.executor.logpath=./log/xxl/
 ### xxl-job executor log-retention-days
 xxl.job.executor.logretentiondays=30
 
-task.gpload-shell-path=/data1/site_info/gpload/gpload.sh
-task.timeout=25
+task.config.min-insert-count=10000
+task.config.import-script=copy.sh
+task.config.timeout=3
+task.config.jdbc-url1=jdbc:postgresql://192.168.10.9:5432/sqmmt
+task.config.driver-class-name1=org.postgresql.Driver
+task.config.db-host1=192.168.10.9
+task.config.db-port1=5432
+task.config.db-username1=pmparse
+task.config.db-password1=Richr00t#
+task.config.db-name1=sqmmt
+task.config.jdbc-url2=jdbc:postgresql://192.168.50.5:5432/sqmmt
+task.config.driver-class-name2=org.postgresql.Driver
+task.config.db-host2=192.168.50.5
+task.config.db-port2=5432
+task.config.db-username2=sqmdb
+task.config.db-password2=sqmdb_1QAZ
+task.config.db-name2=sqmmt
+task.config.jdbc-url3=jdbc:postgresql://192.168.50.3:15432/sqmmt
+task.config.driver-class-name3=org.postgresql.Driver
+task.config.db-host3=192.168.50.3
+task.config.db-port3=15432
+task.config.db-username3=postgres
+task.config.db-password3=NFQCgBA6YhNvgAqG6THw
+task.config.db-name3=sqmmt