pm4G修补了 一些问题

other/gpload/pm_lte_gpload.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 
-source /usr/local/greenplum-db-clients/greenplum_loaders_path.sh
+source /data/xxl-job/greenplum_loaders_path.sh
 
 new_date=$1
 
 # 修改gpload配置文件
-sed -i 's/\/data1\/pm\/distinct\/pm_4g_hour_[0-9]\{10\}.csv/\/data1\/pm\/distinct\/pm_4g_hour_'${new_date}'.csv/' /data1/pm/gpload/pm_lte_gpload2.yml
+sed -i 's/\/data\/xxl-job\/distinct\/pm_4g_hour_[0-9]\{10\}.csv/\/data\/xxl-job\/distinct\/pm_4g_hour_'${new_date}'.csv/' /data/xxl-job/pm_lte_gpload2.yml
 
 # 由于gpload需要输入密码，这里需要使用expect执行
 password=Richr00t#
 
 expect -c "
 set timeout 300
-spawn gpload -f /data1/pm/gpload/pm_lte_gpload2.yml
+spawn gpload -f /data/xxl-job/pm_lte_gpload2.yml
 expect {
 \"connecting (yes/no)?\" { send \"yes\n\";exp_continue }
 \"Password:\" { send \"${password}\n\"; exp_continue}

other/gpload/pm_lte_gpload2.yml

@@ -1,16 +1,16 @@
 VERSION: 1.0.0.1
 DATABASE: sqmmt
 USER: gpadmin
-HOST: 192.168.70.109
-PORT: 5432
+HOST: 192.168.10.7
+PORT: 22
 GPLOAD:
   INPUT:
     - SOURCE:
         LOCAL_HOSTNAME:
-          - 192.168.70.130
-        PORT: 54321
+          - 192.168.50.5
+        PORT: 5432
         FILE:
-          - /data1/pm/distinct/pm_4g_hour_2022052409.csv
+          - /data/xxl-job/distinct/pm_4g_hour_2022052409.csv
     - FORMAT: csv
     - DELIMITER: ','
     - HEADER: true

src/main/java/com/nokia/common/ssh/SSHUtil.java

@@ -77,6 +77,7 @@ public class SSHUtil {
         session = getConnectSession();
         channelSftp = (ChannelSftp) session.openChannel("sftp");
         channelSftp.connect();
+        System.out.println(fileName);
         channelSftp.rm(fileName);
         channelSftp.quit();
         session.disconnect();

src/main/java/com/nokia/task/LtePmTask.java

@@ -1,5 +1,4 @@
 package com.nokia.task;
-
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.SftpException;
 import com.nokia.common.gpload.GploadUtil;
@@ -15,7 +14,6 @@ import org.apache.commons.csv.CSVRecord;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -25,8 +23,6 @@ import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 @Slf4j
 /*@Task*/
@@ -36,9 +32,9 @@ public class LtePmTask {
     private String host;
     @Value("${lte.pm.download.port:22}")
     private Integer port;
-    @Value("${lte.pm.download.username:esbftp}")
+    @Value("${lte.pm.download.username:nokia}")
     private String username;
-    @Value("${lte.pm.download.password:Esb2019ftp!}")
+    @Value("${lte.pm.download.password:Nokia*123}")
     private String password;
     @Value("${lte.pm.download.sourceDir:/data/out2/pm_4g_hour}")
     private String sourceDir;
@@ -49,9 +45,11 @@ public class LtePmTask {
     @Value("${lte.pm.distinct.targetDir:distinct/}")
     private String distinctTargetDir;
 
-    /**
-     * 扫描文件
-     */
+ /*   @XxlJob("execHandler")
+     public void Test(){
+        System.out.println("java------------------");
+    }
+*/
 
     @XxlJob("execHandler")
     public void cronTask() throws JSchException, SSHUtilException, IOException, SftpException {
@@ -81,15 +79,16 @@ public class LtePmTask {
     }*/
 
     /*@AllowedTaskType({TaskType.TIMING, TaskType.IMMEDIATE})*/
-    public void singleTask(String hourString) throws JSchException, SSHUtilException, IOException {
-        download();
-        distinct();
+    public void singleTask(String hourString) throws JSchException, SSHUtilException, IOException, SftpException {
+        download(hourString);
+        distinct(hourString);
         gpload(hourString);
     }
 
-    public void download() throws JSchException, SSHUtilException, IOException {
+    public void download(String hourString) throws JSchException, SSHUtilException, IOException, SftpException {
         SSHUtil sshUtil = new SSHUtil(host, port, username, password);
-        String sourceFilePath = sourceDir;
+        String downloadFileName  = hourString+".csv";
+        String sourceFilePath=sourceDir + "/" + downloadFileName;
         File targetFile = new File(downloadTargetDir);
         if (!targetFile.exists()) {
             targetFile.getParentFile().mkdirs();
@@ -98,15 +97,16 @@ public class LtePmTask {
         boolean b = sshUtil.scpFrom(sourceFilePath, targetPath);
         if (b) {
             log.debug("文件 {} 下载成功...", targetPath);
-            String exec = sshUtil.exec(sourceDir);
-            File file = new File(exec);
-            file.delete();
+           sshUtil.delete(sourceFilePath);
+            System.out.println("删除成功--------------------");
         }
     }
 
-    public void distinct() throws IOException {
-        String inputFilePath = downloadTargetDir ;
-        String outputFilePath = distinctTargetDir;
+    public void distinct(String hourString) throws IOException {
+        String fileName = hourString + ".csv";
+        System.out.println(fileName+"---------------------------------");
+        String inputFilePath = downloadTargetDir +fileName;
+        String outputFilePath = distinctTargetDir+fileName;
         CSVFormat format = CSVFormat.DEFAULT.builder().build();
         CSVParser records = format.parse(new InputStreamReader(Files.newInputStream(Paths.get(inputFilePath)), StandardCharsets.UTF_8));
         Map<String, CSVRecord> recordMap = new HashMap<>();
@@ -125,7 +125,7 @@ public class LtePmTask {
     }
 
     public void gpload(String hourString) {
-        String gploadCommand = "sh /data1/pm/gpload/pm_lte_gpload.sh " + hourString;
+        String gploadCommand = "sh /data/xxl-job/pm_lte_gpload.sh " + hourString;
         GploadResult gpload = GploadUtil.gpload(gploadCommand);
         if (Boolean.TRUE.equals(gpload.getTaskStatus())) {
             log.debug("gpload完成: {}", gpload);

src/main/resources/application.properties

@@ -18,9 +18,8 @@ pm.download.directory=D:\\download
 pm.webdriver.chrome.driver=C:\\Users\\DELL\\Desktop\\chromedriver.exe
 pm.driver.options.headless=false
 pm.download.timeoutSeconds=120
-
 ### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
-xxl.job.admin.addresses=http://127.0.0.1:8087/xxl-job-admin
+xxl.job.admin.addresses=http://192.168.10.7:8087/xxl-job-admin
 ### xxl-job, access token
 xxl.job.accessToken=
 
@@ -32,6 +31,6 @@ xxl.job.executor.address=
 xxl.job.executor.ip=
 xxl.job.executor.port=8888
 ### xxl-job executor log-path
-xxl.job.executor.logpath=/data/applogs/xxl-job/jobhandler
+xxl.job.executor.logpath=/data/xxl-job/logs
 ### xxl-job executor log-retention-days
 xxl.job.executor.logretentiondays=30