init

.gitignore

@@ -0,0 +1,554 @@
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+### Maven template
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+# https://github.com/takari/maven-wrapper#usage-without-binary-jar
+.mvn/wrapper/maven-wrapper.jar
+
+### VisualStudioCode template
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Eclipse template
+.metadata
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
+.cproject
+
+# CDT- autotools
+.autotools
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/
+
+# Annotation Processing
+.apt_generated/
+.apt_generated_test/
+
+# Scala IDE specific (Scala & Java development for Eclipse)
+.cache-main
+.scala_dependencies
+.worksheet
+
+# Uncomment this line if you wish to ignore the project description file.
+# Typically, this file would be tracked if it contains build/dependency configurations:
+#.project
+
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+### VisualStudio template
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp_proj
+*_wpftmp.csproj
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+

other/gpload/pm_lte_gpload.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source /usr/local/greenplum-db-clients/greenplum_loaders_path.sh
+
+new_date=$1
+
+# 修改gpload配置文件
+sed -i 's/\/data1\/pm\/distinct\/pm_4g_hour_[0-9]\{10\}.csv/\/data1\/pm\/distinct\/pm_4g_hour_'${new_date}'.csv/' /data1/pm/gpload/pm_lte_gpload2.yml
+
+# 由于gpload需要输入密码，这里需要使用expect执行
+password=Richr00t#
+
+expect -c "
+set timeout 300
+spawn gpload -f /data1/pm/gpload/pm_lte_gpload2.yml
+expect {
+\"connecting (yes/no)?\" { send \"yes\n\";exp_continue }
+\"Password:\" { send \"${password}\n\"; exp_continue}
+timeout { puts \"超时\" exit 2}
+}
+"

other/gpload/pm_lte_gpload2.yml

@@ -0,0 +1,22 @@
+VERSION: 1.0.0.1
+DATABASE: sqmmt
+USER: gpadmin
+HOST: 192.168.70.109
+PORT: 5432
+GPLOAD:
+  INPUT:
+    - SOURCE:
+        LOCAL_HOSTNAME:
+          - 192.168.70.130
+        PORT: 54321
+        FILE:
+          - /data1/pm/distinct/pm_4g_hour_2022052409.csv
+    - FORMAT: csv
+    - DELIMITER: ','
+    - HEADER: true
+    - ENCODING: utf-8
+    - ERROR_LIMIT: 20000000
+    - LOG_ERRORS: true
+  OUTPUT:
+    - TABLE: tsfx.dw_ft_re_st_eutrancell_ind_h
+    - MODE: insert

pom.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.nokia</groupId>
+    <artifactId>pm_interface</artifactId>
+    <version>1.0</version>
+
+    <parent>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <groupId>org.springframework.boot</groupId>
+        <version>2.3.0.RELEASE</version>
+        <relativePath/>
+    </parent>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <skipTests>true</skipTests>
+    </properties>
+
+    <dependencies>
+       <!-- <dependency>
+            <groupId>com.nokia</groupId>
+            <artifactId>scheduling-core</artifactId>
+            <version>1.1</version>
+        </dependency>-->
+
+        <dependency>
+            <groupId>com.jcraft</groupId>
+            <artifactId>jsch</artifactId>
+            <version>0.1.55</version>
+        </dependency>
+        <dependency>
+            <groupId>com.xuxueli</groupId>
+            <artifactId>xxl-job-core</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>2.6.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+            <version>2.3.0.RELEASE</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

src/main/java/com/nokia/PmInterfaceApplication.java

@@ -0,0 +1,11 @@
+package com.nokia;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class PmInterfaceApplication{
+    public static void main(String[] args) {
+        SpringApplication.run(PmInterfaceApplication.class, args);
+    }
+}

src/main/java/com/nokia/common/gpload/GploadUtil.java

@@ -0,0 +1,86 @@
+package com.nokia.common.gpload;
+
+import com.nokia.common.gpload.entity.GploadResult;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class GploadUtil {
+
+    private static Process process = null;
+    private static BufferedReader reader = null;
+
+    public static GploadResult gpload(String gploadCommand) {
+        GploadResult result = new GploadResult();
+        try {
+            Process process = Runtime.getRuntime().exec(gploadCommand);
+            reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+            List<String> lines = new ArrayList<>();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                lines.add(line);
+            }
+            for (int i = 0; i < lines.size(); i++) {
+                if (lines.get(i).contains("|INFO|rows Inserted")) {
+                    // 找到第一行 对应的是插入数据的数量
+                    // s4可以判断gpload是否成功
+                    String s4 = lines.get(i + 3);
+                    if (s4.endsWith("succeeded")) {
+                        // 插入数量
+                        String s1 = lines.get(i);
+                        s1 = s1.substring(s1.indexOf('=') + 2);
+                        // 更新数量
+                        String s2 = lines.get(i + 1);
+                        s2 = s2.substring(s2.indexOf('=') + 2);
+                        // 错误数量
+                        String s3 = lines.get(i + 2);
+                        s3 = s3.substring(s3.indexOf('=') + 2);
+                        result.setInsertedCount(Integer.parseInt(s1));
+                        result.setUpdatedCount(Integer.parseInt(s2));
+                        result.setErrorCount(Integer.parseInt(s3));
+                        result.setTaskStatus(true);
+                    } else {
+                        result.setTaskStatus(false);
+                        StringBuilder sb = new StringBuilder();
+                        for (int j = i - 1; j > 0; j--) {
+                            sb.insert(0, lines.get(i));
+                            if (lines.get(i).contains("|ERROR|ERROR:")) {
+                                sb.insert(0, lines.get(i));
+                                break;
+                            }
+                        }
+                        result.setMessage(sb.toString());
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            result.setTaskStatus(false);
+            result.setMessage(e.getMessage());
+        } finally {
+            log.debug("gpload的结果为: {}", result);
+            destroy();
+        }
+        return result;
+    }
+
+    private static void destroy() {
+        if (reader != null) {
+            try {
+                reader.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            reader = null;
+        }
+        if (process != null) {
+            process.destroy();
+            process = null;
+        }
+    }
+}

src/main/java/com/nokia/common/gpload/entity/GploadResult.java

@@ -0,0 +1,12 @@
+package com.nokia.common.gpload.entity;
+
+import lombok.Data;
+
+@Data
+public class GploadResult {
+    private Boolean taskStatus;
+    private Integer insertedCount;
+    private Integer updatedCount;
+    private Integer errorCount;
+    private String message;
+}

src/main/java/com/nokia/common/ssh/SSHUtil.java

@@ -0,0 +1,303 @@
+package com.nokia.common.ssh;
+
+import com.jcraft.jsch.*;
+import com.nokia.common.ssh.entity.SSHServer;
+import com.nokia.common.ssh.entity.UserInfoImpl;
+import com.nokia.common.ssh.exception.SSHUtilException;
+import com.nokia.common.ssh.exception.ScpAckErrorException;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.*;
+import java.nio.file.NoSuchFileException;
+import java.util.Properties;
+
+/**
+ * 使用jsch库实现的ssh的工具类
+ * <p>
+ * todo: scpTo和scpFrom 在本机和targetServer默认编码不一致的时候,文件名中的中文会乱码,但是不会影响到文件内容,
+ */
+
+@Slf4j
+public class SSHUtil {
+
+    @Getter
+    @Setter
+    private SSHServer targetServer = new SSHServer();
+    private Session session = null;
+    private Channel channel = null;
+    private JSch jSch = null;
+    private FileInputStream fileInputStream = null;
+    private FileOutputStream fileOutputStream = null;
+    private OutputStream outputStream = null;
+    private InputStream inputStream = null;
+
+    public SSHUtil() {
+    }
+
+    public SSHUtil(String host, String user, String password) {
+        targetServer = new SSHServer(host, 22, user, password);
+    }
+
+    public SSHUtil(String host, Integer port, String user, String password) {
+        targetServer = new SSHServer(host, port, user, password);
+    }
+
+    /**
+     * 远程执行指令
+     */
+    public String exec(String command) throws JSchException, IOException {
+        StringBuilder stringBuilder = new StringBuilder();
+        try {
+            Session session = getConnectSession();
+            channel = session.openChannel("exec");
+            // jsch的登陆是无环境登陆即非login状态登陆,因此是没有环境变量的,
+            String execCommand;
+            // 在命令前添加 bash --login -c "command"以获取环境变量
+            // source .bashrc && command 也可以解决问题, 但是可能环境加载不全
+            if (command.startsWith("bash --login -c")) {
+                execCommand = command;
+            } else {
+                execCommand = String.format("bash --login -c \"%s\"", command);
+            }
+            ((ChannelExec) channel).setCommand(execCommand);
+            channel.setInputStream(null);
+            ((ChannelExec) channel).setErrStream(System.err);
+            InputStream in = channel.getInputStream();
+            channel.connect();
+            byte[] tmp = new byte[1024];
+            while (true) {
+                while (in.available() > 0) {
+                    int i = in.read(tmp, 0, 1024);
+                    if (i < 0) break;
+                    stringBuilder.append(new String(tmp, 0, i));
+                }
+                if (channel.isClosed()) {
+                    if (in.available() > 0) continue;
+                    break;
+                }
+            }
+        } finally {
+            disconnect();
+        }
+        return stringBuilder.toString();
+    }
+
+    /**
+     * 使用SCP把本地文件推送到targetServer目录下
+     * <p>
+     * 注意,文件名不能包含中文
+     */
+    public boolean scpTo(String sourceFilePath, String targetPath) throws JSchException, IOException, SSHUtilException {
+        try {
+            session = getConnectSession();
+            // scp内置了两个参数 -t 和 -f ，这两个参数是隐藏的，不会被用户显式提供，
+            // 两个scp进程之间传输数据时，远端机器上的scp进程被本地scp进程启动起来时提供上去。
+            // 需要说明的是，这是通过本地scp进程经ssh远程过去开启远端机器的scp进程来实现的。
+            // -t 指定为to 也就是目的端模式 指定的对象就是session对应的连接对象targetServer
+            String command = "scp " + "-t " + targetPath;
+            channel = session.openChannel("exec");
+            ((ChannelExec) channel).setCommand(command);
+            outputStream = channel.getOutputStream();
+            inputStream = channel.getInputStream();
+            channel.connect();
+            if (checkAck(inputStream) != 0) {
+                log.error("scpTo 执行失败");
+                return false;
+            }
+            File sourceFile = new File(sourceFilePath);
+            if (sourceFile.isDirectory()) {
+                log.error("sourceFilePath 必须是文件");
+                return false;
+            }
+            long fileSize = sourceFile.length();
+            command = "C0644 " + fileSize + " " + sourceFile.getName() + "\n";
+            outputStream.write(command.getBytes());
+            outputStream.flush();
+            if (checkAck(inputStream) != 0) {
+                log.error("scpTo 执行失败");
+                return false;
+            }
+            fileInputStream = new FileInputStream(sourceFile);
+            byte[] buffer = new byte[1024];
+            while (true) {
+                int len = fileInputStream.read(buffer, 0, buffer.length);
+                if (len <= 0) break;
+                outputStream.write(buffer, 0, len);
+            }
+            buffer[0] = 0;
+            outputStream.write(buffer, 0, 1);
+            outputStream.flush();
+            return checkAck(inputStream) == 0;
+        } finally {
+            disconnect();
+        }
+    }
+
+    /**
+     * 使用scp把targetServer目录下的文件复制到本地
+     */
+    public boolean scpFrom(String sourceFilePath, String targetPath) throws JSchException, IOException, SSHUtilException {
+        try {
+            log.debug(sourceFilePath);
+            session = getConnectSession();
+            // scp内置了两个参数 -t 和 -f ，这两个参数是隐藏的，不会被用户显式提供，
+            // 两个scp进程之间传输数据时，远端机器上的scp进程被本地scp进程启动起来时提供上去。
+            // 需要说明的是，这是通过本地scp进程经ssh远程过去开启远端机器的scp进程来实现的。
+            // -f 指定对端为from 也就是源端模式 指定的对象就是session对应的连接对象targetServer
+            String command = "scp -f " + sourceFilePath;
+            Channel channel = session.openChannel("exec");
+            ((ChannelExec) channel).setCommand(command);
+            outputStream = channel.getOutputStream();
+            inputStream = channel.getInputStream();
+            channel.connect();
+            byte[] buf = new byte[1024];
+            // 发送指令 '0'
+            // 源端会一直等宿端的回应, 直到等到回应才会传输下一条协议文本.
+            // 在送出最后一条协议文本后, 源端会传出一个大小为零的字符'0'来表示真正文件传输的开始.
+            // 当文件接收完成后, 宿端会给源端发送一个'0'
+            buf[0] = 0;
+            outputStream.write(buf, 0, 1);
+            outputStream.flush();
+            // 接收C0644 这条消息携带了文件的信息
+            while (true) {
+                int c = checkAck(inputStream);
+                // 遇到C时跳出循环
+                if (c == 'C') {
+                    break;
+                }
+            }
+            // 接收 '0644 ' 这段字符表示文件的权限
+            inputStream.read(buf, 0, 5);
+            // 获取filesize
+            long filesize = 0L;
+            while (true) {
+                if (inputStream.read(buf, 0, 1) < 0) {
+                    break;
+                }
+                if (buf[0] == ' ') break;
+                filesize = filesize * 10L + (long) (buf[0] - '0');
+            }
+            // 从 C0644命令读取文件名,命令中的文件名是不带路径的
+            String file = null;
+            for (int i = 0; ; i++) {
+                inputStream.read(buf, i, 1);
+                // 0x0a 是LF 换行符
+                if (buf[i] == (byte) 0x0a) {
+                    file = new String(buf, 0, i);
+                    break;
+                }
+            }
+            log.debug("filesize={}, file={}", filesize, file);
+            // 发送 '0'
+            buf[0] = 0;
+            outputStream.write(buf, 0, 1);
+            outputStream.flush();
+            // 如果目标是目录,则需要加上文件名
+            File target = new File(targetPath);
+            if (target.isDirectory()) {
+                log.debug("{} 是目录,需要添加文件名", target.getAbsolutePath());
+                target = new File(targetPath + File.separator + file);
+            }
+
+            fileOutputStream = new FileOutputStream(target);
+            int foo;
+            while (true) {
+                if (buf.length < filesize) {
+                    foo = buf.length;
+                } else {
+                    foo = (int) filesize;
+                }
+                foo = inputStream.read(buf, 0, foo);
+                if (foo < 0) {
+                    break;
+                }
+                fileOutputStream.write(buf, 0, foo);
+                filesize -= foo;
+                if (filesize == 0L) break;
+            }
+            if (checkAck(inputStream) != 0) {
+                return false;
+            }
+            // 发送 '0'
+            buf[0] = 0;
+            outputStream.write(buf, 0, 1);
+            outputStream.flush();
+            log.debug("scp from {}@{}:{}{} to {} 完成", targetServer.getUser(), targetServer.getHost(), targetServer.getPort(), sourceFilePath, target.getAbsolutePath());
+            return true;
+        } finally {
+            disconnect();
+        }
+    }
+
+    private Session getConnectSession() throws JSchException {
+        jSch = new JSch();
+        session = jSch.getSession(targetServer.getUser(), targetServer.getHost(), targetServer.getPort());
+        session.setPassword(targetServer.getPassword());
+        session.setUserInfo(new UserInfoImpl());
+        // 不需要输入保存ssh安全密钥的yes或no
+        Properties properties = new Properties();
+        properties.put("StrictHostKeyChecking", "no");
+        session.setConfig(properties);
+        session.connect();
+        log.debug("已连接到{}@{}:{}", targetServer.getUser(), targetServer.getHost(), targetServer.getPort());
+        return session;
+    }
+
+    private void disconnect() throws IOException {
+        if (fileOutputStream != null) {
+            fileOutputStream.close();
+            fileOutputStream = null;
+        }
+        if (fileInputStream != null) {
+            fileInputStream.close();
+            fileInputStream = null;
+        }
+        if (outputStream != null) {
+            outputStream.close();
+            outputStream = null;
+        }
+        if (channel != null) {
+            channel.disconnect();
+            channel = null;
+        }
+        if (session != null) {
+            session.disconnect();
+            session = null;
+        }
+        jSch = null;
+    }
+
+    /**
+     * 来自源端的每条消息和每个传输完毕的文件都需要宿端的确认和响应.
+     * 宿端会返回三种确认消息: 0(正常), 1(警告)或2(严重错误, 将中断连接).
+     * 消息1和2可以跟一个字符串和一个换行符, 这个字符串将显示在scp的源端. 无论这个字符串是否为空, 换行符都是不可缺少的.
+     */
+    private static int checkAck(InputStream in) throws IOException, SSHUtilException {
+        int b = in.read();
+        // b 取值为0表示成功
+        if (b == 0) return b;
+        if (b == -1) return b;
+
+        // 1表示警告 2表示严重错误,将中断连接
+        // 1和2 后面会携带一条错误信息,以\n结尾
+        if (b == 1 || b == 2) {
+            // 打印消息后面跟的字符串
+            StringBuilder sb = new StringBuilder();
+            int c;
+            do {
+                // 读取字符串直到遇到换行符
+                c = in.read();
+                sb.append((char) c);
+            } while (c != '\n');
+            log.debug("checkAck发现错误消息: ack={}-msg={}", b, sb.toString());
+            if (b == 1 && sb.toString().endsWith("No such file or directory")) {
+                throw new NoSuchFileException(sb.toString());
+            } else {
+                throw new ScpAckErrorException(sb.toString());
+            }
+        }
+        return b;
+    }
+}

src/main/java/com/nokia/common/ssh/entity/SSHServer.java

@@ -0,0 +1,15 @@
+package com.nokia.common.ssh.entity;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class SSHServer {
+    private String host;
+    private int port = 22;
+    private String user;
+    private String password;
+}

src/main/java/com/nokia/common/ssh/entity/UserInfoImpl.java

@@ -0,0 +1,35 @@
+package com.nokia.common.ssh.entity;
+
+import com.jcraft.jsch.UserInfo;
+
+public class UserInfoImpl implements UserInfo {
+    @Override
+    public String getPassphrase() {
+        return null;
+    }
+
+    @Override
+    public String getPassword() {
+        return null;
+    }
+
+    @Override
+    public boolean promptPassword(String s) {
+        return false;
+    }
+
+    @Override
+    public boolean promptPassphrase(String s) {
+        return false;
+    }
+
+    @Override
+    public boolean promptYesNo(String s) {
+        return false;
+    }
+
+    @Override
+    public void showMessage(String s) {
+
+    }
+}

src/main/java/com/nokia/common/ssh/exception/SSHUtilException.java

@@ -0,0 +1,8 @@
+package com.nokia.common.ssh.exception;
+
+public class SSHUtilException extends Exception{
+
+    public SSHUtilException(String message) {
+        super(message);
+    }
+}

src/main/java/com/nokia/common/ssh/exception/ScpAckErrorException.java

@@ -0,0 +1,7 @@
+package com.nokia.common.ssh.exception;
+
+public class ScpAckErrorException extends SSHUtilException {
+    public ScpAckErrorException(String message) {
+        super(message);
+    }
+}

src/main/java/com/nokia/config/XxlJobConfig.java

@@ -0,0 +1,77 @@
+package com.nokia.config;
+
+import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * xxl-job config
+ *
+ * @author xuxueli 2017-04-28
+ */
+@Configuration
+public class XxlJobConfig {
+    private Logger logger = LoggerFactory.getLogger(com.nokia.config.XxlJobConfig.class);
+
+    @Value("${xxl.job.admin.addresses}")
+    private String adminAddresses;
+
+    @Value("${xxl.job.accessToken}")
+    private String accessToken;
+
+    @Value("${xxl.job.executor.appname}")
+    private String appname;
+
+    @Value("${xxl.job.executor.address}")
+    private String address;
+
+    @Value("${xxl.job.executor.ip}")
+    private String ip;
+
+    @Value("${xxl.job.executor.port}")
+    private int port;
+
+    @Value("${xxl.job.executor.logpath}")
+    private String logPath;
+
+    @Value("${xxl.job.executor.logretentiondays}")
+    private int logRetentionDays;
+
+
+    @Bean
+    public XxlJobSpringExecutor xxlJobExecutor() {
+        logger.info(">>>>>>>>>>> xxl-job config init.");
+        XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
+        xxlJobSpringExecutor.setAdminAddresses(adminAddresses);
+        xxlJobSpringExecutor.setAppname(appname);
+        xxlJobSpringExecutor.setAddress(address);
+        xxlJobSpringExecutor.setIp(ip);
+        xxlJobSpringExecutor.setPort(port);
+        xxlJobSpringExecutor.setAccessToken(accessToken);
+        xxlJobSpringExecutor.setLogPath(logPath);
+        xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays);
+        return xxlJobSpringExecutor;
+    }
+
+    /**
+     * 针对多网卡、容器内部署等情况，可借助 "spring-cloud-commons" 提供的 "InetUtils" 组件灵活定制注册IP；
+     *
+     *      1、引入依赖：
+     *          <dependency>
+     *             <groupId>org.springframework.cloud</groupId>
+     *             <artifactId>spring-cloud-commons</artifactId>
+     *             <version>${version}</version>
+     *         </dependency>
+     *
+     *      2、配置文件，或者容器启动变量
+     *          spring.cloud.inetutils.preferred-networks: 'xxx.xxx.xxx.'
+     *
+     *      3、获取IP
+     *          String ip_ = inetUtils.findFirstNonLoopbackHostInfo().getIpAddress();
+     */
+
+
+}

src/main/java/com/nokia/task/LtePmTask.java

@@ -0,0 +1,127 @@
+package com.nokia.task;
+import com.jcraft.jsch.JSchException;
+import com.nokia.common.gpload.GploadUtil;
+import com.nokia.common.gpload.entity.GploadResult;
+import com.nokia.common.ssh.SSHUtil;
+import com.nokia.common.ssh.exception.SSHUtilException;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+@Slf4j
+/*@Task*/
+@Component
+public class LtePmTask {
+
+    @Value("${lte.pm.download.host:10.17.180.55}")
+    private String host;
+    @Value("${lte.pm.download.port:22}")
+    private Integer port;
+    @Value("${lte.pm.download.username:esbftp}")
+    private String username;
+    @Value("${lte.pm.download.password:Esb2019ftp!}")
+    private String password;
+    @Value("${lte.pm.download.sourceDir:/data/out2/pm_4g_hour}")
+    private String sourceDir;
+    @Value("${lte.pm.download.targetDir:download/}")
+    private String downloadTargetDir;
+    @Value("${lte.pm.filePrefix:pm_4g_hour_}")
+    private String filePrefix;
+    @Value("${lte.pm.distinct.targetDir:distinct/}")
+    private String distinctTargetDir;
+
+
+    @XxlJob("execHandler")
+    public void xxlJobCall(){
+        System.out.println("java-----------");
+    }
+
+    /**
+     * 扫描文件
+     */
+    @XxlJob("")
+    public void scan() throws JSchException, SSHUtilException, IOException {
+        SSHUtil sshUtil = new SSHUtil(host, port, username, password);
+        String exec = sshUtil.exec(sourceDir);
+        File targetFile = new File(exec);
+        String[] list = targetFile.list();
+        for (String s : list) {
+            String[] split = s.split("\\.");
+            if (split[1].equals("csv")){
+                singleTask(split[0]);
+            }
+        }
+    }
+
+/*    @AllowedTaskType({TaskType.CRON})
+    public void cronTask(String hourDelay) throws JSchException, SSHUtilException, IOException {
+        long hours = Long.parseLong(hourDelay);
+        String hourString = dateFormat.format(new Date(System.currentTimeMillis() - 3600L * 1000L * hours));
+        singleTask(hourString);
+    }*/
+
+    /*@AllowedTaskType({TaskType.TIMING, TaskType.IMMEDIATE})*/
+    public void singleTask(String hourString) throws JSchException, SSHUtilException, IOException {
+        download();
+        distinct();
+        gpload(hourString);
+    }
+
+    public void download() throws JSchException, SSHUtilException, IOException {
+        SSHUtil sshUtil = new SSHUtil(host, port, username, password);
+        String sourceFilePath = sourceDir;
+        File targetFile = new File(downloadTargetDir);
+        if (!targetFile.exists()) {
+            targetFile.getParentFile().mkdirs();
+        }
+        String targetPath = targetFile.getAbsolutePath();
+        boolean b = sshUtil.scpFrom(sourceFilePath, targetPath);
+        if (b) {
+            log.debug("文件 {} 下载成功...", targetPath);
+            String exec = sshUtil.exec(sourceDir);
+            File file = new File(exec);
+            file.delete();
+        }
+    }
+
+    public void distinct() throws IOException {
+        String inputFilePath = downloadTargetDir ;
+        String outputFilePath = distinctTargetDir;
+        CSVFormat format = CSVFormat.DEFAULT.builder().build();
+        CSVParser records = format.parse(new InputStreamReader(new FileInputStream(inputFilePath), StandardCharsets.UTF_8));
+        Map<String, CSVRecord> recordMap = new HashMap<>();
+        int count = 0;
+        for (CSVRecord record : records) {
+            recordMap.put(record.get(1), record);
+            count++;
+        }
+        CSVPrinter printer = format.print(new File(outputFilePath), StandardCharsets.UTF_8);
+        for (CSVRecord record : recordMap.values()) {
+            printer.printRecord(record);
+        }
+        printer.flush();
+        printer.close();
+        log.debug("去重完成,原文件{}条数据,去重后{}条数据", count, recordMap.size());
+    }
+
+    public void gpload(String hourString) {
+        String gploadCommand = "sh /data1/pm/gpload/pm_lte_gpload.sh " + hourString;
+        GploadResult gpload = GploadUtil.gpload(gploadCommand);
+        if (gpload.getTaskStatus()) {
+            log.debug("gpload完成: {}", gpload);
+        } else {
+            log.error("gpload 失败: {}", gpload.getMessage());
+        }
+    }
+}

src/main/resources/application.properties

@@ -0,0 +1,37 @@
+# log配置
+logging.level.root=info
+logging.level.com.nokia=debug
+logging.file.name=log/ops.log
+# web服务端口号
+server.port=8082
+# 数据源
+#spring.datasource.driver-class-name=org.postgresql.Driver
+#spring.datasource.url=jdbc:postgresql://localhost:5432/postgres
+#spring.datasource.username=postgres
+#spring.datasource.password=fantuan1985
+# vue生成的文件可以直接把dist目录拷贝到resource目录下
+spring.web.resources.static-locations=classpath:/dist
+# 任务调度的线程池大小
+scheduling.scheduler.pool.size=3
+# pm下载相关配置
+pm.download.directory=D:\\download
+pm.webdriver.chrome.driver=C:\\Users\\DELL\\Desktop\\chromedriver.exe
+pm.driver.options.headless=false
+pm.download.timeoutSeconds=120
+
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+xxl.job.admin.addresses=http://127.0.0.1:8087/xxl-job-admin
+### xxl-job, access token
+xxl.job.accessToken=
+
+### xxl-job executor appname
+xxl.job.executor.appname=xxl-job-executor-sample2
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
+xxl.job.executor.address=
+### xxl-job executor server-info
+xxl.job.executor.ip=
+xxl.job.executor.port=8888
+### xxl-job executor log-path
+xxl.job.executor.logpath=/data/applogs/xxl-job/jobhandler
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays=30

src/readme.md

@@ -0,0 +1,8 @@
+# pm_interface
+
+处理pm数据同步及入库
+
+本版本实现LTE pm数据从数据解析数据源同步并入库到GP数据库
+
+数据源 10.17.180.55 /data/out2/pm_4g_hour
+pm_4g_hour_2022053005.csv

src/test/java/com/nokia/task/LtePmTaskTest.java

@@ -0,0 +1,29 @@
+package com.nokia.task;
+
+import com.jcraft.jsch.JSchException;
+import com.nokia.PmInterfaceApplication;
+import com.nokia.common.ssh.exception.SSHUtilException;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@SpringBootTest(classes = {PmInterfaceApplication.class})
+class LtePmTaskTest {
+
+    /*@Autowired
+    private LtePmTask task;
+
+    @Test
+    void download() throws JSchException, SSHUtilException, IOException {
+        task.download("2022053007");
+    }
+
+    @Test
+    void distinct() throws IOException {
+        task.distinct("2022053007");
+    }*/
+}