v1.3 调整为同时入库到GP和PG数据库

.gitignore

@@ -2,6 +2,8 @@ target/
 .vscode/### Java template
 # Compiled class file
 *.class
+.vscode
+csv
 
 # Log file
 *.log

README.MD

@@ -1 +1,5 @@
 # 告警数据入库
+
+## v1.3说明
+
+增加入库到服务请求分析PG数据库

doc/开发文档/20240506.md

@@ -0,0 +1,3 @@
+# 20240506
+
+PG数据库登录信息 192.168.70.172:5432/sqmmt  sqmdb/sqmdb_1QAZ

doc/部署文档/正式环境/正式环境部署情况.md

@@ -1,5 +1,3 @@
 # 正式环境部署情况
 
 当前部署位置 192.168.70.130/data1/s130/alarm
-
-后续会调整

pom.xml

@@ -9,9 +9,27 @@
     </parent>
 
     <artifactId>alarm</artifactId>
-    <version>1.2</version>
+    <version>1.3</version>
     <packaging>jar</packaging>
 
+    <properties>
+        <!-- 跳过测试代码 -->
+        <skipTests>true</skipTests>
+        <!-- 指定java的版本 -->
+        <java.version>1.8</java.version>
+        <!-- 文件拷贝时的编码 -->
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <!-- 指定maven-compiler-plugin的配置属性开始 -->
+        <!-- 编译时的编码 -->
+        <maven.compiler.encoding>UTF-8</maven.compiler.encoding>
+        <!-- 指定编译的版本 -->
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <maven.compiler.compilerVersion>8</maven.compiler.compilerVersion>
+        <!-- 指定maven-compiler-plugin的配置属性结束 -->
+    </properties>
+
     <dependencies>
         <!-- 本地依赖开始 -->
         <dependency>

src/main/java/com/nokia/alarm/config/SchedulingConfig.java

@@ -2,6 +2,7 @@ package com.nokia.alarm.config;
 
 import java.util.Date;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -9,21 +10,20 @@ import org.springframework.scheduling.annotation.SchedulingConfigurer;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
 import org.springframework.scheduling.config.ScheduledTaskRegistrar;
 
-import com.nokia.alarm.service.AlarmSaveToFileService;
+import com.nokia.alarm.service.AlarmSaveToGPandPGService;
+
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Configuration
+@SuppressWarnings("null")
 public class SchedulingConfig implements SchedulingConfigurer {
 
-    @Value("${scheduling.scheduler.pool.size:1}")
+    @Value("${scheduling.scheduler.pool.size:2}")
     private int poolSize;
 
-    private final AlarmSaveToFileService service;
-
-    public SchedulingConfig(AlarmSaveToFileService service) {
-        this.service = service;
-    }
+    @Autowired
+    private AlarmSaveToGPandPGService service;
 
     @Bean
     public ThreadPoolTaskScheduler taskScheduler() {
@@ -42,18 +42,8 @@ public class SchedulingConfig implements SchedulingConfigurer {
         taskRegistrar.setScheduler(taskScheduler());
         taskRegistrar.addCronTask(() -> {
             Date date = new Date();
-            try {
-                service.getHttpSave(date);
-                service.getHttpUpdate(date);
-            } catch (Exception e) {
-                e.printStackTrace();
-                // 延时 2 分钟重新调度一下
-                Date newDate = new Date(date.getTime() + 2 * 60000);
-                taskRegistrar.getScheduler().schedule(() -> {
-                    service.getHttpSave(newDate);
-                    service.getHttpUpdate(newDate);
-                }, newDate);
-            }
+            service.newAlarm(date);
+            service.updateAlarm(date);
         }, "0 */5 * * * ?");
     }
 

src/main/java/com/nokia/alarm/mapper/ResultMapDao.java

@@ -7,30 +7,31 @@ import org.apache.ibatis.annotations.Update;
 
 import com.nokia.alarm.entity.ResultMap;
 
+/* cspell:disable */
 @Mapper
 public interface ResultMapDao {
 
-    @Select("select count(1) from tsfx.dw_complain_realtime_alarm_h where uniqueId=#{uniqueId} and sdate = to_timestamp(#{eventTime}, 'yyyy-MM-dd HH24:00:00')")
-    int findByUniqueId(ResultMap result);
+        @Select("select count(1) from tsfx.dw_complain_realtime_alarm_h where uniqueId=#{uniqueId} and sdate = to_timestamp(#{eventTime}, 'yyyy-MM-dd HH24:00:00')")
+        int findByUniqueId(ResultMap result);
 
-    @Update("update tsfx.dw_complain_realtime_alarm_h set cancelTime=#{cancelTime},status=#{status},projectStatus=#{projectStatus},workStatus=#{workStatus}"
-            +" where uniqueId=#{uniqueId}")
-    void updateData(ResultMap result);
+        @Update("update tsfx.dw_complain_realtime_alarm_h set cancelTime=#{cancelTime},status=#{status},projectStatus=#{projectStatus},workStatus=#{workStatus}"
+                        + " where uniqueId=#{uniqueId}")
+        void updateData(ResultMap result);
 
-    @Insert("insert into tsfx.dw_complain_realtime_alarm_h "
-            + "(sdate,uniqueid,uniqueclearid,eventtime,canceltime,status,projectstatus,vendoralarmserial,alarmtitle,"
-            + "vendorseverity,vendoralarmtype,specificproblem,specificproblemid,equipmentclass,equipmentname,locateneclass,"
-            + "locatenename,province,region,city,grid,area,vendor,specialty,siteid,roomid,towersiteno,proequipmentstatus,"
-            + "projectname,projectno,projectstarttime,projectendtime,worksendstatus,workstatus,workno,workarchivedtime,"
-            + "workdutydep,worksendperson,alarmlogicclass,alarmlogicsubclass,standardalarmname,alarmseverity,standardalarmid,"
-            + "effectonequipment,effectonbusiness,standardflag,networktype,equipmentdn,locatenedn,taccode,tacid,alarmtext) values "
-            + "(to_timestamp(#{eventTime}, 'yyyy-MM-dd HH24:00:00'),#{uniqueId},#{uniqueClearId},#{eventTime},#{cancelTime},"
-            + "#{status},#{projectStatus},#{vendorAlarmSerial},#{alarmTitle},#{vendorSeverity},#{vendorAlarmType},#{specificProblem},"
-            + "#{specificProblemID},#{equipmentClass},#{equipmentName},#{locateNeClass},#{locateNeName},#{province},#{region},#{city},"
-            + "#{grid},#{area},#{vendor},#{specialty},#{siteId},#{roomId},#{towerSiteNo},#{proEquipmentStatus},#{projectName},"
-            + "#{projectNo},#{projectStartTime},#{projectEndTime},#{workSendStatus},#{workStatus},#{workNo},#{workArchivedTime},"
-            + "#{workDutyDep},#{workSendPerson},#{alarmLogicClass},#{alarmLogicSubClass},#{standardAlarmName},#{alarmSeverity},"
-            + "#{standardAlarmId},#{effectOnEquipment},#{effectOnBusiness},#{standardFlag},#{networkType},#{locateNEDN},"
-            + "#{equipmentDN},#{tacCode},#{tacId},#{alarmText})")
-    void insertData(ResultMap result);
+        @Insert("insert into tsfx.dw_complain_realtime_alarm_h "
+                        + "(sdate,uniqueid,uniqueclearid,eventtime,canceltime,status,projectstatus,vendoralarmserial,alarmtitle,"
+                        + "vendorseverity,vendoralarmtype,specificproblem,specificproblemid,equipmentclass,equipmentname,locateneclass,"
+                        + "locatenename,province,region,city,grid,area,vendor,specialty,siteid,roomid,towersiteno,proequipmentstatus,"
+                        + "projectname,projectno,projectstarttime,projectendtime,worksendstatus,workstatus,workno,workarchivedtime,"
+                        + "workdutydep,worksendperson,alarmlogicclass,alarmlogicsubclass,standardalarmname,alarmseverity,standardalarmid,"
+                        + "effectonequipment,effectonbusiness,standardflag,networktype,equipmentdn,locatenedn,taccode,tacid,alarmtext) values "
+                        + "(to_timestamp(#{eventTime}, 'yyyy-MM-dd HH24:00:00'),#{uniqueId},#{uniqueClearId},#{eventTime},#{cancelTime},"
+                        + "#{status},#{projectStatus},#{vendorAlarmSerial},#{alarmTitle},#{vendorSeverity},#{vendorAlarmType},#{specificProblem},"
+                        + "#{specificProblemID},#{equipmentClass},#{equipmentName},#{locateNeClass},#{locateNeName},#{province},#{region},#{city},"
+                        + "#{grid},#{area},#{vendor},#{specialty},#{siteId},#{roomId},#{towerSiteNo},#{proEquipmentStatus},#{projectName},"
+                        + "#{projectNo},#{projectStartTime},#{projectEndTime},#{workSendStatus},#{workStatus},#{workNo},#{workArchivedTime},"
+                        + "#{workDutyDep},#{workSendPerson},#{alarmLogicClass},#{alarmLogicSubClass},#{standardAlarmName},#{alarmSeverity},"
+                        + "#{standardAlarmId},#{effectOnEquipment},#{effectOnBusiness},#{standardFlag},#{networkType},#{locateNEDN},"
+                        + "#{equipmentDN},#{tacCode},#{tacId},#{alarmText})")
+        void insertData(ResultMap result);
 }

src/main/java/com/nokia/alarm/service/AlarmSaveToFileService.java

@@ -44,7 +44,7 @@ public class AlarmSaveToFileService {
                 return;
             }
             String body = (String) resObj.getUNI_NET_BODY();
-//            log.info(body);
+            // log.info(body);
             // body = body.replace("null", "\"\"");
             ResultInfo info;
             try {

src/main/java/com/nokia/alarm/service/AlarmSaveToGPandPGService.java

@@ -0,0 +1,68 @@
+package com.nokia.alarm.service;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.nokia.alarm.entity.ResultMap;
+import com.nokia.alarm.util.ExportUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class AlarmSaveToGPandPGService {
+
+    @Autowired
+    private ApiCallService apiCallService;
+
+    @Autowired
+    private ResultMapService resultMapService;
+
+    public void newAlarm(Date date) {
+        long start = System.currentTimeMillis();
+        // 获取新增告警
+        List<ResultMap> results = apiCallService.getNewAlarm(date);
+        // 使用gpload入库到GP数据库
+        // 写入文件
+        String dString = ExportUtil.saveToCsv(results);
+        // gpload
+        String gploadCommand = "sh /data1/s130/alarm/gpload/alarm.sh " + dString;
+        log.info("新增告警goload:--{}", gploadCommand);
+        try {
+            int exitCode = Runtime.getRuntime().exec(gploadCommand).waitFor();
+            log.info("新增告警gpload完成...{}", exitCode);
+        } catch (InterruptedException | IOException e) {
+            e.printStackTrace();
+            log.error("新增告警gpload失败: {}", e.getMessage());
+        }
+        // 通过轮询批量插入PG数据库
+        resultMapService.insertData(results);
+        log.info("新增告警 {} 时间 入库完成, 耗时 {} 毫秒...", date, (System.currentTimeMillis() - start));
+    }
+
+    public void updateAlarm(Date date) {
+        long start = System.currentTimeMillis();
+        // 获取取消告警
+        List<ResultMap> results = apiCallService.getCanceledAlarm(date);
+        // 通过gpload入库到GP数据库
+        // 写入文件
+        String dString = ExportUtil.saveToCsv(results);
+        // gpload
+        String gploadCommand = "sh /data1/s130/alarm/gpload/alarm.sh " + dString;
+        log.info("消除告警goload:--{}", gploadCommand);
+        try {
+            int exitCode = Runtime.getRuntime().exec(gploadCommand).waitFor();
+            log.info("消除告警gpload完成。。。。。{}", exitCode);
+        } catch (InterruptedException | IOException e) {
+            e.printStackTrace();
+            log.error("消除告警gpload失败: {}", e.getMessage());
+        }
+        // 通过resultMapService更新到PG数据库
+        resultMapService.updateOrInsertData(results);
+        log.info("消除告警 {} 时间 入库完成, 耗时 {} 毫秒...", date, (System.currentTimeMillis() - start));
+    }
+}

src/main/java/com/nokia/alarm/service/ApiCallService.java

@@ -0,0 +1,116 @@
+package com.nokia.alarm.service;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.alibaba.fastjson.JSON;
+import com.hl.opnc.AbilityResponse;
+import com.hl.opnc.ResponseBody;
+import com.hl.opnc.invoke.AbilityInvokeManager;
+import com.hl.opnc.invoke.InvokeManager;
+import com.nokia.alarm.entity.MessageCfg;
+import com.nokia.alarm.entity.ResultInfo;
+import com.nokia.alarm.entity.ResultMap;
+import com.nokia.alarm.util.SDKUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class ApiCallService {
+
+    @Value("${httpurl:http://192.168.70.125:12090/esb/HEB_AlarmCustomQuery/0?authcode=SEVCX1VDSUFQI0BhdXRoQCNTWnFsVk9QeA}")
+    private String methodUrl;
+    @Value("${appId:0f9bf497549343e68ca36b4caffbc8c4}")
+    private String appId;
+    @Value("${secret:087b1ceb698140cd8c65543f5ace0490}")
+    private String secret;
+
+    public List<ResultMap> getNewAlarm(Date date) {
+        log.info("查询新增告警开始...");
+        InvokeManager manager = new AbilityInvokeManager(appId, secret);
+        MessageCfg cfg = SDKUtil.getBody(date);
+        AbilityResponse response = manager.post(methodUrl, null, JSON.toJSONString(cfg));
+        try {
+            String responseStr = response.getRespStr();
+            ResponseBody resObj;
+            try {
+                resObj = JSON.parseObject(responseStr, ResponseBody.class);
+            } catch (Exception e) {
+                log.error("返回消息json解析错误: {}", responseStr);
+                e.printStackTrace();
+                return null;
+            }
+            String body = (String) resObj.getUNI_NET_BODY();
+            // log.info(body);
+            body = body.replace("null", "\"\"");
+            ResultInfo info;
+            try {
+                info = JSON.parseObject(body, ResultInfo.class);
+            } catch (Exception e) {
+                log.error("返回消息json解析错误: {}", responseStr);
+                e.printStackTrace();
+                return null;
+            }
+            int rows = info.getTotal();
+            log.info("新增告警返回数据{}条", rows);
+            return info.getResult();
+        } catch (Exception e) {
+            log.error("查询新增告警出错: {}", e.getMessage());
+            e.printStackTrace();
+            return null;
+        } finally {
+            try {
+                response.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    public List<ResultMap> getCanceledAlarm(Date date) {
+        log.info("查询消除告警开始...");
+        InvokeManager manager = new AbilityInvokeManager(appId, secret);
+        MessageCfg cfg = SDKUtil.gethistoryBody(date);
+        AbilityResponse response = manager.post(methodUrl, null, JSON.toJSONString(cfg));
+        try {
+            String responseStr = response.getRespStr();
+            ResponseBody resObj;
+            try {
+                resObj = JSON.parseObject(responseStr, ResponseBody.class);
+            } catch (Exception e) {
+                log.error("返回消息json解析错误: {}", responseStr);
+                e.printStackTrace();
+                return null;
+            }
+            String body = (String) resObj.getUNI_NET_BODY();
+            body = body.replace("null", "\"\"");
+            ResultInfo info;
+            try {
+                info = JSON.parseObject(body, ResultInfo.class);
+            } catch (Exception e) {
+                log.error("返回消息json解析错误: {}", responseStr);
+                e.printStackTrace();
+                return null;
+            }
+            int rows = info.getTotal();
+            log.info("消除告警返回数据{}条", rows);
+            return info.getResult();
+        } catch (Exception e) {
+            log.error("查询消除告警出错: {}...", e.getMessage());
+            e.printStackTrace();
+            return null;
+        } finally {
+            try {
+                response.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+}

src/main/java/com/nokia/alarm/service/ResultMapService.java

@@ -30,27 +30,17 @@ public class ResultMapService {
         if (results != null) {
             for (ResultMap result : results) {
                 if (isExist(result)) {
-                    log.info("2222============================================================================");
+                    log.debug("更新数据 {}", result.getUniqueId());
                     resultMapDao.updateData(result);
                 } else {
-                    log.info("1111============================================================================");
                     resultMapDao.insertData(result);
                 }
-                try {
-                    log.info("1111============================================================================");
-                    resultMapDao.insertData(result);
-                } catch (Exception e) {
-                    log.info("2222============================================================================");
-                    if (e.getMessage().indexOf("duplicate key value violates unique constraint") != -1) {
-                        resultMapDao.updateData(result);
-                    }
-                }
             }
         }
     }
 
     @Transactional
-    public void insertdata(List<ResultMap> results) {
+    public void insertData(List<ResultMap> results) {
         for (ResultMap result : results) {
             resultMapDao.insertData(result);
         }

src/main/resources/application-prod.properties

@@ -1,14 +1,11 @@
 server.port=8081
 
-# # 本地postgresql
-# spring.datasource.driverClassName=org.postgresql.Driver
-# spring.datasource.url=jdbc:postgresql://localhost:5432/postgres
-# spring.datasource.username=postgres
-# spring.datasource.password=fantuan
+# 入库GP数据库使用的是GPload，不需要配置
 
-# 70.109 GreenPlum数据库
+# 入库PG数据库
+# PG数据库登录信息
 spring.datasource.driverClassName=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://192.168.70.109:5432/sqmmt
+spring.datasource.url=jdbc:postgresql://192.168.70.172:5432/sqmmt
 spring.datasource.username=sqmdb
 spring.datasource.password=sqmdb_1QAZ
 

src/main/resources/application-test.properties

@@ -1,16 +1,10 @@
 server.port=8081
 
 # # 本地postgresql
-# spring.datasource.driverClassName=org.postgresql.Driver
-# spring.datasource.url=jdbc:postgresql://localhost:5432/postgres
-# spring.datasource.username=postgres
-# spring.datasource.password=fantuan
-
-# 70.109 GreenPlum数据库
 spring.datasource.driverClassName=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://192.168.50.5:5432/sqmmt
-spring.datasource.username=sqmdb
-spring.datasource.password=sqmdb_1QAZ
+spring.datasource.url=jdbc:postgresql://localhost:5432/postgres
+spring.datasource.username=postgres
+spring.datasource.password=fantuan1985
 
 # log配置
 logging.level.com.nokia=info

src/main/resources/application.properties

@@ -1,24 +1 @@
-server.port=8081
 spring.profiles.active=prod
-
-# # 本地postgresql
-# spring.datasource.driverClassName=org.postgresql.Driver
-# spring.datasource.url=jdbc:postgresql://localhost:5432/postgres
-# spring.datasource.username=postgres
-# spring.datasource.password=fantuan
-
-# 70.109 GreenPlum数据库
-spring.datasource.driverClassName=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://192.168.70.109:5432/sqmmt
-spring.datasource.username=sqmdb
-spring.datasource.password=sqmdb_1QAZ
-
-# log配置
-logging.level.com.nokia=info
-# logging.file.name=log/alarm.log
-# logging.charset.file=utf-8
-# logging.charset.console=utf-8
-
-httpurl=http://192.168.70.125:12090/esb/HEB_AlarmCustomQuery/0?authcode=SEVCX1VDSUFQI0BhdXRoQCNTWnFsVk9QeA
-
-# httpurl=http://10.160.84.7:8888/esb/HEB_AlarmCustomQuery/0?authcode=SEVCX1VDSUFQI0BhdXRoQCNTWnFsVk9QeA

src/test/com/nokia/alarm/AlarmApplicationTest.java

@@ -0,0 +1,23 @@
+package com.nokia.alarm;
+
+import java.util.Date;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import com.nokia.alarm.service.AlarmSaveToGPandPGService;
+
+@SpringBootTest
+public class AlarmApplicationTest {
+
+    @Autowired
+    private AlarmSaveToGPandPGService alarmSaveToGPandPGService;
+
+    @Test
+    void test() {
+        Date date = new Date();
+        alarmSaveToGPandPGService.newAlarm(date);
+        alarmSaveToGPandPGService.updateAlarm(date);
+    }
+
+}

src/test/com/nokia/alarm/mapper/ResultMapDaoTest.java

@@ -1,41 +0,0 @@
-package com.nokia.alarm.mapper;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-
-import com.nokia.alarm.entity.ResultMap;
-
-@SpringBootTest
-public class ResultMapDaoTest {
-
-    @Autowired
-    private ResultMapDao resultMapDao;
-
-    @Test
-    void testUpdateData() {
-        ResultMap result = new ResultMap();
-        result.setUniqueId("aaa");
-        result.setCancelTime("2022-6-27 11:00:00");
-        resultMapDao.updateData(result);
-        System.out.println("done!");
-    }
-
-    @Test
-    void testInsertdata() {
-        ResultMap result = new ResultMap();
-        result.setUniqueId("aaa");
-        result.setEventTime("2022-06-27 15:57:05");
-        resultMapDao.insertData(result);
-        System.out.println("done!");
-    }
-
-    @Test
-    void findByUniqueId() {
-        ResultMap result = new ResultMap();
-        result.setUniqueId("24604231315990233716931359656937819837");
-        result.setEventTime("2022-06-27 19:24:58");
-        int findByUniqueId = resultMapDao.findByUniqueId(result);
-        System.out.println(findByUniqueId);
-    }
-}

src/test/com/nokia/alarm/service/AlarmSaveToFileServiceTest.java

@@ -1,31 +0,0 @@
-package com.nokia.alarm.service;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-
-@SpringBootTest
-public class AlarmSaveToFileServiceTest {
-
-    @Autowired
-    private AlarmSaveToFileService service;
-
-    @Test
-    void testGetHttpSave() throws ParseException {
-        DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.sss");
-        Date date = format.parse("2022-08-08 05:35:00.000");
-        service.getHttpSave(date);
-    }
-
-    @Test
-    void testGetHttpUpdate() throws ParseException {
-        DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.sss");
-        Date date = format.parse("2022-06-28 16:55:00.000");
-        service.getHttpUpdate(date);
-    }
-}