fix: 修复数据重复

src/main/java/com/nokia/siteinfo/task/UpdateTask.java

@@ -37,10 +37,11 @@ public class UpdateTask {
     @XxlJob("siteInfoJobHandler")
     public void siteInfoJobHandler() {
         try {
-            Map<String, Map<String, Object>> map = new HashMap<>();
-            update4g(map);
-            update5g(map);
-            writeCsvs(map);
+            Map<String, Map<String, Object>> four = new HashMap<>();
+            Map<String, Map<String, Object>> five = new HashMap<>();
+            update4g(four);
+            update5g(five);
+            writeCsvs(four, five);
             backup("customer_service.cfg_p_netconf_std");
             gpload("customer_service.cfg_p_netconf_std");
             backup("customer_service.cfg_cell_info");
@@ -52,20 +53,16 @@ public class UpdateTask {
         }
     }
 
-    public void update4g(Map<String, Map<String, Object>> map) throws IOException {
+    private void update4g(Map<String, Map<String, Object>> map) throws IOException {
         // 查询o2p.cfg_0_4g_siteinfo
         String sql = "select * from o2p.cfg_0_4g_siteinfo";
         List<Map<String, Object>> list = inputJdbcTemplate.queryForList(sql);
-        Map<String, Map<String, Object>> temp = new HashMap<>();
         // 去重
-        list.forEach(t -> {
-            map.put((String) t.get("cgi"), t);
-            temp.put((String) t.get("cgi"), t);
-        });
-        log.info("cfg_0_4g_siteinfo: {} -> {}", list.size(), temp.size());
-        XxlJobHelper.log("cfg_0_4g_siteinfo: {} -> {}", list.size(), temp.size());
+        list.forEach(t -> map.put((String) t.get("cgi"), t));
+        log.info("cfg_0_4g_siteinfo: {} -> {}", list.size(), map.size());
+        XxlJobHelper.log("cfg_0_4g_siteinfo: {} -> {}", list.size(), map.size());
         String table = "cfm.cfg_0_4g_siteinfo";
-        writeCsv(temp, table);
+        writeCsv(map, table);
         backup(table);
         // o2p.cfg_0_4g_siteinfo -> cfm.cfg_0_4g_siteinfo
         gpload(table);
@@ -78,20 +75,16 @@ public class UpdateTask {
         XxlJobHelper.log("cfg_0_4g_siteinfo_gx: {} -> {}", list.size(), map.size());
     }
 
-    public void update5g(Map<String, Map<String, Object>> map) throws IOException {
+    private void update5g(Map<String, Map<String, Object>> map) throws IOException {
         // 查询o2p.cfg_0_5g_siteinfo
         String sql = "select * from o2p.cfg_0_5g_siteinfo";
         List<Map<String, Object>> list = inputJdbcTemplate.queryForList(sql);
-        Map<String, Map<String, Object>> temp = new HashMap<>();
         // 去重
-        list.forEach(t -> {
-            map.put((String) t.get("cgisai"), t);
-            temp.put((String) t.get("cgisai"), t);
-        });
-        log.info("cfg_0_5g_siteinfo: {} -> {}", list.size(), temp.size());
-        XxlJobHelper.log("cfg_0_5g_siteinfo: {} -> {}", list.size(), temp.size());
+        list.forEach(t -> map.put((String) t.get("cgisai"), t));
+        log.info("cfg_0_5g_siteinfo: {} -> {}", list.size(), map.size());
+        XxlJobHelper.log("cfg_0_5g_siteinfo: {} -> {}", list.size(), map.size());
         String table = "cfm.cfg_0_5g_siteinfo";
-        writeCsv(temp, table);
+        writeCsv(map, table);
         backup(table);
         // o2p.cfg_0_5g_siteinfo -> cfm.cfg_0_5g_siteinfo
         gpload(table);
@@ -104,7 +97,10 @@ public class UpdateTask {
         XxlJobHelper.log("cfg_0_5g_siteinfo_gx: {} -> {}", list.size(), map.size());
     }
 
-    public void writeCsvs(Map<String, Map<String, Object>> map) throws IOException {
+    /**
+     * 将cfg_0_4g_siteinfo、cfg_0_5g_siteinfo转成cfg_p_netconf_std、cfg_cell_info写入csv文件
+     */
+    private void writeCsvs(Map<String, Map<String, Object>> four, Map<String, Map<String, Object>> five) throws IOException {
         try (OutputStreamWriter osw1 = new OutputStreamWriter(
                 Files.newOutputStream(Paths.get("customer_service.cfg_p_netconf_std.csv")),
                 StandardCharsets.UTF_8);
@@ -113,22 +109,127 @@ public class UpdateTask {
                      Files.newOutputStream(Paths.get("customer_service.cfg_cell_info.csv")),
                      StandardCharsets.UTF_8);
              CSVPrinter printer2 = new CSVPrinter(osw2, CSVFormat.DEFAULT);) {
-            for (Map<String, Object> t : map.values()) {
-                boolean isFourG = "4G".equals(t.get("network_name"));
-                if (isFourG) {
-                    BigDecimal id = print4gCfgPNetconfStd(printer1, t);
-                    String siteName = (String) t.get("bbu_name");
-                    printCfgCellInfo(printer2, t, id, siteName);
-                } else {
-                    BigDecimal id = print5gCfgPNetconfStd(printer1, t);
-                    String siteName = (String) t.get("station_name");
-                    printCfgCellInfo(printer2, t, id, siteName);
-                }
-            }
+            Map<Object, Map<String, Object>> map = new HashMap<>();
+            fourGCfgPNetconfStd(four, map);
+            fiveGCfgPNetconfStd(five, map);
+            log.info("cfg_p_netconf_std: {}", map.size());
+            XxlJobHelper.log("cfg_p_netconf_std: {}", map.size());
+            printCfgPNetconfStd(printer1, map);
+            printCfgCellInfo(printer2, map);
+        }
+    }
+
+    /**
+     * cfg_0_5g_siteinfo -> cfg_p_netconf_std
+     */
+    private void fiveGCfgPNetconfStd(Map<String, Map<String, Object>> five, Map<Object, Map<String, Object>> map) {
+        for (Map<String, Object> t : five.values()) {
+            Map<String, Object> m = new HashMap<>();
+            BigDecimal gnbid = (BigDecimal) t.get("gnbid");
+            BigDecimal cell_id = (BigDecimal) t.get("cell_id");
+            BigDecimal cellId = gnbid.multiply(new BigDecimal(4096)).add(cell_id);
+            BigDecimal id = cellId.add(new BigDecimal(4600100000000000L));
+            m.put("id", "");
+            m.put("city_id", t.get("city_code"));
+            m.put("city_name", t.get("city_name"));
+            m.put("area_id", t.get("district_name"));
+            m.put("core_ne_name", t.get(""));
+            m.put("radio_ne_name", t.get(""));
+            m.put("site_name", t.get("station_name"));
+            m.put("site_id", gnbid);
+            m.put("cell_no", cell_id);
+            m.put("cell_name", t.get("cell_name"));
+            m.put("cell_id", id);
+            m.put("ecgi", cellId);
+            m.put("gnb_cu_id", "");
+            m.put("gnb_du_id", "");
+            m.put("gsm_cell_id", "");
+            m.put("network_type", 4);
+            m.put("type_5g", "");
+            m.put("tac", "");
+            m.put("cover_type", "宏站".equals(t.get("station_type")) ? "室外" : "室内");
+            m.put("nodeb_type", "");
+            m.put("rac", "");
+            m.put("lac", "");
+            m.put("grid", "");
+            m.put("areatype", t.get("scene"));
+            m.put("sub_areatype", "");
+            m.put("area3", "");
+            m.put("longitude", t.get("lon"));
+            m.put("latitude", t.get("lat"));
+            m.put("vendor", t.get("vender"));
+            m.put("height", t.get("height"));
+            m.put("downtilt", ((BigDecimal) t.get("m_downtilt")).add((BigDecimal) t.get("e_downtilt")));
+            m.put("azimuth", t.get("direction"));
+            m.put("fr", t.get("down_freq"));
+            m.put("scramber", t.get(""));
+            m.put("province_id", t.get("河北省"));
+            m.put("bsbusip", t.get(""));
+            m.put("bsbusip_1", t.get(""));
+            m.put("isanchor", t.get(""));
+            m.put("operator", t.get("construction"));
+            m.put("isshare", "");
+            m.put("slice_type", "");
+            map.put(id, m);
+        }
+    }
+
+    /**
+     * cfg_0_4g_siteinfo -> cfg_p_netconf_std
+     */
+    private void fourGCfgPNetconfStd(Map<String, Map<String, Object>> four, Map<Object, Map<String, Object>> map) {
+        for (Map<String, Object> t : four.values()) {
+            Map<String, Object> m = new HashMap<>();
+            BigDecimal enbid = (BigDecimal) t.get("enbid");
+            BigDecimal cell_id = (BigDecimal) t.get("cell_id");
+            BigDecimal cellId = enbid.multiply(new BigDecimal(256)).add(cell_id);
+            BigDecimal id = cellId.add(new BigDecimal(46001000000000L));
+            m.put("id", "");
+            m.put("city_id", t.get("city_code"));
+            m.put("city_name", t.get("city_name"));
+            m.put("area_id", t.get("district_name"));
+            m.put("core_ne_name", t.get(""));
+            m.put("radio_ne_name", t.get(""));
+            m.put("site_name", t.get("bbu_name"));
+            m.put("site_id", enbid);
+            m.put("cell_no", cell_id);
+            m.put("cell_name", t.get("cell_name"));
+            m.put("cell_id", id);
+            m.put("ecgi", cellId);
+            m.put("gnb_cu_id", "");
+            m.put("gnb_du_id", "");
+            m.put("gsm_cell_id", "");
+            m.put("network_type", 4);
+            m.put("type_5g", "");
+            m.put("tac", "");
+            m.put("cover_type", "宏站".equals(t.get("station_type")) ? "室外" : "室内");
+            m.put("nodeb_type", "");
+            m.put("rac", "");
+            m.put("lac", "");
+            m.put("grid", "");
+            m.put("areatype", t.get("scene"));
+            m.put("sub_areatype", "");
+            m.put("area3", "");
+            m.put("longitude", t.get("lon"));
+            m.put("latitude", t.get("lat"));
+            m.put("vendor", t.get("vender"));
+            m.put("height", t.get("height"));
+            m.put("downtilt", ((BigDecimal) t.get("m_downtilt")).add((BigDecimal) t.get("e_downtilt")));
+            m.put("azimuth", t.get("direction"));
+            m.put("fr", t.get("down_freq"));
+            m.put("scramber", t.get(""));
+            m.put("province_id", t.get("河北省"));
+            m.put("bsbusip", t.get(""));
+            m.put("bsbusip_1", t.get(""));
+            m.put("isanchor", t.get(""));
+            m.put("operator", t.get("construction"));
+            m.put("isshare", "");
+            m.put("slice_type", "");
+            map.put(id, m);
         }
     }
 
-    public void writeCsv(Map<String, Map<String, Object>> map, String table) throws IOException {
+    private void writeCsv(Map<String, Map<String, Object>> map, String table) throws IOException {
         log.info("writeCsv: {}", table);
         XxlJobHelper.log("writeCsv: {}", table);
         try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(Paths.get(table + ".csv")),
@@ -145,7 +246,7 @@ public class UpdateTask {
      *
      * @param table 表
      */
-    public void backup(String table) {
+    private void backup(String table) {
         log.info("备份: {}", table);
         XxlJobHelper.log("备份: {}", table);
         LocalDate now = LocalDate.now();
@@ -176,11 +277,11 @@ public class UpdateTask {
      * @param localDate 当地日期
      * @return {@link String}
      */
-    public String dateFormat(LocalDate localDate) {
+    private String dateFormat(LocalDate localDate) {
         return localDate.format(DateTimeFormatter.ofPattern("MMdd"));
     }
 
-    public void gpload(String table) throws IOException {
+    private void gpload(String table) throws IOException {
         String gploadCommand = "sh /data1/site_info/gpload/gpload.sh " + table;
         GploadResult gpload = GploadUtil.gpload(gploadCommand);
         if (Boolean.TRUE.equals(gpload.getTaskStatus())) {
@@ -194,116 +295,70 @@ public class UpdateTask {
         }
     }
 
-    private BigDecimal print4gCfgPNetconfStd(CSVPrinter p, Map<String, Object> t) throws IOException {
-        BigDecimal enbid = (BigDecimal) t.get("enbid");
-        BigDecimal cell_id = (BigDecimal) t.get("cell_id");
-        BigDecimal cellId = enbid.multiply(new BigDecimal(256)).add(cell_id);
-        BigDecimal id = cellId.add(new BigDecimal(46001000000000L));
-        p.printRecord(
-                "",
-                t.get("city_code"),
-                t.get("city_name"),
-                t.get("district_name"),
-                "",
-                "",
-                t.get("bbu_name"),
-                enbid,
-                cell_id,
-                t.get("cell_name"),
-                id,
-                cellId,
-                "",
-                "",
-                "",
-                4,
-                "",
-                "",
-                "宏站".equals(t.get("station_type")) ? "室外" : "室内",
-                "",
-                "",
-                "",
-                "",
-                t.get("scene"),
-                "",
-                "",
-                t.get("lon"),
-                t.get("lat"),
-                t.get("vender"),
-                t.get("height"),
-                ((BigDecimal) t.get("m_downtilt")).add((BigDecimal) t.get("e_downtilt")),
-                t.get("direction"),
-                t.get("down_freq"),
-                "",
-                "河北省",
-                "",
-                "",
-                "",
-                t.get("construction"),
-                "",
-                ""
-        );
-        return id;
-    }
-
-    private BigDecimal print5gCfgPNetconfStd(CSVPrinter p, Map<String, Object> t) throws IOException {
-        BigDecimal gnbid = (BigDecimal) t.get("gnbid");
-        BigDecimal cell_id = (BigDecimal) t.get("cell_id");
-        BigDecimal cellId = gnbid.multiply(new BigDecimal(4096)).add(cell_id);
-        BigDecimal id = cellId.add(new BigDecimal(4600100000000000L));
-        p.printRecord(
-                "",
-                t.get("city_code"),
-                t.get("city_name"),
-                t.get("district_name"),
-                "",
-                "",
-                t.get("station_name"),
-                gnbid,
-                cell_id,
-                t.get("cell_name"),
-                id,
-                cellId,
-                "",
-                "",
-                "",
-                4,
-                "",
-                "",
-                "宏站".equals(t.get("station_type")) ? "室外" : "室内",
-                "",
-                "",
-                "",
-                "",
-                t.get("scene"),
-                "",
-                "",
-                t.get("lon"),
-                t.get("lat"),
-                t.get("vender"),
-                t.get("height"),
-                ((BigDecimal) t.get("m_downtilt")).add((BigDecimal) t.get("e_downtilt")),
-                t.get("direction"),
-                t.get("down_freq"),
-                "",
-                "河北省",
-                "",
-                "",
-                "",
-                t.get("construction"),
-                "",
-                ""
-        );
-        return id;
+    /**
+     * 将cfg_p_netconf_std写入csv文件
+     */
+    private void printCfgPNetconfStd(CSVPrinter p, Map<Object, Map<String, Object>> m) throws IOException {
+        for (Map<String, Object> t : m.values()) {
+            p.printRecord(
+                    t.get("id"),
+                    t.get("city_id"),
+                    t.get("city_name"),
+                    t.get("area_id"),
+                    t.get("core_ne_name"),
+                    t.get("radio_ne_name"),
+                    t.get("site_name"),
+                    t.get("site_id"),
+                    t.get("cell_no"),
+                    t.get("cell_name"),
+                    t.get("cell_id"),
+                    t.get("ecgi"),
+                    t.get("gnb_cu_id"),
+                    t.get("gnb_du_id"),
+                    t.get("gsm_cell_id"),
+                    t.get("network_type"),
+                    t.get("type_5g"),
+                    t.get("tac"),
+                    t.get("cover_type"),
+                    t.get("nodeb_type"),
+                    t.get("rac"),
+                    t.get("lac"),
+                    t.get("grid"),
+                    t.get("areatype"),
+                    t.get("sub_areatype"),
+                    t.get("area3"),
+                    t.get("longitude"),
+                    t.get("latitude"),
+                    t.get("vendor"),
+                    t.get("height"),
+                    t.get("downtilt"),
+                    t.get("azimuth"),
+                    t.get("fr"),
+                    t.get("scramber"),
+                    t.get("province_id"),
+                    t.get("bsbusip"),
+                    t.get("bsbusip_1"),
+                    t.get("isanchor"),
+                    t.get("operator"),
+                    t.get("isshare"),
+                    t.get("slice_type")
+            );
+        }
     }
 
-    private void printCfgCellInfo(CSVPrinter p, Map<String, Object> t, BigDecimal id, String siteName) throws IOException {
-        p.printRecord(
-                id,
-                siteName,
-                t.get("city_name"),
-                t.get("lon"),
-                t.get("lat"),
-                ""
-        );
+    /**
+     * 将cfg_cell_info写入csv文件
+     */
+    private void printCfgCellInfo(CSVPrinter p, Map<Object, Map<String, Object>> m) throws IOException {
+        for (Map<String, Object> t : m.values()) {
+            p.printRecord(
+                    t.get("cell_id"),
+                    t.get("site_name"),
+                    t.get("city_name"),
+                    t.get("longitude"),
+                    t.get("latitude"),
+                    t.get("tac")
+            );
+        }
     }
 }