feat: 河北价值管理平台北十车辆费用月数据入库定时任务、河北价值管理平台省内车辆费用月数据入库定时任务增加去重逻辑

src/main/java/com/nokia/finance/tasks/jobs/car/shujucangku/CarFeeBsJob.java

@@ -28,8 +28,12 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 /**
@@ -155,7 +159,17 @@ public class CarFeeBsJob {
         for (Map<String, String> map : list) {
             map.put("source", path.getFileName().toString());
         }
-        return list;
+        // 去重
+        return list.stream()
+                .filter(distinctByKey(map -> map.get("month_id") + map.get("area_no") + map.get("kpi_id"))).toList();
+    }
+
+    /**
+     * 去重
+     */
+    private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
+        Set<Object> set = ConcurrentHashMap.newKeySet();
+        return t -> set.add(keyExtractor.apply(t));
     }
 
     /**

src/main/java/com/nokia/finance/tasks/jobs/car/shujucangku/CarFeeHbJob.java

@@ -31,8 +31,12 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 /**
@@ -188,7 +192,17 @@ public class CarFeeHbJob {
             map.put("second_org_name", secondOrgName);
             map.put("source", path.getFileName().toString());
         }
-        return list;
+        // 去重
+        return list.stream()
+                .filter(distinctByKey(map -> map.get("month_id") + map.get("area_no") + map.get("kpi_id"))).toList();
+    }
+
+    /**
+     * 去重
+     */
+    private static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
+        Set<Object> set = ConcurrentHashMap.newKeySet();
+        return t -> set.add(keyExtractor.apply(t));
     }
 
     /**