fix(runeconomy): 优化数据拟合功能

- 修改数据准备、预处理和拟合的逻辑
- 增加对空数据和异常值的处理
- 优化数据组装和解析方法- 调整文件读写和数据库保存操作
- 移除未使用的代码和注释

runeconomy-xk/src/main/java/com/gyee/runeconomy/model/PowerFittingALG.java

@@ -65,6 +65,7 @@ public class PowerFittingALG {
             if (point.getY() > max) {
                 max = point.getY();
             }
+            if(point.getX()<=3) point.setY(0);
 
             points.add(point);
         }

runeconomy-xk/src/main/java/com/gyee/runeconomy/service/auto/impl/NewDataFittingService.java

@@ -12,7 +12,9 @@ import com.gyee.runeconomy.model.LineCurveFitting;
 import com.gyee.runeconomy.model.PowerFittingALG;
 import com.gyee.runeconomy.model.PowerFittingData;
 import com.gyee.runeconomy.model.PowerProcessALG;
-import com.gyee.runeconomy.model.auto.*;
+import com.gyee.runeconomy.model.auto.ProBasicEquipmentPoint;
+import com.gyee.runeconomy.model.auto.ProBasicModelPower;
+import com.gyee.runeconomy.model.auto.ProEconPowerFittingAnalySis;
 import com.gyee.runeconomy.model.vo.NewDataFittingVo;
 import com.gyee.runeconomy.model.vo.PointVo;
 import com.gyee.runeconomy.service.DBSCANPointALG;
@@ -52,9 +54,9 @@ public class NewDataFittingService {
 
     public static final String DATA_FITTING = "fitting";
 
-//    private List<List<TsDoubleData>> res = new ArrayList<>();
+    //    private List<List<TsDoubleData>> res = new ArrayList<>();
 
-//    private List<PowerPointData> dataFilter = new ArrayList<>();
+    //    private List<PowerPointData> dataFilter = new ArrayList<>();
 
 
     @Resource
@@ -79,23 +81,24 @@ public class NewDataFittingService {
 
         //1.数据获取
         List<String> wtIds = Arrays.asList(vo.getWtIds().split(","));
-//        deleteDir(config.getFilePathPrepare());
+        //        deleteDir(config.getFilePathPrepare());
         for (int k = 0; k < wtIds.size(); k++) {
             String wt = wtIds.get(k);
             List<List<TsDoubleData>> result = new ArrayList<>();
 
             try {
                 List<String> points = config.getPoints();
-//                Map<String, List<ProBasicEquipmentPoint>> collect = CacheContext.pointNewMap.get(wt).stream().collect(Collectors.groupingBy(w -> w.getUniformCode()));
+                //                Map<String, List<ProBasicEquipmentPoint>> collect = CacheContext.pointNewMap.get(wt).stream().collect(Collectors.groupingBy(w -> w.getUniformCode()));
                 Map<String, List<ProBasicEquipmentPoint>> collect = CacheContext.pointNewMap.get(wt).stream().collect(Collectors.groupingBy(w -> w.getUniformCode()));
                 if (collect.size() < 8)
                     continue;
                 for (int i = 0; i < points.size(); i++) {
                     ProBasicEquipmentPoint point = collect.get(points.get(i)).get(0);
+                    if (point == null) continue;
                     log.info("测点：" + point.getId() + "----" + point.getName());
                     List<TsDoubleData> data = remoteService.adapterfd().getHistorySnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
-//                    List<TsDoubleData> data = edosUtil.getHistorySnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
-//                    List<PointData> data = edosUtil.getHistoryDatasSnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
+                    //                    List<TsDoubleData> data = edosUtil.getHistorySnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
+                    //                    List<PointData> data = edosUtil.getHistoryDatasSnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
                     if (data == null || data.size() == 0)
                         break;
                     result.add(data);
@@ -105,27 +108,27 @@ public class NewDataFittingService {
                     continue;
 
                 String content = prepareAssemble(result);
-//                 处理的数据保存在本地
+                //                 处理的数据保存在本地
                 String wtCode = CacheContext.wtmap.get(wt).getId();
                 String wtCode2 = CacheContext.wtmap.get(wt).getNemCode();
                 String fileName = config.getFilePathPrepare() + vo.getStation() + "_" + wtCode + "_" + System.currentTimeMillis() / 1000 + ".csv";
                 boolean flag = FileUtil.writeFile(fileName, content);
 
-                if (flag){  // TODO  保存数据库
-                ProEconPowerFittingAnalySis obj = new ProEconPowerFittingAnalySis();
-                obj.setStation(vo.getStation());
-                obj.setStationcn(CacheContext.wtstandardmap.get(vo.getStation()));
-                obj.setWindturbineId(wt);
-                obj.setCode(wtCode2);
-//                    obj.setTime(DateUtil.format(vo.getSt(), "yyyy年MM月dd日") + "-" + DateUtil.format(vo.getEt(), "yyyy年MM月dd日"));
-                SimpleDateFormat sdf = new SimpleDateFormat("yyyy年MM月dd日");
-                obj.setTime(sdf.format(new Date(vo.getSt())) + "-" + sdf.format(new Date(vo.getEt())));
-                obj.setInterval(vo.getInterval().toString());
-                obj.setPath(fileName);
-                obj.setType("prepare");
-                obj.setInterp(vo.getInterval());
-                proEconPowerFittingAnalySisService.saveOrUpdate(obj);
-                prepareMap.put(obj.getId(),fileName);//保存拿到的数据
+                if (flag) {  // TODO  保存数据库
+                    ProEconPowerFittingAnalySis obj = new ProEconPowerFittingAnalySis();
+                    obj.setStation(vo.getStation());
+                    obj.setStationcn(CacheContext.wtstandardmap.get(vo.getStation()));
+                    obj.setWindturbineId(wt);
+                    obj.setCode(wtCode2);
+                    //                    obj.setTime(DateUtil.format(vo.getSt(), "yyyy年MM月dd日") + "-" + DateUtil.format(vo.getEt(), "yyyy年MM月dd日"));
+                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy年MM月dd日");
+                    obj.setTime(sdf.format(new Date(vo.getSt())) + "-" + sdf.format(new Date(vo.getEt())));
+                    obj.setInterval(vo.getInterval().toString());
+                    obj.setPath(fileName);
+                    obj.setType("prepare");
+                    obj.setInterp(vo.getInterval());
+                    proEconPowerFittingAnalySisService.saveOrUpdate(obj);
+                    prepareMap.put(obj.getId(), fileName);//保存拿到的数据
                 }
                 System.out.println("数据准备完成：" + wt);
             } catch (Exception e) {
@@ -141,9 +144,9 @@ public class NewDataFittingService {
                 /** 读取csv数据  转换成对象数组 **/
                 List<PowerPointData> eis = new ArrayList<>();
                 List<String> list = FileUtil.readFile(prepareMap.get(key), true);
-//                String s = prepareAssemble(res);
-//                String[] lines = s.split("\n");
-//                List<String> list = new ArrayList<>(Arrays.asList(lines));
+                //                String s = prepareAssemble(res);
+                //                String[] lines = s.split("\n");
+                //                List<String> list = new ArrayList<>(Arrays.asList(lines));
                 for (int i = 1; i < list.size(); i++) {
                     eis.add(new PowerPointData(list.get(i).split(","), false));
                 }
@@ -163,13 +166,13 @@ public class NewDataFittingService {
                 boolean flag = FileUtil.writeFile(fileName, content);
 
                 if (flag) {  // TODO  保存数据库
-//                    obj.setId(null);
+                    //                    obj.setId(null);
                     obj.setPath(fileName);
-                obj.setFrequency(frequency);
-                obj.setSpeedavg(speed);
-                obj.setType(DATA_PROCESS);
-                proEconPowerFittingAnalySisService.saveOrUpdate(obj);
-                    processMap.put(obj.getId(),fileName);//保存预处理的的数据
+                    obj.setFrequency(frequency);
+                    obj.setSpeedavg(speed);
+                    obj.setType(DATA_PROCESS);
+                    proEconPowerFittingAnalySisService.saveOrUpdate(obj);
+                    processMap.put(obj.getId(), fileName);//保存预处理的的数据
                 }
 
                 System.out.println("功率曲线拟合数据预处理完成：" + obj.getWindturbineId());
@@ -180,16 +183,16 @@ public class NewDataFittingService {
         //3.数据拟合
 
         AtomicReference<ProEconPowerFittingAnalySis> object = new AtomicReference<>();
-//        deleteDir(config.getFilePathFitting());
+        //        deleteDir(config.getFilePathFitting());
         if (vo.getMode() == 0) {  //单台拟合
             for (String processkey : processMap.keySet()) {
                 List<ProEconPowerFittingAnalySis> list = proEconPowerFittingAnalySisService.selectListByIds(processkey);
                 List<Double> arraySpeed = new ArrayList<>();
                 List<Double> arrayPower = new ArrayList<>();
                 List<String> line = FileUtil.readFile(processMap.get(processkey), true);
-//                String content = processAssemble(dataFilter);
-//                String[] lines = content.split("\n");
-//                List<String> line = new ArrayList<>(Arrays.asList(lines));
+                //                String content = processAssemble(dataFilter);
+                //                String[] lines = content.split("\n");
+                //                List<String> line = new ArrayList<>(Arrays.asList(lines));
                 csvParse(line, arraySpeed, arrayPower, vo.getMins(), vo.getMaxs(), vo.getMinp(), vo.getMaxp());
                 List<ProBasicModelPower> mp = CacheContext.modelPowerDetailNewMap.get(CacheContext.wtmap.get(list.get(0).getWindturbineId()).getModelId());
                 Double maxP = mp.stream().map(ProBasicModelPower::getEnsurePower).max(Comparator.comparing(Double::doubleValue)).get();
@@ -207,9 +210,9 @@ public class NewDataFittingService {
                 ids.append(processkey).append(",");
                 ProEconPowerFittingAnalySis fittingAnalySis = proEconPowerFittingAnalySisService.getById(processkey);
                 List<String> line = FileUtil.readFile(processMap.get(processkey), true);
-//                String content = processAssemble(dataFilter);
-//                String[] lines = content.split("\n");
-//                List<String> line = new ArrayList<>(Arrays.asList(lines));
+                //                String content = processAssemble(dataFilter);
+                //                String[] lines = content.split("\n");
+                //                List<String> line = new ArrayList<>(Arrays.asList(lines));
                 csvParse(line, arraySpeed, arrayPower, vo.getMins(), vo.getMaxs(), vo.getMinp(), vo.getMaxp());
                 List<ProBasicModelPower> mp = CacheContext.modelPowerDetailNewMap.get(CacheContext.wtmap.get(fittingAnalySis.getWindturbineId()).getModelId());
                 Double maxPower = mp.stream().map(ProBasicModelPower::getEnsurePower).max(Comparator.comparing(Double::doubleValue)).get();
@@ -222,22 +225,22 @@ public class NewDataFittingService {
         if (vo.getMode() == 2) {
             //同名拟合（暂时不支持）
         }
-        fittingMap.put(object.get().getId(),object.get().getPath());
+        fittingMap.put(object.get().getId(), object.get().getPath());
 
         return object.get();
     }
 
 
-
     /**
      * 曲线，散点等数据
      * 风速 eg:[1,2,3,4。。。。]
      * 曲线 eg:[1,2,3,4。。。。]
      * 散点 eg:[[1,2],[3,2],[3,5]。。。。。]
+     *
      * @param id
      * @return
      */
-    public Map<String, Object> dataFittingCurve(String id){
+    public Map<String, Object> dataFittingCurve(String id) {
 
         Map<String, Object> map = new HashMap<>();
         ProEconPowerFittingAnalySis obj = proEconPowerFittingAnalySisService.selectItemById(id);
@@ -246,7 +249,7 @@ public class NewDataFittingService {
         List<Object> sjglList = new ArrayList<>();
         List<Object> cpzList = new ArrayList<>();
         List<String> ls = FileUtil.readFile(obj.getPath(), true);
-        for (int i = 1; i < ls.size(); i++){
+        for (int i = 1; i < ls.size(); i++) {
             PowerFittingData data = new PowerFittingData(ls.get(i).split(","));
             sjglList.add(new double[]{Double.valueOf(data.getSpeed()), data.getNhdata()});
             cpzList.add(new double[]{Double.valueOf(data.getSpeed()), data.getCpdata()});
@@ -260,10 +263,10 @@ public class NewDataFittingService {
         String[] ids = obj.getProcessid().split(",");
         List<PowerPointData> yyd = new ArrayList<>(); //有用点
         List<PowerPointData> wyd = new ArrayList<>(); //无用点
-        for (String pid : ids){
+        for (String pid : ids) {
             ProEconPowerFittingAnalySis pf = proEconPowerFittingAnalySisService.selectItemById(pid);
             List<String> lp = FileUtil.readFile(pf.getPath(), true);
-            for (int i = 1; i < lp.size(); i++){
+            for (int i = 1; i < lp.size(); i++) {
                 String[] split = lp.get(i).split(",");
                 PowerPointData pd = new PowerPointData(split, true);
                 if (pd.getSpeed() < 0 || pd.getPower() < 0)
@@ -301,11 +304,9 @@ public class NewDataFittingService {
     }
 
 
-
-
-
     /**
      * 通过大点的key获取小散点
+     *
      * @param yk
      * @param wk
      * @return
@@ -313,15 +314,15 @@ public class NewDataFittingService {
     public List<PowerPointData> dataOrigin(String yk, String wk) {
         List<PowerPointData> list = new ArrayList<>();
 
-        if (!StringUtils.isEmpty(yk)){
+        if (!StringUtils.isEmpty(yk)) {
             String[] key = yk.split(",");
-            for (String k : key){
+            for (String k : key) {
                 list.addAll(dataScanService.getMapYY().get(k));
             }
         }
-        if (!StringUtils.isEmpty(wk)){
+        if (!StringUtils.isEmpty(wk)) {
             String[] kew = wk.split(",");
-            for (String k : kew){
+            for (String k : kew) {
                 list.addAll(dataScanService.getMapWY().get(k));
             }
         }
@@ -348,8 +349,8 @@ public class NewDataFittingService {
         /** 添加内容 **/
         List<PowerFittingData> list = new ArrayList<>();
         ProEconPowerFittingAnalySis obj = proEconPowerFittingAnalySisService.selectItemById(id);
-        List<String> ls = FileUtil.readFile(obj.getPath(), false);
-        for (int i = 1; i < ls.size(); i++){
+        List<String> ls = FileUtil.readFile(obj.getPath(), true);
+        for (int i = 1; i < ls.size(); i++) {
             PowerFittingData data = new PowerFittingData(ls.get(i).split(","));
             list.add(data);
         }
@@ -361,34 +362,34 @@ public class NewDataFittingService {
     }
 
 
-        private String prepareAssemble(List<List<TsDoubleData>> list){
-            if (list.size() == 0)
-                return null;
-
-            StringBuilder sb = setTitle();
-            List<TsDoubleData> data = list.get(0);
-            for (int i = 0; i < data.size(); i++){
-//                sb.append(DateUtil.format(data.get(i).getTs(), DateUtil.DATE_TIME_PATTERN)).append(",");
-                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-                sb.append(sdf.format(data.get(i).getTs())).append(",");
-                sb.append(data.get(i).getDoubleValue()).append(",");
-                for (int j = 1; j < list.size(); j++){
-
-                    //
-                    if(null != list.get(j) && list.get(j).size()>0){
-                        sb.append(list.get(j).get(i).getDoubleValue()).append(",");
-                    }else {
-                        sb.append(0).append(",");
-                    }
+    private String prepareAssemble(List<List<TsDoubleData>> list) {
+        if (list.size() == 0)
+            return null;
 
+        StringBuilder sb = setTitle();
+        List<TsDoubleData> data = list.get(0);
+        for (int i = 0; i < data.size(); i++) {
+            //                sb.append(DateUtil.format(data.get(i).getTs(), DateUtil.DATE_TIME_PATTERN)).append(",");
+            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+            sb.append(sdf.format(data.get(i).getTs())).append(",");
+            sb.append(data.get(i).getDoubleValue()).append(",");
+            for (int j = 1; j < list.size(); j++) {
+
+                //
+                if (null != list.get(j) && list.get(j).size() > 0) {
+                    sb.append(list.get(j).get(i).getDoubleValue()).append(",");
+                } else {
+                    sb.append(0).append(",");
                 }
-                sb.deleteCharAt(sb.lastIndexOf(","));
-                sb.append("\n");
-            }
 
-            return sb.toString();
+            }
+            sb.deleteCharAt(sb.lastIndexOf(","));
+            sb.append("\n");
         }
 
+        return sb.toString();
+    }
+
 
     private String processAssemble(List<PowerPointData> list) {
         StringBuilder sb = setTitle();
@@ -458,26 +459,26 @@ public class NewDataFittingService {
         String content = fittingAssemble(lf);
         String processId = "";
         String fileName = null;
-        if (mode == 0){
+        if (mode == 0) {
             processId = obj.getId();
             fileName = config.getFilePathFitting() + obj.getStation() + "_" + obj.getCode() + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
         }
-        if (mode == 1){
+        if (mode == 1) {
             processId = list.stream().map(d -> d.getId()).collect(Collectors.joining(","));
             fileName = config.getFilePathFitting() + obj.getStation() + "_merge" + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
         }
-        if (mode == 2){
+        if (mode == 2) {
             processId = list.stream().map(d -> d.getId()).collect(Collectors.joining(","));
             fileName = config.getFilePathFitting() + obj.getStation() + "_same" + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
         }
         boolean flag = FileUtil.writeFile(fileName, content);
         if (flag) {  // TODO  保存数据库
-//        obj.setId(null);
-        obj.setPath(fileName);
-        obj.setProcessid(processId);
-        obj.setCpavg(lf.getCpAvg());
-        obj.setType(DATA_FITTING);
-        proEconPowerFittingAnalySisService.saveOrUpdate(obj);
+            //        obj.setId(null);
+            obj.setPath(fileName);
+            obj.setProcessid(processId);
+            obj.setCpavg(lf.getCpAvg());
+            obj.setType(DATA_FITTING);
+            proEconPowerFittingAnalySisService.saveOrUpdate(obj);
         }
         System.out.println("功率曲线拟合完成：" + obj.getWindturbineId());
 
@@ -579,21 +580,21 @@ public class NewDataFittingService {
     }
 
 
-//
-//    private boolean deleteDir(String path){
-//         boolean  b  =false;
-//        File directory = new File(path);
-//        for (File file: Objects.requireNonNull(directory.listFiles())) {
-//            if (!file.isDirectory()) {
-//                b = file.delete();
-//                if(!b){
-//                    return b;
-//                }
-//            }
-//        }
-//        return b;
-//
-//    }
+    //
+    //    private boolean deleteDir(String path){
+    //         boolean  b  =false;
+    //        File directory = new File(path);
+    //        for (File file: Objects.requireNonNull(directory.listFiles())) {
+    //            if (!file.isDirectory()) {
+    //                b = file.delete();
+    //                if(!b){
+    //                    return b;
+    //                }
+    //            }
+    //        }
+    //        return b;
+    //
+    //    }
 
 
 }