sis-background/power-fitting-JN/src/main/java/com.gyee.power.fitting/service/custom/curve/NewDataFittingService.java

package com.gyee.power.fitting.service.custom.curve;


import com.gyee.power.fitting.common.alg.DBSCANPointALG;
import com.gyee.power.fitting.common.alg.PowerFittingALG;
import com.gyee.power.fitting.common.alg.PowerProcessALG;
import com.gyee.power.fitting.common.alg.WindDirectionALG;
import com.gyee.power.fitting.common.config.GyeeConfig;
import com.gyee.power.fitting.common.constants.Constants;
import com.gyee.power.fitting.common.feign.RemoteServiceBuilder;
import com.gyee.power.fitting.common.spring.InitialRunner;
import com.gyee.power.fitting.common.util.DateUtil;
import com.gyee.power.fitting.common.util.FileUtil;
import com.gyee.power.fitting.common.util.NumberUtil;
import com.gyee.power.fitting.common.util.SnowFlakeUtil;
import com.gyee.power.fitting.model.*;
import com.gyee.power.fitting.model.ProBasicModelPower;
import com.gyee.power.fitting.model.anno.AnnotationTool;
import com.gyee.power.fitting.model.anno.FixedVo;
import com.gyee.power.fitting.model.custom.*;
import com.gyee.power.fitting.service.ProEconPowerFittingAnalySisService;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.text.DecimalFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

@Slf4j
@Service
public class NewDataFittingService {
    @Autowired
    private GyeeConfig config;

    @Autowired
    private RemoteServiceBuilder remoteService;

    @Autowired
    private ProEconPowerFittingAnalySisService proEconPowerFittingAnalySisService;

    @Resource
    private DataScanService dataScanService;


    //数据map
    private Map<String, String> prepareMap = null;//数据准备map
    private Map<String, String> processMap = null;//数据处理map
    private Map<String, String> fittingMap = null;//数据拟合map

    /**
     * 数据准备拟合
     * @param vo
     * @return
     */
    public ProEconPowerFittingAnalySis newDataFitting(NewDataFittingVo vo) {

        prepareMap = new HashMap<>();
        processMap = new HashMap<>();
        fittingMap = new HashMap<>();

        //1.数据获取
        List<String> wtIds = Arrays.asList(vo.getWtIds().split(","));
        for (int k = 0; k < wtIds.size(); k++) {
            String wt = wtIds.get(k);
            List<List<TsDoubleData>> result = new ArrayList<>();

            try {
                List<String> points = config.getPoints();
                Map<String, List<ProBasicEquipmentPoint>> collect = InitialRunner.pointNewMap.get(wt).stream().collect(Collectors.groupingBy(w -> w.getUniformCode()));
                if (collect.size() < 8)
                    continue;
                for (int i = 0; i < points.size(); i++) {
                    ProBasicEquipmentPoint point = collect.get(points.get(i)).get(0);
                    log.info("测点：" + point.getId() + "----" + point.getName());
                    List<TsDoubleData> data = remoteService.adapter().getHistorySnap(point.getNemCode(), vo.getSt(), vo.getEt(), vo.getInterval());
                    if (data == null || data.size() < 0)
                        break;
                    result.add(data);
                    TimeUnit.MILLISECONDS.sleep(200);
                }
                if (result.size() != points.size())
                    continue;

                String content = prepareAssemble(result);
                // 处理的数据保存在本地
                String wtCode = InitialRunner.wtNewMap.get(wt).getId();
                String fileName = config.getFilePathPrepare() + vo.getStation() + "_" + wtCode + "_" + System.currentTimeMillis() / 1000 + ".csv";
                boolean flag = FileUtil.writeFile(fileName, content);

                if (flag){  // TODO  保存数据库
                    ProEconPowerFittingAnalySis obj = new ProEconPowerFittingAnalySis();
                    obj.setStation(vo.getStation());
                    obj.setStationcn(InitialRunner.stationNewMap.get(vo.getStation()));
                    obj.setWindturbineId(wt);
                    obj.setCode(wtCode);
                    obj.setTime(DateUtil.format(vo.getSt(), DateUtil.YYYY_MM_DD_CHN) + "-" + DateUtil.format(vo.getEt(), DateUtil.YYYY_MM_DD_CHN));
                    obj.setInterval(NumberUtil.toNum(vo.getInterval()));
                    obj.setPath(fileName);
                    obj.setType(Constants.DATA_PREPARE);
                    obj.setInterp(vo.getInterval());
                    proEconPowerFittingAnalySisService.saveOrUpdate(obj);
                    prepareMap.put(obj.getId(),fileName);//保存拿到的数据
                }

                System.out.println("数据准备完成：" + wt);


            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        //2.数据筛选

        try {
            for (String key : prepareMap.keySet()){

                ProEconPowerFittingAnalySis obj = proEconPowerFittingAnalySisService.getById(key);
                /** 读取csv数据  转换成对象数组 **/
                List<PowerPointData> eis = new ArrayList<>();
                List<String> list = FileUtil.readFile(prepareMap.get(key), true);
                for (int i = 1; i < list.size(); i++) {
                    eis.add(new PowerPointData(list.get(i).split(","), false));
                }

                /** 风速  ->  保证功率  来自数据库 **/
                List<ProBasicModelPower> modelPowerList = InitialRunner.modelPowerDetailNewMap.get(InitialRunner.wtNewMap.get(obj.getWindturbineId()).getModelId());
                Map<Double, Double> modelPowerMap = modelPowerList.stream().collect(Collectors.toMap(ProBasicModelPower::getSpeed, ProBasicModelPower::getEnsurePower));
                /** 数据预处理 **/
                List<PowerPointData> data = PowerProcessALG.dataProcess(eis, modelPowerMap, vo.getMaxs(), vo.getMins(), vo.getMaxp(), vo.getMinp(), vo.getIsfbw(), vo.getIsfhl(), vo.getIsbw(), vo.getIstj(), vo.getIsglpc(), vo.getIsqfh(), vo.getQfhdj());
                /** 静风频率 **/
                List<Double> ls = WindDirectionALG.frequency(data.stream().map(PowerPointData::getSpeed).collect(Collectors.toList()), 3);
                double frequency = ls.get(0);
                double speed = ls.get(1);

                String content = processAssemble(data);
                String fileName = config.getFilePathProcess() + vo.getStation() + "_" + obj.getWindturbineId() + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
                boolean flag = FileUtil.writeFile(fileName, content);

                if (flag) {  // TODO  保存数据库
                    obj.setId(null);
                    obj.setPath(fileName);
                    obj.setFrequency(frequency);
                    obj.setSpeedavg(speed);
                    obj.setType(Constants.DATA_PROCESS);
                    proEconPowerFittingAnalySisService.saveOrUpdate(obj);
                    processMap.put(obj.getId(),fileName);//保存预处理的的数据
                }

                System.out.println("功率曲线拟合数据预处理完成：" + obj.getWindturbineId());
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        //3.数据拟合

        AtomicReference<ProEconPowerFittingAnalySis> object = new AtomicReference<>();

        if (vo.getMode() == 0){  //单台拟合
            for (String processkey : processMap.keySet()){
                List<ProEconPowerFittingAnalySis> list = proEconPowerFittingAnalySisService.selectListByIds(processkey);
                List<Double> arraySpeed = new ArrayList<>();
                List<Double> arrayPower = new ArrayList<>();
                List<String> line = FileUtil.readFile(processMap.get(processkey), true);
                csvParse(line, arraySpeed, arrayPower, vo.getMins(), vo.getMaxs(), vo.getMinp(), vo.getMaxp());
                List<ProBasicModelPower> mp = InitialRunner.modelPowerDetailNewMap.get(InitialRunner.wtNewMap.get(list.get(0).getWindturbineId()).getModelId());
                Double maxP = mp.stream().map(ProBasicModelPower::getEnsurePower).max(Comparator.comparing(Double::doubleValue)).get();
                object.set(fittingMode(list, maxP, arraySpeed, arrayPower, vo.getDimension(), vo.getMode()));

            }
        }
        if (vo.getMode() == 1){  //合并拟合

            AtomicReference<Double> maxP = new AtomicReference<>(0.0);
            List<Double> arraySpeed = new ArrayList<>();
            List<Double> arrayPower = new ArrayList<>();
            for (String processkey : processMap.keySet()){
                List<String> line = FileUtil.readFile(processMap.get(processkey), true);
                csvParse(line, arraySpeed, arrayPower, vo.getMins(), vo.getMaxs(), vo.getMinp(), vo.getMaxp());
                List<ProBasicModelPower> mp = InitialRunner.modelPowerDetailNewMap.get(InitialRunner.wtNewMap.get(processkey).getModelId());
                Double maxPower = mp.stream().map(ProBasicModelPower::getEnsurePower).max(Comparator.comparing(Double::doubleValue)).get();
                if (maxPower > maxP.get()) maxP.set(maxPower);
            }
//            object.set(fittingMode(list, maxP.get(), arraySpeed, arrayPower, dimension, mode));
        }
        if (vo.getMode() == 2){  //同名拟合（暂时不支持）
////            List<ProEconPowerFittingAnalySis> list = powerService.selectListByIds(ids);
//            Map<String, List<ProEconPowerFittingAnalySis>> map = list.stream().collect(Collectors.groupingBy(d -> d.getWindturbine()));
//            map.forEach((k, ls) -> {
//                if (ls.size() > 1){
//                    double maxP = 0;
//                    List<ProBasicModelPower> mp = null;
//                    List<Double> arraySpeed = new ArrayList<>();
//                    List<Double> arrayPower = new ArrayList<>();
//                    for (ProEconPowerFittingAnalySis obj : ls){
//                        List<String> line = FileUtil.readFile(obj.getPath(), true);
//                        csvParse(line, arraySpeed, arrayPower, mins, maxs, minp, maxp);
//                        mp = InitialRunner.modelPowerDetailMap.get(InitialRunner.wtMap.get(obj.getWindturbine()).getModelid());
//                        maxP = mp.stream().map(ProBasicModelPower::getEnsurepower).max(Comparator.comparing(Double::doubleValue)).get();
//                    }
//                    object.set(fittingMode(ls, maxP, arraySpeed, arrayPower, dimension, mode));
//                }else {
//                    for (ProEconPowerFittingAnalySis obj : ls) {
//                        List<ProEconPowerFittingAnalySis> collect = new ArrayList<>();
//                        collect.add(obj);
//                        List<Double> arraySpeed = new ArrayList<>();
//                        List<Double> arrayPower = new ArrayList<>();
//                        List<String> line = FileUtil.readFile(collect.get(0).getPath(), true);
//                        csvParse(line, arraySpeed, arrayPower, mins, maxs, minp, maxp);
//                        List<ProBasicModelPower> mp = InitialRunner.modelPowerDetailMap.get(InitialRunner.wtMap.get(list.get(0).getWindturbine()).getModelid());
//                        Double maxP = mp.stream().map(ProBasicModelPower::getEnsurepower).max(Comparator.comparing(Double::doubleValue)).get();
//                        object.set(fittingMode(collect, maxP, arraySpeed, arrayPower, dimension, mode));
//                    }
//                }
//            });
        }

        fittingMap.put(object.get().getId(),object.get().getPath());


        return object.get();

    }




    /**
     * 曲线，散点等数据
     * 风速 eg:[1,2,3,4。。。。]
     * 曲线 eg:[1,2,3,4。。。。]
     * 散点 eg:[[1,2],[3,2],[3,5]。。。。。]
     * @param id
     * @return
     */
    public Map<String, Object> dataFittingCurve(String id){

        Map<String, Object> map = new HashMap<>();
        ProEconPowerFittingAnalySis obj = proEconPowerFittingAnalySisService.selectItemById(id);

        //实际功率、风速、Cp值
        List<Object> sjglList = new ArrayList<>();
        List<Object> cpzList = new ArrayList<>();
        List<String> ls = FileUtil.readFile(obj.getPath(), true);
        for (int i = 1; i < ls.size(); i++){
            PowerFittingData data = new PowerFittingData(ls.get(i).split(","));
            sjglList.add(new double[]{Double.valueOf(data.getSpeed()), data.getNhdata()});
            cpzList.add(new double[]{Double.valueOf(data.getSpeed()), data.getCpdata()});
        }

        //保证功率
        List<ProBasicModelPower> modelPower = InitialRunner.modelPowerDetailNewMap.get(InitialRunner.wtNewMap.get(obj.getWindturbineId()).getModelId());
        List<Object> bzglList = modelPower.stream().sorted(Comparator.comparing(ProBasicModelPower::getSpeed)).map(m -> new double[]{m.getSpeed(), m.getEnsurePower()}).collect(Collectors.toList());

        //散点
        String[] ids = obj.getProcessid().split(",");
        List<PowerPointData> yyd = new ArrayList<>(); //有用点
        List<PowerPointData> wyd = new ArrayList<>(); //无用点
        for (String pid : ids){
            ProEconPowerFittingAnalySis pf = proEconPowerFittingAnalySisService.selectItemById(pid);
            List<String> lp = FileUtil.readFile(pf.getPath(), true);
            for (int i = 1; i < lp.size(); i++){
                String[] split = lp.get(i).split(",");
                PowerPointData pd = new PowerPointData(split, true);
                if (pd.getSpeed() < 0 || pd.getPower() < 0)
                    continue;
                pd.setWtId(pf.getWindturbineId());
                if (0 == pd.getFilter())
                    yyd.add(pd);   //没有过滤
                if (1 == pd.getFilter())
                    wyd.add(pd);   //已过滤
            }
        }

        dataScanService.setMapYY(DBSCANPointALG.dbscan(yyd, 10));
        dataScanService.setMapWY(DBSCANPointALG.dbscan(wyd, 10));

        List<PointVo> listYY = new ArrayList<>();
        List<PointVo> listWY = new ArrayList<>();
        dataScanService.getMapYY().forEach((k, v) -> {
            // k: 前端画圈时的散点数据标记
            listYY.add(new PointVo(v.get(0).getSpeed(), v.get(0).getPower(), dataScanService.getMapYY().get(k).size() + 3, k));
        });
        dataScanService.getMapWY().forEach((k, v) -> {
            // k: 前端画圈时的散点数据标记
            listWY.add(new PointVo(v.get(0).getSpeed(), v.get(0).getPower(), dataScanService.getMapWY().get(k).size() + 3, k));
        });

        map.put("sjgl", sjglList);    //实际功率
        map.put("llgl", bzglList);    //保证功率
        map.put("cpz", cpzList);      //Cp值
        map.put("obj", obj);     //对w象
        map.put("yyd", listYY);  //有用散点
        map.put("wyd", listWY);  //无用散点

        return map;
    }



    private String prepareAssemble(List<List<TsDoubleData>> list){
        if (list.size() == 0)
            return null;

        StringBuilder sb = setTitle();
        List<TsDoubleData> data = list.get(0);
        for (int i = 0; i < data.size(); i++){
            sb.append(DateUtil.format(data.get(i).getTs(), DateUtil.DATE_TIME_PATTERN)).append(",");
            sb.append(data.get(i).getDoubleValue()).append(",");
            for (int j = 1; j < list.size(); j++){
                sb.append(list.get(j).get(i).getDoubleValue()).append(",");
            }
            sb.deleteCharAt(sb.lastIndexOf(","));
            sb.append("\n");
        }

        return sb.toString();
    }



    private String processAssemble(List<PowerPointData> list) {
        StringBuilder sb = setTitle();
        for (PowerPointData obj : list){
            List<FixedVo> ls = AnnotationTool.getValueList(obj);
            String data = ls.stream().filter(f -> !StringUtils.isEmpty(f.getRemark())).map(FixedVo::getKey).collect(Collectors.joining(","));
            sb.append(data).append("\n");
        }

        return sb.toString();
    }



    private StringBuilder setTitle(){
        StringBuilder sb = new StringBuilder();
        val list = AnnotationTool.getFixedVoList(PowerPointData.class);
        String columnName = list.stream().filter(f -> f.getRemark().equals("1")).map(FixedVo::getDes).collect(Collectors.joining(","));
        sb.append(columnName).append("\n");
        return sb;
    }



    /** 读取csv数据 转换成对象数组 **/
    private void csvParse(List<String> line, List<Double> arrayS, List<Double> arrayP, double mins, double maxs, double minp, double maxp){
        for (int i = 1; i < line.size(); i++) {
            String[] split = line.get(i).split(",");
            PowerPointData data = new PowerPointData(split, true);//是否过滤 0：没过滤 1：过滤
            double x = data.getSpeed();    //风速
            double y = data.getPower();    //功率
            int filter = data.getFilter();
            if (filter == 0 && (x >= mins && x <= maxs && y >= minp && y <= maxp)) {
                arrayS.add(x);
                arrayP.add(y);
            }
        }
    }


    private ProEconPowerFittingAnalySis fittingMode(List<ProEconPowerFittingAnalySis> list, Double powerMax, List<Double> arraySpeed, List<Double> arrayPower, Integer dimension, int mode){
        if (list == null || list.size() == 0)
            return null;

        ProEconPowerFittingAnalySis obj = list.get(0);
        //风速0-25，数据不全拟合的则不全，需要补一下
        arraySpeed.add(0.0);
        arraySpeed.add(25.01);
        arrayPower.add(0.0);
        arrayPower.add(powerMax);
        double[] arrX = arraySpeed.stream().sorted().mapToDouble(i->i).toArray();
        double[] arrY = arrayPower.stream().sorted().mapToDouble(i->i).toArray();

        //功率曲线拟合 不合理数据过滤
        List<Point> temp = PowerFittingALG.buildLine(arrX, arrY, arraySpeed.size(), dimension, 0.01);
        //推力系数 CP值
        LineCurveFitting lf = new LineCurveFitting();
        lf.setYLines(temp);
        lf = PowerFittingALG.buildCp(InitialRunner.equipmentNewMap.get(InitialRunner.wtNewMap.get(obj.getWindturbineId()).getModelId()).getSweptArea(), lf);
        lf.getCpValue().forEach(f -> {if(f.getX() <= 2.5) f.setY(0);});
        //曲线偏差率
        dataCurveRatio(lf, obj);

        String content = fittingAssemble(lf);
        String processId = "";
        String fileName = null;
        if (mode == 0){
            processId = obj.getId();
            fileName = config.getFilePathFitting() + obj.getStation() + "_" + obj.getCode() + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
        }
        if (mode == 1){
            processId = list.stream().map(d -> d.getId()).collect(Collectors.joining(","));
            fileName = config.getFilePathFitting() + obj.getStation() + "_merge" + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
        }
        if (mode == 2){
            processId = list.stream().map(d -> d.getId()).collect(Collectors.joining(","));
            fileName = config.getFilePathFitting() + obj.getStation() + "_same" + "_" + SnowFlakeUtil.generateIdL() / 100000 + ".csv";
        }
        boolean flag = FileUtil.writeFile(fileName, content);
        if (flag) {  // TODO  保存数据库
            obj.setId(null);
            obj.setPath(fileName);
            obj.setProcessid(processId);
            obj.setCpavg(lf.getCpAvg());
            obj.setType(Constants.DATA_FITTING);
            proEconPowerFittingAnalySisService.saveOrUpdate(obj);
        }
        System.out.println("功率曲线拟合完成：" + obj.getWindturbineId());

        return obj;
    }
    /**
     * 曲线偏差率  分段的+全部的
     *  3-5  5-10  10-12  12-25
     * @return
     */
    private void dataCurveRatio(LineCurveFitting lf, ProEconPowerFittingAnalySis obj) {
        DecimalFormat df = new DecimalFormat("0.00");
        try{
            //风速、实际功率
            List<Point> point = new ArrayList<>();  //3-25m
            List<Point> point5 = new ArrayList<>();  //分段
            List<Point> point10 = new ArrayList<>();  //分段
            List<Point> point12= new ArrayList<>();  //分段
            List<Point> point25 = new ArrayList<>();  //分段
            List<Point> line = lf.getYLines();
            for (int i = 1; i < line.size(); i++){
                double speed = Double.valueOf(df.format(line.get(i).getX()));
                if (speed >= 3 && speed < 5)
                    point5.add(new Point(speed, line.get(i).getY()));
                if (speed >= 5 && speed < 10)
                    point10.add(new Point(speed, line.get(i).getY()));
                if (speed >= 10 && speed < 12)
                    point12.add(new Point(speed, line.get(i).getY()));
                if (speed >= 12 && speed <= 25)
                    point25.add(new Point(speed, line.get(i).getY()));
                if (speed >= 3 && speed <= 25){
                    point.add(new Point(speed, line.get(i).getY()));
                }
            }

            //保证功率
            List<Point> points = new ArrayList<>();  //3-25m
            List<Point> points5 = new ArrayList<>();  //分段
            List<Point> points10 = new ArrayList<>();  //分段
            List<Point> points12 = new ArrayList<>();  //分段
            List<Point> points25 = new ArrayList<>();  //分段
            List<ProBasicModelPower> list = InitialRunner.modelPowerDetailNewMap.get(InitialRunner.wtNewMap.get(obj.getWindturbineId()).getModelId());
            for (int i = 0; i < list.size(); i++){
                ProBasicModelPower power = list.get(i);
                if (power.getSpeed() >= 3 && power.getSpeed() < 5)
                    points5.add(new Point(power.getSpeed(), power.getEnsurePower()));
                if (power.getSpeed() >= 5 && power.getSpeed() < 10)
                    points10.add(new Point(power.getSpeed(), power.getEnsurePower()));
                if (power.getSpeed() >= 10 && power.getSpeed() < 12)
                    points12.add(new Point(power.getSpeed(), power.getEnsurePower()));
                if (power.getSpeed() >= 12 && power.getSpeed() <= 25)
                    points25.add(new Point(power.getSpeed(), power.getEnsurePower()));
                if (power.getSpeed() >= 3 && power.getSpeed() <= 25)
                    points.add(new Point(power.getSpeed(), power.getEnsurePower()));
            }

            double maxp5 = list.stream().filter(f -> 5.0 == f.getSpeed()).collect(Collectors.toList()).get(0).getEnsurePower();
            double maxp10 = list.stream().filter(f -> 10.0 == f.getSpeed()).collect(Collectors.toList()).get(0).getEnsurePower();
            double maxp12 = list.stream().filter(f -> 12.0 == f.getSpeed()).collect(Collectors.toList()).get(0).getEnsurePower();
            double maxp25 = list.stream().filter(f -> 25.0 == f.getSpeed()).collect(Collectors.toList()).get(0).getEnsurePower();

            //曲线偏差率
            double pcl = PowerFittingALG.curveDeviationRatio2(point, points, maxp25, 3, 25);
            double pcl5 = PowerFittingALG.curveDeviationRatio2(point5, points5, maxp5, 3, 5);
            double pcl10 = PowerFittingALG.curveDeviationRatio2(point10, points10, maxp10, 5, 10);
            double pcl12 = PowerFittingALG.curveDeviationRatio2(point12, points12, maxp12, 10, 12);
            double pcl25 = PowerFittingALG.curveDeviationRatio2(point25, points25, maxp25, 12, 25);

            obj.setPcratio(Double.valueOf(df.format(pcl)));
            obj.setPc5ratio(Double.valueOf(df.format(pcl5)));
            obj.setPc10ratio(Double.valueOf(df.format(pcl10)));
            obj.setPc12ratio(Double.valueOf(df.format(pcl12)));
            obj.setPc25ratio(Double.valueOf(df.format(pcl25)));

        } catch (Exception e){
            log.error("DataFittingService--dataCurveRatio",e);
        }
    }

    private String fittingAssemble(LineCurveFitting lf){
        StringBuilder sb = setTitle();
        for (int i = 0; i < lf.getYLines().size(); i++){
            Point cp = lf.getCpValue().get(i);
            Point gl = lf.getYLines().get(i);

            String stx = String.format("%.2f", gl.getX());
            double x = Double.parseDouble(stx);
            String sty = String.format("%.2f", gl.getY());
            double y = Double.parseDouble(sty);
            String stcp = String.format("%.4f", cp.getY());
            double z = Double.parseDouble(stcp);
            sb.append(x).append(",").append(y).append(",").append(z).append("\n");
        }

        return sb.toString();
    }



}