故障诊断在线获取训练数据

gyee-sample-impala/src/main/java/com/gyee/impala/common/exception/CustomException.java

@@ -16,4 +16,8 @@ public class CustomException extends RuntimeException {
         this.code = result.getCode();
         this.message = result.getMessage();
     }
+
+    public CustomException(Exception e){
+        super();
+    }
 }

gyee-sample-impala/src/main/java/com/gyee/impala/common/result/ResultCode.java

@@ -19,8 +19,7 @@ public enum ResultCode {
     ERROR_CONNECT(4005, "数据库连接异常"),
     ERROR_DATA(4006, "数据库操作失败"),
     ERROR_DATA_REPEAT(4007, "数据已存在"),
-    ERROR_DATA_DIR(4008, "目录创建失败"),
-    ERROR_DATA_FILE(4009, "文件上传失败"),
+    ERROR_SQL(4008, "sql语法不正确"),
 
 
     /* 参数错误：1000～1999 */
@@ -28,6 +27,8 @@ public enum ResultCode {
     PARAM_IS_BLANK(1002, "参数为空"),
     PARAM_TYPE_ERROR(1003, "参数类型错误"),
     PARAM_NOT_COMPLETE(1004, "参数缺失"),
+    ERROR_DATA_DIR(1005, "目录创建失败"),
+    ERROR_DATA_FILE(1006, "文件上传失败"),
 
     /* 用户错误 */
     USER_NOT_LOGIN(2001, "用户未登录"),

gyee-sample-impala/src/main/java/com/gyee/impala/common/spring/InitialRunner.java

@@ -2,7 +2,9 @@ package com.gyee.impala.common.spring;
 
 
 import com.gyee.impala.model.master.*;
+import com.gyee.impala.model.master.diagnose.Diagnosepoint;
 import com.gyee.impala.service.master.*;
+import com.gyee.impala.service.master.diagnose.DiagnosepointService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.core.annotation.Order;
@@ -10,6 +12,7 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * 服务启动前执行，进行全局变量初始化
@@ -26,7 +29,8 @@ public class InitialRunner implements CommandLineRunner {
     private KnowcategoryService faulttypeService;
     @Autowired
     private EquipmentmodelService modelService;
-
+    @Autowired
+    private DiagnosepointService diagnosepointService;
 
     /**场站所有信息**/
     public static List<Windpowerstation> wpList = new ArrayList<>();
@@ -51,6 +55,8 @@ public class InitialRunner implements CommandLineRunner {
     public static List<Knowcategory> customWarnList = new ArrayList<>();
     /** 设备型号配置 <UP82, Equipmentmodel> **/
     public static Map<String, Equipmentmodel> modelMap = new HashMap<>();
+    /** 故障诊断 训练需要的测点**/
+    public static Map<String, Map<String, List<Diagnosepoint>>> mapPoint = new HashMap<>();
 
 
     @Override
@@ -60,6 +66,7 @@ public class InitialRunner implements CommandLineRunner {
         cacheStation();
         cacheKnowCategory();
         cacheEquipmentModel();
+        cacheDiagnosePoint();
 
         System.out.println(">>>>>>>>>>>>>>>数据缓存完成<<<<<<<<<<<<<<");
     }
@@ -119,4 +126,10 @@ public class InitialRunner implements CommandLineRunner {
         List<Equipmentmodel> models = modelService.getAll();
         models.stream().forEach(obj -> modelMap.put(obj.getId(), obj));
     }
+
+    public void cacheDiagnosePoint(){
+        mapPoint.clear();
+        List<Diagnosepoint> points = diagnosepointService.getDiagnosepointList();
+        mapPoint = points.stream().collect(Collectors.groupingBy(Diagnosepoint::getStationen, Collectors.groupingBy(Diagnosepoint::getModel)));
+    }
 }

gyee-sample-impala/src/main/java/com/gyee/impala/controller/diagnose/DiagnosepointController.java

@@ -5,14 +5,18 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.gyee.impala.common.result.JsonResult;
 import com.gyee.impala.common.result.ResultCode;
+import com.gyee.impala.common.spring.InitialRunner;
 import com.gyee.impala.model.master.Windpowerstation;
 import com.gyee.impala.model.master.diagnose.Diagnosepoint;
 import com.gyee.impala.service.master.WindpowerstationService;
 import com.gyee.impala.service.master.diagnose.DiagnosepointService;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 @CrossOrigin
 @RestController
@@ -30,10 +34,15 @@ public class DiagnosepointController {
      * 获取服务器配置参数信息
      */
     @GetMapping("/list")
-    public JSONObject getDiagnosepointList() {
-        List<Diagnosepoint> diagnosepointList = diagnosepointService.getDiagnosepointList();
-        JSONObject jsonObject = JsonResult.successData(ResultCode.SUCCESS, diagnosepointList);
-        return jsonObject;
+    public JSONObject getDiagnosepointList(String station, String model) {
+        if (!StringUtils.isEmpty(station) && !StringUtils.isEmpty(model)) {
+            List<Diagnosepoint> list = InitialRunner.mapPoint.get(station).get(model);
+            return JsonResult.successData(ResultCode.SUCCESS, list);
+        }
+        else {
+            Collection<Map<String, List<Diagnosepoint>>> list = InitialRunner.mapPoint.values();
+            return JsonResult.successData(ResultCode.SUCCESS, list);
+        }
     }
 
 
@@ -42,12 +51,6 @@ public class DiagnosepointController {
      */
     @PostMapping("/addPoint")
     public JSONObject addDiagnosepointList(@RequestBody JSONObject jsonObject) {
-
-
-
-
-
-
         String name = jsonObject.get("name").toString();
         JSONArray values = jsonObject.getJSONArray("values");
 

gyee-sample-impala/src/main/java/com/gyee/impala/controller/diagnose/TrainDataModeController.java

@@ -4,12 +4,19 @@ package com.gyee.impala.controller.diagnose;
 import com.alibaba.fastjson.JSONObject;
 import com.gyee.impala.common.result.JsonResult;
 import com.gyee.impala.common.result.ResultCode;
+import com.gyee.impala.common.util.DateUtil;
+import com.gyee.impala.model.custom.diagnose.DataInfo;
+import com.gyee.impala.model.custom.diagnose.ExecuteInfo;
+import com.gyee.impala.model.master.Casefault;
+import com.gyee.impala.model.master.diagnose.Diagnosepoint;
+import com.gyee.impala.service.custom.diagnose.DataPointService;
+import com.gyee.impala.service.master.CasefaultService;
 import com.gyee.impala.service.master.diagnose.TrainDataModeService;
 import org.apache.kudu.client.ListTablesResponse;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
-import java.util.List;
+import java.util.*;
 
 /**
  * 数据源方式训练模型
@@ -19,20 +26,107 @@ import java.util.List;
 @RequestMapping("/api/traindatamode")
 public class TrainDataModeController {
 
-
+    @Autowired
+    private DataPointService dataService;
+    @Autowired
+    private CasefaultService casefaultService;
     @Autowired
     TrainDataModeService trainDataModeService;
 
-    @GetMapping("/getListTables")
+    /**
+     * 查询数据库的表
+     * @return
+     */
+    @GetMapping("/tables")
     public JSONObject getListTables(){
-
+        List<Map<String, String>> list = new ArrayList<>();
         try {
             List<ListTablesResponse.TableInfo> tables = trainDataModeService.getListTables();
+            tables.stream().filter(a -> a.getTableName().equals("impala::gyee_sample_kudu.casefault")).forEach(obj -> {
+                Map<String, String> map = new HashMap<>();
+                String name = obj.getTableName().substring(obj.getTableName().lastIndexOf(".") + 1);
+                map.put("tableId", obj.getTableId());
+                map.put("tableName", name);
+                list.add(map);
+            });
+        } catch (Exception e) { e.getMessage(); }
 
-            return JsonResult.successData(ResultCode.SUCCESS,tables);
-        } catch (Exception e) {
-            return JsonResult.error(ResultCode.ERROR);
-        }
+        return JsonResult.successData(ResultCode.SUCCESS,list);
+    }
+
+    /**
+     * 查询数据库表的列
+     * @param table
+     * @return
+     */
+    @GetMapping("/columns")
+    public JSONObject getColumns(String table){
+        Object columns = null;
+        try {
+            columns = trainDataModeService.getColumns(table);
+        } catch (Exception e) { e.getMessage(); }
 
+        return JsonResult.successData(ResultCode.SUCCESS, columns);
     }
+
+
+    /** 查询样本原始数据 **/
+    @GetMapping("/data")
+    public JSONObject getData(String sql){
+        List<Casefault> list = casefaultService.executeSql(sql);
+        return JsonResult.successData(ResultCode.SUCCESS, list);
+    }
+
+
+    /** 开始训练 查询 golden 所有原始数据
+     *  flag  ture: 所有数据
+     *  flag  false: 前10条数据
+     * **/
+    @PostMapping("/pointdata")
+    public JSONObject getPointData(@RequestBody JSONObject json){
+        if (json == null)
+            return JsonResult.error(ResultCode.PARAM_IS_BLANK);
+
+        json = json.getJSONObject("params");
+        boolean flag = json.getBooleanValue("flag");
+        List<Diagnosepoint> points = JSONObject.parseArray(json.getJSONArray("points").toString(), Diagnosepoint.class);
+        List<Casefault> faults = JSONObject.parseArray(json.getJSONArray("faults").toString(), Casefault.class);
+
+        Map<String, Object> map = new HashMap<>();
+        /** 组装数据 **/
+        dataService.formatUniformcode(points);
+        ExecuteInfo executeInfo = new ExecuteInfo();
+        Calendar cal = Calendar.getInstance();
+        DataInfo[] dataInfos = new DataInfo[faults.size()];
+        for (int i = 0; i < faults.size(); i++){
+            DataInfo data = new DataInfo();
+            data.setId(Long.valueOf(faults.get(i).getId()));
+            data.setStationId(faults.get(i).getStationen());
+            data.setThingId(faults.get(i).getWindturbineid());
+            data.setModelId(faults.get(i).getModel());
+            data.setTag(faults.get(i).getFaultcode());
+            data.setFaultTime(faults.get(i).getStarttime());
+            data.setStartTs(DateUtil.covertDateTimestamp(faults.get(i).getStarttime()).toString());
+            if (flag){
+                data.setEndTs(DateUtil.covertDateTimestamp(faults.get(i).getEndtime()).toString());
+            }else{
+                //查看前10条数据时结束时间往后推10s
+                cal.setTime(DateUtil.parseStrtoDate(faults.get(i).getStarttime(), DateUtil.DATE_TIME_PATTERN));
+                cal.add(Calendar.SECOND, 10);
+                data.setEndTs(cal.getTimeInMillis() + "");
+            }
+            dataInfos[i] = data;
+        }
+        executeInfo.setDataInfos(dataInfos);
+        if (flag){
+            String file = dataService.getFormDataAll(executeInfo);
+            map.put("info", executeInfo);
+            map.put("filename", file);
+            return JsonResult.successData(ResultCode.SUCCESS, map);
+        }else {
+            Map<String, Object> mp = dataService.getFormData(executeInfo);
+            return JsonResult.successData(ResultCode.SUCCESS, mp);
+        }
+    }
+
 }

gyee-sample-impala/src/main/java/com/gyee/impala/mapper/master/CasefaultMapper.java

@@ -2,6 +2,12 @@ package com.gyee.impala.mapper.master;
 
 import com.gyee.impala.model.master.Casefault;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.ResultType;
+import org.apache.ibatis.annotations.Select;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * <p>
@@ -13,4 +19,7 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 public interface CasefaultMapper extends BaseMapper<Casefault> {
 
+    @Select({"${sql}"})
+    @ResultType(ArrayList.class)
+    List<Casefault> executeQuery(@Param("sql") String sql);
 }

gyee-sample-impala/src/main/java/com/gyee/impala/service/custom/diagnose/DataPointService.java

@@ -0,0 +1,170 @@
+package com.gyee.impala.service.custom.diagnose;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.parser.Feature;
+import com.gyee.impala.common.cache.InfoCache;
+import com.gyee.impala.common.config.GyeeConfig;
+import com.gyee.impala.common.feign.RemoteServiceBuilder;
+import com.gyee.impala.common.util.DateUtil;
+import com.gyee.impala.common.util.FileUtil;
+import com.gyee.impala.model.custom.diagnose.*;
+import com.gyee.impala.model.master.diagnose.Diagnosepoint;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.io.*;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * 数据处理服务   在线工具
+ *
+ * @author cmh
+ */
+@Slf4j
+@Component
+public class DataPointService {
+
+    @Autowired
+    private GyeeConfig gyeeConfig;
+
+    /**
+     * 数据适配器
+     */
+    @Resource
+    private RemoteServiceBuilder dataAdapterBuilder;
+
+    /**
+     * UniformCode信息
+     */
+    private Map<String, List<Diagnosepoint>> uniformCodeInfoMap;
+
+    public void formatUniformcode(List<Diagnosepoint> list) {
+        uniformCodeInfoMap = list.stream().collect(Collectors.groupingBy(Diagnosepoint::getStationen));
+    }
+
+    /**
+     * 获取所有原始数据
+     */
+    public String getFormDataAll(ExecuteInfo executeInfo) {
+        System.out.println("有监督学习获取数据...");
+        String fileName = gyeeConfig.getDiagnoseFile() + executeInfo.getDataInfos()[0].getStationId() + "-"
+                + executeInfo.getDataInfos()[0].getModelId() + "-"
+                + DateUtil.dateTimeNow() + ".csv";
+        log.info("===filename: " + fileName);
+        boolean isFirst = true;
+
+        for (DataInfo di : executeInfo.getDataInfos()) {
+            if (!uniformCodeInfoMap.containsKey(di.getStationId())) {
+                continue;
+            }
+            Map<String, List<PointData>> data = getData(di);
+            String s = getDataString(data, di.getTag(), isFirst);
+            FileUtil.writeFile(fileName, s);
+            isFirst = false;
+        }
+        System.out.println("有监督学习获取数据结束...");
+        return fileName;
+    }
+
+    /**
+     * 获取前10条原始数据
+     */
+    public Map<String, Object> getFormData(ExecuteInfo executeInfo) {
+        System.out.println("有监督学习获取测试数据...");
+        Map<String, Object> map = new HashMap<>();
+        boolean isFirst = true;
+        long count = 0;
+        StringBuilder sb = new StringBuilder();
+        for (DataInfo di : executeInfo.getDataInfos()) {
+            if (!uniformCodeInfoMap.containsKey(di.getStationId())) {
+                continue;
+            }
+            // 查询一台风机的原始数据
+            if (count == 0){
+                Map<String, List<PointData>> data = getData(di);
+                String s = getDataString(data, di.getTag(), isFirst);
+                sb.append(s);
+            }
+
+            /** 评估多少数据量 **/
+            count += (Long.valueOf(di.getEndTs()) - Long.valueOf(di.getStartTs())) / 1000;
+        }
+        map.put("count", count);
+        map.put("data", sb.toString());
+        System.out.println("有监督学习获取测试数据结束...");
+
+        return map;
+    }
+
+    private String getDataString(Map<String, List<PointData>> data, String tag, boolean isFirst) {
+        List<List<PointData>> vals = new ArrayList<>(data.values());
+        if(vals.size()<=0){
+            return "";
+        }
+
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+        StringBuilder stringBuilder = new StringBuilder();
+        if (isFirst) {
+            stringBuilder.append(getFileTitle(data.keySet()));
+        }
+
+        for (int i = 0; i < vals.get(0).size(); ++i) {
+            StringBuilder sb = new StringBuilder();
+            Date dt = new Date(vals.get(0).get(i).getTs());
+            sb.append(sdf.format(dt)).append(',');
+            for (List<PointData> vv : vals) {
+                if (i >= vv.size()) {
+                    sb.append(0).append(',');
+                    continue;
+                }
+                sb.append(vv.get(i).getValue()).append(',');
+            }
+            if (tag != null) {
+                sb.append(tag);
+            }
+            sb.append('\n');
+            stringBuilder.append(sb);
+        }
+        return stringBuilder.toString();
+    }
+
+    /**
+     * @param di
+     * @return
+     */
+    public Map<String, List<PointData>> getData(DataInfo di) {
+        List<Diagnosepoint> list = uniformCodeInfoMap.get(di.getStationId());
+        Map<String, List<PointData>> pairs = new HashMap<>();
+        for (Diagnosepoint ci : list) {
+            if (pairs.containsKey(ci.getName())) {
+                continue;
+            }
+            try {
+                List<PointData> pds = dataAdapterBuilder.adapter().getHistoryByUniformCode(ci.getUniformcode(), di.getStartTs(), di.getEndTs(), di.getThingId());
+                pairs.put(ci.getName(), pds);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+        return pairs;
+    }
+
+
+    private String getFileTitle(Set<String> keySet) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("时间,");
+        for (String st : keySet) {
+            sb.append(st).append(',');
+        }
+        sb.append("故障类型");
+        sb.append('\n');
+        return sb.toString();
+    }
+}

gyee-sample-impala/src/main/java/com/gyee/impala/service/custom/diagnose/DataService.java

@@ -51,7 +51,7 @@ public class DataService {
     /**
      * UniformCode信息
      */
-    private Map<String, UniformCodeInfo> uniformCodeInfoMap;
+    public Map<String, UniformCodeInfo> uniformCodeInfoMap;
 
     public Map<String, UniformCodeInfo> getUniformCodeInfoMap() {
         return uniformCodeInfoMap;

gyee-sample-impala/src/main/java/com/gyee/impala/service/impl/master/CasefaultServiceImpl.java

@@ -11,6 +11,8 @@ import com.gyee.impala.model.master.Casefault;
 import com.gyee.impala.mapper.master.CasefaultMapper;
 import com.gyee.impala.service.master.CasefaultService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.jasper.tagplugins.jstl.core.Catch;
 import org.apache.kudu.client.*;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -88,6 +90,20 @@ public class CasefaultServiceImpl extends ServiceImpl<CasefaultMapper, Casefault
         }
     }
 
+    @Override
+    public List<Casefault> executeSql(String sql) {
+        if (StringUtils.isEmpty(sql) || sql.toLowerCase().contains("drop")
+                || sql.toLowerCase().contains("delete") || sql.toLowerCase().contains("update")
+                || sql.toLowerCase().contains("truncate")){
+            throw new CustomException(ResultCode.ERROR_SQL);
+        }
+        try{
+            return baseMapper.executeQuery(sql);
+        } catch (Exception e){
+            throw new CustomException(e);
+        }
+    }
+
 
     /**
      * 由于mybatis-plus存储的中文乱码

gyee-sample-impala/src/main/java/com/gyee/impala/service/master/CasefaultService.java

@@ -48,4 +48,11 @@ public interface CasefaultService extends IService<Casefault> {
      * @return
      */
     void deleteItem(String id);
+
+    /**
+     * 原生sql查询
+     * @param sql
+     * @return
+     */
+    List<Casefault> executeSql(String sql);
 }

gyee-sample-impala/src/main/java/com/gyee/impala/service/master/diagnose/TrainDataModeService.java

@@ -1,6 +1,8 @@
 package com.gyee.impala.service.master.diagnose;
 
 import com.gyee.impala.common.config.datasource.KuduDataSourceConfig;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.client.KuduTable;
 import org.apache.kudu.client.ListTablesResponse;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -16,8 +18,13 @@ public class TrainDataModeService {
 
     public List<ListTablesResponse.TableInfo> getListTables() throws Exception {
 
-        List<ListTablesResponse.TableInfo> ld = kuduConfig.kuduClient.getTablesList().getTableInfosList();
+        List<ListTablesResponse.TableInfo> tables = kuduConfig.kuduClient.getTablesList().getTableInfosList();
 
-        return ld;
+        return tables;
+    }
+
+    public Object getColumns(String tableName) throws Exception {
+        KuduTable table = kuduConfig.kuduClient.openTable("impala::gyee_sample_kudu." + tableName);
+        return table.getSchema().getColumns();
     }
 }