文件目录调整

gyee-sample-impala/src/main/java/com/gyee/impala/controller/diagnose/HostparamController.java

@@ -2,7 +2,7 @@ package com.gyee.impala.controller.diagnose;
 
 
 import com.alibaba.fastjson.JSONObject;
-import com.gyee.impala.service.master.diagnose.HostparamService;
+import com.gyee.impala.service.custom.diagnose.HostparamService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 

gyee-sample-impala/src/main/java/com/gyee/impala/controller/diagnose/TrainDataModeController.java

@@ -1,133 +0,0 @@
-package com.gyee.impala.controller.diagnose;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.gyee.impala.common.result.JsonResult;
-import com.gyee.impala.common.result.ResultCode;
-import com.gyee.impala.common.util.DateUtil;
-import com.gyee.impala.model.custom.diagnose.DataInfo;
-import com.gyee.impala.model.custom.diagnose.ExecuteInfo;
-import com.gyee.impala.model.master.Casefault;
-import com.gyee.impala.model.master.diagnose.Diagnosepoint;
-import com.gyee.impala.service.custom.diagnose.DataPointService;
-import com.gyee.impala.service.master.CasefaultService;
-import com.gyee.impala.service.master.diagnose.TrainDataModeService;
-import org.apache.kudu.client.ListTablesResponse;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * 数据源方式训练模型
- */
-@CrossOrigin
-@RestController
-@RequestMapping("/api/traindatamode")
-public class TrainDataModeController {
-
-    @Autowired
-    private DataPointService dataService;
-    @Autowired
-    private CasefaultService casefaultService;
-    @Autowired
-    TrainDataModeService trainDataModeService;
-
-    /**
-     * 查询数据库的表
-     * @return
-     */
-    @GetMapping("/tables")
-    public JSONObject getListTables(){
-        List<Map<String, String>> list = new ArrayList<>();
-        try {
-            List<ListTablesResponse.TableInfo> tables = trainDataModeService.getListTables();
-            tables.stream().filter(a -> a.getTableName().equals("impala::gyee_sample_kudu.casefault")).forEach(obj -> {
-                Map<String, String> map = new HashMap<>();
-                String name = obj.getTableName().substring(obj.getTableName().lastIndexOf(".") + 1);
-                map.put("tableId", obj.getTableId());
-                map.put("tableName", name);
-                list.add(map);
-            });
-        } catch (Exception e) { e.getMessage(); }
-
-        return JsonResult.successData(ResultCode.SUCCESS,list);
-    }
-
-    /**
-     * 查询数据库表的列
-     * @param table
-     * @return
-     */
-    @GetMapping("/columns")
-    public JSONObject getColumns(String table){
-        Object columns = null;
-        try {
-            columns = trainDataModeService.getColumns(table);
-        } catch (Exception e) { e.getMessage(); }
-
-        return JsonResult.successData(ResultCode.SUCCESS, columns);
-    }
-
-
-    /** 查询样本原始数据 **/
-    @GetMapping("/data")
-    public JSONObject getData(String sql){
-        List<Casefault> list = casefaultService.executeSql(sql);
-        return JsonResult.successData(ResultCode.SUCCESS, list);
-    }
-
-
-    /** 开始训练 查询 golden 所有原始数据
-     *  flag  ture: 所有数据
-     *  flag  false: 前10条数据
-     * **/
-    @PostMapping("/pointdata")
-    public JSONObject getPointData(@RequestBody JSONObject json){
-        if (json == null)
-            return JsonResult.error(ResultCode.PARAM_IS_BLANK);
-
-        json = json.getJSONObject("params");
-        boolean flag = json.getBooleanValue("flag");
-        List<Diagnosepoint> points = JSONObject.parseArray(json.getJSONArray("points").toString(), Diagnosepoint.class);
-        List<Casefault> faults = JSONObject.parseArray(json.getJSONArray("faults").toString(), Casefault.class);
-
-        Map<String, Object> map = new HashMap<>();
-        /** 组装数据  删除添加的故障类型**/
-        dataService.formatUniformcode(points.stream().filter(a -> !a.getUniformcode().equals("faulttype")).collect(Collectors.toList()));
-        ExecuteInfo executeInfo = new ExecuteInfo();
-        Calendar cal = Calendar.getInstance();
-        DataInfo[] dataInfos = new DataInfo[faults.size()];
-        for (int i = 0; i < faults.size(); i++){
-            DataInfo data = new DataInfo();
-            data.setId(Long.valueOf(faults.get(i).getId()));
-            data.setStationId(faults.get(i).getStationen());
-            data.setThingId(faults.get(i).getWindturbineid());
-            data.setModelId(faults.get(i).getModel());
-            data.setTag(faults.get(i).getFaultcode());
-            data.setFaultTime(faults.get(i).getStarttime());
-            data.setStartTs(DateUtil.covertDateTimestamp(faults.get(i).getStarttime()).toString());
-            if (flag){
-                data.setEndTs(DateUtil.covertDateTimestamp(faults.get(i).getEndtime()).toString());
-            }else{
-                //查看前10条数据时结束时间往后推10s
-                cal.setTime(DateUtil.parseStrtoDate(faults.get(i).getStarttime(), DateUtil.DATE_TIME_PATTERN));
-                cal.add(Calendar.SECOND, 10);
-                data.setEndTs(cal.getTimeInMillis() + "");
-            }
-            dataInfos[i] = data;
-        }
-        executeInfo.setDataInfos(dataInfos);
-        if (flag){
-            String file = dataService.getFormDataAll(executeInfo);
-            map.put("info", executeInfo);
-            map.put("filename", file);
-            return JsonResult.successData(ResultCode.SUCCESS, map);
-        }else {
-            Map<String, Object> mp = dataService.getFormData(executeInfo);
-            return JsonResult.successData(ResultCode.SUCCESS, mp);
-        }
-    }
-
-}

gyee-sample-impala/src/main/java/com/gyee/impala/controller/diagnose/TrainFileModeController.java

@@ -4,12 +4,19 @@ package com.gyee.impala.controller.diagnose;
 import com.alibaba.fastjson.JSONObject;
 import com.gyee.impala.common.result.JsonResult;
 import com.gyee.impala.common.result.ResultCode;
+import com.gyee.impala.common.util.DateUtil;
+import com.gyee.impala.model.custom.diagnose.DataInfo;
+import com.gyee.impala.model.custom.diagnose.ExecuteInfo;
+import com.gyee.impala.model.master.Casefault;
+import com.gyee.impala.model.master.diagnose.Diagnosepoint;
 import com.gyee.impala.model.master.diagnose.Diagnosetrainhistory;
 import com.gyee.impala.model.master.diagnose.TrainInfo;
-import com.gyee.impala.service.custom.ShiroService;
-import com.gyee.impala.service.master.diagnose.TrainFileModeService;
-import org.apache.ibatis.logging.Log;
-import org.apache.ibatis.logging.LogFactory;
+import com.gyee.impala.service.custom.diagnose.DataPointService;
+import com.gyee.impala.service.master.CasefaultService;
+import com.gyee.impala.service.custom.diagnose.TrainDataModeService;
+import com.gyee.impala.service.custom.diagnose.TrainFileModeService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kudu.client.ListTablesResponse;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.web.bind.annotation.*;
@@ -17,19 +24,25 @@ import org.springframework.web.multipart.MultipartFile;
 
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletRequest;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 
 /**
  * 文件方式训练模型
  */
+@Slf4j
 @CrossOrigin
 @RestController
 @RequestMapping("/api/filemode")
 public class TrainFileModeController {
 
-
-    protected Log log = LogFactory.getLog(getClass());
+    @Autowired
+    private DataPointService dataService;
+    @Autowired
+    private CasefaultService casefaultService;
+    @Autowired
+    TrainDataModeService trainDataModeService;
 
     /**
      * 线程池
@@ -48,6 +61,8 @@ public class TrainFileModeController {
     private String[] inputLabel1;
     private String host1;
     private MultipartFile file1;
+    private ExecuteInfo executeInfo;
+    private String fileName;
 
 
     /**
@@ -96,6 +111,128 @@ public class TrainFileModeController {
         trainFileModeService.exec(name1, forecastLabel1, inputLabel1, host1, file1);
     }
 
+    /**   在线训练  **/
+    /**
+     * 查询数据库的表
+     * @return
+     */
+    @GetMapping("/tables")
+    public JSONObject getListTables(){
+        List<Map<String, String>> list = new ArrayList<>();
+        try {
+            List<ListTablesResponse.TableInfo> tables = trainDataModeService.getListTables();
+            tables.stream().filter(a -> a.getTableName().equals("impala::gyee_sample_kudu.casefault")).forEach(obj -> {
+                Map<String, String> map = new HashMap<>();
+                String name = obj.getTableName().substring(obj.getTableName().lastIndexOf(".") + 1);
+                map.put("tableId", obj.getTableId());
+                map.put("tableName", name);
+                list.add(map);
+            });
+        } catch (Exception e) { e.getMessage(); }
+
+        return JsonResult.successData(ResultCode.SUCCESS,list);
+    }
+
+    /**
+     * 查询数据库表的列
+     * @param table
+     * @return
+     */
+    @GetMapping("/columns")
+    public JSONObject getColumns(String table){
+        Object columns = null;
+        try {
+            columns = trainDataModeService.getColumns(table);
+        } catch (Exception e) { e.getMessage(); }
+
+        return JsonResult.successData(ResultCode.SUCCESS, columns);
+    }
+
+
+    /** 查询样本原始数据 **/
+    @GetMapping("/data")
+    public JSONObject getData(String sql){
+        List<Casefault> list = casefaultService.executeSql(sql);
+        return JsonResult.successData(ResultCode.SUCCESS, list);
+    }
+
+
+    /** 开始训练 查询 golden 所有原始数据
+     *  flag  ture: 所有数据
+     *  flag  false: 前10条数据
+     * **/
+    @PostMapping("/pointdata")
+    public JSONObject getPointData(@RequestBody JSONObject json){
+        if (json == null)
+            return JsonResult.error(ResultCode.PARAM_IS_BLANK);
+
+        json = json.getJSONObject("params");
+        boolean flag = json.getBooleanValue("flag");
+        List<Diagnosepoint> points = JSONObject.parseArray(json.getJSONArray("points").toString(), Diagnosepoint.class);
+        List<Casefault> faults = JSONObject.parseArray(json.getJSONArray("faults").toString(), Casefault.class);
+
+        /** 组装数据  删除添加的故障类型**/
+        dataService.formatUniformcode(points.stream().filter(a -> !a.getUniformcode().equals("faulttype")).collect(Collectors.toList()));
+        executeInfo = new ExecuteInfo();
+        Calendar cal = Calendar.getInstance();
+        DataInfo[] dataInfos = new DataInfo[faults.size()];
+        for (int i = 0; i < faults.size(); i++){
+            DataInfo data = new DataInfo();
+            data.setId(Long.valueOf(faults.get(i).getId()));
+            data.setStationId(faults.get(i).getStationen());
+            data.setThingId(faults.get(i).getWindturbineid());
+            data.setModelId(faults.get(i).getModel());
+            data.setTag(faults.get(i).getFaultcode());
+            data.setFaultTime(faults.get(i).getStarttime());
+            data.setStartTs(DateUtil.covertDateTimestamp(faults.get(i).getStarttime()).toString());
+            if (flag){
+                data.setEndTs(DateUtil.covertDateTimestamp(faults.get(i).getEndtime()).toString());
+            }else{
+                //查看前10条数据时结束时间往后推10s
+                cal.setTime(DateUtil.parseStrtoDate(faults.get(i).getStarttime(), DateUtil.DATE_TIME_PATTERN));
+                cal.add(Calendar.SECOND, 10);
+                data.setEndTs(cal.getTimeInMillis() + "");
+            }
+            dataInfos[i] = data;
+        }
+        executeInfo.setDataInfos(dataInfos);
+        if (flag){
+            if (!trainFileModeService.isComplete()) {
+                return JsonResult.error(4000, "已有正在训练的模型...");
+            }
+            synchronized (locker) {
+                taskExecutor.submit(this::execute2);
+            }
+            return JsonResult.success(ResultCode.SUCCESS);
+        }else {
+            Map<String, Object> mp = dataService.getFormData(executeInfo);
+            return JsonResult.successData(ResultCode.SUCCESS, mp);
+        }
+    }
+
+    /**
+     * 调用执行脚本
+     */
+    private void execute2() {
+        fileName = dataService.getFormDataAll(executeInfo);
+        trainFileModeService.exec();
+    }
+
+    /**
+     * py 获取在线训练数据
+     *
+     * @return
+     */
+    @GetMapping("/traindata")
+    public JSONObject getData() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("info", this.executeInfo);
+        map.put("filename", fileName);
+        return JsonResult.successData(ResultCode.SUCCESS, map);
+    }
+    /**   在线训练  **/
+
+
     /**
      * 生产控制台信息
      * @param trainInfo

gyee-sample-impala/src/main/java/com/gyee/impala/service/master/diagnose/HostparamService.java

@@ -1,4 +1,4 @@
-package com.gyee.impala.service.master.diagnose;
+package com.gyee.impala.service.custom.diagnose;
 
 import com.alibaba.fastjson.JSONObject;
 import com.gyee.impala.common.config.GyeeConfig;

gyee-sample-impala/src/main/java/com/gyee/impala/service/master/diagnose/TrainDataModeService.java

@@ -1,4 +1,4 @@
-package com.gyee.impala.service.master.diagnose;
+package com.gyee.impala.service.custom.diagnose;
 
 import com.gyee.impala.common.config.datasource.KuduDataSourceConfig;
 import org.apache.kudu.ColumnSchema;

gyee-sample-impala/src/main/java/com/gyee/impala/service/master/diagnose/TrainFileModeService.java

@@ -1,4 +1,4 @@
-package com.gyee.impala.service.master.diagnose;
+package com.gyee.impala.service.custom.diagnose;
 
 
 import com.alibaba.fastjson.JSONObject;
@@ -10,6 +10,8 @@ import com.gyee.impala.common.util.JudeSystem;
 import com.gyee.impala.model.master.diagnose.Diagnosetrainhistory;
 import com.gyee.impala.model.master.diagnose.TrainInfo;
 import com.gyee.impala.service.custom.SftpFileService;
+import com.gyee.impala.service.master.diagnose.DiagnosetrainhistoryService;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.ibatis.logging.Log;
 import org.apache.ibatis.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,11 +24,10 @@ import java.util.*;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+@Slf4j
 @Service
 public class TrainFileModeService {
 
-
-    protected Log log = LogFactory.getLog(getClass());
     @Autowired
     private SftpFileService fileService;
 
@@ -243,4 +244,46 @@ public class TrainFileModeService {
         return resultvalue.trim();
 
     }
+
+
+    /**
+     * 在线训练调用脚本
+     */
+    public void exec() {
+        //判断当前是否有模型在训练
+        if (!isComplete) {
+            return;
+        }
+
+        try {
+            isComplete = false;
+            System.out.println(new Date() + "在线训练开始执行脚本..." + isComplete);
+            Process p;
+            String cmdPath = gyeeConfig.getDiagnosePath();
+            String[] cmd = {"/bin/sh", "-c", "python " + cmdPath + "Deeplearning.py"};
+            //String[] cmd = {"/bin/sh", "-c", "python " + cmdPath + executeInfo.getAlgorithm().getName() + ".py " + JSON.toJSONString(executeInfo.getDataInfos())};
+            //String s = JSON.toJSONString(Arrays.stream(executeInfo.getAlgorithm().getParameters()).map(Parameter::getValue).toArray());
+            //String order = "cmd /c python " + cmdPath + executeInfo.getAlgorithm().getName() + ".py " + executeInfo.getAlgorithm().getParameter();
+            System.out.println(cmd[0] + " " + cmd[1] + " " + cmd[2]);
+            p = Runtime.getRuntime().exec(cmd);
+            //监听脚本输出信息
+            BufferedReader bri = new BufferedReader(new InputStreamReader(p.getInputStream()));
+            BufferedReader bre = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+            String si = null, se = null;
+            while ((si = bri.readLine()) != null || (se = bre.readLine()) != null) {
+                if (si != null) {
+                    System.out.println(si);
+                }
+                if (se != null) {
+                    System.err.println(se);
+                }
+            }
+            p.waitFor();
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            isComplete = true;
+            log.error("在线训练脚本结束..." + isComplete);
+        }
+    }
 }