任务排序，删除常量接口

app/crud/constant.py

@@ -26,4 +26,16 @@ def find_and_update(db: Session, type: str, value: str):
         })
         db.add(db_item)
         db.commit()
-        db.refresh(db_item)
+        db.refresh(db_item)
+
+def delete_constant(db: Session, type: str, value: str):
+    db_item = db.query(models.Constant)\
+        .filter(models.Constant.type == type)\
+        .filter(models.Constant.value == value).first()
+    if not db_item:
+        raise Exception('未找到该常亮')
+    else:
+        db.query(models.Constant)\
+        .filter(models.Constant.type == type)\
+        .filter(models.Constant.value == value).delete()
+    return db_item

app/crud/jm_homework.py

@@ -12,7 +12,7 @@ def create_jm_homework(db: Session, item: schemas.JmHomeworkCreate):
     db_item = db.query(models.JmHomework).filter(models.JmHomework.name == jm_homework_create['name'])\
         .filter(models.JmHomework.status != 0).first()
     if db_item:
-        raise Exception('JmHomework name already exists')
+        raise Exception('作业名称已存在')
     relation_list = []
     if 'relation_list' in jm_homework_create.keys():
         relation_list = jm_homework_create.pop('relation_list')
@@ -35,7 +35,8 @@ def create_jm_homework(db: Session, item: schemas.JmHomeworkCreate):
 def get_jm_homeworks(db: Session, project_id: str):
     res: List[models.JmHomework] = db.query(models.JmHomework)\
             .filter(models.JmHomework.project_id == project_id)\
-            .filter(models.JmHomework.status == 1).all()
+            .filter(models.JmHomework.status == 1)\
+            .order_by(models.JmHomework.create_time.desc()).all()
     return res
 
 def get_jm_homework_info(db: Session, homework_id: int):
@@ -50,11 +51,11 @@ def update_jm_homework(db: Session, id: int, update_item: schemas.JmHomeworkUpda
     jm_homework_update =update_item.dict(exclude_unset=True)
     db_item = db.query(models.JmHomework).filter(models.JmHomework.id == id).first()
     if not db_item:
-        raise Exception('JmHomework not found')
+        raise Exception('未找到该作业')
     db_name_item = db.query(models.JmHomework).filter(models.JmHomework.name == jm_homework_update['name'])\
         .filter(models.JmHomework.status != 0).first()
     if db_name_item:
-        raise Exception('JmHomework name already exists')
+        raise Exception('作业名称已存在')
     relation_list = []
     if 'relation_list' in jm_homework_update.keys():
         relation_list = jm_homework_update.pop('relation_list')
@@ -75,7 +76,7 @@ def update_jm_homework(db: Session, id: int, update_item: schemas.JmHomeworkUpda
 def delete_jm_homework(db: Session, id: int):
     db_item = db.query(models.JmHomework).filter(models.JmHomework.id == id).first()
     if not db_item:
-        raise Exception('JmHomework not found')
+        raise Exception('未找到该作业')
     db_item.status = 0
     db.commit()
     db.flush()

app/crud/jm_job_info.py

@@ -12,9 +12,9 @@ def create_jm_job_info(db: Session, item: schemas.JmJobInfoCreate):
     db_item = db.query(models.JmJobInfo).filter(models.JmJobInfo.name == jm_job_info_create['name'])\
         .filter(models.JmJobInfo.delete_status != 0).first()
     if db_item:
-        raise Exception('JmJobInfo name already exists')
+        raise Exception('定时任务名称已存在')
     tag = jm_job_info_create['tag']
-    find_and_update(db, '作业分类', tag)
+    find_and_update(db, '任务分类', tag)
     jm_job_info = models.JmJobInfo(**jm_job_info_create,**{
         'status': 1,
         'delete_status': 1,
@@ -26,7 +26,8 @@ def create_jm_job_info(db: Session, item: schemas.JmJobInfoCreate):
 
 def get_jm_job_infos(db: Session):
     res: List[models.JmJobInfo] = db.query(models.JmJobInfo)\
-        .filter(models.JmJobInfo.delete_status != 0).all()
+        .filter(models.JmJobInfo.delete_status != 0)\
+        .order_by(models.JmJobInfo.id.desc()).all()
     return res
 
 def get_jm_job_info(db: Session, jm_job_id: int):
@@ -34,7 +35,7 @@ def get_jm_job_info(db: Session, jm_job_id: int):
         .filter(models.JmJobInfo.id == jm_job_id)\
         .filter(models.JmJobInfo.delete_status != 0).first()
     if not item:
-        raise Exception('JmJobInfo not found')
+        raise Exception('未找到该定时任务')
     return item
 
 def update_jm_job_info(db: Session, item: schemas.JmJobInfoUpdate):
@@ -44,7 +45,7 @@ def update_jm_job_info(db: Session, item: schemas.JmJobInfoUpdate):
     db_item = db.query(models.JmJobInfo)\
         .filter(models.JmJobInfo.id == jm_job_info_update['id']).first()
     if not db_item:
-        raise Exception('JmJobInfo not found')
+        raise Exception('未找到该定时任务')
     for k, v in jm_job_info_update.items():
         setattr(db_item, k, v)
     db.commit()
@@ -56,7 +57,7 @@ def delete_jm_job_info(db: Session, jm_job_id: int):
     jm_job_info = db.query(models.JmJobInfo)\
         .filter(models.JmJobInfo.id == jm_job_id).first()
     if not jm_job_info:
-        raise Exception("JmJobInfo not found")
+        raise Exception('未找到该定时任务')
     jm_job_info.delete_status = 0
     db.commit()
     db.flush()
@@ -68,7 +69,7 @@ def update_jm_job_status(db: Session, item: schemas.JmJobInfoStatusUpdate):
         .filter(models.JmJobInfo.id == item.id)\
         .filter(models.JmJobInfo.delete_status != 0).first()
     if not jm_job_info:
-        raise Exception("JmJobInfo not found")
+        raise Exception('未找到该定时任务')
     jm_job_info.status = item.status
     db.commit()
     db.flush()

app/crud/job_info.py

@@ -30,7 +30,7 @@ def get_job_infos(db: Session, skip: int = 0, limit: int = 20):
 def update_job_info(db: Session, id: int, update_item: schemas.JobInfoUpdate):
     db_item = db.query(models.JobInfo).filter(models.JobInfo.id == id).first()
     if not db_item:
-        raise Exception('JobInfo not found')
+        raise Exception('未找到该任务')
     update_dict = update_item.dict(exclude_unset=True)
     for k, v in update_dict.items():
         setattr(db_item, k, v)
@@ -43,7 +43,7 @@ def update_job_info(db: Session, id: int, update_item: schemas.JobInfoUpdate):
 def delete_job_info(db: Session, job_id: int):
     job_item = db.query(models.JobInfo).filter(models.JobInfo.id == job_id).first()
     if not job_item:
-        raise Exception('JobInfo not found')
+        raise Exception('未找到该任务')
     job_item.delete_status = 0
     db.commit()
     db.flush()

app/crud/job_jdbc_datasource.py

@@ -19,12 +19,12 @@ def _format_datasource(db: Session, item: schemas.JobJdbcDatasourceBase, ds_id:
     if ds_id != 0:
         item = db.query(models.JobJdbcDatasource).filter(models.JobJdbcDatasource.id == ds_id).first()
         if not item:
-            raise Exception('JobJdbcDatasource not found')
+            raise Exception('未找到该数据源')
         item.jdbc_url = _decode(item.jdbc_url, item.datasource, item.database_name)
         item.jdbc_username, item.jdbc_password = decode_user(item.jdbc_username, item.jdbc_password)
     host, port = item.jdbc_url.split(':')
     if not host or not port:
-        raise Exception('jdbc_url is invalid')
+        raise Exception('jdbc_url无效')
 
     ds = DataSrouceFactory.create(item.datasource, {'port': port, 'host': host, 'username': item.jdbc_username, 'password': item.jdbc_password, 'database_name': item.database_name})
     item.jdbc_url = ds.jdbc_url
@@ -83,7 +83,7 @@ def update_job_jdbc_datasources(db: Session, ds_id: int, update_item: schemas.Jo
 
     db_item = db.query(models.JobJdbcDatasource).filter(models.JobJdbcDatasource.id == ds_id).first()
     if not db_item:
-        raise Exception('JobJdbcDatasource not found')
+        raise Exception('未找到该数据源')
     update_dict = update_item.dict(exclude_unset=True)
     for k, v in update_dict.items():
         setattr(db_item, k, v)
@@ -98,7 +98,7 @@ def update_job_jdbc_datasources(db: Session, ds_id: int, update_item: schemas.Jo
 def delete_job_jdbc_datasource(db: Session, ds_id: int):
     db_item = db.query(models.JobJdbcDatasource).filter(models.JobJdbcDatasource.id == ds_id).first()
     if not db_item:
-        raise Exception('JobJdbcDatasource not found')
+        raise Exception('未找到该数据源')
     db_item.status = 0
     db.commit()
     db.flush()

app/models/jm_job_info.py

@@ -16,11 +16,13 @@ class JmJobInfo(BaseModel):
     # 周期类型
     cron_type = Column(Integer, nullable=False)
     # 周期num
-    cron_num = Column(Integer, nullable=False)
+    cron_num = Column(Integer)
     # 周期单位
-    cron_unit = Column(String, nullable=False)
+    cron_unit = Column(String)
     # api
     api = Column(String)
+    # 图形信息
+    json_str = Column(String)
     # 状态
     status = Column(Integer, nullable=False)
     # 是否被删除

app/routers/constants.py

@@ -30,3 +30,9 @@ def get_datasources():
 @sxtimeit
 def get_constants(type: str,db: Session = Depends(get_db)):
     return crud.get_constant_list(db, type)
+
+@router.delete("/")
+@web_try()
+@sxtimeit
+def delete_constants(type: str, value: str, db: Session = Depends(get_db)):
+    return crud.delete_constant(db, type, value)

app/routers/intermediate.py

@@ -31,7 +31,7 @@ def put_intermediate(project_id: str, user_id: str, dag_uuid: str, node_uuid: st
     file_handler = FileHandler("datax")
     file_exist = file_handler.get_file("intermediate/{}/{}/{}/{}/{}".format(project_id, user_id, dag_uuid, node_uuid, file_name))
     if len(file_exist) > 0:
-        raise Exception("file is exists")
+        raise Exception('文件已存在')
     url = file_handler.put_byte_file("intermediate/{}/{}/{}/{}/{}".format(project_id, user_id, dag_uuid, node_uuid, file_name), file.file.read())
     return url
 

app/routers/jm_job_info.py

@@ -173,5 +173,7 @@ def update_jm_job_status(item: schemas.JmJobInfoStatusUpdate, db: Session = Depe
 @sxtimeit
 def execute_jm_job(jm_job_id: int, db: Session = Depends(get_db)):
     jm_job = crud.get_jm_job_info(db,jm_job_id)
+    if jm_job.status == 0:
+        raise Exception('任务已被停用')
     # 进行api调用
     return jm_job

app/schemas/jm_job_info.py

@@ -14,9 +14,11 @@ class JmJobInfoBase(BaseModel):
     # 周期类型(1:单次;2:循环)
     cron_type: int
     # 周期num
-    cron_num: int
+    cron_num: Optional[int]
     # 周期单位
-    cron_unit: str
+    cron_unit: Optional[str]
+    # 图形信息
+    json_str : Optional[str]
     # 用户id
     user_id: str
     # 用户名称
@@ -37,6 +39,7 @@ class JmJobInfoCreate(JmJobInfoBase):
                 "cron_type": "2",
                 "cron_num": "1",
                 "cron_unit": "时",
+                "json_str": "\{图形信息\}",
                 "user_id": "test",
                 "user_name": "test",
                 "project_id": "test",
@@ -77,6 +80,7 @@ class JmJobInfoUpdate(JmJobInfoBase):
                 "cron_type": "2",
                 "cron_num": "1",
                 "cron_unit": "时",
+                "json_str": "\{图形信息\}",
                 "user_id": "test",
                 "user_name": "test",
                 "project_id": "test",

app/schemas/jm_job_node.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 from pydantic import BaseModel
 
 class JmJobNode(BaseModel):
-    id: int
+    id: str
     # 作业id
     homework_id: Optional[int]
     # 作业名称
@@ -13,9 +13,9 @@ class JmJobNode(BaseModel):
 
 class JmJobEdge(BaseModel):
     # 入节点
-    source: int
+    source: str
     # 出节点
-    target: int
+    target: str
 
 
 

dag/format_name.dag

@@ -0,0 +1,451 @@
+{
+  "user_name": "test",
+  "user_id": "test",
+  "project_name": "test",
+  "project_id": "test",
+  "itermidate_data": [
+    "hdfs://host:port/uri"
+  ],
+  "nodes": [
+    {
+      "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "name": "DataSource",
+      "op": "datasource",
+      "data": {
+        "input_source": [
+          {
+            "dataSelect": true,
+            "dataField": "id",
+            "dataType": "int"
+          },
+          {
+            "dataSelect": true,
+            "dataField": "ssn",
+            "dataType": "string"
+          },
+          {
+            "dataSelect": true,
+            "dataField": "test2",
+            "dataType": "int"
+          }
+        ],
+        "input_table": 5
+      }
+    },
+    {
+      "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+      "name": "SQL",
+      "op": "script",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r3"
+          }
+        ],
+        "script": "Select * from table11",
+        "param": "test SQL Param"
+      }
+    },
+    {
+      "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+      "name": "PySpark",
+      "op": "script",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r2"
+          }
+        ],
+        "script": "import pySpark\n\nprint('hello wolrd pySpark')",
+        "param": "test PySpark Param"
+      }
+    },
+    {
+      "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+      "name": "Python",
+      "op": "script",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r1"
+          }
+        ],
+        "script": "import test\nprint('hello world')",
+        "param": "test param",
+        "package": "test = 1.1\ntest1 = 2.2"
+      }
+    }
+  ],
+  "edges": [
+    {
+      "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd"
+    },
+    {
+      "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde"
+    },
+    {
+      "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "6b437370-0468-44da-a0bb-e637e33f3efb"
+    }
+  ],
+  "graph": {
+    "cells": [
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "port": "topPort"
+        }
+      },
+      {
+        "position": {
+          "x": 240,
+          "y": 130
+        },
+        "size": {
+          "width": 180,
+          "height": 80
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+        "data": {
+          "label": "DataSource",
+          "status": "default",
+          "type": "datasource",
+          "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "inputSource": [
+            {
+              "dataSelect": false,
+              "dataField": "name",
+              "dataType": "string"
+            },
+            {
+              "dataSelect": true,
+              "dataField": "address",
+              "dataType": "string"
+            }
+          ],
+          "dataTable": 5
+        },
+        "zIndex": 1
+      },
+      {
+        "position": {
+          "x": 480,
+          "y": 300
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+        "data": {
+          "label": "SQL",
+          "status": "default",
+          "type": "script",
+          "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "paramText": "test SQL Param",
+          "scriptText": "Select * from table11",
+          "outputData": [
+            {
+              "outputVar": "r3"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 2
+      },
+      {
+        "position": {
+          "x": 230,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+        "data": {
+          "label": "PySpark",
+          "status": "default",
+          "type": "script",
+          "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "paramText": "test PySpark Param",
+          "scriptText": "import pySpark\n\nprint('hello wolrd pySpark')",
+          "outputData": [
+            {
+              "outputVar": "r2"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 3
+      },
+      {
+        "position": {
+          "x": -10,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+        "data": {
+          "label": "Python",
+          "status": "default",
+          "type": "script",
+          "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "paramText": "test param",
+          "scriptText": "import test\nprint('hello world')",
+          "outputData": [
+            {
+              "outputVar": "r1"
+            }
+          ],
+          "inputNumber": 1,
+          "packageData": "test = 1.1\ntest1 = 2.2"
+        },
+        "zIndex": 4
+      }
+    ]
+  }
+}