作业管理模块

app/crud/__init__.py

@@ -2,4 +2,7 @@ from app.crud.job_jdbc_datasource import *
 from app.crud.job_info import *
 from app.crud.datax_json import *
 from app.crud.job_log import *
-from app.crud.data_management import *
+from app.crud.data_management import *
+from app.crud.constant import *
+from app.crud.jm_homework import *
+from app.crud.jm_homework_datasource_relation import *

app/crud/constant.py

@@ -0,0 +1,16 @@
+import time
+from typing import List
+from app import models, schemas
+from sqlalchemy.orm import Session
+
+
+def create_constant(db: Session,item: schemas.ConstantCreate):
+    db_item = models.Constant(**item.dict())
+    db.add(db_item)
+    db.commit()
+    db.refresh(db_item)
+    return db_item
+
+def get_constant_list(db: Session, type: str):
+    res: List[models.Constant] = db.query(models.Constant).filter(models.Constant.type == type).all()
+    return [r.value for r in res]

app/crud/jm_homework.py

@@ -0,0 +1,59 @@
+import time
+from typing import List
+from app import models, schemas
+from sqlalchemy.orm import Session
+
+from app.crud.jm_homework_datasource_relation import create_jm_hd_relation, get_jm_relations
+
+
+def create_jm_homework(db: Session, item: schemas.JmHomeworkCreate):
+    jm_homework_create = item.dict()
+    relation_list = jm_homework_create.pop('relation_list')
+    for relation in relation_list:
+        create_jm_hd_relation(db, schemas.JmHomeworkDatasourceRelationCreate(**relation))
+    create_time: int = int(time.time())
+    db_item = models.JmHomework(**jm_homework_create,**{
+        'create_time': create_time,
+        'update_time': create_time,
+        'status': 1
+    })
+    db.add(db_item)
+    db.commit()
+    db.refresh(db_item)
+    return db_item
+
+def get_jm_homeworks(db: Session, project_id: str):
+    res: List[models.JmHomework] = db.query(models.JmHomework)\
+            .filter(models.JmHomework.project_id == project_id)\
+            .filter(models.JmHomework.status == 1).all()
+    return res
+
+def get_jm_homework_info(db: Session, homework_id: int):
+    item  = db.query(models.JmHomework)\
+            .filter(models.JmHomework.id == homework_id).first()
+    relations = get_jm_relations(db,homework_id)
+    item.__dict__.update({"hd_relation":relations})
+    return item
+
+def update_jm_homework(db: Session, id: int, update_item: schemas.JmHomeworkUpdate):
+    db_item = db.query(models.JmHomework).filter(models.JmHomework.id == id).first()
+    if not db_item:
+        raise Exception('JmHomework not found')
+    update_dict = update_item.dict(exclude_unset=True)
+    for k, v in update_dict.items():
+        setattr(db_item, k, v)
+    db_item.update_time = int(time.time())
+    db.commit()
+    db.flush()
+    db.refresh(db_item)
+    return db_item
+
+def delete_jm_homework(db: Session, id: int):
+    db_item = db.query(models.JmHomework).filter(models.JmHomework.id == id).first()
+    if not db_item:
+        raise Exception('JmHomework not found')
+    db_item.status = 0
+    db.commit()
+    db.flush()
+    db.refresh(db_item)
+    return db_item

app/crud/jm_homework_datasource_relation.py

@@ -0,0 +1,17 @@
+import time
+from typing import List
+from app import models, schemas
+from sqlalchemy.orm import Session
+
+
+def create_jm_hd_relation(db: Session, item: schemas.JmHomeworkDatasourceRelationCreate):
+    db_item = models.JmHomeworkDatasourceRelation(**item.dict())
+    db.add(db_item)
+    db.commit()
+    db.refresh(db_item)
+    return db_item
+
+def get_jm_relations(db: Session, homework_id: int):
+    res: List[models.JmHomeworkDatasourceRelation] = db.query(models.JmHomeworkDatasourceRelation)\
+                                                            .filter(models.JmHomeworkDatasourceRelation.homework_id == homework_id).all()
+    return res

app/models/__init__.py

@@ -1,4 +1,7 @@
 from app.models.job_jdbc_datasource import *
 from app.models.job_info import *
 from app.models.job_log import *
-from app.models.data_management import *
+from app.models.data_management import *
+from app.models.constant import *
+from app.models.jm_homework import *
+from app.models.jm_homework_datasource_relation import *

app/models/constant.py

@@ -0,0 +1,13 @@
+from operator import index
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
+
+from app.models.database import BaseModel
+
+class Constant(BaseModel):
+    __tablename__ = 'constant'
+
+    id = Column(Integer, primary_key=True, index=True)
+    # 常量类型
+    type = Column(String, nullable=False)
+    # 常量值
+    value = Column(String, nullable=False)

app/models/jm_homework.py

@@ -0,0 +1,33 @@
+from operator import index
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
+
+from app.models.database import BaseModel
+
+class JmHomework(BaseModel):
+    __tablename__ = 'jm_homework'
+
+    id = Column(Integer, primary_key=True, index=True)
+    # 作业名称
+    name = Column(String)
+    # 作业类型
+    type = Column(String)
+    # 作业分类
+    tag = Column(String)
+    # 执行镜像
+    image_url = Column(String)
+    # DAG_ID
+    dag_uuid = Column(String)
+    # DAG文件地址
+    dag_url = Column(String)
+    # 脚本文件
+    script_file = Column(String)
+    # 更新时间
+    update_time = Column(Integer)
+    # 创建时间
+    create_time = Column(Integer)
+    # 用户ID
+    user_id = Column(String, nullable=False)
+    # 项目ID
+    project_id = Column(String, nullable=False)
+    # 状态（0:已删除，1:未删除）
+    status = Column(Integer, nullable=False)

app/models/jm_homework_datasource_relation.py

@@ -0,0 +1,20 @@
+from operator import index
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String
+
+from app.models.database import BaseModel
+
+
+class JmHomeworkDatasourceRelation(BaseModel):
+    __tablename__ = "jm_homework_datasource_relation"
+
+    id = Column(Integer, primary_key=True, index=True)
+    # 类型
+    type = Column(String, nullable=False)
+    # 数据源id
+    datasource_id = Column(Integer, nullable=False)
+    # 节点uuid
+    node_uuid = Column(String, nullable=False)
+    # 表名
+    table = Column(String, nullable=False)
+    # 作业id
+    homework_id = Column(Integer, nullable=False)

app/routers/constants.py

@@ -1,8 +1,12 @@
 
-
+from fastapi import Depends
+from sqlalchemy.orm import Session
 from fastapi import APIRouter
 from constants.constants import CONSTANTS
 from utils import *
+import app.crud as crud
+
+from app import get_db
 
 router = APIRouter(
     prefix="/jpt/constants",
@@ -10,11 +14,19 @@ router = APIRouter(
 )
 
 
+
 def format_constants(constants: dict):
     return [{'id': k, 'value': v} for k, v in constants.items()]
 
-@router.post("/datasources")
+@router.get("/datasources")
 @web_try()
 @sxtimeit
 def get_datasources():
     return format_constants(CONSTANTS['DATASOURCES'])
+
+
+@router.get("/")
+@web_try()
+@sxtimeit
+def get_constants(type: str,db: Session = Depends(get_db)):
+    return crud.get_constant_list(db, type)

app/routers/dag.py

@@ -0,0 +1,65 @@
+import io
+from multiprocessing.connection import wait
+import os
+from typing import Optional
+from fastapi import APIRouter
+
+from fastapi import Depends
+from sqlalchemy.orm import Session
+from app import schemas
+
+import app.crud as crud
+from utils.sx_time import sxtimeit
+from utils.sx_web import web_try
+from fastapi_pagination import Page, add_pagination, paginate, Params
+from fastapi.responses import StreamingResponse
+
+from app import get_db
+
+
+
+router = APIRouter(
+    prefix="/jpt/dag",
+    tags=["dag-dag管理"],
+)
+
+@router.get("/")
+@web_try()
+@sxtimeit
+def get_dags(project_id: str, user_id: str):
+    path = "./dag"
+    file_list = get_all_dags(path)
+    return file_list
+
+def get_all_dags(path):
+    file_list = []
+    files= os.listdir(path)
+    for file in files:
+        if file == ".DS_Store":
+            continue
+        next_path = path+'/'+file
+        if os.path.isdir(next_path):
+            n_file_list = get_all_dags(next_path)
+            file_list.extend(n_file_list)
+        else:
+            if file[-4:] == ".dag":
+                file_list.append(next_path.replace('./dag',''))
+    return file_list
+
+
+@router.get("/info")
+# @web_try()
+@sxtimeit
+def get_file( uri: str):
+
+    response = StreamingResponse(get_file_byte('./dag'+uri))
+    return response
+
+def get_file_byte(filename, chunk_size=1024):
+    with open(filename, "rb") as f:
+        while True:
+            content = f.read(chunk_size)
+            if content:
+                yield content
+            else:
+                break

app/routers/jm_homework.py

@@ -0,0 +1,53 @@
+from re import S
+from typing import Optional
+from fastapi import APIRouter
+
+from fastapi import Depends
+from sqlalchemy.orm import Session
+from app import schemas
+
+import app.crud as crud
+from utils.sx_time import sxtimeit
+from utils.sx_web import web_try
+from fastapi_pagination import Page, add_pagination, paginate, Params
+
+from app import get_db
+
+
+
+router = APIRouter(
+    prefix="/jpt/jm_homework",
+    tags=["jm_homework-作业管理"],
+)
+
+@router.post("/")
+@web_try()
+@sxtimeit
+def create_jm_homework(item: schemas.JmHomeworkCreate, db: Session = Depends(get_db)):
+    # 根据获取到的文件路径另存一份并改变
+    return crud.create_jm_homework(db, item)
+
+@router.get("/")
+@web_try()
+@sxtimeit
+def get_jm_homeworks(project_id: str, db: Session = Depends(get_db)):
+    return crud.get_jm_homeworks(db, project_id)
+
+@router.get("/info")
+@web_try()
+@sxtimeit
+def get_jm_homework_info(homework_id: str, db: Session = Depends(get_db)):
+    return crud.get_jm_homework_info(db, homework_id)
+
+@router.put("/")
+@web_try()
+@sxtimeit
+def update_jm_homework(jm_id: int, update_item: schemas.JmHomeworkUpdate, db: Session = Depends(get_db)):
+    # 根据获取到的文件路径另存一份并改变
+    return crud.update_jm_homework(db, jm_id, update_item)
+
+@router.delete("/")
+@web_try()
+@sxtimeit
+def delete_jm_homework(jm_id: int, db: Session = Depends(get_db)):
+    return crud.delete_jm_homework(db, jm_id)

app/schemas/__init__.py

@@ -2,4 +2,6 @@ from app.schemas.job_jdbc_datasouce import *
 from app.schemas.job_info import *
 from app.schemas.datax_json import *
 from app.schemas.job_log import *
-from app.schemas.data_management import *
+from app.schemas.data_management import *
+from app.schemas.constant import *
+from app.schemas.jm_homework import *

app/schemas/constant.py

@@ -0,0 +1,24 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+class ConstantBase(BaseModel):
+    # 常量类型
+    type: str
+    # 常量值
+    value: str
+
+class ConstantCreate(ConstantBase):
+    class Config:
+        schema_extra = {
+            "example": {
+                "type": "作业类型",
+                "value": "textValue",
+            }
+        }
+
+class Constant(ConstantBase):
+    id: int
+
+    class Config:
+        orm_mode = True

app/schemas/jm_homework.py

@@ -0,0 +1,82 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from app.schemas.jm_homework_datasource_relation import JmHomeworkDatasourceRelationCreate
+
+class JmHomeworkBase(BaseModel):
+    # 作业名称
+    name: str
+    # 作业类型
+    type: str
+    # 作业分类
+    tag: str
+    # 执行镜像
+    image_url: str
+    # DAG_ID
+    dag_uuid: str
+    # DAG文件地址
+    dag_url: str
+    # 脚本文件
+    script_file: str
+    # 用户ID
+    user_id: str
+    # 项目ID
+    project_id: str
+
+
+class JmHomeworkCreate(JmHomeworkBase):
+    relation_list: List[JmHomeworkDatasourceRelationCreate]
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "name": "监察流水线",
+                "type": "java",
+                "tag": "作业监测",
+                "image_url": "/test/images/example",
+                "dag_uuid": "",
+                "dag_url": "",
+                "script_file": "/test/scripts/example",
+                "user_id": "test",
+                "project_id": "test",
+                "relation_list": [
+                    {
+                        "type": "input",
+                        "datasource_id": 1,
+                        "node_uuid": "77a6e831-bb62-42e1-be8d-97699db00e73",
+                        "table": "jm_homework_datasource_relation",
+                        "homework_id": 1,
+                    }
+                ]
+            }
+        }
+
+class JmHomeworkUpdate(JmHomeworkBase):
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "name": "监察流水线",
+                "type": "java",
+                "tag": "作业监测",
+                "image_url": "/test/images/example",
+                "dag_uuid": "",
+                "dag_url": "",
+                "script_file": "/test/scripts/example",
+                "user_id": "test",
+                "project_id": "test",
+            }
+        }
+
+class JmHomework(JmHomeworkBase):
+    id: int
+    # 创建时间
+    create_time: int
+    # 更新时间
+    update_time: int
+    # 状态
+    status: int
+
+    class Config:
+        orm_mode = True

app/schemas/jm_homework_datasource_relation.py

@@ -0,0 +1,35 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+class JmHomeworkDatasourceRelationBase(BaseModel):
+    # 类型
+    type: str
+    # 数据源id
+    datasource_id: int
+    # 节点uuid
+    node_uuid: str
+    # 表名
+    table: str
+    # 作业id
+    homework_id: int
+
+
+class JmHomeworkDatasourceRelationCreate(JmHomeworkDatasourceRelationBase):
+    class Config:
+        schema_extra = {
+            "example": {
+                "type": "input",
+                "datasource_id": 1,
+                "node_uuid": "77a6e831-bb62-42e1-be8d-97699db00e73",
+                "table": "jm_homework_datasource_relation",
+                "homework_id": 1,
+            }
+        }
+
+
+class JmHomeworkDatasourceRelation(JmHomeworkDatasourceRelationBase):
+    id: int
+
+    class Config:
+        orm_mode = True

dag/test.dag

@@ -0,0 +1,442 @@
+{
+  "user_name": "test",
+  "user_id": "test",
+  "project_name": "test",
+  "project_id": "test",
+  "itermidate_data": [
+    "hdfs://host:port/uri"
+  ],
+  "nodes": [
+    {
+      "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "op": "DataSource",
+      "data": {
+        "input_source": [
+          {
+            "dataSelect": false,
+            "dataField": "name",
+            "dataType": "string"
+          },
+          {
+            "dataSelect": true,
+            "dataField": "address",
+            "dataType": "string"
+          }
+        ],
+        "input_table": 5
+      }
+    },
+    {
+      "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+      "op": "SQL",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r3"
+          }
+        ],
+        "script": "Select * from table11",
+        "param": "test SQL Param"
+      }
+    },
+    {
+      "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+      "op": "PySpark",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r2"
+          }
+        ],
+        "script": "import pySpark\n\nprint('hello wolrd pySpark')",
+        "param": "test PySpark Param"
+      }
+    },
+    {
+      "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+      "op": "Python",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r1"
+          }
+        ],
+        "script": "import test\nprint('hello world')",
+        "param": "test param",
+        "package": "test = 1.1\ntest1 = 2.2"
+      }
+    }
+  ],
+  "edges": [
+    {
+      "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd"
+    },
+    {
+      "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde"
+    },
+    {
+      "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "6b437370-0468-44da-a0bb-e637e33f3efb"
+    }
+  ],
+  "graph": {
+    "cells": [
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "port": "topPort"
+        }
+      },
+      {
+        "position": {
+          "x": 240,
+          "y": 130
+        },
+        "size": {
+          "width": 180,
+          "height": 80
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+        "data": {
+          "label": "DataSource",
+          "status": "default",
+          "type": "datasource",
+          "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "inputSource": [
+            {
+              "dataSelect": false,
+              "dataField": "name",
+              "dataType": "string"
+            },
+            {
+              "dataSelect": true,
+              "dataField": "address",
+              "dataType": "string"
+            }
+          ],
+          "dataTable": 5
+        },
+        "zIndex": 1
+      },
+      {
+        "position": {
+          "x": 480,
+          "y": 300
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+        "data": {
+          "label": "SQL",
+          "status": "default",
+          "type": "script",
+          "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "paramText": "test SQL Param",
+          "scriptText": "Select * from table11",
+          "outputData": [
+            {
+              "outputVar": "r3"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 2
+      },
+      {
+        "position": {
+          "x": 230,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+        "data": {
+          "label": "PySpark",
+          "status": "default",
+          "type": "script",
+          "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "paramText": "test PySpark Param",
+          "scriptText": "import pySpark\n\nprint('hello wolrd pySpark')",
+          "outputData": [
+            {
+              "outputVar": "r2"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 3
+      },
+      {
+        "position": {
+          "x": -10,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+        "data": {
+          "label": "Python",
+          "status": "default",
+          "type": "script",
+          "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "paramText": "test param",
+          "scriptText": "import test\nprint('hello world')",
+          "outputData": [
+            {
+              "outputVar": "r1"
+            }
+          ],
+          "inputNumber": 1,
+          "packageData": "test = 1.1\ntest1 = 2.2"
+        },
+        "zIndex": 4
+      }
+    ]
+  }
+}

dag/test/test.dag

@@ -0,0 +1,442 @@
+{
+  "user_name": "test",
+  "user_id": "test",
+  "project_name": "test",
+  "project_id": "test",
+  "itermidate_data": [
+    "hdfs://host:port/uri"
+  ],
+  "nodes": [
+    {
+      "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "op": "DataSource",
+      "data": {
+        "input_source": [
+          {
+            "dataSelect": false,
+            "dataField": "name",
+            "dataType": "string"
+          },
+          {
+            "dataSelect": true,
+            "dataField": "address",
+            "dataType": "string"
+          }
+        ],
+        "input_table": 5
+      }
+    },
+    {
+      "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+      "op": "SQL",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r3"
+          }
+        ],
+        "script": "Select * from table11",
+        "param": "test SQL Param"
+      }
+    },
+    {
+      "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+      "op": "PySpark",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r2"
+          }
+        ],
+        "script": "import pySpark\n\nprint('hello wolrd pySpark')",
+        "param": "test PySpark Param"
+      }
+    },
+    {
+      "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+      "op": "Python",
+      "data": {
+        "input": 1,
+        "output": [
+          {
+            "outputVar": "r1"
+          }
+        ],
+        "script": "import test\nprint('hello world')",
+        "param": "test param",
+        "package": "test = 1.1\ntest1 = 2.2"
+      }
+    }
+  ],
+  "edges": [
+    {
+      "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd"
+    },
+    {
+      "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde"
+    },
+    {
+      "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+      "source": "77a6e831-bb62-42e1-be8d-97699db00e73",
+      "target": "6b437370-0468-44da-a0bb-e637e33f3efb"
+    }
+  ],
+  "graph": {
+    "cells": [
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "b29bcdc7-b543-4d4a-9bd7-7fa74f3a548d",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "e87690d4-f71d-44f5-85fc-3c14b32b323e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "port": "topPort"
+        }
+      },
+      {
+        "shape": "dag-edge",
+        "attrs": {
+          "line": {
+            "strokeDasharray": ""
+          }
+        },
+        "id": "4c312a41-5abf-47a8-a6ad-c2fbba147c4e",
+        "zIndex": -1,
+        "source": {
+          "cell": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "port": "bottomPort"
+        },
+        "target": {
+          "cell": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "port": "topPort"
+        }
+      },
+      {
+        "position": {
+          "x": 240,
+          "y": 130
+        },
+        "size": {
+          "width": 180,
+          "height": 80
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+        "data": {
+          "label": "DataSource",
+          "status": "default",
+          "type": "datasource",
+          "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
+          "inputSource": [
+            {
+              "dataSelect": false,
+              "dataField": "name",
+              "dataType": "string"
+            },
+            {
+              "dataSelect": true,
+              "dataField": "address",
+              "dataType": "string"
+            }
+          ],
+          "dataTable": 5
+        },
+        "zIndex": 1
+      },
+      {
+        "position": {
+          "x": 480,
+          "y": 300
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+        "data": {
+          "label": "SQL",
+          "status": "default",
+          "type": "script",
+          "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
+          "paramText": "test SQL Param",
+          "scriptText": "Select * from table11",
+          "outputData": [
+            {
+              "outputVar": "r3"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 2
+      },
+      {
+        "position": {
+          "x": 230,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+        "data": {
+          "label": "PySpark",
+          "status": "default",
+          "type": "script",
+          "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
+          "paramText": "test PySpark Param",
+          "scriptText": "import pySpark\n\nprint('hello wolrd pySpark')",
+          "outputData": [
+            {
+              "outputVar": "r2"
+            }
+          ],
+          "inputNumber": 1
+        },
+        "zIndex": 3
+      },
+      {
+        "position": {
+          "x": -10,
+          "y": 310
+        },
+        "size": {
+          "width": 180,
+          "height": 36
+        },
+        "view": "react-shape-view",
+        "shape": "dag-node",
+        "component": {
+          "key": null,
+          "ref": null,
+          "props": {},
+          "_owner": null,
+          "_store": {}
+        },
+        "ports": {
+          "groups": {
+            "top": {
+              "position": "top",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            },
+            "bottom": {
+              "position": "bottom",
+              "attrs": {
+                "circle": {
+                  "r": 4,
+                  "magnet": true,
+                  "stroke": "#C2C8D5",
+                  "strokeWidth": 1,
+                  "fill": "#fff"
+                }
+              }
+            }
+          },
+          "items": [
+            {
+              "id": "topPort",
+              "group": "top"
+            },
+            {
+              "id": "bottomPort",
+              "group": "bottom"
+            }
+          ]
+        },
+        "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+        "data": {
+          "label": "Python",
+          "status": "default",
+          "type": "script",
+          "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
+          "paramText": "test param",
+          "scriptText": "import test\nprint('hello world')",
+          "outputData": [
+            {
+              "outputVar": "r1"
+            }
+          ],
+          "inputNumber": 1,
+          "packageData": "test = 1.1\ntest1 = 2.2"
+        },
+        "zIndex": 4
+      }
+    ]
+  }
+}

data/data.sql

@@ -23,7 +23,7 @@ CREATE TABLE `job_jdbc_datasource`  (
   `update_time` int(20) NULL COMMENT '更新时间',
   `comments` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '备注',
   PRIMARY KEY (`id`) USING BTREE
-) ENGINE = InnoDB AUTO_INCREMENT = 6 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = 'jdbc数据源配置' ROW_FORMAT = Dynamic;
+) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = 'jdbc数据源配置' ROW_FORMAT = Dynamic;
 
 
 
@@ -51,7 +51,7 @@ CREATE TABLE `job_info`  (
   `inc_start_time` int(11) NULL COMMENT '增量初始时间',
   `last_handle_code` int(11) NULL COMMENT '最近一次执行状态',
   PRIMARY KEY (`id`) USING BTREE
-) ENGINE = InnoDB AUTO_INCREMENT = 7 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
+) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
 
 
 -- ----------------------------
@@ -100,7 +100,7 @@ CREATE TABLE `job_log` (
   PRIMARY KEY (`id`) USING BTREE,
   KEY `I_trigger_time` (`trigger_time`) USING BTREE,
   KEY `I_handle_code` (`handle_code`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1581 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
 
 -- ----------------------------
 -- Table structure for data_management
@@ -120,4 +120,54 @@ CREATE TABLE `data_management` (
   `user_id` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '创建人编号',
   `project_id` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '项目编号',
   PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8_unicode_ci COMMENT='数据管理' ROW_FORMAT=DYNAMIC;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_unicode_ci COMMENT='数据管理' ROW_FORMAT=DYNAMIC;
+
+
+-- ----------------------------
+-- Table structure for constant
+-- ----------------------------
+DROP TABLE IF EXISTS `constant`;
+CREATE TABLE `constant` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `type` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '常量类型',
+  `value` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '常量值',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_unicode_ci COMMENT='常量表';
+
+
+-- ----------------------------
+-- Table structure for jm_homework
+-- ----------------------------
+DROP TABLE IF EXISTS `jm_homework`;
+CREATE TABLE `jm_homework` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `name` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '作业名称',
+  `type` varchar(20) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '作业类型',
+  `tag` varchar(20) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '作业分类',
+  `image_url` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '执行镜像',
+  `dag_uuid` varchar(32) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DAG_ID',
+  `dag_url` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DAG文件地址',
+  `script_file` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '脚本文件',
+  `update_time` int(13) DEFAULT NULL COMMENT '更新时间',
+  `create_time` int(13) NOT NULL COMMENT '创建时间',
+  `user_id` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '用户ID',
+  `project_id` varchar(50) COLLATE utf8_unicode_ci NOT NULL COMMENT '项目ID',
+  `status` tinyint(4) NOT NULL COMMENT '状态（0:已删除，1:未删除）',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_unicode_ci COMMENT='作业信息';
+
+
+
+
+-- ----------------------------
+-- Table structure for jm_homework_datasource_relation
+-- ----------------------------
+DROP TABLE IF EXISTS `jm_homework_datasource_relation`;
+CREATE TABLE `jm_homework_datasource_relation` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `type` varchar(255) COLLATE utf8_unicode_ci NOT NULL COMMENT '类型',
+  `datasource_id` bigint(20) NOT NULL COMMENT '数据源ID',
+  `table` varchar(255) COLLATE utf8_unicode_ci NOT NULL COMMENT '数据源对应数据表',
+  `homework_id` bigint(20) NOT NULL COMMENT '作业ID',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_unicode_ci COMMENT='作业与数据源关系表';

server.py

@@ -12,6 +12,8 @@ import app.routers.datax_json as router_datax
 import app.routers.data_management as router_data_management
 import app.routers.files as router_files
 import app.routers.intermediate as router_intermediate
+import app.routers.jm_homework as router_jm_homework
+import app.routers.dag as router_dag
 
 Base.metadata.create_all(bind=engine)
 app = FastAPI( docs_url='/jpt/docs', redoc_url='/jpt/redoc', title="DAG管理系统")
@@ -35,6 +37,9 @@ app.include_router(router_data_management.router)
 app.include_router(router_datax.router)
 app.include_router(router_files.router)
 app.include_router(router_intermediate.router)
+app.include_router(router_jm_homework.router)
+app.include_router(router_dag.router)
+