python 代码校验

app/crud/constant.py

@@ -13,4 +13,17 @@ def create_constant(db: Session,item: schemas.ConstantCreate):
 
 def get_constant_list(db: Session, type: str):
     res: List[models.Constant] = db.query(models.Constant).filter(models.Constant.type == type).all()
-    return [r.value for r in res]
+    return [r.value for r in res]
+
+def find_and_update(db: Session, type: str, value: str):
+    res: List[models.Constant] = db.query(models.Constant)\
+        .filter(models.Constant.type == type)\
+        .filter(models.Constant.value == value).all()
+    if len(res) == 0:
+        db_item = models.Constant(**{
+            'type': type,
+            'value': value
+        })
+        db.add(db_item)
+        db.commit()
+        db.refresh(db_item)

app/crud/jm_homework.py

@@ -2,15 +2,22 @@ import time
 from typing import List
 from app import models, schemas
 from sqlalchemy.orm import Session
+from app.crud.constant import find_and_update
 
 from app.crud.jm_homework_datasource_relation import create_jm_hd_relation, get_jm_relations
 
 
 def create_jm_homework(db: Session, item: schemas.JmHomeworkCreate):
     jm_homework_create = item.dict()
+    db_item = db.query(models.JmHomework).filter(models.JmHomework.name == jm_homework_create['name']).first()
+    if db_item:
+        raise Exception('JmHomework name already exists')
     relation_list = jm_homework_create.pop('relation_list')
-    for relation in relation_list:
-        create_jm_hd_relation(db, schemas.JmHomeworkDatasourceRelationCreate(**relation))
+    if jm_homework_create['type'] != 'DAG':
+        for relation in relation_list:
+            create_jm_hd_relation(db, schemas.JmHomeworkDatasourceRelationCreate(**relation))
+    tag = jm_homework_create['tag']
+    find_and_update(db, '作业分类', tag)
     create_time: int = int(time.time())
     db_item = models.JmHomework(**jm_homework_create,**{
         'create_time': create_time,
@@ -31,15 +38,21 @@ def get_jm_homeworks(db: Session, project_id: str):
 def get_jm_homework_info(db: Session, homework_id: int):
     item  = db.query(models.JmHomework)\
             .filter(models.JmHomework.id == homework_id).first()
-    relations = get_jm_relations(db,homework_id)
-    item.__dict__.update({"hd_relation":relations})
+    if item.type == 'DAG':
+        relations = get_jm_relations(db,homework_id)
+        item.__dict__.update({"hd_relation":relations})
     return item
 
 def update_jm_homework(db: Session, id: int, update_item: schemas.JmHomeworkUpdate):
     db_item = db.query(models.JmHomework).filter(models.JmHomework.id == id).first()
     if not db_item:
         raise Exception('JmHomework not found')
+    db_name_item = db.query(models.JmHomework).filter(models.JmHomework.name == update_item.name).first()
+    if db_name_item:
+        raise Exception('JmHomework name already exists')
     update_dict = update_item.dict(exclude_unset=True)
+    tag = update_dict.tag
+    find_and_update(db, '作业分类', tag)
     for k, v in update_dict.items():
         setattr(db_item, k, v)
     db_item.update_time = int(time.time())

app/routers/code_check.py

@@ -0,0 +1,39 @@
+import json
+import os
+import tempfile
+from typing import Optional
+from fastapi import APIRouter
+from fastapi import Depends
+from app import schemas
+from utils.sx_time import sxtimeit
+from utils.sx_web import web_try
+from pylint import epylint
+
+
+router = APIRouter(
+    prefix="/jpt/code_check",
+    tags=["code_check-代码校验"],
+)
+
+@router.post("/")
+@web_try()
+@sxtimeit
+def code_check(code: str, code_type: str):
+    res = []
+    if code_type == "python":
+        file = tempfile.NamedTemporaryFile(delete=False, mode='w')
+        file.write(code)
+        file.close()
+
+        options = ' '.join([
+            file.name,
+            '--output-format', 'json',
+        ])
+
+        (lint_stdout, lint_stderr) = epylint.py_run(return_std=True, command_options=options)
+
+        os.remove(file.name)
+        res = json.loads(lint_stdout.getvalue())
+    else:
+        res = []
+    return res

dag/test.dag

@@ -9,11 +9,12 @@
   "nodes": [
     {
       "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
-      "op": "DataSource",
+      "name": "DataSource",
+      "op": "datasource",
       "data": {
         "input_source": [
           {
-            "dataSelect": false,
+            "dataSelect": true,
             "dataField": "name",
             "dataType": "string"
           },
@@ -28,7 +29,8 @@
     },
     {
       "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
-      "op": "SQL",
+      "name": "SQL",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [
@@ -42,7 +44,8 @@
     },
     {
       "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
-      "op": "PySpark",
+      "name": "PySpark",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [
@@ -56,7 +59,8 @@
     },
     {
       "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
-      "op": "Python",
+      "name": "Python",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [

dag/test/test.dag

@@ -9,11 +9,12 @@
   "nodes": [
     {
       "id": "77a6e831-bb62-42e1-be8d-97699db00e73",
-      "op": "DataSource",
+      "name": "DataSource",
+      "op": "datasource",
       "data": {
         "input_source": [
           {
-            "dataSelect": false,
+            "dataSelect": true,
             "dataField": "name",
             "dataType": "string"
           },
@@ -28,7 +29,8 @@
     },
     {
       "id": "143cfcb1-f566-4f97-b38b-9b62d40fe4cd",
-      "op": "SQL",
+      "name": "SQL",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [
@@ -42,7 +44,8 @@
     },
     {
       "id": "6b437370-0468-44da-a0bb-e637e33f3efb",
-      "op": "PySpark",
+      "name": "PySpark",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [
@@ -56,7 +59,8 @@
     },
     {
       "id": "c79943c6-9513-4cf8-9c24-d5fa89b6cdde",
-      "op": "Python",
+      "name": "Python",
+      "op": "script",
       "data": {
         "input": 1,
         "output": [

environment.yml

@@ -22,6 +22,7 @@ dependencies:
       - gunicorn
       - fastapi-pagination==0.9.3
       - pymysql==1.0.2
+      - pylint==2.5.3
       - PyHive==0.6.5
       - pure-sasl==0.6.2
       # - sasl==0.3.1

requirements.txt

@@ -4,4 +4,5 @@ PyHive==0.6.5
 pure-sasl==0.6.2
 sasl==0.3.1
 thrift==0.16.0
-thrift-sasl==0.4.3
+thrift-sasl==0.4.3
+pylint==2.5.3

server.py

@@ -14,6 +14,7 @@ import app.routers.files as router_files
 import app.routers.intermediate as router_intermediate
 import app.routers.jm_homework as router_jm_homework
 import app.routers.dag as router_dag
+import app.routers.code_check as router_code_check
 
 Base.metadata.create_all(bind=engine)
 app = FastAPI( docs_url='/jpt/docs', redoc_url='/jpt/redoc', title="DAG管理系统")
@@ -39,6 +40,8 @@ app.include_router(router_files.router)
 app.include_router(router_intermediate.router)
 app.include_router(router_jm_homework.router)
 app.include_router(router_dag.router)
+app.include_router(router_code_check.router)
+