定时清理表任务测试

app/crud/data_table.py

@@ -176,7 +176,7 @@ def get_lake_table(db: Session, project_id: int):
 # 获取数据湖表详情
 def get_lake_table_info(db: Session, dl_id: int):
     data_table: models.DataTable = db.query(models.DataTable)\
-            .filter(models.DataTable.id==dl_id).first()
+            .filter(models.DataTable.id == dl_id).first()
     if not data_table: raise Exception('未找到该数据湖表')
     return data_table
 
@@ -188,9 +188,24 @@ def get_table_info(db: Session, db_name: str, table_name: str):
 # 检查表是否被分享
 def check_share(db: Session, table_name: str):
     data_tables: List[models.DataTable] = db.query(models.DataTable)\
-            .filter(models.DataTable.table_name==table_name)\
+            .filter(models.DataTable.table_name == table_name)\
             .filter(models.DataTable.type == 0).all()
     if len(data_tables) > 0:
         return 1
     return 0
 
+def delete_nonentity_table(db: Session):
+    data_tables: List[models.DataTable] = db.query(models.DataTable) \
+        .filter(models.DataTable.source == 1) \
+        .filter(models.DataTable.type == 1).all()
+    table_ids = []
+    for data_table in data_tables:
+        print(data_table.table_name)
+        res = hiveDs._execute_sql([f"show tables like '{data_table.table_name}'"])
+        print(res)
+        if len(res[0][0]) <= 0:
+            print(f"{data_table.table_name}不存在，需要删除")
+            table_ids.append(data_table.id)
+    res = db.query(models.DataTable).filter(models.DataTable.id.in_(table_ids)).delete()
+    print(f"数据湖中不存在的表已清除-->{res}")
+    return res

app/routers/job_jdbc_datasource.py

@@ -21,6 +21,12 @@ router = APIRouter(
     tags=["datasource-数据源管理"],
 )
 
+@router.delete("/nonentity_table")
+@web_try()
+@sxtimeit
+def delete_nonentity_table(db: Session = Depends(get_db)):
+    return crud.delete_nonentity_table(db)
+
 # 测试数据源连接
 @router.post("/test")
 @web_try()

delete.py

@@ -1,4 +1,5 @@
 import datetime
+import requests
 import os
 import shutil
 from fastapi import FastAPI
@@ -38,10 +39,15 @@ def delete_log():
             dir_path = scheduler_path + '/' + file_name
     print(f'The scheduled task is successfully executed at time {ts}')
 
+def delete_nonentity_table():
+    print("删除数据湖中不存在的表开始")
+    requests.delete("http://localhost:8080/jpt/datasource/nonentity_table")
+
 #创建调度器：BackgroundScheduler
 scheduler = BackgroundScheduler()
 #添加任务,时间间隔2S
 scheduler.add_job(delete_log, 'cron', hour='1', id='delete_job')
+scheduler.add_job(delete_nonentity_table, 'interval', minutes=10)
 scheduler.start()
 print('任务创建成功')