Merge branch 'master' of http://gogsb.soaringnova.com/liutao/ai-proj-demo

src/module/datasource/component/DatasourceAdd.jsx

@@ -9,28 +9,59 @@ import {
 } from '@ant-design/pro-components'
 import { Button, Col, message, Form } from 'antd'
 import { useState } from 'react'
-import {testDataSourceConnection, createDataSource} from '../services'
+import {
+  testDataSourceConnection,
+  createDataSource,
+  uploadFile,
+} from '../services'
 
 const DATA_TYPE_MYSQL = 'mysql'
 const KERBS_VALID_FALSE = false
 
-export default function DatasourceAdd({updateDataSource}) {
+export default function DatasourceAdd({ updateDataSource }) {
   // 数据源类型
   const [dataType, setDataType] = useState(DATA_TYPE_MYSQL)
   // 是否开启kerbsValid验证
   const [kerbsValid, setKerbsValid] = useState(KERBS_VALID_FALSE)
+  const [kerberosServiceName, setKerberosServiceName] = useState('')
+  const [principal, setPrincipal] = useState(KERBS_VALID_FALSE)
   // Loading状态
   const [loading, setLoading] = useState(false)
+
+  const [keytabFile, setKeytabFile] = useState([])
+  const [krd5File, setKrd5File] = useState([])
   // Form表单
   const [form] = Form.useForm()
   // 测试连接
   const testConnect = async () => {
-    const fields = ['datasource_name','datasource','database_name','jdbc_username','jdbc_password','jdbc_url','comments', 'tag']
+    const fields = [
+      'datasource_name',
+      'datasource',
+      'database_name',
+      'jdbc_username',
+      'jdbc_password',
+      'jdbc_url',
+      'comments',
+      'tag',
+    ]
     try {
       await form.validateFields(fields)
       setLoading(true)
       const testData = form.getFieldsValue(fields)
-      const { data } = await testDataSourceConnection({...testData, tag: form.getFieldValue('tag').join('/')})
+      const params = {
+        ...testData,
+        tag: form.getFieldValue('tag').join('/'),
+      }
+      params['kerberos'] = kerbsValid ? 1 : 0
+      if (kerbsValid) {
+        const keytab = await uploadKerbFile(keytabFile)
+        const krb5config = await uploadKerbFile(krd5File)
+        params['keytab'] = keytab
+        params['krb5config'] = krb5config
+        params['kerberos_service_name'] = kerberosServiceName
+        params['principal'] = principal
+      }
+      const { data } = await testDataSourceConnection(params)
       if (data.data) {
         message.success('连接成功')
       } else {
@@ -43,8 +74,30 @@ export default function DatasourceAdd({updateDataSource}) {
   }
   // 完成添加
   const finishAdd = async () => {
-    const fields = ['datasource_name','datasource','database_name','jdbc_username','jdbc_password','jdbc_url','comments', 'tag']
-    const { data } = await createDataSource({...form.getFieldsValue(fields), tag: form.getFieldValue('tag').join('/')})
+    const fields = [
+      'datasource_name',
+      'datasource',
+      'database_name',
+      'jdbc_username',
+      'jdbc_password',
+      'jdbc_url',
+      'comments',
+      'tag',
+    ]
+    const params = {
+      ...form.getFieldsValue(fields),
+      tag: form.getFieldValue('tag').join('/'),
+    }
+    params['kerberos'] = kerbsValid ? 1 : 0
+    if (kerbsValid) {
+      const keytab = await uploadKerbFile(keytabFile)
+      const krb5config = await uploadKerbFile(krd5File)
+      params['keytab'] = keytab
+      params['krb5config'] = krb5config
+      params['kerberos_service_name'] = kerberosServiceName
+      params['principal'] = principal
+    }
+    const { data } = await createDataSource(params)
     if (data.code === 200) {
       message.success('添加成功')
       updateDataSource()
@@ -53,23 +106,66 @@ export default function DatasourceAdd({updateDataSource}) {
     }
     return true
   }
+  const uploadKerbFile = async fileList => {
+    const file = new FormData()
+    fileList.forEach(item => {
+      file.append('file', item)
+    })
+    file.append('project_id', 'test')
+    file.append('file_type', 'kerberos')
+    const { data } = await uploadFile(file)
+    let uri = ''
+    if (data.code === 200) {
+      uri = data.data
+    } else {
+      message.error(data.msg)
+    }
+    return uri
+  }
   // 测试连接按钮
   const testConnectBtn = (
     <Button onClick={testConnect} loading={loading}>
       测试连接
     </Button>
   )
+
+  //上传文件配置
+  const keytabUpProps = {
+    onRemove: file => {
+      const index = keytabFile.indexOf(file)
+      const newFileList = keytabFile.slice()
+      newFileList.splice(index, 1)
+      setKeytabFile(newFileList)
+    },
+    beforeUpload: file => {
+      setKeytabFile([...keytabFile, file])
+      return false
+    },
+    keytabFile,
+  }
+
+  const krd5UpProps = {
+    onRemove: file => {
+      const index = krd5File.indexOf(file)
+      const newFileList = krd5File.slice()
+      newFileList.splice(index, 1)
+      setKrd5File(newFileList)
+    },
+    beforeUpload: file => {
+      setKrd5File([...krd5File, file])
+      return false
+    },
+    krd5File,
+  }
   return (
     <ModalForm
       form={form}
-      trigger={
-        <Button type="primary">添加数据源</Button>
-      }
+      trigger={<Button type="primary">添加数据源</Button>}
       onFinish={finishAdd}
       initialValues={{
         datasource: DATA_TYPE_MYSQL,
         kerbsValid: KERBS_VALID_FALSE,
-        comments: ''
+        comments: '',
       }}
       title="添加数据源"
       modalProps={{
@@ -121,12 +217,12 @@ export default function DatasourceAdd({updateDataSource}) {
         label="数据源标签"
         name="tag"
         fieldProps={{
-          mode: 'tags'
+          mode: 'tags',
         }}
         valueEnum={{
-          '线上': '线上',
-          '测试': '测试',
-          '开发': '开发',
+          线上: '线上',
+          测试: '测试',
+          开发: '开发',
         }}
         rules={[
           {
@@ -160,7 +256,7 @@ export default function DatasourceAdd({updateDataSource}) {
         name="jdbc_username"
         rules={[
           {
-            required: true,
+            required: !kerbsValid ? true : false,
             message: '请输入用户名!',
           },
         ]}
@@ -170,7 +266,7 @@ export default function DatasourceAdd({updateDataSource}) {
         name="jdbc_password"
         rules={[
           {
-            required: true,
+            required: !kerbsValid ? true : false,
             message: '请输入密码!',
           },
         ]}
@@ -182,8 +278,8 @@ export default function DatasourceAdd({updateDataSource}) {
       />
       {dataType === 'hive' ? (
         <ProFormRadio.Group
-          label="kerbs验证"
-          name="kerbsValid"
+          label="kerberos验证"
+          name="kerberosValid"
           fieldProps={{
             onChange: e => setKerbsValid(e.target.value),
           }}
@@ -211,25 +307,38 @@ export default function DatasourceAdd({updateDataSource}) {
       {dataType === 'hive' && kerbsValid ? (
         <>
           <ProFormUploadButton
-            label="keytab"
+            label="keytab文件"
             name="keytab"
-            action="upload.do"
             title="上传keyTabPath"
+            fieldProps={{ ...keytabUpProps }}
             max={1}
           />
           <Col span={12} />
           <ProFormUploadButton
-            label="krd5config"
-            name="krd5config"
-            action="upload.do"
-            title="上传kard5config"
+            label="krb5config文件"
+            name="krb5config"
+            title="上传krb5config"
+            fieldProps={{ ...krd5UpProps }}
             max={1}
           />
           <Col span={12} />
+          <ProFormText
+            label="kerberos_service_name"
+            name="kerberos_service_name"
+            placeholder="输入kerberos_service_name"
+            fieldProps={{
+              onChange: e => setKerberosServiceName(e.target.value),
+            }}
+          />
+          <Col span={12} />
           <ProFormText
             label="Principal"
             name="principal"
-            placeholder="ylaiuser@EMR-56L6ZNTS"></ProFormText>
+            placeholder="ylaiuser@EMR-56L6ZNTS"
+            fieldProps={{
+              onChange: e => setPrincipal(e.target.value),
+            }}
+          />
         </>
       ) : (
         <></>

src/module/datasource/component/DatasourceSyncView.jsx

@@ -42,7 +42,7 @@ export default function DatasourceSyncView({ selectLog }) {
         }
       })
       setJobList(list)
-      setJobData(data.data.items)
+      setJobData(data.data.item)
     }
     setDataLoading(false)
   }

src/module/datasource/services/index.js

@@ -120,3 +120,10 @@ export const refreshLogsStatus = str =>
     url: '/jpt/joblog/logs_status/' + str,
     method: 'get',
   })
+
+export const uploadFile = params =>
+  request({
+    url: '/jpt/files/upload_file',
+    method: 'post',
+    data: params,
+  })

src/module/workmgmt/component/JobCreate.jsx

@@ -20,12 +20,12 @@ import { useNavigate, useLocation } from 'react-router-dom'
 import {
   getJobTypes,
   createJob,
-  getDagInfomation,
+  getFileInfomation,
   getJobInfo,
   deleteTag,
   updateJobInfo,
-  uploadJar,
-  uploadPy,
+  uploadFile,
+  getFileUri,
 } from '../services'
 
 const JobCre = styled.div`
@@ -175,12 +175,34 @@ export default function JobCreate() {
 
   const [fileList, setFileList] = useState([])
 
+  const [javas, setJavas] = useState([])
+  const [pythons, setPythons] = useState([])
+
   useEffect(() => {
     if (state) {
       fetchJobInfo(state.id)
     }
   }, [state])
 
+  useEffect(() => {
+    fetchFileUri('java')
+    fetchFileUri('python')
+  }, [])
+
+  const fetchFileUri = async type => {
+    const params = {
+      project_id: 'test',
+      user_id: 'test',
+      file_type: type,
+    }
+    const { data } = await getFileUri(params)
+    if (data.code === 200) {
+      type === 'jave' ? setJavas(data.data) : setPythons(data.data)
+    } else {
+      message.error(data.msg)
+    }
+  }
+
   const fetchJobInfo = async id => {
     const { data } = await getJobInfo(id)
     if (data.code === 200) {
@@ -289,7 +311,6 @@ export default function JobCreate() {
       setFileList([...fileList, file])
       return false
     },
-    maxCount: 1,
     fileList,
   }
 
@@ -323,10 +344,12 @@ export default function JobCreate() {
 
   //提交DAG
   const DagSubmit = async () => {
-    const res = await getDagInfomation(dagData)
-    if (res.status === 200) {
-      const innodes = res.data.nodes.filter(item => item.op === 'datasource')
-      const outnodes = res.data.nodes.filter(item => item.op === 'outputsource')
+    const { data } = await getFileInfomation(dagData)
+    if (data.code === 200) {
+      const innodes = data.data.nodes.filter(item => item.op === 'datasource')
+      const outnodes = data.data.nodes.filter(
+        item => item.op === 'outputsource'
+      )
       const relation_list = []
       innodes.forEach((item, index) => {
         relation_list.push({
@@ -449,8 +472,9 @@ export default function JobCreate() {
       fileList.forEach(item => {
         file.append('file', item)
       })
-      const { data } =
-        currentJobType === 'Java' ? await uploadJar(file) : await uploadPy(file)
+      file.append('project_id', 'test')
+      file.append('file_type', currentJobType === 'Java' ? 'java' : 'python')
+      const { data } = await uploadFile(file)
       if (data.code === 200) {
         params['script_file'] = data.data
       } else {
@@ -699,7 +723,11 @@ export default function JobCreate() {
                         value={scriptFile}
                         onChange={onScriptFileChange}
                         disabled={radioValue === 'fromList' ? false : true}>
-                        <Option value="test">test</Option>
+                        {pythons.map(item => (
+                          <Option key={item.uri} value={item.name}>
+                            {item.name}
+                          </Option>
+                        ))}
                       </Select>
                     </span>
                   </Radio>
@@ -846,7 +874,11 @@ export default function JobCreate() {
                         value={scriptFile}
                         onChange={onScriptFileChange}
                         disabled={radioValue === 'fromList' ? false : true}>
-                        <Option value="test">test</Option>
+                        {javas.map(item => (
+                          <Option key={item.uri} value={item.name}>
+                            {item.name}
+                          </Option>
+                        ))}
                       </Select>
                     </span>
                   </Radio>

src/module/workmgmt/component/JobStepOne.jsx

@@ -1,7 +1,7 @@
 import { Button, Form, Input, Select, Space, Divider, message } from 'antd'
 import { DeleteOutlined, PlusOutlined } from '@ant-design/icons'
 import React, { useState, useEffect, useRef } from 'react'
-import { getJobTypes, getDagFile, deleteTag } from '../services'
+import { getJobTypes, getFileUri, deleteTag } from '../services'
 const { Option } = Select
 const FormItem = Form.Item
 
@@ -45,9 +45,14 @@ export default function JobStepOne({ ConfigDataForm, SelectDag }) {
     const params = {
       project_id: 'test',
       user_id: 'test',
+      file_type: 'dag',
+    }
+    const { data } = await getFileUri(params)
+    if (data.code === 200) {
+      setDags(data.data)
+    } else {
+      message.error(data.msg)
     }
-    const data = await getDagFile(params)
-    setDags(data.data.data)
   }
 
   const onNameChange = event => {
@@ -169,7 +174,9 @@ export default function JobStepOne({ ConfigDataForm, SelectDag }) {
             allowClear
             onSelect={val => SelectDag(val)}>
             {dags.map(item => (
-              <Option key={item}>{item}</Option>
+              <Option key={item.uri} value={item.uri}>
+                {item.name}
+              </Option>
             ))}
           </Select>
         </FormItem>

src/module/workmgmt/component/JobStepThree.jsx

@@ -1,7 +1,7 @@
-import { Alert, Button, Form, Select, Row, Tooltip } from 'antd'
+import { Alert, Button, Form, Select, Row, Tooltip, message } from 'antd'
 import styled from 'styled-components'
 import {
-  getDagInfomation,
+  getFileInfomation,
   getDataSourceInfo,
   getTableNameInfo,
   getTableSchemaInfo,
@@ -143,12 +143,14 @@ export default function JobStepTwo({ OutputDataForm, dagData, checkSync }) {
   const [nodeState, setNodeState] = useState({})
 
   const getDagInfo = async dag_url => {
-    const data = await getDagInfomation(dag_url)
-    if (data.status === 200) {
+    const { data } = await getFileInfomation(dag_url)
+    if (data.code === 200) {
       const dag_nodes = data.data.nodes.filter(
         item => item.op === 'outputsource'
       )
       setNodes(dag_nodes)
+    } else {
+      message.error(data.msg)
     }
   }
 
@@ -181,7 +183,7 @@ export default function JobStepTwo({ OutputDataForm, dagData, checkSync }) {
           const strs = item.split(':')
           return { dataField: strs[1], dataType: strs[2] }
         })
-        handleSyncData(schemas, nodes[index]?.data.input_source, index)
+        handleSyncData(schemas, nodes[index]?.data.output_source, index)
       }
     }
   }

src/module/workmgmt/component/JobStepTwo.jsx

@@ -1,7 +1,7 @@
-import { Alert, Button, Form, Select, Row, Tooltip } from 'antd'
+import { Alert, Button, Form, Select, Row, Tooltip, message } from 'antd'
 import styled from 'styled-components'
 import {
-  getDagInfomation,
+  getFileInfomation,
   getDataSourceInfo,
   getTableNameInfo,
   getTableSchemaInfo,
@@ -143,10 +143,12 @@ export default function JobStepTwo({ InputDataForm, dagData, checkSync }) {
   const [nodeState, setNodeState] = useState({})
 
   const getDagInfo = async dag_url => {
-    const data = await getDagInfomation(dag_url)
-    if (data.status === 200) {
+    const { data } = await getFileInfomation(dag_url)
+    if (data.code === 200) {
       const dag_nodes = data.data.nodes.filter(item => item.op === 'datasource')
       setNodes(dag_nodes)
+    } else {
+      message.error(data.msg)
     }
   }
 

src/module/workmgmt/services/index.js

@@ -29,17 +29,17 @@ export const deleteJob = params =>
     method: 'delete',
   })
 
-//获取DAG算子
-export const getDagFile = params =>
+//获取文件uri
+export const getFileUri = params =>
   request({
-    url: `/jpt/dag/?project_id=${params.project_id}&user_id=${params.user_id}`,
+    url: `/jpt/files/directory?project_id=${params.project_id}&user_id=${params.user_id}&file_type=${params.file_type}`,
     method: 'get',
   })
 
-//获取DAG详细信息
-export const getDagInfomation = params =>
+//获取文件详细信息
+export const getFileInfomation = params =>
   request({
-    url: `/jpt/dag/info?uri=${params}`,
+    url: `/jpt/files/dag_content?uri=${params}`,
     method: 'get',
   })
 
@@ -84,16 +84,9 @@ export const updateJobInfo = (id, params) =>
     data: { ...params },
   })
 
-export const uploadJar = params =>
+export const uploadFile = params =>
   request({
-    url: '/jpt/files/java/',
-    method: 'post',
-    data: params,
-  })
-
-export const uploadPy = params =>
-  request({
-    url: '/jpt/files/python/',
+    url: '/jpt/files/upload_file',
     method: 'post',
     data: params,
   })