leo 2 лет назад
Родитель
Сommit
ffd502614e

+ 24 - 0
Dockerfile

@@ -0,0 +1,24 @@
+FROM node:14-buster as build
+WORKDIR /workspace
+COPY package.json .
+RUN npm install
+ADD . /workspace
+RUN npm run build
+
+
+FROM nginx:mainline
+WORKDIR /usr/share/nginx/html
+COPY --from=build /workspace/build /usr/share/nginx/html/dag
+RUN echo "\
+server {\n\
+    listen 80;\n\
+    listen [::]:80;\n\
+\n\
+    root /usr/share/nginx/html/dag;\n\
+    index index.html index.htm;\n\
+\n\
+    location / {\n\
+    try_files \$uri \$uri/ =404;\n\
+    }\n\
+}\n\
+" > /etc/nginx/conf.d/default.conf

+ 19 - 11
src/module/datasource/component/StepOne.jsx

@@ -70,7 +70,14 @@ export default function StepOne({ drawDataForm, dataSourceList, updateTableStruc
         }}
         wrapperCol={{
           span: 6,
-        }}>
+        }}
+        initialValues={{
+          query_sql: '',
+          where_param: '',
+          reader_split_pk: '',
+          delimiter: ''
+        }}
+      >
         <Form.Item
           label="选择数据源"
           name="datasource_name"
@@ -100,16 +107,16 @@ export default function StepOne({ drawDataForm, dataSourceList, updateTableStruc
           <>
             <Form.Item
               label="查询字段"
-              name="query_field"
+              name="query_sql"
               wrapperCol={{ span: 10 }}>
               <Input.TextArea rows={5} />
             </Form.Item>
-            <Form.Item label="切分主键" name="datasource_mainkey">
+            <Form.Item label="切分主键" name="reader_split_pk">
               <Input />
             </Form.Item>
             <Form.Item
               label="条件语句"
-              name="centence"
+              name="where_param"
               wrapperCol={{ span: 10 }}>
               <Input.TextArea rows={5} />
             </Form.Item>
@@ -118,7 +125,7 @@ export default function StepOne({ drawDataForm, dataSourceList, updateTableStruc
           <>
             <Form.Item
               label="选择路径"
-              name="dataRoute"
+              name="reader_path"
               wrapperCol={{ span: 10 }}
               rules={[
                 {
@@ -130,18 +137,18 @@ export default function StepOne({ drawDataForm, dataSourceList, updateTableStruc
             </Form.Item>
             <Form.Item
               label="HDFS"
-              name="datasource_hdfs"
+              name="reader_default_fs"
               rules={[
                 {
                   required: true,
                   message: '请输入HDFS',
                 },
               ]}>
-              <Input placeholder="hdfs namenode节点地址" />
+              <Input placeholder="Hadoop hdfs文件系统namenode节点地址" />
             </Form.Item>
             <Form.Item
               label="文件类型"
-              name="file_type"
+              name="reader_file_type"
               rules={[
                 {
                   required: true,
@@ -150,14 +157,15 @@ export default function StepOne({ drawDataForm, dataSourceList, updateTableStruc
               ]}>
               <Select
                 options={[
-                  {label: 'txt', value: 'txt'},
-                  {label: 'ocr', value: 'ocr'}
+                  { label: 'text', value: 'text' },
+                  { label: 'csv', value: 'csv' },
+                  { label: 'ocr', value: 'ocr' },
                 ]}
                 allowClear
                 placeholder="文件的类型"
               />
             </Form.Item>
-            <Form.Item label="分隔符" name="datasource_partition">
+            <Form.Item label="分隔符" name="reader_field_delimiter">
               <Input placeholder="读取字段的分隔符" />
             </Form.Item>
           </>

+ 16 - 10
src/module/datasource/component/StepTwo.jsx

@@ -38,7 +38,6 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
     loadDataForm.setFieldValue('datasource_table', null)
     const type = dataSourceList.find(item => item.key === val).datasource
     setDatasourceType(type)
-    console.log(val)
     const { data } = await getTableNamesList(val)
     if (data.code === 200) {
       const list = data.data.map(item => ({
@@ -75,7 +74,13 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
         }}
         wrapperCol={{
           span: 6,
-        }}>
+        }}
+        initialValues={{
+          pre_sql: '',
+          post_sql: '',
+          writer_field_delimiter: ''
+        }}
+      >
         <Form.Item
           label="选择数据源"
           name="datasource_name"
@@ -115,7 +120,7 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
           <>
             <Form.Item
               label="选择路径"
-              name="dataRoute"
+              name="writer_path"
               wrapperCol={{ span: 10 }}
               rules={[
                 {
@@ -123,22 +128,22 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
                   message: '请选择路径!',
                 },
               ]}>
-              <Input.TextArea rows={5} />
+              <Input.TextArea rows={5} placeholder="要读取的文件路径" />
             </Form.Item>
             <Form.Item
               label="HDFS"
-              name="datasource_hdfs"
+              name="writer_default_fs"
               rules={[
                 {
                   required: true,
                   message: '请输入HDFS',
                 },
               ]}>
-              <Input placeholder="hdfs namenode节点地址" />
+              <Input placeholder="Hadoop hdfs文件系统namenode节点地址" />
             </Form.Item>
             <Form.Item
               label="文件类型"
-              name="file_type"
+              name="writer_file_type"
               rules={[
                 {
                   required: true,
@@ -147,7 +152,8 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
               ]}>
               <Select
                 options={[
-                  { label: 'txt', value: 'txt' },
+                  { label: 'text', value: 'text' },
+                  { label: 'csv', value: 'csv' },
                   { label: 'ocr', value: 'ocr' },
                 ]}
                 allowClear
@@ -156,7 +162,7 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
             </Form.Item>
             <Form.Item
               label="写入模式"
-              name="write_mode"
+              name="writer_mode"
               rules={[
                 {
                   required: true,
@@ -172,7 +178,7 @@ export default function LoadFormConfig({ loadDataForm, dataSourceList, updateTab
                 placeholder="写入模式"
               />
             </Form.Item>
-            <Form.Item label="分隔符" name="datasource_partition">
+            <Form.Item label="分隔符" name="writer_field_delimiter">
               <Input placeholder="读取字段的分隔符" />
             </Form.Item>
           </>

+ 140 - 14
src/module/datasource/component/SyncTaskAdd.jsx

@@ -7,7 +7,7 @@ import React, { useState, useEffect } from 'react'
 import DataTableStruct from './DataTableStruct'
 import styled from 'styled-components'
 import { MenuFoldOutlined, MenuUnfoldOutlined } from '@ant-design/icons'
-import { getDataSourceList, getTableSchema } from '../services'
+import { getDataSourceList, getTableSchema, buildJsonData } from '../services'
 
 const { Step } = Steps
 
@@ -136,14 +136,30 @@ export default function SyncTaskAdd() {
     }
   }
 
+  // 挂载的时候获取SourceList
   useEffect(() => {
     fetchDataSourceList()
   }, [])
 
+  // reader_columns
+  const [reader_columns, setReaderColumns] = useState([])
+
+  // writer_columns
+  const [writer_columns, setWriterColumns] = useState([])
+
   // 下一步
   const nextStep = () => {
     if (currentStep === 2) {
-      console.log(stepThreeRef.current)
+      setReaderColumns(
+        stepThreeRef.current.syncMappings.map(item => {
+          return item.key.drawData
+        })
+      )
+      setWriterColumns(
+        stepThreeRef.current.syncMappings.map(item => {
+          return item.key.loadData
+        })
+      )
     }
     currentForm
       .validateFields()
@@ -155,21 +171,131 @@ export default function SyncTaskAdd() {
       })
   }
 
-  // 完成构建
-  const finishBuild = async () => {
-    setBuilding(true)
-    setTimeout(() => {
-      message.success('构建成功')
-      setBuilding(false)
-      setIsFinishBuild(true)
-    }, 2000)
+  // 格式化构建请求参数数据
+  const formatBuildData = () => {
+    // 获取表单数据
+    const reader_form = drawDataForm.getFieldValue()
+    const writer_form = loadDataForm.getFieldValue()
+    // 获取id tables
+    const reader_datasource_id = reader_form['datasource_name']
+    const reader_tables = [reader_form['datasource_table']]
+    const writer_datasource_id = writer_form['datasource_name']
+    const writer_tables = [writer_form['datasource_table']]
+    const writer_filename = dataSourceList.find(
+      item => item.key === writer_datasource_id
+    ).datasource_name
+    // 基础固定参数
+    const base_params = {
+      reader_datasource_id,
+      reader_tables,
+      reader_columns,
+      writer_datasource_id,
+      writer_tables,
+      writer_columns,
+    }
+    // 获取表类型
+    const reader_type = dataSourceList.find(
+      item => item.key === reader_datasource_id
+    ).datasource
+    const writer_type = dataSourceList.find(
+      item => item.key === writer_datasource_id
+    ).datasource
+    // 结构表单参数
+    const {
+      reader_split_pk,
+      where_param,
+      query_sql,
+      reader_default_fs,
+      reader_file_type,
+      reader_path,
+      reader_field_delimiter,
+    } = reader_form
+    const {
+      pre_sql,
+      post_sql,
+      writer_default_fs,
+      writer_file_type,
+      writer_path,
+      writer_mode,
+      writer_field_delimiter,
+    } = writer_form
+    // 根据类型区分参数
+    switch (`${reader_type}2${writer_type}`) {
+      case 'mysql2mysql':
+        return {
+          ...base_params,
+          rdbms_reader: { reader_split_pk, where_param, query_sql },
+          rdbms_writer: { pre_sql, post_sql },
+        }
+      case 'mysql2hive':
+        return {
+          ...base_params,
+          rdbms_reader: { reader_split_pk, where_param, query_sql },
+          hive_writer: {
+            writer_default_fs,
+            writer_file_type,
+            writer_path,
+            writer_mode,
+            writer_field_delimiter,
+            writer_filename,
+          },
+        }
+      case 'hive2mysql':
+        return {
+          ...base_params,
+          hive_reader: {
+            reader_default_fs,
+            reader_file_type,
+            reader_path,
+            reader_field_delimiter,
+          },
+          rdbms_writer: { pre_sql, post_sql },
+        }
+      case 'hive2hive':
+        return {
+          ...base_params,
+          hive_reader: {
+            reader_default_fs,
+            reader_file_type,
+            reader_path,
+            reader_field_delimiter,
+          },
+          hive_writer: {
+            writer_default_fs,
+            writer_file_type,
+            writer_path,
+            writer_mode,
+            writer_field_delimiter,
+            writer_filename,
+          },
+        }
+      default:
+        break
+    }
+    return null
   }
 
   // 构建表单
-  const build = () => {
-    console.log('loadDataForm.getFieldsValue()', loadDataForm.getFieldValue())
-    showDrawer()
-    // finishBuild()
+  const build = async () => {
+    const buildParams = formatBuildData()
+    if (!buildParams) {
+      message.error('获取表单参数失败')
+      return
+    }
+    setBuilding(true)
+    console.log(buildParams)
+    const { data } = await buildJsonData(buildParams)
+    console.log(data)
+    if (data.code === 200) {
+      const jsonData = data.json
+      console.log(jsonData)
+      message.success('构建成功')
+      showDrawer()
+    } else {
+      message.error('构建失败,请检查表单数据', data.msg)
+    }
+    setBuilding(false)
+    setIsFinishBuild(true)
   }
 
   // 完成提交

+ 6 - 0
src/module/datasource/services/index.js

@@ -62,4 +62,10 @@ export const getJoblog = () => request({
 export const getOnceJoblog = params => request({
   url: `/jpt/joblog/getOnce/` + params,
   method: 'get',
+})
+
+export const buildJsonData = params => request({
+  url: '/jpt/datax',
+  method: 'post',
+  data: { ...params }
 })