first commit

Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu
+RUN sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list \
+    && sed -i 's/security.debian.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list
+RUN set -xe && apt-get update
+
+RUN apt-get -y install python3-pip zsh
+# 安装miniconda
+ENV PYTHON_VERSION 3
+RUN chsh -s `which zsh`
+RUN apt install -y curl
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh  && \
+    chmod +x ~/miniconda.sh && \
+    ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh
+RUN ln /opt/conda/bin/conda /usr/local/bin/conda
+RUN conda init zsh
+RUN conda install mamba -n base -c conda-forge
+RUN ln /opt/conda/bin/mamba /usr/local/bin/mamba && mamba init zsh
+
+ADD environment.yml /environment.yml
+ADD main.py /main.py
+ADD utils.py /utils.py
+ADD datax.py /datax.py
+ADD user.keytab /user.keytab
+ADD krb5.conf /krb5.conf
+RUN cp krb5.conf /etc
+RUN apt-get install -y iputils-ping
+ RUN conda env create -f /environment.yml && rm -rf /root/.cache
+# RUN python3 -m venv venv
+RUN apt-get install -y vim

README.md

@@ -0,0 +1,94 @@
+## Get Started with test ailab
+
+### 简介
+
+众所周知，公司自主开发的AI平台目前还存在诸多bug，为了减轻晏姐的工作量，立哥让我开发一个自动化测试脚本，可以自动测试AI平台是否可以正常运行。于是我开发了此项目，可以检测线上或是线下的AI平台是否可以跑通案例，主要是通过requests库进行检测。
+
+### 快速开始
+
+#### 项目结构
+
+```shell
+├── datax.py
+├── Dockerfile
+├── Dockerfile_krb5
+├── environment.yml
+├── krb5.conf
+├── main.py
+├── test.json
+├── user.keytab
+└── utils.py
+```
+
+#### 构建镜像
+
+`docker build -t test_agent:v1 .`
+
+#### 启动容器
+
+`docker run -it test_agent:v1 zsh`
+
+#### 开启VPN
+
+重新启动一个终端，`ssh sxwl1070@192.168.199.107`，密码为`sx`
+
+输入指令`sudo openvpn --config ./openvpn-client.ovpn`
+
+密码为`sx`
+
+然后输入用户名`liangzhongquan`
+
+最后输入密码`Hello123!`，即可启动VPN
+
+#### 运行容器中的代码
+
+然后返回刚才的镜像终端中，
+
+运行`python datax.py`，即可进入选择阶段:
+
+`请输入平台：`
+
+输入`online`即为线上环境，程序会自动加载线上AI平台的各个URL，从而应对后续的各个测试流程
+
+输入`offline`即为线下环境，程序会自动加载线上AI平台的各个URL，从而应对后续的各个测试流程
+
+### 函数介绍
+
+```python
+class Agent(object):
+    def __init__(self)
+    def test_test(self, name)
+    def test_add_datasource(self, name)
+    def test_add_job_sql2sql(self, name)
+    def test_add_job_sql2hive(self, name)
+    def test_add_job_hive2sql(self, name)
+    def test_add_job_hive2hive(self, name)
+    def test(self, json_data)
+
+
+if __name__ == "__main__":
+    agent = Agent()
+    agent.test_add_job_sql2sql("liyuqi_test")
+```
+
+`___init__`：请求输入，从而判断将在哪个平台上运行，输入目前仅支持`online`和`offline`，即线上和线下，通过用户的输入进行配置文件的加载
+
+`test_test`：测试AI平台的添加数据源的`测试连接`，判断当前数据源是否可以添加
+
+`test_add_datasource`：测试AI平台的添加数据源的`确认添加`，测试AI平台的数据源添加功能是否异常
+
+`test`：集成了一些测试流程，先后`添加任务`，`启动任务`，`执行任务`
+
+`test_add_job_sql2sql`：测试AI平台的添加任务，启动任务，执行任务功能，数据迁移模式为从mysql数据源到mysql数据源
+
+`test_add_job_sql2hive`：测试AI平台的添加任务，启动任务，执行任务功能，数据迁移模式为从mysql数据源到hive数据源
+
+`test_add_job_hive2sql`：测试AI平台的添加任务，启动任务，执行任务功能，数据迁移模式为从hive数据源到mysql数据源
+
+`test_add_hive2hive`：测试AI平台的添加任务，启动任务，执行任务功能，数据迁移模式为从hive数据源到hive数据源
+
+用户可以根据自身需要在`__main__`方法中添加相应的指令，从而对AI平台进行测试。
+
+### 注意点
+
+最后的最后，记得在启动脚本之前对任务改个名字，否则会报错名称重复

datax.py

@@ -0,0 +1,193 @@
+import requests as R
+import json
+from utils import ConfigOnline, ConfigOffline
+
+
+class Agent(object):
+    def __init__(self):
+        self.type = input("请输入平台：")
+        if self.type == "offline":
+            self.add_job_url = ConfigOffline.add_job_url
+            self.update_url = ConfigOffline.update_url
+            self.execute_url = ConfigOffline.execute_url
+            self.test_url = ConfigOffline.test_url
+            self.add_datasource_url = ConfigOffline.add_datasource_url
+            print("加载url成功")
+        elif self.type == "online":
+            self.add_job_url = ConfigOnline.add_job_url
+            self.update_url = ConfigOnline.update_url
+            self.execute_url = ConfigOnline.execute_url
+            self.test_url = ConfigOnline.test_url
+            self.add_datasource_url = ConfigOnline.add_datasource_url
+            print("加载url成功")
+        self.job_id = 0
+
+    def test_test(self, name):
+        data = {}
+        if self.type == "online":
+            data = {
+                "datasource_name": name,
+                "datasource": "hive",
+                "database_name": "ailab",
+                "jdbc_url": "10.254.20.22:7001",
+                "comments": "",
+                "tag": "线上",
+                "use_ssl": 0,
+                "kerberos": 1,
+                "keytab": "/Users/btobab/Downloads/user.keytab",
+                "krb5config": "/Users/btobab/Downloads/krb5.conf",
+                "kerberos_service_name": "hadoop",
+                "principal": "ailab@EMR-5XJSY31F"
+            }
+        elif self.type == "offline":
+            data = {
+                "datasource_name": name,
+                "datasource": ConfigOffline.datasource,
+                "database_name": ConfigOffline.database_name,
+                "jdbc_username": ConfigOffline.jdbc_username,
+                "jdbc_password": ConfigOffline.jdbc_password,
+                "jdbc_url": ConfigOffline.jdbc_url,
+                "comments": "",
+                "tag": "测试",
+                "use_ssl": ConfigOffline.use_ssl,
+                "kerberos": ConfigOffline.kerberos
+            }
+        r = R.post(url=self.test_url, json=data)
+        js = json.loads(r.text)
+        if js["data"]["code"] == 200:
+            print("测试连接成功")
+        else:
+            print("测试连接失败")
+        print(r.text)
+
+    def test_add_datasource(self, name):
+        data = {}
+        if self.type == "online":
+            data = {
+                "datasource_name": name,
+                "datasource": ConfigOnline.datasource,
+                "database_name": ConfigOnline.database_name,
+                "jdbc_url": ConfigOnline.jdbc_url,
+                "comments": "",
+                "tag": "线上",
+                "use_ssl": ConfigOnline.use_ssl,
+                "kerberos": ConfigOnline.kerberos,
+                "keytab": ConfigOnline.keytab,
+                "krb5config": ConfigOnline.krb5config,
+                "kerberos_service_name": ConfigOnline.kerberos_service_name,
+                "principal": ConfigOnline.principal
+            }
+        elif self.type == "offline":
+            data = {
+                "datasource_name": name,
+                "datasource": ConfigOffline.datasource,
+                "database_name": ConfigOffline.database_name,
+                "jdbc_username": ConfigOffline.jdbc_username,
+                "jdbc_password": ConfigOffline.jdbc_password,
+                "jdbc_url": ConfigOffline.jdbc_url,
+                "comments": "",
+                "tag": "测试",
+                "use_ssl": ConfigOffline.use_ssl,
+                "kerberos": ConfigOffline.kerberos
+            }
+        r = R.post(url=self.add_datasource_url, json=data)
+        print(r.text)
+
+    def test_add_job_sql2sql(self, name):
+        data = {
+            "cron_expression":
+                {
+                    "cron_select_type": 1, "hour": 17, "minute": 1
+                },
+            "executor_block_strategy": "SERIAL_EXECUTION",
+            "executor_fail_retry_count": 0,
+            "executor_route_strategy": "FIRST",
+            "executor_timeout": 0,
+            "inc_start_time": None,
+            "job_desc": name,
+            "job_json":
+                "{\"job\":{\"content\":[{\"reader\":{\"name\":\"mysqlreader\",\"parameter\":{\"column\":[\"id\",\"txn_amount\",\"txn_type\",\"txn_date\"],\"where\":\"\",\"splitPk\":\"\",\"connection\":[{\"jdbcUrl\":[\"jdbc:mysql://192.168.199.107:10086/test-db?useSSL=false\"],\"table\":[\"test_liyuqi\"]}],\"username\":\"root\",\"password\":\"happylay\"}},\"writer\":{\"name\":\"mysqlwriter\",\"parameter\":{\"connection\":[{\"jdbcUrl\":\"jdbc:mysql://192.168.199.107:10086/test-db?useSSL=false\",\"table\":[\"test_liyuqi\"]}],\"column\":[\"id\",\"txn_amount\",\"txn_type\",\"txn_date\"],\"username\":\"root\",\"password\":\"happylay\",\"preSql\":[\"\"],\"postSql\":[\"\"]}}}],\"setting\":{\"speed\":{\"channel\":\"1\"}}}}",
+            "partition_info": "",
+            "partition_num": 0,
+            "user_id": "test"
+        }
+        self.test(data)
+
+    def test_add_job_sql2hive(self, name):
+        data = {
+            "partition_info": "",
+            "executor_timeout": 0,
+            "executor_fail_retry_count": 0,
+            "inc_start_time": None,
+            "job_desc": name,
+            "executor_route_strategy": "FIRST",
+            "executor_block_strategy": "SERIAL_EXECUTION",
+            "cron_expression": {
+                "cron_select_type": 0,
+                "hour": 1
+            },
+            "job_json": "{\"job\":{\"content\":[{\"reader\":{\"name\":\"mysqlreader\",\"parameter\":{\"column\":[\"id\",\"name\",\"ct\"],\"where\":\"\",\"splitPk\":\"\",\"connection\":[{\"jdbcUrl\":[\"jdbc:mysql://58.87.83.174:3306/test\"],\"table\":[\"test_1\"]}],\"username\":\"root\",\"password\":\"root\"}},\"writer\":{\"name\":\"hdfswriter\",\"parameter\":{\"defaultFS\":\"hdfs://HDFS8000912\",\"hadoopConfig\":{\"dfs.nameservices\":\"HDFS8000912\",\"dfs.ha.namenodes.HDFS8000912\":\"nn1,nn2\",\"dfs.namenode.rpc-address.HDFS8000912.nn1\":\"10.254.20.18:4007\",\"dfs.namenode.rpc-address.HDFS8000912.nn2\":\"10.254.20.22:4007\",\"dfs.client.failover.proxy.provider.HDFS8000912\":\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"},\"haveKerberos\":\"true\",\"kerberosKeytabFilePath\":\"/workspace/confs/test/user.keytab\",\"kerberosPrincipal\":\"ailab@EMR-5XJSY31F\",\"fileType\":\"text\",\"path\":\"/usr/hive/warehouse/ailab.db/my_test_2\",\"fileName\":\"test\",\"writeMode\":\"append\",\"fieldDelimiter\":\",\",\"column\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"ct\",\"type\":\"string\"}]}}}],\"setting\":{\"speed\":{\"channel\":\"5\"}}}}",
+            "user_id": "test",
+            "partition_num": 0
+        }
+        self.test(data)
+
+    def test_add_job_hive2sql(self, name):
+        data = {
+            "partition_info": "",
+            "executor_timeout": 0,
+            "executor_fail_retry_count": 0,
+            "inc_start_time": None,
+            "job_desc": name,
+            "executor_route_strategy": "FIRST",
+            "executor_block_strategy": "SERIAL_EXECUTION",
+            "cron_expression": {
+                "cron_select_type": 0,
+                "hour": 1
+            },
+            "job_json": "{\"job\":{\"content\":[{\"reader\":{\"name\":\"hdfsreader\",\"parameter\":{\"defaultFS\":\"hdfs://HDFS8000912\",\"hadoopConfig\":{\"dfs.nameservices\":\"HDFS8000912\",\"dfs.ha.namenodes.HDFS8000912\":\"nn1,nn2\",\"dfs.namenode.rpc-address.HDFS8000912.nn1\":\"10.254.20.18:4007\",\"dfs.namenode.rpc-address.HDFS8000912.nn2\":\"10.254.20.22:4007\",\"dfs.client.failover.proxy.provider.HDFS8000912\":\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"},\"haveKerberos\":\"true\",\"kerberosKeytabFilePath\":\"/workspace/confs/test/user.keytab\",\"kerberosPrincipal\":\"ailab@EMR-5XJSY31F\",\"path\":\"/usr/hive/warehouse/ailab.db/my_test_2\",\"fileType\":\"text\",\"fieldDelimiter\":\",\",\"column\":[{\"index\":\"0\",\"type\":\"long\"},{\"index\":\"1\",\"type\":\"string\"},{\"index\":\"2\",\"type\":\"string\"}]}},\"writer\":{\"name\":\"mysqlwriter\",\"parameter\":{\"connection\":[{\"jdbcUrl\":\"jdbc:mysql://58.87.83.174:3306/test\",\"table\":[\"test_1\"]}],\"column\":[\"id\",\"name\",\"ct\"],\"username\":\"root\",\"password\":\"root\",\"preSql\":[\"\"],\"postSql\":[\"\"]}}}],\"setting\":{\"speed\":{\"channel\":\"5\"}}}}",
+            "user_id": "test",
+            "partition_num": 0
+        }
+        self.test(data)
+
+    def test_add_job_hive2hive(self, name):
+        data = {
+            "partition_info": "",
+            "executor_timeout": 0,
+            "executor_fail_retry_count": 0,
+            "inc_start_time": None,
+            "job_desc": name,
+            "executor_route_strategy": "FIRST",
+            "executor_block_strategy": "SERIAL_EXECUTION",
+            "cron_expression": {
+                "cron_select_type": 0,
+                "hour": 1
+            },
+            "job_json": "{\"job\":{\"content\":[{\"reader\":{\"name\":\"hdfsreader\",\"parameter\":{\"defaultFS\":\"hdfs://HDFS8000912\",\"hadoopConfig\":{\"dfs.nameservices\":\"HDFS8000912\",\"dfs.ha.namenodes.HDFS8000912\":\"nn1,nn2\",\"dfs.namenode.rpc-address.HDFS8000912.nn1\":\"10.254.20.18:4007\",\"dfs.namenode.rpc-address.HDFS8000912.nn2\":\"10.254.20.22:4007\",\"dfs.client.failover.proxy.provider.HDFS8000912\":\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"},\"haveKerberos\":\"true\",\"kerberosKeytabFilePath\":\"/workspace/confs/test/user.keytab\",\"kerberosPrincipal\":\"ailab@EMR-5XJSY31F\",\"path\":\"/usr/hive/warehouse/ailab.db/my_test_2\",\"fileType\":\"text\",\"fieldDelimiter\":\",\",\"column\":[{\"index\":\"0\",\"type\":\"long\"},{\"index\":\"1\",\"type\":\"string\"},{\"index\":\"2\",\"type\":\"string\"}]}},\"writer\":{\"name\":\"hdfswriter\",\"parameter\":{\"defaultFS\":\"hdfs://HDFS8000912\",\"hadoopConfig\":{\"dfs.nameservices\":\"HDFS8000912\",\"dfs.ha.namenodes.HDFS8000912\":\"nn1,nn2\",\"dfs.namenode.rpc-address.HDFS8000912.nn1\":\"10.254.20.18:4007\",\"dfs.namenode.rpc-address.HDFS8000912.nn2\":\"10.254.20.22:4007\",\"dfs.client.failover.proxy.provider.HDFS8000912\":\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"},\"haveKerberos\":\"true\",\"kerberosKeytabFilePath\":\"/workspace/confs/test/user.keytab\",\"kerberosPrincipal\":\"ailab@EMR-5XJSY31F\",\"fileType\":\"text\",\"path\":\"/usr/hive/warehouse/ailab.db/my_test_p\",\"fileName\":\"test\",\"writeMode\":\"append\",\"fieldDelimiter\":\",\",\"column\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"ct\",\"type\":\"string\"}]}}}],\"setting\":{\"speed\":{\"channel\":\"5\"}}}}",
+            "user_id": "test",
+            "partition_num": 0
+        }
+        self.test(data)
+
+    def test(self, json_data):
+        r = R.post(url=self.add_job_url, json=json_data)
+        print("输出添加任务日志")
+        js = json.loads(r.text)
+        print(js)
+        self.job_id = js["data"]["id"]
+        data = {
+            "id": self.job_id,
+            "trigger_status": 1
+        }
+        r = R.put(url=self.update_url, json=data)
+        print(r.text)
+        print("更改状态成功")
+        r = R.post(url="{}?job_id={}".format(self.execute_url, self.job_id))
+        print(r.text)
+        print("启动任务成功")
+
+
+if __name__ == "__main__":
+    agent = Agent()
+    agent.test_add_datasource("test_1003844955")

environment.yml

@@ -0,0 +1,13 @@
+name: py38
+channels:
+  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/ # Anocanda清华镜像
+  - defaults
+  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/
+dependencies:
+  - python=3.8
+  - ipython
+  - pip
+  - pip:
+      - requests
+      - -i https://mirror.baidu.com/pypi/simple
+prefix: /opt/conda/envs/py38

krb5.conf

@@ -0,0 +1,27 @@
+[libdefaults]
+    dns_lookup_realm = false
+    dns_lookup_kdc = false
+    ticket_lifetime = 24h
+    renew_lifetime = 7d
+    forwardable = true
+    rdns = false
+    default_realm = EMR-5XJSY31F
+    default_tgs_enctypes = des3-cbc-sha1
+    default_tkt_enctypes = des3-cbc-sha1
+    permitted_enctypes = des3-cbc-sha1
+    kdc_timeout = 3000
+    max_retries = 3
+[realms]
+    EMR-5XJSY31F = {
+
+        kdc = 10.254.20.18:88
+        admin_server = 10.254.20.18
+        kdc = 10.254.20.22:88
+		admin_server = 10.254.20.22
+
+    }
+
+[domain_realm]
+# .example.com = EXAMPLE.COM
+
+

utils.py

@@ -0,0 +1,32 @@
+class ConfigOnline(object):
+    test_url = "http://aihub-dag-test.digitalyili.com/jpt/datasource/test"
+    add_job_url = "http://aihub-dag-test.digitalyili.com/jpt/jobinfo/"
+    update_url = "http://aihub-dag-test.digitalyili.com/jpt/jobinfo/update_trigger_status/"
+    execute_url = "http://aihub-dag-test.digitalyili.com/jpt/jobinfo/execute"
+    add_datasource_url = "http://aihub-dag-test.digitalyili.com/jpt/datasource/"
+
+    datasource = "hive"
+    database_name = "ailab"
+    jdbc_url = "10.254.20.22:7001"
+    use_ssl = 0
+    kerberos = 1
+    keytab = "test/kerberos/1667353194_user.keytab"
+    krb5config = "test/kerberos/1667353194_krb5.conf"
+    kerberos_service_name = "hadoop"
+    principal = "ailab@EMR-5XJSY31F"
+
+
+class ConfigOffline(object):
+    test_url = "http://192.168.199.107:18082/jpt/datasource/test"
+    add_job_url = "http://192.168.199.107:18082/jpt/jobinfo/"
+    update_url = "http://192.168.199.107:18082/jpt/jobinfo/update_trigger_status/"
+    execute_url = "http://192.168.199.107:18082/jpt/jobinfo/execute"
+    add_datasource_url = "http://192.168.199.107:18082/jpt/datasource/"
+
+    datasource = "mysql"
+    jdbc_username = "root"
+    jdbc_password = "aGFwcHlsYXk="
+    database_name = "test-db"
+    jdbc_url = "192.168.199.107:10086"
+    use_ssl = 0
+    kerberos = 0