test_bootstrapper.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847
  1. #
  2. # Copyright 2018-2022 Elyra Authors
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. #
  16. import hashlib
  17. import json
  18. import logging
  19. import os
  20. from pathlib import Path
  21. import subprocess
  22. from subprocess import CalledProcessError
  23. from subprocess import CompletedProcess
  24. from subprocess import run
  25. import sys
  26. from tempfile import TemporaryFile
  27. import time
  28. from typing import Optional
  29. import minio
  30. import mock
  31. import nbformat
  32. import papermill
  33. import pytest
  34. from elyra.kfp import bootstrapper
  35. # To run this test from an IDE:
  36. # 1. set PYTHONPATH='`path-to-repo`/etc/docker-scripts' and working directory to `path-to-repo`
  37. # 2. Manually launch test_minio container: docker run --name test_minio -d -p 9000:9000 minio/minio server /data
  38. # (this is located in Makefile)
  39. #
  40. # NOTE: Any changes to elyra/tests/kfp/resources/test-notebookA.ipynb require an
  41. # update of elyra/tests/kfp/resources/test-archive.tgz using the command below:
  42. # tar -cvzf test-archive.tgz test-notebookA.ipynb
  43. MINIO_HOST_PORT = os.getenv("MINIO_HOST_PORT", "127.0.0.1:9000")
  44. @pytest.fixture(scope="module", autouse=True)
  45. def start_minio():
  46. """Start the minio container to simulate COS."""
  47. # The docker run command will fail if an instance of the test_minio container is running.
  48. # We'll make a "silent" attempt to start. If that fails, assume its due to the container
  49. # conflict, force its shutdown, and try once more. If successful, yield the minio instance
  50. # but also shutdown on the flip-side of the yield (when the fixture is cleaned up).
  51. #
  52. # Although actions like SIGINT (ctrl-C) should still trigger the post-yield logic, urgent
  53. # interrupts like SIGQUIT (ctrl-\) or multiple SIGINTs can still orphan the container, so
  54. # we still need the pre-yield behavior.
  55. minio = start_minio_container(False)
  56. if minio is None: # Got a failure. Shutdown (assumed) container and try once more.
  57. stop_minio_container()
  58. minio = start_minio_container(True)
  59. time.sleep(3) # give container a chance to start
  60. yield minio
  61. stop_minio_container()
  62. def start_minio_container(raise_on_failure: bool = False) -> Optional[CompletedProcess]:
  63. minio = None
  64. try:
  65. minio = run(
  66. ["docker", "run", "--name", "test_minio", "-d", "-p", "9000:9000", "minio/minio", "server", "/data"],
  67. cwd=os.getcwd(),
  68. check=True,
  69. )
  70. except CalledProcessError as ex:
  71. if raise_on_failure:
  72. raise RuntimeError(f"Error executing docker process: {ex}") from ex
  73. return minio
  74. def stop_minio_container():
  75. run(["docker", "rm", "-f", "test_minio"], check=True)
  76. @pytest.fixture(scope="function")
  77. def s3_setup():
  78. bucket_name = "test-bucket"
  79. cos_client = minio.Minio(MINIO_HOST_PORT, access_key="minioadmin", secret_key="minioadmin", secure=False)
  80. cos_client.make_bucket(bucket_name)
  81. yield cos_client
  82. cleanup_files = cos_client.list_objects(bucket_name, recursive=True)
  83. for file in cleanup_files:
  84. cos_client.remove_object(bucket_name, file.object_name)
  85. cos_client.remove_bucket(bucket_name)
  86. def main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict):
  87. """Primary body for main method testing..."""
  88. monkeypatch.setattr(bootstrapper.OpUtil, "parse_arguments", lambda x: argument_dict)
  89. monkeypatch.setattr(bootstrapper.OpUtil, "package_install", mock.Mock(return_value=True))
  90. monkeypatch.setenv("AWS_ACCESS_KEY_ID", "minioadmin")
  91. monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "minioadmin")
  92. monkeypatch.setenv("TEST_ENV_VAR1", "test_env_var1")
  93. s3_setup.fput_object(
  94. bucket_name=argument_dict["cos-bucket"],
  95. object_name="test-directory/test-file.txt",
  96. file_path="elyra/tests/kfp/resources/test-requirements-elyra.txt",
  97. )
  98. s3_setup.fput_object(
  99. bucket_name=argument_dict["cos-bucket"],
  100. object_name="test-directory/test,file.txt",
  101. file_path="elyra/tests/kfp/resources/test-bad-requirements-elyra.txt",
  102. )
  103. s3_setup.fput_object(
  104. bucket_name=argument_dict["cos-bucket"],
  105. object_name="test-directory/test-archive.tgz",
  106. file_path="elyra/tests/kfp/resources/test-archive.tgz",
  107. )
  108. with tmpdir.as_cwd():
  109. bootstrapper.main()
  110. test_file_list = [
  111. "test-archive.tgz",
  112. "test-file.txt",
  113. "test,file.txt",
  114. "test-file/test-file-copy.txt",
  115. "test-file/test,file/test,file-copy.txt",
  116. "test-notebookA.ipynb",
  117. "test-notebookA-output.ipynb",
  118. "test-notebookA.html",
  119. ]
  120. # Ensure working directory has all the files.
  121. for file in test_file_list:
  122. assert os.path.isfile(file)
  123. # Ensure upload directory has all the files EXCEPT the output notebook
  124. # since it was it is uploaded as the input notebook (test-notebookA.ipynb)
  125. # (which is included in the archive at start).
  126. for file in test_file_list:
  127. if file != "test-notebookA-output.ipynb":
  128. assert s3_setup.stat_object(
  129. bucket_name=argument_dict["cos-bucket"], object_name="test-directory/" + file
  130. )
  131. if file == "test-notebookA.html":
  132. with open("test-notebookA.html") as html_file:
  133. assert "TEST_ENV_VAR1: test_env_var1" in html_file.read()
  134. def _get_operation_instance(monkeypatch, s3_setup):
  135. config = {
  136. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  137. "cos-user": "minioadmin",
  138. "cos-password": "minioadmin",
  139. "cos-bucket": "test-bucket",
  140. "filepath": "untitled.ipynb",
  141. }
  142. op = bootstrapper.FileOpBase.get_instance(**config)
  143. # use the same minio instance used by the test
  144. # to avoid access denied errors when two minio
  145. # instances exist
  146. monkeypatch.setattr(op, "cos_client", s3_setup)
  147. return op
  148. def test_main_method(monkeypatch, s3_setup, tmpdir):
  149. argument_dict = {
  150. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  151. "cos-bucket": "test-bucket",
  152. "cos-directory": "test-directory",
  153. "cos-dependencies-archive": "test-archive.tgz",
  154. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  155. "inputs": "test-file.txt;test,file.txt",
  156. "outputs": "test-file/test-file-copy.txt;test-file/test,file/test,file-copy.txt",
  157. "user-volume-path": None,
  158. }
  159. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  160. def test_main_method_with_wildcard_outputs(monkeypatch, s3_setup, tmpdir):
  161. argument_dict = {
  162. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  163. "cos-bucket": "test-bucket",
  164. "cos-directory": "test-directory",
  165. "cos-dependencies-archive": "test-archive.tgz",
  166. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  167. "inputs": "test-file.txt;test,file.txt",
  168. "outputs": "test-file/*",
  169. "user-volume-path": None,
  170. }
  171. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  172. def test_main_method_with_dir_outputs(monkeypatch, s3_setup, tmpdir):
  173. argument_dict = {
  174. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  175. "cos-bucket": "test-bucket",
  176. "cos-directory": "test-directory",
  177. "cos-dependencies-archive": "test-archive.tgz",
  178. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  179. "inputs": "test-file.txt;test,file.txt",
  180. "outputs": "test-file", # this is the directory that contains the outputs
  181. "user-volume-path": None,
  182. }
  183. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  184. def is_writable_dir(path):
  185. """Helper method determines whether 'path' is a writable directory"""
  186. try:
  187. with TemporaryFile(mode="w", dir=path) as t:
  188. t.write("1")
  189. return True
  190. except Exception:
  191. return False
  192. def remove_file(filename, fail_ok=True):
  193. """Removes filename. If fail_ok is False an assert is raised
  194. if removal failed for any reason, e.g. filenotfound
  195. """
  196. try:
  197. os.remove(filename)
  198. except OSError as ose:
  199. if fail_ok is False:
  200. raise AssertionError(f"Cannot remove {filename}: {str(ose)} {ose}")
  201. def test_process_metrics_method_not_writable_dir(monkeypatch, s3_setup, tmpdir):
  202. """Test for process_metrics_and_metadata
  203. Validates that the method can handle output directory that is not writable
  204. """
  205. # remove "default" output file if it already exists
  206. output_metadata_file = Path("/tmp") / "mlpipeline-ui-metadata.json"
  207. remove_file(output_metadata_file)
  208. try:
  209. monkeypatch.setenv("ELYRA_WRITABLE_CONTAINER_DIR", "/good/time/to/fail")
  210. argument_dict = {
  211. "cos-endpoint": f"http://{MINIO_HOST_PORT}",
  212. "cos-bucket": "test-bucket",
  213. "cos-directory": "test-directory",
  214. "cos-dependencies-archive": "test-archive.tgz",
  215. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  216. "inputs": "test-file.txt;test,file.txt",
  217. "outputs": "test-file/test-file-copy.txt;test-file/test,file/test,file-copy.txt",
  218. "user-volume-path": None,
  219. }
  220. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  221. except Exception as ex:
  222. print(f"Writable dir test failed: {str(ex)} {ex}")
  223. assert False
  224. assert output_metadata_file.exists() is False
  225. def test_process_metrics_method_no_metadata_file(monkeypatch, s3_setup, tmpdir):
  226. """Test for process_metrics_and_metadata
  227. Verifies that the method produces a valid KFP UI metadata file if
  228. the node's script | notebook did not generate this metadata file.
  229. """
  230. argument_dict = {
  231. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  232. "cos-bucket": "test-bucket",
  233. "cos-directory": "test-directory",
  234. "cos-dependencies-archive": "test-archive.tgz",
  235. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  236. "inputs": "test-file.txt;test,file.txt",
  237. "outputs": "test-file/test-file-copy.txt;test-file/test,file/test,file-copy.txt",
  238. "user-volume-path": None,
  239. }
  240. output_path = Path(tmpdir)
  241. # metadata file name and location
  242. metadata_file = output_path / "mlpipeline-ui-metadata.json"
  243. # remove file if it already exists
  244. remove_file(metadata_file)
  245. # override the default output directory to make this test platform
  246. # independent
  247. monkeypatch.setenv("ELYRA_WRITABLE_CONTAINER_DIR", str(tmpdir))
  248. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  249. # process_metrics should have generated a file named mlpipeline-ui-metadata.json
  250. # in tmpdir
  251. try:
  252. with open(metadata_file, "r") as f:
  253. metadata = json.load(f)
  254. assert metadata.get("outputs") is not None
  255. assert isinstance(metadata["outputs"], list)
  256. assert len(metadata["outputs"]) == 1
  257. assert metadata["outputs"][0]["storage"] == "inline"
  258. assert metadata["outputs"][0]["type"] == "markdown"
  259. assert (
  260. f"{argument_dict['cos-endpoint']}/{argument_dict['cos-bucket']}/{argument_dict['cos-directory']}"
  261. in metadata["outputs"][0]["source"]
  262. )
  263. assert argument_dict["cos-dependencies-archive"] in metadata["outputs"][0]["source"]
  264. except AssertionError:
  265. raise
  266. except Exception as ex:
  267. # Potential reasons for failures:
  268. # file not found, invalid JSON
  269. print(f'Validation of "{str(ex)}" failed: {ex}')
  270. assert False
  271. def test_process_metrics_method_valid_metadata_file(monkeypatch, s3_setup, tmpdir):
  272. """Test for process_metrics_and_metadata
  273. Verifies that the method produces a valid KFP UI metadata file if
  274. the node's script | notebook generated this metadata file.
  275. """
  276. argument_dict = {
  277. "cos-endpoint": "http://" + MINIO_HOST_PORT,
  278. "cos-bucket": "test-bucket",
  279. "cos-directory": "test-directory",
  280. "cos-dependencies-archive": "test-archive.tgz",
  281. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  282. "inputs": "test-file.txt;test,file.txt",
  283. "outputs": "test-file/test-file-copy.txt;test-file/test,file/test,file-copy.txt",
  284. "user-volume-path": None,
  285. }
  286. output_path = Path(tmpdir)
  287. # metadata file name and location
  288. input_metadata_file = "mlpipeline-ui-metadata.json"
  289. output_metadata_file = output_path / input_metadata_file
  290. # remove output_metadata_file if it already exists
  291. remove_file(output_metadata_file)
  292. #
  293. # Simulate some custom metadata that the script | notebook produced
  294. #
  295. custom_metadata = {
  296. "some_property": "some property value",
  297. "outputs": [{"source": "gs://project/bucket/file.md", "type": "markdown"}],
  298. }
  299. with tmpdir.as_cwd():
  300. with open(input_metadata_file, "w") as f:
  301. json.dump(custom_metadata, f)
  302. # override the default output directory to make this test platform
  303. # independent
  304. monkeypatch.setenv("ELYRA_WRITABLE_CONTAINER_DIR", str(tmpdir))
  305. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  306. # output_metadata_file should now exist
  307. try:
  308. with open(output_metadata_file, "r") as f:
  309. metadata = json.load(f)
  310. assert metadata.get("some_property") is not None
  311. assert metadata["some_property"] == custom_metadata["some_property"]
  312. assert metadata.get("outputs") is not None
  313. assert isinstance(metadata["outputs"], list)
  314. assert len(metadata["outputs"]) == 2
  315. for output in metadata["outputs"]:
  316. if output.get("storage") is not None:
  317. assert output["storage"] == "inline"
  318. assert output["type"] == "markdown"
  319. assert (
  320. f"{argument_dict['cos-endpoint']}/{argument_dict['cos-bucket']}/{argument_dict['cos-directory']}" # noqa
  321. in output["source"]
  322. )
  323. assert argument_dict["cos-dependencies-archive"] in output["source"]
  324. else:
  325. assert output["type"] == custom_metadata["outputs"][0]["type"]
  326. assert output["source"] == custom_metadata["outputs"][0]["source"]
  327. except AssertionError:
  328. raise
  329. except Exception as ex:
  330. # Potential reasons for failures:
  331. # file not found, invalid JSON
  332. print(f'Validation of "{str(ex)}" failed: {ex}')
  333. assert False
  334. def test_process_metrics_method_invalid_metadata_file(monkeypatch, s3_setup, tmpdir):
  335. """Test for process_metrics_and_metadata
  336. Verifies that the method produces a valid KFP UI metadata file if
  337. the node's script | notebook generated an invalid metadata file.
  338. """
  339. argument_dict = {
  340. "cos-endpoint": f"http://{MINIO_HOST_PORT}",
  341. "cos-bucket": "test-bucket",
  342. "cos-directory": "test-directory",
  343. "cos-dependencies-archive": "test-archive.tgz",
  344. "filepath": "elyra/tests/kfp/resources/test-notebookA.ipynb",
  345. "inputs": "test-file.txt;test,file.txt",
  346. "outputs": "test-file/test-file-copy.txt;test-file/test,file/test,file-copy.txt",
  347. "user-volume-path": None,
  348. }
  349. output_path = Path(tmpdir)
  350. # metadata file name and location
  351. input_metadata_file = "mlpipeline-ui-metadata.json"
  352. output_metadata_file = output_path / input_metadata_file
  353. # remove output_metadata_file if it already exists
  354. remove_file(output_metadata_file)
  355. #
  356. # Populate the metadata file with some custom data that's not JSON
  357. #
  358. with tmpdir.as_cwd():
  359. with open(input_metadata_file, "w") as f:
  360. f.write("I am not a valid JSON data structure")
  361. f.write("1,2,3,4,5,6,7")
  362. # override the default output directory to make this test platform
  363. # independent
  364. monkeypatch.setenv("ELYRA_WRITABLE_CONTAINER_DIR", str(tmpdir))
  365. main_method_setup_execution(monkeypatch, s3_setup, tmpdir, argument_dict)
  366. # process_metrics replaces the existing metadata file
  367. # because its content cannot be merged
  368. try:
  369. with open(output_metadata_file, "r") as f:
  370. metadata = json.load(f)
  371. assert metadata.get("outputs") is not None
  372. assert isinstance(metadata["outputs"], list)
  373. assert len(metadata["outputs"]) == 1
  374. assert metadata["outputs"][0]["storage"] == "inline"
  375. assert metadata["outputs"][0]["type"] == "markdown"
  376. assert (
  377. f"{argument_dict['cos-endpoint']}/{argument_dict['cos-bucket']}/{argument_dict['cos-directory']}"
  378. in metadata["outputs"][0]["source"]
  379. )
  380. assert argument_dict["cos-dependencies-archive"] in metadata["outputs"][0]["source"]
  381. except AssertionError:
  382. raise
  383. except Exception as ex:
  384. # Potential reasons for failures:
  385. # file not found, invalid JSON
  386. print(f'Validation of "{str(ex)}" failed: {ex}')
  387. assert False
  388. def test_fail_bad_notebook_main_method(monkeypatch, s3_setup, tmpdir):
  389. argument_dict = {
  390. "cos-endpoint": f"http://{MINIO_HOST_PORT}",
  391. "cos-bucket": "test-bucket",
  392. "cos-directory": "test-directory",
  393. "cos-dependencies-archive": "test-bad-archiveB.tgz",
  394. "filepath": "elyra/tests/kfp/resources/test-bad-notebookB.ipynb",
  395. "inputs": "test-file.txt",
  396. "outputs": "test-file/test-copy-file.txt",
  397. "user-volume-path": None,
  398. }
  399. monkeypatch.setattr(bootstrapper.OpUtil, "parse_arguments", lambda x: argument_dict)
  400. monkeypatch.setattr(bootstrapper.OpUtil, "package_install", mock.Mock(return_value=True))
  401. mocked_func = mock.Mock(
  402. return_value="default",
  403. side_effect=[
  404. "test-bad-archiveB.tgz",
  405. "test-file.txt",
  406. "test-bad-notebookB-output.ipynb",
  407. "test-bad-notebookB.html",
  408. "test-file.txt",
  409. ],
  410. )
  411. monkeypatch.setattr(bootstrapper.FileOpBase, "get_object_storage_filename", mocked_func)
  412. monkeypatch.setenv("AWS_ACCESS_KEY_ID", "minioadmin")
  413. monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "minioadmin")
  414. s3_setup.fput_object(bucket_name=argument_dict["cos-bucket"], object_name="test-file.txt", file_path="README.md")
  415. s3_setup.fput_object(
  416. bucket_name=argument_dict["cos-bucket"],
  417. object_name="test-bad-archiveB.tgz",
  418. file_path="elyra/tests/kfp/resources/test-bad-archiveB.tgz",
  419. )
  420. with tmpdir.as_cwd():
  421. with pytest.raises(papermill.exceptions.PapermillExecutionError):
  422. bootstrapper.main()
  423. def test_package_installation(monkeypatch, virtualenv):
  424. elyra_dict = {
  425. "ipykernel": "5.3.0",
  426. "ansiwrap": "0.8.4",
  427. "packaging": "20.0",
  428. "text-extensions-for-pandas": "0.0.1-prealpha",
  429. }
  430. to_install_dict = {
  431. "bleach": "3.1.5",
  432. "ansiwrap": "0.7.0",
  433. "packaging": "20.4",
  434. "text-extensions-for-pandas": "0.0.1-prealpha",
  435. }
  436. correct_dict = {
  437. "ipykernel": "5.3.0",
  438. "ansiwrap": "0.8.4",
  439. "packaging": "20.4",
  440. "text-extensions-for-pandas": "0.0.1-prealpha",
  441. }
  442. mocked_func = mock.Mock(return_value="default", side_effect=[elyra_dict, to_install_dict])
  443. monkeypatch.setattr(bootstrapper.OpUtil, "package_list_to_dict", mocked_func)
  444. monkeypatch.setattr(sys, "executable", virtualenv.python)
  445. virtualenv.run("python3 -m pip install bleach==3.1.5")
  446. virtualenv.run("python3 -m pip install ansiwrap==0.7.0")
  447. virtualenv.run("python3 -m pip install packaging==20.4")
  448. virtualenv.run(
  449. "python3 -m pip install git+https://github.com/akchinSTC/"
  450. "text-extensions-for-pandas@3de5ce17ab0493dcdf88b51e8727f580c08d6997"
  451. )
  452. bootstrapper.OpUtil.package_install(user_volume_path=None)
  453. virtual_env_dict = {}
  454. output = virtualenv.run("python3 -m pip freeze", capture=True)
  455. print("This is the [pip freeze] output :\n" + output)
  456. for line in output.strip().split("\n"):
  457. if " @ " in line:
  458. package_name, package_version = line.strip("\n").split(sep=" @ ")
  459. elif "===" in line:
  460. package_name, package_version = line.strip("\n").split(sep="===")
  461. else:
  462. package_name, package_version = line.strip("\n").split(sep="==")
  463. virtual_env_dict[package_name] = package_version
  464. for package, version in correct_dict.items():
  465. assert virtual_env_dict[package] == version
  466. def test_package_installation_with_target_path(monkeypatch, virtualenv, tmpdir):
  467. # TODO : Need to add test for direct-source e.g. ' @ '
  468. elyra_dict = {
  469. "ipykernel": "5.3.0",
  470. "ansiwrap": "0.8.4",
  471. "packaging": "20.0",
  472. "text-extensions-for-pandas": "0.0.1-prealpha",
  473. }
  474. to_install_dict = {
  475. "bleach": "3.1.5",
  476. "ansiwrap": "0.7.0",
  477. "packaging": "21.0",
  478. "text-extensions-for-pandas": "0.0.1-prealpha",
  479. }
  480. correct_dict = {
  481. "ipykernel": "5.3.0",
  482. "ansiwrap": "0.8.4",
  483. "packaging": "21.0",
  484. "text-extensions-for-pandas": "0.0.1-prealpha",
  485. }
  486. mocked_func = mock.Mock(return_value="default", side_effect=[elyra_dict, to_install_dict])
  487. monkeypatch.setattr(bootstrapper.OpUtil, "package_list_to_dict", mocked_func)
  488. monkeypatch.setattr(sys, "executable", virtualenv.python)
  489. virtualenv.run("python3 -m pip install --upgrade pip")
  490. virtualenv.run(f"python3 -m pip install --target={tmpdir} bleach==3.1.5")
  491. virtualenv.run(f"python3 -m pip install --target={tmpdir} ansiwrap==0.7.0")
  492. virtualenv.run(f"python3 -m pip install --target={tmpdir} packaging==20.9")
  493. virtualenv.run(
  494. f"python3 -m pip install --target={tmpdir} git+https://github.com/akchinSTC/"
  495. "text-extensions-for-pandas@3de5ce17ab0493dcdf88b51e8727f580c08d6997"
  496. )
  497. bootstrapper.OpUtil.package_install(user_volume_path=str(tmpdir))
  498. virtual_env_dict = {}
  499. output = virtualenv.run(f"python3 -m pip freeze --path={tmpdir}", capture=True)
  500. print("This is the [pip freeze] output :\n" + output)
  501. for line in output.strip().split("\n"):
  502. if " @ " in line:
  503. package_name, package_version = line.strip("\n").split(sep=" @ ")
  504. elif "===" in line:
  505. package_name, package_version = line.strip("\n").split(sep="===")
  506. else:
  507. package_name, package_version = line.strip("\n").split(sep="==")
  508. virtual_env_dict[package_name] = package_version
  509. for package, version in correct_dict.items():
  510. assert virtual_env_dict[package].split(".")[0] == version.split(".")[0]
  511. def test_convert_notebook_to_html(tmpdir):
  512. notebook_file = os.getcwd() + "/elyra/tests/kfp/resources/test-notebookA.ipynb"
  513. notebook_output_html_file = "test-notebookA.html"
  514. with tmpdir.as_cwd():
  515. bootstrapper.NotebookFileOp.convert_notebook_to_html(notebook_file, notebook_output_html_file)
  516. assert os.path.isfile(notebook_output_html_file)
  517. # Validate that an html file got generated from the notebook
  518. with open(notebook_output_html_file, "r") as html_file:
  519. html_data = html_file.read()
  520. assert html_data.startswith("<!DOCTYPE html>")
  521. assert "TEST_ENV_VAR1" in html_data # from os.getenv("TEST_ENV_VAR1")
  522. assert html_data.endswith("</html>\n")
  523. def test_fail_convert_notebook_to_html(tmpdir):
  524. notebook_file = os.getcwd() + "/elyra/tests/kfp/resources/test-bad-notebookA.ipynb"
  525. notebook_output_html_file = "bad-notebookA.html"
  526. with tmpdir.as_cwd():
  527. # Recent versions raising typeError due to #1130
  528. # https://github.com/jupyter/nbconvert/pull/1130
  529. with pytest.raises((TypeError, nbformat.validator.NotebookValidationError)):
  530. bootstrapper.NotebookFileOp.convert_notebook_to_html(notebook_file, notebook_output_html_file)
  531. def test_get_file_object_store(monkeypatch, s3_setup, tmpdir):
  532. file_to_get = "README.md"
  533. current_directory = os.getcwd() + "/"
  534. bucket_name = "test-bucket"
  535. s3_setup.fput_object(bucket_name=bucket_name, object_name=file_to_get, file_path=file_to_get)
  536. with tmpdir.as_cwd():
  537. op = _get_operation_instance(monkeypatch, s3_setup)
  538. op.get_file_from_object_storage(file_to_get)
  539. assert os.path.isfile(file_to_get)
  540. assert _fileChecksum(file_to_get) == _fileChecksum(current_directory + file_to_get)
  541. def test_fail_get_file_object_store(monkeypatch, s3_setup, tmpdir):
  542. file_to_get = "test-file.txt"
  543. with tmpdir.as_cwd():
  544. with pytest.raises(minio.error.S3Error) as exc_info:
  545. op = _get_operation_instance(monkeypatch, s3_setup)
  546. op.get_file_from_object_storage(file_to_get=file_to_get)
  547. assert exc_info.value.code == "NoSuchKey"
  548. def test_put_file_object_store(monkeypatch, s3_setup, tmpdir):
  549. bucket_name = "test-bucket"
  550. file_to_put = "LICENSE"
  551. current_directory = os.getcwd() + "/"
  552. op = _get_operation_instance(monkeypatch, s3_setup)
  553. op.put_file_to_object_storage(file_to_upload=file_to_put)
  554. with tmpdir.as_cwd():
  555. s3_setup.fget_object(bucket_name, file_to_put, file_to_put)
  556. assert os.path.isfile(file_to_put)
  557. assert _fileChecksum(file_to_put) == _fileChecksum(current_directory + file_to_put)
  558. def test_fail_invalid_filename_put_file_object_store(monkeypatch, s3_setup):
  559. file_to_put = "LICENSE_NOT_HERE"
  560. with pytest.raises(FileNotFoundError):
  561. op = _get_operation_instance(monkeypatch, s3_setup)
  562. op.put_file_to_object_storage(file_to_upload=file_to_put)
  563. def test_fail_bucket_put_file_object_store(monkeypatch, s3_setup):
  564. bucket_name = "test-bucket-not-exist"
  565. file_to_put = "LICENSE"
  566. with pytest.raises(minio.error.S3Error) as exc_info:
  567. op = _get_operation_instance(monkeypatch, s3_setup)
  568. monkeypatch.setattr(op, "cos_bucket", bucket_name)
  569. op.put_file_to_object_storage(file_to_upload=file_to_put)
  570. assert exc_info.value.code == "NoSuchBucket"
  571. def test_find_best_kernel_nb(tmpdir):
  572. source_nb_file = os.path.join(os.getcwd(), "elyra/tests/kfp/resources/test-notebookA.ipynb")
  573. nb_file = os.path.join(tmpdir, "test-notebookA.ipynb")
  574. # "Copy" nb file to destination - this test does not update the kernel or language.
  575. nb = nbformat.read(source_nb_file, 4)
  576. nbformat.write(nb, nb_file)
  577. with tmpdir.as_cwd():
  578. kernel_name = bootstrapper.NotebookFileOp.find_best_kernel(nb_file)
  579. assert kernel_name == nb.metadata.kernelspec["name"]
  580. def test_find_best_kernel_lang(tmpdir, caplog):
  581. caplog.set_level(logging.INFO)
  582. source_nb_file = os.path.join(os.getcwd(), "elyra/tests/kfp/resources/test-notebookA.ipynb")
  583. nb_file = os.path.join(tmpdir, "test-notebookA.ipynb")
  584. # "Copy" nb file to destination after updating the kernel name - forcing a language match
  585. nb = nbformat.read(source_nb_file, 4)
  586. nb.metadata.kernelspec["name"] = "test-kernel"
  587. nb.metadata.kernelspec["language"] = "PYTHON" # test case-insensitivity
  588. nbformat.write(nb, nb_file)
  589. with tmpdir.as_cwd():
  590. kernel_name = bootstrapper.NotebookFileOp.find_best_kernel(nb_file)
  591. assert kernel_name == "python3"
  592. assert len(caplog.records) == 1
  593. assert caplog.records[0].message.startswith("Matched kernel by language (PYTHON)")
  594. def test_find_best_kernel_nomatch(tmpdir, caplog):
  595. source_nb_file = os.path.join(os.getcwd(), "elyra/tests/kfp/resources/test-notebookA.ipynb")
  596. nb_file = os.path.join(tmpdir, "test-notebookA.ipynb")
  597. # "Copy" nb file to destination after updating the kernel name and language - forcing use of updated name
  598. nb = nbformat.read(source_nb_file, 4)
  599. nb.metadata.kernelspec["name"] = "test-kernel"
  600. nb.metadata.kernelspec["language"] = "test-language"
  601. nbformat.write(nb, nb_file)
  602. with tmpdir.as_cwd():
  603. kernel_name = bootstrapper.NotebookFileOp.find_best_kernel(nb_file)
  604. assert kernel_name == "test-kernel"
  605. assert len(caplog.records) == 1
  606. assert caplog.records[0].message.startswith("Reverting back to missing notebook kernel 'test-kernel'")
  607. def test_parse_arguments():
  608. test_args = [
  609. "-e",
  610. "http://test.me.now",
  611. "-d",
  612. "test-directory",
  613. "-t",
  614. "test-archive.tgz",
  615. "-f",
  616. "test-notebook.ipynb",
  617. "-b",
  618. "test-bucket",
  619. "-p",
  620. "/tmp/lib",
  621. "-n",
  622. "test-pipeline",
  623. ]
  624. args_dict = bootstrapper.OpUtil.parse_arguments(test_args)
  625. assert args_dict["cos-endpoint"] == "http://test.me.now"
  626. assert args_dict["cos-directory"] == "test-directory"
  627. assert args_dict["cos-dependencies-archive"] == "test-archive.tgz"
  628. assert args_dict["cos-bucket"] == "test-bucket"
  629. assert args_dict["filepath"] == "test-notebook.ipynb"
  630. assert args_dict["user-volume-path"] == "/tmp/lib"
  631. assert args_dict["pipeline-name"] == "test-pipeline"
  632. assert not args_dict["inputs"]
  633. assert not args_dict["outputs"]
  634. def test_fail_missing_notebook_parse_arguments():
  635. test_args = ["-e", "http://test.me.now", "-d", "test-directory", "-t", "test-archive.tgz", "-b", "test-bucket"]
  636. with pytest.raises(SystemExit):
  637. bootstrapper.OpUtil.parse_arguments(test_args)
  638. def test_fail_missing_endpoint_parse_arguments():
  639. test_args = ["-d", "test-directory", "-t", "test-archive.tgz", "-f", "test-notebook.ipynb", "-b", "test-bucket"]
  640. with pytest.raises(SystemExit):
  641. bootstrapper.OpUtil.parse_arguments(test_args)
  642. def test_fail_missing_archive_parse_arguments():
  643. test_args = ["-e", "http://test.me.now", "-d", "test-directory", "-f", "test-notebook.ipynb", "-b", "test-bucket"]
  644. with pytest.raises(SystemExit):
  645. bootstrapper.OpUtil.parse_arguments(test_args)
  646. def test_fail_missing_bucket_parse_arguments():
  647. test_args = [
  648. "-e",
  649. "http://test.me.now",
  650. "-d",
  651. "test-directory",
  652. "-t",
  653. "test-archive.tgz",
  654. "-f",
  655. "test-notebook.ipynb",
  656. ]
  657. with pytest.raises(SystemExit):
  658. bootstrapper.OpUtil.parse_arguments(test_args)
  659. def test_fail_missing_directory_parse_arguments():
  660. test_args = ["-e", "http://test.me.now", "-t", "test-archive.tgz", "-f", "test-notebook.ipynb", "-b", "test-bucket"]
  661. with pytest.raises(SystemExit):
  662. bootstrapper.OpUtil.parse_arguments(test_args)
  663. def test_requirements_file(monkeypatch, tmpdir, caplog):
  664. elyra_requirements_file = Path(__file__).parent / "resources/test-requirements-elyra.txt"
  665. elyra_correct_number_of_packages = 19
  666. elyra_list_dict = bootstrapper.OpUtil.package_list_to_dict(elyra_requirements_file)
  667. assert len(elyra_list_dict) == elyra_correct_number_of_packages
  668. current_requirements_file = Path(__file__).parent / "resources/test-requirements-current.txt"
  669. current_correct_number_of_packages = 15
  670. current_list_dict = bootstrapper.OpUtil.package_list_to_dict(current_requirements_file)
  671. assert len(current_list_dict) == current_correct_number_of_packages
  672. mocked_package_list_to_dict = mock.Mock(return_value="default", side_effect=[elyra_list_dict, current_list_dict])
  673. monkeypatch.setattr(bootstrapper.OpUtil, "package_list_to_dict", mocked_package_list_to_dict)
  674. mocked_subprocess_run = mock.Mock(return_value="default")
  675. monkeypatch.setattr(subprocess, "run", mocked_subprocess_run)
  676. bootstrapper.OpUtil.package_install(user_volume_path=str(tmpdir))
  677. assert "WARNING: Source package 'jupyter-client' found already installed as an editable package" in caplog.text
  678. assert "WARNING: Source package 'requests' found already installed as an editable package" in caplog.text
  679. assert "WARNING: Source package 'tornado' found already installed from git" in caplog.text
  680. def test_fail_requirements_file_bad_delimiter():
  681. bad_requirements_file = Path(__file__).parent / "resources/test-bad-requirements-elyra.txt"
  682. with open(bad_requirements_file, "r") as f:
  683. file_content = f.readlines()
  684. valid_package_list = [
  685. line.strip("\n").split("==")[0] for line in file_content if not line.startswith("#") and "==" in line
  686. ]
  687. package_dict = bootstrapper.OpUtil.package_list_to_dict(bad_requirements_file)
  688. assert valid_package_list == list(package_dict.keys())
  689. def _fileChecksum(filename):
  690. hasher = hashlib.sha256()
  691. with open(filename, "rb") as afile:
  692. buf = afile.read(65536)
  693. while len(buf) > 0:
  694. hasher.update(buf)
  695. buf = afile.read(65536)
  696. checksum = hasher.hexdigest()
  697. return checksum