Update generic component operator to support python 3.7 (#2727)

Co-authored-by: Patrick Titzler <ptitzler@us.ibm.com>

Makefile

@@ -333,13 +333,17 @@ validate-runtime-images: # Validates delivered runtime-images meet minimum crite
 				IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm $$image $$cmd --version | cut -d' ' -f2 | cut -d'.' -f2` ; \
 				if [[ $$IMAGE_PYTHON3_MINOR_VERSION -lt 8 ]]; then \
 					echo WARNING: Image $$image requires at Python 3.8 or greater for latest generic component dependency installation; \
-					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image python3 -m pip install -r /opt/elyra/requirements-elyra-py37.txt > /dev/null ; \
+					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra-py37.txt && \
+								   curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \
+								   python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \
 					if [ $$? -ne 0 ]; then \
 						echo ERROR: Image $$image did not meet python requirements criteria in requirements-elyra-py37.txt ; \
 						fail=1; \
 					fi; \
 				elif [[ $$IMAGE_PYTHON3_MINOR_VERSION -ge 8 ]]; then \
-					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image python3 -m pip install -r /opt/elyra/requirements-elyra.txt > /dev/null ; \
+					docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \
+								   curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \
+								   python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \
 					if [ $$? -ne 0 ]; then \
 						echo ERROR: Image $$image did not meet python requirements criteria in requirements-elyra.txt ; \
 						fail=1; \

elyra/airflow/bootstrapper.py

@@ -330,7 +330,8 @@ class OpUtil(object):
     def package_install(cls) -> None:
         OpUtil.log_operation_info("Installing packages")
         t0 = time.time()
-        elyra_packages = cls.package_list_to_dict("requirements-elyra.txt")
+        requirements_file = cls.determine_elyra_requirements()
+        elyra_packages = cls.package_list_to_dict(requirements_file)
         current_packages = cls.package_list_to_dict("requirements-current.txt")
         to_install_list = []
 
@@ -372,6 +373,19 @@ class OpUtil(object):
         duration = time.time() - t0
         OpUtil.log_operation_info("Packages installed", duration)
 
+    @classmethod
+    def determine_elyra_requirements(cls) -> Any:
+        if sys.version_info.major == 3:
+            if sys.version_info.minor == 7:
+                return "requirements-elyra-py37.txt"
+            elif sys.version_info.minor in [8, 9, 10]:
+                return "requirements-elyra.txt"
+        logger.error(
+            f"This version of Python '{sys.version_info.major}.{sys.version_info.minor}' "
+            f"is not supported for Elyra generic components"
+        )
+        return None
+
     @classmethod
     def package_list_to_dict(cls, filename: str) -> dict:
         package_dict = {}

elyra/airflow/operator.py

@@ -42,6 +42,12 @@ ELYRA_REQUIREMENTS_URL = os.getenv(
     f"elyra/{ELYRA_GITHUB_BRANCH}/etc/generic/requirements-elyra.txt",
 )
 
+ELYRA_REQUIREMENTS_URL_PY37 = os.getenv(
+    "ELYRA_REQUIREMENTS_URL_PY37",
+    f"https://raw.githubusercontent.com/{ELYRA_GITHUB_ORG}/"
+    f"elyra/{ELYRA_GITHUB_BRANCH}/etc/generic/requirements-elyra-py37.txt",
+)
+
 
 class BootscriptBuilder(object):
     def __init__(
@@ -93,6 +99,8 @@ class BootscriptBuilder(object):
             f"echo 'Downloading {ELYRA_BOOTSCRIPT_URL}' && "
             f"curl {common_curl_options} -L {ELYRA_BOOTSCRIPT_URL} --output bootstrapper.py && "
             f"echo 'Downloading {ELYRA_REQUIREMENTS_URL}' && "
+            f"echo 'Downloading {ELYRA_REQUIREMENTS_URL_PY37}' && "
+            f"curl {common_curl_options} -L {ELYRA_REQUIREMENTS_URL_PY37} --output requirements-elyra-py37.txt && "
             f"curl {common_curl_options} -L {ELYRA_REQUIREMENTS_URL} "
             f"--output requirements-elyra.txt && "
             "python3 -m pip install packaging && "

elyra/kfp/bootstrapper.py

@@ -504,7 +504,8 @@ class OpUtil(object):
     def package_install(cls, user_volume_path) -> None:
         OpUtil.log_operation_info("Installing packages")
         t0 = time.time()
-        elyra_packages = cls.package_list_to_dict("requirements-elyra.txt")
+        requirements_file = cls.determine_elyra_requirements()
+        elyra_packages = cls.package_list_to_dict(requirements_file)
         current_packages = cls.package_list_to_dict("requirements-current.txt")
         to_install_list = []
 
@@ -553,6 +554,19 @@ class OpUtil(object):
         duration = time.time() - t0
         OpUtil.log_operation_info("Packages installed", duration)
 
+    @classmethod
+    def determine_elyra_requirements(cls) -> Any:
+        if sys.version_info.major == 3:
+            if sys.version_info.minor == 7:
+                return "requirements-elyra-py37.txt"
+            elif sys.version_info.minor in [8, 9, 10]:
+                return "requirements-elyra.txt"
+        logger.error(
+            f"This version of Python '{sys.version_info.major}.{sys.version_info.minor}' "
+            f"is not supported for Elyra generic components"
+        )
+        return None
+
     @classmethod
     def package_list_to_dict(cls, filename: str) -> dict:
         package_dict = {}

elyra/kfp/operator.py

@@ -57,6 +57,11 @@ ELYRA_REQUIREMENTS_URL = os.getenv(
     f"https://raw.githubusercontent.com/{ELYRA_GITHUB_ORG}/"
     f"elyra/{ELYRA_GITHUB_BRANCH}/etc/generic/requirements-elyra.txt",
 )
+ELYRA_REQUIREMENTS_URL_PY37 = os.getenv(
+    "ELYRA_REQUIREMENTS_URL_PY37",
+    f"https://raw.githubusercontent.com/{ELYRA_GITHUB_ORG}/"
+    f"elyra/{ELYRA_GITHUB_BRANCH}/etc/generic/requirements-elyra-py37.txt",
+)
 
 
 class ExecuteFileOp(ContainerOp):
@@ -185,14 +190,15 @@ class ExecuteFileOp(ContainerOp):
                 f"curl {common_curl_options} -L {self.bootstrap_script_url} --output bootstrapper.py && "
                 f"echo 'Downloading {self.requirements_url}' && "
                 f"curl {common_curl_options} -L {self.requirements_url} --output requirements-elyra.txt && "
+                f"echo 'Downloading {ELYRA_REQUIREMENTS_URL_PY37}' && "
+                f"curl {common_curl_options} -L {ELYRA_REQUIREMENTS_URL_PY37} --output requirements-elyra-py37.txt && "
             )
 
             if self.emptydir_volume_size:
                 argument_list.append(
                     f"mkdir {self.container_python_dir_name} && cd {self.container_python_dir_name} && "
                     f"echo 'Downloading {self.python_pip_config_url}' && "
-                    f"curl {common_curl_options} -L {self.python_pip_config_url} "
-                    "--output pip.conf && cd .. &&"
+                    f"curl {common_curl_options} -L {self.python_pip_config_url} --output pip.conf && cd .. &&"
                 )
 
             argument_list.append(

etc/generic/requirements-elyra-py37.txt

@@ -1,12 +1,12 @@
 # This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks.
 ipykernel==6.13.0
 ipython==7.33.0
-jinja2==3.1.2
+jinja2==3.0.3
 jupyter-client==7.3.1
 jupyter-core==4.10.0
 minio==7.1.8
 nbclient==0.6.3
-nbconvert==5.6.1
+nbconvert==6.5.0
 nbformat==5.4.0
 papermill==2.3.4
 pyzmq==22.3.0

etc/generic/requirements-elyra.txt

@@ -2,7 +2,7 @@
 ipykernel==6.13.0
 ipython==8.3.0
 ipython-genutils==0.2.0
-jinja2==3.1.2
+jinja2==3.0.3
 jupyter-client==7.3.1
 jupyter-core==4.10.0
 MarkupSafe==2.1.1