123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394 |
- #
- # Copyright 2018-2022 Elyra Authors
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #
- .PHONY: help purge uninstall-src uninstall clean
- .PHONY: lint-dependencies lint-server black-format prettier-check-ui eslint-check-ui prettier-ui eslint-ui lint-ui lint
- .PHONY: dev-link dev-unlink
- .PHONY: build-dependencies dev-dependencies yarn-install build-ui package-ui package-ui-dev
- .PHONY: build-server install-server-package install-server
- .PHONY: install install-all install-dev install-examples install-gitlab-dependency check-install watch release
- .PHONY: test-dependencies pytest test-server test-ui-unit test-integration test-integration-debug test-ui test
- .PHONY: docs-dependencies docs
- .PHONY: elyra-image elyra-image-env publish-elyra-image kf-notebook-image publish-kf-notebook-image
- .PHONY: container-images publish-container-images validate-runtime-image validate-runtime-images
- SHELL:=/bin/bash
- # Container execs
- CONTAINER_EXEC=docker
- # Python execs
- PYTHON?=python
- PYTHON_PIP=$(PYTHON) -m pip
- PYTHON_VERSION?=3.8
- CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate
- ELYRA_VERSION:=$$(grep __version__ elyra/_version.py | cut -d"\"" -f2)
- TAG:=dev
- IMAGE_IS_LATEST=False
- ELYRA_IMAGE=elyra/elyra:$(TAG)
- ELYRA_IMAGE_LATEST=elyra/elyra:latest
- ELYRA_IMAGE_ENV?=elyra-image-env
- KF_NOTEBOOK_IMAGE=elyra/kf-notebook:$(TAG)
- KF_NOTEBOOK_IMAGE_LATEST=elyra/kf-notebook:latest
- # Contains the set of commands required to be used by elyra
- REQUIRED_RUNTIME_IMAGE_COMMANDS?="curl python3"
- REMOVE_RUNTIME_IMAGE?=0 # Invoke `make REMOVE_RUNTIME_IMAGE=1 validate-runtime-images` to have images removed after validation
- UPGRADE_STRATEGY?=only-if-needed
- # Black CMD for code formatting
- BLACK_CMD:=$(PYTHON) -m black --check --diff --color .
- help:
- # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
- @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
- ## Clean targets
- purge:
- rm -rf build *.egg-info yarn-error.log
- rm -rf node_modules lib dist
- rm -rf $$(find packages -name node_modules -type d -maxdepth 2)
- rm -rf $$(find packages -name dist -type d)
- rm -rf $$(find packages -name lib -type d)
- rm -rf $$(find packages -name *.egg-info -type d)
- rm -rf $$(find . -name __pycache__ -type d)
- rm -rf $$(find . -name tsconfig.tsbuildinfo)
- rm -rf $$(find . -name package-lock.json)
- rm -rf $$(find . -name .pytest_cache)
- rm -rf $(yarn cache dir)
- uninstall-src: # Uninstalls source extensions if they're still installed
- - jupyter labextension unlink --no-build @elyra/services
- - jupyter labextension unlink --no-build @elyra/ui-components
- - jupyter labextension unlink --no-build @elyra/metadata-common
- - jupyter labextension unlink --no-build @elyra/script-editor
- - jupyter labextension uninstall --no-build @elyra/theme-extension
- - jupyter labextension uninstall --no-build @elyra/code-snippet-extension
- - jupyter labextension uninstall --no-build @elyra/metadata-extension
- - jupyter labextension uninstall --no-build @elyra/pipeline-editor-extension
- - jupyter labextension uninstall --no-build @elyra/python-editor-extension
- - jupyter labextension uninstall --no-build @elyra/r-editor-extension
- - jupyter labextension uninstall --no-build @elyra/scala-editor-extension
- - jupyter labextension uninstall --no-build @elyra/code-viewer-extension
- - jupyter labextension uninstall --no-build @elyra/script-debugger-extension
- - jupyter labextension unlink --no-build @elyra/pipeline-services
- - jupyter labextension unlink --no-build @elyra/pipeline-editor
- uninstall: uninstall-src
- $(PYTHON_PIP) uninstall -y jupyterlab-git
- $(PYTHON_PIP) uninstall -y nbdime
- $(PYTHON_PIP) uninstall -y jupyter-lsp
- - jupyter labextension uninstall @krassowski/jupyterlab-lsp
- $(PYTHON_PIP) uninstall -y jupyterlab-lsp
- $(PYTHON_PIP) uninstall -y python-lsp-server
- $(PYTHON_PIP) uninstall -y jupyter-resource-usage
- - jupyter labextension uninstall @jupyter-server/resource-usage
- $(PYTHON_PIP) uninstall -y elyra
- - jupyter lab clean
- # remove Kubeflow Pipelines example components
- - $(PYTHON_PIP) uninstall -y elyra-examples-kfp-catalog
- # remove GitLab dependency
- - $(PYTHON_PIP) uninstall -y python-gitlab
- clean: purge uninstall ## Make a clean source tree and uninstall extensions
- ## Lint targets
- lint-dependencies:
- @$(PYTHON_PIP) install -q -r lint_requirements.txt
- lint-server: lint-dependencies
- $(PYTHON) -m flake8 elyra .github
- @echo $(BLACK_CMD)
- @$(BLACK_CMD) || (echo "Black formatting encountered issues. Use 'make black-format' to apply the suggested changes."; exit 1)
- black-format: # Apply black formatter to Python source code
- $(PYTHON) -m black .
- prettier-check-ui:
- yarn prettier:check
- eslint-check-ui:
- yarn eslint:check --max-warnings=0
- prettier-ui:
- yarn prettier
- eslint-ui:
- yarn eslint --max-warnings=0
- lint-ui: prettier-ui eslint-ui
- lint: lint-ui lint-server ## Run linters
- ## Library linking targets
- dev-link:
- - yarn link @elyra/pipeline-services
- - yarn link @elyra/pipeline-editor
- - lerna run link:dev
- dev-unlink:
- - yarn unlink @elyra/pipeline-services
- - yarn unlink @elyra/pipeline-editor
- - lerna run unlink:dev
- yarn install --force
- ## Build and install targets
- build-dependencies:
- @$(PYTHON_PIP) install -q --upgrade pip
- @$(PYTHON_PIP) install -q -r build_requirements.txt
- dev-dependencies:
- @$(PYTHON_PIP) install -q --upgrade pip
- @$(PYTHON_PIP) install -q jupyter-packaging
- yarn-install:
- yarn install
- build-ui: # Build packages
- yarn lerna run build --stream
- package-ui: build-dependencies yarn-install lint-ui build-ui
- package-ui-dev: dev-dependencies yarn-install dev-link lint-ui build-ui
- build-server: # Build backend
- $(PYTHON) -m setup bdist_wheel sdist
- uninstall-server-package:
- @$(PYTHON_PIP) uninstall elyra -y
- install-server-package: uninstall-server-package
- $(PYTHON_PIP) install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) "$(shell find dist -name "elyra-*-py3-none-any.whl")[kfp-tekton]"
- install-server: build-dependencies lint-server build-server install-server-package ## Build and install backend
- install: package-ui install-server check-install ## Build and install
- install-all: package-ui install-server install-examples install-gitlab-dependency check-install ## Build and install, including examples
- install-dev: package-ui-dev install-server install-examples install-gitlab-dependency check-install
- install-examples: ## Install example pipeline components
- # install Kubeflow Pipelines example components
- # -> https://github.com/elyra-ai/examples/tree/main/component-catalog-connectors/kfp-example-components-connector
- - $(PYTHON_PIP) install --upgrade elyra-examples-kfp-catalog
- install-gitlab-dependency:
- # install GitLab support for Airflow
- - $(PYTHON_PIP) install --upgrade python-gitlab
- check-install:
- jupyter server extension list
- jupyter labextension list
- watch: ## Watch packages. For use alongside jupyter lab --watch
- yarn lerna run watch --parallel
- release: yarn-install build-ui build-server ## Build wheel file for release
- elyra-image-env: ## Creates a conda env consisting of the dependencies used in images
- conda env remove -y -n $(ELYRA_IMAGE_ENV)
- conda create -y -n $(ELYRA_IMAGE_ENV) python=$(PYTHON_VERSION)
- if [ "$(PYTHON_VERSION)" == "3.7" ]; then \
- $(CONDA_ACTIVATE) $(ELYRA_IMAGE_ENV) && \
- $(PYTHON_PIP) install -r etc/generic/requirements-elyra-py37.txt && \
- conda deactivate; \
- else \
- $(CONDA_ACTIVATE) $(ELYRA_IMAGE_ENV) && \
- $(PYTHON_PIP) install -r etc/generic/requirements-elyra.txt && \
- conda deactivate; \
- fi
- ## Test targets
- test-dependencies:
- @$(PYTHON_PIP) install -q -r test_requirements.txt
- pytest:
- $(PYTHON) -m pytest -v --durations=0 --durations-min=60 elyra
- test-server: test-dependencies pytest # Run python unit tests
- test-ui-unit: # Run frontend jest unit tests
- yarn test:unit
- test-integration: # Run frontend cypress integration tests
- yarn test:integration
- test-integration-debug: # Open cypress integration test debugger
- yarn test:integration:debug
- test-ui: lint-ui test-ui-unit test-integration # Run frontend tests
- test: test-server test-ui ## Run all tests (backend, frontend and cypress integration tests)
- ## Doc targets
- docs-dependencies:
- @$(PYTHON_PIP) install -q -r docs/requirements.txt
- docs: docs-dependencies ## Build docs
- make -C docs clean html
- ## Docker targets
- elyra-image: # Build Elyra stand-alone container image
- @mkdir -p build/docker
- cp etc/docker/elyra/Dockerfile build/docker/Dockerfile
- cp etc/docker/elyra/start-elyra.sh build/docker/start-elyra.sh
- cp etc/docker/elyra/requirements.txt build/docker/requirements.txt
- @mkdir -p build/docker/elyra
- cp dist/elyra-$(ELYRA_VERSION)-py3-none-any.whl build/docker/
- $(CONTAINER_EXEC) buildx build \
- --progress=plain \
- --output=type=docker \
- --tag docker.io/$(ELYRA_IMAGE) \
- --tag quay.io/$(ELYRA_IMAGE) \
- --build-arg TAG=$(TAG) \
- --build-arg ELYRA_VERSION=$(ELYRA_VERSION) \
- build/docker/;
- publish-elyra-image: elyra-image # Publish Elyra stand-alone container image
- # this is a privileged operation; a `docker login` might be required
- $(CONTAINER_EXEC) push docker.io/$(ELYRA_IMAGE)
- $(CONTAINER_EXEC) push quay.io/$(ELYRA_IMAGE)
- # If we're building a release from main, tag latest and push
- if [ "$(IMAGE_IS_LATEST)" == "True" ]; then \
- $(CONTAINER_EXEC) tag docker.io/$(ELYRA_IMAGE) docker.io/$(ELYRA_IMAGE_LATEST); \
- $(CONTAINER_EXEC) push docker.io/$(ELYRA_IMAGE_LATEST); \
- $(CONTAINER_EXEC) tag quay.io/$(ELYRA_IMAGE) quay.io/$(ELYRA_IMAGE_LATEST); \
- $(CONTAINER_EXEC) push quay.io/$(ELYRA_IMAGE_LATEST); \
- fi
- kf-notebook-image: # Build elyra image for use with Kubeflow Notebook Server
- @mkdir -p build/docker-kubeflow
- cp etc/docker/kubeflow/* build/docker-kubeflow/
- cp dist/elyra-$(ELYRA_VERSION)-py3-none-any.whl build/docker-kubeflow/
- $(CONTAINER_EXEC) buildx build \
- --progress=plain \
- --output=type=docker \
- --tag docker.io/$(KF_NOTEBOOK_IMAGE) \
- --tag quay.io/$(KF_NOTEBOOK_IMAGE) \
- --build-arg TAG=$(TAG) \
- --build-arg ELYRA_VERSION=$(ELYRA_VERSION) \
- build/docker-kubeflow;
- publish-kf-notebook-image: kf-notebook-image # Publish elyra image for use with Kubeflow Notebook Server
- # this is a privileged operation; a `docker login` might be required
- $(CONTAINER_EXEC) push docker.io/$(KF_NOTEBOOK_IMAGE)
- $(CONTAINER_EXEC) push quay.io/$(KF_NOTEBOOK_IMAGE)
- # If we're building a release from main, tag latest and push
- if [ "$(IMAGE_IS_LATEST)" == "True" ]; then \
- $(CONTAINER_EXEC) tag docker.io/$(KF_NOTEBOOK_IMAGE) docker.io/$(KF_NOTEBOOK_IMAGE_LATEST); \
- $(CONTAINER_EXEC) push docker.io/$(KF_NOTEBOOK_IMAGE_LATEST); \
- $(CONTAINER_EXEC) tag quay.io/$(KF_NOTEBOOK_IMAGE) quay.io/$(KF_NOTEBOOK_IMAGE_LATEST); \
- $(CONTAINER_EXEC) push quay.io/$(KF_NOTEBOOK_IMAGE_LATEST); \
- fi
- container-images: elyra-image kf-notebook-image ## Build all container images
- $(CONTAINER_EXEC) images $(ELYRA_IMAGE)
- $(CONTAINER_EXEC) images quay.io/$(ELYRA_IMAGE)
- $(CONTAINER_EXEC) images $(KF_NOTEBOOK_IMAGE)
- $(CONTAINER_EXEC) images quay.io/$(KF_NOTEBOOK_IMAGE)
- publish-container-images: publish-elyra-image publish-kf-notebook-image ## Publish all container images
- validate-runtime-images: # Validates delivered runtime-images meet minimum criteria
- @required_commands=$(REQUIRED_RUNTIME_IMAGE_COMMANDS) ; \
- $(PYTHON_PIP) install jq ; \
- for file in `find etc/config/metadata/runtime-images -name "*.json"` ; do \
- image=`cat $$file | jq -e -r '.metadata.image_name'` ; \
- if [ $$? -ne 0 ]; then \
- echo ERROR: $$file does not define the image_name property ; \
- exit 1; \
- fi; \
- make validate-runtime-image image=$$image ; \
- done
- validate-runtime-image: # Validate that runtime image meets minimum criteria
- @required_commands=$(REQUIRED_RUNTIME_IMAGE_COMMANDS) ; \
- if [[ $$image == "" ]] ; then \
- echo "Usage: make validate-runtime-image image=<container-image-name>" ; \
- exit 1 ; \
- fi ; \
- $(PYTHON_PIP) install -q jq ; \
- fail=0; \
- echo "***********************************************************" ; \
- echo "Validating container image $$image" ; \
- echo "-----------------------------------------------------------" ; \
- echo "=> Loading container image ..." ; \
- docker inspect $$image > /dev/null 2>&1 ; \
- if [ $$? -ne 0 ]; then \
- echo Container image $$image is not present, pulling... ; \
- docker pull $$image ; \
- if [ $$? -ne 0 ]; then \
- echo "ERROR: pull of container image $$image failed" ; \
- exit 1; \
- fi; \
- fi; \
- for cmd in $$required_commands ; do \
- echo "=> Checking container image $$image for $$cmd..." ; \
- docker run --rm $$image which $$cmd > /dev/null 2>&1 ; \
- if [ $$? -ne 0 ]; then \
- echo "ERROR: Container image $$image does not meet criteria for command: $$cmd" ; \
- fail=1; \
- continue; \
- fi; \
- if [ $$cmd == "python3" ]; then \
- IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm $$image $$cmd --version | cut -d' ' -f2 | cut -d'.' -f2` ; \
- if [[ $$IMAGE_PYTHON3_MINOR_VERSION -lt 8 ]]; then \
- echo "WARNING: Container image $$image requires Python 3.8 or greater for latest generic component dependency installation" ; \
- echo "=> Checking notebook execution..." ; \
- docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra-py37.txt && \
- curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \
- python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \
- if [ $$? -ne 0 ]; then \
- echo "ERROR: Container image $$image does not meet Python requirements criteria in requirements-elyra-py37.txt" ; \
- fail=1; \
- fi; \
- elif [[ $$IMAGE_PYTHON3_MINOR_VERSION -ge 8 ]]; then \
- echo "=> Checking notebook execution..." ; \
- docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \
- curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \
- python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \
- if [ $$? -ne 0 ]; then \
- echo "ERROR: Image $$image does not meet Python requirements criteria in requirements-elyra.txt" ; \
- fail=1; \
- fi; \
- else \
- echo "ERROR: Container image $$image: unable to parse Python version" ; \
- fail=1; \
- fi; \
- fi; \
- done ; \
- if [ $(REMOVE_RUNTIME_IMAGE) -eq 1 ]; then \
- echo Removing container image $$image... ; \
- docker rmi $$image > /dev/null ; \
- fi; \
- echo "-----------------------------------------------------------" ; \
- if [ $$fail -eq 1 ]; then \
- echo "=> ERROR: Container image $$image is not a suitable Elyra runtime image" ; \
- exit 1 ; \
- else \
- echo "=> Container image $$image is a suitable Elyra runtime image" ; \
- fi; \
|