Carry end-to-end tests on docker (#10498)

* Carry end-to-end tests on docker

* Fix integrity test

* Remove unused install instruction

.dockerignore

@@ -0,0 +1,44 @@
+.git
+.dockerignore
+Dockerfile
+
+# project files
+coverage/
+
+dev_mode/listings
+dev_mode/schemas
+dev_mode/static
+dev_mode/themes
+dev_mode/workspaces
+dev_mode/stats.json
+
+# Skip docs
+docs/
+**/docs/source/_build
+
+# Skip examples
+examples/
+
+junit.xml
+
+jupyterlab/geckodriver
+jupyterlab/static
+jupyterlab/schemas
+jupyterlab/themes
+jupyterlab/style.js
+jupyterlab/tests/**/static
+jupyterlab/tests/**/labextension
+
+# Remove after next release
+jupyterlab/imports.css
+
+packages/nbconvert-css/style/
+packages/services/examples/node/config.json
+packages/services/examples/browser/tmp
+packages/theme-*/static
+
+tests/**/coverage
+tests/**/.cache-loader
+
+**/node_modules
+ui-tests/

.github/workflows/ui-tests.yml

@@ -6,76 +6,15 @@ jobs:
   ui-tests:
     name: Visual Regression
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        group: [ui-tests]
-        python: [3.8]
-      fail-fast: false
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Set up Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12.x'
-
-      - name: Cache pip on Linux
-        uses: actions/cache@v2
-        if: startsWith(runner.os, 'Linux')
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt', 'setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ matrix.python }}
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Cache yarn
-        uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install dependencies
-        env:
-          GROUP: ${{ matrix.group }}
-        run: |
-          bash ./scripts/ci_install.sh
-          jlpm
-          jlpm build
-          jupyter lab build
-
-      - name: Install Galata
-        run: |
-          cd ui-tests
-          jlpm install --frozen-lockfile
-
-      - name: Launch JupyterLab
-        run: |
-          cd ui-tests
-          mkdir jlab_root
-          jlpm run start-jlab:detached
-
-      - name: Wait for JupyterLab
-        uses: ifaxity/wait-on-action@v1
-        with:
-          resource: http-get://localhost:8888/api
-          timeout: 20000
 
       - name: Run UI Tests
         run: |
-          cd ui-tests
-          jlpm run test
-          rm -rf jlab_root
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" down || true
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" pull
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" run --rm e2e
 
       - name: Upload UI Test artifacts
         if: always()
@@ -85,14 +24,18 @@ jobs:
           path: |
             ui-tests/test-output
 
+      - name: Stop containers
+        if: always()
+        run: |
+          # Print jupyterlab logs before removing the containers using the container name set in docker-compose file
+          docker logs jupyterlab
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" down
+          
       - name: Run UI Tests
         if: ${{ failure() }}
         run: |
-          cd ui-tests
-          [ -d "jlab_root" ] && rm -rf jlab_root
-          mkdir jlab_root
-          jlpm run test:create-references
-          rm -rf jlab_root
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" run --rm e2e yarn run test:create-references --jlab-base-url=http://jupyterlab:8888
+          docker-compose -f "./ui-tests/docker/docker-compose.yml" down
 
       - name: Upload UI Test new reference artifacts
         if: ${{ failure() }}

Dockerfile

@@ -0,0 +1,60 @@
+# Build image
+FROM mambaorg/micromamba:0.14.0 as build
+
+# Install basic tools
+RUN micromamba install -qy -c conda-forge python nodejs yarn=1.21 build
+
+# Install python dependencies - faster build thanks to caching
+COPY setup.cfg /tmp
+
+RUN list_package=$(python -c "from configparser import ConfigParser; c = ConfigParser(); c.read('/tmp/setup.cfg'); print(' '.join(c['options']['install_requires'].strip().splitlines()))") \
+    && micromamba install -qy -c conda-forge $list_package
+
+# Build JupyterLab wheel
+COPY ./builder /tmp/jupyterlab-dev/builder
+COPY ./buildutils /tmp/jupyterlab-dev/buildutils
+COPY ./dev_mode /tmp/jupyterlab-dev/dev_mode
+COPY ./jupyterlab /tmp/jupyterlab-dev/jupyterlab
+COPY ./packages /tmp/jupyterlab-dev/packages
+COPY ./scripts /tmp/jupyterlab-dev/scripts
+COPY ./*.* ./LICENSE /tmp/jupyterlab-dev/
+
+RUN pushd /tmp/jupyterlab-dev \
+    && pip install -e . \
+    && jlpm install \
+    && jlpm run build \
+    && python -m build .
+
+# Runtime image
+FROM mambaorg/micromamba:0.14.0
+
+RUN micromamba install -qy -c conda-forge python \
+    && useradd --shell /bin/bash jovyan \
+    && chown jovyan $HOME
+
+# Install python dependencies - faster build thanks to caching
+COPY setup.cfg /tmp
+
+RUN list_package=$(python -c "from configparser import ConfigParser; c = ConfigParser(); c.read('/tmp/setup.cfg'); print(' '.join(c['options']['install_requires'].strip().splitlines()))") \
+    && micromamba install -qy -c conda-forge $list_package \
+    && micromamba clean --all --yes \
+    && rm /tmp/setup.cfg
+
+USER jovyan
+WORKDIR ${HOME}
+
+# Install JupyterLab
+ENV PATH="/home/micromamba/.local/bin:$PATH"
+
+COPY --from=build /tmp/jupyterlab-dev/dist/jupyterlab*.whl /tmp
+
+RUN pip install /tmp/jupyterlab*.whl \
+    # TODO remove when ipykernel 6 is released
+    && pip install --pre --upgrade ipykernel \
+    && mkdir -p /home/micromamba/jlab_root
+
+COPY ./docker/jupyter_server_config.json /etc/jupyter/
+
+EXPOSE 8888
+
+ENTRYPOINT ["jupyter", "lab"]

MANIFEST.in

@@ -32,4 +32,9 @@ prune jupyterlab/staging/build
 recursive-exclude jupyterlab *.pyc
 recursive-exclude jupyterlab *.js.map
 
+# Docker
+include Dockerfile
+include .dockerignore
+recursive-include docker *.json
+
 prune ui-tests

docker/jupyter_server_config.json

@@ -0,0 +1,9 @@
+{
+  "ServerApp": {
+    "allow_origin": "*",
+    "ip": "0.0.0.0",
+    "port": 8888,
+    "open_browser": false,
+    "root_dir": "/home/micromamba/jlab_root"
+  }
+}

scripts/ci_install.sh

@@ -22,11 +22,6 @@ pip --version
 pip install -e ".[test]" || pip install -v -e ".[test]"
 jlpm versions
 jlpm config current
-jupyter server extension enable jupyterlab
-jupyter server extension list 1>serverextensions 2>&1
-cat serverextensions
-cat serverextensions | grep -i "jupyterlab.*enabled"
-cat serverextensions | grep -i "jupyterlab.*OK"
 
 # TODO: remove when we no longer support classic notebook
 jupyter serverextension enable jupyterlab
@@ -55,12 +50,6 @@ if [[ $GROUP == js-debugger ]]; then
     pip install xeus-python">=0.9.0,<0.10.0"
 fi
 
-# The UI tests use ipykernel with support for debugging, to not
-# extra launcher items and other UI elements to the reference screenshots
-if [[ $GROUP == ui-tests ]]; then
-    pip install --pre --upgrade ipykernel
-fi
-
 # Pin the jedi dependency to prevent a temporary breakage
 # See https://github.com/ipython/ipython/issues/12740, https://github.com/ipython/ipython/pull/12751
 pip install 'jedi<0.18'

ui-tests/.dockerignore

@@ -0,0 +1,7 @@
+.dockerignore
+Dockerfile
+docker/docker-compose.yml
+
+jlab_root/
+node_modules/
+test-output/

ui-tests/Dockerfile

@@ -0,0 +1,14 @@
+# Playwright version should match the one in the yarn.lock file
+FROM mcr.microsoft.com/playwright:v1.11.1-focal
+
+COPY *.* /tmp/galata/
+
+WORKDIR /tmp/galata
+
+RUN yarn install --frozen-lockfile
+
+COPY ./docker/*.sh /opt/galata/
+
+ENTRYPOINT [ "/opt/galata/run-e2e.sh" ]
+
+CMD ["yarn", "run", "test"]

ui-tests/README.md

@@ -13,41 +13,19 @@ import { galata, describe, test } from '@jupyterlab/galata';
 
 3. Add your test suite with step by step tasks. Galata runs steps in series, so your steps can rely on the state in the previous step. Check [Galata API](https://github.com/jupyterlab/galata/blob/main/packages/galata/src/galata.ts) for available functionality to interact with JupyterLab.
 4. When doing visual regression tests, it is important to use reference images that were generated in the same environment. Please read _Reference Image Captures_ section [here](https://github.com/jupyterlab/galata#reference-image-captures) if you are adding visual regression tests.
-5. If you added visual regression tests, make sure you tested them locally enough number of times to make sure they do not produce false positives due to async nature of UI actions. Use the following steps to test locally.
+5. If you added visual regression tests, make sure you tested them locally enough number of times to make sure they do not produce false positives due to async nature of UI actions. Use the following steps to test locally (it requires [docker](https://docs.docker.com/engine/) and [docker-compose](https://docs.docker.com/compose/install/)).
 
 ```bash
-# install the ipykernel pre-release with support for debugging
-python -m pip install --pre --upgrade ipykernel
-
-# inside ui-tests directory:
-
-# install dependencies
-jlpm install --frozen-lockfile
-
-# launch JupyterLab
-#   create empty working directory
-mkdir jlab_root
-#  launch JupyterLab with test configuration
-jlpm run start-jlab
-
-# run UI tests once to create test captures to use as reference images
-jlpm run test:create-references
+# run UI tests once to create test captures to use as reference images for your new feature
+docker-compose -f "./ui-tests/docker/docker-compose.yml" run --rm e2e yarn run test:create-references --jlab-base-url=http://jupyterlab:8888
 
 # copy test captures into reference-output directory to use as references
-#  delete existing references
-rm -rf reference-output/screenshots/*.*
-#  set locally generated test output as references
-cp test-output/test/screenshots/*.* reference-output/screenshots
+# cp test-output/test/screenshots/*.* reference-output/screenshots
 
 # run UI tests locally, repeatedly. make sure no test fails. wait for 10-20 successful repeats
-./repeated_test_run.sh
+docker-compose -f "./ui-tests/docker/docker-compose.yml" run --rm e2e
 ```
 
-6. Once you are done testing locally:
-
-- Revert changes in your `reference-output/screenshots/` directory
-- Update [CI script](../.github/workflows/ui-tests.yml) to use `jlpm run test:create-references` instead of `jlpm run test`. Push your test suite files with change to CI script and create a PR. Do not push reference images generated on your local computer.
-- Once CI workflow runs for your PR on GitHub, it will generate test artifacts from your test output, named `ui-test-output`. Download it from GitHub Actions page and copy screenshots from test-output directory into `ui-tests/reference-output/screenshots` directory. Make sure the changes are limited your newly added / modified tests.
-- Update [CI script](../.github/workflows/ui-tests.yml) to use `jlpm run test` now and push to your PR along with newly generated reference screenshots.
+6. Once you are done testing locally, push the new references on your PR and check CI is passing.
 
-7. If your tests are failing or if you want to debug UI tests, you can use the script `jlpm run test:debug` by specifying the particular test suite(s) in [package.json](package.json) you want to debug. Check [Galata CLI Options](https://github.com/jupyterlab/galata#command-line-options) for list of available command-line options.
+7. If your tests are failing or if you want to debug UI tests, you can use the script `yarn run test:debug --include=...` by specifying the particular test suite(s) you want to debug. Check [Galata CLI Options](https://github.com/jupyterlab/galata#command-line-options) for list of available command-line options.

ui-tests/docker/docker-compose.yml

@@ -0,0 +1,39 @@
+version: '3.5'
+
+services:
+  lab:
+    container_name: jupyterlab
+    build:
+      context: ../..
+      dockerfile: ./Dockerfile
+    command:
+      [
+        '--ServerApp.token=',
+        '--ServerApp.password=',
+        '--ServerApp.disable_check_xsrf=True',
+        '--LabApp.expose_app_in_browser=True',
+      ]
+    networks:
+      - frontend
+    ports:
+      - 8888:8888
+
+  e2e:
+    build:
+      context: ..
+    command: ['yarn', 'run', 'test', '--jlab-base-url=http://jupyterlab:8888']
+    # See https://playwright.dev/docs/docker/#run-the-image
+    ipc: host
+    networks:
+      - frontend
+    depends_on:
+      - lab
+    volumes:
+      # Mount everything in a shared drive to be sure we are not using files from docker cache
+      - ../reference-output:/tmp/galata/reference-output
+      - ../test-output:/tmp/galata/test-output
+      - ../tests:/tmp/galata/tests
+    working_dir: /tmp/galata
+
+networks:
+  frontend:

ui-tests/docker/run-e2e.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Trick to get the galata test outputs writable on the host
+umask 0000
+# Wait for jupyterlab container to be ready
+/opt/galata/wait-for-it.sh jupyterlab:8888 --strict --timeout=10 -- $*

ui-tests/docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi