Clean up CI

Break up the CI checks into three workflows

Update workflow names

Break changelog into a separate job

cleanup script

exclude changelog on python 3.5

Deselect changelog instead of deleting it

Remove azure pipelines config

remove a layer of nesting

Revert "remove a layer of nesting"

This reverts commit 02ea871ffdd9325343042b8e622bc0127999b91c.

.github/workflows/linuxjs-tests.yml

@@ -0,0 +1,54 @@
+name: Linux JS Tests
+
+on: [push, pull_request]
+
+jobs:
+  linuxjs:
+    name: JS
+    strategy:
+      matrix:
+        group: [js-services, js-application, js-apputils, js-cells, js-codeeditor, js-codemirror, js-completer, js-console, js-coreutils, js-csvviewer, js-docmanager, js-docregistry, js-filebrowser, js-fileeditor, js-imageviewer, js-inspector, js-logconsole, js-mainmenu, js-nbformat, js-notebook, js-observables, js-outputarea, js-rendermime,  js-settingregistry, js-statedb, js-statusbar, js-terminal, js-ui-components]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Set up Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: Cache pip on Linux
+        uses: actions/cache@v1
+        if: startsWith(runner.os, 'Linux')
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python }}
+
+      # Cache yarn
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Cache yarn
+        uses: actions/cache@v1
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install dependencies
+        env:
+          GROUP: ${{ matrix.group }}
+        run: |
+          bash ./scripts/ci_install.sh
+      - name: Run test ${{ matrix.group }}
+        env:
+          GROUP: ${{ matrix.group }}
+        run: |
+          bash ./scripts/ci_script.sh

.github/workflows/linuxtests.yml

@@ -0,0 +1,62 @@
+name: Linux Tests
+
+on: [push, pull_request]
+
+jobs:
+  linuxtests:
+    name: Linux
+    strategy:
+      matrix:
+        group: [integrity, docs, changelog, python, usage, nonode]
+        python: [3.5, 3.8]
+        exclude:
+          - group: integrity
+            python: 3.5
+          - group: docs
+            python: 3.5
+          - group: changelog
+            python: 3.5
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Set up Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: Cache pip on Linux
+        uses: actions/cache@v1
+        if: startsWith(runner.os, 'Linux')
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python }}
+
+      # Cache yarn
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Cache yarn
+        uses: actions/cache@v1
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install dependencies
+        env:
+          GROUP: ${{ matrix.group }}
+        run: |
+          bash ./scripts/ci_install.sh
+      - name: Run test ${{ matrix.group }}
+        env:
+          GROUP: ${{ matrix.group }}
+        run: |
+          bash ./scripts/ci_script.sh

.github/workflows/tests.yml

@@ -1,165 +0,0 @@
-name: Test
-
-on: [push, pull_request]
-
-jobs:
-  linuxjs:
-    name: JS
-    strategy:
-      matrix:
-        group: [js-application, js-apputils, js-cells, js-codeeditor, js-codemirror, js-completer, js-console, js-coreutils, js-csvviewer, js-docmanager, js-docregistry, js-filebrowser, js-fileeditor, js-imageviewer, js-inspector, js-logconsole, js-mainmenu, js-nbformat, js-notebook, js-observables, js-outputarea, js-rendermime, js-services, js-settingregistry, js-statedb, js-statusbar, js-terminal, js-ui-components]
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Set up Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12.x'
-      - name: Cache pip on Linux
-        uses: actions/cache@v1
-        if: startsWith(runner.os, 'Linux')
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ matrix.python }}
-
-      # Cache yarn
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Cache yarn
-        uses: actions/cache@v1
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install dependencies
-        env:
-          GROUP: ${{ matrix.group }}
-        run: |
-          bash ./scripts/ci_install.sh
-      - name: Run test ${{ matrix.group }}
-        env:
-          GROUP: ${{ matrix.group }}
-        run: |
-          bash ./scripts/ci_script.sh
-
-  linuxtests:
-    name: Linux
-    strategy:
-      matrix:
-        group: [integrity, docs, python, usage, nonode]
-        python: [3.5, 3.8]
-        exclude:
-          - group: integrity
-            python: 3.5
-          - group: docs
-            python: 3.5
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Set up Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12.x'
-      - name: Cache pip on Linux
-        uses: actions/cache@v1
-        if: startsWith(runner.os, 'Linux')
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ matrix.python }}
-
-      # Cache yarn
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Cache yarn
-        uses: actions/cache@v1
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install dependencies
-        env:
-          GROUP: ${{ matrix.group }}
-        run: |
-          bash ./scripts/ci_install.sh
-      - name: Run test ${{ matrix.group }}
-        env:
-          GROUP: ${{ matrix.group }}
-        run: |
-          bash ./scripts/ci_script.sh
-
-  windowstests:
-    name: Windows
-    strategy:
-      matrix:
-        group: [javascript, python, integrity]
-      fail-fast: false
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.5
-      - name: Set up Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12.x'
-      - name: Cache pip on Windows
-        uses: actions/cache@v1
-        if: startsWith(runner.os, 'Windows')
-        with:
-          path: ~\AppData\Local\pip\Cache
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      # Cache yarn
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Cache yarn
-        uses: actions/cache@v1
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-      - name: Enable long paths
-        run: powershell Set-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1
-      - name: Upgrade pip
-        run: python -m pip install -U pip
-      - name: Install pip deps
-        run: pip install --upgrade -e ".[test]"
-      - name: List kernels
-        run: jupyter kernelspec list
-      - name: List jlpm versions
-        run: jlpm versions
-      - name: List jlpm config
-        run: jlpm config current
-      - name: Run test ${{ matrix.group }}
-        env:
-          NAME: ${{ matrix.group }}
-        run: cmd /E:ON /V:ON /C .\\scripts\\appveyor.cmd

.github/workflows/windowstests.yml

@@ -0,0 +1,59 @@
+name: Windows Tests
+
+on: [push, pull_request]
+
+jobs:
+  windowstests:
+    name: Windows
+    strategy:
+      matrix:
+        group: [javascript, python, integrity]
+      fail-fast: false
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.5
+      - name: Set up Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: Cache pip on Windows
+        uses: actions/cache@v1
+        if: startsWith(runner.os, 'Windows')
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      # Cache yarn
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Cache yarn
+        uses: actions/cache@v1
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: Enable long paths
+        run: powershell Set-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1
+      - name: Upgrade pip
+        run: python -m pip install -U pip
+      - name: Install pip deps
+        run: pip install --upgrade -e ".[test]"
+      - name: List kernels
+        run: jupyter kernelspec list
+      - name: List jlpm versions
+        run: jlpm versions
+      - name: List jlpm config
+        run: jlpm config current
+      - name: Run test ${{ matrix.group }}
+        env:
+          NAME: ${{ matrix.group }}
+        run: cmd /E:ON /V:ON /C .\\scripts\\appveyor.cmd

azure-pipelines.yml

@@ -1,99 +0,0 @@
-jobs:
-  - job: 'Linux'
-    pool:
-      vmImage: 'ubuntu-16.04'
-    variables:
-      python.version: '3.6'
-      gh.ref: github.com/jupyterlab/jupyterlab.git
-    strategy:
-      matrix:
-        JS:
-          group: 'js'
-          testResultsFiles: 'tests/**/junit.xml'
-        Integrity:
-          group: 'integrity'
-        Python:
-          group: 'python'
-          testResultsFiles: 'junit.xml'
-        Usage:
-          group: 'usage'
-        UsageNoNodeJS:
-          group: 'nonode'
-        Docs:
-          group: 'docs'
-
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: 'x64'
-
-      - task: Bash@3
-        displayName: 'install'
-        inputs:
-          targetType: 'filePath'
-          filePath: ./scripts/ci_install.sh
-      - script: python -m pip install virtualenv
-      - task: Bash@3
-        displayName: 'script'
-        inputs:
-          targetType: 'filePath'
-          filePath: ./scripts/ci_script.sh
-      - task: PublishTestResults@2
-        displayName: 'publish test results'
-        condition: variables['testResultsFiles']
-        inputs:
-          testResultsFiles: '$(testResultsFiles)'
-          testRunTitle: 'Linux - $(group)'
-          mergeTestResults: true
-
-  - job: 'Windows'
-    pool:
-      vmImage: 'vs2017-win2016'
-    variables:
-      python.version: '3.6'
-    strategy:
-      matrix:
-        JS:
-          name: 'javascript'
-          testResultsFiles: 'tests/**/junit.xml'
-          python.version: '3.5'
-        Python:
-          name: 'python'
-        Integrity:
-          name: 'integrity'
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'install python'
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: 'x64'
-
-      - script: powershell Set-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1
-        displayName: 'set long path'
-      - script: 'python -m pip install -U pip'
-        displayName: 'upgrade pip'
-      - script: 'pip install --upgrade -e ".[test]"'
-        displayName: 'install pip deps'
-      - script: 'jupyter kernelspec list'
-        displayName: 'list kernels'
-      - script: 'jlpm versions'
-        displayName: 'list jlpm versions'
-      - script: 'jlpm config current'
-        displayName: 'list jlpm config'
-      - script: cmd /E:ON /V:ON /C .\\scripts\\appveyor.cmd
-        displayName: 'run tests'
-      - task: PublishTestResults@2
-        displayName: 'publish results'
-        condition: variables['testResultsFiles']
-        inputs:
-          testResultsFiles: '$(testResultsFiles)'
-          testRunTitle: 'Windows - $(name)'
-          mergeTestResults: true
-
-trigger:
-  branches:
-    include:
-    - master
-    - 1.x

scripts/ci_script.sh

@@ -67,12 +67,23 @@ if [[ $GROUP == docs ]]; then
     # Remove internal sphinx files and use pytest-check-links on the generated html
     rm build/html/genindex.html
     rm build/html/search.html
-    py.test --check-links -k .html build/html || py.test --check-links -k .html --lf build/html
+
+    # Changelog as a lot of links and is covered in a separate job.
+    changelog=./docs/source/getting_started/changelog.rst
+    py.test --check-links -k .html ----deselect=$changelog build/html || py.test --check-links -k .html --lf build/html
 
     popd
 fi
 
 
+
+if [[ $GROUP == changelog ]]; then
+    # Run the link check - allow for a link to fail once (--lf means only run last failed)
+    changelog=./docs/source/getting_started/changelog.rst
+    py.test --check-links $changlog || py.test --check-links --lf $changelog
+fi
+
+
 if [[ $GROUP == integrity ]]; then
     # Run the integrity script first
     jlpm run integrity --force