Clean up Release Scripts and Test in CI

.github/workflows/linuxtests.yml

@@ -7,7 +7,7 @@ jobs:
     name: Linux
     strategy:
       matrix:
-        group: [integrity, integrity2, integrity3, docs, usage, usage2, python, examples, interop, nonode, linkcheck, lint]
+        group: [integrity, integrity2, integrity3, release_check, docs, usage, usage2, python, examples, interop, nonode, linkcheck, lint]
         python: [3.6, 3.8]
         exclude:
           - group: integrity
@@ -16,6 +16,8 @@ jobs:
             python: 3.6
           - group: integrity3
             python: 3.6
+          - group: release_check
+            python: 3.6
           - group: docs
             python: 3.6
           - group: usage

RELEASE.md

@@ -71,16 +71,27 @@ Results:
 
 ## Publishing Packages
 
-Now publish the JS packages and build the python packages
+Now publish the JS packages
 
 ```bash
-npm run publish:all
+npm run publish:js
 ```
 
-If there is a network error during JS publish, run `npm run publish:all --skip-build` to resume publish without requiring another clean and build phase of the JS packages.
+If there is a network error during JS publish, run `npm run publish:js --skip-build` to resume publish without requiring another clean and build phase of the JS packages.
 
 Note that the use of `npm` instead of `jlpm` is [significant on Windows](https://github.com/jupyterlab/jupyterlab/issues/6733).
 
+Next, prepare the python release by running:
+
+```bash
+npm run prepare:python-release
+```
+
+This will update the Python package to use the new JS packages and
+create the Python release assets. Note: sometimes the npm registry
+is slow to update with the new packages, so this script tries to fetch
+the packages until they are available.
+
 At this point, run the `./scripts/release_test.sh` to test the wheel in
 a fresh conda environment with and without extensions installed. Open and run
 the Outputs notebook and verify everything runs properly. Also add a cell with the following code and make sure the widget renders:

buildutils/src/prepare-python-release.ts

@@ -0,0 +1,106 @@
+/* -----------------------------------------------------------------------------
+| Copyright (c) Jupyter Development Team.
+| Distributed under the terms of the Modified BSD License.
+|----------------------------------------------------------------------------*/
+
+import commander from 'commander';
+import * as crypto from 'crypto';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import * as utils from './utils';
+
+/**
+ * Verify that a package specifier is published and available on npm.
+ *
+ * @param specifier The package specifier to verify.
+ */
+function verifyPublished(specifier: string): void {
+  const cmd = `npm info ${specifier}`;
+  const output = utils.run(cmd, { stdio: 'pipe' }, true);
+  console.log(specifier);
+  if (output.indexOf('dist-tags') === -1) {
+    throw new Error(`${specifier} is not yet available`);
+  }
+}
+
+/**
+ * Sleep for a specified period.
+ *
+ * @param wait The time in milliseconds to wait.
+ */
+async function sleep(wait: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, wait));
+}
+
+// Specify the program signature.
+commander
+  .description('Prepare the Python package for release')
+  .action(async (options: any) => {
+    // Make sure all current JS packages are published.
+    console.log('Checking for published packages...');
+    utils.getCorePaths().forEach(async pkgPath => {
+      const pkgJson = path.join(pkgPath, 'package.json');
+      const pkgData = utils.readJSONFile(pkgJson);
+      const specifier = `${pkgData.name}@${pkgData.version}`;
+      let attempt = 0;
+      while (attempt < 10) {
+        try {
+          verifyPublished(specifier);
+          break;
+        } catch (e) {
+          console.error(e);
+          console.log('Sleeping for one minute...');
+          await sleep(1 * 60 * 1000);
+          attempt += 1;
+        }
+      }
+    });
+
+    const distDir = './dist';
+
+    // Clean the dist directory.
+    if (fs.existsSync(distDir)) {
+      fs.removeSync(distDir);
+    }
+
+    // Update core mode.  This cannot be done until the JS packages are
+    // released.
+    utils.run('node buildutils/lib/update-core-mode.js');
+
+    // Make the Python release.
+    utils.run('python -m pip install -U twine build');
+    utils.run('python -m build .');
+    utils.run('twine check dist/*');
+
+    const files = fs.readdirSync(distDir);
+    const hashes = new Map<string, string>();
+    files.forEach(file => {
+      const shasum = crypto.createHash('sha256');
+      const hash = shasum.update(fs.readFileSync(path.join(distDir, file)));
+      hashes.set(file, hash.digest('hex'));
+    });
+
+    const hashString = Array.from(hashes.entries())
+      .map(entry => `${entry[0]}: ${entry[1]}`)
+      .join('" -m "');
+
+    // Make the commit and the tag.
+    const curr = utils.getPythonVersion();
+    utils.run(
+      `git commit -am "Publish ${curr}" -m "SHA256 hashes:" -m "${hashString}"`
+    );
+    utils.run(`git tag v${curr}`);
+
+    // Prompt the user to finalize.
+    console.debug('*'.repeat(40));
+    console.debug('*'.repeat(40));
+    console.debug('Ready to publish!');
+    console.debug('Run these command when ready:');
+    console.debug('twine upload dist/*');
+    console.debug('git push origin <BRANCH> --tags');
+
+    // Emit a system beep.
+    process.stdout.write('\x07');
+  });
+
+commander.parse(process.argv);

buildutils/src/publish.ts

@@ -4,47 +4,40 @@
 |----------------------------------------------------------------------------*/
 
 import commander from 'commander';
-import * as crypto from 'crypto';
-import * as fs from 'fs-extra';
 import * as path from 'path';
 import { handlePackage } from './update-dist-tag';
 import * as utils from './utils';
 
-async function sleep(wait: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, wait));
-}
-
 // Specify the program signature.
 commander
-  .description('Publish the JS packages and prep the Python package')
+  .description('Publish the JS packages')
   .option(
     '--skip-build',
     'Skip the clean and build step (if there was a network error during a JS publish'
   )
+  .option('--dry-run', 'Do not actually push any assets')
   .action(async (options: any) => {
-    // Make sure we are logged in.
-    if (utils.checkStatus('npm whoami') !== 0) {
-      console.error('Please run `npm login`');
+    if (!options.skipBuild) {
+      utils.run('jlpm run build:packages');
     }
-    const distDir = './dist';
 
-    // Optionally clean and build the python packages.
-    if (!options.skipBuild) {
-      // Ensure a clean state.
-      utils.run('npm run clean:slate');
-    } else {
-      // Still clean the dist directory.
-      if (fs.existsSync(distDir)) {
-        fs.removeSync(distDir);
+    if (!options.dryRun) {
+      // Make sure we are logged in.
+      if (utils.checkStatus('npm whoami') !== 0) {
+        console.error('Please run `npm login`');
       }
     }
 
     // Publish JS to the appropriate tag.
     const curr = utils.getPythonVersion();
+    let cmd = 'lerna publish from-package ';
+    if (options.dryRun) {
+      cmd += '--no-git-tag-version --no-push ';
+    }
     if (curr.indexOf('rc') === -1 && curr.indexOf('a') === -1) {
-      utils.run('lerna publish from-package -m "Publish"');
+      utils.run(`${cmd} -m "Publish"`);
     } else {
-      utils.run('lerna publish from-package --dist-tag=next -m "Publish"');
+      utils.run(`${cmd} --dist-tag=next -m "Publish"`);
     }
 
     // Fix up any tagging issues.
@@ -53,52 +46,14 @@ commander
     const cmds = await Promise.all(paths.map(handlePackage));
     cmds.forEach(cmdList => {
       cmdList.forEach(cmd => {
-        utils.run(cmd);
+        if (options.dryRun) {
+          utils.run(cmd);
+        } else {
+          throw new Error(`Tag is out of sync: ${cmd}`);
+        }
       });
     });
 
-    // Pause to allow npm some time to update their servers to list the published packages.
-    const pause = 5; // minutes
-    console.log(
-      `Pausing ${pause} minutes after publishing to allow npmjs.com to update their package listing.`
-    );
-    await sleep(pause * 60 * 1000);
-
-    // Update core mode.  This cannot be done until the JS packages are
-    // released.
-    utils.run('node buildutils/lib/update-core-mode.js');
-
-    // Make the Python release.
-    utils.run('python -m pip install -U twine build');
-    utils.run('python -m build .');
-    utils.run('twine check dist/*');
-
-    const files = fs.readdirSync(distDir);
-    const hashes = new Map<string, string>();
-    files.forEach(file => {
-      const shasum = crypto.createHash('sha256');
-      const hash = shasum.update(fs.readFileSync(path.join(distDir, file)));
-      hashes.set(file, hash.digest('hex'));
-    });
-
-    const hashString = Array.from(hashes.entries())
-      .map(entry => `${entry[0]}: ${entry[1]}`)
-      .join('" -m "');
-
-    // Make the commit and the tag.
-    utils.run(
-      `git commit -am "Publish ${curr}" -m "SHA256 hashes:" -m "${hashString}"`
-    );
-    utils.run(`git tag v${curr}`);
-
-    // Prompt the user to finalize.
-    console.debug('*'.repeat(40));
-    console.debug('*'.repeat(40));
-    console.debug('Ready to publish!');
-    console.debug('Run these command when ready:');
-    console.debug('twine upload dist/*');
-    console.debug('git push origin <BRANCH> --tags');
-
     // Emit a system beep.
     process.stdout.write('\x07');
   });

jupyterlab/browser_check.py

@@ -180,6 +180,7 @@ class BrowserApp(LabApp):
     serverapp_config = {
         "base_url": "/foo/"
     }
+    default_url = "/lab?reset"
     ip = '127.0.0.1'
     flags = test_flags
     aliases = test_aliases

package.json

@@ -69,10 +69,11 @@
     "lint": "jlpm && jlpm run prettier && jlpm run eslint",
     "lint:check": "jlpm run prettier:check && jlpm run eslint:check",
     "patch:release": "node buildutils/lib/patch-release.js",
+    "prepare:python-release": "node buildutils/lib/prepare-python-release.js",
     "prepublish:check": "node buildutils/lib/prepublish-check.js",
     "prettier": "prettier --write \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"",
     "prettier:check": "prettier --list-different \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"",
-    "publish:all": "node buildutils/lib/publish.js",
+    "publish:js": "node buildutils/lib/publish.js",
     "remove:dependency": "node buildutils/lib/remove-dependency.js",
     "remove:package": "node buildutils/lib/remove-package.js",
     "remove:sibling": "node buildutils/lib/remove-package.js",

scripts/ci_install.ps1

@@ -13,9 +13,6 @@ if ($LASTEXITCODE -ne 0) { throw "Command failed. See above errors for details"
 pip --version
 if ($LASTEXITCODE -ne 0) { throw "Command failed. See above errors for details" }
 
-pip install jupyter_packaging
-if ($LASTEXITCODE -ne 0) { throw "Command failed. See above errors for details" }
-
 # Show a verbose install if the install fails, for debugging
 pip install -e ".[test]" || pip install -v -e ".[test]"
 if ($LASTEXITCODE -ne 0) { throw "Command failed. See above errors for details" }

scripts/ci_install.sh

@@ -18,7 +18,6 @@ mkdir ~/.jupyter
 # Install and enable the server extension
 pip install -q --upgrade pip --user
 pip --version
-pip install jupyter_packaging
 # Show a verbose install if the install fails, for debugging
 pip install -e ".[test]" || pip install -v -e ".[test]"
 jlpm versions

scripts/ci_script.sh

@@ -156,6 +156,12 @@ if [[ $GROUP == integrity3 ]]; then
 fi
 
 
+if [[ $GROUP == release_check ]]; then
+    jlpm run publish:js --dry-run
+    jlpm run prepare:python-release
+    ./scripts/release_test.sh
+fi
+
 if [[ $GROUP == examples ]]; then
     # Run the integrity script to link binary files
     jlpm integrity

scripts/release_test.sh

@@ -26,16 +26,17 @@ jupyter labextension develop extension
 jupyter labextension build extension
 popd
 
-# Install third party widgets - allow to fail
-pip install --pre jupyterlab_widgets; true
-
 pushd $TEST_DIR
 
-conda install --override-channels --strict-channel-priority -c conda-forge -c anaconda -y ipywidgets altair matplotlib vega_datasets
+conda install --override-channels --strict-channel-priority -c conda-forge -c anaconda -y ipywidgets altair matplotlib vega_datasets jupyterlab_widgets
 
 jupyter lab build
 python -m jupyterlab.browser_check
-jupyter lab
+
+# if not running on github actions, start JupyterLab
+if [[ -z "${GITHUB_ACTIONS}" ]]; then
+    jupyter lab
+fi
 
 # undo our environment changes just in case we are sourcing this script
 conda deactivate

setup.py

@@ -146,6 +146,7 @@ setup_args['install_requires'] = [
     'packaging',
     'tornado>=6.1.0',
     'jupyter_core',
+    'jupyter_packaging~=0.7.3',
     'jupyterlab_server~=2.2',
     'jupyter_server~=1.2',
     'nbclassic~=0.2',