Merge pull request #9310 from jasongrout/sharedeps

Built-in extensions using federated dependencies

buildutils/src/update-core-mode.ts

@@ -39,6 +39,7 @@ const notice =
 
 [
   'index.js',
+  'bootstrap.js',
   'publicpath.js',
   'webpack.config.js',
   'webpack.prod.config.js',

dev_mode/bootstrap.js

@@ -0,0 +1,115 @@
+/*-----------------------------------------------------------------------------
+| Copyright (c) Jupyter Development Team.
+| Distributed under the terms of the Modified BSD License.
+|----------------------------------------------------------------------------*/
+
+// We copy some of the pageconfig parsing logic in @jupyterlab/coreutils
+// below, since this must run before any other files are loaded (including
+// @jupyterlab/coreutils).
+
+/**
+ * Get global configuration data for the Jupyter application.
+ *
+ * @param name - The name of the configuration option.
+ *
+ * @returns The config value or an empty string if not found.
+ *
+ * #### Notes
+ * All values are treated as strings. For browser based applications, it is
+ * assumed that the page HTML includes a script tag with the id
+ * `jupyter-config-data` containing the configuration as valid JSON.
+ */
+let _CONFIG_DATA = null;
+function getOption(name) {
+  if (_CONFIG_DATA === null) {
+    let configData;
+    // Use script tag if available.
+    if (typeof document !== 'undefined' && document) {
+      const el = document.getElementById('jupyter-config-data');
+
+      if (el) {
+        configData = JSON.parse(el.textContent || '{}');
+      }
+    }
+    _CONFIG_DATA = configData ?? Object.create(null);
+  }
+
+  return _CONFIG_DATA[name] || '';
+}
+
+// eslint-disable-next-line no-undef
+__webpack_public_path__ = getOption('fullStaticUrl') + '/';
+
+// Promise.allSettled polyfill, until our supported browsers implement it
+// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled
+if (Promise.allSettled === undefined) {
+  Promise.allSettled = promises =>
+    Promise.all(
+      promises.map(promise =>
+        promise.then(
+          value => ({
+            status: 'fulfilled',
+            value
+          }),
+          reason => ({
+            status: 'rejected',
+            reason
+          })
+        )
+      )
+    );
+}
+
+function loadScript(url) {
+  return new Promise((resolve, reject) => {
+    const newScript = document.createElement('script');
+    newScript.onerror = reject;
+    newScript.onload = resolve;
+    newScript.async = true;
+    document.head.appendChild(newScript);
+    newScript.src = url;
+  });
+}
+
+async function loadComponent(url, scope) {
+  await loadScript(url);
+
+  // From https://webpack.js.org/concepts/module-federation/#dynamic-remote-containers
+  await __webpack_init_sharing__('default');
+  const container = window._JUPYTERLAB[scope];
+  // Initialize the container, it may provide shared modules and may need ours
+  await container.init(__webpack_share_scopes__.default);
+}
+
+void (async function bootstrap() {
+  // This is all the data needed to load and activate plugins. This should be
+  // gathered by the server and put onto the initial page template.
+  const extension_data = getOption('federated_extensions');
+
+  // We first load all federated components so that the shared module
+  // deduplication can run and figure out which shared modules from all
+  // components should be actually used. We have to do this before importing
+  // and using the module that actually uses these components so that all
+  // dependencies are initialized.
+  let labExtensionUrl = getOption('fullLabextensionsUrl');
+  const extensions = await Promise.allSettled(
+    extension_data.map(async data => {
+      await loadComponent(
+        `${labExtensionUrl}/${data.name}/${data.load}`,
+        data.name
+      );
+    })
+  );
+
+  extensions.forEach(p => {
+    if (p.status === 'rejected') {
+      // There was an error loading the component
+      console.error(p.reason);
+    }
+  });
+
+  // Now that all federated containers are initialized with the main
+  // container, we can import the main function.
+  let main = (await import('./index.out.js')).main;
+  window.addEventListener('load', main);
+})();

dev_mode/index.js

@@ -3,58 +3,13 @@
 | Distributed under the terms of the Modified BSD License.
 |----------------------------------------------------------------------------*/
 
-import {
-  PageConfig,
-  URLExt,
-} from '@jupyterlab/coreutils';
+import { PageConfig } from '@jupyterlab/coreutils';
 
-// eslint-disable-next-line no-undef
-__webpack_public_path__ = PageConfig.getOption('fullStaticUrl') + '/';
-
-// Promise.allSettled polyfill, until our supported browsers implement it
-// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled
-if (Promise.allSettled === undefined) {
-  Promise.allSettled = promises =>
-    Promise.all(
-      promises.map(promise =>
-        promise
-          .then(value => ({
-            status: "fulfilled",
-            value,
-          }), reason => ({
-            status: "rejected",
-            reason,
-          }))
-      )
-    );
-}
 
 // This must be after the public path is set.
 // This cannot be extracted because the public path is dynamic.
 require('./imports.css');
 
-
-function loadScript(url) {
-  return new Promise((resolve, reject) => {
-    const newScript = document.createElement('script');
-    newScript.onerror = reject;
-    newScript.onload = resolve;
-    newScript.async = true;
-    document.head.appendChild(newScript);
-    newScript.src = url;
-  });
-}
-
-async function loadComponent(url, scope) {
-  await loadScript(url);
-
-  // From MIT-licensed https://github.com/module-federation/module-federation-examples/blob/af043acd6be1718ee195b2511adf6011fba4233c/advanced-api/dynamic-remotes/app1/src/App.js#L6-L12
-  await __webpack_init_sharing__('default');
-  const container = window._JUPYTERLAB[scope];
-  // Initialize the container, it may provide shared modules and may need ours
-  await container.init(__webpack_share_scopes__.default);
-}
-
 async function createModule(scope, module) {
   try {
     const factory = await window._JUPYTERLAB[scope].get(module);
@@ -68,42 +23,24 @@ async function createModule(scope, module) {
 /**
  * The main entry point for the application.
  */
-async function main() {
+export async function main() {
   var JupyterLab = require('@jupyterlab/application').JupyterLab;
   var disabled = [];
   var deferred = [];
   var ignorePlugins = [];
   var register = [];
 
-  // This is all the data needed to load and activate plugins. This should be
-  // gathered by the server and put onto the initial page template.
-  const extension_data = JSON.parse(
-    PageConfig.getOption('federated_extensions')
-  );
 
   const federatedExtensionPromises = [];
   const federatedMimeExtensionPromises = [];
   const federatedStylePromises = [];
 
-  // We first load all federated components so that the shared module
-  // deduplication can run and figure out which shared modules from all
-  // components should be actually used.
-  const extensions = await Promise.allSettled(extension_data.map( async data => {
-    await loadComponent(
-      `${URLExt.join(PageConfig.getOption('fullLabextensionsUrl'), data.name, data.load)}`,
-      data.name
-    );
-    return data;
-  }));
-
-  extensions.forEach(p => {
-    if (p.status === "rejected") {
-      // There was an error loading the component
-      console.error(p.reason);
-      return;
-    }
+  // Start initializing the federated extensions
+  const extensions = JSON.parse(
+    PageConfig.getOption('federated_extensions')
+  );
 
-    const data = p.value;
+  extensions.forEach(data => {
     if (data.extension) {
       federatedExtensionPromises.push(createModule(data.name, data.extension));
     }
@@ -262,4 +199,3 @@ async function main() {
 
 }
 
-window.addEventListener('load', main);

dev_mode/webpack.config.js

@@ -20,17 +20,10 @@ const package_data = require('./package.json');
 
 // Handle the extensions.
 const jlab = package_data.jupyterlab;
-const extensions = jlab.extensions;
-const mimeExtensions = jlab.mimeExtensions;
-const { externalExtensions } = jlab;
-const packageNames = Object.keys(mimeExtensions).concat(
-  Object.keys(extensions),
-  Object.keys(externalExtensions)
-);
-
-// go throught each external extension
-// add to mapping of extension and mime extensions, of package name
-// to path of the extension.
+const { extensions, mimeExtensions, externalExtensions } = jlab;
+
+// Add external extensions to the extensions/mimeExtensions data as
+// appropriate
 for (const key in externalExtensions) {
   const {
     jupyterlab: { extension, mimeExtension }
@@ -43,6 +36,11 @@ for (const key in externalExtensions) {
   }
 }
 
+// Deduplicated list of extension package names.
+const extensionPackages = [
+  ...new Set([...Object.keys(extensions), ...Object.keys(mimeExtensions)])
+];
+
 // Ensure a clear build directory.
 const buildDir = plib.resolve(jlab.buildDir);
 if (fs.existsSync(buildDir)) {
@@ -54,42 +52,25 @@ const outputDir = plib.resolve(jlab.outputDir);
 
 // Build the assets
 const extraConfig = Build.ensureAssets({
-  packageNames: packageNames,
+  // Deduplicate the extension package names
+  packageNames: extensionPackages,
   output: outputDir
 });
 
-// Build up singleton metadata for module federation.
-const singletons = {};
-
-package_data.jupyterlab.singletonPackages.forEach(element => {
-  singletons[element] = { singleton: true };
-});
-
-// Go through each external extension
-// add to mapping of extension and mime extensions, of package name
-// to path of the extension.
-for (const key in externalExtensions) {
-  const {
-    jupyterlab: { extension, mimeExtension }
-  } = require(`${key}/package.json`);
-  if (extension !== undefined) {
-    extensions[key] = extension === true ? '' : extension;
-  }
-  if (mimeExtension !== undefined) {
-    mimeExtensions[key] = mimeExtension === true ? '' : mimeExtension;
-  }
-}
-
-// Create the entry point file.
+// Create the entry point and other assets in build directory.
 const source = fs.readFileSync('index.js').toString();
 const template = Handlebars.compile(source);
 const extData = {
   jupyterlab_extensions: extensions,
   jupyterlab_mime_extensions: mimeExtensions
 };
-const result = template(extData);
+fs.writeFileSync(plib.join(buildDir, 'index.out.js'), template(extData));
+
+// Create the bootstrap file that loads federated extensions and calls the
+// initialization logic in index.out.js
+const entryPoint = plib.join(buildDir, 'bootstrap.js');
+fs.copySync('./bootstrap.js', entryPoint);
 
-fs.writeFileSync(plib.join(buildDir, 'index.out.js'), result);
 fs.copySync('./package.json', plib.join(buildDir, 'package.json'));
 if (outputDir !== buildDir) {
   fs.copySync(
@@ -98,11 +79,6 @@ if (outputDir !== buildDir) {
   );
 }
 
-// Make a bootstrap entrypoint
-const entryPoint = plib.join(buildDir, 'bootstrap.js');
-const bootstrap = 'import("./index.out.js");';
-fs.writeFileSync(entryPoint, bootstrap);
-
 // Set up variables for the watch mode ignore plugins
 const watched = {};
 const ignoreCache = Object.create(null);
@@ -181,19 +157,93 @@ function ignored(path) {
   return ignore;
 }
 
-const shared = {
-  ...package_data.resolutions,
-  ...singletons
-};
+// Set up module federation sharing config
+const shared = {};
+
+// Make sure any resolutions are shared
+for (let [key, requiredVersion] of Object.entries(package_data.resolutions)) {
+  shared[key] = { requiredVersion };
+}
 
-// Webpack module sharing expects version numbers, so if a resolution was a
-// filename, extract the right version number from what was installed
-Object.keys(shared).forEach(k => {
-  const v = shared[k];
-  if (typeof v === 'string' && v.startsWith('file:')) {
-    shared[k] = require(`${k}/package.json`).version;
+// Add any extension packages that are not in resolutions (i.e., installed from npm)
+for (let pkg of extensionPackages) {
+  if (shared[pkg] === undefined) {
+    shared[pkg] = { requiredVersion: require(`${pkg}/package.json`).version };
   }
-});
+}
+
+// Add dependencies and sharedPackage config from extension packages if they
+// are not already in the shared config. This means that if there is a
+// conflict, the resolutions package version is the one that is shared.
+extraShared = [];
+for (let pkg of extensionPackages) {
+  let pkgShared = {};
+  let {
+    dependencies = {},
+    jupyterlab: { sharedPackages = {} } = {}
+  } = require(`${pkg}/package.json`);
+  for (let [dep, requiredVersion] of Object.entries(dependencies)) {
+    if (!shared[dep]) {
+      pkgShared[dep] = { requiredVersion };
+    }
+  }
+
+  // Overwrite automatic dependency sharing with custom sharing config
+  for (let [pkg, config] of Object.entries(sharedPackages)) {
+    if (config === false) {
+      delete pkgShared[pkg];
+    } else {
+      if ('bundled' in config) {
+        config.import = config.bundled;
+        delete config.bundled;
+      }
+      pkgShared[pkg] = config;
+    }
+  }
+  extraShared.push(pkgShared);
+}
+
+// Now merge the extra shared config
+const mergedShare = {};
+for (let sharedConfig of extraShared) {
+  for (let [pkg, config] of Object.entries(sharedConfig)) {
+    // Do not override the basic share config from resolutions
+    if (shared[pkg]) {
+      continue;
+    }
+
+    // Add if we haven't seen the config before
+    if (!mergedShare[pkg]) {
+      mergedShare[pkg] = config;
+      continue;
+    }
+
+    // Choose between the existing config and this new config. We do not try
+    // to merge configs, which may yield a config no one wants
+    let oldConfig = mergedShare[pkg];
+
+    // if the old one has import: false, use the new one
+    if (oldConfig.import === false) {
+      mergedShare[pkg] = config;
+    }
+  }
+}
+
+Object.assign(shared, mergedShare);
+
+// Transform any file:// requiredVersion to the version number from the
+// imported package. This assumes (for simplicity) that the version we get
+// importing was installed from the file.
+for (let [key, { requiredVersion }] of Object.entries(shared)) {
+  if (requiredVersion.startsWith('file:')) {
+    shared[key].requiredVersion = require(`${key}/package.json`).version;
+  }
+}
+
+// Add singleton package information
+for (let key of jlab.singletonPackages) {
+  shared[key].singleton = true;
+}
 
 const plugins = [
   new WPPlugin.NowatchDuplicatePackageCheckerPlugin({

jupyterlab/commands.py

@@ -1127,7 +1127,7 @@ class _AppHandler(object):
                 _rmtree(staging, self.logger)
                 os.makedirs(staging)
 
-        for fname in ['index.js', 'publicpath.js',
+        for fname in ['index.js', 'bootstrap.js', 'publicpath.js',
                       'webpack.config.js',
                       'webpack.prod.config.js',
                       'webpack.prod.minimize.config.js',