Merge pull request #4224 from saulshanabrook/file-limit

Allow larger files uploads

jupyterlab/extension.py

@@ -29,9 +29,12 @@ def load_jupyter_server_extension(nbapp):
     """Load the JupyterLab server extension.
     """
     # Delay imports to speed up jlpmapp
+    from json import dumps
     from jupyterlab_launcher import add_handlers, LabConfig
     from notebook.utils import url_path_join as ujoin, url_escape
+    from notebook._version import version_info
     from tornado.ioloop import IOLoop
+    from markupsafe import Markup
     from .build_handler import build_path, Builder, BuildHandler
     from .commands import (
         get_app_dir, get_user_settings_dir, watch, ensure_dev, watch_dev,
@@ -86,6 +89,13 @@ def load_jupyter_server_extension(nbapp):
     page_config['buildCheck'] = not core_mode and not dev_mode
     page_config['token'] = nbapp.token
     page_config['devMode'] = dev_mode
+    # Export the version info tuple to a JSON array. This get's printed
+    # inside double quote marks, so we render it to a JSON string of the
+    # JSON data (so that we can call JSON.parse on the frontend on it).
+    # We also have to wrap it in `Markup` so that it isn't escaped
+    # by Jinja. Otherwise, if the version has string parts these will be
+    # escaped and then will have to be unescaped on the frontend.
+    page_config['notebookVersion'] = Markup(dumps(dumps(version_info))[1:-1])
 
     if nbapp.file_to_run and type(nbapp).__name__ == "LabApp":
         relpath = os.path.relpath(nbapp.file_to_run, nbapp.notebook_dir)

packages/coreutils/src/pageconfig.ts

@@ -156,6 +156,19 @@ namespace PageConfig {
     return getOption('token') || Private.getBodyData('jupyterApiToken');
   }
 
+  /**
+   * Get the Notebook version info [major, minor, patch].
+   */
+  export
+  function getNotebookVersion(): [number, number, number] {
+    const notebookVersion = getOption('notebookVersion');
+    if (notebookVersion === '') {
+      return [0, 0, 0];
+    }
+    return JSON.parse(notebookVersion);
+  }
+
+
   /**
    * Private page config data for the Jupyter application.
    */

packages/filebrowser/src/model.ts

@@ -2,7 +2,7 @@
 // Distributed under the terms of the Modified BSD License.
 
 import {
-  IChangedArgs, IStateDB, PathExt
+  IChangedArgs, IStateDB, PathExt, PageConfig
 } from '@jupyterlab/coreutils';
 
 import {
@@ -29,6 +29,11 @@ import {
   ISignal, Signal
 } from '@phosphor/signaling';
 
+import {
+  showDialog,
+  Dialog
+} from '@jupyterlab/apputils';
+
 
 /**
  * The duration of auto-refresh in ms.
@@ -41,6 +46,28 @@ const REFRESH_DURATION = 10000;
 const MIN_REFRESH = 1000;
 
 
+/**
+ * The maximum upload size (in bytes) for notebook version < 5.1.0
+ */
+export const LARGE_FILE_SIZE = 15 * 1024 * 1024;
+
+/**
+ * The size (in bytes) of the biggest chunk we should upload at once.
+ */
+export const CHUNK_SIZE = 1024 * 1024;
+
+
+/**
+ * An upload progress event for a file at `path`.
+ */
+export interface IUploadModel {
+  path: string;
+  /**
+   * % uploaded [0, 1)
+   */
+  progress: number;
+}
+
 /**
  * An implementation of a file browser model.
  *
@@ -74,6 +101,15 @@ class FileBrowserModel implements IDisposable {
     services.contents.fileChanged.connect(this._onFileChanged, this);
     services.sessions.runningChanged.connect(this._onRunningChanged, this);
 
+    this._unloadEventListener = (e: Event) => {
+      if (this._uploads.length > 0) {
+        const confirmationMessage = 'Files still uploading';
+
+        (e as any).returnValue = confirmationMessage;
+        return confirmationMessage;
+      }
+    };
+    window.addEventListener('beforeunload', this._unloadEventListener);
     this._scheduleUpdate();
     this._startTimer();
   }
@@ -139,6 +175,20 @@ class FileBrowserModel implements IDisposable {
     return this._isDisposed;
   }
 
+  /**
+   * A signal emitted when an upload progresses.
+   */
+  get uploadChanged(): ISignal<this, IChangedArgs<IUploadModel>> {
+    return this._uploadChanged;
+  }
+
+  /**
+   * Create an iterator over the status of all in progress uploads.
+   */
+  uploads(): IIterator<IUploadModel> {
+    return new ArrayIterator(this._uploads);
+  }
+
   /**
    * Dispose of the resources held by the model.
    */
@@ -146,6 +196,7 @@ class FileBrowserModel implements IDisposable {
     if (this.isDisposed) {
       return;
     }
+    window.removeEventListener('beforeunload', this._unloadEventListener);
     this._isDisposed = true;
     clearTimeout(this._timeoutId);
     this._sessions.length = 0;
@@ -308,76 +359,133 @@ class FileBrowserModel implements IDisposable {
    * @returns A promise containing the new file contents model.
    *
    * #### Notes
-   * This will fail to upload files that are too big to be sent in one
-   * request to the server.
+   * On Notebook version < 5.1.0, this will fail to upload files that are too
+   * big to be sent in one request to the server. On newer versions, it will
+   * ask for confirmation then upload the file in 1 MB chunks.
    */
-  upload(file: File): Promise<Contents.IModel> {
-    // Skip large files with a warning.
-    if (file.size > this._maxUploadSizeMb * 1024 * 1024) {
-      let msg = `Cannot upload file (>${this._maxUploadSizeMb} MB) `;
-      msg += `"${file.name}"`;
+  async upload(file: File): Promise<Contents.IModel> {
+    const supportsChunked = PageConfig.getNotebookVersion() >= [5, 1, 0];
+    const largeFile = file.size > LARGE_FILE_SIZE;
+    const isNotebook = file.name.indexOf('.ipynb') !== -1;
+    const canSendChunked = supportsChunked && !isNotebook;
+
+    if (largeFile && !canSendChunked) {
+      let msg = `Cannot upload file (>${LARGE_FILE_SIZE / (1024 * 1024) } MB). ${file.name}`;
       console.warn(msg);
-      return Promise.reject<Contents.IModel>(new Error(msg));
+      throw msg;
     }
 
-    return this.refresh().then(() => {
-      if (this.isDisposed) {
-        return Promise.resolve(false);
-      }
-      let item = find(this._items, i => i.name === file.name);
-      if (item) {
-        return shouldOverwrite(file.name);
-      }
-      return Promise.resolve(true);
-    }).then(value => {
-      if (value) {
-        return this._upload(file);
-      }
-      return Promise.reject('File not uploaded');
+    const err = 'File not uploaded';
+    if (largeFile && !await this._shouldUploadLarge(file)) {
+      throw 'Cancelled large file upload';
+    }
+    await this._uploadCheckDisposed();
+    await this.refresh();
+    await this._uploadCheckDisposed();
+    if (find(this._items, i => i.name === file.name) && !await shouldOverwrite(file.name)) {
+      throw err;
+    }
+    await this._uploadCheckDisposed();
+    const chunkedUpload = supportsChunked && file.size > CHUNK_SIZE;
+    return await this._upload(file, isNotebook, chunkedUpload);
+  }
+
+
+  private async _shouldUploadLarge(file: File): Promise<boolean> {
+    const {button} = await showDialog({
+      title: 'Large file size warning',
+      body: `The file size is ${Math.round(file.size / (1024 * 1024))} MB. Do you still want to upload it?`,
+      buttons: [Dialog.cancelButton(), Dialog.warnButton({ label: 'UPLOAD'})]
     });
+    return button.accept;
   }
 
   /**
    * Perform the actual upload.
    */
-  private _upload(file: File): Promise<Contents.IModel> {
+  private async _upload(file: File, isNotebook: boolean, chunked: boolean): Promise<Contents.IModel> {
     // Gather the file model parameters.
     let path = this._model.path;
     path = path ? path + '/' + file.name : file.name;
     let name = file.name;
-    let isNotebook = file.name.indexOf('.ipynb') !== -1;
     let type: Contents.ContentType = isNotebook ? 'notebook' : 'file';
     let format: Contents.FileFormat = isNotebook ? 'json' : 'base64';
 
-    // Get the file content.
-    let reader = new FileReader();
-    if (isNotebook) {
-      reader.readAsText(file);
-    } else {
-      reader.readAsArrayBuffer(file);
+    const uploadInner = async (blob: Blob, chunk?: number): Promise<Contents.IModel> => {
+      await this._uploadCheckDisposed();
+      let reader = new FileReader();
+      if (isNotebook) {
+        reader.readAsText(blob);
+      } else {
+        reader.readAsArrayBuffer(blob);
+      }
+      await new Promise((resolve, reject) => {
+        reader.onload = resolve;
+        reader.onerror = event => reject(`Failed to upload "${file.name}":` + event);
+      });
+      await this._uploadCheckDisposed();
+      let model: Partial<Contents.IModel> = {
+        type,
+        format,
+        name,
+        chunk,
+        content: Private.getContent(reader)
+      };
+      return await this.manager.services.contents.save(path, model);
+    };
+
+    if (!chunked) {
+      return await uploadInner(file);
     }
 
-    return new Promise<Contents.IModel>((resolve, reject) => {
-      reader.onload = (event: Event) => {
-        let model: Partial<Contents.IModel> = {
-          type: type,
-          format,
-          name,
-          content: Private.getContent(reader)
-        };
-
-        this.manager.services.contents.save(path, model).then(contents => {
-          resolve(contents);
-        }).catch(reject);
-      };
+    let finalModel: Contents.IModel;
 
-      reader.onerror = (event: Event) => {
-        reject(Error(`Failed to upload "${file.name}":` + event));
-      };
+    let upload = {path, progress: 0};
+    this._uploadChanged.emit({name: 'start',
+      newValue: upload,
+      oldValue: null
     });
 
+    for (let start = 0; !finalModel; start += CHUNK_SIZE) {
+      const end = start + CHUNK_SIZE;
+      const lastChunk = end >= file.size;
+      const chunk = lastChunk ? -1 : end / CHUNK_SIZE;
+
+      const newUpload = {path, progress: start / file.size};
+      this._uploads.splice(this._uploads.indexOf(upload));
+      this._uploads.push(newUpload);
+      this._uploadChanged.emit({
+        name: 'update',
+        newValue: newUpload,
+        oldValue: upload
+      });
+      upload = newUpload;
+
+      const currentModel = await uploadInner(file.slice(start, end), chunk);
+
+      if (lastChunk) {
+        finalModel = currentModel;
+      }
+    }
+
+    this._uploads.splice(this._uploads.indexOf(upload));
+    this._uploadChanged.emit({
+      name: 'finish',
+      newValue: null,
+      oldValue: upload
+    });
+
+    return finalModel;
+  }
+
+  private _uploadCheckDisposed(): Promise<void> {
+    if (this.isDisposed) {
+      return Promise.reject('Filemanager disposed. File upload canceled');
+    }
+    return Promise.resolve();
   }
 
+
   /**
    * Handle an updated contents model.
    */
@@ -478,7 +586,6 @@ class FileBrowserModel implements IDisposable {
   private _fileChanged = new Signal<this, Contents.IChangedArgs>(this);
   private _items: Contents.IModel[] = [];
   private _key: string = '';
-  private _maxUploadSizeMb = 15;
   private _model: Contents.IModel;
   private _pathChanged = new Signal<this, IChangedArgs<string>>(this);
   private _paths = new Set<string>();
@@ -494,6 +601,9 @@ class FileBrowserModel implements IDisposable {
   private _driveName: string;
   private _isDisposed = false;
   private _restored = new PromiseDelegate<void>();
+  private _uploads: IUploadModel[] = [];
+  private _uploadChanged = new Signal<this, IChangedArgs<IUploadModel>>(this);
+  private _unloadEventListener: (e: Event) => string;
 }
 
 

packages/services/src/contents/index.ts

@@ -103,6 +103,11 @@ namespace Contents {
      */
     readonly content: any;
 
+    /**
+     * The chunk of the file upload.
+     */
+    readonly chunk?: number;
+
     /**
      * The format of the file `content`.
      *

tests/test-filebrowser/package.json

@@ -20,6 +20,7 @@
     "@jupyterlab/docregistry": "^0.16.2",
     "@jupyterlab/filebrowser": "^0.16.2",
     "@jupyterlab/services": "^2.0.2",
+    "@phosphor/algorithm": "^1.1.2",
     "@phosphor/messaging": "^1.2.2",
     "@phosphor/signaling": "^1.2.2",
     "@phosphor/widgets": "^1.6.0",

tests/test-filebrowser/src/model.spec.ts

@@ -4,7 +4,7 @@
 import expect = require('expect.js');
 
 import {
-  StateDB, uuid
+  StateDB, uuid, PageConfig
 } from '@jupyterlab/coreutils';
 
 import {
@@ -20,12 +20,14 @@ import {
 } from '@jupyterlab/services';
 
 import {
-  FileBrowserModel
+  FileBrowserModel, LARGE_FILE_SIZE, CHUNK_SIZE
 } from '@jupyterlab/filebrowser';
 
 import {
   acceptDialog, dismissDialog
 } from '../../utils';
+import { ISignal } from '@phosphor/signaling';
+import { IIterator } from '@phosphor/algorithm';
 
 
 describe('filebrowser/model', () => {
@@ -346,8 +348,118 @@ describe('filebrowser/model', () => {
         model.upload(file).catch(done);
       });
 
+      describe('older notebook version', () => {
+        let prevNotebookVersion: string;
+
+        before(() => {
+          prevNotebookVersion = PageConfig.setOption('notebookVersion', JSON.stringify([5, 0, 0]));
+        });
+
+        it('should not upload large file', () => {
+          const fname = uuid() + '.html';
+          const file = new File([new ArrayBuffer(LARGE_FILE_SIZE + 1)], fname);
+          return model.upload(file).then(() => {
+            expect().fail('Upload should have failed');
+          }).catch(err => {
+            expect(err).to.be(`Cannot upload file (>15 MB). ${fname}`);
+          });
+        });
+
+        after(() => {
+          PageConfig.setOption('notebookVersion', prevNotebookVersion);
+        });
+      });
+
+      describe('newer notebook version', () => {
+        let prevNotebookVersion: string;
+
+        before(() => {
+          prevNotebookVersion = PageConfig.setOption('notebookVersion', JSON.stringify([5, 1, 0]));
+        });
+
+        it('should not upload large notebook file', () => {
+          const fname = uuid() + '.ipynb';
+          const file = new File([new ArrayBuffer(LARGE_FILE_SIZE + 1)], fname);
+          return model.upload(file).then(() => {
+            expect().fail('Upload should have failed');
+          }).catch(err => {
+            expect(err).to.be(`Cannot upload file (>15 MB). ${fname}`);
+          });
+        });
+
+        for (const size of [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, 2 * CHUNK_SIZE]) {
+          it(`should upload a large file of size ${size}`, async () => {
+            const fname = uuid() + '.txt';
+            const content = 'a'.repeat(size);
+            const file = new File([content], fname);
+            await model.upload(file);
+            const contentsModel = await model.manager.services.contents.get(fname);
+            expect(contentsModel.content).to.be(content);
+          });
+        }
+        it(`should produce progress as a large file uploads`, async () => {
+          const fname = uuid() + '.txt';
+          const file = new File([new ArrayBuffer(2 * CHUNK_SIZE)], fname);
+
+          const {cleanup, values: [start, first, second, finished]} = signalToPromises(model.uploadChanged, 4);
+
+          model.upload(file);
+          expect(iteratorToList(model.uploads())).to.eql([]);
+          expect(await start).to.eql([model, {name: 'start', oldValue: null, newValue: {path: fname, progress: 0}}]);
+          expect(iteratorToList(model.uploads())).to.eql([{path: fname, progress: 0}]);
+          expect(await first).to.eql([model, {name: 'update', oldValue: {path: fname, progress: 0}, newValue: {path: fname, progress: 0}}]);
+          expect(iteratorToList(model.uploads())).to.eql([{path: fname, progress: 0}]);
+          expect(await second).to.eql([model, {name: 'update', oldValue: {path: fname, progress: 0}, newValue: {path: fname, progress: 1 / 2}}]);
+          expect(iteratorToList(model.uploads())).to.eql([{path: fname, progress: 1 / 2}]);
+          expect(await finished).to.eql([model, {name: 'finish', oldValue: {path: fname, progress: 1 / 2}, newValue: null}]);
+          expect(iteratorToList(model.uploads())).to.eql([]);
+          cleanup();
+        });
+
+        after(() => {
+          PageConfig.setOption('notebookVersion', prevNotebookVersion);
+        });
+      });
+
     });
 
   });
 
 });
+
+/**
+ * Creates a number of promises from a signal, which each resolve to the successive values in the signal.
+ */
+function signalToPromises<T, U>(signal: ISignal<T, U>, numberValues: number): {values: Promise<[T, U]>[], cleanup: () => void} {
+  const values: Promise<[T, U]>[] = new Array(numberValues);
+  const resolvers: Array<((value: [T, U]) => void)> = new Array(numberValues);
+
+  for (let i = 0; i < numberValues; i++) {
+    values[i] = new Promise<[T, U]>(resolve => {
+      resolvers[i] = resolve;
+    });
+  }
+
+  let current = 0;
+  function slot(sender: T, args: U) {
+    resolvers[current++]([sender, args]);
+  }
+  signal.connect(slot);
+
+  function cleanup() {
+    signal.disconnect(slot);
+  }
+  return {values, cleanup};
+}
+
+
+/**
+ * Convert an IIterator into a list.
+ */
+function iteratorToList<T>(i: IIterator<T>): T[] {
+  const a: T[] = [];
+  for (let v = i.next(); v !== undefined; v = i.next()) {
+    a.push(v);
+  }
+  return a;
+}