From 3a164f34737f76796d3f6eb7094a1601a2d75997 Mon Sep 17 00:00:00 2001 From: rebornix Date: Sun, 15 May 2022 17:56:07 -0700 Subject: [PATCH 01/12] Experiment nbconvert for web. --- src/kernels/common/baseJupyterSession.ts | 11 +- .../launcher/jupyterNotebookProvider.ts | 2 + src/kernels/jupyter/session/jupyterSession.ts | 255 +++++++++++++++++- src/kernels/jupyter/types.ts | 7 +- src/kernels/types.ts | 8 +- src/platform/export/exportBase.web.ts | 121 +++++++++ src/platform/export/fileConverter.web.ts | 19 +- src/platform/serviceRegistry.web.ts | 4 +- 8 files changed, 409 insertions(+), 18 deletions(-) create mode 100644 src/platform/export/exportBase.web.ts diff --git a/src/kernels/common/baseJupyterSession.ts b/src/kernels/common/baseJupyterSession.ts index c37aa45b1ef..d14fd4e23ce 100644 --- a/src/kernels/common/baseJupyterSession.ts +++ b/src/kernels/common/baseJupyterSession.ts @@ -20,7 +20,13 @@ import { JupyterInvalidKernelError } from '../../platform/errors/jupyterInvalidK import { JupyterWaitForIdleError } from '../../platform/errors/jupyterWaitForIdleError'; import { KernelInterruptTimeoutError } from '../../platform/errors/kernelInterruptTimeoutError'; import { SessionDisposedError } from '../../platform/errors/sessionDisposedError'; -import { IJupyterSession, ISessionWithSocket, KernelConnectionMetadata, KernelSocketInformation } from '../types'; +import { + IJupyterServerSession, + IJupyterSession, + ISessionWithSocket, + KernelConnectionMetadata, + KernelSocketInformation +} from '../types'; import { ChainingExecuteRequester } from './chainingExecuteRequester'; import { getResourceType } from '../../platform/common/utils'; import { KernelProgressReporter } from '../../platform/progress/kernelProgressReporter'; @@ -127,6 +133,9 @@ export abstract class BaseJupyterSession implements IJupyterSession { traceInfo(`Unhandled message found: ${m.header.msg_type}`); }; } + isServerSession(): this is IJupyterServerSession { + return false; + } public async dispose(): Promise { await this.shutdownImplementation(false); } diff --git a/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts b/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts index ddf7a344e07..b5f5edb5ff8 100644 --- a/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts +++ b/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts @@ -55,6 +55,8 @@ export class JupyterNotebookProvider implements IJupyterNotebookProvider { }; const server = await this.serverProvider.getOrCreateServer(serverOptions); Cancellation.throwIfCanceled(options.token); + + console.log(server.connection.rootDirectory); return server.createNotebook( options.resource, options.kernelConnection, diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index 4d98599ddce..8107415cd25 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; +import * as nbformat from '@jupyterlab/nbformat'; import type { ContentsManager, Kernel, KernelSpecManager, Session, SessionManager } from '@jupyterlab/services'; import * as uuid from 'uuid/v4'; import { CancellationToken, CancellationTokenSource } from 'vscode-jsonrpc'; @@ -21,15 +22,27 @@ import { isLocalConnection, IJupyterConnection, ISessionWithSocket, - KernelActionSource + KernelActionSource, + IJupyterServerSession } from '../../types'; import { DisplayOptions } from '../../displayOptions'; -import { IJupyterBackingFileCreator, IJupyterKernelService, IJupyterRequestCreator } from '../types'; -import { Uri } from 'vscode'; +import { IBackupFile, IJupyterBackingFileCreator, IJupyterKernelService, IJupyterRequestCreator } from '../types'; +import { + NotebookCell, + NotebookCellData, + NotebookCellKind, + NotebookData, + NotebookDocument, + Uri, + workspace +} from 'vscode'; import { generateBackingIPyNbFileName } from './backingFileCreator.base'; // function is -export class JupyterSession extends BaseJupyterSession { +export class JupyterSession extends BaseJupyterSession implements IJupyterServerSession { + public override readonly kind: 'remoteJupyter' | 'localJupyter'; + private backingFile: IBackupFile | undefined; + constructor( resource: Resource, private connInfo: IJupyterConnection, @@ -53,6 +66,12 @@ export class JupyterSession extends BaseJupyterSession { workingDirectory, interruptTimeout ); + + this.kind = connInfo.localLaunch ? 'localJupyter' : 'remoteJupyter'; + } + + public override isServerSession(): this is IJupyterServerSession { + return true; } @captureTelemetry(Telemetry.WaitForIdleJupyter, undefined, true) @@ -61,6 +80,11 @@ export class JupyterSession extends BaseJupyterSession { return this.waitForIdleOnSession(this.session, timeout); } + public override async shutdown(): Promise { + await this.disposeBackingFile(); + await super.shutdown(); + } + public override get kernel(): Kernel.IKernelConnection | undefined { return this.session?.kernel || undefined; } @@ -98,6 +122,23 @@ export class JupyterSession extends BaseJupyterSession { ...this.kernelConnectionMetadata.kernelModel, model: this.kernelConnectionMetadata.kernelModel.model }) as ISessionWithSocket; + + const request = newSession.kernel?.requestExecute( + { + code: 'import os; os.getcwd()', + silent: false, + stop_on_error: false, + allow_stdin: true, + store_history: false + }, + true + ); + request!.onIOPub = (msg) => { + console.log(msg); + }; + + await request!.done; + newSession.kernelConnectionMetadata = this.kernelConnectionMetadata; newSession.kernelSocketInformation = { socket: this.requestCreator.getWebsocket(this.kernelConnectionMetadata.id), @@ -140,6 +181,8 @@ export class JupyterSession extends BaseJupyterSession { } } + // try print again + return newSession; } @@ -152,6 +195,7 @@ export class JupyterSession extends BaseJupyterSession { if (!session || !this.contentsManager || !this.sessionManager) { throw new SessionDisposedError(); } + await this.disposeBackingFile(); let result: ISessionWithSocket | undefined; let tryCount = 0; const ui = new DisplayOptions(disableUI); @@ -190,12 +234,67 @@ export class JupyterSession extends BaseJupyterSession { return promise; } + async invokeWithFileSynced(handler: (file: IBackupFile) => Promise): Promise { + if (!this.resource) { + return; + } + + const document = workspace.notebookDocuments.find( + (document) => document.uri.toString() === this.resource!.toString() + ); + + if (!document) { + return; + } + + if (!this.backingFile) { + this.backingFile = await this.backingFileCreator.createBackingFile( + this.resource, + this.workingDirectory, + this.kernelConnectionMetadata, + this.connInfo, + this.contentsManager + ); + } + + const content = await this.getContent(document); + + await this.contentsManager + .save(this.backingFile!.filePath, { + content: content, + type: 'notebook' + }) + .ignoreErrors(); + await handler({ + filePath: this.backingFile!.filePath, + dispose: this.backingFile!.dispose.bind(this.backingFile!) + }); + await this.disposeBackingFile(); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private async getContent(document: NotebookDocument): Promise { + const notebookContent = getNotebookMetadata(document); + const preferredCellLanguage = + notebookContent.metadata?.language_info?.name ?? document.cellAt(0).document.languageId; + notebookContent.cells = document + .getCells() + .map((cell) => createJupyterCellFromNotebookCell(cell, preferredCellLanguage)); + // .map(pruneCell); + + // const indentAmount = document.metadata && 'indentAmount' in document.metadata && typeof document.metadata.indentAmount === 'string' ? + // document.metadata.indentAmount : + (' '); + // ipynb always ends with a trailing new line (we add this so that SCMs do not show unnecesary changes, resulting from a missing trailing new line). + return sortObjectPropertiesRecursively(notebookContent); + } + private async createSession(options: { token: CancellationToken; ui: IDisplayOptions; }): Promise { // Create our backing file for the notebook - const backingFile = await this.backingFileCreator.createBackingFile( + this.backingFile = await this.backingFileCreator.createBackingFile( this.resource, this.workingDirectory, this.kernelConnectionMetadata, @@ -220,8 +319,8 @@ export class JupyterSession extends BaseJupyterSession { ); } catch (ex) { // If we failed to create the kernel, we need to clean up the file. - if (this.connInfo && backingFile) { - this.contentsManager.delete(backingFile.filePath).ignoreErrors(); + if (this.connInfo && this.backingFile) { + this.contentsManager.delete(this.backingFile.filePath).ignoreErrors(); } throw ex; } @@ -235,7 +334,7 @@ export class JupyterSession extends BaseJupyterSession { // Create our session options using this temporary notebook and our connection info const sessionOptions: Session.ISessionOptions = { - path: backingFile?.filePath || generateBackingIPyNbFileName(this.resource), // Name has to be unique + path: this.backingFile?.filePath || generateBackingIPyNbFileName(this.resource), // Name has to be unique kernel: { name: kernelName }, @@ -283,18 +382,150 @@ export class JupyterSession extends BaseJupyterSession { throw new JupyterSessionStartError(new Error(`No kernel created`)); }) .catch((ex) => Promise.reject(new JupyterSessionStartError(ex))) - .finally(() => { - if (this.connInfo && backingFile) { - this.contentsManager.delete(backingFile.filePath).ignoreErrors(); - } + .finally(async () => { + await this.disposeBackingFile(); }), options.token ); } + private async disposeBackingFile() { + if (this.connInfo && this.backingFile) { + await this.backingFile.dispose(); + await this.contentsManager.delete(this.backingFile.filePath).ignoreErrors(); + } + } + private logRemoteOutput(output: string) { if (!isLocalConnection(this.kernelConnectionMetadata)) { this.outputChannel.appendLine(output); } } } + +export function createJupyterCellFromNotebookCell( + vscCell: NotebookCell, + preferredLanguage: string | undefined +): nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell { + let cell: nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell; + if (vscCell.kind === NotebookCellKind.Markup) { + cell = createMarkdownCellFromNotebookCell(vscCell); + } else if (vscCell.document.languageId === 'raw') { + cell = createRawCellFromNotebookCell(vscCell); + } else { + cell = createCodeCellFromNotebookCell(vscCell, preferredLanguage); + } + return cell; +} + +function createMarkdownCellFromNotebookCell(cell: NotebookCell): nbformat.IMarkdownCell { + const cellMetadata = getCellMetadata(cell); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const markdownCell: any = { + cell_type: 'markdown', + source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), + metadata: cellMetadata?.metadata || {} // This cannot be empty. + }; + if (cellMetadata?.attachments) { + markdownCell.attachments = cellMetadata.attachments; + } + if (cellMetadata?.id) { + markdownCell.id = cellMetadata.id; + } + return markdownCell; +} + +function createRawCellFromNotebookCell(cell: NotebookCell): nbformat.IRawCell { + const cellMetadata = getCellMetadata(cell); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const rawCell: any = { + cell_type: 'raw', + source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), + metadata: cellMetadata?.metadata || {} // This cannot be empty. + }; + if (cellMetadata?.attachments) { + rawCell.attachments = cellMetadata.attachments; + } + if (cellMetadata?.id) { + rawCell.id = cellMetadata.id; + } + return rawCell; +} + +function createCodeCellFromNotebookCell(cell: NotebookCell, preferredLanguage: string | undefined): nbformat.ICodeCell { + const cellMetadata = getCellMetadata(cell); + let metadata = cellMetadata?.metadata || {}; // This cannot be empty. + if (cell.document.languageId !== preferredLanguage) { + metadata = { + ...metadata, + vscode: { + languageId: cell.document.languageId + } + }; + } else { + // cell current language is the same as the preferred cell language in the document, flush the vscode custom language id metadata + metadata.vscode = undefined; + } + metadata.trusted = true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const codeCell: any = { + cell_type: 'code', + execution_count: cell.executionSummary?.executionOrder ?? null, + source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), + outputs: [], //.map(translateCellDisplayOutput), + metadata: metadata + }; + if (cellMetadata?.id) { + codeCell.id = cellMetadata.id; + } + return codeCell; +} + +export function getCellMetadata(cell: NotebookCell | NotebookCellData) { + return cell.metadata?.custom; +} + +function splitMultilineString(source: nbformat.MultilineString): string[] { + if (Array.isArray(source)) { + return source as string[]; + } + const str = source.toString(); + if (str.length > 0) { + // Each line should be a separate entry, but end with a \n if not last entry + const arr = str.split('\n'); + return arr + .map((s, i) => { + if (i < arr.length - 1) { + return `${s}\n`; + } + return s; + }) + .filter((s) => s.length > 0); // Skip last one if empty (it's the only one that could be length 0) + } + return []; +} + +/* eslint-disable @typescript-eslint/no-explicit-any */ +export function sortObjectPropertiesRecursively(obj: any): any { + if (Array.isArray(obj)) { + return obj.map(sortObjectPropertiesRecursively); + } + if (obj !== undefined && obj !== null && typeof obj === 'object' && Object.keys(obj).length > 0) { + return Object.keys(obj) + .sort() + .reduce>((sortedObj, prop) => { + sortedObj[prop] = sortObjectPropertiesRecursively(obj[prop]); + return sortedObj; + }, {}) as any; + } + return obj; +} + +export function getNotebookMetadata(document: NotebookDocument | NotebookData) { + const notebookContent: Partial = document.metadata?.custom || {}; + notebookContent.cells = notebookContent.cells || []; + notebookContent.nbformat = notebookContent.nbformat || 4; + notebookContent.nbformat_minor = notebookContent.nbformat_minor ?? 2; + notebookContent.metadata = notebookContent.metadata || { orig_nbformat: 4 }; + return notebookContent; +} diff --git a/src/kernels/jupyter/types.ts b/src/kernels/jupyter/types.ts index 0156e9726ad..96311635974 100644 --- a/src/kernels/jupyter/types.ts +++ b/src/kernels/jupyter/types.ts @@ -241,6 +241,11 @@ export interface IJupyterServerUriStorage { setUriToRemote(uri: string, displayName: string): Promise; } +export interface IBackupFile { + dispose: () => Promise; + filePath: string; +} + export const IJupyterBackingFileCreator = Symbol('IJupyterBackingFileCreator'); export interface IJupyterBackingFileCreator { createBackingFile( @@ -249,7 +254,7 @@ export interface IJupyterBackingFileCreator { kernel: KernelConnectionMetadata, connInfo: IJupyterConnection, contentsManager: ContentsManager - ): Promise<{ dispose: () => Promise; filePath: string } | undefined>; + ): Promise; } export const IJupyterKernelService = Symbol('IJupyterKernelService'); diff --git a/src/kernels/types.ts b/src/kernels/types.ts index ab738a7f26c..9bc052bcc38 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -19,7 +19,7 @@ import type * as nbformat from '@jupyterlab/nbformat'; import { PythonEnvironment } from '../platform/pythonEnvironments/info'; import { IAsyncDisposable, IDisplayOptions, Resource } from '../platform/common/types'; import { WebSocketData } from '../platform/api/extension'; -import { IJupyterKernel } from './jupyter/types'; +import { IBackupFile, IJupyterKernel } from './jupyter/types'; import { PythonEnvironment_PythonApi } from '../platform/api/types'; export type LiveKernelModel = IJupyterKernel & @@ -262,6 +262,7 @@ export interface IJupyterSession extends IAsyncDisposable { readonly status: KernelMessage.Status; readonly kernelId: string; readonly kernelSocket: Observable; + isServerSession(): this is IJupyterServerSession; onSessionStatusChanged: Event; onDidDispose: Event; onIOPubMessage: Event; @@ -293,6 +294,11 @@ export interface IJupyterSession extends IAsyncDisposable { shutdown(): Promise; } +export interface IJupyterServerSession extends IJupyterSession { + readonly kind: 'remoteJupyter' | 'localJupyter'; + invokeWithFileSynced(handler: (file: IBackupFile) => Promise): Promise; +} + export type ISessionWithSocket = Session.ISessionConnection & { /** * The resource associated with this session. diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts new file mode 100644 index 00000000000..1c683577c60 --- /dev/null +++ b/src/platform/export/exportBase.web.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import * as nbformat from '@jupyterlab/nbformat'; +import { inject, injectable } from 'inversify'; +import { Uri, CancellationToken, workspace } from 'vscode'; +import { DisplayOptions } from '../../kernels/displayOptions'; +import { executeSilently } from '../../kernels/helpers'; +import { IKernel, IKernelProvider } from '../../kernels/types'; +import { concatMultilineString } from '../../webviews/webview-side/common'; +import { IFileSystem } from '../common/platform/types'; +import { PythonEnvironment } from '../pythonEnvironments/info'; +import { ExportFormat, INbConvertExport } from './types'; + +@injectable() +export class ExportBase implements INbConvertExport { + constructor( + @inject(IKernelProvider) private readonly kernelProvider: IKernelProvider, // @inject(IExtensions) private readonly extensions: IExtensions + @inject(IFileSystem) private readonly fs: IFileSystem + ) {} + + public async export( + _source: Uri, + _target: Uri, + _interpreter: PythonEnvironment, + _token: CancellationToken + // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function + ): Promise {} + + // @reportAction(ReportableAction.PerformingExport) + async execute( + source: Uri, + target: Uri, + _format: ExportFormat, + _interpreter: PythonEnvironment, + _token: CancellationToken + ): Promise { + const kernel = this.kernelProvider.get(source); + if (!kernel) { + // trace error + return; + } + + if (!kernel.session) { + await kernel.start(new DisplayOptions(false)); + } + + if (!kernel.session) { + return; + } + + const document = workspace.notebookDocuments.find((doc) => doc.uri.toString() === source.toString()); + + if (!document) { + return; + } + + if (kernel.session!.isServerSession()) { + await kernel.session!.invokeWithFileSynced(async (file) => { + const pwd = await this.getCWD(kernel); + console.log(pwd); + + const filePath = `${pwd}/${file.filePath}`; + + const outputs = await executeSilently( + kernel.session!, + `!jupyter nbconvert ${filePath} --to html --stdout` + ); + + if (outputs.length === 0) { + return; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const output: nbformat.IStream = outputs[0] as any; + if (output.name !== 'stdout' && output.output_type !== 'stream') { + return; + } + + const text = concatMultilineString(output.text).trim().toLowerCase(); + const headerRemoved = text + .split(/\r\n|\r|\n/g) + .slice(1) + .join('\n'); + + await this.fs.writeFile(target, headerRemoved); + }); + } else { + // no op + } + } + + private async getCWD(kernel: IKernel) { + const outputs = await executeSilently(kernel.session!, `import os;os.getcwd();`); + if (outputs.length === 0) { + return; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const output: nbformat.IExecuteResult = outputs[0] as any; + if (output.output_type !== 'execute_result') { + return undefined; + } + + return output.data['text/plain']; + } +} + +@injectable() +export class ExportToHTML extends ExportBase { + public override async export( + source: Uri, + target: Uri, + interpreter: PythonEnvironment, + token: CancellationToken + ): Promise { + await this.execute(source, target, ExportFormat.html, interpreter, token); + } +} diff --git a/src/platform/export/fileConverter.web.ts b/src/platform/export/fileConverter.web.ts index 6311061633f..748531ee117 100644 --- a/src/platform/export/fileConverter.web.ts +++ b/src/platform/export/fileConverter.web.ts @@ -8,12 +8,13 @@ import { CancellationToken, CancellationTokenSource, NotebookDocument, Uri } fro import { traceError } from '../logging'; import { PythonEnvironment } from '../pythonEnvironments/info'; import { ExportFileOpener } from './exportFileOpener'; -import { ExportFormat, IExport, IExportDialog, IFileConverter } from './types'; +import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './types'; @injectable() export class FileConverter implements IFileConverter { constructor( @inject(IExport) @named(ExportFormat.python) private readonly exportToPythonPlain: IExport, + @inject(INbConvertExport) @named(ExportFormat.html) private readonly exportToHTML: INbConvertExport, @inject(IExportDialog) private readonly filePicker: IExportDialog, @inject(ExportFileOpener) private readonly exportFileOpener: ExportFileOpener ) {} @@ -66,7 +67,8 @@ export class FileConverter implements IFileConverter { await this.performPlainExport(format, sourceDocument, target, token); await this.exportFileOpener.openFile(format, target, true); } else { - throw new Error('Method not implemented.'); + await this.performNbConvertExport(format, sourceDocument, target, token); + await this.exportFileOpener.openFile(format, target, true); } } @@ -83,6 +85,19 @@ export class FileConverter implements IFileConverter { } } + private async performNbConvertExport( + format: ExportFormat, + sourceDocument: NotebookDocument, + target: Uri, + cancelToken: CancellationToken + ) { + switch (format) { + case ExportFormat.html: + await this.exportToHTML.export(sourceDocument.uri, target, undefined, cancelToken); + break; + } + } + private async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { let target = await this.filePicker.showDialog(format, source, defaultFileName); diff --git a/src/platform/serviceRegistry.web.ts b/src/platform/serviceRegistry.web.ts index b8cbd890fc1..9d01f7b754c 100644 --- a/src/platform/serviceRegistry.web.ts +++ b/src/platform/serviceRegistry.web.ts @@ -28,12 +28,13 @@ import { IExtensionSingleActivationService } from './activation/types'; import { ExtensionSideRenderer, IExtensionSideRenderer } from '../webviews/extension-side/renderer'; import { OutputCommandListener } from './logging/outputCommandListener'; import { ExportDialog } from './export/exportDialog'; -import { ExportFormat, IExport, IExportDialog, IFileConverter } from './export/types'; +import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './export/types'; import { FileConverter } from './export/fileConverter.web'; import { ExportFileOpener } from './export/exportFileOpener'; import { ExportToPythonPlain } from './export/exportToPythonPlain.web'; import { IFileSystem } from './common/platform/types'; import { FileSystem } from './common/platform/fileSystem'; +import { ExportToHTML } from './export/exportBase.web'; export function registerTypes(context: IExtensionContext, serviceManager: IServiceManager, isDevMode: boolean) { serviceManager.addSingleton(IFileSystem, FileSystem); @@ -51,6 +52,7 @@ export function registerTypes(context: IExtensionContext, serviceManager: IServi serviceManager.addSingleton(IExportDialog, ExportDialog); serviceManager.addSingleton(IFileConverter, FileConverter); serviceManager.addSingleton(IExport, ExportToPythonPlain, ExportFormat.python); + serviceManager.addSingleton(INbConvertExport, ExportToHTML, ExportFormat.html); registerCommonTypes(serviceManager); registerApiTypes(serviceManager); From 9a50e086dc9d672afdfce917f569d64037440700 Mon Sep 17 00:00:00 2001 From: rebornix Date: Tue, 17 May 2022 21:13:47 -0700 Subject: [PATCH 02/12] Merge file converter node and web --- ...upyterInterpreterDependencyService.node.ts | 2 +- .../nbconvertExportToPythonService.node.ts | 2 +- src/kernels/jupyter/session/jupyterSession.ts | 169 +------------ src/kernels/types.ts | 2 +- src/platform/export/export.index.node.ts | 15 -- src/platform/export/exportBase.node.ts | 52 +++- src/platform/export/exportBase.web.ts | 42 ++-- src/platform/export/exportToHTML.node.ts | 17 -- src/platform/export/exportToHTML.ts | 17 ++ src/platform/export/exportToPDF.node.ts | 17 -- src/platform/export/exportToPDF.ts | 18 ++ src/platform/export/exportToPython.node.ts | 20 -- src/platform/export/exportToPython.ts | 18 ++ .../export/exportToPythonPlain.node.ts | 20 -- src/platform/export/exportToPythonPlain.ts | 15 +- .../export/exportToPythonPlain.web.ts | 20 -- src/platform/export/exportUtil.node.ts | 14 +- src/platform/export/exportUtil.ts | 35 +++ src/platform/export/fileConverter.node.ts | 228 +++--------------- src/platform/export/fileConverter.ts | 176 ++++++++++++++ src/platform/export/fileConverter.web.ts | 110 --------- src/platform/export/types.ts | 13 +- .../{decorator.node.ts => decorator.ts} | 0 ...ssReporter.node.ts => progressReporter.ts} | 2 +- src/platform/serviceRegistry.node.ts | 28 +-- src/platform/serviceRegistry.web.ts | 18 +- .../export/exportFileOpener.unit.test.ts | 2 +- .../export/exportUtil.vscode.test.ts | 4 +- .../export/fileConverter.vscode.test.ts | 13 +- .../ms-ai-tools-test/package.json | 132 +++++----- .../progress/decorators.unit.test.ts | 4 +- .../progress/progressReporter.unit.test.ts | 2 +- 32 files changed, 494 insertions(+), 733 deletions(-) delete mode 100644 src/platform/export/export.index.node.ts delete mode 100644 src/platform/export/exportToHTML.node.ts create mode 100644 src/platform/export/exportToHTML.ts delete mode 100644 src/platform/export/exportToPDF.node.ts create mode 100644 src/platform/export/exportToPDF.ts delete mode 100644 src/platform/export/exportToPython.node.ts create mode 100644 src/platform/export/exportToPython.ts delete mode 100644 src/platform/export/exportToPythonPlain.node.ts delete mode 100644 src/platform/export/exportToPythonPlain.web.ts create mode 100644 src/platform/export/exportUtil.ts create mode 100644 src/platform/export/fileConverter.ts delete mode 100644 src/platform/export/fileConverter.web.ts rename src/platform/progress/{decorator.node.ts => decorator.ts} (100%) rename src/platform/progress/{progressReporter.node.ts => progressReporter.ts} (98%) diff --git a/src/kernels/jupyter/interpreter/jupyterInterpreterDependencyService.node.ts b/src/kernels/jupyter/interpreter/jupyterInterpreterDependencyService.node.ts index 7e7c36fd3b0..13d204e016f 100644 --- a/src/kernels/jupyter/interpreter/jupyterInterpreterDependencyService.node.ts +++ b/src/kernels/jupyter/interpreter/jupyterInterpreterDependencyService.node.ts @@ -17,7 +17,7 @@ import { JupyterInstallError } from '../../../platform/errors/jupyterInstallErro import { ProductNames } from '../../installer/productNames'; import { Product, IInstaller, InstallerResponse } from '../../installer/types'; import { HelpLinks } from '../../../platform/common/constants'; -import { reportAction } from '../../../platform/progress/decorator.node'; +import { reportAction } from '../../../platform/progress/decorator'; import { ReportableAction } from '../../../platform/progress/types'; import { JupyterInterpreterDependencyResponse } from '../types'; import { IJupyterCommandFactory } from '../types.node'; diff --git a/src/kernels/jupyter/interpreter/nbconvertExportToPythonService.node.ts b/src/kernels/jupyter/interpreter/nbconvertExportToPythonService.node.ts index 04b86446c2a..9f41ea62578 100644 --- a/src/kernels/jupyter/interpreter/nbconvertExportToPythonService.node.ts +++ b/src/kernels/jupyter/interpreter/nbconvertExportToPythonService.node.ts @@ -7,7 +7,7 @@ import { inject, injectable } from 'inversify'; import { CancellationToken, Uri } from 'vscode'; import { traceError } from '../../../platform/logging'; import { IPythonExecutionFactory, IPythonDaemonExecutionService } from '../../../platform/common/process/types.node'; -import { reportAction } from '../../../platform/progress/decorator.node'; +import { reportAction } from '../../../platform/progress/decorator'; import { ReportableAction } from '../../../platform/progress/types'; import { PythonEnvironment } from '../../../platform/pythonEnvironments/info'; import { JupyterDaemonModule } from '../../../webviews/webview-side/common/constants'; diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index 8107415cd25..7da8c6798e3 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import * as nbformat from '@jupyterlab/nbformat'; import type { ContentsManager, Kernel, KernelSpecManager, Session, SessionManager } from '@jupyterlab/services'; import * as uuid from 'uuid/v4'; import { CancellationToken, CancellationTokenSource } from 'vscode-jsonrpc'; @@ -27,15 +26,7 @@ import { } from '../../types'; import { DisplayOptions } from '../../displayOptions'; import { IBackupFile, IJupyterBackingFileCreator, IJupyterKernelService, IJupyterRequestCreator } from '../types'; -import { - NotebookCell, - NotebookCellData, - NotebookCellKind, - NotebookData, - NotebookDocument, - Uri, - workspace -} from 'vscode'; +import { Uri } from 'vscode'; import { generateBackingIPyNbFileName } from './backingFileCreator.base'; // function is @@ -234,19 +225,11 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer return promise; } - async invokeWithFileSynced(handler: (file: IBackupFile) => Promise): Promise { + async invokeWithFileSynced(contents: string, handler: (file: IBackupFile) => Promise): Promise { if (!this.resource) { return; } - const document = workspace.notebookDocuments.find( - (document) => document.uri.toString() === this.resource!.toString() - ); - - if (!document) { - return; - } - if (!this.backingFile) { this.backingFile = await this.backingFileCreator.createBackingFile( this.resource, @@ -257,11 +240,9 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer ); } - const content = await this.getContent(document); - await this.contentsManager .save(this.backingFile!.filePath, { - content: content, + content: JSON.parse(contents), type: 'notebook' }) .ignoreErrors(); @@ -272,23 +253,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer await this.disposeBackingFile(); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private async getContent(document: NotebookDocument): Promise { - const notebookContent = getNotebookMetadata(document); - const preferredCellLanguage = - notebookContent.metadata?.language_info?.name ?? document.cellAt(0).document.languageId; - notebookContent.cells = document - .getCells() - .map((cell) => createJupyterCellFromNotebookCell(cell, preferredCellLanguage)); - // .map(pruneCell); - - // const indentAmount = document.metadata && 'indentAmount' in document.metadata && typeof document.metadata.indentAmount === 'string' ? - // document.metadata.indentAmount : - (' '); - // ipynb always ends with a trailing new line (we add this so that SCMs do not show unnecesary changes, resulting from a missing trailing new line). - return sortObjectPropertiesRecursively(notebookContent); - } - private async createSession(options: { token: CancellationToken; ui: IDisplayOptions; @@ -402,130 +366,3 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer } } } - -export function createJupyterCellFromNotebookCell( - vscCell: NotebookCell, - preferredLanguage: string | undefined -): nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell { - let cell: nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell; - if (vscCell.kind === NotebookCellKind.Markup) { - cell = createMarkdownCellFromNotebookCell(vscCell); - } else if (vscCell.document.languageId === 'raw') { - cell = createRawCellFromNotebookCell(vscCell); - } else { - cell = createCodeCellFromNotebookCell(vscCell, preferredLanguage); - } - return cell; -} - -function createMarkdownCellFromNotebookCell(cell: NotebookCell): nbformat.IMarkdownCell { - const cellMetadata = getCellMetadata(cell); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const markdownCell: any = { - cell_type: 'markdown', - source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), - metadata: cellMetadata?.metadata || {} // This cannot be empty. - }; - if (cellMetadata?.attachments) { - markdownCell.attachments = cellMetadata.attachments; - } - if (cellMetadata?.id) { - markdownCell.id = cellMetadata.id; - } - return markdownCell; -} - -function createRawCellFromNotebookCell(cell: NotebookCell): nbformat.IRawCell { - const cellMetadata = getCellMetadata(cell); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const rawCell: any = { - cell_type: 'raw', - source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), - metadata: cellMetadata?.metadata || {} // This cannot be empty. - }; - if (cellMetadata?.attachments) { - rawCell.attachments = cellMetadata.attachments; - } - if (cellMetadata?.id) { - rawCell.id = cellMetadata.id; - } - return rawCell; -} - -function createCodeCellFromNotebookCell(cell: NotebookCell, preferredLanguage: string | undefined): nbformat.ICodeCell { - const cellMetadata = getCellMetadata(cell); - let metadata = cellMetadata?.metadata || {}; // This cannot be empty. - if (cell.document.languageId !== preferredLanguage) { - metadata = { - ...metadata, - vscode: { - languageId: cell.document.languageId - } - }; - } else { - // cell current language is the same as the preferred cell language in the document, flush the vscode custom language id metadata - metadata.vscode = undefined; - } - metadata.trusted = true; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const codeCell: any = { - cell_type: 'code', - execution_count: cell.executionSummary?.executionOrder ?? null, - source: splitMultilineString(cell.document.getText().replace(/\r\n/g, '\n')), - outputs: [], //.map(translateCellDisplayOutput), - metadata: metadata - }; - if (cellMetadata?.id) { - codeCell.id = cellMetadata.id; - } - return codeCell; -} - -export function getCellMetadata(cell: NotebookCell | NotebookCellData) { - return cell.metadata?.custom; -} - -function splitMultilineString(source: nbformat.MultilineString): string[] { - if (Array.isArray(source)) { - return source as string[]; - } - const str = source.toString(); - if (str.length > 0) { - // Each line should be a separate entry, but end with a \n if not last entry - const arr = str.split('\n'); - return arr - .map((s, i) => { - if (i < arr.length - 1) { - return `${s}\n`; - } - return s; - }) - .filter((s) => s.length > 0); // Skip last one if empty (it's the only one that could be length 0) - } - return []; -} - -/* eslint-disable @typescript-eslint/no-explicit-any */ -export function sortObjectPropertiesRecursively(obj: any): any { - if (Array.isArray(obj)) { - return obj.map(sortObjectPropertiesRecursively); - } - if (obj !== undefined && obj !== null && typeof obj === 'object' && Object.keys(obj).length > 0) { - return Object.keys(obj) - .sort() - .reduce>((sortedObj, prop) => { - sortedObj[prop] = sortObjectPropertiesRecursively(obj[prop]); - return sortedObj; - }, {}) as any; - } - return obj; -} - -export function getNotebookMetadata(document: NotebookDocument | NotebookData) { - const notebookContent: Partial = document.metadata?.custom || {}; - notebookContent.cells = notebookContent.cells || []; - notebookContent.nbformat = notebookContent.nbformat || 4; - notebookContent.nbformat_minor = notebookContent.nbformat_minor ?? 2; - notebookContent.metadata = notebookContent.metadata || { orig_nbformat: 4 }; - return notebookContent; -} diff --git a/src/kernels/types.ts b/src/kernels/types.ts index 9bc052bcc38..1ac4ebbc06f 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -296,7 +296,7 @@ export interface IJupyterSession extends IAsyncDisposable { export interface IJupyterServerSession extends IJupyterSession { readonly kind: 'remoteJupyter' | 'localJupyter'; - invokeWithFileSynced(handler: (file: IBackupFile) => Promise): Promise; + invokeWithFileSynced(contents: string, handler: (file: IBackupFile) => Promise): Promise; } export type ISessionWithSocket = Session.ISessionConnection & { diff --git a/src/platform/export/export.index.node.ts b/src/platform/export/export.index.node.ts deleted file mode 100644 index f075f8fca16..00000000000 --- a/src/platform/export/export.index.node.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -'use strict'; - -export * from './exportBase.node'; -export * from './exportDialog'; -export * from './exportFileOpener'; -export * from './exportInterpreterFinder.node'; -export * from './exportToHTML.node'; -export * from './exportToPDF.node'; -export * from './exportToPython.node'; -export * from './exportToPythonPlain.node'; -export * from './exportUtil.node'; -export * from './fileConverter.node'; diff --git a/src/platform/export/exportBase.node.ts b/src/platform/export/exportBase.node.ts index 7d7ff89d1d7..d75fe57eda0 100644 --- a/src/platform/export/exportBase.node.ts +++ b/src/platform/export/exportBase.node.ts @@ -1,28 +1,33 @@ import { inject, injectable } from 'inversify'; import * as path from '../../platform/vscode-path/path'; -import { CancellationToken, Uri } from 'vscode'; +import { CancellationToken, NotebookDocument, Uri } from 'vscode'; import { INotebookImporter } from '../../kernels/jupyter/types'; import { IJupyterSubCommandExecutionService } from '../../kernels/jupyter/types.node'; import { IFileSystemNode } from '../common/platform/types.node'; import { IPythonExecutionFactory, IPythonExecutionService } from '../common/process/types.node'; -import { reportAction } from '../progress/decorator.node'; +import { reportAction } from '../progress/decorator'; import { ReportableAction } from '../progress/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportFormat, INbConvertExport } from './types'; +import { ExportFormat, IExportBase, INbConvertExport } from './types'; +import { ExportUtil } from './exportUtil.node'; +import { TemporaryDirectory } from '../common/platform/types'; +import { ExportInterpreterFinder } from './exportInterpreterFinder.node'; @injectable() -export class ExportBase implements INbConvertExport { +export class ExportBase implements INbConvertExport, IExportBase { constructor( @inject(IPythonExecutionFactory) protected readonly pythonExecutionFactory: IPythonExecutionFactory, @inject(IJupyterSubCommandExecutionService) protected jupyterService: IJupyterSubCommandExecutionService, @inject(IFileSystemNode) protected readonly fs: IFileSystemNode, - @inject(INotebookImporter) protected readonly importer: INotebookImporter + @inject(ExportUtil) protected readonly exportUtil: ExportUtil, + @inject(INotebookImporter) protected readonly importer: INotebookImporter, + @inject(ExportInterpreterFinder) private exportInterpreterFinder: ExportInterpreterFinder ) {} public async export( - _source: Uri, + _sourceDocument: NotebookDocument, _target: Uri, _interpreter: PythonEnvironment, _token: CancellationToken @@ -31,16 +36,40 @@ export class ExportBase implements INbConvertExport { @reportAction(ReportableAction.PerformingExport) public async executeCommand( - source: Uri, + sourceDocument: NotebookDocument, target: Uri, format: ExportFormat, - interpreter: PythonEnvironment, + interpreter: PythonEnvironment | undefined, token: CancellationToken ): Promise { if (token.isCancellationRequested) { return; } + interpreter = await this.exportInterpreterFinder.getExportInterpreter(interpreter); + + if (format === ExportFormat.python) { + const contents = await this.importer.importFromFile(sourceDocument.uri, interpreter); + await this.fs.writeFile(target, contents); + return; + } + + let contents = await this.exportUtil.getContent(sourceDocument); + + if (format === ExportFormat.pdf) { + // When exporting to PDF we need to remove any SVG output. This is due to an error + // with nbconvert and a dependency of its called InkScape. + contents = await this.exportUtil.removeSvgs(contents); + } + + /* Need to make a temp directory here, instead of just a temp file. This is because + we need to store the contents of the notebook in a file that is named the same + as what we want the title of the exported file to be. To ensure this file path will be unique + we store it in a temp directory. The name of the file matters because when + exporting to certain formats the filename is used within the exported document as the title. */ + const tempDir = await this.exportUtil.generateTempDir(); + const source = await this.makeSourceFile(target, contents, tempDir); + const service = await this.getExecutionService(source, interpreter); if (!service) { return; @@ -84,6 +113,13 @@ export class ExportBase implements INbConvertExport { } } + private async makeSourceFile(target: Uri, contents: string, tempDir: TemporaryDirectory): Promise { + // Creates a temporary file with the same base name as the target file + const fileName = path.basename(target.fsPath, path.extname(target.fsPath)); + const sourceFilePath = await this.exportUtil.makeFileInDirectory(contents, `${fileName}.ipynb`, tempDir.path); + return Uri.file(sourceFilePath); + } + protected async getExecutionService( source: Uri, interpreter: PythonEnvironment diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 1c683577c60..f1461c6fe3a 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -5,24 +5,26 @@ import * as nbformat from '@jupyterlab/nbformat'; import { inject, injectable } from 'inversify'; -import { Uri, CancellationToken, workspace } from 'vscode'; +import { Uri, CancellationToken, NotebookDocument } from 'vscode'; import { DisplayOptions } from '../../kernels/displayOptions'; import { executeSilently } from '../../kernels/helpers'; import { IKernel, IKernelProvider } from '../../kernels/types'; import { concatMultilineString } from '../../webviews/webview-side/common'; import { IFileSystem } from '../common/platform/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportFormat, INbConvertExport } from './types'; +import { ExportUtilBase } from './exportUtil'; +import { ExportFormat, IExportBase, INbConvertExport } from './types'; @injectable() -export class ExportBase implements INbConvertExport { +export class ExportBase implements INbConvertExport, IExportBase { constructor( - @inject(IKernelProvider) private readonly kernelProvider: IKernelProvider, // @inject(IExtensions) private readonly extensions: IExtensions - @inject(IFileSystem) private readonly fs: IFileSystem + @inject(IKernelProvider) private readonly kernelProvider: IKernelProvider, + @inject(IFileSystem) private readonly fs: IFileSystem, + @inject(ExportUtilBase) protected readonly exportUtil: ExportUtilBase ) {} public async export( - _source: Uri, + _sourceDocument: NotebookDocument, _target: Uri, _interpreter: PythonEnvironment, _token: CancellationToken @@ -30,14 +32,14 @@ export class ExportBase implements INbConvertExport { ): Promise {} // @reportAction(ReportableAction.PerformingExport) - async execute( - source: Uri, + async executeCommand( + sourceDocument: NotebookDocument, target: Uri, _format: ExportFormat, _interpreter: PythonEnvironment, _token: CancellationToken ): Promise { - const kernel = this.kernelProvider.get(source); + const kernel = this.kernelProvider.get(sourceDocument.uri); if (!kernel) { // trace error return; @@ -51,14 +53,10 @@ export class ExportBase implements INbConvertExport { return; } - const document = workspace.notebookDocuments.find((doc) => doc.uri.toString() === source.toString()); - - if (!document) { - return; - } - if (kernel.session!.isServerSession()) { - await kernel.session!.invokeWithFileSynced(async (file) => { + let contents = await this.exportUtil.getContent(sourceDocument); + + await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); console.log(pwd); @@ -107,15 +105,3 @@ export class ExportBase implements INbConvertExport { return output.data['text/plain']; } } - -@injectable() -export class ExportToHTML extends ExportBase { - public override async export( - source: Uri, - target: Uri, - interpreter: PythonEnvironment, - token: CancellationToken - ): Promise { - await this.execute(source, target, ExportFormat.html, interpreter, token); - } -} diff --git a/src/platform/export/exportToHTML.node.ts b/src/platform/export/exportToHTML.node.ts deleted file mode 100644 index a2a922fe212..00000000000 --- a/src/platform/export/exportToHTML.node.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { injectable } from 'inversify'; -import { CancellationToken, Uri } from 'vscode'; -import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportBase } from './exportBase.node'; -import { ExportFormat } from './types'; - -@injectable() -export class ExportToHTML extends ExportBase { - public override async export( - source: Uri, - target: Uri, - interpreter: PythonEnvironment, - token: CancellationToken - ): Promise { - await this.executeCommand(source, target, ExportFormat.html, interpreter, token); - } -} diff --git a/src/platform/export/exportToHTML.ts b/src/platform/export/exportToHTML.ts new file mode 100644 index 00000000000..9514c326b77 --- /dev/null +++ b/src/platform/export/exportToHTML.ts @@ -0,0 +1,17 @@ +import { inject, injectable } from 'inversify'; +import { CancellationToken, NotebookDocument, Uri } from 'vscode'; +import { PythonEnvironment } from '../pythonEnvironments/info'; +import { ExportFormat, IExportBase, INbConvertExport } from './types'; + +@injectable() +export class ExportToHTML implements INbConvertExport { + constructor(@inject(IExportBase) protected readonly exportBase: IExportBase) {} + public async export( + sourceDocument: NotebookDocument, + target: Uri, + interpreter: PythonEnvironment, + token: CancellationToken + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.html, interpreter, token); + } +} diff --git a/src/platform/export/exportToPDF.node.ts b/src/platform/export/exportToPDF.node.ts deleted file mode 100644 index 8d8ca858db7..00000000000 --- a/src/platform/export/exportToPDF.node.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { injectable } from 'inversify'; -import { CancellationToken, Uri } from 'vscode'; -import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportBase } from './exportBase.node'; -import { ExportFormat } from './types'; - -@injectable() -export class ExportToPDF extends ExportBase { - public override async export( - source: Uri, - target: Uri, - interpreter: PythonEnvironment, - token: CancellationToken - ): Promise { - await this.executeCommand(source, target, ExportFormat.pdf, interpreter, token); - } -} diff --git a/src/platform/export/exportToPDF.ts b/src/platform/export/exportToPDF.ts new file mode 100644 index 00000000000..97c6464921a --- /dev/null +++ b/src/platform/export/exportToPDF.ts @@ -0,0 +1,18 @@ +import { inject, injectable } from 'inversify'; +import { CancellationToken, NotebookDocument, Uri } from 'vscode'; +import { PythonEnvironment } from '../pythonEnvironments/info'; +import { ExportFormat, IExportBase, INbConvertExport } from './types'; + +@injectable() +export class ExportToPDF implements INbConvertExport { + constructor(@inject(IExportBase) protected readonly exportBase: IExportBase) {} + + public async export( + sourceDocument: NotebookDocument, + target: Uri, + interpreter: PythonEnvironment, + token: CancellationToken + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.pdf, interpreter, token); + } +} diff --git a/src/platform/export/exportToPython.node.ts b/src/platform/export/exportToPython.node.ts deleted file mode 100644 index 0ed12f6645e..00000000000 --- a/src/platform/export/exportToPython.node.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { injectable } from 'inversify'; -import { CancellationToken, Uri } from 'vscode'; -import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportBase } from './exportBase.node'; - -@injectable() -export class ExportToPython extends ExportBase { - public override async export( - source: Uri, - target: Uri, - interpreter: PythonEnvironment, - token: CancellationToken - ): Promise { - if (token.isCancellationRequested) { - return; - } - const contents = await this.importer.importFromFile(source, interpreter); - await this.fs.writeFile(target, contents); - } -} diff --git a/src/platform/export/exportToPython.ts b/src/platform/export/exportToPython.ts new file mode 100644 index 00000000000..fdd57c4d2cf --- /dev/null +++ b/src/platform/export/exportToPython.ts @@ -0,0 +1,18 @@ +import { inject, injectable } from 'inversify'; +import { CancellationToken, NotebookDocument, Uri } from 'vscode'; +import { PythonEnvironment } from '../pythonEnvironments/info'; +import { ExportFormat, IExportBase, INbConvertExport } from './types'; + +@injectable() +export class ExportToPython implements INbConvertExport { + constructor(@inject(IExportBase) protected readonly exportBase: IExportBase) {} + + public async export( + sourceDocument: NotebookDocument, + target: Uri, + interpreter: PythonEnvironment, + token: CancellationToken + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.python, interpreter, token); + } +} diff --git a/src/platform/export/exportToPythonPlain.node.ts b/src/platform/export/exportToPythonPlain.node.ts deleted file mode 100644 index 1f26e95f00e..00000000000 --- a/src/platform/export/exportToPythonPlain.node.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { inject, injectable } from 'inversify'; -import * as os from 'os'; -import { IFileSystem } from '../common/platform/types'; -import { IConfigurationService } from '../common/types'; -import { ExportToPythonPlainBase } from './exportToPythonPlain'; - -// Handles exporting a NotebookDocument to python -@injectable() -export class ExportToPythonPlain extends ExportToPythonPlainBase { - public constructor( - @inject(IFileSystem) fs: IFileSystem, - @inject(IConfigurationService) configuration: IConfigurationService - ) { - super(fs, configuration); - } - - override getEOL(): string { - return os.EOL; - } -} diff --git a/src/platform/export/exportToPythonPlain.ts b/src/platform/export/exportToPythonPlain.ts index d8f4f61dca8..751476d97a1 100644 --- a/src/platform/export/exportToPythonPlain.ts +++ b/src/platform/export/exportToPythonPlain.ts @@ -3,18 +3,20 @@ 'use strict'; +import { inject, injectable } from 'inversify'; import { CancellationToken, NotebookCell, NotebookCellKind, NotebookDocument, Uri } from 'vscode'; import { appendLineFeed } from '../../webviews/webview-side/common'; -import { IFileSystem } from '../common/platform/types'; +import { IFileSystem, IPlatformService } from '../common/platform/types'; import { IConfigurationService } from '../common/types'; import { IExport } from './types'; // Handles exporting a NotebookDocument to python -export class ExportToPythonPlainBase implements IExport { +@injectable() +export class ExportToPythonPlain implements IExport { public constructor( - private readonly fs: IFileSystem, - - protected readonly configuration: IConfigurationService + @inject(IFileSystem) private readonly fs: IFileSystem, + @inject(IConfigurationService) private readonly configuration: IConfigurationService, + @inject(IPlatformService) private platform: IPlatformService ) {} async writeFile(target: Uri, contents: string): Promise { @@ -22,6 +24,9 @@ export class ExportToPythonPlainBase implements IExport { } getEOL(): string { + if (this.platform.isWindows) { + return '\r\n'; + } return '\n'; } diff --git a/src/platform/export/exportToPythonPlain.web.ts b/src/platform/export/exportToPythonPlain.web.ts deleted file mode 100644 index 8ab1b92251a..00000000000 --- a/src/platform/export/exportToPythonPlain.web.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -'use strict'; - -import { inject, injectable } from 'inversify'; -import { IFileSystem } from '../common/platform/types'; -import { IConfigurationService } from '../common/types'; -import { ExportToPythonPlainBase } from './exportToPythonPlain'; - -// Handles exporting a NotebookDocument to python -@injectable() -export class ExportToPythonPlain extends ExportToPythonPlainBase { - public constructor( - @inject(IFileSystem) fs: IFileSystem, - @inject(IConfigurationService) configuration: IConfigurationService - ) { - super(fs, configuration); - } -} diff --git a/src/platform/export/exportUtil.node.ts b/src/platform/export/exportUtil.node.ts index d86ac131a9d..8aabca428d3 100644 --- a/src/platform/export/exportUtil.node.ts +++ b/src/platform/export/exportUtil.node.ts @@ -3,14 +3,17 @@ import { inject, injectable } from 'inversify'; import * as os from 'os'; import * as path from '../../platform/vscode-path/path'; import * as uuid from 'uuid/v4'; -import { Uri } from 'vscode'; import { TemporaryDirectory } from '../common/platform/types'; import { IFileSystemNode } from '../common/platform/types.node'; import { sleep } from '../common/utils/async'; +import { ExportUtilBase } from './exportUtil'; +import { IExtensions } from '../common/types'; @injectable() -export class ExportUtil { - constructor(@inject(IFileSystemNode) private fs: IFileSystemNode) {} +export class ExportUtil extends ExportUtilBase { + constructor(@inject(IFileSystemNode) private fs: IFileSystemNode, @inject(IExtensions) extensions: IExtensions) { + super(extensions); + } public async generateTempDir(): Promise { const resultDir = path.join(os.tmpdir(), uuid()); @@ -44,8 +47,7 @@ export class ExportUtil { return newFilePath; } - public async removeSvgs(source: Uri) { - const model = await this.fs.readFile(source); + public async removeSvgs(model: string) { const content = JSON.parse(model) as nbformat.INotebookContent; for (const cell of content.cells) { const outputs = 'outputs' in cell ? (cell.outputs as nbformat.IOutput[]) : undefined; @@ -53,7 +55,7 @@ export class ExportUtil { this.removeSvgFromOutputs(outputs); } } - await this.fs.writeFile(source, JSON.stringify(content, undefined, 4)); + return JSON.stringify(content, undefined, 4); } private removeSvgFromOutputs(outputs: nbformat.IOutput[]) { diff --git a/src/platform/export/exportUtil.ts b/src/platform/export/exportUtil.ts new file mode 100644 index 00000000000..ad87a11aed4 --- /dev/null +++ b/src/platform/export/exportUtil.ts @@ -0,0 +1,35 @@ +import { inject, injectable } from 'inversify'; +import { NotebookCellData, NotebookData, NotebookDocument } from 'vscode'; +import { IExtensions } from '../common/types'; + +@injectable() +export class ExportUtilBase { + constructor(@inject(IExtensions) private readonly extensions: IExtensions) {} + + async getContent(document: NotebookDocument): Promise { + const serializerApi = this.extensions.getExtension<{ exportNotebook: (notebook: NotebookData) => string }>( + 'vscode.ipynb' + ); + if (!serializerApi) { + throw new Error( + 'Unable to export notebook as the built-in vscode.ipynb extension is currently unavailable.' + ); + } + // Via the interactive window export this might not be activated + if (!serializerApi.isActive) { + await serializerApi.activate(); + } + + const cells = document.getCells(); + const cellData = cells.map((c) => { + const data = new NotebookCellData(c.kind, c.document.getText(), c.document.languageId); + data.metadata = c.metadata; + data.mime = c.mime; + data.outputs = [...c.outputs]; + return data; + }); + const notebookData = new NotebookData(cellData); + notebookData.metadata = document.metadata; + return serializerApi.exports.exportNotebook(notebookData); + } +} diff --git a/src/platform/export/fileConverter.node.ts b/src/platform/export/fileConverter.node.ts index 05dccb722c8..2bac13bc0a6 100644 --- a/src/platform/export/fileConverter.node.ts +++ b/src/platform/export/fileConverter.node.ts @@ -1,104 +1,42 @@ import { inject, injectable, named } from 'inversify'; -import * as path from '../../platform/vscode-path/path'; -import { CancellationToken, NotebookCellData, NotebookData, NotebookDocument, Uri, workspace } from 'vscode'; +import { CancellationToken, NotebookDocument, Uri } from 'vscode'; import { IApplicationShell } from '../common/application/types'; -import { traceError } from '../logging'; -import { TemporaryDirectory } from '../common/platform/types'; -import { IConfigurationService, IExtensions } from '../common/types'; -import * as localize from '../common/utils/localize'; +import { IConfigurationService } from '../common/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; -import { sendTelemetryEvent } from '../../telemetry'; -import { ProgressReporter } from '../progress/progressReporter.node'; +import { ProgressReporter } from '../progress/progressReporter'; import { ExportFileOpener } from './exportFileOpener'; -import { ExportInterpreterFinder } from './exportInterpreterFinder.node'; -import { ExportUtil } from './exportUtil.node'; import { ExportFormat, INbConvertExport, IExportDialog, IFileConverter, IExport } from './types'; -import { Telemetry } from '../common/constants'; import { IFileSystemNode } from '../common/platform/types.node'; +import { FileConverter as FileConverterBase } from './fileConverter'; // Class is responsible for file conversions (ipynb, py, pdf, html) and managing nb convert for some of those conversions @injectable() -export class FileConverter implements IFileConverter { +export class FileConverter extends FileConverterBase implements IFileConverter { constructor( - @inject(INbConvertExport) @named(ExportFormat.pdf) private readonly exportToPDF: INbConvertExport, - @inject(INbConvertExport) @named(ExportFormat.html) private readonly exportToHTML: INbConvertExport, - @inject(INbConvertExport) @named(ExportFormat.python) private readonly exportToPython: INbConvertExport, - @inject(IExport) @named(ExportFormat.python) private readonly exportToPythonPlain: IExport, - @inject(IFileSystemNode) private readonly fs: IFileSystemNode, - @inject(IExportDialog) private readonly filePicker: IExportDialog, - @inject(ProgressReporter) private readonly progressReporter: ProgressReporter, - @inject(ExportUtil) private readonly exportUtil: ExportUtil, - @inject(IApplicationShell) private readonly applicationShell: IApplicationShell, - @inject(ExportFileOpener) private readonly exportFileOpener: ExportFileOpener, - @inject(ExportInterpreterFinder) private exportInterpreterFinder: ExportInterpreterFinder, - @inject(IExtensions) private readonly extensions: IExtensions, - @inject(IConfigurationService) private readonly configuration: IConfigurationService - ) {} - - // Import a notebook file on disk to a .py file - public async importIpynb(source: Uri): Promise { - const reporter = this.progressReporter.createProgressIndicator(localize.DataScience.importingIpynb(), true); - let nbDoc; - try { - // Open the source as a NotebookDocument, note that this doesn't actually show an editor, and we don't need - // a specific close action as VS Code owns the lifetime - nbDoc = await workspace.openNotebookDocument(source); - await this.exportImpl(ExportFormat.python, nbDoc, reporter.token); - } finally { - reporter.dispose(); - } - } - - public async export( - format: ExportFormat, - sourceDocument: NotebookDocument, - defaultFileName?: string, - candidateInterpreter?: PythonEnvironment - ): Promise { - const reporter = this.progressReporter.createProgressIndicator( - localize.DataScience.exportingToFormat().format(format.toString()), - true + @inject(INbConvertExport) @named(ExportFormat.pdf) exportToPDF: INbConvertExport, + @inject(INbConvertExport) @named(ExportFormat.html) exportToHTML: INbConvertExport, + @inject(INbConvertExport) @named(ExportFormat.python) exportToPython: INbConvertExport, + @inject(IExport) @named(ExportFormat.python) exportToPythonPlain: IExport, + @inject(IExportDialog) filePicker: IExportDialog, + @inject(ProgressReporter) progressReporter: ProgressReporter, + @inject(IApplicationShell) applicationShell: IApplicationShell, + @inject(ExportFileOpener) exportFileOpener: ExportFileOpener, + @inject(IConfigurationService) readonly configuration: IConfigurationService, + @inject(IFileSystemNode) readonly fs: IFileSystemNode + ) { + super( + exportToPythonPlain, + exportToPDF, + exportToHTML, + exportToPython, + filePicker, + progressReporter, + applicationShell, + exportFileOpener ); - - try { - await this.exportImpl(format, sourceDocument, reporter.token, defaultFileName, candidateInterpreter); - } finally { - reporter.dispose(); - } - - if (reporter.token.isCancellationRequested) { - sendTelemetryEvent(Telemetry.ExportNotebookAs, undefined, { format: format, cancelled: true }); - return; - } } - public async exportImpl( - format: ExportFormat, - sourceDocument: NotebookDocument, - token: CancellationToken, - defaultFileName?: string, - candidateInterpreter?: PythonEnvironment - ): Promise { - let target; - try { - target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (!target) { - return; - } - await this.performExport(format, sourceDocument, target, token, candidateInterpreter); - } catch (e) { - traceError('Export failed', e); - sendTelemetryEvent(Telemetry.ExportNotebookAsFailed, undefined, { format: format }); - - if (format === ExportFormat.pdf) { - traceError(localize.DataScience.exportToPDFDependencyMessage()); - } - - this.showExportFailed(localize.DataScience.exportFailedGeneralMessage()); - } - } - - private async performExport( + override async performExport( format: ExportFormat, sourceDocument: NotebookDocument, target: Uri, @@ -111,52 +49,17 @@ export class FileConverter implements IFileConverter { // Unless selected by the setting use plain conversion for python script convert await this.performPlainExport(format, sourceDocument, target, token); } else { - // For all others (or if 'nbconvert' set for python export method) use nbconvert path - // Get the interpreter to use for the export, checking the candidate interpreter first - const exportInterpreter = await this.exportInterpreterFinder.getExportInterpreter(candidateInterpreter); - const contents = await this.getContent(sourceDocument); - await this.performNbConvertExport(format, contents, target, exportInterpreter, token); + await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); } await this.exportFileOpener.openFile(format, target); } - private async performPlainExport( - format: ExportFormat, - sourceDocument: NotebookDocument, - target: Uri, - cancelToken: CancellationToken - ) { - switch (format) { - case ExportFormat.python: - await this.exportToPythonPlain.export(sourceDocument, target, cancelToken); - break; - } - } - - private async performNbConvertExport( + override async getTargetFile( format: ExportFormat, - contents: string, - target: Uri, - interpreter: PythonEnvironment, - cancelToken: CancellationToken - ) { - /* Need to make a temp directory here, instead of just a temp file. This is because - we need to store the contents of the notebook in a file that is named the same - as what we want the title of the exported file to be. To ensure this file path will be unique - we store it in a temp directory. The name of the file matters because when - exporting to certain formats the filename is used within the exported document as the title. */ - const tempDir = await this.exportUtil.generateTempDir(); - const source = await this.makeSourceFile(target, contents, tempDir); - - try { - await this.exportToFormat(source, target, format, interpreter, cancelToken); - } finally { - tempDir.dispose(); - } - } - - private async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { + source: Uri, + defaultFileName?: string + ): Promise { let target; if (format !== ExportFormat.python) { @@ -167,73 +70,4 @@ export class FileConverter implements IFileConverter { return target; } - - private async makeSourceFile(target: Uri, contents: string, tempDir: TemporaryDirectory): Promise { - // Creates a temporary file with the same base name as the target file - const fileName = path.basename(target.fsPath, path.extname(target.fsPath)); - const sourceFilePath = await this.exportUtil.makeFileInDirectory(contents, `${fileName}.ipynb`, tempDir.path); - return Uri.file(sourceFilePath); - } - - private showExportFailed(msg: string) { - // eslint-disable-next-line - this.applicationShell.showErrorMessage(`${localize.DataScience.failedExportMessage()} ${msg}`).then(); - } - - private async exportToFormat( - source: Uri, - target: Uri, - format: ExportFormat, - interpreter: PythonEnvironment, - cancelToken: CancellationToken - ) { - if (format === ExportFormat.pdf) { - // When exporting to PDF we need to remove any SVG output. This is due to an error - // with nbconvert and a dependency of its called InkScape. - await this.exportUtil.removeSvgs(source); - } - - switch (format) { - case ExportFormat.python: - await this.exportToPython.export(source, target, interpreter, cancelToken); - break; - - case ExportFormat.pdf: - await this.exportToPDF.export(source, target, interpreter, cancelToken); - break; - - case ExportFormat.html: - await this.exportToHTML.export(source, target, interpreter, cancelToken); - break; - - default: - break; - } - } - private async getContent(document: NotebookDocument): Promise { - const serializerApi = this.extensions.getExtension<{ exportNotebook: (notebook: NotebookData) => string }>( - 'vscode.ipynb' - ); - if (!serializerApi) { - throw new Error( - 'Unable to export notebook as the built-in vscode.ipynb extension is currently unavailable.' - ); - } - // Via the interactive window export this might not be activated - if (!serializerApi.isActive) { - await serializerApi.activate(); - } - - const cells = document.getCells(); - const cellData = cells.map((c) => { - const data = new NotebookCellData(c.kind, c.document.getText(), c.document.languageId); - data.metadata = c.metadata; - data.mime = c.mime; - data.outputs = [...c.outputs]; - return data; - }); - const notebookData = new NotebookData(cellData); - notebookData.metadata = document.metadata; - return serializerApi.exports.exportNotebook(notebookData); - } } diff --git a/src/platform/export/fileConverter.ts b/src/platform/export/fileConverter.ts new file mode 100644 index 00000000000..feb830493d1 --- /dev/null +++ b/src/platform/export/fileConverter.ts @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable, named } from 'inversify'; +import { CancellationToken, NotebookDocument, Uri, workspace } from 'vscode'; +import { sendTelemetryEvent } from '../../telemetry'; +import { IApplicationShell } from '../common/application/types'; +import { Telemetry } from '../common/constants'; +import * as localize from '../common/utils/localize'; +import { traceError } from '../logging'; +import { ProgressReporter } from '../progress/progressReporter'; +import { PythonEnvironment } from '../pythonEnvironments/info'; +import { ExportFileOpener } from './exportFileOpener'; +import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './types'; + +@injectable() +export class FileConverter implements IFileConverter { + constructor( + @inject(IExport) @named(ExportFormat.python) private readonly exportToPythonPlain: IExport, + @inject(INbConvertExport) @named(ExportFormat.pdf) private readonly exportToPDF: INbConvertExport, + @inject(INbConvertExport) @named(ExportFormat.html) private readonly exportToHTML: INbConvertExport, + @inject(INbConvertExport) @named(ExportFormat.python) private readonly exportToPython: INbConvertExport, + @inject(IExportDialog) protected readonly filePicker: IExportDialog, + @inject(ProgressReporter) private readonly progressReporter: ProgressReporter, + @inject(IApplicationShell) private readonly applicationShell: IApplicationShell, + @inject(ExportFileOpener) protected readonly exportFileOpener: ExportFileOpener + ) {} + + async importIpynb(source: Uri): Promise { + const reporter = this.progressReporter.createProgressIndicator(localize.DataScience.importingIpynb(), true); + let nbDoc; + try { + // Open the source as a NotebookDocument, note that this doesn't actually show an editor, and we don't need + // a specific close action as VS Code owns the lifetime + nbDoc = await workspace.openNotebookDocument(source); + await this.exportImpl(ExportFormat.python, nbDoc, reporter.token); + } finally { + reporter.dispose(); + } + } + + async export( + format: ExportFormat, + sourceDocument: NotebookDocument, + defaultFileName?: string | undefined, + candidateInterpreter?: PythonEnvironment + ): Promise { + const reporter = this.progressReporter.createProgressIndicator( + localize.DataScience.exportingToFormat().format(format.toString()), + true + ); + + try { + await this.exportImpl(format, sourceDocument, reporter.token, defaultFileName, candidateInterpreter); + } finally { + reporter.dispose(); + } + + if (reporter.token.isCancellationRequested) { + sendTelemetryEvent(Telemetry.ExportNotebookAs, undefined, { format: format, cancelled: true }); + return; + } + } + + public async exportImpl( + format: ExportFormat, + sourceDocument: NotebookDocument, + token: CancellationToken, + defaultFileName?: string, + candidateInterpreter?: PythonEnvironment + ): Promise { + let target; + try { + target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); + if (!target) { + return; + } + await this.performExport(format, sourceDocument, target, token, candidateInterpreter); + } catch (e) { + traceError('Export failed', e); + sendTelemetryEvent(Telemetry.ExportNotebookAsFailed, undefined, { format: format }); + + if (format === ExportFormat.pdf) { + traceError(localize.DataScience.exportToPDFDependencyMessage()); + } + + this.showExportFailed(localize.DataScience.exportFailedGeneralMessage()); + } + } + + protected async performExport( + format: ExportFormat, + sourceDocument: NotebookDocument, + target: Uri, + token: CancellationToken, + candidateInterpreter?: PythonEnvironment + ) { + // For web, we perform plain export for Python + if (format === ExportFormat.python) { + // Unless selected by the setting use plain conversion for python script convert + await this.performPlainExport(format, sourceDocument, target, token); + } else { + await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); + } + + await this.exportFileOpener.openFile(format, target, true); + } + + protected async performPlainExport( + format: ExportFormat, + sourceDocument: NotebookDocument, + target: Uri, + cancelToken: CancellationToken + ) { + switch (format) { + case ExportFormat.python: + await this.exportToPythonPlain.export(sourceDocument, target, cancelToken); + break; + } + } + + protected async performNbConvertExport( + sourceDocument: NotebookDocument, + format: ExportFormat, + target: Uri, + interpreter: PythonEnvironment | undefined, + cancelToken: CancellationToken + ) { + try { + await this.exportToFormat(sourceDocument, target, format, interpreter, cancelToken); + } finally { + } + } + + protected async getTargetFile( + format: ExportFormat, + source: Uri, + defaultFileName?: string + ): Promise { + let target = await this.filePicker.showDialog(format, source, defaultFileName); + + return target; + } + + protected async exportToFormat( + sourceDocument: NotebookDocument, + target: Uri, + format: ExportFormat, + interpreter: PythonEnvironment | undefined, + cancelToken: CancellationToken + ) { + switch (format) { + case ExportFormat.python: + await this.exportToPython.export(sourceDocument, target, interpreter, cancelToken); + break; + + case ExportFormat.pdf: + await this.exportToPDF.export(sourceDocument, target, interpreter, cancelToken); + break; + + case ExportFormat.html: + await this.exportToHTML.export(sourceDocument, target, interpreter, cancelToken); + break; + + default: + break; + } + } + + private showExportFailed(msg: string) { + // eslint-disable-next-line + this.applicationShell.showErrorMessage(`${localize.DataScience.failedExportMessage()} ${msg}`).then(); + } +} diff --git a/src/platform/export/fileConverter.web.ts b/src/platform/export/fileConverter.web.ts deleted file mode 100644 index 748531ee117..00000000000 --- a/src/platform/export/fileConverter.web.ts +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -'use strict'; - -import { inject, injectable, named } from 'inversify'; -import { CancellationToken, CancellationTokenSource, NotebookDocument, Uri } from 'vscode'; -import { traceError } from '../logging'; -import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportFileOpener } from './exportFileOpener'; -import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './types'; - -@injectable() -export class FileConverter implements IFileConverter { - constructor( - @inject(IExport) @named(ExportFormat.python) private readonly exportToPythonPlain: IExport, - @inject(INbConvertExport) @named(ExportFormat.html) private readonly exportToHTML: INbConvertExport, - @inject(IExportDialog) private readonly filePicker: IExportDialog, - @inject(ExportFileOpener) private readonly exportFileOpener: ExportFileOpener - ) {} - - async export( - format: ExportFormat, - sourceDocument: NotebookDocument, - defaultFileName?: string | undefined, - candidateInterpreter?: PythonEnvironment - ): Promise { - try { - const cancellation = new CancellationTokenSource(); - await this.exportImpl(format, sourceDocument, cancellation.token, defaultFileName, candidateInterpreter); - cancellation.dispose(); - } finally { - } - - return; - } - - public async exportImpl( - format: ExportFormat, - sourceDocument: NotebookDocument, - token: CancellationToken, - defaultFileName?: string, - candidateInterpreter?: PythonEnvironment - ): Promise { - let target; - try { - target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (!target) { - return; - } - await this.performExport(format, sourceDocument, target, token, candidateInterpreter); - } catch (e) { - traceError('Export failed', e); - } - } - - private async performExport( - format: ExportFormat, - sourceDocument: NotebookDocument, - target: Uri, - token: CancellationToken, - _candidateInterpreter?: PythonEnvironment - ) { - // For web, we perform plain export for Python - if (format === ExportFormat.python) { - // Unless selected by the setting use plain conversion for python script convert - await this.performPlainExport(format, sourceDocument, target, token); - await this.exportFileOpener.openFile(format, target, true); - } else { - await this.performNbConvertExport(format, sourceDocument, target, token); - await this.exportFileOpener.openFile(format, target, true); - } - } - - private async performPlainExport( - format: ExportFormat, - sourceDocument: NotebookDocument, - target: Uri, - cancelToken: CancellationToken - ) { - switch (format) { - case ExportFormat.python: - await this.exportToPythonPlain.export(sourceDocument, target, cancelToken); - break; - } - } - - private async performNbConvertExport( - format: ExportFormat, - sourceDocument: NotebookDocument, - target: Uri, - cancelToken: CancellationToken - ) { - switch (format) { - case ExportFormat.html: - await this.exportToHTML.export(sourceDocument.uri, target, undefined, cancelToken); - break; - } - } - - private async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { - let target = await this.filePicker.showDialog(format, source, defaultFileName); - - return target; - } - - importIpynb(_source: Uri): Promise { - throw new Error('Method not implemented.'); - } -} diff --git a/src/platform/export/types.ts b/src/platform/export/types.ts index 8b43802918c..691048fd693 100644 --- a/src/platform/export/types.ts +++ b/src/platform/export/types.ts @@ -22,13 +22,24 @@ export interface IFileConverter { export const INbConvertExport = Symbol('INbConvertExport'); export interface INbConvertExport { export( - source: Uri, + sourceDocument: NotebookDocument, target: Uri, interpreter: PythonEnvironment | undefined, token: CancellationToken ): Promise; } +export const IExportBase = Symbol('IExportBase'); +export interface IExportBase { + executeCommand( + sourceDocument: NotebookDocument, + target: Uri, + format: ExportFormat, + interpreter: PythonEnvironment, + token: CancellationToken + ): Promise; +} + export const IExport = Symbol('IExport'); export interface IExport { export(sourceDocument: NotebookDocument, target: Uri, token: CancellationToken): Promise; diff --git a/src/platform/progress/decorator.node.ts b/src/platform/progress/decorator.ts similarity index 100% rename from src/platform/progress/decorator.node.ts rename to src/platform/progress/decorator.ts diff --git a/src/platform/progress/progressReporter.node.ts b/src/platform/progress/progressReporter.ts similarity index 98% rename from src/platform/progress/progressReporter.node.ts rename to src/platform/progress/progressReporter.ts index 519f5b0544c..90d82c0e53c 100644 --- a/src/platform/progress/progressReporter.node.ts +++ b/src/platform/progress/progressReporter.ts @@ -9,7 +9,7 @@ import { IApplicationShell } from '../common/application/types'; import { IDisposable } from '../common/types'; import { createDeferred } from '../common/utils/async'; import { noop } from '../common/utils/misc'; -import { registerReporter } from './decorator.node'; +import { registerReporter } from './decorator'; import { getUserMessageForAction } from './messages'; import { IProgressReporter, Progress, ReportableAction } from './types'; diff --git a/src/platform/serviceRegistry.node.ts b/src/platform/serviceRegistry.node.ts index 899dc870813..032908667f3 100644 --- a/src/platform/serviceRegistry.node.ts +++ b/src/platform/serviceRegistry.node.ts @@ -37,22 +37,20 @@ import { DebuggingManager } from './debugger/jupyter/debuggingManager.node'; import { IDebugLocationTracker, IDebuggingManager } from './debugger/types'; import { DataScienceErrorHandler } from './errors/errorHandler'; import { IDataScienceErrorHandler } from './errors/types'; -import { - ExportBase, - ExportDialog, - ExportFileOpener, - ExportInterpreterFinder, - ExportToHTML, - ExportToPDF, - ExportToPython, - ExportToPythonPlain, - ExportUtil, - FileConverter -} from './export/export.index.node'; -import { IFileConverter, INbConvertExport, ExportFormat, IExport, IExportDialog } from './export/types'; +import { ExportBase } from './export/exportBase.node'; +import { ExportDialog } from './export/exportDialog'; +import { ExportFileOpener } from './export/exportFileOpener'; +import { ExportInterpreterFinder } from './export/exportInterpreterFinder.node'; +import { ExportToHTML } from './export/exportToHTML'; +import { ExportToPDF } from './export/exportToPDF'; +import { ExportToPython } from './export/exportToPython'; +import { ExportToPythonPlain } from './export/exportToPythonPlain'; +import { ExportUtil } from './export/exportUtil.node'; +import { FileConverter } from './export/fileConverter.node'; +import { IFileConverter, INbConvertExport, ExportFormat, IExport, IExportDialog, IExportBase } from './export/types'; import { GitHubIssueCommandListener } from './logging/gitHubIssueCommandListener.node'; import { KernelProgressReporter } from './progress/kernelProgressReporter'; -import { ProgressReporter } from './progress/progressReporter.node'; +import { ProgressReporter } from './progress/progressReporter'; import { StatusProvider } from './progress/statusProvider'; import { IStatusProvider } from './progress/types'; import { ApplicationShell } from './common/application/applicationShell'; @@ -109,6 +107,8 @@ export function registerTypes(context: IExtensionContext, serviceManager: IServi serviceManager.addSingleton(IFileConverter, FileConverter); serviceManager.addSingleton(ExportInterpreterFinder, ExportInterpreterFinder); serviceManager.addSingleton(ExportFileOpener, ExportFileOpener); + + serviceManager.addSingleton(IExportBase, ExportBase); serviceManager.addSingleton(INbConvertExport, ExportToPDF, ExportFormat.pdf); serviceManager.addSingleton(INbConvertExport, ExportToHTML, ExportFormat.html); serviceManager.addSingleton(INbConvertExport, ExportToPython, ExportFormat.python); diff --git a/src/platform/serviceRegistry.web.ts b/src/platform/serviceRegistry.web.ts index 9d01f7b754c..86d5e29e110 100644 --- a/src/platform/serviceRegistry.web.ts +++ b/src/platform/serviceRegistry.web.ts @@ -18,6 +18,7 @@ import { registerTypes as registerActivationTypes } from './activation/serviceRe import { registerTypes as registerDevToolTypes } from './devTools/serviceRegistry'; import { IConfigurationService, IDataScienceCommandListener, IExtensionContext } from './common/types'; import { IServiceManager } from './ioc/types'; +import { ProgressReporter } from './progress/progressReporter'; import { StatusProvider } from './progress/statusProvider'; import { IStatusProvider } from './progress/types'; import { WorkspaceService } from './common/application/workspace.web'; @@ -28,13 +29,17 @@ import { IExtensionSingleActivationService } from './activation/types'; import { ExtensionSideRenderer, IExtensionSideRenderer } from '../webviews/extension-side/renderer'; import { OutputCommandListener } from './logging/outputCommandListener'; import { ExportDialog } from './export/exportDialog'; -import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './export/types'; -import { FileConverter } from './export/fileConverter.web'; +import { ExportFormat, IExport, IExportBase, IExportDialog, IFileConverter, INbConvertExport } from './export/types'; +import { FileConverter } from './export/fileConverter'; import { ExportFileOpener } from './export/exportFileOpener'; -import { ExportToPythonPlain } from './export/exportToPythonPlain.web'; +import { ExportToPythonPlain } from './export/exportToPythonPlain'; import { IFileSystem } from './common/platform/types'; import { FileSystem } from './common/platform/fileSystem'; -import { ExportToHTML } from './export/exportBase.web'; +import { ExportBase } from './export/exportBase.web'; +import { ExportUtilBase } from './export/exportUtil'; +import { ExportToHTML } from './export/exportToHTML'; +import { ExportToPDF } from './export/exportToPDF'; +import { ExportToPython } from './export/exportToPython'; export function registerTypes(context: IExtensionContext, serviceManager: IServiceManager, isDevMode: boolean) { serviceManager.addSingleton(IFileSystem, FileSystem); @@ -49,10 +54,15 @@ export function registerTypes(context: IExtensionContext, serviceManager: IServi serviceManager.addSingletonInstance(IExtensionSideRenderer, new ExtensionSideRenderer()); serviceManager.addSingleton(IDataScienceCommandListener, OutputCommandListener); serviceManager.addSingleton(ExportFileOpener, ExportFileOpener); + serviceManager.addSingleton(IExportBase, ExportBase); serviceManager.addSingleton(IExportDialog, ExportDialog); + serviceManager.addSingleton(ProgressReporter, ProgressReporter); serviceManager.addSingleton(IFileConverter, FileConverter); serviceManager.addSingleton(IExport, ExportToPythonPlain, ExportFormat.python); serviceManager.addSingleton(INbConvertExport, ExportToHTML, ExportFormat.html); + serviceManager.addSingleton(INbConvertExport, ExportToPDF, ExportFormat.pdf); + serviceManager.addSingleton(INbConvertExport, ExportToPython, ExportFormat.python); + serviceManager.addSingleton(ExportUtilBase, ExportUtilBase); registerCommonTypes(serviceManager); registerApiTypes(serviceManager); diff --git a/src/test/datascience/export/exportFileOpener.unit.test.ts b/src/test/datascience/export/exportFileOpener.unit.test.ts index c9e607be35a..d14c7a2a6a3 100644 --- a/src/test/datascience/export/exportFileOpener.unit.test.ts +++ b/src/test/datascience/export/exportFileOpener.unit.test.ts @@ -10,7 +10,7 @@ import { IFileSystem } from '../../../platform/common/platform/types.node'; import { IBrowserService, IDisposable } from '../../../platform/common/types'; import { ExportFileOpener } from '../../../platform/export/exportFileOpener'; import { ExportFormat } from '../../../platform/export/types'; -import { ProgressReporter } from '../../../platform/progress/progressReporter.node'; +import { ProgressReporter } from '../../../platform/progress/progressReporter'; import { getLocString } from '../../../webviews/webview-side/react-common/locReactSide'; suite('DataScience - Export File Opener', () => { diff --git a/src/test/datascience/export/exportUtil.vscode.test.ts b/src/test/datascience/export/exportUtil.vscode.test.ts index d92e3b26a07..2789b32d3d9 100644 --- a/src/test/datascience/export/exportUtil.vscode.test.ts +++ b/src/test/datascience/export/exportUtil.vscode.test.ts @@ -32,8 +32,10 @@ suite('DataScience - Export Util', () => { suiteTeardown(() => closeActiveWindows(testDisposables)); test('Remove svgs from model', async () => { const exportUtil = api.serviceContainer.get(ExportUtil); + const contents = fs.readFileSync(testPdfIpynb.fsPath).toString(); - await exportUtil.removeSvgs(testPdfIpynb); + const contentsWithoutSvg = await exportUtil.removeSvgs(contents); + await fs.writeFile(testPdfIpynb.fsPath, contentsWithoutSvg); const model = JSON.parse(fs.readFileSync(testPdfIpynb.fsPath).toString()) as nbformat.INotebookContent; // make sure no svg exists in model diff --git a/src/test/datascience/export/fileConverter.vscode.test.ts b/src/test/datascience/export/fileConverter.vscode.test.ts index ec3951178bd..932f43bc3f7 100644 --- a/src/test/datascience/export/fileConverter.vscode.test.ts +++ b/src/test/datascience/export/fileConverter.vscode.test.ts @@ -9,18 +9,13 @@ import * as sinon from 'sinon'; import { Uri } from 'vscode'; import { IApplicationShell } from '../../../platform/common/application/types'; import { IFileSystemNode } from '../../../platform/common/platform/types.node'; -import { - IConfigurationService, - IDisposable, - IExtensions, - IWatchableJupyterSettings -} from '../../../platform/common/types'; +import { IConfigurationService, IDisposable, IWatchableJupyterSettings } from '../../../platform/common/types'; import { ExportFileOpener } from '../../../platform/export/exportFileOpener'; import { ExportInterpreterFinder } from '../../../platform/export/exportInterpreterFinder.node'; import { ExportUtil } from '../../../platform/export/exportUtil.node'; import { FileConverter } from '../../../platform/export/fileConverter.node'; import { INbConvertExport, IExport, IExportDialog, ExportFormat } from '../../../platform/export/types'; -import { ProgressReporter } from '../../../platform/progress/progressReporter.node'; +import { ProgressReporter } from '../../../platform/progress/progressReporter'; suite('DataScience - File Converter', () => { let fileConverter: FileConverter; @@ -34,7 +29,6 @@ suite('DataScience - File Converter', () => { let appShell: IApplicationShell; let exportFileOpener: ExportFileOpener; let exportInterpreterFinder: ExportInterpreterFinder; - let extensions: IExtensions; let configuration: IConfigurationService; let settings: IWatchableJupyterSettings; setup(async () => { @@ -49,7 +43,6 @@ suite('DataScience - File Converter', () => { appShell = mock(); exportFileOpener = mock(); exportInterpreterFinder = mock(); - extensions = mock(); configuration = mock(); settings = mock(); when(configuration.getSettings(anything())).thenReturn(instance(settings)); @@ -80,11 +73,9 @@ suite('DataScience - File Converter', () => { instance(fileSystem), instance(filePicker), instance(reporter), - instance(exportUtil), instance(appShell), instance(exportFileOpener), instance(exportInterpreterFinder), - instance(extensions), instance(configuration) ); diff --git a/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json b/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json index 008d3f5709a..9b68ca94e32 100644 --- a/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json +++ b/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json @@ -1,68 +1,70 @@ { - "name": "ms-toolsai-test", - "displayName": "AI Tools Test Extension", - "description": "Extension for testing the API for talking to the ms-toolsai.jupyter extension", - "version": "0.0.1", - "publisher": "ms-toolsai", - "engines": { - "vscode": "^1.32.0" - }, - "license": "MIT", - "homepage": "https://github.com/Microsoft/vscode-jupyter", - "repository": { - "type": "git", - "url": "https://github.com/Microsoft/vscode-jupyter" - }, - "bugs": { - "url": "https://github.com/Microsoft/vscode-jupyter/issues" - }, - "qna": "https://stackoverflow.com/questions/tagged/visual-studio-code+python", - "categories": [ - "Other" - ], - "activationEvents": ["*"], - "main": "./dist/extension", - "contributes": { - "pythonRemoteServerProvider": [ - { - "id": "RemoteServerPickerExample" - } - ], - "commands": [ - { - "command": "ms-toolsai-test.createBlankNotebook", - "title": "Create new blank Julia notebook", - "category": "Notebook" - } - ], + "name": "ms-toolsai-test", + "displayName": "AI Tools Test Extension", + "description": "Extension for testing the API for talking to the ms-toolsai.jupyter extension", + "version": "0.0.1", + "publisher": "ms-toolsai", + "engines": { + "vscode": "^1.32.0" + }, + "license": "MIT", + "homepage": "https://github.com/Microsoft/vscode-jupyter", + "repository": { + "type": "git", + "url": "https://github.com/Microsoft/vscode-jupyter" + }, + "bugs": { + "url": "https://github.com/Microsoft/vscode-jupyter/issues" + }, + "qna": "https://stackoverflow.com/questions/tagged/visual-studio-code+python", + "categories": [ + "Other" + ], + "activationEvents": [ + "*" + ], + "main": "./dist/extension", + "contributes": { + "pythonRemoteServerProvider": [ + { + "id": "RemoteServerPickerExample" + } + ], + "commands": [ + { + "command": "ms-toolsai-test.createBlankNotebook", + "title": "Create new blank Julia notebook", + "category": "Notebook" + } + ], "jupyter.kernels": [ - { - "title": "Julia", - "defaultLanguage": "julia" - } - ] - }, - "scripts": { - "vscode:prepublish": "webpack --mode production", - "webpack": "webpack --mode development", - "webpack-dev": "webpack --mode development --watch", - "test-compile": "tsc -p ./", - "lint": "eslint . --ext .ts,.tsx", - "package": "npm run vscode:prepublish && vsce package -o ms-toolsai-test.vsix" - }, - "devDependencies": { - "@types/jquery": "^3.5.0", - "@types/node": "^12.12.0", - "@types/vscode": "^1.32.0", - "@typescript-eslint/eslint-plugin": "^3.0.2", - "@typescript-eslint/parser": "^3.0.2", - "eslint": "^7.1.0", - "ts-loader": "^7.0.5", - "typescript": "^3.9.4", - "webpack": "^4.43.0", - "webpack-cli": "^3.3.11" - }, - "dependencies": { - "uuid": "^8.2.0" - } + { + "title": "Julia", + "defaultLanguage": "julia" + } + ] + }, + "scripts": { + "vscode:prepublish": "webpack --mode production", + "webpack": "webpack --mode development", + "webpack-dev": "webpack --mode development --watch", + "test-compile": "tsc -p ./", + "lint": "eslint . --ext .ts,.tsx", + "package": "npm run vscode:prepublish && vsce package -o ms-toolsai-test.vsix" + }, + "devDependencies": { + "@types/jquery": "^3.5.0", + "@types/node": "^12.12.0", + "@types/vscode": "^1.32.0", + "@typescript-eslint/eslint-plugin": "^3.0.2", + "@typescript-eslint/parser": "^3.0.2", + "eslint": "^7.1.0", + "ts-loader": "^7.0.5", + "typescript": "^3.9.4", + "webpack": "^4.43.0", + "webpack-cli": "^3.3.11" + }, + "dependencies": { + "uuid": "^8.2.0" + } } diff --git a/src/test/datascience/progress/decorators.unit.test.ts b/src/test/datascience/progress/decorators.unit.test.ts index 2853c419fd6..7fbb35b48ad 100644 --- a/src/test/datascience/progress/decorators.unit.test.ts +++ b/src/test/datascience/progress/decorators.unit.test.ts @@ -5,8 +5,8 @@ import { anything, deepEqual, instance, mock, verify } from 'ts-mockito'; import { createDeferred } from '../../../platform/common/utils/async'; -import { reportAction, registerReporter, disposeRegisteredReporters } from '../../../platform/progress/decorator.node'; -import { ProgressReporter } from '../../../platform/progress/progressReporter.node'; +import { reportAction, registerReporter, disposeRegisteredReporters } from '../../../platform/progress/decorator'; +import { ProgressReporter } from '../../../platform/progress/progressReporter'; import { IProgressReporter, ReportableAction } from '../../../platform/progress/types'; import { noop } from '../../core'; diff --git a/src/test/datascience/progress/progressReporter.unit.test.ts b/src/test/datascience/progress/progressReporter.unit.test.ts index 8d7461c4b54..8b3179ec9ec 100644 --- a/src/test/datascience/progress/progressReporter.unit.test.ts +++ b/src/test/datascience/progress/progressReporter.unit.test.ts @@ -9,7 +9,7 @@ import { CancellationToken, CancellationTokenSource, Progress as VSCProgress } f import { ApplicationShell } from '../../../platform/common/application/applicationShell'; import { IApplicationShell } from '../../../platform/common/application/types'; import { getUserMessageForAction } from '../../../platform/progress/messages'; -import { ProgressReporter } from '../../../platform/progress/progressReporter.node'; +import { ProgressReporter } from '../../../platform/progress/progressReporter'; import { ReportableAction } from '../../../platform/progress/types'; import { noop, sleep } from '../../core'; type Task = ( From 3055fbfa571e4480a045d89efdfa0a3069d7e7ef Mon Sep 17 00:00:00 2001 From: rebornix Date: Tue, 17 May 2022 21:24:12 -0700 Subject: [PATCH 03/12] export format --- src/platform/export/exportBase.web.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index f1461c6fe3a..17444d034ae 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -35,7 +35,7 @@ export class ExportBase implements INbConvertExport, IExportBase { async executeCommand( sourceDocument: NotebookDocument, target: Uri, - _format: ExportFormat, + format: ExportFormat, _interpreter: PythonEnvironment, _token: CancellationToken ): Promise { @@ -64,7 +64,7 @@ export class ExportBase implements INbConvertExport, IExportBase { const outputs = await executeSilently( kernel.session!, - `!jupyter nbconvert ${filePath} --to html --stdout` + `!jupyter nbconvert ${filePath} --to ${format} --stdout` ); if (outputs.length === 0) { From 9e822d3e8d3a7a4f3fdef01afcfcb15e1a21ee92 Mon Sep 17 00:00:00 2001 From: rebornix Date: Mon, 16 May 2022 15:55:21 -0700 Subject: [PATCH 04/12] get download path --- src/kernels/jupyter/session/jupyterSession.ts | 12 ++++++++++++ src/kernels/types.ts | 2 ++ src/platform/export/exportBase.web.ts | 6 +++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index 7da8c6798e3..e9829114999 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -253,6 +253,18 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer await this.disposeBackingFile(); } + async createTempfile(): Promise { + const tempFile = await this.contentsManager.newUntitled({ type: 'file' }); + return tempFile.path; + } + + async getDownloadPath(file: string): Promise { + const baseUrl = this.connInfo.baseUrl; + const token = this.connInfo.token; + const url = `${baseUrl}/files/${file}?token=${token}`; + return url; + } + private async createSession(options: { token: CancellationToken; ui: IDisplayOptions; diff --git a/src/kernels/types.ts b/src/kernels/types.ts index 1ac4ebbc06f..9bcce508103 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -297,6 +297,8 @@ export interface IJupyterSession extends IAsyncDisposable { export interface IJupyterServerSession extends IJupyterSession { readonly kind: 'remoteJupyter' | 'localJupyter'; invokeWithFileSynced(contents: string, handler: (file: IBackupFile) => Promise): Promise; + createTempfile(): Promise; + getDownloadPath(file: string): Promise; } export type ISessionWithSocket = Session.ISessionConnection & { diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 17444d034ae..412fe7521ea 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. 'use strict'; - import * as nbformat from '@jupyterlab/nbformat'; import { inject, injectable } from 'inversify'; import { Uri, CancellationToken, NotebookDocument } from 'vscode'; @@ -55,6 +54,8 @@ export class ExportBase implements INbConvertExport, IExportBase { if (kernel.session!.isServerSession()) { let contents = await this.exportUtil.getContent(sourceDocument); + // const tempTarget = await kernel.session!.createTempfile(); + // const outputFolder = path.dirname(tempTarget); await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); @@ -71,6 +72,9 @@ export class ExportBase implements INbConvertExport, IExportBase { return; } + // const downloadUrl = await session.getDownloadPath(file.filePath); + // console.log(downloadUrl); + // eslint-disable-next-line @typescript-eslint/no-explicit-any const output: nbformat.IStream = outputs[0] as any; if (output.name !== 'stdout' && output.output_type !== 'stream') { From 322b0f66d59534a4b85a277fb65ebbcf8f5189f9 Mon Sep 17 00:00:00 2001 From: rebornix Date: Wed, 18 May 2022 16:47:49 -0700 Subject: [PATCH 05/12] construct download path --- src/kernels/jupyter/session/jupyterSession.ts | 49 ++++++-- .../jupyter/session/jupyterSessionManager.ts | 1 + src/kernels/types.ts | 2 +- src/platform/export/exportBase.node.ts | 34 ++++- src/platform/export/exportBase.web.ts | 116 ++++++++++++------ src/platform/export/exportToHTML.ts | 12 +- src/platform/export/exportToPDF.ts | 12 +- src/platform/export/exportToPython.ts | 12 +- src/platform/export/exportToPythonPlain.ts | 15 ++- src/platform/export/fileConverter.node.ts | 31 ++--- src/platform/export/fileConverter.ts | 63 ++++------ src/platform/export/types.ts | 14 ++- .../jupyter/jupyterSession.unit.test.ts | 2 + 13 files changed, 243 insertions(+), 120 deletions(-) diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index e9829114999..fce2b96c21e 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -25,7 +25,13 @@ import { IJupyterServerSession } from '../../types'; import { DisplayOptions } from '../../displayOptions'; -import { IBackupFile, IJupyterBackingFileCreator, IJupyterKernelService, IJupyterRequestCreator } from '../types'; +import { + IBackupFile, + IJupyterBackingFileCreator, + IJupyterKernelService, + IJupyterPasswordConnect, + IJupyterRequestCreator +} from '../types'; import { Uri } from 'vscode'; import { generateBackingIPyNbFileName } from './backingFileCreator.base'; @@ -37,6 +43,7 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer constructor( resource: Resource, private connInfo: IJupyterConnection, + private jupyterPasswordConnect: IJupyterPasswordConnect, kernelConnectionMetadata: KernelConnectionMetadata, private specsManager: KernelSpecManager, private sessionManager: SessionManager, @@ -253,16 +260,44 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer await this.disposeBackingFile(); } - async createTempfile(): Promise { - const tempFile = await this.contentsManager.newUntitled({ type: 'file' }); + async createTempfile(ext: string): Promise { + const tempFile = await this.contentsManager.newUntitled({ type: 'file', ext }); return tempFile.path; } async getDownloadPath(file: string): Promise { - const baseUrl = this.connInfo.baseUrl; - const token = this.connInfo.token; - const url = `${baseUrl}/files/${file}?token=${token}`; - return url; + let baseUrl = this.connInfo.baseUrl; + let token = this.connInfo.token; + let xsrfToken: string | undefined = undefined; + + if (token === '' || token === null) { + const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo(baseUrl); + + if (pwSettings && pwSettings.requestHeaders) { + if (pwSettings.remappedBaseUrl) { + baseUrl = pwSettings.remappedBaseUrl; + } + + if (pwSettings.remappedToken) { + token = pwSettings.remappedToken; + } + + // eslint-disable-next-line + xsrfToken = (pwSettings.requestHeaders as any)['X-XSRFToken']; + } + } + + if (token !== '' && token !== null) { + return `${baseUrl}files/${file}?token=${token}`; + } else if (xsrfToken) { + // If we don't have a token, fall back to xsrfToken + const fullUrl = new URL(`${baseUrl}files/${file}`); + fullUrl.searchParams.append('_xsrf', xsrfToken); + const url = fullUrl.toString(); + return url; + } else { + throw new Error(DataScience.passwordFailure()); + } } private async createSession(options: { diff --git a/src/kernels/jupyter/session/jupyterSessionManager.ts b/src/kernels/jupyter/session/jupyterSessionManager.ts index 9490e15bcbb..987860a8947 100644 --- a/src/kernels/jupyter/session/jupyterSessionManager.ts +++ b/src/kernels/jupyter/session/jupyterSessionManager.ts @@ -192,6 +192,7 @@ export class JupyterSessionManager implements IJupyterSessionManager { const session = new JupyterSession( resource, this.connInfo, + this.jupyterPasswordConnect, kernelConnection, this.specsManager, this.sessionManager, diff --git a/src/kernels/types.ts b/src/kernels/types.ts index 9bcce508103..1e78e167c61 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -297,7 +297,7 @@ export interface IJupyterSession extends IAsyncDisposable { export interface IJupyterServerSession extends IJupyterSession { readonly kind: 'remoteJupyter' | 'localJupyter'; invokeWithFileSynced(contents: string, handler: (file: IBackupFile) => Promise): Promise; - createTempfile(): Promise; + createTempfile(ext: string): Promise; getDownloadPath(file: string): Promise; } diff --git a/src/platform/export/exportBase.node.ts b/src/platform/export/exportBase.node.ts index d75fe57eda0..aca4f27dab6 100644 --- a/src/platform/export/exportBase.node.ts +++ b/src/platform/export/exportBase.node.ts @@ -9,7 +9,7 @@ import { IPythonExecutionFactory, IPythonExecutionService } from '../common/proc import { reportAction } from '../progress/decorator'; import { ReportableAction } from '../progress/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; -import { ExportFormat, IExportBase, INbConvertExport } from './types'; +import { ExportFormat, IExportBase, IExportDialog, INbConvertExport } from './types'; import { ExportUtil } from './exportUtil.node'; import { TemporaryDirectory } from '../common/platform/types'; import { ExportInterpreterFinder } from './exportInterpreterFinder.node'; @@ -21,6 +21,7 @@ export class ExportBase implements INbConvertExport, IExportBase { @inject(IJupyterSubCommandExecutionService) protected jupyterService: IJupyterSubCommandExecutionService, @inject(IFileSystemNode) protected readonly fs: IFileSystemNode, + @inject(IExportDialog) protected readonly filePicker: IExportDialog, @inject(ExportUtil) protected readonly exportUtil: ExportUtil, @inject(INotebookImporter) protected readonly importer: INotebookImporter, @inject(ExportInterpreterFinder) private exportInterpreterFinder: ExportInterpreterFinder @@ -28,24 +29,31 @@ export class ExportBase implements INbConvertExport, IExportBase { public async export( _sourceDocument: NotebookDocument, - _target: Uri, _interpreter: PythonEnvironment, + _defaultFileName: string | undefined, _token: CancellationToken - // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function - ): Promise {} + ): Promise { + return undefined; + } @reportAction(ReportableAction.PerformingExport) public async executeCommand( sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, format: ExportFormat, interpreter: PythonEnvironment | undefined, token: CancellationToken - ): Promise { + ): Promise { if (token.isCancellationRequested) { return; } + let target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); + + if (!target) { + return; + } + interpreter = await this.exportInterpreterFinder.getExportInterpreter(interpreter); if (format === ExportFormat.python) { @@ -111,6 +119,20 @@ export class ExportBase implements INbConvertExport, IExportBase { } finally { tempTarget.dispose(); } + + return target; + } + + async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { + let target; + + if (format !== ExportFormat.python) { + target = await this.filePicker.showDialog(format, source, defaultFileName); + } else { + target = Uri.file((await this.fs.createTemporaryLocalFile('.py')).filePath); + } + + return target; } private async makeSourceFile(target: Uri, contents: string, tempDir: TemporaryDirectory): Promise { diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 412fe7521ea..4878b97780b 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -5,6 +5,7 @@ import * as nbformat from '@jupyterlab/nbformat'; import { inject, injectable } from 'inversify'; import { Uri, CancellationToken, NotebookDocument } from 'vscode'; +import * as path from '../../platform/vscode-path/path'; import { DisplayOptions } from '../../kernels/displayOptions'; import { executeSilently } from '../../kernels/helpers'; import { IKernel, IKernelProvider } from '../../kernels/types'; @@ -12,32 +13,35 @@ import { concatMultilineString } from '../../webviews/webview-side/common'; import { IFileSystem } from '../common/platform/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; import { ExportUtilBase } from './exportUtil'; -import { ExportFormat, IExportBase, INbConvertExport } from './types'; +import { ExportFormat, IExportBase, IExportDialog, INbConvertExport } from './types'; +import { traceError } from '../logging'; @injectable() export class ExportBase implements INbConvertExport, IExportBase { constructor( @inject(IKernelProvider) private readonly kernelProvider: IKernelProvider, @inject(IFileSystem) private readonly fs: IFileSystem, + @inject(IExportDialog) protected readonly filePicker: IExportDialog, @inject(ExportUtilBase) protected readonly exportUtil: ExportUtilBase ) {} public async export( _sourceDocument: NotebookDocument, - _target: Uri, _interpreter: PythonEnvironment, + _defaultFileName: string | undefined, _token: CancellationToken - // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function - ): Promise {} + ): Promise { + return undefined; + } // @reportAction(ReportableAction.PerformingExport) async executeCommand( sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, format: ExportFormat, _interpreter: PythonEnvironment, _token: CancellationToken - ): Promise { + ): Promise { const kernel = this.kernelProvider.get(sourceDocument.uri); if (!kernel) { // trace error @@ -53,45 +57,88 @@ export class ExportBase implements INbConvertExport, IExportBase { } if (kernel.session!.isServerSession()) { + const session = kernel.session!; let contents = await this.exportUtil.getContent(sourceDocument); - // const tempTarget = await kernel.session!.createTempfile(); - // const outputFolder = path.dirname(tempTarget); + + let target: Uri | undefined; await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); - console.log(pwd); - const filePath = `${pwd}/${file.filePath}`; - const outputs = await executeSilently( - kernel.session!, - `!jupyter nbconvert ${filePath} --to ${format} --stdout` - ); - - if (outputs.length === 0) { - return; + if (format === ExportFormat.pdf) { + const tempTarget = await session.createTempfile('.pdf'); + const outputs = await executeSilently( + session, + `!jupyter nbconvert ${filePath} --to pdf --output ${path.basename(tempTarget)}` + ); + + const text = this.parseStreamOutput(outputs); + + if (this.exportSucceed(text)) { + const downloadUrl = await session.getDownloadPath(tempTarget); + target = Uri.parse(downloadUrl); + } else { + traceError(text || 'Failed to export to PDF'); + throw new Error(text || 'Failed to export to PDF'); + } + } else { + target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); + if (target === undefined) { + return; + } + + const outputs = await executeSilently( + session, + `!jupyter nbconvert ${filePath} --to ${format} --stdout` + ); + + const text = this.parseStreamOutput(outputs); + if (!text) { + return; + } + + const headerRemoved = text + .split(/\r\n|\r|\n/g) + .slice(1) + .join('\n'); + + await this.fs.writeFile(target!, headerRemoved); } + }); + + return target; + } else { + // no op + } + } - // const downloadUrl = await session.getDownloadPath(file.filePath); - // console.log(downloadUrl); + private exportSucceed(message: string | undefined) { + if (!message) { + return false; + } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const output: nbformat.IStream = outputs[0] as any; - if (output.name !== 'stdout' && output.output_type !== 'stream') { - return; - } + return /\[NbConvertApp\].* successfully created/g.exec(message); + } - const text = concatMultilineString(output.text).trim().toLowerCase(); - const headerRemoved = text - .split(/\r\n|\r|\n/g) - .slice(1) - .join('\n'); + private parseStreamOutput(outputs: nbformat.IOutput[]): string | undefined { + if (outputs.length === 0) { + return; + } - await this.fs.writeFile(target, headerRemoved); - }); - } else { - // no op + const output: nbformat.IStream = outputs[0] as unknown as nbformat.IStream; + if (output.name !== 'stdout' && output.output_type !== 'stream') { + return; } + + const text = concatMultilineString(output.text).trim(); + return text; + } + + private async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { + let target = await this.filePicker.showDialog(format, source, defaultFileName); + + return target; } private async getCWD(kernel: IKernel) { @@ -100,8 +147,7 @@ export class ExportBase implements INbConvertExport, IExportBase { return; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const output: nbformat.IExecuteResult = outputs[0] as any; + const output: nbformat.IExecuteResult = outputs[0] as unknown as nbformat.IExecuteResult; if (output.output_type !== 'execute_result') { return undefined; } diff --git a/src/platform/export/exportToHTML.ts b/src/platform/export/exportToHTML.ts index 9514c326b77..afc2a593fdb 100644 --- a/src/platform/export/exportToHTML.ts +++ b/src/platform/export/exportToHTML.ts @@ -8,10 +8,16 @@ export class ExportToHTML implements INbConvertExport { constructor(@inject(IExportBase) protected readonly exportBase: IExportBase) {} public async export( sourceDocument: NotebookDocument, - target: Uri, interpreter: PythonEnvironment, + defaultFileName: string | undefined, token: CancellationToken - ): Promise { - await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.html, interpreter, token); + ): Promise { + return await this.exportBase.executeCommand( + sourceDocument, + defaultFileName, + ExportFormat.html, + interpreter, + token + ); } } diff --git a/src/platform/export/exportToPDF.ts b/src/platform/export/exportToPDF.ts index 97c6464921a..0c108c49540 100644 --- a/src/platform/export/exportToPDF.ts +++ b/src/platform/export/exportToPDF.ts @@ -9,10 +9,16 @@ export class ExportToPDF implements INbConvertExport { public async export( sourceDocument: NotebookDocument, - target: Uri, interpreter: PythonEnvironment, + defaultFileName: string | undefined, token: CancellationToken - ): Promise { - await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.pdf, interpreter, token); + ): Promise { + return await this.exportBase.executeCommand( + sourceDocument, + defaultFileName, + ExportFormat.pdf, + interpreter, + token + ); } } diff --git a/src/platform/export/exportToPython.ts b/src/platform/export/exportToPython.ts index fdd57c4d2cf..e538fa9435b 100644 --- a/src/platform/export/exportToPython.ts +++ b/src/platform/export/exportToPython.ts @@ -9,10 +9,16 @@ export class ExportToPython implements INbConvertExport { public async export( sourceDocument: NotebookDocument, - target: Uri, interpreter: PythonEnvironment, + defaultFileName: string | undefined, token: CancellationToken - ): Promise { - await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.python, interpreter, token); + ): Promise { + return await this.exportBase.executeCommand( + sourceDocument, + defaultFileName, + ExportFormat.python, + interpreter, + token + ); } } diff --git a/src/platform/export/exportToPythonPlain.ts b/src/platform/export/exportToPythonPlain.ts index 751476d97a1..462eb0e2759 100644 --- a/src/platform/export/exportToPythonPlain.ts +++ b/src/platform/export/exportToPythonPlain.ts @@ -8,7 +8,7 @@ import { CancellationToken, NotebookCell, NotebookCellKind, NotebookDocument, Ur import { appendLineFeed } from '../../webviews/webview-side/common'; import { IFileSystem, IPlatformService } from '../common/platform/types'; import { IConfigurationService } from '../common/types'; -import { IExport } from './types'; +import { ExportFormat, IExport, IExportDialog } from './types'; // Handles exporting a NotebookDocument to python @injectable() @@ -16,6 +16,7 @@ export class ExportToPythonPlain implements IExport { public constructor( @inject(IFileSystem) private readonly fs: IFileSystem, @inject(IConfigurationService) private readonly configuration: IConfigurationService, + @inject(IExportDialog) private readonly filePicker: IExportDialog, @inject(IPlatformService) private platform: IPlatformService ) {} @@ -31,11 +32,21 @@ export class ExportToPythonPlain implements IExport { } // Export the given document to the target source file - public async export(sourceDocument: NotebookDocument, target: Uri, token: CancellationToken): Promise { + public async export( + sourceDocument: NotebookDocument, + defaultFileName: string | undefined, + token: CancellationToken + ): Promise { if (token.isCancellationRequested) { return; } + let target = await this.filePicker.showDialog(ExportFormat.python, sourceDocument.uri, defaultFileName); + + if (!target) { + return; + } + const contents = this.exportDocument(sourceDocument); await this.writeFile(target, contents); } diff --git a/src/platform/export/fileConverter.node.ts b/src/platform/export/fileConverter.node.ts index 2bac13bc0a6..ac0dd7ffa33 100644 --- a/src/platform/export/fileConverter.node.ts +++ b/src/platform/export/fileConverter.node.ts @@ -39,35 +39,28 @@ export class FileConverter extends FileConverterBase implements IFileConverter { override async performExport( format: ExportFormat, sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, token: CancellationToken, candidateInterpreter?: PythonEnvironment ) { + let target: Uri | undefined; const pythonNbconvert = this.configuration.getSettings(sourceDocument.uri).pythonExportMethod === 'nbconvert'; if (format === ExportFormat.python && !pythonNbconvert) { // Unless selected by the setting use plain conversion for python script convert - await this.performPlainExport(format, sourceDocument, target, token); + target = await this.performPlainExport(format, sourceDocument, defaultFileName, token); } else { - await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); + target = await this.performNbConvertExport( + sourceDocument, + format, + defaultFileName, + candidateInterpreter, + token + ); } - await this.exportFileOpener.openFile(format, target); - } - - override async getTargetFile( - format: ExportFormat, - source: Uri, - defaultFileName?: string - ): Promise { - let target; - - if (format !== ExportFormat.python) { - target = await this.filePicker.showDialog(format, source, defaultFileName); - } else { - target = Uri.file((await this.fs.createTemporaryLocalFile('.py')).filePath); + if (target) { + await this.exportFileOpener.openFile(format, target); } - - return target; } } diff --git a/src/platform/export/fileConverter.ts b/src/platform/export/fileConverter.ts index feb830493d1..1a2f5aac178 100644 --- a/src/platform/export/fileConverter.ts +++ b/src/platform/export/fileConverter.ts @@ -35,7 +35,7 @@ export class FileConverter implements IFileConverter { // Open the source as a NotebookDocument, note that this doesn't actually show an editor, and we don't need // a specific close action as VS Code owns the lifetime nbDoc = await workspace.openNotebookDocument(source); - await this.exportImpl(ExportFormat.python, nbDoc, reporter.token); + await this.exportImpl(ExportFormat.python, nbDoc, undefined, reporter.token); } finally { reporter.dispose(); } @@ -53,7 +53,7 @@ export class FileConverter implements IFileConverter { ); try { - await this.exportImpl(format, sourceDocument, reporter.token, defaultFileName, candidateInterpreter); + await this.exportImpl(format, sourceDocument, defaultFileName, reporter.token, candidateInterpreter); } finally { reporter.dispose(); } @@ -67,17 +67,12 @@ export class FileConverter implements IFileConverter { public async exportImpl( format: ExportFormat, sourceDocument: NotebookDocument, + defaultFileName: string | undefined, token: CancellationToken, - defaultFileName?: string, candidateInterpreter?: PythonEnvironment - ): Promise { - let target; + ): Promise { try { - target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (!target) { - return; - } - await this.performExport(format, sourceDocument, target, token, candidateInterpreter); + await this.performExport(format, sourceDocument, defaultFileName, token, candidateInterpreter); } catch (e) { traceError('Export failed', e); sendTelemetryEvent(Telemetry.ExportNotebookAsFailed, undefined, { format: format }); @@ -93,30 +88,39 @@ export class FileConverter implements IFileConverter { protected async performExport( format: ExportFormat, sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, token: CancellationToken, candidateInterpreter?: PythonEnvironment ) { + let target: Uri | undefined; // For web, we perform plain export for Python if (format === ExportFormat.python) { // Unless selected by the setting use plain conversion for python script convert - await this.performPlainExport(format, sourceDocument, target, token); + target = await this.performPlainExport(format, sourceDocument, defaultFileName, token); } else { - await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); + target = await this.performNbConvertExport( + sourceDocument, + format, + defaultFileName, + candidateInterpreter, + token + ); } - await this.exportFileOpener.openFile(format, target, true); + if (target) { + await this.exportFileOpener.openFile(format, target, true); + } } protected async performPlainExport( format: ExportFormat, sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, cancelToken: CancellationToken - ) { + ): Promise { switch (format) { case ExportFormat.python: - await this.exportToPythonPlain.export(sourceDocument, target, cancelToken); + return await this.exportToPythonPlain.export(sourceDocument, defaultFileName, cancelToken); break; } } @@ -124,45 +128,32 @@ export class FileConverter implements IFileConverter { protected async performNbConvertExport( sourceDocument: NotebookDocument, format: ExportFormat, - target: Uri, + defaultFileName: string | undefined, interpreter: PythonEnvironment | undefined, cancelToken: CancellationToken ) { try { - await this.exportToFormat(sourceDocument, target, format, interpreter, cancelToken); + return await this.exportToFormat(sourceDocument, defaultFileName, format, interpreter, cancelToken); } finally { } } - protected async getTargetFile( - format: ExportFormat, - source: Uri, - defaultFileName?: string - ): Promise { - let target = await this.filePicker.showDialog(format, source, defaultFileName); - - return target; - } - protected async exportToFormat( sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, format: ExportFormat, interpreter: PythonEnvironment | undefined, cancelToken: CancellationToken ) { switch (format) { case ExportFormat.python: - await this.exportToPython.export(sourceDocument, target, interpreter, cancelToken); - break; + return await this.exportToPython.export(sourceDocument, interpreter, defaultFileName, cancelToken); case ExportFormat.pdf: - await this.exportToPDF.export(sourceDocument, target, interpreter, cancelToken); - break; + return await this.exportToPDF.export(sourceDocument, interpreter, defaultFileName, cancelToken); case ExportFormat.html: - await this.exportToHTML.export(sourceDocument, target, interpreter, cancelToken); - break; + return await this.exportToHTML.export(sourceDocument, interpreter, defaultFileName, cancelToken); default: break; diff --git a/src/platform/export/types.ts b/src/platform/export/types.ts index 691048fd693..dcc0fab900c 100644 --- a/src/platform/export/types.ts +++ b/src/platform/export/types.ts @@ -23,26 +23,30 @@ export const INbConvertExport = Symbol('INbConvertExport'); export interface INbConvertExport { export( sourceDocument: NotebookDocument, - target: Uri, interpreter: PythonEnvironment | undefined, + defaultFileName: string | undefined, token: CancellationToken - ): Promise; + ): Promise; } export const IExportBase = Symbol('IExportBase'); export interface IExportBase { executeCommand( sourceDocument: NotebookDocument, - target: Uri, + defaultFileName: string | undefined, format: ExportFormat, interpreter: PythonEnvironment, token: CancellationToken - ): Promise; + ): Promise; } export const IExport = Symbol('IExport'); export interface IExport { - export(sourceDocument: NotebookDocument, target: Uri, token: CancellationToken): Promise; + export( + sourceDocument: NotebookDocument, + defaultFileName: string | undefined, + token: CancellationToken + ): Promise; } export const IExportDialog = Symbol('IExportDialog'); diff --git a/src/test/datascience/jupyter/jupyterSession.unit.test.ts b/src/test/datascience/jupyter/jupyterSession.unit.test.ts index aa5482e7110..b14dc92b1bd 100644 --- a/src/test/datascience/jupyter/jupyterSession.unit.test.ts +++ b/src/test/datascience/jupyter/jupyterSession.unit.test.ts @@ -35,6 +35,7 @@ import { FileSystem } from '../../../platform/common/platform/fileSystem.node'; import { BackingFileCreator } from '../../../kernels/jupyter/session/backingFileCreator.node'; import * as path from '../../../platform/vscode-path/path'; import { JupyterRequestCreator } from '../../../kernels/jupyter/session/jupyterRequestCreator.node'; +import { IJupyterPasswordConnect } from '../../../kernels/jupyter/types'; /* eslint-disable , @typescript-eslint/no-explicit-any */ suite('DataScience - JupyterSession', () => { @@ -134,6 +135,7 @@ suite('DataScience - JupyterSession', () => { jupyterSession = new JupyterSession( resource, instance(connection), + mock(IJupyterPasswordConnect), mockKernelSpec, instance(specManager), instance(sessionManager), From 666a53023ffb0fa39fead96a5a090dcd40e2d6ba Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 10:17:00 -0700 Subject: [PATCH 06/12] fetch content through api other than stdout --- src/kernels/jupyter/session/jupyterSession.ts | 56 +++------ .../jupyter/session/jupyterSessionManager.ts | 1 - src/kernels/types.ts | 5 +- src/platform/export/exportBase.web.ts | 110 +++++++++++------- .../jupyter/jupyterSession.unit.test.ts | 2 - 5 files changed, 86 insertions(+), 88 deletions(-) diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index fce2b96c21e..45414baab36 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -1,7 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import type { ContentsManager, Kernel, KernelSpecManager, Session, SessionManager } from '@jupyterlab/services'; +import type { + Contents, + ContentsManager, + Kernel, + KernelSpecManager, + Session, + SessionManager +} from '@jupyterlab/services'; import * as uuid from 'uuid/v4'; import { CancellationToken, CancellationTokenSource } from 'vscode-jsonrpc'; import { Cancellation } from '../../../platform/common/cancellation'; @@ -25,13 +32,7 @@ import { IJupyterServerSession } from '../../types'; import { DisplayOptions } from '../../displayOptions'; -import { - IBackupFile, - IJupyterBackingFileCreator, - IJupyterKernelService, - IJupyterPasswordConnect, - IJupyterRequestCreator -} from '../types'; +import { IBackupFile, IJupyterBackingFileCreator, IJupyterKernelService, IJupyterRequestCreator } from '../types'; import { Uri } from 'vscode'; import { generateBackingIPyNbFileName } from './backingFileCreator.base'; @@ -43,7 +44,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer constructor( resource: Resource, private connInfo: IJupyterConnection, - private jupyterPasswordConnect: IJupyterPasswordConnect, kernelConnectionMetadata: KernelConnectionMetadata, private specsManager: KernelSpecManager, private sessionManager: SessionManager, @@ -265,39 +265,13 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer return tempFile.path; } - async getDownloadPath(file: string): Promise { - let baseUrl = this.connInfo.baseUrl; - let token = this.connInfo.token; - let xsrfToken: string | undefined = undefined; - - if (token === '' || token === null) { - const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo(baseUrl); - - if (pwSettings && pwSettings.requestHeaders) { - if (pwSettings.remappedBaseUrl) { - baseUrl = pwSettings.remappedBaseUrl; - } - - if (pwSettings.remappedToken) { - token = pwSettings.remappedToken; - } - - // eslint-disable-next-line - xsrfToken = (pwSettings.requestHeaders as any)['X-XSRFToken']; - } - } + async deleteTempfile(file: string): Promise { + await this.contentsManager.delete(file); + } - if (token !== '' && token !== null) { - return `${baseUrl}files/${file}?token=${token}`; - } else if (xsrfToken) { - // If we don't have a token, fall back to xsrfToken - const fullUrl = new URL(`${baseUrl}files/${file}`); - fullUrl.searchParams.append('_xsrf', xsrfToken); - const url = fullUrl.toString(); - return url; - } else { - throw new Error(DataScience.passwordFailure()); - } + async getContents(file: string, format: Contents.FileFormat): Promise { + const data = await this.contentsManager.get(file, { type: 'file', format: format, content: true }); + return data; } private async createSession(options: { diff --git a/src/kernels/jupyter/session/jupyterSessionManager.ts b/src/kernels/jupyter/session/jupyterSessionManager.ts index 987860a8947..9490e15bcbb 100644 --- a/src/kernels/jupyter/session/jupyterSessionManager.ts +++ b/src/kernels/jupyter/session/jupyterSessionManager.ts @@ -192,7 +192,6 @@ export class JupyterSessionManager implements IJupyterSessionManager { const session = new JupyterSession( resource, this.connInfo, - this.jupyterPasswordConnect, kernelConnection, this.specsManager, this.sessionManager, diff --git a/src/kernels/types.ts b/src/kernels/types.ts index 1e78e167c61..7cd37c4cd2f 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -3,7 +3,7 @@ 'use strict'; -import type { Kernel, KernelMessage, Session } from '@jupyterlab/services'; +import type { Contents, Kernel, KernelMessage, Session } from '@jupyterlab/services'; import type { Observable } from 'rxjs/Observable'; import type { JSONObject } from '@lumino/coreutils'; import type { @@ -298,7 +298,8 @@ export interface IJupyterServerSession extends IJupyterSession { readonly kind: 'remoteJupyter' | 'localJupyter'; invokeWithFileSynced(contents: string, handler: (file: IBackupFile) => Promise): Promise; createTempfile(ext: string): Promise; - getDownloadPath(file: string): Promise; + deleteTempfile(file: string): Promise; + getContents(file: string, format: Contents.FileFormat): Promise; } export type ISessionWithSocket = Session.ISessionConnection & { diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 4878b97780b..9f65704ff6d 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -34,6 +34,31 @@ export class ExportBase implements INbConvertExport, IExportBase { return undefined; } + b64toBlob(b64Data: string, contentType: string | undefined) { + contentType = contentType || ''; + var sliceSize = 512; + b64Data = b64Data.replace(/^[^,]+,/, ''); + b64Data = b64Data.replace(/\s/g, ''); + var byteCharacters = atob(b64Data); + var byteArrays = []; + + for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) { + var slice = byteCharacters.slice(offset, offset + sliceSize); + + var byteNumbers = new Array(slice.length); + for (var i = 0; i < slice.length; i++) { + byteNumbers[i] = slice.charCodeAt(i); + } + + var byteArray = new Uint8Array(byteNumbers); + + byteArrays.push(byteArray); + } + + var blob = new Blob(byteArrays, { type: contentType }); + return blob; + } + // @reportAction(ReportableAction.PerformingExport) async executeCommand( sourceDocument: NotebookDocument, @@ -62,48 +87,49 @@ export class ExportBase implements INbConvertExport, IExportBase { let target: Uri | undefined; + let fileExt = ''; + + switch (format) { + case ExportFormat.html: + fileExt = '.html'; + break; + case ExportFormat.pdf: + fileExt = '.pdf'; + break; + case ExportFormat.python: + fileExt = '.py'; + break; + } + await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); const filePath = `${pwd}/${file.filePath}`; + const tempTarget = await session.createTempfile(fileExt); + const outputs = await executeSilently( + session, + `!jupyter nbconvert ${filePath} --to ${format} --output ${path.basename(tempTarget)}` + ); + + const text = this.parseStreamOutput(outputs); + if (text) { + traceError(text || `Failed to export to ${format}`); + } + + target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); + if (target === undefined) { + return; + } if (format === ExportFormat.pdf) { - const tempTarget = await session.createTempfile('.pdf'); - const outputs = await executeSilently( - session, - `!jupyter nbconvert ${filePath} --to pdf --output ${path.basename(tempTarget)}` - ); - - const text = this.parseStreamOutput(outputs); - - if (this.exportSucceed(text)) { - const downloadUrl = await session.getDownloadPath(tempTarget); - target = Uri.parse(downloadUrl); - } else { - traceError(text || 'Failed to export to PDF'); - throw new Error(text || 'Failed to export to PDF'); - } + const content = await session.getContents(tempTarget, 'base64'); + const bytes = this.b64toBlob(content.content, 'application/pdf'); + const buffer = await bytes.arrayBuffer(); + await this.fs.writeFile(target, Buffer.from(buffer)); + await session.deleteTempfile(tempTarget); } else { - target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (target === undefined) { - return; - } - - const outputs = await executeSilently( - session, - `!jupyter nbconvert ${filePath} --to ${format} --stdout` - ); - - const text = this.parseStreamOutput(outputs); - if (!text) { - return; - } - - const headerRemoved = text - .split(/\r\n|\r|\n/g) - .slice(1) - .join('\n'); - - await this.fs.writeFile(target!, headerRemoved); + const content = await session.getContents(tempTarget, 'text'); + await this.fs.writeFile(target, content.content as string); + await session.deleteTempfile(tempTarget); } }); @@ -113,13 +139,13 @@ export class ExportBase implements INbConvertExport, IExportBase { } } - private exportSucceed(message: string | undefined) { - if (!message) { - return false; - } + // private exportSucceed(message: string | undefined) { + // if (!message) { + // return false; + // } - return /\[NbConvertApp\].* successfully created/g.exec(message); - } + // return /\[NbConvertApp\].* successfully created/g.exec(message); + // } private parseStreamOutput(outputs: nbformat.IOutput[]): string | undefined { if (outputs.length === 0) { diff --git a/src/test/datascience/jupyter/jupyterSession.unit.test.ts b/src/test/datascience/jupyter/jupyterSession.unit.test.ts index b14dc92b1bd..aa5482e7110 100644 --- a/src/test/datascience/jupyter/jupyterSession.unit.test.ts +++ b/src/test/datascience/jupyter/jupyterSession.unit.test.ts @@ -35,7 +35,6 @@ import { FileSystem } from '../../../platform/common/platform/fileSystem.node'; import { BackingFileCreator } from '../../../kernels/jupyter/session/backingFileCreator.node'; import * as path from '../../../platform/vscode-path/path'; import { JupyterRequestCreator } from '../../../kernels/jupyter/session/jupyterRequestCreator.node'; -import { IJupyterPasswordConnect } from '../../../kernels/jupyter/types'; /* eslint-disable , @typescript-eslint/no-explicit-any */ suite('DataScience - JupyterSession', () => { @@ -135,7 +134,6 @@ suite('DataScience - JupyterSession', () => { jupyterSession = new JupyterSession( resource, instance(connection), - mock(IJupyterPasswordConnect), mockKernelSpec, instance(specManager), instance(sessionManager), From 203e671303794024df9a9c7a64d7d4d6fbfdc31d Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 11:48:07 -0700 Subject: [PATCH 07/12] :lipstick: --- src/kernels/common/baseJupyterSession.ts | 2 +- .../launcher/jupyterNotebookProvider.ts | 2 - src/kernels/jupyter/session/jupyterSession.ts | 74 ++++------ src/platform/export/exportBase.web.ts | 90 ++++++------ src/platform/export/exportToPythonPlain.ts | 5 +- src/platform/export/fileConverter.node.ts | 4 +- .../export/exportToHTML.vscode.test.ts | 19 +-- .../export/exportToPython.vscode.test.ts | 19 +-- .../export/fileConverter.vscode.test.ts | 1 - .../ms-ai-tools-test/package.json | 134 +++++++++--------- 10 files changed, 154 insertions(+), 196 deletions(-) diff --git a/src/kernels/common/baseJupyterSession.ts b/src/kernels/common/baseJupyterSession.ts index d14fd4e23ce..881c210ec71 100644 --- a/src/kernels/common/baseJupyterSession.ts +++ b/src/kernels/common/baseJupyterSession.ts @@ -133,7 +133,7 @@ export abstract class BaseJupyterSession implements IJupyterSession { traceInfo(`Unhandled message found: ${m.header.msg_type}`); }; } - isServerSession(): this is IJupyterServerSession { + public isServerSession(): this is IJupyterServerSession { return false; } public async dispose(): Promise { diff --git a/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts b/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts index b5f5edb5ff8..ddf7a344e07 100644 --- a/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts +++ b/src/kernels/jupyter/launcher/jupyterNotebookProvider.ts @@ -55,8 +55,6 @@ export class JupyterNotebookProvider implements IJupyterNotebookProvider { }; const server = await this.serverProvider.getOrCreateServer(serverOptions); Cancellation.throwIfCanceled(options.token); - - console.log(server.connection.rootDirectory); return server.createNotebook( options.resource, options.kernelConnection, diff --git a/src/kernels/jupyter/session/jupyterSession.ts b/src/kernels/jupyter/session/jupyterSession.ts index 45414baab36..375a85277d9 100644 --- a/src/kernels/jupyter/session/jupyterSession.ts +++ b/src/kernels/jupyter/session/jupyterSession.ts @@ -39,7 +39,6 @@ import { generateBackingIPyNbFileName } from './backingFileCreator.base'; // function is export class JupyterSession extends BaseJupyterSession implements IJupyterServerSession { public override readonly kind: 'remoteJupyter' | 'localJupyter'; - private backingFile: IBackupFile | undefined; constructor( resource: Resource, @@ -78,11 +77,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer return this.waitForIdleOnSession(this.session, timeout); } - public override async shutdown(): Promise { - await this.disposeBackingFile(); - await super.shutdown(); - } - public override get kernel(): Kernel.IKernelConnection | undefined { return this.session?.kernel || undefined; } @@ -120,23 +114,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer ...this.kernelConnectionMetadata.kernelModel, model: this.kernelConnectionMetadata.kernelModel.model }) as ISessionWithSocket; - - const request = newSession.kernel?.requestExecute( - { - code: 'import os; os.getcwd()', - silent: false, - stop_on_error: false, - allow_stdin: true, - store_history: false - }, - true - ); - request!.onIOPub = (msg) => { - console.log(msg); - }; - - await request!.done; - newSession.kernelConnectionMetadata = this.kernelConnectionMetadata; newSession.kernelSocketInformation = { socket: this.requestCreator.getWebsocket(this.kernelConnectionMetadata.id), @@ -179,8 +156,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer } } - // try print again - return newSession; } @@ -193,7 +168,6 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer if (!session || !this.contentsManager || !this.sessionManager) { throw new SessionDisposedError(); } - await this.disposeBackingFile(); let result: ISessionWithSocket | undefined; let tryCount = 0; const ui = new DisplayOptions(disableUI); @@ -237,27 +211,32 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer return; } - if (!this.backingFile) { - this.backingFile = await this.backingFileCreator.createBackingFile( - this.resource, - this.workingDirectory, - this.kernelConnectionMetadata, - this.connInfo, - this.contentsManager - ); + const backingFile = await this.backingFileCreator.createBackingFile( + this.resource, + this.workingDirectory, + this.kernelConnectionMetadata, + this.connInfo, + this.contentsManager + ); + + if (!backingFile) { + return; } await this.contentsManager - .save(this.backingFile!.filePath, { + .save(backingFile!.filePath, { content: JSON.parse(contents), type: 'notebook' }) .ignoreErrors(); + await handler({ - filePath: this.backingFile!.filePath, - dispose: this.backingFile!.dispose.bind(this.backingFile!) + filePath: backingFile.filePath, + dispose: backingFile.dispose.bind(backingFile) }); - await this.disposeBackingFile(); + + await backingFile.dispose(); + await this.contentsManager.delete(backingFile.filePath).ignoreErrors(); } async createTempfile(ext: string): Promise { @@ -279,7 +258,7 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer ui: IDisplayOptions; }): Promise { // Create our backing file for the notebook - this.backingFile = await this.backingFileCreator.createBackingFile( + const backingFile = await this.backingFileCreator.createBackingFile( this.resource, this.workingDirectory, this.kernelConnectionMetadata, @@ -304,8 +283,8 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer ); } catch (ex) { // If we failed to create the kernel, we need to clean up the file. - if (this.connInfo && this.backingFile) { - this.contentsManager.delete(this.backingFile.filePath).ignoreErrors(); + if (this.connInfo && backingFile) { + this.contentsManager.delete(backingFile.filePath).ignoreErrors(); } throw ex; } @@ -319,7 +298,7 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer // Create our session options using this temporary notebook and our connection info const sessionOptions: Session.ISessionOptions = { - path: this.backingFile?.filePath || generateBackingIPyNbFileName(this.resource), // Name has to be unique + path: backingFile?.filePath || generateBackingIPyNbFileName(this.resource), // Name has to be unique kernel: { name: kernelName }, @@ -368,19 +347,14 @@ export class JupyterSession extends BaseJupyterSession implements IJupyterServer }) .catch((ex) => Promise.reject(new JupyterSessionStartError(ex))) .finally(async () => { - await this.disposeBackingFile(); + if (this.connInfo && backingFile) { + this.contentsManager.delete(backingFile.filePath).ignoreErrors(); + } }), options.token ); } - private async disposeBackingFile() { - if (this.connInfo && this.backingFile) { - await this.backingFile.dispose(); - await this.contentsManager.delete(this.backingFile.filePath).ignoreErrors(); - } - } - private logRemoteOutput(output: string) { if (!isLocalConnection(this.kernelConnectionMetadata)) { this.outputChannel.appendLine(output); diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 9f65704ff6d..0c350bcd828 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -14,7 +14,9 @@ import { IFileSystem } from '../common/platform/types'; import { PythonEnvironment } from '../pythonEnvironments/info'; import { ExportUtilBase } from './exportUtil'; import { ExportFormat, IExportBase, IExportDialog, INbConvertExport } from './types'; -import { traceError } from '../logging'; +import { traceLog } from '../logging'; +import { reportAction } from '../progress/decorator'; +import { ReportableAction } from '../progress/types'; @injectable() export class ExportBase implements INbConvertExport, IExportBase { @@ -34,32 +36,7 @@ export class ExportBase implements INbConvertExport, IExportBase { return undefined; } - b64toBlob(b64Data: string, contentType: string | undefined) { - contentType = contentType || ''; - var sliceSize = 512; - b64Data = b64Data.replace(/^[^,]+,/, ''); - b64Data = b64Data.replace(/\s/g, ''); - var byteCharacters = atob(b64Data); - var byteArrays = []; - - for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) { - var slice = byteCharacters.slice(offset, offset + sliceSize); - - var byteNumbers = new Array(slice.length); - for (var i = 0; i < slice.length; i++) { - byteNumbers[i] = slice.charCodeAt(i); - } - - var byteArray = new Uint8Array(byteNumbers); - - byteArrays.push(byteArray); - } - - var blob = new Blob(byteArrays, { type: contentType }); - return blob; - } - - // @reportAction(ReportableAction.PerformingExport) + @reportAction(ReportableAction.PerformingExport) async executeCommand( sourceDocument: NotebookDocument, defaultFileName: string | undefined, @@ -85,8 +62,6 @@ export class ExportBase implements INbConvertExport, IExportBase { const session = kernel.session!; let contents = await this.exportUtil.getContent(sourceDocument); - let target: Uri | undefined; - let fileExt = ''; switch (format) { @@ -101,6 +76,11 @@ export class ExportBase implements INbConvertExport, IExportBase { break; } + let target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); + if (target === undefined) { + return; + } + await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); const filePath = `${pwd}/${file.filePath}`; @@ -111,24 +91,22 @@ export class ExportBase implements INbConvertExport, IExportBase { ); const text = this.parseStreamOutput(outputs); - if (text) { - traceError(text || `Failed to export to ${format}`); - } - - target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (target === undefined) { - return; + if (this.isExportFailed(text)) { + throw new Error(text || `Failed to export to ${format}`); + } else if (text) { + // trace the output in case we didn't identify all errors + traceLog(text); } if (format === ExportFormat.pdf) { const content = await session.getContents(tempTarget, 'base64'); const bytes = this.b64toBlob(content.content, 'application/pdf'); const buffer = await bytes.arrayBuffer(); - await this.fs.writeFile(target, Buffer.from(buffer)); + await this.fs.writeFile(target!, Buffer.from(buffer)); await session.deleteTempfile(tempTarget); } else { const content = await session.getContents(tempTarget, 'text'); - await this.fs.writeFile(target, content.content as string); + await this.fs.writeFile(target!, content.content as string); await session.deleteTempfile(tempTarget); } }); @@ -139,13 +117,37 @@ export class ExportBase implements INbConvertExport, IExportBase { } } - // private exportSucceed(message: string | undefined) { - // if (!message) { - // return false; - // } + private b64toBlob(b64Data: string, contentType: string | undefined) { + contentType = contentType || ''; + const sliceSize = 512; + b64Data = b64Data.replace(/^[^,]+,/, ''); + b64Data = b64Data.replace(/\s/g, ''); + const byteCharacters = atob(b64Data); + let byteArrays = []; + + for (let offset = 0; offset < byteCharacters.length; offset += sliceSize) { + const slice = byteCharacters.slice(offset, offset + sliceSize); + + let byteNumbers = new Array(slice.length); + for (let i = 0; i < slice.length; i++) { + byteNumbers[i] = slice.charCodeAt(i); + } + + const byteArray = new Uint8Array(byteNumbers); + byteArrays.push(byteArray); + } + + const blob = new Blob(byteArrays, { type: contentType }); + return blob; + } + + private isExportFailed(message: string | undefined) { + if (!message) { + return true; + } - // return /\[NbConvertApp\].* successfully created/g.exec(message); - // } + return /Traceback \(most recent call last\)/g.exec(message); + } private parseStreamOutput(outputs: nbformat.IOutput[]): string | undefined { if (outputs.length === 0) { diff --git a/src/platform/export/exportToPythonPlain.ts b/src/platform/export/exportToPythonPlain.ts index 462eb0e2759..25b8fc4727b 100644 --- a/src/platform/export/exportToPythonPlain.ts +++ b/src/platform/export/exportToPythonPlain.ts @@ -25,10 +25,7 @@ export class ExportToPythonPlain implements IExport { } getEOL(): string { - if (this.platform.isWindows) { - return '\r\n'; - } - return '\n'; + return this.platform.isWindows ? '\r\n' : '\n'; } // Export the given document to the target source file diff --git a/src/platform/export/fileConverter.node.ts b/src/platform/export/fileConverter.node.ts index ac0dd7ffa33..039cc767650 100644 --- a/src/platform/export/fileConverter.node.ts +++ b/src/platform/export/fileConverter.node.ts @@ -17,12 +17,12 @@ export class FileConverter extends FileConverterBase implements IFileConverter { @inject(INbConvertExport) @named(ExportFormat.html) exportToHTML: INbConvertExport, @inject(INbConvertExport) @named(ExportFormat.python) exportToPython: INbConvertExport, @inject(IExport) @named(ExportFormat.python) exportToPythonPlain: IExport, + @inject(IFileSystemNode) readonly fs: IFileSystemNode, @inject(IExportDialog) filePicker: IExportDialog, @inject(ProgressReporter) progressReporter: ProgressReporter, @inject(IApplicationShell) applicationShell: IApplicationShell, @inject(ExportFileOpener) exportFileOpener: ExportFileOpener, - @inject(IConfigurationService) readonly configuration: IConfigurationService, - @inject(IFileSystemNode) readonly fs: IFileSystemNode + @inject(IConfigurationService) readonly configuration: IConfigurationService ) { super( exportToPythonPlain, diff --git a/src/test/datascience/export/exportToHTML.vscode.test.ts b/src/test/datascience/export/exportToHTML.vscode.test.ts index dcebe165da6..fa5517a5564 100644 --- a/src/test/datascience/export/exportToHTML.vscode.test.ts +++ b/src/test/datascience/export/exportToHTML.vscode.test.ts @@ -4,7 +4,7 @@ /* eslint-disable @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports, no-invalid-this, @typescript-eslint/no-explicit-any */ import { assert } from 'chai'; import * as path from '../../../platform/vscode-path/path'; -import { CancellationTokenSource, Uri } from 'vscode'; +import { CancellationTokenSource, Uri, workspace } from 'vscode'; import { IFileSystemNode } from '../../../platform/common/platform/types.node'; import { ExportInterpreterFinder } from '../../../platform/export/exportInterpreterFinder.node'; import { INbConvertExport, ExportFormat } from '../../../platform/export/types'; @@ -24,20 +24,15 @@ suite('DataScience - Export HTML', function () { const fileSystem = api.serviceContainer.get(IFileSystemNode); const exportToHTML = api.serviceContainer.get(INbConvertExport, ExportFormat.html); const exportInterpreterFinder = api.serviceContainer.get(ExportInterpreterFinder); - const file = await fileSystem.createTemporaryLocalFile('.html'); - const target = Uri.file(file.filePath); - await file.dispose(); const token = new CancellationTokenSource(); const interpreter = await exportInterpreterFinder.getExportInterpreter(); - await exportToHTML.export( - Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')), - target, - interpreter, - token.token + const document = await workspace.openNotebookDocument( + Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')) ); - - assert.equal(await fileSystem.localFileExists(target.fsPath), true); - const fileContents = await fileSystem.readLocalFile(target.fsPath); + const target = await exportToHTML.export(document, interpreter, undefined, token.token); + assert.exists(target); + assert.equal(await fileSystem.localFileExists(target!.fsPath), true); + const fileContents = await fileSystem.readLocalFile(target!.fsPath); assert.include(fileContents, ''); // this is the content of a cell assert.include(fileContents, 'f6886df81f3d4023a2122cc3f55fdbec'); diff --git a/src/test/datascience/export/exportToPython.vscode.test.ts b/src/test/datascience/export/exportToPython.vscode.test.ts index ca0022fed0b..0140d5263fa 100644 --- a/src/test/datascience/export/exportToPython.vscode.test.ts +++ b/src/test/datascience/export/exportToPython.vscode.test.ts @@ -4,9 +4,8 @@ /* eslint-disable @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports, no-invalid-this, @typescript-eslint/no-explicit-any */ import { assert } from 'chai'; import * as path from '../../../platform/vscode-path/path'; -import { CancellationTokenSource, Uri } from 'vscode'; +import { CancellationTokenSource, Uri, workspace } from 'vscode'; import { IDocumentManager } from '../../../platform/common/application/types'; -import { IFileSystemNode } from '../../../platform/common/platform/types.node'; import { ExportInterpreterFinder } from '../../../platform/export/exportInterpreterFinder.node'; import { INbConvertExport, ExportFormat } from '../../../platform/export/types'; import { IExtensionTestApi } from '../../common.node'; @@ -22,21 +21,17 @@ suite('DataScience - Export Python', function () { teardown(closeActiveWindows); suiteTeardown(closeActiveWindows); test('Export To Python', async () => { - const fileSystem = api.serviceContainer.get(IFileSystemNode); const exportToPython = api.serviceContainer.get(INbConvertExport, ExportFormat.python); - const target = Uri.file((await fileSystem.createTemporaryLocalFile('.py')).filePath); const token = new CancellationTokenSource(); const exportInterpreterFinder = api.serviceContainer.get(ExportInterpreterFinder); const interpreter = await exportInterpreterFinder.getExportInterpreter(); - await exportToPython.export( - Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')), - target, - interpreter, - token.token + const document = await workspace.openNotebookDocument( + Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')) ); - + const target = await exportToPython.export(document, interpreter, undefined, token.token); + assert.exists(target); const documentManager = api.serviceContainer.get(IDocumentManager); - const document = await documentManager.openTextDocument(target); - assert.include(document.getText(), 'tim = 1'); + const targetDocument = await documentManager.openTextDocument(target!); + assert.include(targetDocument.getText(), 'tim = 1'); }); }); diff --git a/src/test/datascience/export/fileConverter.vscode.test.ts b/src/test/datascience/export/fileConverter.vscode.test.ts index 932f43bc3f7..137ec8a426b 100644 --- a/src/test/datascience/export/fileConverter.vscode.test.ts +++ b/src/test/datascience/export/fileConverter.vscode.test.ts @@ -75,7 +75,6 @@ suite('DataScience - File Converter', () => { instance(reporter), instance(appShell), instance(exportFileOpener), - instance(exportInterpreterFinder), instance(configuration) ); diff --git a/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json b/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json index 9b68ca94e32..e59cad34e39 100644 --- a/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json +++ b/src/test/datascience/extensionapi/exampleextension/ms-ai-tools-test/package.json @@ -1,70 +1,68 @@ { - "name": "ms-toolsai-test", - "displayName": "AI Tools Test Extension", - "description": "Extension for testing the API for talking to the ms-toolsai.jupyter extension", - "version": "0.0.1", - "publisher": "ms-toolsai", - "engines": { - "vscode": "^1.32.0" - }, - "license": "MIT", - "homepage": "https://github.com/Microsoft/vscode-jupyter", - "repository": { - "type": "git", - "url": "https://github.com/Microsoft/vscode-jupyter" - }, - "bugs": { - "url": "https://github.com/Microsoft/vscode-jupyter/issues" - }, - "qna": "https://stackoverflow.com/questions/tagged/visual-studio-code+python", - "categories": [ - "Other" - ], - "activationEvents": [ - "*" - ], - "main": "./dist/extension", - "contributes": { - "pythonRemoteServerProvider": [ - { - "id": "RemoteServerPickerExample" - } - ], - "commands": [ - { - "command": "ms-toolsai-test.createBlankNotebook", - "title": "Create new blank Julia notebook", - "category": "Notebook" - } - ], + "name": "ms-toolsai-test", + "displayName": "AI Tools Test Extension", + "description": "Extension for testing the API for talking to the ms-toolsai.jupyter extension", + "version": "0.0.1", + "publisher": "ms-toolsai", + "engines": { + "vscode": "^1.32.0" + }, + "license": "MIT", + "homepage": "https://github.com/Microsoft/vscode-jupyter", + "repository": { + "type": "git", + "url": "https://github.com/Microsoft/vscode-jupyter" + }, + "bugs": { + "url": "https://github.com/Microsoft/vscode-jupyter/issues" + }, + "qna": "https://stackoverflow.com/questions/tagged/visual-studio-code+python", + "categories": [ + "Other" + ], + "activationEvents": ["*"], + "main": "./dist/extension", + "contributes": { + "pythonRemoteServerProvider": [ + { + "id": "RemoteServerPickerExample" + } + ], + "commands": [ + { + "command": "ms-toolsai-test.createBlankNotebook", + "title": "Create new blank Julia notebook", + "category": "Notebook" + } + ], "jupyter.kernels": [ - { - "title": "Julia", - "defaultLanguage": "julia" - } - ] - }, - "scripts": { - "vscode:prepublish": "webpack --mode production", - "webpack": "webpack --mode development", - "webpack-dev": "webpack --mode development --watch", - "test-compile": "tsc -p ./", - "lint": "eslint . --ext .ts,.tsx", - "package": "npm run vscode:prepublish && vsce package -o ms-toolsai-test.vsix" - }, - "devDependencies": { - "@types/jquery": "^3.5.0", - "@types/node": "^12.12.0", - "@types/vscode": "^1.32.0", - "@typescript-eslint/eslint-plugin": "^3.0.2", - "@typescript-eslint/parser": "^3.0.2", - "eslint": "^7.1.0", - "ts-loader": "^7.0.5", - "typescript": "^3.9.4", - "webpack": "^4.43.0", - "webpack-cli": "^3.3.11" - }, - "dependencies": { - "uuid": "^8.2.0" - } -} + { + "title": "Julia", + "defaultLanguage": "julia" + } + ] + }, + "scripts": { + "vscode:prepublish": "webpack --mode production", + "webpack": "webpack --mode development", + "webpack-dev": "webpack --mode development --watch", + "test-compile": "tsc -p ./", + "lint": "eslint . --ext .ts,.tsx", + "package": "npm run vscode:prepublish && vsce package -o ms-toolsai-test.vsix" + }, + "devDependencies": { + "@types/jquery": "^3.5.0", + "@types/node": "^12.12.0", + "@types/vscode": "^1.32.0", + "@typescript-eslint/eslint-plugin": "^3.0.2", + "@typescript-eslint/parser": "^3.0.2", + "eslint": "^7.1.0", + "ts-loader": "^7.0.5", + "typescript": "^3.9.4", + "webpack": "^4.43.0", + "webpack-cli": "^3.3.11" + }, + "dependencies": { + "uuid": "^8.2.0" + } +} \ No newline at end of file From 552d2b1f92f36f11d7b9d6cc77dce4232c783afb Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 12:01:39 -0700 Subject: [PATCH 08/12] Update when clauses --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 2fa4daff732..f364afda35f 100644 --- a/package.json +++ b/package.json @@ -347,25 +347,25 @@ "command": "jupyter.export", "title": "%DataScience.notebookExportAs%", "category": "Jupyter", - "enablement": "isWorkspaceTrusted && !jupyter.webExtension" + "enablement": "isWorkspaceTrusted && !jupyter.webExtension || isWorkspaceTrusted && jupyter.webExtension && notebookKernel =~ /^ms-toolsai.jupyter\\//" }, { "command": "jupyter.exportAsPythonScript", "title": "%jupyter.command.jupyter.exportAsPythonScript.title%", "category": "Jupyter", - "enablement": "isWorkspaceTrusted && !jupyter.webExtension" + "enablement": "isWorkspaceTrusted && !jupyter.webExtension || isWorkspaceTrusted && jupyter.webExtension && notebookKernel =~ /^ms-toolsai.jupyter\\//" }, { "command": "jupyter.exportToHTML", "title": "%jupyter.command.jupyter.exportToHTML.title%", "category": "Jupyter", - "enablement": "isWorkspaceTrusted && !jupyter.webExtension" + "enablement": "isWorkspaceTrusted && !jupyter.webExtension || isWorkspaceTrusted && jupyter.webExtension && notebookKernel =~ /^ms-toolsai.jupyter\\//" }, { "command": "jupyter.exportToPDF", "title": "%jupyter.command.jupyter.exportToPDF.title%", "category": "Jupyter", - "enablement": "isWorkspaceTrusted && !jupyter.webExtension" + "enablement": "isWorkspaceTrusted && !jupyter.webExtension || isWorkspaceTrusted && jupyter.webExtension && notebookKernel =~ /^ms-toolsai.jupyter\\//" }, { "command": "jupyter.selectJupyterInterpreter", From 602d41b75109e269a67e4434c96103d0cd03a4c8 Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 14:54:36 -0700 Subject: [PATCH 09/12] target uri. --- src/platform/export/exportBase.node.ts | 30 +++--------- src/platform/export/exportBase.web.ts | 21 ++------- src/platform/export/exportToHTML.ts | 12 ++--- src/platform/export/exportToPDF.ts | 12 ++--- src/platform/export/exportToPython.ts | 12 ++--- src/platform/export/exportToPythonPlain.ts | 15 +----- src/platform/export/exportUtil.node.ts | 26 +++++++++- src/platform/export/exportUtil.ts | 18 ++++++- src/platform/export/fileConverter.node.ts | 16 +++---- src/platform/export/fileConverter.ts | 47 ++++++++++--------- src/platform/export/types.ts | 14 ++---- .../export/exportToHTML.vscode.test.ts | 5 +- .../export/exportToPython.vscode.test.ts | 5 +- .../export/fileConverter.vscode.test.ts | 1 + 14 files changed, 107 insertions(+), 127 deletions(-) diff --git a/src/platform/export/exportBase.node.ts b/src/platform/export/exportBase.node.ts index aca4f27dab6..ff76cc531e8 100644 --- a/src/platform/export/exportBase.node.ts +++ b/src/platform/export/exportBase.node.ts @@ -29,31 +29,25 @@ export class ExportBase implements INbConvertExport, IExportBase { public async export( _sourceDocument: NotebookDocument, + _target: Uri, _interpreter: PythonEnvironment, - _defaultFileName: string | undefined, _token: CancellationToken - ): Promise { - return undefined; + ): Promise { + return; } @reportAction(ReportableAction.PerformingExport) public async executeCommand( sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, format: ExportFormat, interpreter: PythonEnvironment | undefined, token: CancellationToken - ): Promise { + ): Promise { if (token.isCancellationRequested) { return; } - let target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - - if (!target) { - return; - } - interpreter = await this.exportInterpreterFinder.getExportInterpreter(interpreter); if (format === ExportFormat.python) { @@ -120,19 +114,7 @@ export class ExportBase implements INbConvertExport, IExportBase { tempTarget.dispose(); } - return target; - } - - async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { - let target; - - if (format !== ExportFormat.python) { - target = await this.filePicker.showDialog(format, source, defaultFileName); - } else { - target = Uri.file((await this.fs.createTemporaryLocalFile('.py')).filePath); - } - - return target; + return; } private async makeSourceFile(target: Uri, contents: string, tempDir: TemporaryDirectory): Promise { diff --git a/src/platform/export/exportBase.web.ts b/src/platform/export/exportBase.web.ts index 0c350bcd828..edd4fef3c43 100644 --- a/src/platform/export/exportBase.web.ts +++ b/src/platform/export/exportBase.web.ts @@ -29,21 +29,21 @@ export class ExportBase implements INbConvertExport, IExportBase { public async export( _sourceDocument: NotebookDocument, + _target: Uri, _interpreter: PythonEnvironment, - _defaultFileName: string | undefined, _token: CancellationToken - ): Promise { + ): Promise { return undefined; } @reportAction(ReportableAction.PerformingExport) async executeCommand( sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, format: ExportFormat, _interpreter: PythonEnvironment, _token: CancellationToken - ): Promise { + ): Promise { const kernel = this.kernelProvider.get(sourceDocument.uri); if (!kernel) { // trace error @@ -76,11 +76,6 @@ export class ExportBase implements INbConvertExport, IExportBase { break; } - let target = await this.getTargetFile(format, sourceDocument.uri, defaultFileName); - if (target === undefined) { - return; - } - await kernel.session!.invokeWithFileSynced(contents, async (file) => { const pwd = await this.getCWD(kernel); const filePath = `${pwd}/${file.filePath}`; @@ -111,7 +106,7 @@ export class ExportBase implements INbConvertExport, IExportBase { } }); - return target; + return; } else { // no op } @@ -163,12 +158,6 @@ export class ExportBase implements INbConvertExport, IExportBase { return text; } - private async getTargetFile(format: ExportFormat, source: Uri, defaultFileName?: string): Promise { - let target = await this.filePicker.showDialog(format, source, defaultFileName); - - return target; - } - private async getCWD(kernel: IKernel) { const outputs = await executeSilently(kernel.session!, `import os;os.getcwd();`); if (outputs.length === 0) { diff --git a/src/platform/export/exportToHTML.ts b/src/platform/export/exportToHTML.ts index afc2a593fdb..9514c326b77 100644 --- a/src/platform/export/exportToHTML.ts +++ b/src/platform/export/exportToHTML.ts @@ -8,16 +8,10 @@ export class ExportToHTML implements INbConvertExport { constructor(@inject(IExportBase) protected readonly exportBase: IExportBase) {} public async export( sourceDocument: NotebookDocument, + target: Uri, interpreter: PythonEnvironment, - defaultFileName: string | undefined, token: CancellationToken - ): Promise { - return await this.exportBase.executeCommand( - sourceDocument, - defaultFileName, - ExportFormat.html, - interpreter, - token - ); + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.html, interpreter, token); } } diff --git a/src/platform/export/exportToPDF.ts b/src/platform/export/exportToPDF.ts index 0c108c49540..97c6464921a 100644 --- a/src/platform/export/exportToPDF.ts +++ b/src/platform/export/exportToPDF.ts @@ -9,16 +9,10 @@ export class ExportToPDF implements INbConvertExport { public async export( sourceDocument: NotebookDocument, + target: Uri, interpreter: PythonEnvironment, - defaultFileName: string | undefined, token: CancellationToken - ): Promise { - return await this.exportBase.executeCommand( - sourceDocument, - defaultFileName, - ExportFormat.pdf, - interpreter, - token - ); + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.pdf, interpreter, token); } } diff --git a/src/platform/export/exportToPython.ts b/src/platform/export/exportToPython.ts index e538fa9435b..fdd57c4d2cf 100644 --- a/src/platform/export/exportToPython.ts +++ b/src/platform/export/exportToPython.ts @@ -9,16 +9,10 @@ export class ExportToPython implements INbConvertExport { public async export( sourceDocument: NotebookDocument, + target: Uri, interpreter: PythonEnvironment, - defaultFileName: string | undefined, token: CancellationToken - ): Promise { - return await this.exportBase.executeCommand( - sourceDocument, - defaultFileName, - ExportFormat.python, - interpreter, - token - ); + ): Promise { + await this.exportBase.executeCommand(sourceDocument, target, ExportFormat.python, interpreter, token); } } diff --git a/src/platform/export/exportToPythonPlain.ts b/src/platform/export/exportToPythonPlain.ts index 25b8fc4727b..507649c87e4 100644 --- a/src/platform/export/exportToPythonPlain.ts +++ b/src/platform/export/exportToPythonPlain.ts @@ -8,7 +8,7 @@ import { CancellationToken, NotebookCell, NotebookCellKind, NotebookDocument, Ur import { appendLineFeed } from '../../webviews/webview-side/common'; import { IFileSystem, IPlatformService } from '../common/platform/types'; import { IConfigurationService } from '../common/types'; -import { ExportFormat, IExport, IExportDialog } from './types'; +import { IExport } from './types'; // Handles exporting a NotebookDocument to python @injectable() @@ -16,7 +16,6 @@ export class ExportToPythonPlain implements IExport { public constructor( @inject(IFileSystem) private readonly fs: IFileSystem, @inject(IConfigurationService) private readonly configuration: IConfigurationService, - @inject(IExportDialog) private readonly filePicker: IExportDialog, @inject(IPlatformService) private platform: IPlatformService ) {} @@ -29,21 +28,11 @@ export class ExportToPythonPlain implements IExport { } // Export the given document to the target source file - public async export( - sourceDocument: NotebookDocument, - defaultFileName: string | undefined, - token: CancellationToken - ): Promise { + public async export(sourceDocument: NotebookDocument, target: Uri, token: CancellationToken): Promise { if (token.isCancellationRequested) { return; } - let target = await this.filePicker.showDialog(ExportFormat.python, sourceDocument.uri, defaultFileName); - - if (!target) { - return; - } - const contents = this.exportDocument(sourceDocument); await this.writeFile(target, contents); } diff --git a/src/platform/export/exportUtil.node.ts b/src/platform/export/exportUtil.node.ts index 8aabca428d3..365ed5dada2 100644 --- a/src/platform/export/exportUtil.node.ts +++ b/src/platform/export/exportUtil.node.ts @@ -8,11 +8,17 @@ import { IFileSystemNode } from '../common/platform/types.node'; import { sleep } from '../common/utils/async'; import { ExportUtilBase } from './exportUtil'; import { IExtensions } from '../common/types'; +import { ExportFormat, IExportDialog } from './types'; +import { Uri } from 'vscode'; @injectable() export class ExportUtil extends ExportUtilBase { - constructor(@inject(IFileSystemNode) private fs: IFileSystemNode, @inject(IExtensions) extensions: IExtensions) { - super(extensions); + constructor( + @inject(IFileSystemNode) private fs: IFileSystemNode, + @inject(IExtensions) extensions: IExtensions, + @inject(IExportDialog) filePicker: IExportDialog + ) { + super(extensions, filePicker); } public async generateTempDir(): Promise { @@ -39,6 +45,22 @@ export class ExportUtil extends ExportUtilBase { }; } + override async getTargetFile( + format: ExportFormat, + source: Uri, + defaultFileName?: string | undefined + ): Promise { + let target; + + if (format !== ExportFormat.python) { + target = await this.filePicker.showDialog(format, source, defaultFileName); + } else { + target = Uri.file((await this.fs.createTemporaryLocalFile('.py')).filePath); + } + + return target; + } + public async makeFileInDirectory(contents: string, fileName: string, dirPath: string): Promise { const newFilePath = path.join(dirPath, fileName); diff --git a/src/platform/export/exportUtil.ts b/src/platform/export/exportUtil.ts index ad87a11aed4..dca3d2757b3 100644 --- a/src/platform/export/exportUtil.ts +++ b/src/platform/export/exportUtil.ts @@ -1,10 +1,14 @@ import { inject, injectable } from 'inversify'; -import { NotebookCellData, NotebookData, NotebookDocument } from 'vscode'; +import { NotebookCellData, NotebookData, NotebookDocument, Uri } from 'vscode'; import { IExtensions } from '../common/types'; +import { ExportFormat, IExportDialog } from './types'; @injectable() export class ExportUtilBase { - constructor(@inject(IExtensions) private readonly extensions: IExtensions) {} + constructor( + @inject(IExtensions) private readonly extensions: IExtensions, + @inject(IExportDialog) protected readonly filePicker: IExportDialog + ) {} async getContent(document: NotebookDocument): Promise { const serializerApi = this.extensions.getExtension<{ exportNotebook: (notebook: NotebookData) => string }>( @@ -32,4 +36,14 @@ export class ExportUtilBase { notebookData.metadata = document.metadata; return serializerApi.exports.exportNotebook(notebookData); } + + async getTargetFile( + format: ExportFormat, + source: Uri, + defaultFileName?: string | undefined + ): Promise { + let target = await this.filePicker.showDialog(format, source, defaultFileName); + + return target; + } } diff --git a/src/platform/export/fileConverter.node.ts b/src/platform/export/fileConverter.node.ts index 039cc767650..fa8b680eb2b 100644 --- a/src/platform/export/fileConverter.node.ts +++ b/src/platform/export/fileConverter.node.ts @@ -8,6 +8,7 @@ import { ExportFileOpener } from './exportFileOpener'; import { ExportFormat, INbConvertExport, IExportDialog, IFileConverter, IExport } from './types'; import { IFileSystemNode } from '../common/platform/types.node'; import { FileConverter as FileConverterBase } from './fileConverter'; +import { ExportUtil } from './exportUtil.node'; // Class is responsible for file conversions (ipynb, py, pdf, html) and managing nb convert for some of those conversions @injectable() @@ -17,6 +18,7 @@ export class FileConverter extends FileConverterBase implements IFileConverter { @inject(INbConvertExport) @named(ExportFormat.html) exportToHTML: INbConvertExport, @inject(INbConvertExport) @named(ExportFormat.python) exportToPython: INbConvertExport, @inject(IExport) @named(ExportFormat.python) exportToPythonPlain: IExport, + @inject(ExportUtil) override readonly exportUtil: ExportUtil, @inject(IFileSystemNode) readonly fs: IFileSystemNode, @inject(IExportDialog) filePicker: IExportDialog, @inject(ProgressReporter) progressReporter: ProgressReporter, @@ -30,6 +32,7 @@ export class FileConverter extends FileConverterBase implements IFileConverter { exportToHTML, exportToPython, filePicker, + exportUtil, progressReporter, applicationShell, exportFileOpener @@ -39,24 +42,17 @@ export class FileConverter extends FileConverterBase implements IFileConverter { override async performExport( format: ExportFormat, sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, token: CancellationToken, candidateInterpreter?: PythonEnvironment ) { - let target: Uri | undefined; const pythonNbconvert = this.configuration.getSettings(sourceDocument.uri).pythonExportMethod === 'nbconvert'; if (format === ExportFormat.python && !pythonNbconvert) { // Unless selected by the setting use plain conversion for python script convert - target = await this.performPlainExport(format, sourceDocument, defaultFileName, token); + await this.performPlainExport(format, sourceDocument, target, token); } else { - target = await this.performNbConvertExport( - sourceDocument, - format, - defaultFileName, - candidateInterpreter, - token - ); + await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); } if (target) { diff --git a/src/platform/export/fileConverter.ts b/src/platform/export/fileConverter.ts index 1a2f5aac178..8964dfa95dd 100644 --- a/src/platform/export/fileConverter.ts +++ b/src/platform/export/fileConverter.ts @@ -13,6 +13,7 @@ import { traceError } from '../logging'; import { ProgressReporter } from '../progress/progressReporter'; import { PythonEnvironment } from '../pythonEnvironments/info'; import { ExportFileOpener } from './exportFileOpener'; +import { ExportUtilBase } from './exportUtil'; import { ExportFormat, IExport, IExportDialog, IFileConverter, INbConvertExport } from './types'; @injectable() @@ -23,6 +24,7 @@ export class FileConverter implements IFileConverter { @inject(INbConvertExport) @named(ExportFormat.html) private readonly exportToHTML: INbConvertExport, @inject(INbConvertExport) @named(ExportFormat.python) private readonly exportToPython: INbConvertExport, @inject(IExportDialog) protected readonly filePicker: IExportDialog, + @inject(ExportUtilBase) protected readonly exportUtil: ExportUtilBase, @inject(ProgressReporter) private readonly progressReporter: ProgressReporter, @inject(IApplicationShell) private readonly applicationShell: IApplicationShell, @inject(ExportFileOpener) protected readonly exportFileOpener: ExportFileOpener @@ -72,7 +74,11 @@ export class FileConverter implements IFileConverter { candidateInterpreter?: PythonEnvironment ): Promise { try { - await this.performExport(format, sourceDocument, defaultFileName, token, candidateInterpreter); + let target = await this.exportUtil.getTargetFile(format, sourceDocument.uri, defaultFileName); + if (!target) { + return; + } + await this.performExport(format, sourceDocument, target, token, candidateInterpreter); } catch (e) { traceError('Export failed', e); sendTelemetryEvent(Telemetry.ExportNotebookAsFailed, undefined, { format: format }); @@ -88,23 +94,16 @@ export class FileConverter implements IFileConverter { protected async performExport( format: ExportFormat, sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, token: CancellationToken, candidateInterpreter?: PythonEnvironment ) { - let target: Uri | undefined; // For web, we perform plain export for Python if (format === ExportFormat.python) { // Unless selected by the setting use plain conversion for python script convert - target = await this.performPlainExport(format, sourceDocument, defaultFileName, token); + await this.performPlainExport(format, sourceDocument, target, token); } else { - target = await this.performNbConvertExport( - sourceDocument, - format, - defaultFileName, - candidateInterpreter, - token - ); + await this.performNbConvertExport(sourceDocument, format, target, candidateInterpreter, token); } if (target) { @@ -115,45 +114,49 @@ export class FileConverter implements IFileConverter { protected async performPlainExport( format: ExportFormat, sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, cancelToken: CancellationToken ): Promise { - switch (format) { - case ExportFormat.python: - return await this.exportToPythonPlain.export(sourceDocument, defaultFileName, cancelToken); - break; + if (target) { + switch (format) { + case ExportFormat.python: + await this.exportToPythonPlain.export(sourceDocument, target, cancelToken); + break; + } } + + return target; } protected async performNbConvertExport( sourceDocument: NotebookDocument, format: ExportFormat, - defaultFileName: string | undefined, + target: Uri, interpreter: PythonEnvironment | undefined, cancelToken: CancellationToken ) { try { - return await this.exportToFormat(sourceDocument, defaultFileName, format, interpreter, cancelToken); + return await this.exportToFormat(sourceDocument, target, format, interpreter, cancelToken); } finally { } } protected async exportToFormat( sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, format: ExportFormat, interpreter: PythonEnvironment | undefined, cancelToken: CancellationToken ) { switch (format) { case ExportFormat.python: - return await this.exportToPython.export(sourceDocument, interpreter, defaultFileName, cancelToken); + return await this.exportToPython.export(sourceDocument, target, interpreter, cancelToken); case ExportFormat.pdf: - return await this.exportToPDF.export(sourceDocument, interpreter, defaultFileName, cancelToken); + return await this.exportToPDF.export(sourceDocument, target, interpreter, cancelToken); case ExportFormat.html: - return await this.exportToHTML.export(sourceDocument, interpreter, defaultFileName, cancelToken); + return await this.exportToHTML.export(sourceDocument, target, interpreter, cancelToken); default: break; diff --git a/src/platform/export/types.ts b/src/platform/export/types.ts index dcc0fab900c..691048fd693 100644 --- a/src/platform/export/types.ts +++ b/src/platform/export/types.ts @@ -23,30 +23,26 @@ export const INbConvertExport = Symbol('INbConvertExport'); export interface INbConvertExport { export( sourceDocument: NotebookDocument, + target: Uri, interpreter: PythonEnvironment | undefined, - defaultFileName: string | undefined, token: CancellationToken - ): Promise; + ): Promise; } export const IExportBase = Symbol('IExportBase'); export interface IExportBase { executeCommand( sourceDocument: NotebookDocument, - defaultFileName: string | undefined, + target: Uri, format: ExportFormat, interpreter: PythonEnvironment, token: CancellationToken - ): Promise; + ): Promise; } export const IExport = Symbol('IExport'); export interface IExport { - export( - sourceDocument: NotebookDocument, - defaultFileName: string | undefined, - token: CancellationToken - ): Promise; + export(sourceDocument: NotebookDocument, target: Uri, token: CancellationToken): Promise; } export const IExportDialog = Symbol('IExportDialog'); diff --git a/src/test/datascience/export/exportToHTML.vscode.test.ts b/src/test/datascience/export/exportToHTML.vscode.test.ts index fa5517a5564..1dba5e45fc1 100644 --- a/src/test/datascience/export/exportToHTML.vscode.test.ts +++ b/src/test/datascience/export/exportToHTML.vscode.test.ts @@ -24,12 +24,15 @@ suite('DataScience - Export HTML', function () { const fileSystem = api.serviceContainer.get(IFileSystemNode); const exportToHTML = api.serviceContainer.get(INbConvertExport, ExportFormat.html); const exportInterpreterFinder = api.serviceContainer.get(ExportInterpreterFinder); + const file = await fileSystem.createTemporaryLocalFile('.html'); + const target = Uri.file(file.filePath); + await file.dispose(); const token = new CancellationTokenSource(); const interpreter = await exportInterpreterFinder.getExportInterpreter(); const document = await workspace.openNotebookDocument( Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')) ); - const target = await exportToHTML.export(document, interpreter, undefined, token.token); + await exportToHTML.export(document, target, interpreter, token.token); assert.exists(target); assert.equal(await fileSystem.localFileExists(target!.fsPath), true); const fileContents = await fileSystem.readLocalFile(target!.fsPath); diff --git a/src/test/datascience/export/exportToPython.vscode.test.ts b/src/test/datascience/export/exportToPython.vscode.test.ts index 0140d5263fa..0972bf99d26 100644 --- a/src/test/datascience/export/exportToPython.vscode.test.ts +++ b/src/test/datascience/export/exportToPython.vscode.test.ts @@ -6,6 +6,7 @@ import { assert } from 'chai'; import * as path from '../../../platform/vscode-path/path'; import { CancellationTokenSource, Uri, workspace } from 'vscode'; import { IDocumentManager } from '../../../platform/common/application/types'; +import { IFileSystemNode } from '../../../platform/common/platform/types.node'; import { ExportInterpreterFinder } from '../../../platform/export/exportInterpreterFinder.node'; import { INbConvertExport, ExportFormat } from '../../../platform/export/types'; import { IExtensionTestApi } from '../../common.node'; @@ -21,14 +22,16 @@ suite('DataScience - Export Python', function () { teardown(closeActiveWindows); suiteTeardown(closeActiveWindows); test('Export To Python', async () => { + const fileSystem = api.serviceContainer.get(IFileSystemNode); const exportToPython = api.serviceContainer.get(INbConvertExport, ExportFormat.python); + const target = Uri.file((await fileSystem.createTemporaryLocalFile('.py')).filePath); const token = new CancellationTokenSource(); const exportInterpreterFinder = api.serviceContainer.get(ExportInterpreterFinder); const interpreter = await exportInterpreterFinder.getExportInterpreter(); const document = await workspace.openNotebookDocument( Uri.file(path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'export', 'test.ipynb')) ); - const target = await exportToPython.export(document, interpreter, undefined, token.token); + await exportToPython.export(document, target, interpreter, token.token); assert.exists(target); const documentManager = api.serviceContainer.get(IDocumentManager); const targetDocument = await documentManager.openTextDocument(target!); diff --git a/src/test/datascience/export/fileConverter.vscode.test.ts b/src/test/datascience/export/fileConverter.vscode.test.ts index 137ec8a426b..15c131e55a9 100644 --- a/src/test/datascience/export/fileConverter.vscode.test.ts +++ b/src/test/datascience/export/fileConverter.vscode.test.ts @@ -70,6 +70,7 @@ suite('DataScience - File Converter', () => { instance(exportHtml), instance(exportPython), instance(exportPythonPlain), + instance(exportUtil), instance(fileSystem), instance(filePicker), instance(reporter), From 2a6e055184428a214a86bef587773222e36bda45 Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 15:28:29 -0700 Subject: [PATCH 10/12] false negative integration test. --- src/test/datascience/export/fileConverter.vscode.test.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/datascience/export/fileConverter.vscode.test.ts b/src/test/datascience/export/fileConverter.vscode.test.ts index 15c131e55a9..bcb944ee6a8 100644 --- a/src/test/datascience/export/fileConverter.vscode.test.ts +++ b/src/test/datascience/export/fileConverter.vscode.test.ts @@ -56,6 +56,7 @@ suite('DataScience - File Converter', () => { // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function when(exportUtil.generateTempDir()).thenResolve({ path: 'test', dispose: () => {} }); when(exportUtil.makeFileInDirectory(anything(), anything(), anything())).thenResolve('foo'); + when(exportUtil.getTargetFile(anything(), anything(), anything())).thenResolve(Uri.file('bar')); // eslint-disable-next-line no-empty,@typescript-eslint/no-empty-function when(fileSystem.createTemporaryLocalFile(anything())).thenResolve({ filePath: 'test', dispose: () => {} }); when(exportPdf.export(anything(), anything(), anything(), anything())).thenResolve(); @@ -80,15 +81,11 @@ suite('DataScience - File Converter', () => { ); // Stub out the getContent inner method of the ExportManager we don't care about the content returned - const getContentStub = sinon.stub(FileConverter.prototype, 'getContent' as any); + const getContentStub = sinon.stub(ExportUtil.prototype, 'getContent' as any); getContentStub.resolves('teststring'); }); teardown(() => sinon.restore()); - test('Remove svg is called when exporting to PDF', async () => { - await fileConverter.export(ExportFormat.pdf, {} as any); - verify(exportUtil.removeSvgs(anything())).once(); - }); test('Erorr message is shown if export fails', async () => { when(exportHtml.export(anything(), anything(), anything(), anything())).thenThrow(new Error('failed...')); await fileConverter.export(ExportFormat.html, {} as any); From f437946fe8c07788febe79f27b2d085906a7c525 Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 16:25:01 -0700 Subject: [PATCH 11/12] Trigger document to be saved. --- src/platform/serviceRegistry.node.ts | 2 ++ src/test/datascience/notebook/exportFull.vscode.test.ts | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/platform/serviceRegistry.node.ts b/src/platform/serviceRegistry.node.ts index 1c51a5b45bf..24b0b058283 100644 --- a/src/platform/serviceRegistry.node.ts +++ b/src/platform/serviceRegistry.node.ts @@ -63,6 +63,7 @@ import { FileSystem } from './common/platform/fileSystem.node'; import { WorkspaceService } from './common/application/workspace.node'; import { ExtensionSideRenderer, IExtensionSideRenderer } from '../webviews/extension-side/renderer'; import { OutputCommandListener } from './logging/outputCommandListener'; +import { ExportUtilBase } from './export/exportUtil'; export function registerTypes(context: IExtensionContext, serviceManager: IServiceManager, isDevMode: boolean) { serviceManager.addSingleton(FileSystem, FileSystem); @@ -115,6 +116,7 @@ export function registerTypes(context: IExtensionContext, serviceManager: IServi serviceManager.addSingleton(INbConvertExport, ExportToPython, ExportFormat.python); serviceManager.addSingleton(INbConvertExport, ExportBase, 'Export Base'); serviceManager.addSingleton(IExport, ExportToPythonPlain, ExportFormat.python); + serviceManager.addSingleton(ExportUtilBase, ExportUtilBase); serviceManager.addSingleton(ExportUtil, ExportUtil); serviceManager.addSingleton(IExportDialog, ExportDialog); serviceManager.addSingleton(INotebookWatcher, NotebookWatcher); diff --git a/src/test/datascience/notebook/exportFull.vscode.test.ts b/src/test/datascience/notebook/exportFull.vscode.test.ts index 475deb6f2cf..defa22ddd24 100644 --- a/src/test/datascience/notebook/exportFull.vscode.test.ts +++ b/src/test/datascience/notebook/exportFull.vscode.test.ts @@ -20,6 +20,7 @@ import { hijackPrompt, insertCodeCell, insertMarkdownCell, + saveActiveNotebook, startJupyterServer, workAroundVSCodeNotebookStartPages } from './helper.node'; @@ -33,7 +34,7 @@ import { Product } from '../../../kernels/installer/types'; const expectedPromptMessageSuffix = `requires ${ProductNames.get(Product.ipykernel)!} to be installed.`; /* eslint-disable @typescript-eslint/no-explicit-any, no-invalid-this */ -suite('DataScience - VSCode Notebook - (Export) (slow)', function () { +suite.only('DataScience - VSCode Notebook - (Export) (slow)', function () { let api: IExtensionTestApi; const disposables: IDisposable[] = []; let vscodeNotebook: IVSCodeNotebook; @@ -91,6 +92,7 @@ suite('DataScience - VSCode Notebook - (Export) (slow)', function () { await insertCodeCell('print("Hello World")', { index: 0 }); await insertMarkdownCell('# Markdown Header\nmarkdown string', { index: 1 }); await insertCodeCell('%whos', { index: 2 }); + await saveActiveNotebook(); const deferred = createDeferred(); const onDidChangeDispose = window.onDidChangeActiveTextEditor((te) => { @@ -121,6 +123,7 @@ suite('DataScience - VSCode Notebook - (Export) (slow)', function () { await insertCodeCell('print("Hello World")', { index: 0 }); await insertMarkdownCell('# Markdown Header\nmarkdown string', { index: 1 }); await insertCodeCell('%whos\n!shellcmd', { index: 2 }); + await saveActiveNotebook(); const deferred = createDeferred(); const onDidChangeDispose = window.onDidChangeActiveTextEditor((te) => { @@ -153,6 +156,7 @@ suite('DataScience - VSCode Notebook - (Export) (slow)', function () { await insertCodeCell('print("Hello World")', { index: 0 }); await insertMarkdownCell('# Markdown Header\nmarkdown string', { index: 1 }); await insertCodeCell('%whos\n!shellcmd', { index: 2 }); + await saveActiveNotebook(); const deferred = createDeferred(); const onDidChangeDispose = window.onDidChangeActiveTextEditor((te) => { From 53d6c57e1127d8abf5db175ab486b15375476dc7 Mon Sep 17 00:00:00 2001 From: rebornix Date: Thu, 19 May 2022 16:32:59 -0700 Subject: [PATCH 12/12] :lipstick: --- src/test/datascience/notebook/exportFull.vscode.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/datascience/notebook/exportFull.vscode.test.ts b/src/test/datascience/notebook/exportFull.vscode.test.ts index defa22ddd24..2ea571c42c7 100644 --- a/src/test/datascience/notebook/exportFull.vscode.test.ts +++ b/src/test/datascience/notebook/exportFull.vscode.test.ts @@ -34,7 +34,7 @@ import { Product } from '../../../kernels/installer/types'; const expectedPromptMessageSuffix = `requires ${ProductNames.get(Product.ipykernel)!} to be installed.`; /* eslint-disable @typescript-eslint/no-explicit-any, no-invalid-this */ -suite.only('DataScience - VSCode Notebook - (Export) (slow)', function () { +suite('DataScience - VSCode Notebook - (Export) (slow)', function () { let api: IExtensionTestApi; const disposables: IDisposable[] = []; let vscodeNotebook: IVSCodeNotebook;