Debug attach first steps

news/1 Enhancements/5900.md

@@ -0,0 +1 @@
+Hook up ptvsd debugger to jupyter UI

package.json

@@ -328,6 +328,11 @@
                 "title": "%python.command.python.datascience.runallcellsabove.palette.title%",
                 "category": "Python"
             },
+            {
+                "command": "python.datascience.debugcurrentcell.palette",
+                "title": "%python.command.python.datascience.debugcurrentcell.palette.title%",
+                "category": "Python"
+            },
             {
                 "command": "python.datascience.execSelectionInteractive",
                 "title": "%python.command.python.datascience.execSelectionInteractive.title%",
@@ -616,6 +621,12 @@
                     "category": "Python",
                     "when": "python.datascience.hascodecells && python.datascience.featureenabled"
                 },
+                {
+                    "command": "python.datascience.debugcurrentcell.palette",
+                    "title": "%python.command.python.datascience.debugcurrentcell.palette.title%",
+                    "category": "Python",
+                    "when": "python.datascience.hascodecells && python.datascience.featureenabled"
+                },
                 {
                     "command": "python.datascience.showhistorypane",
                     "title": "%python.command.python.datascience.showhistorypane.title%",
@@ -1327,6 +1338,12 @@
                     "description": "Allow for connecting the Python Interactive window to a https Jupyter server that does not have valid certificates. This can be a security risk, so only use for known and trusted servers.",
                     "scope": "resource"
                 },
+                "python.dataScience.enableDebugging": {
+                    "type": "boolean",
+                    "default": false,
+                    "description": "When starting a Python Interactive session import and enable the ptvsd debugger in the kernel.",
+                    "scope": "resource"
+                },
                 "python.disableInstallationCheck": {
                     "type": "boolean",
                     "default": false,

package.nls.json

@@ -33,6 +33,7 @@
     "python.command.python.datascience.runcellandallbelow.title": "Run Below",
     "python.command.python.datascience.runallcellsabove.palette.title": "Run Cells Above Current Cell",
     "python.command.python.datascience.runcurrentcellandallbelow.palette.title": "Run Current Cell and Below",
+    "python.command.python.datascience.debugcurrentcell.palette.title": "Debug Current Cell",
     "python.command.python.datascience.runtoline.title": "Run To Line in Python Interactive window",
     "python.command.python.datascience.runfromline.title": "Run From Line in Python Interactive window",
     "python.command.python.datascience.runcurrentcell.title": "Run Current Cell",

src/client/common/application/commands.ts

@@ -22,6 +22,7 @@ interface ICommandNameWithoutArgumentTypeMapping {
     [Commands.Set_ShebangInterpreter]: [];
     [Commands.Run_Linter]: [];
     [Commands.Enable_Linter]: [];
+    ['workbench.action.debug.stop']: [];
     ['workbench.action.reloadWindow']: [];
     ['editor.action.formatDocument']: [];
     ['editor.action.rename']: [];
@@ -100,6 +101,7 @@ export interface ICommandNameArgumentTypeMapping extends ICommandNameWithoutArgu
     [DSCommands.RunCellAndAllBelow]: [string, number, number];
     [DSCommands.RunAllCellsAbovePalette]: [];
     [DSCommands.RunCellAndAllBelowPalette]: [];
+    [DSCommands.DebugCurrentCellPalette]: [];
     [DSCommands.RunToLine]: [string, number, number];
     [DSCommands.RunFromLine]: [string, number, number];
     [DSCommands.ImportNotebook]: [undefined | Uri, undefined | CommandSource];

src/client/common/types.ts

@@ -319,6 +319,7 @@ export interface IDataScienceSettings {
     autoPreviewNotebooksInInteractivePane?: boolean;
     allowUnauthorizedRemoteConnection?: boolean;
     askForKernelRestart?: boolean;
+    enableDebugging?: boolean;
 }
 
 export const IConfigurationService = Symbol('IConfigurationService');

src/client/datascience/constants.ts

@@ -33,6 +33,7 @@ export namespace Commands {
     export const ExecSelectionInInteractiveWindow = 'python.datascience.execSelectionInteractive';
     export const RunFileInInteractiveWindows = 'python.datascience.runFileInteractive';
     export const AddCellBelow = 'python.datascience.addcellbelow';
+    export const DebugCurrentCellPalette = 'python.datascience.debugcurrentcell.palette';
 }
 
 export namespace EditorContexts {
@@ -125,6 +126,7 @@ export enum Telemetry {
     GetPasswordFailure = 'DATASCIENCE.GET_PASSWORD_FAILURE',
     GetPasswordSuccess = 'DATASCIENCE.GET_PASSWORD_SUCCESS',
     OpenPlotViewer = 'DATASCIENCE.OPEN_PLOT_VIEWER',
+    DebugCurrentCell = 'DATASCIENCE.DEBUG_CURRENT_CELL',
     CodeLensAverageAcquisitionTime = 'DATASCIENCE.CODE_LENS_ACQ_TIME',
     ClassConstructionTime = 'DATASCIENCE.CLASS_CONSTRUCTION_TIME',
     FindJupyterCommand = 'DATASCIENCE.FIND_JUPYTER_COMMAND',

src/client/datascience/datascience.ts

@@ -287,6 +287,20 @@ export class DataScience implements IDataScience {
         }
     }
 
+    private async debugCurrentCell(): Promise<void> {
+        this.dataScienceSurveyBanner.showBanner().ignoreErrors();
+
+        const currentCodeLens = this.getCurrentCodeLens();
+        if (currentCodeLens) {
+            const activeCodeWatcher = this.getCurrentCodeWatcher();
+            if (activeCodeWatcher) {
+                return activeCodeWatcher.debugCurrentCell();
+            }
+        } else {
+            return Promise.resolve();
+        }
+    }
+
     private validateURI = (testURI: string): string | undefined | null => {
         try {
             // tslint:disable-next-line:no-unused-expression
@@ -360,6 +374,8 @@ export class DataScience implements IDataScience {
         this.disposableRegistry.push(disposable);
         disposable = this.commandManager.registerCommand(Commands.AddCellBelow, this.addCellBelow, this);
         this.disposableRegistry.push(disposable);
+        disposable = this.commandManager.registerCommand(Commands.DebugCurrentCellPalette, this.debugCurrentCell, this);
+        this.disposableRegistry.push(disposable);
         this.commandListeners.forEach((listener: IDataScienceCommandListener) => {
             listener.register(this.commandManager);
         });

src/client/datascience/editor-integration/codewatcher.ts

@@ -94,6 +94,16 @@ export class CodeWatcher implements ICodeWatcher {
         return this.codeLenses;
     }
 
+    @captureTelemetry(Telemetry.DebugCurrentCell)
+    public async debugCurrentCell() {
+        if (!this.documentManager.activeTextEditor || !this.documentManager.activeTextEditor.document) {
+            return Promise.resolve();
+        }
+
+        // Run the cell that matches the current cursor position.
+        return this.runMatchingCell(this.documentManager.activeTextEditor.selection, false, true);
+    }
+
     @captureTelemetry(Telemetry.RunAllCells)
     public async runAllCells() {
         // Run all of our code lenses, they should always be ordered in the file so we can just
@@ -245,17 +255,21 @@ export class CodeWatcher implements ICodeWatcher {
         }
     }
 
-    private async addCode(code: string, file: string, line: number, editor?: TextEditor) : Promise<void> {
+    private async addCode(code: string, file: string, line: number, editor?: TextEditor, debug?: boolean) : Promise<void> {
         try {
             const stopWatch = new StopWatch();
             const activeInteractiveWindow = await this.interactiveWindowProvider.getOrCreateActive();
-            await activeInteractiveWindow.addCode(code, file, line, editor, stopWatch);
+            if (debug) {
+                await activeInteractiveWindow.debugCode(code, file, line, editor, stopWatch);
+            } else {
+                await activeInteractiveWindow.addCode(code, file, line, editor, stopWatch);
+            }
         } catch (err) {
             this.handleError(err);
         }
     }
 
-    private async runMatchingCell(range: Range, advance?: boolean) {
+    private async runMatchingCell(range: Range, advance?: boolean, debug?: boolean) {
         const currentRunCellLens = this.getCurrentCellLens(range.start);
         const nextRunCellLens = this.getNextCellLens(range.start);
 
@@ -279,7 +293,7 @@ export class CodeWatcher implements ICodeWatcher {
             if (this.document) {
                 // Use that to get our code.
                 const code = this.document.getText(currentRunCellLens.range);
-                await this.addCode(code, this.getFileName(), range.start.line, this.documentManager.activeTextEditor);
+                await this.addCode(code, this.getFileName(), range.start.line, this.documentManager.activeTextEditor, debug);
             }
         }
     }

src/client/datascience/interactive-window/interactiveWindow.ts

@@ -48,6 +48,7 @@ import {
     IInteractiveWindowInfo,
     IInteractiveWindowListener,
     IInteractiveWindowProvider,
+    IJupyterDebugger,
     IJupyterExecution,
     IJupyterVariable,
     IJupyterVariables,
@@ -113,7 +114,8 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
         @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider,
         @inject(IDataViewerProvider) private dataExplorerProvider: IDataViewerProvider,
         @inject(IJupyterVariables) private jupyterVariables: IJupyterVariables,
-        @inject(INotebookImporter) private jupyterImporter: INotebookImporter
+        @inject(INotebookImporter) private jupyterImporter: INotebookImporter,
+        @inject(IJupyterDebugger) private jupyterDebugger: IJupyterDebugger
         ) {
         super(
             configuration,
@@ -178,7 +180,12 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
 
     public addCode(code: string, file: string, line: number, editor?: TextEditor, runningStopWatch?: StopWatch) : Promise<void> {
         // Call the internal method.
-        return this.submitCode(code, file, line, undefined, editor, runningStopWatch);
+        return this.submitCode(code, file, line, undefined, editor, runningStopWatch, false);
+    }
+
+    public debugCode(code: string, file: string, line: number, editor?: TextEditor, runningStopWatch?: StopWatch) : Promise<void> {
+        // Call the internal method.
+        return this.submitCode(code, file, line, undefined, editor, runningStopWatch, true);
     }
 
     // tslint:disable-next-line: no-any no-empty cyclomatic-complexity max-func-body-length
@@ -691,7 +698,7 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
         }
     }
 
-    private async submitCode(code: string, file: string, line: number, id?: string, _editor?: TextEditor, runningStopWatch?: StopWatch) : Promise<void> {
+    private async submitCode(code: string, file: string, line: number, id?: string, _editor?: TextEditor, runningStopWatch?: StopWatch, debug?: boolean) : Promise<void> {
         this.logger.logInformation(`Submitting code for ${this.id}`);
 
         // Start a status item
@@ -740,6 +747,11 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
                     await this.jupyterServer.setInitialDirectory(path.dirname(file));
                 }
 
+                if (debug) {
+                    // Attach our debugger
+                    await this.jupyterDebugger.startDebugging(this.jupyterServer);
+                }
+
                 // Attempt to evaluate this cell in the jupyter notebook
                 const observable = this.jupyterServer.executeObservable(code, file, line, id, false);
 
@@ -774,6 +786,12 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
 
             const message = localize.DataScience.executingCodeFailure().format(err);
             this.applicationShell.showErrorMessage(message);
+        } finally {
+            if (debug) {
+                if (this.jupyterServer) {
+                    await this.jupyterDebugger.stopDebugging(this.jupyterServer);
+                }
+            }
         }
     }
 
@@ -1013,6 +1031,11 @@ export class InteractiveWindow extends WebViewHost<IInteractiveWindowMapping> im
         // Now try to create a notebook server
         this.jupyterServer = await this.jupyterExecution.connectToNotebookServer(options);
 
+        // Enable debugging support if set
+        if (options.enableDebugging && this.jupyterServer) {
+            await this.jupyterDebugger.enableAttach(this.jupyterServer);
+        }
+
         // Before we run any cells, update the dark setting
         if (this.jupyterServer) {
             await this.jupyterServer.setMatplotLibStyle(knownDark);

src/client/datascience/interactive-window/interactiveWindowProvider.ts

@@ -82,13 +82,15 @@ export class InteractiveWindowProvider implements IInteractiveWindowProvider, IA
         const settings = this.configService.getSettings();
         let serverURI: string | undefined = settings.datascience.jupyterServerURI;
         const useDefaultConfig: boolean | undefined = settings.datascience.useDefaultConfigForJupyter;
+        const enableDebugging: boolean | undefined = settings.datascience.enableDebugging;
 
         // For the local case pass in our URI as undefined, that way connect doesn't have to check the setting
         if (serverURI === Settings.JupyterServerLocalLaunch) {
             serverURI = undefined;
         }
 
         return {
+            enableDebugging: enableDebugging,
             uri: serverURI,
             useDefaultConfig,
             purpose: Identifiers.HistoryPurpose

src/client/datascience/jupyter/jupyterDebugger.ts

@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+'use strict';
+import { nbformat } from '@jupyterlab/coreutils';
+import { inject, injectable } from 'inversify';
+import * as uuid from 'uuid/v4';
+import { DebugConfiguration } from 'vscode';
+
+import { ICommandManager, IDebugService } from '../../common/application/types';
+import { traceInfo } from '../../common/logger';
+import { Identifiers } from '../constants';
+import { CellState, ICell, IDebuggerConnectInfo, IJupyterDebugger, INotebookServer } from '../types';
+
+@injectable()
+export class JupyterDebugger implements IJupyterDebugger {
+    private connectInfo: IDebuggerConnectInfo | undefined;
+
+    constructor(
+            @inject(ICommandManager) private commandManager: ICommandManager,
+            @inject(IDebugService) private debugService: IDebugService
+        ) {}
+
+    public async enableAttach(server: INotebookServer): Promise<void> {
+        traceInfo('enable debugger attach');
+
+        // Current version of ptvsd doesn't support returning the value that we need so you need to install the correct version or use my hardcoded line above
+        // tslint:disable-next-line:no-multiline-string
+        //const enableDebuggerResults = await this.executeSilently(server, `import sys\r\nsys.path.append('d:/ptvsd-drop/kdrop/src')\r\nimport os\r\nos.environ["PTVSD_LOG_DIR"] = "d:/note_dbg/logs"\r\nimport ptvsd\r\nptvsd.enable_attach(('localhost', 0))`);
+        // tslint:disable-next-line:no-multiline-string
+        const enableDebuggerResults = await this.executeSilently(server, `import ptvsd\r\nptvsd.enable_attach(('localhost', 0))`);
+
+        // Save our connection info to this server
+        this.connectInfo = this.parseConnectInfo(enableDebuggerResults);
+    }
+
+    public async startDebugging(server: INotebookServer): Promise<void> {
+        traceInfo('start debugging');
+        if (this.connectInfo) {
+            // First connect the VSCode UI
+            const config: DebugConfiguration = {
+                name: 'IPython',
+                request: 'attach',
+                type: 'python',
+                port: this.connectInfo.port,
+                host: this.connectInfo.hostName
+            };
+
+            await this.debugService.startDebugging(undefined, config);
+
+            // Wait for attach before we turn on tracing and allow the code to run, if the IDE is already attached this is just a no-op
+            // tslint:disable-next-line:no-multiline-string
+            await this.executeSilently(server, `import ptvsd\r\nptvsd.wait_for_attach()`);
+
+            // Then enable tracing
+            // tslint:disable-next-line:no-multiline-string
+            await this.executeSilently(server, `from ptvsd import tracing\r\ntracing(True)`);
+        }
+    }
+
+    public async stopDebugging(server: INotebookServer): Promise<void> {
+        traceInfo('stop debugging');
+        // Disable tracing
+        // tslint:disable-next-line:no-multiline-string
+        await this.executeSilently(server, `from ptvsd import tracing\r\ntracing(False)`);
+
+        // Stop our debugging UI session, no await as we just want it stopped
+        this.commandManager.executeCommand('workbench.action.debug.stop');
+    }
+
+    private executeSilently(server: INotebookServer, code: string): Promise<ICell[]> {
+        return server.execute(code, Identifiers.EmptyFileName, 0, uuid(), undefined, true);
+    }
+
+    // Pull our connection info out from the cells returned by enable_attach
+    private parseConnectInfo(cells: ICell[]): IDebuggerConnectInfo | undefined {
+        if (cells.length > 0) {
+            let enableAttachString = this.extractOutput(cells[0]);
+            if (enableAttachString) {
+                enableAttachString = enableAttachString.trimQuotes();
+
+                const debugInfoRegEx = /\('(.*?)', ([0-9]*)\)/;
+
+                const debugInfoMatch = debugInfoRegEx.exec(enableAttachString);
+                if (debugInfoMatch) {
+                    return { hostName: debugInfoMatch[1], port: parseInt(debugInfoMatch[2], 10) };
+                }
+            }
+        }
+        return undefined;
+    }
+
+    private extractOutput(cell: ICell): string | undefined {
+        if (cell.state === CellState.error || cell.state === CellState.finished) {
+            const outputs = cell.data.outputs as nbformat.IOutput[];
+            if (outputs.length > 0) {
+                const data = outputs[0].data;
+                if (data && data.hasOwnProperty('text/plain')) {
+                    // tslint:disable-next-line:no-any
+                    return ((data as any)['text/plain']);
+                }
+            }
+        }
+        return undefined;
+    }
+}

src/client/datascience/jupyter/jupyterExecution.ts

@@ -155,7 +155,8 @@ export class JupyterExecutionBase implements IJupyterExecution {
                         kernelSpec: startInfo.kernelSpec,
                         workingDir: options ? options.workingDir : undefined,
                         uri: options ? options.uri : undefined,
-                        purpose: options ? options.purpose : uuid()
+                        purpose: options ? options.purpose : uuid(),
+                        enableDebugging: options ? options.enableDebugging : false
                     };
 
                     traceInfo(`Connecting to process for ${options ? options.purpose : 'unknown type of'} server`);

src/client/datascience/jupyter/liveshare/serverCache.ts

@@ -61,6 +61,7 @@ export class ServerCache implements IAsyncDisposable {
 
     public async generateDefaultOptions(options?: INotebookServerOptions): Promise<INotebookServerOptions> {
         return {
+            enableDebugging: options ? options.enableDebugging : false,
             uri: options ? options.uri : undefined,
             useDefaultConfig: options ? options.useDefaultConfig : true, // Default for this is true.
             usingDarkTheme: options ? options.usingDarkTheme : undefined,
@@ -76,9 +77,10 @@ export class ServerCache implements IAsyncDisposable {
             // combine all the values together to make a unique key
             const uri = options.uri ? options.uri : '';
             const useFlag = options.useDefaultConfig ? 'true' : 'false';
+            const debug = options.enableDebugging ? 'true' : 'false';
             // tslint:disable-next-line:no-suspicious-comment
             // TODO: Should there be some separator in the key?
-            return `${options.purpose}${uri}${useFlag}${options.workingDir}`;
+            return `${options.purpose}${uri}${useFlag}${options.workingDir}${debug}`;
         }
     }