From ad60b5753787e19e977e3c97b26b4bd0e090e642 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 2 Jul 2019 20:25:53 -0700 Subject: [PATCH 1/3] Add logging to see perf times of tests --- uitests/uitests/environment.py | 11 ++++++++++- uitests/uitests/tools.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/uitests/uitests/environment.py b/uitests/uitests/environment.py index e2ddb8ada41c..3e57a4c0bc2b 100644 --- a/uitests/uitests/environment.py +++ b/uitests/uitests/environment.py @@ -51,7 +51,7 @@ def f_restore_context(*args, **kwargs): return deco_context - +@uitests.tools.trace @uitests.tools.retry((TimeoutError, WebDriverException), tries=5, delay=5) @uitests.tools.log_exceptions() def before_all(context): @@ -61,10 +61,12 @@ def before_all(context): _start_and_clear(context, options) +@uitests.tools.trace def after_all(context): _exit(context) +@uitests.tools.trace def before_feature(context, feature): for scenario in feature.scenarios: # If we're working on a scenario, then don't retry. @@ -86,6 +88,7 @@ def before_feature(context, feature): patch_scenario_with_autoretry(scenario, max_attempts=2) +@uitests.tools.trace @uitests.tools.retry((PermissionError, FileNotFoundError), tries=2) @uitests.tools.log_exceptions() @restore_context() @@ -141,6 +144,7 @@ def before_scenario(context, scenario): _dismiss_one_time_messages(context, retry_count=2) +@uitests.tools.trace @uitests.tools.log_exceptions() @restore_context() def after_scenario(context, scenario): @@ -195,12 +199,14 @@ def after_scenario(context, scenario): os.makedirs(context.options.logfiles_dir, exist_ok=True) +@uitests.tools.trace @uitests.tools.log_exceptions() @restore_context() def before_step(context, step): logging.info("Before step") +@uitests.tools.trace @uitests.tools.log_exceptions() @restore_context() def after_step(context, step): @@ -253,12 +259,14 @@ def after_step(context, step): pass +@uitests.tools.trace @restore_context() def _exit(context): uitests.vscode.application.exit(context) uitests.vscode.application.CONTEXT["driver"] = None +@uitests.tools.trace def _start_and_clear(context, options): # Clear VS Code folders (do not let VSC save state). # During tests, this can be done as a step `When I load VSC for the first time`. @@ -287,6 +295,7 @@ def _start_and_clear(context, options): raise +@uitests.tools.trace def _dismiss_one_time_messages(context, retry_count=100, retry_interval=0.1): # Dismiss one time VSC messages. # Dismiss one time extension messages. diff --git a/uitests/uitests/tools.py b/uitests/uitests/tools.py index 5710d7f5f5c2..dc19aa94de54 100644 --- a/uitests/uitests/tools.py +++ b/uitests/uitests/tools.py @@ -83,6 +83,21 @@ def wrapper(*args, **kwargs): return deco_log_exceptions +def trace(f): + """Decorator to just log before and after execution of a function.""" + @wraps(f) + def wrapper(*args, **kwargs): + start_time = time.time() + try: + logging.info(f"Before {f.__name__}") + return f(*args, **kwargs) + finally: + elapsed_time = time.time() - start_time + elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) + logging.info(f"After {f.__name__} ({elapsed_time})") + return wrapper + + def run_command(command, *, cwd=None, silent=False, progress_message=None, env=None): """Run the specified command in a subprocess shell with the following options: - Pipe output from subprocess into current console. From 0b8f257285f19125fa1404a05f0d8f25b18cadf3 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 2 Jul 2019 20:26:30 -0700 Subject: [PATCH 2/3] Run all tests --- build/ci/vscode-python-pr-validation.yaml | 36 +++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/build/ci/vscode-python-pr-validation.yaml b/build/ci/vscode-python-pr-validation.yaml index 25f3a14dad89..02121149cc15 100644 --- a/build/ci/vscode-python-pr-validation.yaml +++ b/build/ci/vscode-python-pr-validation.yaml @@ -25,14 +25,44 @@ variables: jobs: - template: templates/build_compile_jobs.yml +# - template: templates/uitest_jobs.yml +# parameters: +# # In PRs, test only against stable version of VSC. +# vscodeChannels: ['stable'] +# # In PRs, run smoke tests against 3.7 and 2.7 (excluding others). +# jobs: +# - test: "Smoke" +# tags: "--tags=@smoke" +# ignorePythonVersions: "3.6,3.5" + - template: templates/uitest_jobs.yml parameters: - # In PRs, test only against stable version of VSC. - vscodeChannels: ['stable'] - # In PRs, run smoke tests against 3.7 and 2.7 (excluding others). jobs: - test: "Smoke" tags: "--tags=@smoke" + # Smoke tests are cheap, so run them against all Python Versions. + - test: "Test" + tags: "--tags=@testing" + # We have python code that is involved in running/discovering tests. + # Hence test against all versions, until we have CI running for the Python code. + # I.e. when all test dicovery/running is done purely in Python. + - test: "Terminal" + tags: "--tags=@terminal --tags=~@terminal.pipenv" + # No need to run tests against all versions. + # This is faster/cheaper, besides activation of terminals is generic enough + # not to warrant testing against all versions. + ignorePythonVersions: "3.6,3.5" + - test: "Debugging" + tags: "--tags=@debugging" + # No need to run tests against all versions. + # This is faster/cheaper, and these are external packages. + # We expect them to work (or 3rd party packages to test against all PY versions). + ignorePythonVersions: "3.6,3.5" + - test: "Jedi_Language_Server" + tags: "--tags=@ls" + # No need to run tests against all versions. + # This is faster/cheaper, and these are external packages. + # We expect them to work (or 3rd party packages to test against all PY versions). ignorePythonVersions: "3.6,3.5" - job: 'PR' From 2d9ed7559af97c53ae0ea7e58771018fc48af5de Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 2 Jul 2019 20:49:16 -0700 Subject: [PATCH 3/3] Revert --- build/ci/vscode-python-pr-validation.yaml | 36 ++--------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/build/ci/vscode-python-pr-validation.yaml b/build/ci/vscode-python-pr-validation.yaml index 02121149cc15..25f3a14dad89 100644 --- a/build/ci/vscode-python-pr-validation.yaml +++ b/build/ci/vscode-python-pr-validation.yaml @@ -25,44 +25,14 @@ variables: jobs: - template: templates/build_compile_jobs.yml -# - template: templates/uitest_jobs.yml -# parameters: -# # In PRs, test only against stable version of VSC. -# vscodeChannels: ['stable'] -# # In PRs, run smoke tests against 3.7 and 2.7 (excluding others). -# jobs: -# - test: "Smoke" -# tags: "--tags=@smoke" -# ignorePythonVersions: "3.6,3.5" - - template: templates/uitest_jobs.yml parameters: + # In PRs, test only against stable version of VSC. + vscodeChannels: ['stable'] + # In PRs, run smoke tests against 3.7 and 2.7 (excluding others). jobs: - test: "Smoke" tags: "--tags=@smoke" - # Smoke tests are cheap, so run them against all Python Versions. - - test: "Test" - tags: "--tags=@testing" - # We have python code that is involved in running/discovering tests. - # Hence test against all versions, until we have CI running for the Python code. - # I.e. when all test dicovery/running is done purely in Python. - - test: "Terminal" - tags: "--tags=@terminal --tags=~@terminal.pipenv" - # No need to run tests against all versions. - # This is faster/cheaper, besides activation of terminals is generic enough - # not to warrant testing against all versions. - ignorePythonVersions: "3.6,3.5" - - test: "Debugging" - tags: "--tags=@debugging" - # No need to run tests against all versions. - # This is faster/cheaper, and these are external packages. - # We expect them to work (or 3rd party packages to test against all PY versions). - ignorePythonVersions: "3.6,3.5" - - test: "Jedi_Language_Server" - tags: "--tags=@ls" - # No need to run tests against all versions. - # This is faster/cheaper, and these are external packages. - # We expect them to work (or 3rd party packages to test against all PY versions). ignorePythonVersions: "3.6,3.5" - job: 'PR'