From 60c733d8168312b28db30ab6299857b876766872 Mon Sep 17 00:00:00 2001 From: plocket <52798256+plocket@users.noreply.github.com> Date: Wed, 24 Jun 2026 10:31:52 -0400 Subject: [PATCH 1/4] Save HTML on pages, even if they have sensitive answers, as the HTML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit excludes field values. ā€¼ļø NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML. šŸ–Šļø Before merging remove temporary debugging logs. Also breaks out the anonymous function in `Before()` to attempt better error tracing. Goal: try to repeat this for other functions in that file. Addresses #1097 in trying to see more information about our current test failure. --- .github/workflows/github_server.yml | 2 +- .github/workflows/playground.yml | 2 +- CHANGELOG.md | 4 ++ lib/scope.js | 63 +++++++++++++++++++---------- lib/steps.js | 58 +++++++++++++++++--------- package.json | 2 +- 6 files changed, 86 insertions(+), 45 deletions(-) diff --git a/.github/workflows/github_server.yml b/.github/workflows/github_server.yml index 1d78bca6..92c3808f 100644 --- a/.github/workflows/github_server.yml +++ b/.github/workflows/github_server.yml @@ -160,7 +160,7 @@ jobs: #### Developer note: You can probably leave the rest out #### To learn more, see https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#optional-inputs ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}" - # ALKILN_VERSION: + ALKILN_VERSION: "screenshot" #### Developer note: Example of making an issue when tests fail #### that includes the text of the failure output file diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml index 46a75ccd..104f7a62 100644 --- a/.github/workflows/playground.yml +++ b/.github/workflows/playground.yml @@ -83,4 +83,4 @@ jobs: # want to check up on this. ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}" #### Developer note: You can probably leave this out - # ALKILN_VERSION: + ALKILN_VERSION: "screenshot" diff --git a/CHANGELOG.md b/CHANGELOG.md index b0cd69d3..58d69569 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ Format: ### Changed +- On pages with sensitive answers, store the HTML of the page. The HTML excludes field values, so those sensitive answers will not be in the saved file. Still avoid taking a pic of the screen, which would reveal sensitive answers. NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML. + - GitHub action release: Restored our GitHub action's default for ALKiln version to the latest version 5 again. Released first on GitHub actions. NPM release will come in time, but npm has no impact on GitHub action releases. ### Fixed @@ -55,6 +57,8 @@ Format: ### Internal +- Break out the function in `Before()` to attempt better error tracing. Goal: try to repeat this for other functions in that file. + - Updated both of our actions' dependencies (the checkout, setup-node, setup-python, upload-artifacts, download-artifacts actions). Closes [#1095](https://github.com/suffolkLITLab/aLKiln/issues/1095). Once again, action related. ## [5.16.1] - 2026-06-01 diff --git a/lib/scope.js b/lib/scope.js index 8a586c67..3c714a76 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -2711,8 +2711,23 @@ module.exports = { return scope; }, // Ends scope.throwPageError() - take_a_screenshot: async ( scope,{ path }) => { - /* Takes a jpeg screenshot. Avoids destroying signatures. */ + take_a_screenshot: async ( scope, { + path, disable_pic=false, disable_html=false + }) => { + /** + * Saves a jpeg screenshot and page HTML to `path`. Avoids destroying + * signatures. + * + * @param {obj} scope - State and internal functions + * @param {obj} obj - Named arguments + * @param {str} obj.path - name of path where to save the files + * @param {bool} [obj.disable_pic=false] - Optional. Whether to disable a + * puppeteer screenshot + * @param {bool} [obj.disable_html=false] - Optional. Whether to disable + * saving the HTML of the page + * + * @returns undefined + * */ let fullPage = true; let signature_elem = await scope.page.$(scope.signature_selector); @@ -2721,28 +2736,32 @@ module.exports = { fullPage = false; } - await scope.page.screenshot({ - path: path, - type: 'jpeg', - fullPage: fullPage, - }); - - let html_path = path; - if (path.endsWith(".jpg")) { - html_path = html_path.substring(0, html_path.length - 4) + ".html"; - } else { - html_path = html_path + ".html"; + if ( !disable_pic ){ + await scope.page.screenshot({ + path: path, + type: 'jpeg', + fullPage: fullPage, + }); } - // Also save the HTML of the page - await scope.page.content().then(content => { - let server_url = session_vars.get_da_server_url(); - content = content.replaceAll(/"(\/static\/.*\.css\?v=[^"]+")/g, server_url + "$1"); - content = content.replaceAll(/"(\/static\/.*\.js\?v=[^"]+")/g, server_url + "$1"); - content = content.replaceAll(/"(\/packagestatic\/.*\.css\?v=[^"]+")/g, server_url + "$1"); - content = content.replaceAll(/"(\/packagestatic\/.*\.js\?v=[^"]+")/g, server_url + "$1"); - fs.writeFileSync(html_path, content) - }); + if ( !disable_html ) { + let html_path = path; + if (path.endsWith(".jpg")) { + html_path = html_path.substring(0, html_path.length - 4) + ".html"; + } else { + html_path = html_path + ".html"; + } + + // Also save the HTML of the page + await scope.page.content().then(content => { + let server_url = session_vars.get_da_server_url(); + content = content.replaceAll(/"(\/static\/.*\.css\?v=[^"]+")/g, server_url + "$1"); + content = content.replaceAll(/"(\/static\/.*\.js\?v=[^"]+")/g, server_url + "$1"); + content = content.replaceAll(/"(\/packagestatic\/.*\.css\?v=[^"]+")/g, server_url + "$1"); + content = content.replaceAll(/"(\/packagestatic\/.*\.js\?v=[^"]+")/g, server_url + "$1"); + fs.writeFileSync(html_path, content) + }); + } }, // Ends scope.take_a_screenshot() diff --git a/lib/steps.js b/lib/steps.js index f3b5bc02..ac2fcb51 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -114,8 +114,9 @@ BeforeAll(async function() { reports.create( scope ); }); -Before(async (scenario) => { - +Before( beforeScenario ); +async function beforeScenario( scenario ) { + console.log('šŸ‘ļø 111 start beforeScenario'); // Start the running "progress bar" for the Scenario log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `); @@ -125,15 +126,20 @@ Before(async (scenario) => { // Will only run in the Playground outside of a sandbox. TODO: There's a // better way to do this, though it's more complicated. See comments in // https://github.com/SuffolkLITLab/ALKiln/issues/661 + console.log('šŸ‘ļø 112 no browser, await new local browser'); scope.browser = await scope.driver.launch({ args: ['--no-sandbox'] }); } else { + console.log('šŸ‘ļø 113 no browser, await new remote browser'); scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() }); } } + console.log('šŸ‘ļø 114 we have ensured browser exists, awaiting pages'); // Clean up all previously existing pages for (const page of await scope.browser.pages()) { + console.log('šŸ‘ļø 115 awaiting closing page'); await page.close(); } + console.log('šŸ‘ļø 116 awaiting opening a new page'); // Make a new page scope.page = await scope.browser.newPage() @@ -147,8 +153,10 @@ Before(async (scenario) => { scope.server_reload_promise = null; reports.addReportHeading(scope, {scenario}); + console.log('šŸ‘ļø 117 awaiting getting safe scenario name'); // Make folder for this Scenario in the all-tests artifacts folder scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario}); + console.log('šŸ‘ļø 118 got safe scenario name'); // Add a date for uniqueness in case dev has accidentally copied a Scenario description let date = Date.now(); scope.paths.scenario = `${ scope.paths.artifacts }/${ scope.base_filename }-${ date }`; @@ -171,7 +179,7 @@ Before(async (scenario) => { // Reset default timeout scope.timeout = default_timeout; -}); +} // Add a check for an error page before each step? After each step? @@ -1379,25 +1387,35 @@ After(async function(scenario) { reports.addToReport(scope, { type: `row info`, code: `ALK0095`, - value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen.` + // Discuss: Might people use secret variables to prevent any + // information about a screen from getting out? For example, they have + // proprietary info in the HTML itself. + value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML, which omits the values in the fields.` }); - } else { + } + + // Save/download a picture of the screen that's showing during the unexpected status + // Save one copy in the outer-most artifact folder + let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` }); + let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`; + await scope.take_a_screenshot( scope, { + path: path_outer, + disable_pic: scope.disable_error_screenshot, + disable_html: false, + }); + + // Save another copy in the artifact's Scenario folder + let screenshot_name = `error_on`; + let { id } = await scope.examinePageID( scope, 'none to match' ); + let short_id = `${ id }`.substring(0, 20); + screenshot_name += `-${ short_id }`; + let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`; + await scope.take_a_screenshot( scope, { + path: path_scenario, + disable_pic: scope.disable_error_screenshot, + disable_html: false, + }); - // Save/download a picture of the screen that's showing during the unexpected status - // Save one copy in the outer-most artifact folder - let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` }); - let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`; - await scope.take_a_screenshot( scope, { path: path_outer }); - - // Save another copy in the artifact's Scenario folder - let screenshot_name = `error_on`; - let { id } = await scope.examinePageID( scope, 'none to match' ); - let short_id = `${ id }`.substring(0, 20); - screenshot_name += `-${ short_id }`; - let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`; - await scope.take_a_screenshot( scope, { path: path_scenario }); - - } // ends if scope.disable_error_screenshot } // ends if scope.page exists // This has to come after the security message or security message doesn't print. Not sure why. diff --git a/package.json b/package.json index 5067bd23..20c64703 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@suffolklitlab/alkiln", - "version": "5.16.1", + "version": "5.16.1-always-html-2", "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.", "main": "lib/index.js", "scripts": { From e5942ef6f2d0c75013d1e329d545294e921a1eda Mon Sep 17 00:00:00 2001 From: plocket <52798256+plocket@users.noreply.github.com> Date: Thu, 25 Jun 2026 09:58:04 -0400 Subject: [PATCH 2/4] Add nav wait after sign in to allow a screenshot --- lib/scope.js | 81 +++++++++++++++++++++++++++++++++++++++++---------- lib/steps.js | 82 ++++++++++++++++++++++++++++++++++++++++++---------- package.json | 2 +- 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/lib/scope.js b/lib/scope.js index 3c714a76..cc84096d 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -3090,6 +3090,7 @@ module.exports = { /** Allow the developer to log a user into their server using GitHub * secrets to authenticate. */ + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-1.jpg`}); // Don't take a picture of a failed login in case one of the inputs is correct scope.disable_error_screenshot = true; @@ -3101,6 +3102,7 @@ module.exports = { // Go to the sign in page let login_url = `${ session_vars.get_da_server_url() }/user/sign-in`; + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-2.jpg`}); // TODO: implement and use scope.handle_possible_timeout() try { @@ -3108,10 +3110,13 @@ module.exports = { await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout }); await scope.page.waitForSelector( `.dabody` ); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-3.jpg`}); + } catch ( error ) { let err_msg = `Error occurred when ALKiln tried to go to "${ login_url }".` if ( error.name === `TimeoutError` ) { + // console.log(`šŸ–Šļø 110`); let non_reload_report_msg = `It took too long to load "${ login_url }"`; await scope.handle_page_timeout_error( scope, { non_reload_report_data: { code: `ALK0159`, @@ -3119,6 +3124,8 @@ module.exports = { }, error }); } else { // Throw any non-timeout error + // console.log(`šŸ–Šļø 111`); + await scope.take_a_screenshot(scope, {path:`./_alkiln_temp/Temp-4.jpg`}); reports.addToReport( scope, { type: `error`, code: `ALK0160`, value: err_msg }); throw error; } // ends if error is timeout error @@ -3159,6 +3166,8 @@ module.exports = { scope.scenarios.get( scope.scenario_id ).api_keys.push( api_key ); } } + // console.log(`šŸ–Šļø 112`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-4.jpg`}); expect( email, email_msg ).to.not.equal( undefined ); expect( password, password_msg ).to.not.equal( undefined ); @@ -3168,9 +3177,14 @@ module.exports = { await scope.page.type( `#password`, password ); let elem = await scope.page.$( `button[type="submit"]` ); await scope.guard_against_missing_tap_element(scope, { elem }); + // console.log(`šŸ–Šļø 113`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-5.jpg`}); // Submit and see what happens - let winner = await scope.steps.race_sign_in_navigation( scope, { elem }); + let winner = await scope.steps.race_sign_in_navigation( scope, { elem, login_url }); + // console.log(`šŸ–Šļø 113.5 ${ winner }`); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-6.jpg`}); + console.log(`šŸ–Šļø 113.6 ${ winner }`); // Add the result to the report and possibly throw errors if ( winner[0] === `success` ) { reports.addToReport( scope, { @@ -3183,6 +3197,7 @@ module.exports = { type: `error`, code: `ALK0209`, value: `Failed to sign into ${ login_url }. Make sure you followed the instructions at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#sign-in.` }); + // console.log(`šŸ‰ ${ error_msg }`); throw new Error( error_msg ); } else if ( winner[0] === `error` ) { let error_msg = reports.addToReport( scope, { @@ -3191,59 +3206,94 @@ module.exports = { }); throw new Error( error_msg ); } + // console.log(`šŸ–Šļø 114`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-7.jpg`}); + if (error_msg) { + return [ error_msg ]; + } else { + return []; + } }, // Ends scope.steps.sign_in() - race_sign_in_navigation: async function ( scope, { elem }) { + race_sign_in_navigation: async function ( scope, { elem, login_url='unknown' }) { /** Wait for sign in navigation success or failure, or system error. */ + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-0.jpg`}); + // After everything, clean up incomplete promises const controller = new AbortController; + // console.log(`ā³ 200`); // Redirect - let redirect_promise = scope.page.waitForResponse(function ( response ) { + let redirect_promise = scope.page.waitForResponse(async ( response ) => { + // console.log(`ā³ 300`); return response.status() === 302; }, { signal: controller.signal }) - .catch(( catch_info ) => { + .catch(async ( catch_info ) => { + // console.log(`ā³ 201`); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-1.jpg`}); log.debug({ code: `ALK0204`, context: `nav`, }, `302 sign-in wait catch error:`, catch_info.name, catch_info, ); }); // Invalid credentials, no navigation, no sign in - let wrong_sign_in_promise = scope.page.waitForResponse(function ( response ) { + let wrong_sign_in_promise = scope.page.waitForResponse(async ( response ) => { + // console.log(`ā³ 301`); return response.status() >= 200 && response.status() < 300 && response.url().includes(`/sign-in`); }, { signal: controller.signal }) - .catch(( catch_info ) => { + .catch(async ( catch_info ) => { + // errors? + // console.log(`ā³ 202`); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-2.jpg`}); log.debug({ code: `ALK0205`, context: `nav`, }, `200s sign-in wait catch error:`, catch_info.name, catch_info, ); }); // System error - let error_promise = scope.page.waitForResponse(function ( response ) { + let error_promise = scope.page.waitForResponse(async ( response ) => { + // console.log(`ā³ 302`); return response.status() >= 500 && response.status() < 600; }, { signal: controller.signal }) - .catch(( catch_info ) => { + .catch(async ( catch_info ) => { + // console.log(`ā³ 203`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-3.jpg`}); log.debug({ code: `ALK0206`, context: `nav`, }, `500s sign-in wait catch error:`, catch_info.name, catch_info, ); }); + // Duh, the other catches were triggered because those promises were aborted + let click_promise = elem.click(); // MUST complete + let nav_promise = scope.nav_race(scope, {}).result; + let pic_promise = true; // scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-10.jpg`}); + // console.log(`ā³ 204`); const winner = await Promise.race([ - Promise.all([ `success`, click_promise, redirect_promise ]), - Promise.all([ `failure`, click_promise, wrong_sign_in_promise ]), - Promise.all([ `error`, click_promise, error_promise ]), - ]).catch(function ( error ) { + Promise.all([ `success`, click_promise, pic_promise, redirect_promise ]), + Promise.all([ `failure`, click_promise, pic_promise, wrong_sign_in_promise ]), + Promise.all([ `error`, click_promise, pic_promise, error_promise ]), + ]).catch(async ( error ) => { + // console.log(`ā³ 205`); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-4.jpg`}); let error_msg = reports.addToReport( scope, { type: `error`, code: `ALK0207`, value: `Unknown error waiting for results during sign in at ${ login_url }.` }); - throw new Error( error ); + // console.log(`ā³ 205.1`, error); + throw error; }); + // console.log(`ā³ 206`); + // // await waitForTimeout(500); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-5.jpg`}); + // Clean up unresolved promises controller.abort(); + // console.log(`ā³ 207`); + await nav_promise; + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp3-1.jpg`}); log.debug({ code: `ALK0208`, context: `nav` }, `Sign-in winner:`, winner @@ -3517,14 +3567,15 @@ module.exports = { let scenario = scope.report.get( scope.scenario_id ); let report = reports.getPrintableScenario( scenario ); let all_are_included = true; + let missing = []; for ( let one_expectation of expected ) { if ( !report.includes( one_expectation )) { all_are_included = false; - expect( report ).to.contain( one_expectation ); + missing.push( one_expectation ); } } - return all_are_included; + return { all_included: all_are_included, missing }; }, // Ends scope.reportIncludesAllExpected() reportDoesNotInclude: async function ( scope, { not_expected=[] }) { diff --git a/lib/steps.js b/lib/steps.js index ac2fcb51..a836a9a0 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -116,7 +116,7 @@ BeforeAll(async function() { Before( beforeScenario ); async function beforeScenario( scenario ) { - console.log('šŸ‘ļø 111 start beforeScenario'); + // console.log('šŸ‘ļø 111 start beforeScenario'); // Start the running "progress bar" for the Scenario log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `); @@ -126,20 +126,20 @@ async function beforeScenario( scenario ) { // Will only run in the Playground outside of a sandbox. TODO: There's a // better way to do this, though it's more complicated. See comments in // https://github.com/SuffolkLITLab/ALKiln/issues/661 - console.log('šŸ‘ļø 112 no browser, await new local browser'); + // console.log('šŸ‘ļø 112 no browser, await new local browser'); scope.browser = await scope.driver.launch({ args: ['--no-sandbox'] }); } else { - console.log('šŸ‘ļø 113 no browser, await new remote browser'); + // console.log('šŸ‘ļø 113 no browser, await new remote browser'); scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() }); } } - console.log('šŸ‘ļø 114 we have ensured browser exists, awaiting pages'); + // console.log('šŸ‘ļø 114 we have ensured browser exists, awaiting pages'); // Clean up all previously existing pages for (const page of await scope.browser.pages()) { - console.log('šŸ‘ļø 115 awaiting closing page'); + // console.log('šŸ‘ļø 115 awaiting closing page'); await page.close(); } - console.log('šŸ‘ļø 116 awaiting opening a new page'); + // console.log('šŸ‘ļø 116 awaiting opening a new page'); // Make a new page scope.page = await scope.browser.newPage() @@ -153,10 +153,10 @@ async function beforeScenario( scenario ) { scope.server_reload_promise = null; reports.addReportHeading(scope, {scenario}); - console.log('šŸ‘ļø 117 awaiting getting safe scenario name'); + // console.log('šŸ‘ļø 117 awaiting getting safe scenario name'); // Make folder for this Scenario in the all-tests artifacts folder scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario}); - console.log('šŸ‘ļø 118 got safe scenario name'); + // console.log('šŸ‘ļø 118 got safe scenario name'); // Add a date for uniqueness in case dev has accidentally copied a Scenario description let date = Date.now(); scope.paths.scenario = `${ scope.paths.artifacts }/${ scope.base_filename }-${ date }`; @@ -297,6 +297,7 @@ Given(/I go to "([^"]+)"/i, {timeout: -1}, async ( url ) => { return result; }); +foo = null Given( /I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^"]+)",?(?: and)?(?: the password)? "([^"]+)"(?: SECRETs)?(?:,?(?: and)?(?: the API key)? "([^"]+)")?/i, { timeout: -1 }, @@ -314,11 +315,21 @@ Given( // `page` timeout will deal with custom timeout // Couldn't test the timeout for tapping the button because there's not // enough of a pause between initial navigation and pressing button. - await scope.steps.sign_in( scope, { + foo = scope.page + // console.log(`ā­ļø ${ JSON.stringify(scope.page, null, 2) }`) + let result = await scope.steps.sign_in( scope, { email_secret_name: email, password_secret_name: password, api_key_secret_name: api_key }); + + + // console.log(`šŸ–Šļø 115`); + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-8.jpg`}); + + // if ( !result.ok && result.errors.length > 0 ) { + // throw result.errors[ result.errors.length - 1 ]; + // } }); // I am using a mobile/pc @@ -1332,6 +1343,8 @@ Then(/I fail to delete (\d) detected interview(?:s)? and get no error __internal AfterStep(async function({ result }) { // TODO: Abstract this + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-13.jpg`}); + // console.log(`šŸ›‘ 100`); if ( result.status === `PASSED` ) { log.stdout({ records_only: true }, `.`); } else if ( result.status === `FAILED` ) { log.stdout({ records_only: true }, `F`); } else if ( result.status === `UNDEFINED` ) { log.stdout({ records_only: true }, `U`); } @@ -1339,10 +1352,13 @@ AfterStep(async function({ result }) { else if ( result.status === `PENDING` ) { log.stdout({ records_only: true }, `P`); } reports.outdent(); + // console.log(`šŸ›‘ 101`); }); -After(async function(scenario) { - +After(afterScenario); +async function afterScenario(scenario) { + // console.log(`šŸ‘€ 1 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-30.jpg`}); // Log errors if ( scenario.result.message ) { log.debug({ code: `ALK0091`, context: `scenario`, }, @@ -1361,6 +1377,8 @@ After(async function(scenario) { value: `Accessibility standards tests failed. See information above.` }); } + // console.log(`šŸ‘€ 2`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-9.jpg`}); if ( scope.failed_pdf_compares.length > 0) { let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n―――\n${ new_msg }`) @@ -1382,6 +1400,8 @@ After(async function(scenario) { ); if ( scope.page ) { + // console.log(`šŸ‘€ 3`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-10.jpg`}); if ( !!scope.disable_error_screenshot ) { reports.addToReport(scope, { @@ -1393,6 +1413,8 @@ After(async function(scenario) { value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML, which omits the values in the fields.` }); } + // console.log(`šŸ‘€ 4 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-11.jpg`}); // Save/download a picture of the screen that's showing during the unexpected status // Save one copy in the outer-most artifact folder @@ -1403,6 +1425,8 @@ After(async function(scenario) { disable_pic: scope.disable_error_screenshot, disable_html: false, }); + // console.log(`šŸ‘€ 5 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); + await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-12.jpg`}); // Save another copy in the artifact's Scenario folder let screenshot_name = `error_on`; @@ -1415,6 +1439,7 @@ After(async function(scenario) { disable_pic: scope.disable_error_screenshot, disable_html: false, }); + // console.log(`šŸ‘€ 6`); } // ends if scope.page exists @@ -1422,37 +1447,59 @@ After(async function(scenario) { if (changeable_test_status === `FAILED`) { reports.addToReport(scope, { type: `outcome failure info`, code: `ALK0096`, value: `**-- Scenario Failed --**` }); } + // console.log(`šŸ‘€ 7`); } // ends if not passed + // console.log(`šŸ‘€ 8`); // Add report text to Scenario folder after everything has been added to the report let scenario_report_obj = scope.report.get( scope.scenario_id ); + // console.log(`šŸ‘€ 8.01`); let report = reports.getPrintableScenario( scenario_report_obj ); // Save the report as in the Scenario folder + // console.log(`šŸ‘€ 8.02`); fs.writeFileSync( scope.paths.scenario_report, report ); + // console.log(`šŸ‘€ 8.03`); // TODO: Save any cucumber failure message for the Scenario here (instead of waiting till run_cucumber.js) // ---------------- Check internal test results ---------------- if ( scope.expected_in_report && scope.expected_in_report.length > 0) { - let report_includes_all_expected_strings = await scope.reportIncludesAllExpected( + // console.log(`šŸ‘€ 8.04`); + let data = await scope.reportIncludesAllExpected( scope, { expected: scope.expected_in_report } ); - if ( !report_includes_all_expected_strings ) { changeable_test_status = `FAILED`; } + let report_includes_all_expected_strings = data.all_included; + // console.log(`šŸ‘€ 8.05`); + if ( !report_includes_all_expected_strings ) { + changeable_test_status = `FAILED`; + let msg = `These strings are missing from the report: + ${ JSON.stringify(data.missing, null, 2) } + Instead the report had this text: + ${ report }`; + expect( report, msg ).to.contain( data.missing[0] ); + } // Reset report values no matter what so they don't mess up future scenarios scope.expected_in_report = null; + } + // console.log(`šŸ‘€ 8.06`); if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) { + // console.log(`šŸ‘€ 8.07`); let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude( scope, { not_expected: scope.expected_not_in_report } ); + // console.log(`šŸ‘€ 8.08`); if ( !all_prohibited_strings_were_absent ) { changeable_test_status = `FAILED`;} + // console.log(`šŸ‘€ 8.09`); // Reset report values no matter what so they don't mess up future scenarios scope.expected_not_in_report = null; + // console.log(`šŸ‘€ 8.091`); } + // console.log(`šŸ‘€ 9`); // ---------------- Ends internal results check ---------------- @@ -1490,6 +1537,7 @@ After(async function(scenario) { ); } } + // console.log(`šŸ‘€ 10`); scope.expected_status = null; // reset for next Scenario let need_to_force_failed_status = false; @@ -1530,12 +1578,15 @@ After(async function(scenario) { let signout_error_msg = ``; // If there is a page open, then sign out and close it if ( scope.page ) { + // console.log(`šŸ‘€ 11`); // Catch a possible reload error. try { // puppeteer will ensure proper timeout. + // console.log(`šŸ‘€ 12`); await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`}); } catch ( error ) { + // console.log(`šŸ‘€ 13`); signout_succeeded = false; if ( error.name === `TimeoutError` ) { @@ -1562,6 +1613,7 @@ After(async function(scenario) { } // ends try/catch } // ends if scope.page + // console.log(`šŸ‘€ 14`); // Now that we're no longer on the interview page, one way or // another, try to delete the interviews created during the test @@ -1618,13 +1670,13 @@ After(async function(scenario) { internal_errors ); } + // console.log(`šŸ‘€ 15`); // Find the race condition. log.debug({ code: `ALK0100`, context: `scenario`, }, `Scenario After() message:`, scenario.result.message ); - -}); +} // Ends afterScenario() AfterAll(async function() { // Stop collecting server response statuses diff --git a/package.json b/package.json index 20c64703..750e5b2f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@suffolklitlab/alkiln", - "version": "5.16.1-always-html-2", + "version": "5.16.1-always-html-3", "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.", "main": "lib/index.js", "scripts": { From 9ff498db0f8d53be7369052af4d4eabc78c44dc4 Mon Sep 17 00:00:00 2001 From: plocket <52798256+plocket@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:18:33 -0400 Subject: [PATCH 3/4] Bump version --- lib/scope.js | 14 +++++++------- lib/steps.js | 2 +- package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/scope.js b/lib/scope.js index cc84096d..3cdcabb4 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -3184,7 +3184,7 @@ module.exports = { let winner = await scope.steps.race_sign_in_navigation( scope, { elem, login_url }); // console.log(`šŸ–Šļø 113.5 ${ winner }`); // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-6.jpg`}); - console.log(`šŸ–Šļø 113.6 ${ winner }`); + // console.log(`šŸ–Šļø 113.6 ${ winner }`); // Add the result to the report and possibly throw errors if ( winner[0] === `success` ) { reports.addToReport( scope, { @@ -3207,12 +3207,12 @@ module.exports = { throw new Error( error_msg ); } // console.log(`šŸ–Šļø 114`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-7.jpg`}); - if (error_msg) { - return [ error_msg ]; - } else { - return []; - } + // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-7.jpg`}); + // if ( error_msg ) { // undefined sometimes now + // return [ error_msg ]; + // } else { + // return []; + // } }, // Ends scope.steps.sign_in() race_sign_in_navigation: async function ( scope, { elem, login_url='unknown' }) { diff --git a/lib/steps.js b/lib/steps.js index a836a9a0..890a96a2 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -317,7 +317,7 @@ Given( // enough of a pause between initial navigation and pressing button. foo = scope.page // console.log(`ā­ļø ${ JSON.stringify(scope.page, null, 2) }`) - let result = await scope.steps.sign_in( scope, { + await scope.steps.sign_in( scope, { email_secret_name: email, password_secret_name: password, api_key_secret_name: api_key diff --git a/package.json b/package.json index 750e5b2f..79d280a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@suffolklitlab/alkiln", - "version": "5.16.1-always-html-3", + "version": "5.16.1-always-html-5", "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.", "main": "lib/index.js", "scripts": { From 79abdf8bf7798bcbfd3f0e400888909ff6cc671b Mon Sep 17 00:00:00 2001 From: plocket <52798256+plocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 09:10:42 -0400 Subject: [PATCH 4/4] Clean up, guard against irrelevant noisy screenshot errors --- CHANGELOG.md | 11 ++--- lib/scope.js | 68 +++++------------------------ lib/steps.js | 120 ++++++++++++++++++--------------------------------- package.json | 2 +- 4 files changed, 60 insertions(+), 141 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58d69569..5aea6d47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,19 +47,20 @@ Format: ### Changed -- On pages with sensitive answers, store the HTML of the page. The HTML excludes field values, so those sensitive answers will not be in the saved file. Still avoid taking a pic of the screen, which would reveal sensitive answers. NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML. - +- On pages with sensitive answers, store the HTML of the page. The HTML excludes field values, so those sensitive answers will not be in the saved file. Still avoid taking a pic of the screen, which would reveal sensitive answers. ā€¼ļø NEVER USE REAL USERS' ANSWERS IN ALKILN TESTS. This HTML can still reveal information about a user's answers. For example, some answers will reveal new questions. That will change the code of the revealed fields and that code will be in the HTML. See #1099 - GitHub action release: Restored our GitHub action's default for ALKiln version to the latest version 5 again. Released first on GitHub actions. NPM release will come in time, but npm has no impact on GitHub action releases. ### Fixed -- Fixed GitHub+You action outdated docassemble cli version causing Scenario timeouts. +- Updated docassemble cli version. +- Better detect navigation when email and password fail to sign in to a docassemble server account. ### Internal -- Break out the function in `Before()` to attempt better error tracing. Goal: try to repeat this for other functions in that file. - +- Break out the function in `Before()` and `After()` to attempt better error tracing. Goal: try to repeat this for other functions in that file. - Updated both of our actions' dependencies (the checkout, setup-node, setup-python, upload-artifacts, download-artifacts actions). Closes [#1095](https://github.com/suffolkLITLab/aLKiln/issues/1095). Once again, action related. +- When our report phrases are missing, list all missing phrases at one time in our custom message. The cucumber message will stay the same. +- Silence errors from screenshots & HTML downloads when trying to provide more information about failing tests. Those records are nice to have, but not absolutely necessary, and if a server is busy puppeteer will rightly have lost execution context, preventing interactions with the page and we don't need a noisy error about it. Log any problems. ## [5.16.1] - 2026-06-01 diff --git a/lib/scope.js b/lib/scope.js index 3cdcabb4..f7a78321 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -3090,7 +3090,6 @@ module.exports = { /** Allow the developer to log a user into their server using GitHub * secrets to authenticate. */ - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-1.jpg`}); // Don't take a picture of a failed login in case one of the inputs is correct scope.disable_error_screenshot = true; @@ -3102,7 +3101,6 @@ module.exports = { // Go to the sign in page let login_url = `${ session_vars.get_da_server_url() }/user/sign-in`; - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-2.jpg`}); // TODO: implement and use scope.handle_possible_timeout() try { @@ -3110,13 +3108,10 @@ module.exports = { await scope.page.goto( login_url, { waitUntil: `domcontentloaded`, timeout: scope.timeout }); await scope.page.waitForSelector( `.dabody` ); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-3.jpg`}); - } catch ( error ) { let err_msg = `Error occurred when ALKiln tried to go to "${ login_url }".` if ( error.name === `TimeoutError` ) { - // console.log(`šŸ–Šļø 110`); let non_reload_report_msg = `It took too long to load "${ login_url }"`; await scope.handle_page_timeout_error( scope, { non_reload_report_data: { code: `ALK0159`, @@ -3124,8 +3119,6 @@ module.exports = { }, error }); } else { // Throw any non-timeout error - // console.log(`šŸ–Šļø 111`); - await scope.take_a_screenshot(scope, {path:`./_alkiln_temp/Temp-4.jpg`}); reports.addToReport( scope, { type: `error`, code: `ALK0160`, value: err_msg }); throw error; } // ends if error is timeout error @@ -3166,8 +3159,6 @@ module.exports = { scope.scenarios.get( scope.scenario_id ).api_keys.push( api_key ); } } - // console.log(`šŸ–Šļø 112`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-4.jpg`}); expect( email, email_msg ).to.not.equal( undefined ); expect( password, password_msg ).to.not.equal( undefined ); @@ -3177,14 +3168,9 @@ module.exports = { await scope.page.type( `#password`, password ); let elem = await scope.page.$( `button[type="submit"]` ); await scope.guard_against_missing_tap_element(scope, { elem }); - // console.log(`šŸ–Šļø 113`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-5.jpg`}); // Submit and see what happens let winner = await scope.steps.race_sign_in_navigation( scope, { elem, login_url }); - // console.log(`šŸ–Šļø 113.5 ${ winner }`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-6.jpg`}); - // console.log(`šŸ–Šļø 113.6 ${ winner }`); // Add the result to the report and possibly throw errors if ( winner[0] === `success` ) { reports.addToReport( scope, { @@ -3197,7 +3183,6 @@ module.exports = { type: `error`, code: `ALK0209`, value: `Failed to sign into ${ login_url }. Make sure you followed the instructions at https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#sign-in.` }); - // console.log(`šŸ‰ ${ error_msg }`); throw new Error( error_msg ); } else if ( winner[0] === `error` ) { let error_msg = reports.addToReport( scope, { @@ -3206,94 +3191,63 @@ module.exports = { }); throw new Error( error_msg ); } - // console.log(`šŸ–Šļø 114`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-7.jpg`}); - // if ( error_msg ) { // undefined sometimes now - // return [ error_msg ]; - // } else { - // return []; - // } }, // Ends scope.steps.sign_in() race_sign_in_navigation: async function ( scope, { elem, login_url='unknown' }) { /** Wait for sign in navigation success or failure, or system error. */ - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-0.jpg`}); - // After everything, clean up incomplete promises const controller = new AbortController; - // console.log(`ā³ 200`); // Redirect - let redirect_promise = scope.page.waitForResponse(async ( response ) => { - // console.log(`ā³ 300`); + let redirect_promise = scope.page.waitForResponse(function ( response ) { return response.status() === 302; }, { signal: controller.signal }) - .catch(async ( catch_info ) => { - // console.log(`ā³ 201`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-1.jpg`}); + .catch(( catch_info ) => { log.debug({ code: `ALK0204`, context: `nav`, }, `302 sign-in wait catch error:`, catch_info.name, catch_info, ); }); // Invalid credentials, no navigation, no sign in - let wrong_sign_in_promise = scope.page.waitForResponse(async ( response ) => { - // console.log(`ā³ 301`); + let wrong_sign_in_promise = scope.page.waitForResponse(function ( response ) { return response.status() >= 200 && response.status() < 300 && response.url().includes(`/sign-in`); }, { signal: controller.signal }) - .catch(async ( catch_info ) => { - // errors? - // console.log(`ā³ 202`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-2.jpg`}); + .catch(( catch_info ) => { log.debug({ code: `ALK0205`, context: `nav`, }, `200s sign-in wait catch error:`, catch_info.name, catch_info, ); }); // System error - let error_promise = scope.page.waitForResponse(async ( response ) => { - // console.log(`ā³ 302`); + let error_promise = scope.page.waitForResponse(function ( response ) { return response.status() >= 500 && response.status() < 600; }, { signal: controller.signal }) - .catch(async ( catch_info ) => { - // console.log(`ā³ 203`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-3.jpg`}); + .catch(( catch_info ) => { log.debug({ code: `ALK0206`, context: `nav`, }, `500s sign-in wait catch error:`, catch_info.name, catch_info, ); }); - // Duh, the other catches were triggered because those promises were aborted - let click_promise = elem.click(); // MUST complete let nav_promise = scope.nav_race(scope, {}).result; - let pic_promise = true; // scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-10.jpg`}); - // console.log(`ā³ 204`); const winner = await Promise.race([ - Promise.all([ `success`, click_promise, pic_promise, redirect_promise ]), - Promise.all([ `failure`, click_promise, pic_promise, wrong_sign_in_promise ]), - Promise.all([ `error`, click_promise, pic_promise, error_promise ]), + Promise.all([ `success`, click_promise, redirect_promise ]), + Promise.all([ `failure`, click_promise, wrong_sign_in_promise ]), + Promise.all([ `error`, click_promise, error_promise ]), ]).catch(async ( error ) => { - // console.log(`ā³ 205`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-4.jpg`}); let error_msg = reports.addToReport( scope, { type: `error`, code: `ALK0207`, value: `Unknown error waiting for results during sign in at ${ login_url }.` }); - // console.log(`ā³ 205.1`, error); throw error; }); - // console.log(`ā³ 206`); - // // await waitForTimeout(500); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp2-5.jpg`}); - // Clean up unresolved promises controller.abort(); - // console.log(`ā³ 207`); + // Wait to finish navigating + // TODO: Test the winner 'error' under this circumstance await nav_promise; - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp3-1.jpg`}); log.debug({ code: `ALK0208`, context: `nav` }, `Sign-in winner:`, winner diff --git a/lib/steps.js b/lib/steps.js index 890a96a2..ef39ee3e 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -116,7 +116,6 @@ BeforeAll(async function() { Before( beforeScenario ); async function beforeScenario( scenario ) { - // console.log('šŸ‘ļø 111 start beforeScenario'); // Start the running "progress bar" for the Scenario log.stdout({}, `\nScenario: ${ scenario.pickle.name }: `); @@ -126,20 +125,15 @@ async function beforeScenario( scenario ) { // Will only run in the Playground outside of a sandbox. TODO: There's a // better way to do this, though it's more complicated. See comments in // https://github.com/SuffolkLITLab/ALKiln/issues/661 - // console.log('šŸ‘ļø 112 no browser, await new local browser'); scope.browser = await scope.driver.launch({ args: ['--no-sandbox'] }); } else { - // console.log('šŸ‘ļø 113 no browser, await new remote browser'); scope.browser = await scope.driver.launch({ headless: !session_vars.get_debug(), devtools: session_vars.get_debug() }); } } - // console.log('šŸ‘ļø 114 we have ensured browser exists, awaiting pages'); // Clean up all previously existing pages for (const page of await scope.browser.pages()) { - // console.log('šŸ‘ļø 115 awaiting closing page'); await page.close(); } - // console.log('šŸ‘ļø 116 awaiting opening a new page'); // Make a new page scope.page = await scope.browser.newPage() @@ -153,10 +147,8 @@ async function beforeScenario( scenario ) { scope.server_reload_promise = null; reports.addReportHeading(scope, {scenario}); - // console.log('šŸ‘ļø 117 awaiting getting safe scenario name'); // Make folder for this Scenario in the all-tests artifacts folder scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario}); - // console.log('šŸ‘ļø 118 got safe scenario name'); // Add a date for uniqueness in case dev has accidentally copied a Scenario description let date = Date.now(); scope.paths.scenario = `${ scope.paths.artifacts }/${ scope.base_filename }-${ date }`; @@ -297,10 +289,8 @@ Given(/I go to "([^"]+)"/i, {timeout: -1}, async ( url ) => { return result; }); -foo = null Given( /I (?:sign|log) ?(?:in)?(?:on)?(?:to the server)? with(?: the email)? "([^"]+)",?(?: and)?(?: the password)? "([^"]+)"(?: SECRETs)?(?:,?(?: and)?(?: the API key)? "([^"]+)")?/i, - { timeout: -1 }, async ( email, password, api_key ) => { /** Uses the names of environment variables (most often GitHub SECRETs) to * log into an account on the user's server. Must be secure. @@ -315,21 +305,11 @@ Given( // `page` timeout will deal with custom timeout // Couldn't test the timeout for tapping the button because there's not // enough of a pause between initial navigation and pressing button. - foo = scope.page - // console.log(`ā­ļø ${ JSON.stringify(scope.page, null, 2) }`) await scope.steps.sign_in( scope, { email_secret_name: email, password_secret_name: password, api_key_secret_name: api_key }); - - - // console.log(`šŸ–Šļø 115`); - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-8.jpg`}); - - // if ( !result.ok && result.errors.length > 0 ) { - // throw result.errors[ result.errors.length - 1 ]; - // } }); // I am using a mobile/pc @@ -1343,8 +1323,6 @@ Then(/I fail to delete (\d) detected interview(?:s)? and get no error __internal AfterStep(async function({ result }) { // TODO: Abstract this - // await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-13.jpg`}); - // console.log(`šŸ›‘ 100`); if ( result.status === `PASSED` ) { log.stdout({ records_only: true }, `.`); } else if ( result.status === `FAILED` ) { log.stdout({ records_only: true }, `F`); } else if ( result.status === `UNDEFINED` ) { log.stdout({ records_only: true }, `U`); } @@ -1352,13 +1330,10 @@ AfterStep(async function({ result }) { else if ( result.status === `PENDING` ) { log.stdout({ records_only: true }, `P`); } reports.outdent(); - // console.log(`šŸ›‘ 101`); }); After(afterScenario); async function afterScenario(scenario) { - // console.log(`šŸ‘€ 1 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-30.jpg`}); // Log errors if ( scenario.result.message ) { log.debug({ code: `ALK0091`, context: `scenario`, }, @@ -1377,8 +1352,6 @@ async function afterScenario(scenario) { value: `Accessibility standards tests failed. See information above.` }); } - // console.log(`šŸ‘€ 2`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-9.jpg`}); if ( scope.failed_pdf_compares.length > 0) { let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n―――\n${ new_msg }`) @@ -1400,46 +1373,54 @@ async function afterScenario(scenario) { ); if ( scope.page ) { - // console.log(`šŸ‘€ 3`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-10.jpg`}); if ( !!scope.disable_error_screenshot ) { reports.addToReport(scope, { type: `row info`, code: `ALK0095`, // Discuss: Might people use secret variables to prevent any - // information about a screen from getting out? For example, they have - // proprietary info in the HTML itself. - value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML, which omits the values in the fields.` + // information about a screen from getting out? For example, they may + // have proprietary info in the HTML itself. + value: `For security, ALKiln will avoid creating a picture of the page for this error. It's possible a secret is being used on this screen. ALKiln will save the HTML as a file. The HTML omits the values in the fields.` + }); + } + + // Could have an error if page can't load or something similar + try { + // Save/download a picture of the screen that's showing during the unexpected status + // Save one copy in the outer-most artifact folder + let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` }); + let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`; + await scope.take_a_screenshot( scope, { + path: path_outer, + disable_pic: scope.disable_error_screenshot, + disable_html: false, + }); + + // Save another copy in the artifact's Scenario folder + let screenshot_name = `error_on`; + let { id } = await scope.examinePageID( scope, 'none to match' ); + let short_id = `${ id }`.substring(0, 20); + screenshot_name += `-${ short_id }`; + let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`; + await scope.take_a_screenshot( scope, { + path: path_scenario, + disable_pic: scope.disable_error_screenshot, + disable_html: false, }); + + } catch ( page_error ) { + // Fail silently. Our inability to take a pic shouldn't cause confusion + // about why a test failed + if ( page_error.message.lower().includes(`execution context`) ) { + reports.addToReport(scope, { + type: `row warning`, + code: `ALK0281`, + value: `ALKiln is unable to get any record of this page whatsoever, even the page HTML. Your server may be busy.` + }); + } + log.debug({ code: `ALK0282`, level: `note`, }, page_error ); } - // console.log(`šŸ‘€ 4 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-11.jpg`}); - - // Save/download a picture of the screen that's showing during the unexpected status - // Save one copy in the outer-most artifact folder - let scenario_filename = await scope.getSafeScenarioFilename( scope, { prefix: `error_on` }); - let path_outer = `${ scope.paths.artifacts }/${ scenario_filename }.jpg`; - await scope.take_a_screenshot( scope, { - path: path_outer, - disable_pic: scope.disable_error_screenshot, - disable_html: false, - }); - // console.log(`šŸ‘€ 5 ${ foo == scope.page } ${ JSON.stringify(scope.page, null, 2) }`); - await scope.take_a_screenshot(scope, {path:`${scope.paths.artifacts}/Temp-12.jpg`}); - - // Save another copy in the artifact's Scenario folder - let screenshot_name = `error_on`; - let { id } = await scope.examinePageID( scope, 'none to match' ); - let short_id = `${ id }`.substring(0, 20); - screenshot_name += `-${ short_id }`; - let path_scenario = `${ scope.paths.scenario }/${ screenshot_name }.jpg`; - await scope.take_a_screenshot( scope, { - path: path_scenario, - disable_pic: scope.disable_error_screenshot, - disable_html: false, - }); - // console.log(`šŸ‘€ 6`); } // ends if scope.page exists @@ -1447,59 +1428,48 @@ async function afterScenario(scenario) { if (changeable_test_status === `FAILED`) { reports.addToReport(scope, { type: `outcome failure info`, code: `ALK0096`, value: `**-- Scenario Failed --**` }); } - // console.log(`šŸ‘€ 7`); } // ends if not passed - // console.log(`šŸ‘€ 8`); // Add report text to Scenario folder after everything has been added to the report let scenario_report_obj = scope.report.get( scope.scenario_id ); - // console.log(`šŸ‘€ 8.01`); let report = reports.getPrintableScenario( scenario_report_obj ); // Save the report as in the Scenario folder - // console.log(`šŸ‘€ 8.02`); fs.writeFileSync( scope.paths.scenario_report, report ); - // console.log(`šŸ‘€ 8.03`); // TODO: Save any cucumber failure message for the Scenario here (instead of waiting till run_cucumber.js) // ---------------- Check internal test results ---------------- if ( scope.expected_in_report && scope.expected_in_report.length > 0) { - // console.log(`šŸ‘€ 8.04`); let data = await scope.reportIncludesAllExpected( scope, { expected: scope.expected_in_report } ); let report_includes_all_expected_strings = data.all_included; - // console.log(`šŸ‘€ 8.05`); if ( !report_includes_all_expected_strings ) { changeable_test_status = `FAILED`; let msg = `These strings are missing from the report: ${ JSON.stringify(data.missing, null, 2) } Instead the report had this text: ${ report }`; + // These failure messages won't match exactly, which is a shame, but at + // least our devs will have more data _somewhere_ about their failure. expect( report, msg ).to.contain( data.missing[0] ); } // Reset report values no matter what so they don't mess up future scenarios scope.expected_in_report = null; } - // console.log(`šŸ‘€ 8.06`); if (scope.expected_not_in_report && scope.expected_not_in_report.length > 0) { - // console.log(`šŸ‘€ 8.07`); let all_prohibited_strings_were_absent = await scope.reportDoesNotInclude( scope, { not_expected: scope.expected_not_in_report } ); - // console.log(`šŸ‘€ 8.08`); if ( !all_prohibited_strings_were_absent ) { changeable_test_status = `FAILED`;} - // console.log(`šŸ‘€ 8.09`); // Reset report values no matter what so they don't mess up future scenarios scope.expected_not_in_report = null; - // console.log(`šŸ‘€ 8.091`); } - // console.log(`šŸ‘€ 9`); // ---------------- Ends internal results check ---------------- @@ -1537,7 +1507,6 @@ async function afterScenario(scenario) { ); } } - // console.log(`šŸ‘€ 10`); scope.expected_status = null; // reset for next Scenario let need_to_force_failed_status = false; @@ -1578,15 +1547,12 @@ async function afterScenario(scenario) { let signout_error_msg = ``; // If there is a page open, then sign out and close it if ( scope.page ) { - // console.log(`šŸ‘€ 11`); // Catch a possible reload error. try { // puppeteer will ensure proper timeout. - // console.log(`šŸ‘€ 12`); await scope.page.goto(`${ session_vars.get_da_server_url() }/user/sign-out`, {waitUntil: `domcontentloaded`}); } catch ( error ) { - // console.log(`šŸ‘€ 13`); signout_succeeded = false; if ( error.name === `TimeoutError` ) { @@ -1613,7 +1579,6 @@ async function afterScenario(scenario) { } // ends try/catch } // ends if scope.page - // console.log(`šŸ‘€ 14`); // Now that we're no longer on the interview page, one way or // another, try to delete the interviews created during the test @@ -1670,7 +1635,6 @@ async function afterScenario(scenario) { internal_errors ); } - // console.log(`šŸ‘€ 15`); // Find the race condition. log.debug({ code: `ALK0100`, context: `scenario`, }, diff --git a/package.json b/package.json index 79d280a0..43656cc2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@suffolklitlab/alkiln", - "version": "5.16.1-always-html-5", + "version": "5.16.1-always-html-6", "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.", "main": "lib/index.js", "scripts": {