diff --git a/src/actions/receive-profile.ts b/src/actions/receive-profile.ts index faa49d33d4..7eb4182cdd 100644 --- a/src/actions/receive-profile.ts +++ b/src/actions/receive-profile.ts @@ -2,6 +2,13 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { oneLine } from 'common-tags'; +import { + fetchProfile, + getProfileUrlForHash, + type ProfileOrZip, + deduceContentType, + extractJsonFromArrayBuffer, +} from 'firefox-profiler/utils/profile-fetch'; import queryString from 'query-string'; import type JSZip from 'jszip'; import { @@ -20,10 +27,8 @@ import { } from 'firefox-profiler/profile-logic/symbolication'; import * as MozillaSymbolicationAPI from 'firefox-profiler/profile-logic/mozilla-symbolication-api'; import { mergeProfilesForDiffing } from 'firefox-profiler/profile-logic/merge-compare'; -import { decompress, isGzip } from 'firefox-profiler/utils/gz'; import { expandUrl } from 'firefox-profiler/utils/shorten-url'; import { TemporaryError } from 'firefox-profiler/utils/errors'; -import { isLocalURL } from 'firefox-profiler/utils/url'; import { getSelectedThreadIndexesOrNull, getGlobalTrackOrder, @@ -67,7 +72,6 @@ import { import { setDataSource } from './profile-view'; import { fatalError } from './errors'; import { batchLoadDataUrlIcons } from './icons'; -import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants'; import { determineTimelineType, hasUsefulSamples, @@ -547,7 +551,7 @@ async function _unpackGeckoProfileFromBrowser( // global. This happens especially with tests but could happen in the future // in Firefox too. if (Object.prototype.toString.call(profile) === '[object ArrayBuffer]') { - return _extractJsonFromArrayBuffer(profile as ArrayBuffer); + return extractJsonFromArrayBuffer(profile as ArrayBuffer); } return profile; } @@ -557,9 +561,9 @@ function getSymbolStore( symbolServerUrl: string, browserConnection: BrowserConnection | null ): SymbolStore | null { - if (!window.indexedDB) { - // We could be running in a test environment with no indexedDB support. Do not - // return a symbol store in this case. + if (typeof window === 'undefined' || !window.indexedDB) { + // We could be running in a test environment or Node.js with no indexedDB support. + // Do not return a symbol store in this case. return null; } @@ -983,265 +987,6 @@ export function temporaryError(error: TemporaryError): Action { }; } -function _wait(delayMs: number): Promise { - return new Promise((resolve) => setTimeout(resolve, delayMs)); -} - -function _loadProbablyFailedDueToSafariLocalhostHTTPRestriction( - url: string, - error: Error -): boolean { - if (!navigator.userAgent.match(/Safari\/\d+\.\d+/)) { - return false; - } - // Check if Safari considers this mixed content. - const parsedUrl = new URL(url); - return ( - error.name === 'TypeError' && - parsedUrl.protocol === 'http:' && - isLocalURL(parsedUrl) && - location.protocol === 'https:' - ); -} - -class SafariLocalhostHTTPLoadError extends Error { - override name = 'SafariLocalhostHTTPLoadError'; -} - -type FetchProfileArgs = { - url: string; - onTemporaryError: (param: TemporaryError) => void; - // Allow tests to capture the reported error, but normally use console.error. - reportError?: (...data: Array) => void; -}; - -type ProfileOrZip = - | { responseType: 'PROFILE'; profile: unknown } - | { responseType: 'ZIP'; zip: JSZip }; - -/** - * Tries to fetch a profile on `url`. If the profile is not found, - * `onTemporaryError` is called with an appropriate error, we wait 1 second, and - * then tries again. If we still can't find the profile after 11 tries, the - * returned promise is rejected with a fatal error. - * If we can retrieve the profile properly, the returned promise is resolved - * with the JSON.parsed profile. - */ -export async function _fetchProfile( - args: FetchProfileArgs -): Promise { - const MAX_WAIT_SECONDS = 10; - let i = 0; - const { url, onTemporaryError } = args; - // Allow tests to capture the reported error, but normally use console.error. - const reportError = args.reportError || console.error; - - while (true) { - let response; - try { - response = await fetch(url); - } catch (e) { - // Case 1: Exception. - if (_loadProbablyFailedDueToSafariLocalhostHTTPRestriction(url, e)) { - throw new SafariLocalhostHTTPLoadError(); - } - throw e; - } - - // Case 2: successful answer. - if (response.ok) { - return _extractProfileOrZipFromResponse(url, response, reportError); - } - - // case 3: unrecoverable error. - if (response.status !== 403) { - throw new Error(oneLine` - Could not fetch the profile on remote server. - Response was: ${response.status} ${response.statusText}. - `); - } - - // case 4: 403 errors can be transient while a profile is uploaded. - - if (i++ === MAX_WAIT_SECONDS) { - // In the last iteration we don't send a temporary error because we'll - // throw an error right after the while loop. - break; - } - - onTemporaryError( - new TemporaryError( - 'Profile not found on remote server.', - { count: i, total: MAX_WAIT_SECONDS + 1 } // 11 tries during 10 seconds - ) - ); - - await _wait(1000); - } - - throw new Error(oneLine` - Could not fetch the profile on remote server: - still not found after ${MAX_WAIT_SECONDS} seconds. - `); -} - -/** - * Deduce the file type from a url and content type. Third parties can give us - * arbitrary information, so make sure that we try out best to extract the proper - * information about it. - */ -function _deduceContentType( - url: string, - contentType: string | null -): 'application/json' | 'application/zip' | null { - if (contentType === 'application/zip' || contentType === 'application/json') { - return contentType; - } - if (url.match(/\.zip$/)) { - return 'application/zip'; - } - if (url.match(/\.json/)) { - return 'application/json'; - } - return null; -} - -/** - * This function guesses the correct content-type (even if one isn't sent) and then - * attempts to use the proper method to extract the response. - */ -async function _extractProfileOrZipFromResponse( - url: string, - response: Response, - reportError: (...data: Array) => void -): Promise { - const contentType = _deduceContentType( - url, - response.headers.get('content-type') - ); - switch (contentType) { - case 'application/zip': - return { - responseType: 'ZIP', - zip: await _extractZipFromResponse(response, reportError), - }; - case 'application/json': - case null: - // The content type is null if it is unknown, or an unsupported type. Go ahead - // and try to process it as a profile. - return { - responseType: 'PROFILE', - profile: await _extractJsonFromResponse( - response, - reportError, - contentType - ), - }; - default: - throw assertExhaustiveCheck(contentType); - } -} - -/** - * Attempt to load a zip file from a third party. This process can fail, so make sure - * to handle and report the error if it does. - */ -async function _extractZipFromResponse( - response: Response, - reportError: (...data: Array) => void -): Promise { - const buffer = await response.arrayBuffer(); - // Workaround for https://github.com/Stuk/jszip/issues/941 - // When running this code in tests, `buffer` doesn't inherits from _this_ - // realm's ArrayBuffer object, and this breaks JSZip which doesn't account for - // this case. We workaround the issue by wrapping the buffer in an Uint8Array - // that comes from this realm. - const typedBuffer = new Uint8Array(buffer); - try { - const { default: JSZip } = await import('jszip'); - const zip = await JSZip.loadAsync(typedBuffer); - // Catch the error if unable to load the zip. - return zip; - } catch (error) { - const message = 'Unable to open the archive file.'; - reportError(message); - reportError('Error:', error); - reportError('Fetch response:', response); - throw new Error( - `${message} The full error information has been printed out to the DevTool’s console.` - ); - } -} - -/** - * Parse JSON from an optionally gzipped array buffer. - */ -async function _extractJsonFromArrayBuffer( - arrayBuffer: ArrayBuffer -): Promise { - let profileBytes = new Uint8Array(arrayBuffer); - // Check for the gzip magic number in the header. - if (isGzip(profileBytes)) { - profileBytes = await decompress(profileBytes); - } - - const textDecoder = new TextDecoder(); - return JSON.parse(textDecoder.decode(profileBytes)); -} - -/** - * Don't trust third party responses, try and handle a variety of responses gracefully. - */ -async function _extractJsonFromResponse( - response: Response, - reportError: (...data: Array) => void, - fileType: 'application/json' | null -): Promise { - let arrayBuffer: ArrayBuffer | null = null; - try { - // await before returning so that we can catch JSON parse errors. - arrayBuffer = await response.arrayBuffer(); - return await _extractJsonFromArrayBuffer(arrayBuffer); - } catch (error) { - // Change the error message depending on the circumstance: - let message; - if (error && typeof error === 'object' && error.name === 'AbortError') { - message = 'The network request to load the profile was aborted.'; - } else if (fileType === 'application/json') { - message = 'The profile’s JSON could not be decoded.'; - } else if (fileType === null && arrayBuffer !== null) { - // If the content type is not specified, use a raw array buffer - // to fallback to other supported profile formats. - return arrayBuffer; - } else { - message = oneLine` - The profile could not be downloaded and decoded. This does not look like a supported file - type. - `; - } - - // Provide helpful debugging information to the console. - reportError(message); - reportError('JSON parsing error:', error); - reportError('Fetch response:', response); - - throw new Error( - `${message} The full error information has been printed out to the DevTool’s console.` - ); - } -} - -export function getProfileUrlForHash(hash: string): string { - // See https://cloud.google.com/storage/docs/access-public-data - // The URL is https://storage.googleapis.com//. - // https://.storage.googleapis.com/ seems to also work but - // is not documented nowadays. - - // By convention, "profile-store" is the name of our bucket, and the file path - // is the hash we receive in the URL. - return `https://storage.googleapis.com/${GOOGLE_STORAGE_BUCKET}/${hash}`; -} - export function retrieveProfileFromStore( hash: string, initialLoad: boolean = false @@ -1262,7 +1007,7 @@ export function retrieveProfileOrZipFromUrl( dispatch(waitingForProfileFromUrl(profileUrl)); try { - const response: ProfileOrZip = await _fetchProfile({ + const response: ProfileOrZip = await fetchProfile({ url: profileUrl, onTemporaryError: (e: TemporaryError) => { dispatch(temporaryError(e)); @@ -1293,7 +1038,7 @@ export function retrieveProfileOrZipFromUrl( default: throw assertExhaustiveCheck( response as never, - 'Expected to receive an archive or profile from _fetchProfile.' + 'Expected to receive an archive or profile from fetchProfile.' ); } } catch (error) { @@ -1349,7 +1094,7 @@ export function retrieveProfileFromFile( dispatch(waitingForProfileFromFile()); try { - if (_deduceContentType(file.name, file.type) === 'application/zip') { + if (deduceContentType(file.name, file.type) === 'application/zip') { // Open a zip file in the zip file viewer const buffer = await fileReader(file).asArrayBuffer(); const { default: JSZip } = await import('jszip'); @@ -1446,14 +1191,14 @@ export function retrieveProfilesToCompare( const profileUrl = getProfileFetchUrl(url); - const response: ProfileOrZip = await _fetchProfile({ + const response: ProfileOrZip = await fetchProfile({ url: profileUrl, onTemporaryError: (e: TemporaryError) => { dispatch(temporaryError(e)); }, }); if (response.responseType !== 'PROFILE') { - throw new Error('Expected to receive a profile from _fetchProfile'); + throw new Error('Expected to receive a profile from fetchProfile'); } const upgradeInfo: ProfileUpgradeInfo = {}; diff --git a/src/test/components/Root-history.test.tsx b/src/test/components/Root-history.test.tsx index e2c34a6bf2..13a130aa71 100644 --- a/src/test/components/Root-history.test.tsx +++ b/src/test/components/Root-history.test.tsx @@ -11,7 +11,7 @@ import { import { Root } from '../../components/app/Root'; import { autoMockCanvasContext } from '../fixtures/mocks/canvas-context'; import { fireFullClick } from '../fixtures/utils'; -import { getProfileUrlForHash } from '../../actions/receive-profile'; +import { getProfileUrlForHash } from '../../utils/profile-fetch'; import { blankStore } from '../fixtures/stores'; import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile'; import { diff --git a/src/test/fixtures/utils.ts b/src/test/fixtures/utils.ts index 0f50030d9d..84290f4f7b 100644 --- a/src/test/fixtures/utils.ts +++ b/src/test/fixtures/utils.ts @@ -641,6 +641,26 @@ function isControlInput(element: HTMLElement): boolean { ); } +/** + * Returns an ArrayBuffer which contains only the bytes that are covered by the + * Uint8Array, making a copy if needed. + */ +export function extractArrayBuffer( + bufferView: Uint8Array +): ArrayBuffer { + if ( + bufferView.byteOffset === 0 && + bufferView.byteLength === bufferView.buffer.byteLength + ) { + return bufferView.buffer; + } + + // There was extra data at the start or at the end. Make a copy. + const copy = new Uint8Array(bufferView.byteLength); + copy.set(bufferView); + return copy.buffer; +} + /** * Adds a source entry to the sources table and returns the index. * If a source with the same URL already exists, returns the existing index. diff --git a/src/test/store/__snapshots__/receive-profile.test.ts.snap b/src/test/store/__snapshots__/receive-profile.test.ts.snap index 04b089ee95..02c2f06f5a 100644 --- a/src/test/store/__snapshots__/receive-profile.test.ts.snap +++ b/src/test/store/__snapshots__/receive-profile.test.ts.snap @@ -1,59 +1,5 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`actions/receive-profile _fetchProfile fails if a bad profile JSON is passed in 1`] = `[Error: The profile’s JSON could not be decoded. The full error information has been printed out to the DevTool’s console.]`; - -exports[`actions/receive-profile _fetchProfile fails if a bad profile JSON is passed in 2`] = ` -Array [ - Array [ - "The profile’s JSON could not be decoded.", - ], - Array [ - "JSON parsing error:", - [SyntaxError: Unexpected token 'i', "invalid" is not valid JSON], - ], - Array [ - "Fetch response:", - Response {}, - ], -] -`; - -exports[`actions/receive-profile _fetchProfile fails if a bad profile JSON is passed in, with no content type 1`] = `[Error: The profile’s JSON could not be decoded. The full error information has been printed out to the DevTool’s console.]`; - -exports[`actions/receive-profile _fetchProfile fails if a bad profile JSON is passed in, with no content type 2`] = ` -Array [ - Array [ - "The profile’s JSON could not be decoded.", - ], - Array [ - "JSON parsing error:", - [SyntaxError: Unexpected token 'i', "invalid" is not valid JSON], - ], - Array [ - "Fetch response:", - Response {}, - ], -] -`; - -exports[`actions/receive-profile _fetchProfile fails if a bad zip file is passed in 1`] = `[Error: Unable to open the archive file. The full error information has been printed out to the DevTool’s console.]`; - -exports[`actions/receive-profile _fetchProfile fails if a bad zip file is passed in 2`] = ` -Array [ - Array [ - "Unable to open the archive file.", - ], - Array [ - "Error:", - [Error: Can't find end of central directory : is this a zip file ? If it is, see https://stuk.github.io/jszip/documentation/howto/read_zip.html], - ], - Array [ - "Fetch response:", - Response {}, - ], -] -`; - exports[`actions/receive-profile retrieveProfileFromFile will be an error to view a profile with no threads 1`] = `"No threads were captured in this profile, there is nothing to display."`; exports[`actions/receive-profile retrieveProfileFromFile will give an error when unable to decompress a zipped profile 1`] = `[Error: Can't find end of central directory : is this a zip file ? If it is, see https://stuk.github.io/jszip/documentation/howto/read_zip.html]`; diff --git a/src/test/store/receive-profile.test.ts b/src/test/store/receive-profile.test.ts index 0b336a05d8..08fa5b3601 100644 --- a/src/test/store/receive-profile.test.ts +++ b/src/test/store/receive-profile.test.ts @@ -27,9 +27,9 @@ import { retrieveProfileOrZipFromUrl, retrieveProfileFromFile, retrieveProfilesToCompare, - _fetchProfile, retrieveProfileForRawUrl, } from '../../actions/receive-profile'; +import { fetchProfile as _fetchProfile } from '../../utils/profile-fetch'; import { SymbolsNotFoundError } from '../../profile-logic/errors'; import { createGeckoProfile } from '../fixtures/profiles/gecko-profile'; @@ -46,7 +46,7 @@ import { getProfileWithThreadCPUDelta, } from '../fixtures/profiles/processed-profile'; import { getHumanReadableTracks } from '../fixtures/profiles/tracks'; -import { waitUntilState } from '../fixtures/utils'; +import { waitUntilState, extractArrayBuffer } from '../fixtures/utils'; import { dataUrlToBytes } from 'firefox-profiler/utils/base64'; import { compress } from '../../utils/gz'; @@ -88,22 +88,6 @@ function simulateSymbolStoreHasNoCache() { })); } -// Returns an ArrayBuffer which contains only the bytes that -// are covered by the Uint8Array, making a copy if needed. -function extractArrayBuffer(bufferView: Uint8Array): ArrayBuffer { - if ( - bufferView.byteOffset === 0 && - bufferView.byteLength === bufferView.buffer.byteLength - ) { - return bufferView.buffer; - } - - // There was extra data at the start or at the end. Make a copy. - const copy = new Uint8Array(bufferView.byteLength); - copy.set(bufferView); - return copy.buffer; -} - describe('actions/receive-profile', function () { beforeEach(() => { // The SymbolStore requires the use of IndexedDB, ensure that it exists so that @@ -1150,193 +1134,6 @@ describe('actions/receive-profile', function () { }); }); - /** - * _fetchProfile is a helper function for the actions, but it is tested separately - * since it has a decent amount of complexity around different issues with loading - * in different support URL formats. It's mainly testing what happens when JSON - * and zip file is sent, and what happens when things fail. - */ - describe('_fetchProfile', function () { - /** - * This helper function encapsulates various configurations for the type of content - * as well and response headers. - */ - async function configureFetch(obj: { - url: string; - contentType?: string; - content: 'generated-zip' | 'generated-json' | Uint8Array; - }) { - const { url, contentType, content } = obj; - const stringProfile = serializeProfile(_getSimpleProfile()); - const profile = JSON.parse(stringProfile); - let arrayBuffer; - - switch (content) { - case 'generated-zip': { - const zip = new JSZip(); - zip.file('profile.json', stringProfile); - arrayBuffer = await zip.generateAsync({ type: 'uint8array' }); - break; - } - case 'generated-json': - arrayBuffer = encode(stringProfile); - break; - default: - arrayBuffer = content; - break; - } - - window.fetchMock.catch(403).get(url, { - body: arrayBuffer, - headers: { - 'content-type': contentType, - }, - }); - - const reportError = jest.fn(); - const args = { - url, - onTemporaryError: () => {}, - reportError, - }; - - // Return fetch's args, based on the inputs. - return { profile, args, reportError }; - } - - it('fetches a normal profile with the correct content-type headers', async function () { - const { profile, args } = await configureFetch({ - url: 'https://example.com/profile.json', - contentType: 'application/json', - content: 'generated-json', - }); - - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); - }); - - it('fetches a zipped profile with correct content-type headers', async function () { - const { args, reportError } = await configureFetch({ - url: 'https://example.com/profile.zip', - contentType: 'application/zip', - content: 'generated-zip', - }); - - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip.responseType).toBe('ZIP'); - expect(reportError.mock.calls.length).toBe(0); - }); - - it('fetches a zipped profile with incorrect content-type headers, but .zip extension', async function () { - const { args, reportError } = await configureFetch({ - url: 'https://example.com/profile.zip', - content: 'generated-zip', - }); - - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip.responseType).toBe('ZIP'); - expect(reportError.mock.calls.length).toBe(0); - }); - - it('fetches a profile with incorrect content-type headers, but .json extension', async function () { - const { profile, args, reportError } = await configureFetch({ - url: 'https://example.com/profile.json', - content: 'generated-json', - }); - - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); - expect(reportError.mock.calls.length).toBe(0); - }); - - it('fetches a profile with incorrect content-type headers, no known extension, and attempts to JSON parse it it', async function () { - const { profile, args, reportError } = await configureFetch({ - url: 'https://example.com/profile.file', - content: 'generated-json', - }); - - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); - expect(reportError.mock.calls.length).toBe(0); - }); - - it('fails if a bad zip file is passed in', async function () { - const { args, reportError } = await configureFetch({ - url: 'https://example.com/profile.file', - contentType: 'application/zip', - content: new Uint8Array([0, 1, 2, 3]), - }); - - let userFacingError; - try { - await _fetchProfile(args); - } catch (error) { - userFacingError = error; - } - expect(userFacingError).toMatchSnapshot(); - expect(reportError.mock.calls.length).toBeGreaterThan(0); - expect(reportError.mock.calls).toMatchSnapshot(); - }); - - it('fails if a bad profile JSON is passed in', async function () { - const invalidJSON = 'invalid'; - const { args, reportError } = await configureFetch({ - url: 'https://example.com/profile.json', - contentType: 'application/json', - content: encode(invalidJSON), - }); - - let userFacingError; - try { - await _fetchProfile(args); - } catch (error) { - userFacingError = error; - } - expect(userFacingError).toMatchSnapshot(); - expect(reportError.mock.calls.length).toBeGreaterThan(0); - expect(reportError.mock.calls).toMatchSnapshot(); - }); - - it('fails if a bad profile JSON is passed in, with no content type', async function () { - const invalidJSON = 'invalid'; - const { args, reportError } = await configureFetch({ - url: 'https://example.com/profile.json', - content: encode(invalidJSON), - }); - - let userFacingError; - try { - await _fetchProfile(args); - } catch (error) { - userFacingError = error; - } - expect(userFacingError).toMatchSnapshot(); - expect(reportError.mock.calls.length).toBeGreaterThan(0); - expect(reportError.mock.calls).toMatchSnapshot(); - }); - - it('fallback behavior if a completely unknown file is passed in', async function () { - const invalidJSON = 'invalid'; - const profile = encode(invalidJSON); - const { args } = await configureFetch({ - url: 'https://example.com/profile.unknown', - content: profile, - }); - - let userFacingError = null; - try { - const profileOrZip = await _fetchProfile(args); - expect(profileOrZip).toEqual({ - responseType: 'PROFILE', - profile: extractArrayBuffer(profile), - }); - } catch (error) { - userFacingError = error; - } - expect(userFacingError).toBeNull(); - }); - }); - describe('retrieveProfileFromFile', function () { /** * Bypass all of Flow's checks, and mock out the file interface. diff --git a/src/test/unit/__snapshots__/profile-fetch.test.ts.snap b/src/test/unit/__snapshots__/profile-fetch.test.ts.snap new file mode 100644 index 0000000000..1de4f42d93 --- /dev/null +++ b/src/test/unit/__snapshots__/profile-fetch.test.ts.snap @@ -0,0 +1,55 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`fetchProfile fails if a bad profile JSON is passed in 1`] = `[Error: The profile’s JSON could not be decoded. The full error information has been printed out to the DevTool’s console.]`; + +exports[`fetchProfile fails if a bad profile JSON is passed in 2`] = ` +Array [ + Array [ + "The profile’s JSON could not be decoded.", + ], + Array [ + "JSON parsing error:", + [SyntaxError: Unexpected token 'i', "invalid" is not valid JSON], + ], + Array [ + "Fetch response:", + Response {}, + ], +] +`; + +exports[`fetchProfile fails if a bad profile JSON is passed in, with no content type 1`] = `[Error: The profile’s JSON could not be decoded. The full error information has been printed out to the DevTool’s console.]`; + +exports[`fetchProfile fails if a bad profile JSON is passed in, with no content type 2`] = ` +Array [ + Array [ + "The profile’s JSON could not be decoded.", + ], + Array [ + "JSON parsing error:", + [SyntaxError: Unexpected token 'i', "invalid" is not valid JSON], + ], + Array [ + "Fetch response:", + Response {}, + ], +] +`; + +exports[`fetchProfile fails if a bad zip file is passed in 1`] = `[Error: Unable to open the archive file. The full error information has been printed out to the DevTool’s console.]`; + +exports[`fetchProfile fails if a bad zip file is passed in 2`] = ` +Array [ + Array [ + "Unable to open the archive file.", + ], + Array [ + "Error:", + [Error: Can't find end of central directory : is this a zip file ? If it is, see https://stuk.github.io/jszip/documentation/howto/read_zip.html], + ], + Array [ + "Fetch response:", + Response {}, + ], +] +`; diff --git a/src/test/unit/profile-fetch.test.ts b/src/test/unit/profile-fetch.test.ts new file mode 100644 index 0000000000..f971b291ad --- /dev/null +++ b/src/test/unit/profile-fetch.test.ts @@ -0,0 +1,208 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import JSZip from 'jszip'; + +import { fetchProfile } from '../../utils/profile-fetch'; +import { serializeProfile } from '../../profile-logic/process-profile'; +import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile'; +import { extractArrayBuffer } from '../fixtures/utils'; + +import type { Profile } from 'firefox-profiler/types'; + +function encode(string: string): Uint8Array { + return new TextEncoder().encode(string); +} + +/** + * This profile will have a single sample, and a single thread. + */ +function _getSimpleProfile(): Profile { + return getProfileFromTextSamples('A').profile; +} + +/** + * fetchProfile has a decent amount of complexity around different issues with loading + * in different support URL formats. It's mainly testing what happens when JSON + * and zip file is sent, and what happens when things fail. + */ +describe('fetchProfile', function () { + /** + * This helper function encapsulates various configurations for the type of content + * as well and response headers. + */ + async function configureFetch(obj: { + url: string; + contentType?: string; + content: 'generated-zip' | 'generated-json' | Uint8Array; + }) { + const { url, contentType, content } = obj; + const stringProfile = serializeProfile(_getSimpleProfile()); + const profile = JSON.parse(stringProfile); + let arrayBuffer; + + switch (content) { + case 'generated-zip': { + const zip = new JSZip(); + zip.file('profile.json', stringProfile); + arrayBuffer = await zip.generateAsync({ type: 'uint8array' }); + break; + } + case 'generated-json': + arrayBuffer = encode(stringProfile); + break; + default: + arrayBuffer = content; + break; + } + + window.fetchMock.catch(403).get(url, { + body: arrayBuffer, + headers: { + 'content-type': contentType, + }, + }); + + const reportError = jest.fn(); + const args = { + url, + onTemporaryError: () => {}, + reportError, + }; + + // Return fetch's args, based on the inputs. + return { profile, args, reportError }; + } + + it('fetches a normal profile with the correct content-type headers', async function () { + const { profile, args } = await configureFetch({ + url: 'https://example.com/profile.json', + contentType: 'application/json', + content: 'generated-json', + }); + + const profileOrZip = await fetchProfile(args); + expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); + }); + + it('fetches a zipped profile with correct content-type headers', async function () { + const { args, reportError } = await configureFetch({ + url: 'https://example.com/profile.zip', + contentType: 'application/zip', + content: 'generated-zip', + }); + + const profileOrZip = await fetchProfile(args); + expect(profileOrZip.responseType).toBe('ZIP'); + expect(reportError.mock.calls.length).toBe(0); + }); + + it('fetches a zipped profile with incorrect content-type headers, but .zip extension', async function () { + const { args, reportError } = await configureFetch({ + url: 'https://example.com/profile.zip', + content: 'generated-zip', + }); + + const profileOrZip = await fetchProfile(args); + expect(profileOrZip.responseType).toBe('ZIP'); + expect(reportError.mock.calls.length).toBe(0); + }); + + it('fetches a profile with incorrect content-type headers, but .json extension', async function () { + const { profile, args, reportError } = await configureFetch({ + url: 'https://example.com/profile.json', + content: 'generated-json', + }); + + const profileOrZip = await fetchProfile(args); + expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); + expect(reportError.mock.calls.length).toBe(0); + }); + + it('fetches a profile with incorrect content-type headers, no known extension, and attempts to JSON parse it', async function () { + const { profile, args, reportError } = await configureFetch({ + url: 'https://example.com/profile.file', + content: 'generated-json', + }); + + const profileOrZip = await fetchProfile(args); + expect(profileOrZip).toEqual({ responseType: 'PROFILE', profile }); + expect(reportError.mock.calls.length).toBe(0); + }); + + it('fails if a bad zip file is passed in', async function () { + const { args, reportError } = await configureFetch({ + url: 'https://example.com/profile.file', + contentType: 'application/zip', + content: new Uint8Array([0, 1, 2, 3]), + }); + + let userFacingError; + try { + await fetchProfile(args); + } catch (error) { + userFacingError = error; + } + expect(userFacingError).toMatchSnapshot(); + expect(reportError.mock.calls.length).toBeGreaterThan(0); + expect(reportError.mock.calls).toMatchSnapshot(); + }); + + it('fails if a bad profile JSON is passed in', async function () { + const invalidJSON = 'invalid'; + const { args, reportError } = await configureFetch({ + url: 'https://example.com/profile.json', + contentType: 'application/json', + content: encode(invalidJSON), + }); + + let userFacingError; + try { + await fetchProfile(args); + } catch (error) { + userFacingError = error; + } + expect(userFacingError).toMatchSnapshot(); + expect(reportError.mock.calls.length).toBeGreaterThan(0); + expect(reportError.mock.calls).toMatchSnapshot(); + }); + + it('fails if a bad profile JSON is passed in, with no content type', async function () { + const invalidJSON = 'invalid'; + const { args, reportError } = await configureFetch({ + url: 'https://example.com/profile.json', + content: encode(invalidJSON), + }); + + let userFacingError; + try { + await fetchProfile(args); + } catch (error) { + userFacingError = error; + } + expect(userFacingError).toMatchSnapshot(); + expect(reportError.mock.calls.length).toBeGreaterThan(0); + expect(reportError.mock.calls).toMatchSnapshot(); + }); + + it('fallback behavior if a completely unknown file is passed in', async function () { + const invalidJSON = 'invalid'; + const profile = encode(invalidJSON); + const { args } = await configureFetch({ + url: 'https://example.com/profile.unknown', + content: profile, + }); + + let userFacingError = null; + try { + const profileOrZip = await fetchProfile(args); + expect(profileOrZip).toEqual({ + responseType: 'PROFILE', + profile: extractArrayBuffer(profile), + }); + } catch (error) { + userFacingError = error; + } + expect(userFacingError).toBeNull(); + }); +}); diff --git a/src/utils/profile-fetch.ts b/src/utils/profile-fetch.ts new file mode 100644 index 0000000000..3a7f1638ba --- /dev/null +++ b/src/utils/profile-fetch.ts @@ -0,0 +1,346 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { oneLine } from 'common-tags'; +import { assertExhaustiveCheck } from './types'; +import { TemporaryError } from './errors'; +import { decompress, isGzip } from './gz'; +import { isLocalURL } from './url'; +import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants'; +import type JSZip from 'jszip'; + +/** + * Shared utilities for fetching profiles from URLs. + * Used by both the web app (receive-profile.ts) and the CLI (profile-query). + * + * This module was extracted from receive-profile.ts to make the fetching + * logic reusable across different contexts (Redux vs CLI). + */ + +/** + * Convert a profile hash to its Google Cloud Storage URL. + * Public profiles are stored in Google Cloud Storage in the profile-store bucket. + * See https://cloud.google.com/storage/docs/access-public-data + */ +export function getProfileUrlForHash(hash: string): string { + return `https://storage.googleapis.com/${GOOGLE_STORAGE_BUCKET}/${hash}`; +} + +/** + * Extract the actual profile URL from a profiler.firefox.com URL. + * + * Parses URLs like: + * - https://profiler.firefox.com/from-url/http%3A%2F%2F127.0.0.1%3A3000%2Fprofile.json/ + * - https://profiler.firefox.com/public/g9w0fmjjx4bqrky4zg0wb90n65b8g3w0qjjx1t0/calltree/ + * + * Returns the decoded profile URL, or null if this is not a supported datasource. + * This mimics the logic in retrieveProfileFromStore and retrieveProfileForRawUrl + * from receive-profile.ts + */ +export function extractProfileUrlFromProfilerUrl( + profilerUrl: string +): string | null { + try { + // Handle both full URLs and just pathnames + let pathname: string; + if ( + profilerUrl.startsWith('http://') || + profilerUrl.startsWith('https://') + ) { + const url = new URL(profilerUrl); + pathname = url.pathname; + } else { + pathname = profilerUrl; + } + + const pathParts = pathname.split('/').filter((d) => d); + + // Check if this is a from-url datasource + // URL structure: /from-url/{encoded-profile-url}/... + if (pathParts[0] === 'from-url' && pathParts[1]) { + return decodeURIComponent(pathParts[1]); + } + + // Check if this is a public datasource + // URL structure: /public/{hash}/... + // Profile is stored in Google Cloud Storage + if (pathParts[0] === 'public' && pathParts[1]) { + const hash = pathParts[1]; + return getProfileUrlForHash(hash); + } + + return null; + } catch (error) { + console.error('Failed to parse profiler URL:', error); + return null; + } +} + +function _wait(delayMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +/** + * Check if a load failure is likely due to Safari's localhost HTTP restriction. + * Safari blocks mixed content (HTTP on HTTPS page) even for localhost. + * This check works in both browser and Node.js (returns false in Node). + */ +function _loadProbablyFailedDueToSafariLocalhostHTTPRestriction( + url: string, + error: Error +): boolean { + // In Node.js, navigator won't exist + if ( + typeof navigator === 'undefined' || + !navigator.userAgent.match(/Safari\/\d+\.\d+/) + ) { + return false; + } + // Check if Safari considers this mixed content. + try { + const parsedUrl = new URL(url); + return ( + error.name === 'TypeError' && + parsedUrl.protocol === 'http:' && + isLocalURL(parsedUrl) && + typeof location !== 'undefined' && + location.protocol === 'https:' + ); + } catch { + return false; + } +} + +export class SafariLocalhostHTTPLoadError extends Error { + override name = 'SafariLocalhostHTTPLoadError'; +} + +/** + * Deduce the file type from a URL and content type. + * This is used to detect zip files vs profile files. + * Exported for use in receive-profile.ts for file handling. + */ +export function deduceContentType( + url: string, + contentType: string | null +): 'application/json' | 'application/zip' | null { + if (contentType === 'application/zip' || contentType === 'application/json') { + return contentType; + } + if (url.match(/\.zip$/)) { + return 'application/zip'; + } + if (url.match(/\.json/)) { + return 'application/json'; + } + return null; +} + +/** + * Parse JSON from an optionally gzipped array buffer. + * Exported for use in receive-profile.ts for direct file processing. + */ +export async function extractJsonFromArrayBuffer( + arrayBuffer: ArrayBuffer +): Promise { + let profileBytes = new Uint8Array(arrayBuffer); + // Check for the gzip magic number in the header. + if (isGzip(profileBytes)) { + profileBytes = await decompress(profileBytes); + } + + const textDecoder = new TextDecoder(); + return JSON.parse(textDecoder.decode(profileBytes)); +} + +/** + * Don't trust third party responses, try and handle a variety of responses gracefully. + */ +async function _extractJsonFromResponse( + response: Response, + reportError: (...data: Array) => void, + fileType: 'application/json' | null +): Promise { + let arrayBuffer: ArrayBuffer | null = null; + try { + // await before returning so that we can catch JSON parse errors. + arrayBuffer = await response.arrayBuffer(); + return await extractJsonFromArrayBuffer(arrayBuffer); + } catch (error) { + // Change the error message depending on the circumstance: + let message; + if (error && typeof error === 'object' && error.name === 'AbortError') { + message = 'The network request to load the profile was aborted.'; + } else if (fileType === 'application/json') { + message = 'The profile’s JSON could not be decoded.'; + } else if (fileType === null && arrayBuffer !== null) { + // If the content type is not specified, use a raw array buffer + // to fallback to other supported profile formats. + return arrayBuffer; + } else { + message = oneLine` + The profile could not be downloaded and decoded. This does not look like a supported file + type. + `; + } + + // Provide helpful debugging information to the console. + reportError(message); + reportError('JSON parsing error:', error); + reportError('Fetch response:', response); + + throw new Error( + `${message} The full error information has been printed out to the DevTool’s console.` + ); + } +} + +/** + * Attempt to load a zip file from a third party. This process can fail, so make sure + * to handle and report the error if it does. + */ +async function _extractZipFromResponse( + response: Response, + reportError: (...data: Array) => void +): Promise { + const buffer = await response.arrayBuffer(); + // Workaround for https://github.com/Stuk/jszip/issues/941 + // When running this code in tests, `buffer` doesn't inherits from _this_ + // realm's ArrayBuffer object, and this breaks JSZip which doesn't account for + // this case. We workaround the issue by wrapping the buffer in an Uint8Array + // that comes from this realm. + const typedBuffer = new Uint8Array(buffer); + try { + const { default: JSZip } = await import('jszip'); + const zip = await JSZip.loadAsync(typedBuffer); + // Catch the error if unable to load the zip. + return zip; + } catch (error) { + const message = 'Unable to open the archive file.'; + reportError(message); + reportError('Error:', error); + reportError('Fetch response:', response); + throw new Error( + `${message} The full error information has been printed out to the DevTool’s console.` + ); + } +} + +export type ProfileOrZip = + | { responseType: 'PROFILE'; profile: unknown } + | { responseType: 'ZIP'; zip: JSZip }; + +/** + * This function guesses the correct content-type (even if one isn't sent) and then + * attempts to use the proper method to extract the response. + */ +async function _extractProfileOrZipFromResponse( + url: string, + response: Response, + reportError: (...data: Array) => void +): Promise { + const contentType = deduceContentType( + url, + response.headers.get('content-type') + ); + switch (contentType) { + case 'application/zip': + return { + responseType: 'ZIP', + zip: await _extractZipFromResponse(response, reportError), + }; + case 'application/json': + case null: + // The content type is null if it is unknown, or an unsupported type. Go ahead + // and try to process it as a profile. + return { + responseType: 'PROFILE', + profile: await _extractJsonFromResponse( + response, + reportError, + contentType + ), + }; + default: + throw assertExhaustiveCheck(contentType); + } +} + +export type FetchProfileArgs = { + url: string; + onTemporaryError: (param: TemporaryError) => void; + // Allow tests to capture the reported error, but normally use console.error. + reportError?: (...data: Array) => void; +}; + +/** + * Tries to fetch a profile on `url`. If the profile is not found, + * `onTemporaryError` is called with an appropriate error, we wait 1 second, and + * then tries again. If we still can't find the profile after 11 tries, the + * returned promise is rejected with a fatal error. + * If we can retrieve the profile properly, the returned promise is resolved + * with the parsed profile or zip file. + * + * This function was moved from receive-profile.ts to make it reusable by + * both the web app and CLI. + */ +export async function fetchProfile( + args: FetchProfileArgs +): Promise { + const MAX_WAIT_SECONDS = 10; + let i = 0; + const { url, onTemporaryError } = args; + // Allow tests to capture the reported error, but normally use console.error. + const reportError = args.reportError || console.error; + + while (true) { + let response; + try { + response = await fetch(url); + } catch (e) { + // Case 1: Exception. + if ( + _loadProbablyFailedDueToSafariLocalhostHTTPRestriction(url, e as Error) + ) { + throw new SafariLocalhostHTTPLoadError(); + } + throw e; + } + + // Case 2: successful answer. + if (response.ok) { + return _extractProfileOrZipFromResponse(url, response, reportError); + } + + // case 3: unrecoverable error. + if (response.status !== 403) { + throw new Error(oneLine` + Could not fetch the profile on remote server. + Response was: ${response.status} ${response.statusText}. + `); + } + + // case 4: 403 errors can be transient while a profile is uploaded. + + if (i++ === MAX_WAIT_SECONDS) { + // In the last iteration we don't send a temporary error because we'll + // throw an error right after the while loop. + break; + } + + onTemporaryError( + new TemporaryError( + 'Profile not found on remote server.', + { count: i, total: MAX_WAIT_SECONDS + 1 } // 11 tries during 10 seconds + ) + ); + + await _wait(1000); + } + + throw new Error(oneLine` + Could not fetch the profile on remote server: + still not found after ${MAX_WAIT_SECONDS} seconds. + `); +}