diff --git a/src/client/utils/ajax/ajax.mocha.ts b/src/client/utils/ajax/ajax.mocha.ts new file mode 100644 index 000000000..d13c0ed41 --- /dev/null +++ b/src/client/utils/ajax/ajax.mocha.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import axios from "axios"; +import { expect } from "chai"; +import * as sinon from "sinon"; +import { Ajax } from "./ajax"; + +describe("Ajax", () => { + + describe("fetchDataCubes", () => { + let get: sinon.SinonStub; + + describe("happy path", () => { + before(() => { + get = sinon.stub(axios, "get"); + get.withArgs("sources/dataCubes?page=0") + .resolves({ data: { dataCubes: ["first-cube", "second-cube"], next: 1 } }); + get.withArgs("sources/dataCubes?page=1") + .resolves({ data: { dataCubes: ["third-cube", "fourth-cube"], next: 2 } }); + get.withArgs("sources/dataCubes?page=2") + .resolves({ data: { dataCubes: ["fifth-cube", "sixth-cube"], next: 3 } }); + get.withArgs("sources/dataCubes?page=3") + .resolves({ data: { dataCubes: ["last-cube"] } }); + }); + + it("should fetch all data cubes across pages", async () => { + const dataCubes = await Ajax.fetchDataCubes({}); + expect(dataCubes).to.be.deep.equal([ + "first-cube", + "second-cube", + "third-cube", + "fourth-cube", + "fifth-cube", + "sixth-cube", + "last-cube" + ]); + }); + + after(() => { + get.restore(); + }); + }); + + describe("error handling", () => { + before(() => { + get = sinon.stub(axios, "get"); + get.withArgs("sources/dataCubes?page=0") + .resolves({ data: { dataCubes: ["first-cube", "second-cube"], next: 1 } }); + get.withArgs("sources/dataCubes?page=1") + .rejects(new Error("Couldn't fetch")); + }); + + /* + NOTE: + Couldn't write this test with any fancy helpers. + If you try to refactor it, please check if your solution isn't a false positive! + */ + it("should rethrow network error", async () => { + let error; + try { + await Ajax.fetchDataCubes({}); + } catch (e) { + error = e; + } + expect(error).to.have.property("message", "Couldn't fetch"); + }); + + after(() => { + get.restore(); + }); + }); + }); +}); diff --git a/src/client/utils/ajax/ajax.ts b/src/client/utils/ajax/ajax.ts index 96728c111..0dceb65c7 100644 --- a/src/client/utils/ajax/ajax.ts +++ b/src/client/utils/ajax/ajax.ts @@ -18,8 +18,10 @@ import axios from "axios"; import { Dataset, DatasetJS, Environment, Executor, Expression } from "plywood"; import { ClientAppSettings } from "../../../common/models/app-settings/app-settings"; +import { SerializedCluster } from "../../../common/models/cluster/cluster"; +import { SerializedDataCube } from "../../../common/models/data-cube/data-cube"; import { isEnabled, Oauth } from "../../../common/models/oauth/oauth"; -import { ClientSources, SerializedSources } from "../../../common/models/sources/sources"; +import { ClientSources } from "../../../common/models/sources/sources"; import { deserialize } from "../../deserializers/sources"; import { getToken, mapOauthError } from "../../oauth/oauth"; @@ -73,13 +75,44 @@ export class Ajax { }; } - static sources(appSettings: ClientAppSettings): Promise { - const headers = Ajax.headers(appSettings.oauth); - return axios.get("sources", { headers }) - .then(resp => resp.data) - .catch(error => { - throw mapOauthError(appSettings.oauth, error); - }) - .then(sourcesJS => deserialize(sourcesJS, appSettings)); + static async fetchClusters(headers: Record): Promise { + return axios.get("sources/clusters", { headers }).then(resp => resp.data); + } + + static async fetchDataCubes(headers: Record): Promise { + async function* fetchDataCubesPage(page: number): AsyncIterableIterator { + const { dataCubes, next } = (await axios.get(`sources/dataCubes?page=${page}`, { headers })).data; + yield dataCubes; + if (next) { + yield* fetchDataCubesPage(next); + } + + } + + async function* fetchAllPages() { + yield* fetchDataCubesPage(0); + } + + const dataCubes: SerializedDataCube[] = []; + for await (const cubes of fetchAllPages()) { + dataCubes.push(...cubes); + } + + return dataCubes; + } + + static async sources(appSettings: ClientAppSettings): Promise { + try { + const headers = Ajax.headers(appSettings.oauth); + const clusters = Ajax.fetchClusters(headers); + const dataCubes = Ajax.fetchDataCubes(headers); + + return deserialize({ + clusters: await clusters, + dataCubes: await dataCubes + }, appSettings); + } catch (e) { + throw mapOauthError(appSettings.oauth, e); + } } } diff --git a/src/common/models/sources/sources.ts b/src/common/models/sources/sources.ts index cf46006b8..036356af4 100644 --- a/src/common/models/sources/sources.ts +++ b/src/common/models/sources/sources.ts @@ -22,7 +22,6 @@ import { Cluster, ClusterJS, fromConfig as clusterFromConfig, - serialize as serializeCluster, SerializedCluster } from "../cluster/cluster"; import { findCluster } from "../cluster/find-cluster"; @@ -96,22 +95,6 @@ export function fromConfig(config: SourcesJS, logger: Logger): Sources { }; } -export function serialize({ - clusters: serverClusters, - dataCubes: serverDataCubes - }: Sources): SerializedSources { - const clusters = serverClusters.map(serializeCluster); - - const dataCubes = serverDataCubes - .filter(dc => isQueryable(dc)) - .map(serializeDataCube); - - return { - clusters, - dataCubes - }; -} - export function getDataCubesForCluster(sources: Sources, clusterName: string): DataCube[] { return sources.dataCubes.filter(dataCube => dataCube.clusterName === clusterName); } @@ -134,3 +117,7 @@ export function deleteDataCube(sources: Sources, dataCube: DataCube): Sources { dataCubes: sources.dataCubes.filter(dc => dc.name !== dataCube.name) }; } + +export function serializeDataCubes(dataCubes: DataCube[]): SerializedDataCube[] { + return dataCubes.filter(isQueryable).map(serializeDataCube); +} diff --git a/src/server/routes/sources/sources.ts b/src/server/routes/sources/sources.ts index fd62ef964..e7aea64cf 100644 --- a/src/server/routes/sources/sources.ts +++ b/src/server/routes/sources/sources.ts @@ -16,8 +16,10 @@ import { Request, Response, Router } from "express"; import { errorToMessage } from "../../../common/logger/logger"; -import { serialize } from "../../../common/models/sources/sources"; +import { serialize as serializeCluster } from "../../../common/models/cluster/cluster"; +import { serializeDataCubes } from "../../../common/models/sources/sources"; import { checkAccess } from "../../utils/datacube-guard/datacube-guard"; +import { getDataCubesPage, getPageNumber } from "../../utils/datacubes/pagination"; import { SettingsManager } from "../../utils/settings-manager/settings-manager"; export function sourcesRouter(settings: Pick) { @@ -25,20 +27,31 @@ export function sourcesRouter(settings: Pick { - + router.get("/clusters", async (req: Request, res: Response) => { try { - const { clusters, dataCubes } = await settings.getSources(); - res.json(serialize({ - clusters, - dataCubes: dataCubes.filter( dataCube => checkAccess(dataCube, req.headers) ) - })); + const { clusters } = await settings.getSources(); + res.json(clusters.map(serializeCluster)); } catch (error) { - logger.error(errorToMessage(error)); + logger.error(errorToMessage(error)); + res.status(500).send({ + error: "Can't fetch clusters", + message: error.message + }); + } + }); - res.status(500).send({ - error: "Can't fetch settings", + router.get("/dataCubes", async (req: Request, res: Response) => { + try { + const sources = await settings.getSources(); + const dataCubes = sources.dataCubes + .filter(dataCube => checkAccess(dataCube, req.headers)); + const serializedDataCubes = serializeDataCubes(dataCubes); + const page = getPageNumber(req.query.page); + res.json(getDataCubesPage(serializedDataCubes, page)); + } catch (error) { + logger.error(errorToMessage(error)); + res.status(500).send({ + error: "Can't fetch data cubes", message: error.message }); } diff --git a/src/server/utils/datacubes/pagination.mocha.ts b/src/server/utils/datacubes/pagination.mocha.ts new file mode 100644 index 000000000..1148d12c4 --- /dev/null +++ b/src/server/utils/datacubes/pagination.mocha.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from "chai"; +import { serialize, SerializedDataCube } from "../../../common/models/data-cube/data-cube"; +import { twitterDataCube } from "../../../common/models/data-cube/data-cube.fixtures"; +import { getDataCubesPage, getPageNumber } from "./pagination"; + +const DATA_CUBE: SerializedDataCube = serialize(twitterDataCube); + +const nCubes = (n: number) => Array.from({ length: n }).map(() => DATA_CUBE); + +describe("DataCube Pagination", () => { + describe("getPageNumber", () => { + it("should parse string", () => { + expect(getPageNumber("10")).to.be.equal(10); + }); + + it("should return default 0 if empty", () => { + expect(getPageNumber(undefined)).to.be.equal(0); + }); + + describe("URLSearchQuery types", () => { + it("should return default 0 if passed an array", () => { + expect(getPageNumber(["foobar", "bazz"])).to.be.equal(0); + }); + + it("should return default 0 if passed an object", () => { + expect(getPageNumber({ foobar: 42 })).to.be.equal(0); + }); + }); + }); + + describe("getDataCubesPage", () => { + describe("DataCubes fit into first page", () => { + const cubes = nCubes(50); + + it("should return all data cubes", () => { + expect(getDataCubesPage(cubes, 0).dataCubes).to.have.length(50); + }); + + it("should return empty next page", () => { + expect(getDataCubesPage(cubes, 0).next).to.be.undefined; + }); + }); + + describe("DataCubes does not fit into first page", () => { + const cubes = nCubes(1500); + + describe("first page", () => { + it("should return first thousand data cubes", () => { + expect(getDataCubesPage(cubes, 0).dataCubes).to.have.length(1000); + }); + + it("should return index of next page", () => { + expect(getDataCubesPage(cubes, 0).next).to.be.equal(1); + }); + }); + + describe("second page", () => { + it("should return first thousand data cubes", () => { + expect(getDataCubesPage(cubes, 1).dataCubes).to.have.length(500); + }); + + it("should return empty next page", () => { + expect(getDataCubesPage(cubes, 1).next).to.be.undefined; + }); + }); + }); + }); +}); diff --git a/src/server/utils/datacubes/pagination.ts b/src/server/utils/datacubes/pagination.ts new file mode 100644 index 000000000..e6ca9d73c --- /dev/null +++ b/src/server/utils/datacubes/pagination.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2017-2022 Allegro.pl + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SerializedDataCube } from "../../../common/models/data-cube/data-cube"; + +const PAGE_SIZE = 1000; + +interface DataCubesSlice { + dataCubes: SerializedDataCube[]; + next?: number; +} + +export function getDataCubesPage(dataCubes: SerializedDataCube[], page: number): DataCubesSlice { + const sliceStart = page * PAGE_SIZE; + const sliceEnd = sliceStart + PAGE_SIZE; + const next = sliceEnd < dataCubes.length ? page + 1 : undefined; + return { + dataCubes: dataCubes.slice(sliceStart, sliceEnd), + next + }; +} + +export function getPageNumber(page: unknown): number { + if (typeof page === "string") return parseInt(page, 10); + return 0; +}