support large ammount of datasources

src/client/utils/ajax/ajax.mocha.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2017-2022 Allegro.pl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import axios from "axios";
+import { expect } from "chai";
+import * as sinon from "sinon";
+import { Ajax } from "./ajax";
+
+describe("Ajax", () => {
+
+  describe("fetchDataCubes", () => {
+    let get: sinon.SinonStub;
+
+    describe("happy path", () => {
+      before(() => {
+        get = sinon.stub(axios, "get");
+        get.withArgs("sources/dataCubes?page=0")
+          .resolves({ data: { dataCubes: ["first-cube", "second-cube"], next: 1 } });
+        get.withArgs("sources/dataCubes?page=1")
+          .resolves({ data: { dataCubes: ["third-cube", "fourth-cube"], next: 2 } });
+        get.withArgs("sources/dataCubes?page=2")
+          .resolves({ data: { dataCubes: ["fifth-cube", "sixth-cube"], next: 3 } });
+        get.withArgs("sources/dataCubes?page=3")
+          .resolves({ data: { dataCubes: ["last-cube"] } });
+      });
+
+      it("should fetch all data cubes across pages", async () => {
+        const dataCubes = await Ajax.fetchDataCubes({});
+        expect(dataCubes).to.be.deep.equal([
+          "first-cube",
+          "second-cube",
+          "third-cube",
+          "fourth-cube",
+          "fifth-cube",
+          "sixth-cube",
+          "last-cube"
+        ]);
+      });
+
+      after(() => {
+        get.restore();
+      });
+    });
+
+    describe("error handling", () => {
+      before(() => {
+        get = sinon.stub(axios, "get");
+        get.withArgs("sources/dataCubes?page=0")
+          .resolves({ data: { dataCubes: ["first-cube", "second-cube"], next: 1 } });
+        get.withArgs("sources/dataCubes?page=1")
+          .rejects(new Error("Couldn't fetch"));
+      });
+
+      /*
+       NOTE:
+       Couldn't write this test with any fancy helpers.
+       If you try to refactor it, please check if your solution isn't a false positive!
+      */
+      it("should rethrow network error", async () => {
+        let error;
+        try {
+          await Ajax.fetchDataCubes({});
+        } catch (e) {
+          error = e;
+        }
+        expect(error).to.have.property("message", "Couldn't fetch");
+      });
+
+      after(() => {
+        get.restore();
+      });
+    });
+  });
+});

src/client/utils/ajax/ajax.ts

@@ -18,8 +18,10 @@
 import axios from "axios";
 import { Dataset, DatasetJS, Environment, Executor, Expression } from "plywood";
 import { ClientAppSettings } from "../../../common/models/app-settings/app-settings";
+import { SerializedCluster } from "../../../common/models/cluster/cluster";
+import { SerializedDataCube } from "../../../common/models/data-cube/data-cube";
 import { isEnabled, Oauth } from "../../../common/models/oauth/oauth";
-import { ClientSources, SerializedSources } from "../../../common/models/sources/sources";
+import { ClientSources } from "../../../common/models/sources/sources";
 import { deserialize } from "../../deserializers/sources";
 import { getToken, mapOauthError } from "../../oauth/oauth";
 
@@ -73,13 +75,44 @@ export class Ajax {
     };
   }
 
-  static sources(appSettings: ClientAppSettings): Promise<ClientSources> {
-    const headers = Ajax.headers(appSettings.oauth);
-    return axios.get<SerializedSources>("sources", { headers })
-      .then(resp => resp.data)
-      .catch(error => {
-        throw mapOauthError(appSettings.oauth, error);
-      })
-      .then(sourcesJS => deserialize(sourcesJS, appSettings));
+  static async fetchClusters(headers: Record<string, string>): Promise<SerializedCluster[]> {
+    return axios.get("sources/clusters", { headers }).then(resp => resp.data);
+  }
+
+  static async fetchDataCubes(headers: Record<string, string>): Promise<SerializedDataCube[]> {
+    async function* fetchDataCubesPage(page: number): AsyncIterableIterator<SerializedDataCube[]> {
+      const { dataCubes, next } = (await axios.get(`sources/dataCubes?page=${page}`, { headers })).data;
+      yield dataCubes;
+      if (next) {
+        yield* fetchDataCubesPage(next);
+      }
+
+    }
+
+    async function* fetchAllPages() {
+      yield* fetchDataCubesPage(0);
+    }
+
+    const dataCubes: SerializedDataCube[] = [];
+    for await (const cubes of fetchAllPages()) {
+      dataCubes.push(...cubes);
+    }
+
+    return dataCubes;
+  }
+
+  static async sources(appSettings: ClientAppSettings): Promise<ClientSources> {
+    try {
+      const headers = Ajax.headers(appSettings.oauth);
+      const clusters = Ajax.fetchClusters(headers);
+      const dataCubes = Ajax.fetchDataCubes(headers);
+
+      return deserialize({
+        clusters: await clusters,
+        dataCubes: await dataCubes
+      }, appSettings);
+    } catch (e) {
+      throw mapOauthError(appSettings.oauth, e);
+    }
   }
 }

src/common/models/sources/sources.ts

@@ -22,7 +22,6 @@ import {
   Cluster,
   ClusterJS,
   fromConfig as clusterFromConfig,
-  serialize as serializeCluster,
   SerializedCluster
 } from "../cluster/cluster";
 import { findCluster } from "../cluster/find-cluster";
@@ -96,22 +95,6 @@ export function fromConfig(config: SourcesJS, logger: Logger): Sources {
   };
 }
 
-export function serialize({
-                            clusters: serverClusters,
-                            dataCubes: serverDataCubes
-                          }: Sources): SerializedSources {
-  const clusters = serverClusters.map(serializeCluster);
-
-  const dataCubes = serverDataCubes
-    .filter(dc => isQueryable(dc))
-    .map(serializeDataCube);
-
-  return {
-    clusters,
-    dataCubes
-  };
-}
-
 export function getDataCubesForCluster(sources: Sources, clusterName: string): DataCube[] {
   return sources.dataCubes.filter(dataCube => dataCube.clusterName === clusterName);
 }
@@ -134,3 +117,7 @@ export function deleteDataCube(sources: Sources, dataCube: DataCube): Sources {
     dataCubes: sources.dataCubes.filter(dc => dc.name !== dataCube.name)
   };
 }
+
+export function serializeDataCubes(dataCubes: DataCube[]): SerializedDataCube[] {
+  return dataCubes.filter(isQueryable).map(serializeDataCube);
+}

src/server/routes/sources/sources.ts

@@ -16,29 +16,42 @@
 
 import { Request, Response, Router } from "express";
 import { errorToMessage } from "../../../common/logger/logger";
-import { serialize } from "../../../common/models/sources/sources";
+import { serialize as serializeCluster } from "../../../common/models/cluster/cluster";
+import { serializeDataCubes } from "../../../common/models/sources/sources";
 import { checkAccess } from "../../utils/datacube-guard/datacube-guard";
+import { getDataCubesPage, getPageNumber } from "../../utils/datacubes/pagination";
 import { SettingsManager } from "../../utils/settings-manager/settings-manager";
 
 export function sourcesRouter(settings: Pick<SettingsManager, "getSources" | "logger">) {
 
   const logger = settings.logger.setLoggerId("Sources");
 
   const router = Router();
-
-  router.get("/", async (req: Request, res: Response) => {
-
+  router.get("/clusters", async (req: Request, res: Response) => {
     try {
-      const { clusters, dataCubes } = await settings.getSources();
-      res.json(serialize({
-        clusters,
-        dataCubes: dataCubes.filter( dataCube => checkAccess(dataCube, req.headers) )
-      }));
+      const { clusters } = await settings.getSources();
+      res.json(clusters.map(serializeCluster));
     } catch (error) {
-     logger.error(errorToMessage(error));
+      logger.error(errorToMessage(error));
+      res.status(500).send({
+        error: "Can't fetch clusters",
+        message: error.message
+      });
+    }
+  });
 
-     res.status(500).send({
-        error: "Can't fetch settings",
+  router.get("/dataCubes", async (req: Request, res: Response) => {
+    try {
+      const sources = await settings.getSources();
+      const dataCubes = sources.dataCubes
+        .filter(dataCube => checkAccess(dataCube, req.headers));
+      const serializedDataCubes = serializeDataCubes(dataCubes);
+      const page = getPageNumber(req.query.page);
+      res.json(getDataCubesPage(serializedDataCubes, page));
+    } catch (error) {
+      logger.error(errorToMessage(error));
+      res.status(500).send({
+        error: "Can't fetch data cubes",
         message: error.message
       });
     }

src/server/utils/datacubes/pagination.mocha.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017-2022 Allegro.pl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from "chai";
+import { serialize, SerializedDataCube } from "../../../common/models/data-cube/data-cube";
+import { twitterDataCube } from "../../../common/models/data-cube/data-cube.fixtures";
+import { getDataCubesPage, getPageNumber } from "./pagination";
+
+const DATA_CUBE: SerializedDataCube = serialize(twitterDataCube);
+
+const nCubes = (n: number) => Array.from({ length: n }).map(() => DATA_CUBE);
+
+describe("DataCube Pagination", () => {
+  describe("getPageNumber", () => {
+    it("should parse string", () => {
+      expect(getPageNumber("10")).to.be.equal(10);
+    });
+
+    it("should return default 0 if empty", () => {
+      expect(getPageNumber(undefined)).to.be.equal(0);
+    });
+
+    describe("URLSearchQuery types", () => {
+      it("should return default 0 if passed an array", () => {
+        expect(getPageNumber(["foobar", "bazz"])).to.be.equal(0);
+      });
+
+      it("should return default 0 if passed an object", () => {
+        expect(getPageNumber({ foobar: 42 })).to.be.equal(0);
+      });
+    });
+  });
+
+  describe("getDataCubesPage", () => {
+    describe("DataCubes fit into first page", () => {
+      const cubes = nCubes(50);
+
+      it("should return all data cubes", () => {
+        expect(getDataCubesPage(cubes, 0).dataCubes).to.have.length(50);
+      });
+
+      it("should return empty next page", () => {
+        expect(getDataCubesPage(cubes, 0).next).to.be.undefined;
+      });
+    });
+
+    describe("DataCubes does not fit into first page", () => {
+      const cubes = nCubes(1500);
+
+      describe("first page", () => {
+        it("should return first thousand data cubes", () => {
+          expect(getDataCubesPage(cubes, 0).dataCubes).to.have.length(1000);
+        });
+
+        it("should return index of next page", () => {
+          expect(getDataCubesPage(cubes, 0).next).to.be.equal(1);
+        });
+      });
+
+      describe("second page", () => {
+        it("should return first thousand data cubes", () => {
+          expect(getDataCubesPage(cubes, 1).dataCubes).to.have.length(500);
+        });
+
+        it("should return empty next page", () => {
+          expect(getDataCubesPage(cubes, 1).next).to.be.undefined;
+        });
+      });
+    });
+  });
+});

src/server/utils/datacubes/pagination.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2017-2022 Allegro.pl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SerializedDataCube } from "../../../common/models/data-cube/data-cube";
+
+const PAGE_SIZE = 1000;
+
+interface DataCubesSlice {
+  dataCubes: SerializedDataCube[];
+  next?: number;
+}
+
+export function getDataCubesPage(dataCubes: SerializedDataCube[], page: number): DataCubesSlice {
+  const sliceStart = page * PAGE_SIZE;
+  const sliceEnd = sliceStart + PAGE_SIZE;
+  const next = sliceEnd < dataCubes.length ? page + 1 : undefined;
+  return {
+    dataCubes: dataCubes.slice(sliceStart, sliceEnd),
+    next
+  };
+}
+
+export function getPageNumber(page: unknown): number {
+  if (typeof page === "string") return parseInt(page, 10);
+  return 0;
+}