Cursor-based pagination APIs

foundations/core/packages/api-client/src/client.ts

@@ -22,10 +22,13 @@ import {
   type Doc,
   type DocumentQuery,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FindResult,
   type Hierarchy,
   type ModelDb,
   type Ref,
+  type IterateOptions,
   type Space,
   type TxResult,
   type WithLookup,
@@ -156,6 +159,22 @@ class PlatformClientImpl implements PlatformClient {
     return await this.client.findAll(_class, query, options)
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    return await this.client.findAllPage(_class, query, options)
+  }
+
+  iterateAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ): AsyncIterable<WithLookup<T>> {
+    return this.client.iterateAll(_class, query, options)
+  }
+
   async close (): Promise<void> {
     await this.connection.close()
   }

foundations/core/packages/api-client/src/rest/adapter.ts

@@ -22,10 +22,13 @@ import {
   type DomainRequestOptions,
   type DomainResult,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FindResult,
   Hierarchy,
   ModelDb,
   OperationDomain,
+  type IterateOptions,
   type Ref,
   type SearchOptions,
   type SearchQuery,
@@ -60,6 +63,22 @@ export class RestClientAdapter implements Client {
     return await this.client.findAll(_class, query, options)
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    return await this.client.findAllPage(_class, query, options)
+  }
+
+  iterateAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ): AsyncIterable<WithLookup<T>> {
+    return this.client.iterateAll(_class, query, options)
+  }
+
   async tx (tx: Tx): Promise<TxResult> {
     return await this.client.tx(tx)
   }

foundations/core/packages/api-client/src/rest/rest.ts

@@ -28,7 +28,10 @@ import {
   type DomainRequestOptions,
   type DomainResult,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FindResult,
+  type IterateOptions,
   Hierarchy,
   MeasureMetricsContext,
   type Mixin,
@@ -165,6 +168,81 @@ export class RestClientImpl implements RestClient {
     return result
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    const requestUrl = concatLink(this.endpoint, `/api/v1/find-page/${this.workspace}`)
+    const result = await withRetry<FindPageResult<T> & { error?: Status }>(async () => {
+      const response = await fetch(requestUrl, {
+        method: 'POST',
+        keepalive: true,
+        headers: this.jsonHeaders(),
+        body: JSON.stringify({ _class, query, options })
+      })
+      if (!response.ok) {
+        await this.checkRateLimits(response)
+        throw new PlatformError(unknownError(response.statusText))
+      }
+      this.updateRateLimit(response)
+      return await extractJson<FindPageResult<T>>(response)
+    }, isRLE)
+
+    if (result.error !== undefined) {
+      throw new PlatformError(result.error)
+    }
+    if (result.lookupMap !== undefined) {
+      for (const doc of result.docs) {
+        if (doc.$lookup !== undefined) {
+          const lookup = doc.$lookup as Record<string, unknown>
+          for (const [key, value] of Object.entries(lookup)) {
+            if (Array.isArray(value)) {
+              lookup[key] = value.map((item) => result.lookupMap?.[item])
+            } else {
+              lookup[key] = result.lookupMap[value as string]
+            }
+          }
+        }
+      }
+      delete result.lookupMap
+    }
+    for (const doc of result.docs) {
+      const docRecord = doc as Record<string, unknown>
+      for (const [key, value] of Object.entries(query)) {
+        if (
+          (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') &&
+          docRecord[key] == null
+        ) {
+          docRecord[key] = value
+        }
+      }
+      if (doc._class == null) {
+        doc._class = _class
+      }
+    }
+    return result
+  }
+
+  async * iterateAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ): AsyncIterable<WithLookup<T>> {
+    let cursor: string | undefined
+    do {
+      const page = await this.findAllPage(_class, query, {
+        ...options,
+        limit: options?.limit ?? 500,
+        cursor
+      })
+      for (const doc of page.docs) {
+        yield doc
+      }
+      cursor = page.nextCursor
+    } while (cursor !== undefined)
+  }
+
   private async checkRate (): Promise<void> {
     if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) {
       if (this.slowDownTimer < 50) {

foundations/core/packages/api-client/src/rest/tx.ts

@@ -23,9 +23,12 @@ import {
   type DomainRequestOptions,
   type DomainResult,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FindResult,
   Hierarchy,
   ModelDb,
+  type IterateOptions,
   type OperationDomain,
   type Ref,
   type SearchOptions,
@@ -77,6 +80,37 @@ class RestTxClient implements Client {
     return toFindResult(result, data.total)
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    const page = await this.client.findAllPage(_class, query, options)
+    return {
+      ...page,
+      docs: page.docs.map((doc) => this.hierarchy.updateLookupMixin(_class, doc, options))
+    }
+  }
+
+  async * iterateAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ): AsyncIterable<WithLookup<T>> {
+    let cursor: string | undefined
+    do {
+      const page = await this.findAllPage(_class, query, {
+        ...options,
+        limit: options?.limit ?? 500,
+        cursor
+      })
+      for (const doc of page.docs) {
+        yield doc
+      }
+      cursor = page.nextCursor
+    } while (cursor !== undefined)
+  }
+
   async domainRequest<T>(
     domain: OperationDomain,
     params: DomainParams,

foundations/core/packages/api-client/src/rest/types.ts

@@ -26,8 +26,11 @@ import {
   type DomainRequestOptions,
   type DomainResult,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FulltextStorage,
   type Hierarchy,
+  type IterateOptions,
   type Mixin,
   type MixinData,
   type MixinUpdate,
@@ -53,6 +56,18 @@ export interface RestClient extends Storage, FulltextStorage {
     options?: FindOptions<T>
   ) => Promise<WithLookup<T> | undefined>
 
+  findAllPage: <T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ) => Promise<FindPageResult<T>>
+
+  iterateAll: <T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ) => AsyncIterable<WithLookup<T>>
+
   getModel: () => Promise<{ hierarchy: Hierarchy, model: ModelDb }>
 
   domainRequest: <T>(

foundations/core/packages/api-client/src/types.ts

@@ -25,12 +25,15 @@ import {
   type DocumentQuery,
   type DocumentUpdate,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
   type FindResult,
   type Hierarchy,
   type Mixin,
   type MixinData,
   type MixinUpdate,
   type ModelDb,
+  type IterateOptions,
   type Ref,
   type Space,
   type TxResult,
@@ -78,6 +81,18 @@ export interface FindOperations {
     options?: FindOptions<T> | undefined
   ) => Promise<FindResult<T>>
 
+  findAllPage: <T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ) => Promise<FindPageResult<T>>
+
+  iterateAll: <T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ) => AsyncIterable<WithLookup<T>>
+
   findOne: <T extends Doc>(
     _class: Ref<Class<T>>,
     query: DocumentQuery<T>,

foundations/core/packages/client-resources/src/connection.ts

@@ -36,6 +36,8 @@ import core, {
   type DomainRequestOptions,
   type DomainResult,
   FindOptions,
+  FindPageOptions,
+  FindPageResult,
   FindResult,
   generateId,
   platformNow,
@@ -956,6 +958,49 @@ class Connection implements ClientConnection {
     return result
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    const result = (await this.sendRequest({
+      method: 'findAllPage',
+      params: [_class, query, options]
+    })) as FindPageResult<T>
+
+    if (result.lookupMap !== undefined) {
+      for (const doc of result.docs) {
+        if (doc.$lookup !== undefined) {
+          const lookup = doc.$lookup as Record<string, unknown>
+          for (const [key, value] of Object.entries(lookup)) {
+            if (Array.isArray(value)) {
+              lookup[key] = value.map((item) => result.lookupMap?.[item])
+            } else {
+              lookup[key] = result.lookupMap[value as string]
+            }
+          }
+        }
+      }
+      delete result.lookupMap
+    }
+
+    for (const doc of result.docs) {
+      const docRecord = doc as Record<string, unknown>
+      for (const [key, value] of Object.entries(query)) {
+        if (
+          (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') &&
+          docRecord[key] == null
+        ) {
+          docRecord[key] = value
+        }
+      }
+      if (doc._class == null) {
+        doc._class = _class
+      }
+    }
+    return result
+  }
+
   tx (tx: Tx): Promise<TxResult> {
     return this.sendRequest({
       method: 'tx',

foundations/core/packages/core/src/__tests__/memdb.test.ts

@@ -22,6 +22,9 @@ import { TxOperations } from '../operations'
 import {
   type DocumentQuery,
   type FindOptions,
+  type FindPageOptions,
+  type FindPageResult,
+  type IterateOptions,
   type SearchOptions,
   type SearchQuery,
   type SearchResult,
@@ -52,6 +55,41 @@ class ClientModel extends ModelDb implements Client {
     return (await this.findAll(_class, query, options)).shift()
   }
 
+  async findAllPage<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options: FindPageOptions<T>
+  ): Promise<FindPageResult<T>> {
+    const { cursor, limit, ...findOptions } = options
+    const docs = await this.findAll(_class, query, findOptions)
+    const offset = cursor === undefined ? 0 : Number.parseInt(cursor, 10)
+    const start = Number.isNaN(offset) ? 0 : offset
+    const pageDocs = docs.slice(start, start + limit)
+    const nextOffset = start + pageDocs.length
+    return {
+      docs: pageDocs,
+      nextCursor: nextOffset < docs.length ? String(nextOffset) : undefined,
+      total: options.total === true ? docs.length : undefined
+    }
+  }
+
+  async * iterateAll<T extends Doc>(
+    _class: Ref<Class<T>>,
+    query: DocumentQuery<T>,
+    options?: IterateOptions<T>
+  ): AsyncIterable<WithLookup<T>> {
+    let cursor: string | undefined
+    do {
+      const page = await this.findAllPage(_class, query, {
+        ...options,
+        limit: options?.limit ?? 500,
+        cursor
+      })
+      yield * page.docs
+      cursor = page.nextCursor
+    } while (cursor !== undefined)
+  }
+
   async searchFulltext (query: SearchQuery, options: SearchOptions): Promise<SearchResult> {
     return { docs: [] }
   }