From 598434d14075454e3ae49d24097a19b8d1cfcf63 Mon Sep 17 00:00:00 2001 From: Rimon Hanna Date: Tue, 27 May 2025 10:52:50 +0200 Subject: [PATCH 1/3] added logic to fetch a page's relations --- examples/minimal/pages/[pageId].tsx | 2 +- packages/notion-client/src/notion-api.ts | 69 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/examples/minimal/pages/[pageId].tsx b/examples/minimal/pages/[pageId].tsx index b15060ec..0f0bde8f 100644 --- a/examples/minimal/pages/[pageId].tsx +++ b/examples/minimal/pages/[pageId].tsx @@ -6,7 +6,7 @@ import notion from '../lib/notion' export const getStaticProps = async (context: any) => { const pageId = (context.params.pageId as string) || rootNotionPageId - const recordMap = await notion.getPage(pageId) + const recordMap = await notion.getPage(pageId, { fetchRelationPages: true }) return { props: { diff --git a/packages/notion-client/src/notion-api.ts b/packages/notion-client/src/notion-api.ts index b39c110c..b438fdfe 100644 --- a/packages/notion-client/src/notion-api.ts +++ b/packages/notion-client/src/notion-api.ts @@ -53,6 +53,7 @@ export class NotionAPI { chunkNumber = 0, throwOnCollectionErrors = false, collectionReducerLimit = 999, + fetchRelationPages = true, // New option kyOptions }: { concurrency?: number @@ -63,6 +64,7 @@ export class NotionAPI { chunkNumber?: number throwOnCollectionErrors?: boolean collectionReducerLimit?: number + fetchRelationPages?: boolean // New option kyOptions?: KyOptions } = {} ): Promise { @@ -220,6 +222,73 @@ export class NotionAPI { await this.addSignedUrls({ recordMap, contentBlockIds, kyOptions }) } + if (fetchRelationPages) { + const maxIterations = 10 // Limit iterations to prevent infinite loops + for (let i = 0; i < maxIterations; ++i) { + const allRelationPageIdsInThisIteration = new Set() + + for (const blockId of Object.keys(recordMap.block)) { + const blockValue = recordMap.block[blockId]?.value + if ( + blockValue?.parent_table === 'collection' && + blockValue?.parent_id + ) { + const collection = recordMap.collection[blockValue.parent_id]?.value + if (collection?.schema) { + for (const propertyId of Object.keys( + blockValue.properties || {} + )) { + const schema = collection.schema[propertyId] + if (schema?.type === 'relation') { + const decorations = blockValue.properties![propertyId] + if (decorations && Array.isArray(decorations)) { + for (const decoration of decorations) { + if ( + Array.isArray(decoration) && + decoration.length > 1 && + decoration[0] === '‣' + ) { + const pagePointer = decoration[1]?.[0] + if ( + Array.isArray(pagePointer) && + pagePointer.length > 1 && + pagePointer[0] === 'p' + ) { + allRelationPageIdsInThisIteration.add(pagePointer[1]) + } + } + } + } + } + } + } + } + } + + const pendingRelationPageIds = Array.from( + allRelationPageIdsInThisIteration + ).filter((id) => !recordMap.block[id]?.value) + + if (!pendingRelationPageIds.length) { + break // No new related pages to fetch + } + + try { + const newBlocks = await this.getBlocks( + pendingRelationPageIds, + kyOptions + ).then((res) => res.recordMap.block) + recordMap.block = { ...recordMap.block, ...newBlocks } + } catch (err: any) { + console.warn( + 'NotionAPI getBlocks error during fetchRelationPages:', + err.message + ) + //TODO: Decide if we should break or continue if some blocks fail + } + } + } + return recordMap } From 9c1145fe47278d9c83116e9b9b3159d3fefab050 Mon Sep 17 00:00:00 2001 From: Rimon Hanna Date: Tue, 27 May 2025 11:01:47 +0200 Subject: [PATCH 2/3] ogranised it in separate functions --- packages/notion-client/src/notion-api.ts | 137 +++++++++++++---------- 1 file changed, 79 insertions(+), 58 deletions(-) diff --git a/packages/notion-client/src/notion-api.ts b/packages/notion-client/src/notion-api.ts index b438fdfe..a7ed3b45 100644 --- a/packages/notion-client/src/notion-api.ts +++ b/packages/notion-client/src/notion-api.ts @@ -223,73 +223,94 @@ export class NotionAPI { } if (fetchRelationPages) { - const maxIterations = 10 // Limit iterations to prevent infinite loops - for (let i = 0; i < maxIterations; ++i) { - const allRelationPageIdsInThisIteration = new Set() + const newBlocks = await this.fetchRelationPages(recordMap, kyOptions) + recordMap.block = { ...recordMap.block, ...newBlocks } + } - for (const blockId of Object.keys(recordMap.block)) { - const blockValue = recordMap.block[blockId]?.value - if ( - blockValue?.parent_table === 'collection' && - blockValue?.parent_id - ) { - const collection = recordMap.collection[blockValue.parent_id]?.value - if (collection?.schema) { - for (const propertyId of Object.keys( - blockValue.properties || {} - )) { - const schema = collection.schema[propertyId] - if (schema?.type === 'relation') { - const decorations = blockValue.properties![propertyId] - if (decorations && Array.isArray(decorations)) { - for (const decoration of decorations) { - if ( - Array.isArray(decoration) && - decoration.length > 1 && - decoration[0] === '‣' - ) { - const pagePointer = decoration[1]?.[0] - if ( - Array.isArray(pagePointer) && - pagePointer.length > 1 && - pagePointer[0] === 'p' - ) { - allRelationPageIdsInThisIteration.add(pagePointer[1]) - } - } - } - } - } - } - } + return recordMap + } + + fetchRelationPages = async ( + recordMap: notion.ExtendedRecordMap, + kyOptions: KyOptions | undefined + ): Promise => { + const maxIterations = 10 + + for (let i = 0; i < maxIterations; ++i) { + const relationPageIdsThisIteration = new Set() + + for (const blockId of Object.keys(recordMap.block)) { + const blockValue = recordMap.block[blockId]?.value + if ( + blockValue?.parent_table === 'collection' && + blockValue?.parent_id + ) { + const collection = recordMap.collection[blockValue.parent_id]?.value + if (collection?.schema) { + const ids = this.extractRelationPageIdsFromBlock( + blockValue, + collection.schema + ) + for (const id of ids) relationPageIdsThisIteration.add(id) } } + } - const pendingRelationPageIds = Array.from( - allRelationPageIdsInThisIteration - ).filter((id) => !recordMap.block[id]?.value) + const missingRelationPageIds = Array.from( + relationPageIdsThisIteration + ).filter((id) => !recordMap.block[id]?.value) - if (!pendingRelationPageIds.length) { - break // No new related pages to fetch - } + if (!missingRelationPageIds.length) break - try { - const newBlocks = await this.getBlocks( - pendingRelationPageIds, - kyOptions - ).then((res) => res.recordMap.block) - recordMap.block = { ...recordMap.block, ...newBlocks } - } catch (err: any) { - console.warn( - 'NotionAPI getBlocks error during fetchRelationPages:', - err.message - ) - //TODO: Decide if we should break or continue if some blocks fail - } + try { + const newBlocks = await this.getBlocks( + missingRelationPageIds, + kyOptions + ).then((res) => res.recordMap.block) + recordMap.block = { ...recordMap.block, ...newBlocks } + } catch (err: any) { + console.warn( + 'NotionAPI getBlocks error during fetchRelationPages:', + err + ) + // consider break or delay/retry here } } - return recordMap + return recordMap.block + } + + extractRelationPageIdsFromBlock = ( + blockValue: any, + collectionSchema: any + ): Set => { + const pageIds = new Set() + + for (const propertyId of Object.keys(blockValue.properties || {})) { + const schema = collectionSchema[propertyId] + if (schema?.type === 'relation') { + const decorations = blockValue.properties[propertyId] + if (Array.isArray(decorations)) { + for (const decoration of decorations) { + if ( + Array.isArray(decoration) && + decoration.length > 1 && + decoration[0] === '‣' + ) { + const pagePointer = decoration[1]?.[0] + if ( + Array.isArray(pagePointer) && + pagePointer.length > 1 && + pagePointer[0] === 'p' + ) { + pageIds.add(pagePointer[1]) + } + } + } + } + } + } + return pageIds } public async addSignedUrls({ From 056626731402a348a08797670d0120aa079a00f9 Mon Sep 17 00:00:00 2001 From: Rimon Hanna Date: Fri, 13 Jun 2025 16:42:20 +0200 Subject: [PATCH 3/3] moved the fetchRelationPages example to the full example --- examples/full/lib/notion.ts | 2 +- examples/minimal/pages/[pageId].tsx | 2 +- packages/notion-client/src/notion-api.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/full/lib/notion.ts b/examples/full/lib/notion.ts index bc5eb8d1..87da813a 100644 --- a/examples/full/lib/notion.ts +++ b/examples/full/lib/notion.ts @@ -21,7 +21,7 @@ if (useOfficialNotionAPI) { } export async function getPage(pageId: string): Promise { - const recordMap = await notion.getPage(pageId) + const recordMap = await notion.getPage(pageId, { fetchRelationPages: true }) if (previewImagesEnabled) { const previewImageMap = await getPreviewImageMap(recordMap) diff --git a/examples/minimal/pages/[pageId].tsx b/examples/minimal/pages/[pageId].tsx index 0f0bde8f..b15060ec 100644 --- a/examples/minimal/pages/[pageId].tsx +++ b/examples/minimal/pages/[pageId].tsx @@ -6,7 +6,7 @@ import notion from '../lib/notion' export const getStaticProps = async (context: any) => { const pageId = (context.params.pageId as string) || rootNotionPageId - const recordMap = await notion.getPage(pageId, { fetchRelationPages: true }) + const recordMap = await notion.getPage(pageId) return { props: { diff --git a/packages/notion-client/src/notion-api.ts b/packages/notion-client/src/notion-api.ts index a7ed3b45..8cfdc308 100644 --- a/packages/notion-client/src/notion-api.ts +++ b/packages/notion-client/src/notion-api.ts @@ -53,7 +53,7 @@ export class NotionAPI { chunkNumber = 0, throwOnCollectionErrors = false, collectionReducerLimit = 999, - fetchRelationPages = true, // New option + fetchRelationPages = false, kyOptions }: { concurrency?: number @@ -64,7 +64,7 @@ export class NotionAPI { chunkNumber?: number throwOnCollectionErrors?: boolean collectionReducerLimit?: number - fetchRelationPages?: boolean // New option + fetchRelationPages?: boolean kyOptions?: KyOptions } = {} ): Promise {