From 857891816fc36e3c72241b6abe3c33821803fa23 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 15 Jan 2026 12:49:40 -0500 Subject: [PATCH 01/33] eng-1344 f10b upload obsidian relations and their schemas --- apps/obsidian/src/utils/conceptConversion.ts | 146 +++++++++++++++++- .../src/utils/syncDgNodesToSupabase.ts | 58 ++++++- apps/roam/src/utils/conceptConversion.ts | 56 +++++-- 3 files changed, 238 insertions(+), 22 deletions(-) diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index 74e8853b6..1dffaf7e1 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -1,7 +1,13 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { TFile } from "obsidian"; -import type { DiscourseNode } from "~/types"; +import type DiscourseGraphPlugin from "~/index"; +import type { + DiscourseNode, + DiscourseRelation, + DiscourseRelationType, +} from "~/types"; import type { SupabaseContext } from "./supabaseContext"; +import type { DiscourseNodeInVault } from "./syncDgNodesToSupabase"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase"; import type { Json } from "@repo/database/dbTypes"; @@ -52,21 +58,150 @@ export const discourseNodeSchemaToLocalConcept = ({ }; }; +const STANDARD_ROLES = ["source", "destination"]; + +export const discourseRelationTypeToLocalConcept = ({ + context, + relationType, + accountLocalId, +}: { + context: SupabaseContext; + relationType: DiscourseRelationType; + accountLocalId: string; +}): LocalConceptDataInput => { + const { id, label, complement, created, modified, ...otherData } = + relationType; + return { + space_id: context.spaceId, + name: label, + source_local_id: id, + is_schema: true, + author_local_id: accountLocalId, + created: new Date(created).toISOString(), + last_modified: new Date(modified).toISOString(), + literal_content: { + label, + complement, + source_data: otherData, + } as unknown as Json, + }; +}; + +export const discourseRelationSchemaToLocalConcept = ({ + context, + relation, + accountLocalId, + nodeTypesById, + relationTypesById, +}: { + context: SupabaseContext; + relation: DiscourseRelation; + accountLocalId: string; + nodeTypesById: Record; + relationTypesById: Record; +}): LocalConceptDataInput => { + const { id, relationshipTypeId, sourceId, destinationId, created, modified } = + relation; + const sourceName = nodeTypesById[sourceId]?.name ?? sourceId; + const destinationName = nodeTypesById[destinationId]?.name ?? destinationId; + const relationType = relationTypesById[relationshipTypeId]; + if (!relationType) + throw new Error(`missing relation type ${relationshipTypeId}`); + const { label, complement } = relationType; + + return { + space_id: context.spaceId, + name: `${sourceName} -${label}-> ${destinationName}`, + source_local_id: id, + is_schema: true, + author_local_id: accountLocalId, + created: new Date(created).toISOString(), + last_modified: new Date(modified).toISOString(), + literal_content: { + roles: STANDARD_ROLES, + label, + complement, + }, + local_reference_content: { + relation_type: relationshipTypeId, + source: sourceId, + destination: destinationId, + }, + }; +}; + /** * Convert discourse node instance (file) to LocalConceptDataInput */ -export const discourseNodeInstanceToLocalConcept = ({ +export const discourseNodeInstanceToLocalConcepts = ({ + plugin, + allNodesByName, context, nodeData, accountLocalId, }: { + plugin: DiscourseGraphPlugin; + allNodesByName: Record; context: SupabaseContext; nodeData: ObsidianDiscourseNodeData; accountLocalId: string; -}): LocalConceptDataInput => { +}): LocalConceptDataInput[] => { const extraData = getNodeExtraData(nodeData.file, accountLocalId); const { nodeInstanceId, nodeTypeId, ...otherData } = nodeData.frontmatter; - return { + const response: LocalConceptDataInput[] = []; + for (const relType of plugin.settings.relationTypes) { + const rels = otherData[relType.id]; + if (rels) { + delete otherData[relType.id]; + const triples = plugin.settings.discourseRelations.filter( + (r) => r.relationshipTypeId === relType.id && r.sourceId === nodeTypeId, + ); + if (!triples.length) { + // we're probably the target. + continue; + } + const tripleIdByDestType = Object.fromEntries( + triples.map((rel) => [rel.destinationId, rel.id]), + ); + for (let rel of rels as string[]) { + if (rel.startsWith("[[") && rel.endsWith("]]")) + rel = rel.substring(2, rel.length - 2); + if (rel.endsWith(".md")) rel = rel.substring(0, rel.length - 3); + const target = allNodesByName[rel]; + if (!target) { + console.error(`Could not find node name ${rel}`); + continue; + } + const targetTypeId = target.frontmatter.nodeTypeId as string; + const targetInstanceId = target.frontmatter.nodeInstanceId as string; + const relSchemaId = tripleIdByDestType[targetTypeId]; + if (relSchemaId === undefined) { + console.error( + `Found a relation of type ${relType.id} between ${nodeData.file.path} and ${rel} but no relation fits`, + ); + continue; + } + const compositeInstanceId = [ + relSchemaId, + nodeInstanceId as string, + targetInstanceId, + ].join(":"); + response.push({ + space_id: context.spaceId, + name: `[[${nodeData.file.basename}]] -${relType.label}-> [[${target.file.basename}]]`, + source_local_id: compositeInstanceId, + schema_represented_by_local_id: relSchemaId, + is_schema: false, + local_reference_content: { + source: nodeInstanceId as string, + destination: targetInstanceId, + }, + ...extraData, + }); + } + } + } + response.push({ space_id: context.spaceId, name: nodeData.file.path, source_local_id: nodeInstanceId as string, @@ -77,7 +212,8 @@ export const discourseNodeInstanceToLocalConcept = ({ source_data: otherData as unknown as Json, }, ...extraData, - }; + }); + return response; }; export const relatedConcepts = (concept: LocalConceptDataInput): string[] => { diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 94ba12d13..017219e2c 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -12,8 +12,10 @@ import { default as DiscourseGraphPlugin } from "~/index"; import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings"; import { orderConceptsByDependency, - discourseNodeInstanceToLocalConcept, + discourseNodeInstanceToLocalConcepts, discourseNodeSchemaToLocalConcept, + discourseRelationSchemaToLocalConcept, + discourseRelationTypeToLocalConcept, } from "./conceptConversion"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; @@ -176,7 +178,7 @@ const getLastSchemaSyncTime = async ( return new Date((data?.last_modified || DEFAULT_TIME) + "Z"); }; -type DiscourseNodeInVault = { +export type DiscourseNodeInVault = { file: TFile; frontmatter: Record; nodeTypeId: string; @@ -202,7 +204,7 @@ const mergeChangeTypes = ( * Step 1: Collect all discourse nodes from the vault * Filters markdown files that have nodeTypeId in frontmatter */ -const collectDiscourseNodesFromVault = async ( +export const collectDiscourseNodesFromVault = async ( plugin: DiscourseGraphPlugin, ): Promise => { const allFiles = plugin.app.vault.getMarkdownFiles(); @@ -506,13 +508,27 @@ const convertDgToSupabaseConcepts = async ({ context: SupabaseContext; accountLocalId: string; plugin: DiscourseGraphPlugin; + convertRelations?: boolean; }): Promise => { const lastSchemaSync = ( await getLastSchemaSyncTime(supabaseClient, context.spaceId) ).getTime(); - const newNodeTypes = (plugin.settings.nodeTypes ?? []).filter( + const nodeTypes = plugin.settings.nodeTypes ?? []; + const newNodeTypes = nodeTypes.filter((n) => n.modified > lastSchemaSync); + const relationTypes = (plugin.settings.relationTypes ?? []).filter( (n) => n.modified > lastSchemaSync, ); + const discourseRelations = (plugin.settings.discourseRelations ?? []).filter( + (n) => n.modified > lastSchemaSync, + ); + const allNodes = await collectDiscourseNodesFromVault(plugin); + const allNodesByName = Object.fromEntries( + allNodes.map((n) => [n.file.basename, n]), + ); + + const nodeTypesById = Object.fromEntries( + nodeTypes.map((nodeType) => [nodeType.id, nodeType]), + ); const nodesTypesToLocalConcepts = newNodeTypes.map((nodeType) => discourseNodeSchemaToLocalConcept({ @@ -522,16 +538,44 @@ const convertDgToSupabaseConcepts = async ({ }), ); - const nodeInstanceToLocalConcepts = nodesSince.map((node) => - discourseNodeInstanceToLocalConcept({ + const relationTypesById = Object.fromEntries( + relationTypes.map((relationType) => [relationType.id, relationType]), + ); + + const relationTypesToLocalConcepts = relationTypes.map((relationType) => + discourseRelationTypeToLocalConcept({ context, - nodeData: node, + relationType, accountLocalId, }), ); + const discourseRelationsToLocalConcepts = discourseRelations.map((relation) => + discourseRelationSchemaToLocalConcept({ + context, + relation, + accountLocalId, + nodeTypesById, + relationTypesById, + }), + ); + + const nodeInstanceToLocalConcepts = nodesSince + .map((node) => { + return discourseNodeInstanceToLocalConcepts({ + plugin, + allNodesByName, + context, + nodeData: node, + accountLocalId, + }); + }) + .flat(); + const conceptsToUpsert: LocalConceptDataInput[] = [ ...nodesTypesToLocalConcepts, + ...relationTypesToLocalConcepts, + ...discourseRelationsToLocalConcepts, ...nodeInstanceToLocalConcepts, ]; diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index e257e1880..df1ddfbde 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -1,9 +1,13 @@ +import { InputTextNode } from "roamjs-components/types"; +import getBlockProps from "./getBlockProps"; import { DiscourseNode } from "./getDiscourseNodes"; import getDiscourseRelations from "./getDiscourseRelations"; import type { DiscourseRelation } from "./getDiscourseRelations"; import type { SupabaseContext } from "~/utils/supabaseContext"; +import { DISCOURSE_GRAPH_PROP_NAME } from "~/utils/createReifiedBlock"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; +import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageUid"; const getNodeExtraData = ( node_uid: string, @@ -51,18 +55,39 @@ const getNodeExtraData = ( }; }; +const indent = (s: string): string => + s + .split("\n") + .map((l) => " " + l) + .join("\n") + "\n"; + +const templateToText = (template: InputTextNode[]): string => + template + .filter((itn) => !itn.text.startsWith("{{")) + .map( + (itn) => + `* ${itn.text}\n${itn.children?.length ? indent(templateToText(itn.children)) : ""}`, + ) + .join(""); + export const discourseNodeSchemaToLocalConcept = ( context: SupabaseContext, node: DiscourseNode, ): LocalConceptDataInput => { const titleParts = node.text.split("/"); - return { + const result: LocalConceptDataInput = { space_id: context.spaceId, - name: titleParts[titleParts.length - 1], + name: node.text, source_local_id: node.type, is_schema: true, ...getNodeExtraData(node.type), }; + if (node.template !== undefined) + result.literal_content = { + label: titleParts[titleParts.length - 1], + template: templateToText(node.template), + }; + return result; }; export const discourseNodeBlockToLocalConcept = ( @@ -87,7 +112,7 @@ export const discourseNodeBlockToLocalConcept = ( }; }; -const STANDARD_ROLES = ["source", "target"]; +const STANDARD_ROLES = ["source", "destination"]; export const discourseRelationSchemaToLocalConcept = ( context: SupabaseContext, @@ -97,7 +122,7 @@ export const discourseRelationSchemaToLocalConcept = ( space_id: context.spaceId, source_local_id: relation.id, // Not using the label directly, because it is not unique and name should be unique - name: `${relation.id}-${relation.label}`, + name: getPageTitleByPageUid(relation.id), is_schema: true, local_reference_content: Object.fromEntries( Object.entries(relation).filter(([key, v]) => @@ -116,9 +141,21 @@ export const discourseRelationSchemaToLocalConcept = ( export const discourseRelationDataToLocalConcept = ( context: SupabaseContext, - relationSchemaUid: string, - relationNodes: { [role: string]: string }, + relationUid: string, ): LocalConceptDataInput => { + // assuming reified + const relationProps = getBlockProps(relationUid); + const relationSchemaData = relationProps[DISCOURSE_GRAPH_PROP_NAME] as Record< + string, + string + >; + if (!relationSchemaData) { + throw new Error(`Missing relation data for ${relationUid}`); + } + const relationSchemaUid = relationSchemaData.hasSchema; + if (!relationSchemaUid) { + throw new Error(`Missing relation schema uid for ${relationUid}`); + } const roamRelation = getDiscourseRelations().find( (r) => r.id === relationSchemaUid, ); @@ -132,7 +169,7 @@ export const discourseRelationDataToLocalConcept = ( const roles = (litContent["roles"] as string[] | undefined) || STANDARD_ROLES; const casting: { [role: string]: string } = Object.fromEntries( roles - .map((role) => [role, relationNodes[role]]) + .map((role) => [role, relationSchemaData[role + "Uid"]]) .filter(([, uid]) => uid !== undefined), ); if (Object.keys(casting).length === 0) { @@ -152,14 +189,13 @@ export const discourseRelationDataToLocalConcept = ( Math.max(...nodeData.map((nd) => new Date(nd.created).getTime())), ).toISOString(); const author_local_id: string = nodeData[0].author_uid; // take any one; again until I get the relation object - const source_local_id = casting["target"] || Object.values(casting)[0]; // This one is tricky. Prefer the target for now. return { space_id: context.spaceId, - source_local_id, + source_local_id: relationUid, author_local_id, created, last_modified, - name: `${relationSchemaUid}-${Object.values(casting).join("-")}`, + name: relationUid, is_schema: false, schema_represented_by_local_id: relationSchemaUid, local_reference_content: casting, From 53221134df82855b8990818a782bae25c2be3a85 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Fri, 23 Jan 2026 14:40:49 -0500 Subject: [PATCH 02/33] current progress --- .../src/components/ImportNodesModal.tsx | 374 ++++++++++++ apps/obsidian/src/index.ts | 9 +- apps/obsidian/src/types.ts | 16 + apps/obsidian/src/utils/fileChangeListener.ts | 20 +- apps/obsidian/src/utils/importNodes.ts | 547 ++++++++++++++++++ apps/obsidian/src/utils/registerCommands.ts | 18 + .../src/utils/syncDgNodesToSupabase.ts | 9 + 7 files changed, 986 insertions(+), 7 deletions(-) create mode 100644 apps/obsidian/src/components/ImportNodesModal.tsx create mode 100644 apps/obsidian/src/utils/importNodes.ts diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx new file mode 100644 index 000000000..9bea1f099 --- /dev/null +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -0,0 +1,374 @@ +import { App, Modal, Notice } from "obsidian"; +import { createRoot, Root } from "react-dom/client"; +import { StrictMode, useState, useEffect, useCallback } from "react"; +import type DiscourseGraphPlugin from "../index"; +import type { ImportableNode, GroupWithNodes } from "~/types"; +import { + getAvailableGroups, + getPublishedNodesForGroups, + getLocalNodeInstanceIds, + getSpaceNames, + importSelectedNodes, +} from "~/utils/importNodes"; +import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; + +type ImportNodesModalProps = { + plugin: DiscourseGraphPlugin; + onClose: () => void; +}; + +const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { + const [step, setStep] = useState<"loading" | "select" | "importing">( + "loading", + ); + const [groupsWithNodes, setGroupsWithNodes] = useState( + [], + ); + const [isLoading, setIsLoading] = useState(true); + const [importProgress, setImportProgress] = useState({ + current: 0, + total: 0, + }); + + const loadImportableNodes = useCallback(async () => { + setIsLoading(true); + try { + const client = await getLoggedInClient(plugin); + if (!client) { + new Notice("Cannot get Supabase client"); + onClose(); + return; + } + + const context = await getSupabaseContext(plugin); + if (!context) { + new Notice("Cannot get Supabase context"); + onClose(); + return; + } + + // Get user's groups + const groups = await getAvailableGroups(client); + if (groups.length === 0) { + new Notice("You are not a member of any groups"); + onClose(); + return; + } + + const groupIds = groups.map((g) => g.group_id); + + // Get published nodes for these groups + const publishedNodes = await getPublishedNodesForGroups({ + client, + groupIds, + currentSpaceId: context.spaceId, + }); + + // Get local nodeInstanceIds to filter out existing nodes + const localNodeInstanceIds = await getLocalNodeInstanceIds(plugin); + const localIdsSet = new Set(localNodeInstanceIds); + + // Filter out nodes that already exist locally + const importableNodes = publishedNodes.filter( + (node) => !localIdsSet.has(node.source_local_id), + ); + + // Get space names + const uniqueSpaceIds = [ + ...new Set(importableNodes.map((n) => n.space_id)), + ]; + const spaceNames = await getSpaceNames(client, uniqueSpaceIds); + console.log("spaceNames", spaceNames); + // Group nodes by group and space + const grouped: Map = new Map(); + + for (const node of importableNodes) { + const groupId = node.account_uid; + if (!grouped.has(groupId)) { + grouped.set(groupId, { + groupId, + groupName: `Group ${groupId.slice(0, 8)}...`, + nodes: [], + }); + } + + const group = grouped.get(groupId)!; + group.nodes.push({ + nodeInstanceId: node.source_local_id, + title: node.text, + spaceId: node.space_id, + spaceName: spaceNames.get(node.space_id) ?? `Space ${node.space_id}`, + groupId: node.account_uid, + selected: false, + }); + } + + setGroupsWithNodes(Array.from(grouped.values())); + setStep("select"); + } catch (error) { + console.error("Error loading importable nodes:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + new Notice(`Failed to load nodes: ${errorMessage}`, 5000); + onClose(); + } finally { + setIsLoading(false); + } + }, [plugin, onClose]); + + useEffect(() => { + void loadImportableNodes(); + }, [loadImportableNodes]); + + const handleNodeToggle = (groupId: string, nodeIndex: number) => { + setGroupsWithNodes((prev) => + prev.map((group) => { + if (group.groupId !== groupId) return group; + return { + ...group, + nodes: group.nodes.map((node, idx) => + idx === nodeIndex + ? { ...node, selected: !node.selected } + : node, + ), + }; + }), + ); + }; + + + const handleImport = async () => { + const selectedNodes: ImportableNode[] = []; + for (const group of groupsWithNodes) { + for (const node of group.nodes) { + if (node.selected) { + selectedNodes.push(node); + } + } + } + + if (selectedNodes.length === 0) { + new Notice("Please select at least one node to import"); + return; + } + + setStep("importing"); + setImportProgress({ current: 0, total: selectedNodes.length }); + + try { + const result = await importSelectedNodes({ plugin, selectedNodes }); + + if (result.failed > 0) { + new Notice( + `Import completed with some issues:\n${result.success} files imported successfully\n${result.failed} files failed`, + 5000, + ); + } else { + new Notice( + `Successfully imported ${result.success} node(s)`, + 3000, + ); + } + + onClose(); + } catch (error) { + console.error("Error importing nodes:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + new Notice(`Import failed: ${errorMessage}`, 5000); + setStep("select"); + } + }; + + const renderLoadingStep = () => ( +
+

Loading importable nodes...

+
Fetching groups and published nodes
+
+ ); + + const renderSelectStep = () => { + const totalNodes = groupsWithNodes.reduce( + (sum, group) => sum + group.nodes.length, + 0, + ); + const selectedCount = groupsWithNodes.reduce( + (sum, group) => + sum + group.nodes.filter((n) => n.selected).length, + 0, + ); + + // Group nodes by space for better organization + const nodesBySpace = new Map< + number, + { spaceName: string; nodes: Array<{ node: ImportableNode; groupId: string; nodeIndex: number }> } + >(); + + for (const group of groupsWithNodes) { + for (const [nodeIndex, node] of group.nodes.entries()) { + if (!nodesBySpace.has(node.spaceId)) { + nodesBySpace.set(node.spaceId, { + spaceName: node.spaceName, + nodes: [], + }); + } + nodesBySpace.get(node.spaceId)!.nodes.push({ + node, + groupId: group.groupId, + nodeIndex, + }); + } + } + + return ( +
+

Select Nodes to Import

+

+ {totalNodes} importable node(s) found. Select which nodes to import + into your vault. +

+ +
+ + +
+ +
+ {Array.from(nodesBySpace.entries()).map(([spaceId, { spaceName, nodes }]) => { + return ( +
+
+ 📂 + + {spaceName} + + + ({nodes.length} node{nodes.length !== 1 ? "s" : ""}) + +
+ + {nodes.map(({ node, groupId, nodeIndex }) => ( +
+ handleNodeToggle(groupId, nodeIndex)} + className="mr-3 mt-1 flex-shrink-0" + /> +
+
+ {node.title} +
+
+
+ ))} +
+ ); + })} +
+ +
+ + +
+
+ ); + }; + + const renderImportingStep = () => ( +
+

Importing nodes

+
+
+
+
+
+ {importProgress.current} of {importProgress.total} node(s) processed +
+
+
+ ); + + if (isLoading || step === "loading") { + return renderLoadingStep(); + } + + switch (step) { + case "select": + return renderSelectStep(); + case "importing": + return renderImportingStep(); + default: + return null; + } +}; + +export class ImportNodesModal extends Modal { + private plugin: DiscourseGraphPlugin; + private root: Root | null = null; + + constructor(app: App, plugin: DiscourseGraphPlugin) { + super(app); + this.plugin = plugin; + } + + onOpen() { + const { contentEl } = this; + contentEl.empty(); + this.root = createRoot(contentEl); + this.root.render( + + this.close()} /> + , + ); + } + + onClose() { + if (this.root) { + this.root.unmount(); + this.root = null; + } + } +} diff --git a/apps/obsidian/src/index.ts b/apps/obsidian/src/index.ts index 0b139c2be..fe473b9c4 100644 --- a/apps/obsidian/src/index.ts +++ b/apps/obsidian/src/index.ts @@ -272,7 +272,14 @@ export default class DiscourseGraphPlugin extends Plugin { const keysToHide: string[] = []; if (!this.settings.showIdsInFrontmatter) { - keysToHide.push("nodeTypeId"); + keysToHide.push( + ...[ + "nodeTypeId", + "importedFromSpaceId", + "nodeInstanceId", + "publishedToGroups", + ], + ); keysToHide.push(...this.settings.relationTypes.map((rt) => rt.id)); } diff --git a/apps/obsidian/src/types.ts b/apps/obsidian/src/types.ts index b9f849e08..f7e40d78b 100644 --- a/apps/obsidian/src/types.ts +++ b/apps/obsidian/src/types.ts @@ -61,4 +61,20 @@ export type BulkImportPattern = { enabled: boolean; }; +export type ImportableNode = { + nodeInstanceId: string; + title: string; + spaceId: number; + spaceName: string; + groupId: string; + groupName?: string; + selected: boolean; +}; + +export type GroupWithNodes = { + groupId: string; + groupName?: string; + nodes: ImportableNode[]; +}; + export const VIEW_TYPE_DISCOURSE_CONTEXT = "discourse-context-view"; diff --git a/apps/obsidian/src/utils/fileChangeListener.ts b/apps/obsidian/src/utils/fileChangeListener.ts index f01de6270..36197b331 100644 --- a/apps/obsidian/src/utils/fileChangeListener.ts +++ b/apps/obsidian/src/utils/fileChangeListener.ts @@ -80,7 +80,7 @@ export class FileChangeListener { /** * Check if a file is a DG node (has nodeTypeId in frontmatter that matches a node type in settings) */ - private isDiscourseNode(file: TAbstractFile): boolean { + private shouldSyncFile(file: TAbstractFile): boolean { if (!(file instanceof TFile)) { return false; } @@ -91,13 +91,17 @@ export class FileChangeListener { } const cache = this.plugin.app.metadataCache.getFileCache(file); - const nodeTypeId = cache?.frontmatter?.nodeTypeId as string | undefined; + const frontmatter = cache?.frontmatter; + const nodeTypeId = frontmatter?.nodeTypeId as string | undefined; if (!nodeTypeId || typeof nodeTypeId !== "string") { return false; } - // Verify that the nodeTypeId matches one of the node types in settings + if (frontmatter?.importedFromSpaceId) { + return false; + } + return !!getNodeTypeById(this.plugin, nodeTypeId); } @@ -115,7 +119,7 @@ export class FileChangeListener { this.pendingCreates.add(file.path); - if (this.isDiscourseNode(file)) { + if (this.shouldSyncFile(file)) { this.queueChange(file.path, "title"); this.queueChange(file.path, "content"); this.pendingCreates.delete(file.path); @@ -126,7 +130,7 @@ export class FileChangeListener { * Handle file modification event */ private handleFileModify(file: TAbstractFile): void { - if (!this.isDiscourseNode(file)) { + if (!this.shouldSyncFile(file)) { return; } @@ -151,6 +155,10 @@ export class FileChangeListener { * Handle file rename event */ private handleFileRename(file: TAbstractFile, oldPath: string): void { + if (!this.shouldSyncFile(file)) { + return; + } + console.log(`File renamed: ${oldPath} -> ${file.path}`); this.queueChange(file.path, "title", oldPath); } @@ -159,7 +167,7 @@ export class FileChangeListener { * Handle metadata changes (placeholder for relation metadata) */ private handleMetadataChange(file: TFile): void { - if (!this.isDiscourseNode(file)) { + if (!this.shouldSyncFile(file)) { return; } diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts new file mode 100644 index 000000000..e6c36514e --- /dev/null +++ b/apps/obsidian/src/utils/importNodes.ts @@ -0,0 +1,547 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type DiscourseGraphPlugin from "~/index"; +import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; +import type { DiscourseNode, ImportableNode } from "~/types"; +import generateUid from "~/utils/generateUid"; + +export const getAvailableGroups = async ( + client: DGSupabaseClient, +): Promise<{ group_id: string }[]> => { + const { data, error } = await client + .from("group_membership") + .select("group_id") + .eq("member_id", (await client.auth.getUser()).data.user?.id || ""); + + if (error) { + console.error("Error fetching groups:", error); + throw new Error(`Failed to fetch groups: ${error.message}`); + } + + return data || []; +}; + +export const getPublishedNodesForGroups = async ({ + client, + groupIds, + currentSpaceId, +}: { + client: DGSupabaseClient; + groupIds: string[]; + currentSpaceId: number; +}): Promise< + Array<{ + source_local_id: string; + space_id: number; + text: string; + account_uid: string; + }> +> => { + if (groupIds.length === 0) { + return []; + } + + // First get all ResourceAccess entries for these groups + const { data: resourceAccessData, error: raError } = await client + .from("ResourceAccess") + .select("source_local_id, space_id, account_uid") + .in("account_uid", groupIds) + .neq("space_id", currentSpaceId); + + if (raError) { + console.error("Error fetching resource access:", raError); + throw new Error(`Failed to fetch resource access: ${raError.message}`); + } + + if (!resourceAccessData || resourceAccessData.length === 0) { + return []; + } + + // Group by space_id and source_local_id to fetch content efficiently + const nodeKeys = new Set(); + for (const ra of resourceAccessData) { + if (ra.source_local_id && ra.space_id) { + nodeKeys.add(`${ra.space_id}:${ra.source_local_id}`); + } + } + + // Fetch Content with variant "direct" for all these nodes + const spaceIdSourceIdPairs = Array.from(nodeKeys).map((key) => { + const [spaceId, sourceLocalId] = key.split(":"); + return { spaceId: parseInt(spaceId || "0", 10), sourceLocalId: sourceLocalId || "" }; + }); + + const nodes: Array<{ + source_local_id: string; + space_id: number; + text: string; + account_uid: string; + }> = []; + + for (const { spaceId, sourceLocalId } of spaceIdSourceIdPairs) { + const { data: contentData } = await client + .from("Content") + .select("text") + .eq("source_local_id", sourceLocalId) + .eq("space_id", spaceId) + .eq("variant", "direct") + .maybeSingle(); + + if (contentData?.text) { + const relevantRAs = resourceAccessData.filter( + (ra) => + ra.source_local_id === sourceLocalId && ra.space_id === spaceId, + ); + + for (const ra of relevantRAs) { + nodes.push({ + source_local_id: sourceLocalId, + space_id: spaceId, + text: contentData.text, + account_uid: ra.account_uid, + }); + } + } + } + + return nodes; +}; + +export const getLocalNodeInstanceIds = async ( + plugin: DiscourseGraphPlugin, +): Promise> => { + const allFiles = plugin.app.vault.getMarkdownFiles(); + const nodeInstanceIds = new Set(); + + for (const file of allFiles) { + const cache = plugin.app.metadataCache.getFileCache(file); + const frontmatter = cache?.frontmatter; + + if (frontmatter?.nodeInstanceId) { + const nodeInstanceId = frontmatter.nodeInstanceId as string; + if (nodeInstanceId) { + nodeInstanceIds.add(nodeInstanceId); + } + } + } + + return nodeInstanceIds; +}; + +export const getSpaceName = async ( + client: DGSupabaseClient, + spaceId: number, +): Promise => { + const { data, error } = await client + .from("Space") + .select("name") + .eq("id", spaceId) + .maybeSingle(); + + if (error || !data) { + console.error("Error fetching space name:", error); + return `space-${spaceId}`; + } + + return data.name; +}; + +export const getSpaceNames = async ( + client: DGSupabaseClient, + spaceIds: number[], +): Promise> => { + if (spaceIds.length === 0) { + return new Map(); + } + + const { data, error } = await client + .from("Space") + .select("id, name") + .in("id", spaceIds); + + if (error) { + console.error("Error fetching space names:", error); + return new Map(); + } + + const spaceMap = new Map(); + (data || []).forEach((space) => { + spaceMap.set(space.id, space.name); + }); + + return spaceMap; +}; + +export const fetchNodeContent = async ({ + client, + spaceId, + nodeInstanceId, + variant, +}: { + client: DGSupabaseClient; + spaceId: number; + nodeInstanceId: string; + variant: "direct" | "full"; +}): Promise => { + const { data, error } = await client + .from("Content") + .select("text") + .eq("source_local_id", nodeInstanceId) + .eq("space_id", spaceId) + .eq("variant", variant) + .maybeSingle(); + + if (error || !data) { + console.error( + `Error fetching node content (${variant}):`, + error || "No data", + ); + return null; + } + + return data.text; +}; + +export const fetchNodeMetadata = async ({ + client, + spaceId, + nodeInstanceId, +}: { + client: DGSupabaseClient; + spaceId: number; + nodeInstanceId: string; +}): Promise<{ nodeTypeId?: string }> => { + // Try to get nodeTypeId from Concept table + const { data: conceptData } = await client + .from("Concept") + .select("literal_content") + .eq("source_local_id", nodeInstanceId) + .eq("space_id", spaceId) + .eq("is_schema", false) + .maybeSingle(); + + return { + nodeTypeId: (conceptData?.literal_content as unknown as { nodeTypeId?: string })?.nodeTypeId || undefined, + }; +}; + +const sanitizeFileName = (fileName: string): string => { + // Remove invalid characters for file names + return fileName + .replace(/[<>:"/\\|?*]/g, "") + .replace(/\s+/g, " ") + .trim(); +}; + +type ParsedFrontmatter = { + nodeTypeId?: string; + nodeInstanceId?: string; + publishedToGroups?: string[]; + [key: string]: unknown; +}; + +const parseFrontmatter = (content: string): { + frontmatter: ParsedFrontmatter; + body: string; +} => { + // Updated regex to handle files with only frontmatter (no body content) + // The body is optional - it can be empty or not exist at all + // Pattern: ---\n(frontmatter)\n---\n(body - optional) + const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*(?:\n([\s\S]*))?$/; + const match = content.match(frontmatterRegex); + + if (!match || !match[1]) { + // No frontmatter, return empty frontmatter and full content as body + return { frontmatter: {}, body: content }; + } + + const frontmatterText = match[1]; + // Body is optional - if there's no body content, match[2] will be undefined + const body = match[2] ?? ""; + + // Parse YAML-like frontmatter (simple parser for key: value pairs) + const frontmatter: ParsedFrontmatter = {}; + const lines = frontmatterText.split("\n"); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (!line) continue; + + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + + const colonIndex = trimmed.indexOf(":"); + if (colonIndex === -1) continue; + + const key = trimmed.slice(0, colonIndex).trim(); + const valueStr = trimmed.slice(colonIndex + 1).trim(); + let value: unknown = valueStr; + + // Handle array values (simple parsing for publishedToGroups) + if (valueStr.startsWith("-")) { + const arrayValues: string[] = []; + let currentLine: string | undefined = trimmed; + let nextLineIndex = i + 1; + + // Collect array items + while (currentLine && currentLine.trim().startsWith("-")) { + const itemValue = currentLine.slice(1).trim(); + if (itemValue) arrayValues.push(itemValue); + if (nextLineIndex < lines.length) { + currentLine = lines[nextLineIndex]; + nextLineIndex++; + } else { + break; + } + } + value = arrayValues.length > 0 ? arrayValues : [trimmed.slice(1).trim()]; + } else { + // Remove quotes if present + if ( + (valueStr.startsWith('"') && valueStr.endsWith('"')) || + (valueStr.startsWith("'") && valueStr.endsWith("'")) + ) { + value = valueStr.slice(1, -1); + } else { + value = valueStr; + } + } + + frontmatter[key] = value; + } + + return { frontmatter, body }; +}; + +/** + * Parse literal_content from a Concept schema into fields for DiscourseNode. + * Handles both nested form { label, template, source_data: { format, color, tag } } + * and flat form { id, name, color, format, tag }. + */ +const parseSchemaLiteralContent = ( + literalContent: unknown, + fallbackName: string, +): Pick => { + const obj = + typeof literalContent === "string" + ? (JSON.parse(literalContent) as Record) + : (literalContent as Record) || {}; + const src = (obj.source_data as Record) || obj; + const name = (obj.name as string) || (obj.label as string) || fallbackName; + const formatFromSchema = + (src.format as string) || (obj.format as string) || ""; + const format = + formatFromSchema || + `${name.slice(0, 3).toUpperCase()} - {content}`; + return { + name, + format, + color: (src.color as string) || (obj.color as string) || undefined, + tag: (src.tag as string) || (obj.tag as string) || undefined, + template: (obj.template as string) || undefined, + keyImage: (src.keyImage as boolean) ?? (obj.keyImage as boolean) ?? undefined, + }; +}; + +const mapNodeTypeIdToLocal = async ({ + plugin, + client, + sourceSpaceId, + sourceNodeTypeId, +}: { + plugin: DiscourseGraphPlugin; + client: DGSupabaseClient; + sourceSpaceId: number; + sourceNodeTypeId: string; +}): Promise => { + // Find the schema in the source space with this nodeTypeId + const { data: schemaData } = await client + .from("Concept") + .select("name, literal_content") + .eq("space_id", sourceSpaceId) + .eq("is_schema", true) + .eq("source_local_id", sourceNodeTypeId) + .maybeSingle(); + + if (!schemaData?.name) { + return sourceNodeTypeId; + } + + const schemaName = schemaData.name; + + // Find a local nodeType with the same name (use plugin.settings so we see newly created types) + const matchingLocalNodeType = plugin.settings.nodeTypes.find( + (nt) => nt.name === schemaName, + ); + + if (matchingLocalNodeType) { + return matchingLocalNodeType.id; + } + + // No matching local nodeType: create one from literal_content and add to settings + const parsed = parseSchemaLiteralContent( + schemaData.literal_content, + schemaName, + ); + + const newNodeType: DiscourseNode = { + id: generateUid("node"), + name: parsed.name, + format: parsed.format, + color: parsed.color, + tag: parsed.tag, + template: parsed.template, + keyImage: parsed.keyImage, + }; + plugin.settings.nodeTypes = [ + ...plugin.settings.nodeTypes, + newNodeType, + ]; + await plugin.saveSettings(); + return newNodeType.id; +}; + +const processFileContent = async ({ + plugin, + client, + sourceSpaceId, + rawContent, + filePath, +}: { + plugin: DiscourseGraphPlugin; + client: DGSupabaseClient; + sourceSpaceId: number; + rawContent: string; + filePath: string; +}): Promise => { + const { frontmatter } = parseFrontmatter(rawContent); + const sourceNodeTypeId = frontmatter.nodeTypeId + + let mappedNodeTypeId: string | undefined; + if (sourceNodeTypeId && typeof sourceNodeTypeId === "string") { + mappedNodeTypeId = await mapNodeTypeIdToLocal({ + plugin, + client, + sourceSpaceId, + sourceNodeTypeId, + }); + } + + const file = await plugin.app.vault.create(filePath, rawContent); + + await plugin.app.fileManager.processFrontMatter(file, (fm) => { + if (mappedNodeTypeId !== undefined) { + (fm as Record).nodeTypeId = mappedNodeTypeId; + } + (fm as Record).importedFromSpaceId = sourceSpaceId; + }); +}; + +export const importSelectedNodes = async ({ + plugin, + selectedNodes, +}: { + plugin: DiscourseGraphPlugin; + selectedNodes: ImportableNode[]; +}): Promise<{ success: number; failed: number }> => { + const client = await getLoggedInClient(plugin); + if (!client) { + throw new Error("Cannot get Supabase client"); + } + + const context = await getSupabaseContext(plugin); + if (!context) { + throw new Error("Cannot get Supabase context"); + } + + let successCount = 0; + let failedCount = 0; + + // Group nodes by space to create folders efficiently + const nodesBySpace = new Map(); + for (const node of selectedNodes) { + if (!nodesBySpace.has(node.spaceId)) { + nodesBySpace.set(node.spaceId, []); + } + nodesBySpace.get(node.spaceId)!.push(node); + } + + // Process each space + for (const [spaceId, nodes] of nodesBySpace.entries()) { + const spaceName = await getSpaceName(client, spaceId); + const importFolderPath = `import/${sanitizeFileName(spaceName)}`; + + // Ensure the folder exists + const folderExists = await plugin.app.vault.adapter.exists(importFolderPath); + if (!folderExists) { + await plugin.app.vault.createFolder(importFolderPath); + } + + // Process each node in this space + for (const node of nodes) { + try { + // Fetch the file name (direct variant) and content (full variant) + const fileName = await fetchNodeContent({ + client, + spaceId: node.spaceId, + nodeInstanceId: node.nodeInstanceId, + variant: "direct", + }); + + if (!fileName) { + console.warn( + `No direct variant found for node ${node.nodeInstanceId}`, + ); + failedCount++; + continue; + } + + const content = await fetchNodeContent({ + client, + spaceId: node.spaceId, + nodeInstanceId: node.nodeInstanceId, + variant: "full", + }); + + if (content === null) { + console.warn( + `No full variant found for node ${node.nodeInstanceId}`, + ); + failedCount++; + continue; + } + + // Sanitize file name + const sanitizedFileName = sanitizeFileName(fileName); + const filePath = `${importFolderPath}/${sanitizedFileName}.md`; + + // Check if file already exists and handle duplicates + let finalFilePath = filePath; + let counter = 1; + while (await plugin.app.vault.adapter.exists(finalFilePath)) { + finalFilePath = `${importFolderPath}/${sanitizedFileName} (${counter}).md`; + counter++; + } + + // Process the file content (maps nodeTypeId, handles frontmatter) + // This creates the file and uses Obsidian's processFrontMatter API + await processFileContent({ + plugin, + client, + sourceSpaceId: node.spaceId, + rawContent: content, + filePath: finalFilePath, + }); + successCount++; + } catch (error) { + console.error( + `Error importing node ${node.nodeInstanceId}:`, + error, + ); + failedCount++; + } + } + } + + return { success: successCount, failed: failedCount }; +}; diff --git a/apps/obsidian/src/utils/registerCommands.ts b/apps/obsidian/src/utils/registerCommands.ts index ffe07b1b3..4fbccbfb9 100644 --- a/apps/obsidian/src/utils/registerCommands.ts +++ b/apps/obsidian/src/utils/registerCommands.ts @@ -3,6 +3,7 @@ import type DiscourseGraphPlugin from "~/index"; import { NodeTypeModal } from "~/components/NodeTypeModal"; import ModifyNodeModal from "~/components/ModifyNodeModal"; import { BulkIdentifyDiscourseNodesModal } from "~/components/BulkIdentifyDiscourseNodesModal"; +import { ImportNodesModal } from "~/components/ImportNodesModal"; import { createDiscourseNode } from "./createNode"; import { VIEW_TYPE_MARKDOWN, VIEW_TYPE_TLDRAW_DG_PREVIEW } from "~/constants"; import { createCanvas } from "~/components/canvas/utils/tldraw"; @@ -71,6 +72,23 @@ export const registerCommands = (plugin: DiscourseGraphPlugin) => { }, }); + plugin.addCommand({ + id: "import-nodes-from-another-space", + name: "Import nodes from another space", + checkCallback: (checking: boolean) => { + if (!plugin.settings.syncModeEnabled) { + if (!checking) { + new Notice("Sync mode is not enabled", 3000); + } + return false; + } + if (!checking) { + new ImportNodesModal(plugin.app, plugin).open(); + } + return true; + }, + }); + plugin.addCommand({ id: "toggle-discourse-context", name: "Toggle discourse context", diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 017219e2c..fd0746024 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -219,6 +219,10 @@ export const collectDiscourseNodesFromVault = async ( continue; } + if (frontmatter.importedFromSpaceId) { + continue; + } + const nodeTypeId = frontmatter.nodeTypeId as string; if (!nodeTypeId) { continue; @@ -671,6 +675,11 @@ const collectDiscourseNodesFromPaths = async ( continue; } + if (frontmatter.importedFromSpaceId) { + console.debug(`Skipping imported file: ${filePath}`); + continue; + } + const nodeTypeId = frontmatter.nodeTypeId as string; if (!nodeTypeId) { continue; From 455b4a8611902864f42130501746944c857ea832 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Fri, 23 Jan 2026 16:32:42 -0500 Subject: [PATCH 03/33] feature finished --- .../src/components/DiscourseContextView.tsx | 44 ++++- apps/obsidian/src/services/QueryEngine.ts | 30 ++++ apps/obsidian/src/types.ts | 1 - apps/obsidian/src/utils/importNodes.ts | 156 ++++++++++++++++-- apps/obsidian/src/utils/registerCommands.ts | 42 +++++ 5 files changed, 254 insertions(+), 19 deletions(-) diff --git a/apps/obsidian/src/components/DiscourseContextView.tsx b/apps/obsidian/src/components/DiscourseContextView.tsx index 7adc4d1ec..bbf9f7e99 100644 --- a/apps/obsidian/src/components/DiscourseContextView.tsx +++ b/apps/obsidian/src/components/DiscourseContextView.tsx @@ -1,4 +1,4 @@ -import { ItemView, TFile, WorkspaceLeaf } from "obsidian"; +import { ItemView, TFile, WorkspaceLeaf, Notice } from "obsidian"; import { createRoot, Root } from "react-dom/client"; import DiscourseGraphPlugin from "~/index"; import { getDiscourseNodeFormatExpression } from "~/utils/getDiscourseNodeFormatExpression"; @@ -6,6 +6,8 @@ import { RelationshipSection } from "~/components/RelationshipSection"; import { VIEW_TYPE_DISCOURSE_CONTEXT } from "~/types"; import { PluginProvider, usePlugin } from "~/components/PluginContext"; import { getNodeTypeById } from "~/utils/typeUtils"; +import { refreshImportedFile } from "~/utils/importNodes"; +import { useState } from "react"; type DiscourseContextProps = { activeFile: TFile | null; @@ -13,6 +15,7 @@ type DiscourseContextProps = { const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { const plugin = usePlugin(); + const [isRefreshing, setIsRefreshing] = useState(false); const extractContentFromTitle = (format: string, title: string): string => { if (!format) return ""; @@ -21,6 +24,30 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { return match?.[1] ?? title; }; + const handleRefresh = async () => { + if (!activeFile || isRefreshing) return; + + setIsRefreshing(true); + try { + const result = await refreshImportedFile({ plugin, file: activeFile }); + if (result.success) { + new Notice("File refreshed successfully", 3000); + } else { + new Notice( + `Failed to refresh file: ${result.error || "Unknown error"}`, + 5000, + ); + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + new Notice(`Refresh failed: ${errorMessage}`, 5000); + console.error("Refresh failed:", error); + } finally { + setIsRefreshing(false); + } + }; + const renderContent = () => { if (!activeFile) { return
No file is open
; @@ -45,6 +72,9 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { if (!nodeType) { return
Unknown node type: {frontmatter.nodeTypeId}
; } + + const isImported = !!frontmatter.importedFromSpaceId; + return ( <>
@@ -56,6 +86,18 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { /> )} {nodeType.name || "Unnamed Node Type"} + {isImported && ( + + )}
{nodeType.format && ( diff --git a/apps/obsidian/src/services/QueryEngine.ts b/apps/obsidian/src/services/QueryEngine.ts index cbf3d090d..3b9b43b52 100644 --- a/apps/obsidian/src/services/QueryEngine.ts +++ b/apps/obsidian/src/services/QueryEngine.ts @@ -290,6 +290,36 @@ export class QueryEngine { } } + /** + * Find an existing imported file by nodeInstanceId and importedFromSpaceId + * Returns the file if found, null otherwise + */ + findExistingImportedFile = ( + nodeInstanceId: string, + importedFromSpaceId: number, + ): TFile | null => { + if (this.dc) { + try { + const dcQuery = `@page and nodeInstanceId = "${nodeInstanceId}" and importedFromSpaceId = ${importedFromSpaceId}`; + const results = this.dc.query(dcQuery); + + for (const page of results) { + if (page.$path) { + const file = this.app.vault.getAbstractFileByPath(page.$path); + if (file && file instanceof TFile) { + return file; + } + } + } + return null; + } catch (error) { + console.warn("Error querying DataCore for imported file:", error); + return null; + } + } + return null; + }; + private async fallbackScanVault( patterns: BulkImportPattern[], validNodeTypes: DiscourseNode[], diff --git a/apps/obsidian/src/types.ts b/apps/obsidian/src/types.ts index f7e40d78b..b0a0925b1 100644 --- a/apps/obsidian/src/types.ts +++ b/apps/obsidian/src/types.ts @@ -67,7 +67,6 @@ export type ImportableNode = { spaceId: number; spaceName: string; groupId: string; - groupName?: string; selected: boolean; }; diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index e6c36514e..526548a7f 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,9 +1,11 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import { TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import type DiscourseGraphPlugin from "~/index"; import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; import type { DiscourseNode, ImportableNode } from "~/types"; import generateUid from "~/utils/generateUid"; +import { QueryEngine } from "~/services/QueryEngine"; export const getAvailableGroups = async ( client: DGSupabaseClient, @@ -225,6 +227,7 @@ export const fetchNodeMetadata = async ({ }; }; + const sanitizeFileName = (fileName: string): string => { // Remove invalid characters for file names return fileName @@ -413,9 +416,9 @@ const processFileContent = async ({ sourceSpaceId: number; rawContent: string; filePath: string; -}): Promise => { +}): Promise<{ file: TFile; error?: string }> => { const { frontmatter } = parseFrontmatter(rawContent); - const sourceNodeTypeId = frontmatter.nodeTypeId + const sourceNodeTypeId = frontmatter.nodeTypeId; let mappedNodeTypeId: string | undefined; if (sourceNodeTypeId && typeof sourceNodeTypeId === "string") { @@ -427,14 +430,20 @@ const processFileContent = async ({ }); } - const file = await plugin.app.vault.create(filePath, rawContent); - + let file: TFile | null = plugin.app.vault.getFileByPath(filePath); + if (!file) { + file = await plugin.app.vault.create(filePath, rawContent); + } else { + await plugin.app.vault.modify(file, rawContent); + } await plugin.app.fileManager.processFrontMatter(file, (fm) => { if (mappedNodeTypeId !== undefined) { (fm as Record).nodeTypeId = mappedNodeTypeId; } (fm as Record).importedFromSpaceId = sourceSpaceId; }); + + return { file }; }; export const importSelectedNodes = async ({ @@ -454,6 +463,8 @@ export const importSelectedNodes = async ({ throw new Error("Cannot get Supabase context"); } + const queryEngine = new QueryEngine(plugin.app); + let successCount = 0; let failedCount = 0; @@ -480,6 +491,12 @@ export const importSelectedNodes = async ({ // Process each node in this space for (const node of nodes) { try { + // Check if file already exists by nodeInstanceId + importedFromSpaceId + const existingFile = queryEngine.findExistingImportedFile( + node.nodeInstanceId, + node.spaceId, + ); + // Fetch the file name (direct variant) and content (full variant) const fileName = await fetchNodeContent({ client, @@ -504,34 +521,62 @@ export const importSelectedNodes = async ({ }); if (content === null) { - console.warn( - `No full variant found for node ${node.nodeInstanceId}`, - ); + console.warn(`No full variant found for node ${node.nodeInstanceId}`); failedCount++; continue; } // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); - const filePath = `${importFolderPath}/${sanitizedFileName}.md`; - - // Check if file already exists and handle duplicates - let finalFilePath = filePath; - let counter = 1; - while (await plugin.app.vault.adapter.exists(finalFilePath)) { - finalFilePath = `${importFolderPath}/${sanitizedFileName} (${counter}).md`; - counter++; + let finalFilePath: string; + + if (existingFile) { + // Update existing file - use its current path + finalFilePath = existingFile.path; + } else { + // Create new file in the import folder + finalFilePath = `${importFolderPath}/${sanitizedFileName}.md`; + + // Check if file path already exists (edge case: same title but different nodeInstanceId) + let counter = 1; + while (await plugin.app.vault.adapter.exists(finalFilePath)) { + finalFilePath = `${importFolderPath}/${sanitizedFileName} (${counter}).md`; + counter++; + } } // Process the file content (maps nodeTypeId, handles frontmatter) - // This creates the file and uses Obsidian's processFrontMatter API - await processFileContent({ + // This updates existing file or creates new one + const result = await processFileContent({ plugin, client, sourceSpaceId: node.spaceId, rawContent: content, filePath: finalFilePath, }); + + if (result.error) { + console.error( + `Error processing file content for node ${node.nodeInstanceId}:`, + result.error, + ); + failedCount++; + continue; + } + + // If title changed and file exists, rename it to match the new title + const processedFile = result.file; + if (existingFile && processedFile.basename !== sanitizedFileName) { + const newPath = `${importFolderPath}/${sanitizedFileName}.md`; + let targetPath = newPath; + let counter = 1; + while (await plugin.app.vault.adapter.exists(targetPath)) { + targetPath = `${importFolderPath}/${sanitizedFileName} (${counter}).md`; + counter++; + } + await plugin.app.fileManager.renameFile(processedFile, targetPath); + } + successCount++; } catch (error) { console.error( @@ -545,3 +590,80 @@ export const importSelectedNodes = async ({ return { success: successCount, failed: failedCount }; }; + +/** + * Refresh a single imported file by fetching the latest content from the database + * Reuses the same logic as importSelectedNodes by treating it as a single-node import + */ +export const refreshImportedFile = async ({ + plugin, + file, + client, +}: { + plugin: DiscourseGraphPlugin; + file: TFile; + client?: DGSupabaseClient; +}): Promise<{ success: boolean; error?: string }> => { + const supabaseClient = client || await getLoggedInClient(plugin); + if (!supabaseClient) { + throw new Error("Cannot get Supabase client"); + } + const cache = plugin.app.metadataCache.getFileCache(file); + const frontmatter = cache?.frontmatter; + const spaceName = await getSpaceName(supabaseClient, frontmatter?.importedFromSpaceId as number); + const result = await importSelectedNodes({ plugin, selectedNodes: [{ + nodeInstanceId: frontmatter?.nodeInstanceId as string, + title: file.basename, + spaceId: frontmatter?.importedFromSpaceId as number, + spaceName: spaceName, + groupId: frontmatter?.publishedToGroups?.[0] as string, + selected: false, + }] }); + return { success: result.success > 0, error: result.failed > 0 ? "Failed to refresh imported file" : undefined }; +}; + +/** + * Refresh all imported files in the vault + */ +export const refreshAllImportedFiles = async ( + plugin: DiscourseGraphPlugin, +): Promise<{ success: number; failed: number; errors: Array<{ file: string; error: string }> }> => { + const allFiles = plugin.app.vault.getMarkdownFiles(); + const importedFiles: TFile[] = []; + const client = await getLoggedInClient(plugin); + if (!client) { + throw new Error("Cannot get Supabase client"); + } + // Find all imported files + for (const file of allFiles) { + const cache = plugin.app.metadataCache.getFileCache(file); + const frontmatter = cache?.frontmatter; + if (frontmatter?.importedFromSpaceId && frontmatter?.nodeInstanceId) { + importedFiles.push(file); + } + } + + if (importedFiles.length === 0) { + return { success: 0, failed: 0, errors: [] }; + } + + let successCount = 0; + let failedCount = 0; + const errors: Array<{ file: string; error: string }> = []; + + // Refresh each file + for (const file of importedFiles) { + const result = await refreshImportedFile({ plugin, file, client }); + if (result.success) { + successCount++; + } else { + failedCount++; + errors.push({ + file: file.path, + error: result.error || "Unknown error", + }); + } + } + + return { success: successCount, failed: failedCount, errors }; +}; diff --git a/apps/obsidian/src/utils/registerCommands.ts b/apps/obsidian/src/utils/registerCommands.ts index 4fbccbfb9..a44c9bf80 100644 --- a/apps/obsidian/src/utils/registerCommands.ts +++ b/apps/obsidian/src/utils/registerCommands.ts @@ -5,6 +5,7 @@ import ModifyNodeModal from "~/components/ModifyNodeModal"; import { BulkIdentifyDiscourseNodesModal } from "~/components/BulkIdentifyDiscourseNodesModal"; import { ImportNodesModal } from "~/components/ImportNodesModal"; import { createDiscourseNode } from "./createNode"; +import { refreshAllImportedFiles } from "./importNodes"; import { VIEW_TYPE_MARKDOWN, VIEW_TYPE_TLDRAW_DG_PREVIEW } from "~/constants"; import { createCanvas } from "~/components/canvas/utils/tldraw"; import { createOrUpdateDiscourseEmbedding } from "./syncDgNodesToSupabase"; @@ -89,6 +90,47 @@ export const registerCommands = (plugin: DiscourseGraphPlugin) => { }, }); + plugin.addCommand({ + id: "refresh-imported-nodes", + name: "Fetch latest content from imported nodes", + checkCallback: (checking: boolean) => { + if (!plugin.settings.syncModeEnabled) { + if (!checking) { + new Notice("Sync mode is not enabled", 3000); + } + return false; + } + if (!checking) { + void refreshAllImportedFiles(plugin) + .then((result) => { + if (result.failed > 0) { + new Notice( + `Refresh completed with some issues:\n${result.success} file(s) refreshed successfully\n${result.failed} file(s) failed`, + 5000, + ); + if (result.errors.length > 0) { + console.error("Refresh errors:", result.errors); + } + } else if (result.success > 0) { + new Notice( + `Successfully refreshed ${result.success} imported node(s)`, + 3000, + ); + } else { + new Notice("No imported files found to refresh", 3000); + } + }) + .catch((error) => { + const errorMessage = + error instanceof Error ? error.message : String(error); + new Notice(`Refresh failed: ${errorMessage}`, 5000); + console.error("Refresh failed:", error); + }); + } + return true; + }, + }); + plugin.addCommand({ id: "toggle-discourse-context", name: "Toggle discourse context", From b9deb258a5eca2969693247a02eb8f0d83f17ff5 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 25 Jan 2026 00:31:40 -0500 Subject: [PATCH 04/33] address PR comments --- apps/obsidian/TEST_PLAN_IMPORT.md | 789 ++++++++++++++++++ .../src/components/ImportNodesModal.tsx | 17 +- apps/obsidian/src/services/QueryEngine.ts | 19 +- apps/obsidian/src/utils/importNodes.ts | 205 +++-- 4 files changed, 952 insertions(+), 78 deletions(-) create mode 100644 apps/obsidian/TEST_PLAN_IMPORT.md diff --git a/apps/obsidian/TEST_PLAN_IMPORT.md b/apps/obsidian/TEST_PLAN_IMPORT.md new file mode 100644 index 000000000..eb7266c80 --- /dev/null +++ b/apps/obsidian/TEST_PLAN_IMPORT.md @@ -0,0 +1,789 @@ +# Import Functionality Test Plan + +## Overview +This document outlines comprehensive test cases for the node import and refresh functionality in the Obsidian Discourse Graph plugin. + +--- + +## 1. Basic Import Functionality + +### 1.1 Happy Path - Single Node Import +**Test Case**: Import a single node from another space +- **Prerequisites**: + - User is member of at least one group + - Group has published nodes + - Node is not already imported +- **Steps**: + 1. Open "Import nodes from another space" command + 2. Select one node from the list + 3. Click "Import" +- **Expected**: + - File created in `import/[spaceName]/[nodeTitle].md` + - File contains correct frontmatter with `nodeInstanceId` and `importedFromSpaceId` + - File content matches database "full" variant + - File title matches database "direct" variant + - Success notice displayed + +### 1.2 Multiple Nodes Import +**Test Case**: Import multiple nodes from same space +- **Steps**: + 1. Open import modal + 2. Select 3-5 nodes from same space + 3. Click "Import" +- **Expected**: + - All files created in same `import/[spaceName]/` folder + - All files have correct metadata + - Success count matches number of selected nodes + +### 1.3 Multiple Nodes from Different Spaces +**Test Case**: Import nodes from multiple spaces simultaneously +- **Steps**: + 1. Select nodes from 2-3 different spaces + 2. Click "Import" +- **Expected**: + - Files organized in separate folders: `import/[space1]/`, `import/[space2]/`, etc. + - Each file has correct `importedFromSpaceId` matching its source space + +### 1.4 Import All Nodes from Group +**Test Case**: Select and import all available nodes +- **Steps**: + 1. Use "Select All" if available + 2. Import all nodes +- **Expected**: + - All nodes imported successfully + - No duplicates created + - Performance is acceptable (no timeout) + +--- + +## 2. Node Type Mapping + +### 2.1 Matching Local Node Type +**Test Case**: Import node with nodeTypeId that matches local node type by name +- **Prerequisites**: + - Source space has node type "Question" with ID `node_abc123` + - Local vault has node type "Question" with ID `node_xyz789` +- **Steps**: Import a node of type "Question" +- **Expected**: + - Imported file has `nodeTypeId: node_xyz789` (local ID) + - Not `node_abc123` (source ID) + - File displays correctly with local node type formatting + +### 2.2 Non-Matching Node Type - Auto-Create +**Test Case**: Import node with nodeTypeId that doesn't exist locally +- **Prerequisites**: + - Source space has node type "Claim" with specific format/color + - Local vault doesn't have "Claim" node type +- **Steps**: Import a "Claim" node +- **Expected**: + - New local node type "Claim" is created + - New node type has same name, format, color, tag, template, keyImage as source + - New node type appears in settings + - Imported file uses the newly created node type ID + +### 2.3 Node Type with Missing Schema Data +**Test Case**: Import node where source schema can't be fetched +- **Prerequisites**: + - Node has `nodeTypeId` but schema not published/accessible +- **Steps**: Import the node +- **Expected**: + - Import proceeds with original `nodeTypeId` (no mapping) + - Or creates node type with fallback values + - Error logged but import doesn't fail completely + +### 2.4 Node Type with Complex Literal Content +**Test Case**: Import node type with nested `literal_content` structure +- **Prerequisites**: + - Schema has `literal_content` with nested `source_data` object +- **Steps**: Import node of this type +- **Expected**: + - Node type created correctly with all nested properties extracted + - Format, color, tag, template, keyImage all preserved + +### 2.5 Node Type with Flat Literal Content +**Test Case**: Import node type with flat `literal_content` structure +- **Prerequisites**: + - Schema has flat `literal_content` (no `source_data` nesting) +- **Steps**: Import node of this type +- **Expected**: + - Node type created correctly from flat structure + - All properties extracted properly + +--- + +## 3. File Existence and Updates + +### 3.1 Import Existing File - Update +**Test Case**: Import node that already exists (same nodeInstanceId + importedFromSpaceId) +- **Prerequisites**: + - File already exists with matching `nodeInstanceId` and `importedFromSpaceId` +- **Steps**: Import the same node again +- **Expected**: + - Existing file is updated (not duplicated) + - File content refreshed from database + - File remains in same location + - No duplicate files created + +### 3.2 Import with Title Change +**Test Case**: Refresh imported file where title changed in source +- **Prerequisites**: + - File exists: `import/Space1/Old Title.md` + - Source title changed to "New Title" +- **Steps**: Refresh the file +- **Expected**: + - File renamed to `import/Space1/New Title.md` + - Content updated + - Frontmatter preserved (`nodeInstanceId`, `importedFromSpaceId`) + +### 3.3 Import with Path Conflict (Different nodeInstanceId) +**Test Case**: Import node where target path exists but different nodeInstanceId +- **Prerequisites**: + - File exists: `import/Space1/Title.md` with different `nodeInstanceId` + - Importing node with same title but different `nodeInstanceId` +- **Steps**: Import the node +- **Expected**: + - New file created as `import/Space1/Title (1).md` + - Both files coexist correctly + - Each has correct `nodeInstanceId` + +### 3.4 File in Wrong Location +**Test Case**: File exists but in wrong folder (e.g., root instead of import/) +- **Prerequisites**: + - File exists at root: `Title.md` with matching `nodeInstanceId` + `importedFromSpaceId` +- **Steps**: Import/refresh the node +- **Expected**: + - File moved to correct `import/[spaceName]/` folder + - Or file updated in place (depending on implementation) + +--- + +## 4. Space Name Handling + +### 4.1 Space Name with Special Characters +**Test Case**: Import from space with special characters in name +- **Prerequisites**: + - Space name: "Test Space / With-Special_Chars" +- **Steps**: Import nodes from this space +- **Expected**: + - Folder name sanitized: `import/Test Space With-Special_Chars/` + - Files created successfully + - No filesystem errors + +### 4.2 Space Name Fetch Failure +**Test Case**: Space name can't be fetched (RLS/permission issue) +- **Prerequisites**: + - User has access to nodes but not space metadata +- **Steps**: Import nodes from this space +- **Expected**: + - `ensureSpaceNames` retries and handles gracefully + - Uses fallback: `import/space-[id]/` or throws descriptive error + - Import proceeds if possible + +### 4.3 Space Name with Very Long Name +**Test Case**: Import from space with very long name +- **Prerequisites**: + - Space name > 200 characters +- **Steps**: Import nodes +- **Expected**: + - Folder name truncated or handled appropriately + - No filesystem path length errors + +### 4.4 Multiple Spaces with Similar Names +**Test Case**: Import from spaces with similar/duplicate names +- **Prerequisites**: + - Two spaces both named "Test Space" +- **Steps**: Import from both +- **Expected**: + - Files organized correctly (may need space ID in folder name) + - No conflicts or overwrites + +--- + +## 5. DataCore Integration + +### 5.1 DataCore Available +**Test Case**: Find existing file with DataCore enabled +- **Prerequisites**: + - DataCore plugin installed and enabled + - File exists with matching `nodeInstanceId` + `importedFromSpaceId` +- **Steps**: Import/refresh node +- **Expected**: + - `findExistingImportedFile` uses DataCore query + - Fast lookup (no full vault scan) + - File found correctly + +### 5.2 DataCore Unavailable +**Test Case**: Find existing file without DataCore +- **Prerequisites**: + - DataCore plugin not installed or disabled +- **Steps**: Import/refresh node +- **Expected**: + - Falls back gracefully (returns null) + - Import still works (creates new file) + - No errors thrown + +### 5.3 DataCore Query Error +**Test Case**: DataCore query throws error +- **Prerequisites**: + - DataCore installed but query fails +- **Steps**: Import/refresh node +- **Expected**: + - Error caught and logged + - Falls back gracefully + - Import continues + +### 5.4 DataCore Returns Multiple Matches +**Test Case**: DataCore query returns multiple files (shouldn't happen but test) +- **Prerequisites**: + - Multiple files with same `nodeInstanceId` + `importedFromSpaceId` (data inconsistency) +- **Steps**: Find existing file +- **Expected**: + - Returns first match + - Warning logged about multiple matches + - Import proceeds with first file + +--- + +## 6. Database and Network Errors + +### 6.1 Missing Direct Variant +**Test Case**: Node exists but "direct" variant missing +- **Prerequisites**: + - Node in database without "direct" variant content +- **Steps**: Import node +- **Expected**: + - Import fails for this node + - Error message: "No direct variant found" + - Other nodes still import successfully + - Failed count incremented + +### 6.2 Missing Full Variant +**Test Case**: Node exists but "full" variant missing +- **Prerequisites**: + - Node in database without "full" variant content +- **Steps**: Import node +- **Expected**: + - Import fails for this node + - Error message: "No full variant found" + - Other nodes still import successfully + +### 6.3 Database Connection Error +**Test Case**: Database connection fails during import +- **Prerequisites**: + - Network disconnected or database unavailable +- **Steps**: Attempt import +- **Expected**: + - Error caught and displayed to user + - Partial imports rolled back or marked as failed + - Clear error message + +### 6.4 RLS Policy Blocking Access +**Test Case**: User doesn't have permission to access node content +- **Prerequisites**: + - Node exists but RLS policy blocks access +- **Steps**: Attempt import +- **Expected**: + - Query returns null/empty + - Import fails for this node + - Error logged appropriately + - Other accessible nodes still import + +### 6.5 Schema Not Published +**Test Case**: Node type schema not published to group +- **Prerequisites**: + - Node exists but its schema not accessible via ResourceAccess +- **Steps**: Import node +- **Expected**: + - Node type mapping fails gracefully + - Uses original `nodeTypeId` or creates with fallback + - Import still succeeds + +--- + +## 7. Frontmatter Handling + +### 7.1 File with Only Frontmatter (No Body) +**Test Case**: Import node with only YAML frontmatter, no body content +- **Prerequisites**: + - Source content: `---\nnodeTypeId: ...\n---\n` (no body) +- **Steps**: Import node +- **Expected**: + - File created correctly + - Frontmatter parsed properly + - Empty body handled gracefully + +### 7.2 Frontmatter with Special Characters +**Test Case**: Import node with special characters in frontmatter values +- **Prerequisites**: + - Frontmatter contains quotes, colons, newlines in values +- **Steps**: Import node +- **Expected**: + - Frontmatter preserved correctly + - No parsing errors + - Special characters escaped properly + +### 7.3 Frontmatter with Arrays +**Test Case**: Import node with array values in frontmatter +- **Prerequisites**: + - Frontmatter contains `publishedToGroups: [id1, id2]` +- **Steps**: Import node +- **Expected**: + - Arrays preserved in frontmatter + - `publishedToGroups` removed (as per implementation) + - Other arrays handled correctly + +### 7.4 Missing nodeInstanceId in Source +**Test Case**: Source content missing nodeInstanceId in frontmatter +- **Prerequisites**: + - Database content has frontmatter without `nodeInstanceId` +- **Steps**: Import node +- **Expected**: + - `nodeInstanceId` added from import process + - File created with correct `nodeInstanceId` + +### 7.5 Frontmatter Update After Import +**Test Case**: Verify frontmatter is correctly set after import +- **Steps**: + 1. Import node + 2. Check file frontmatter +- **Expected**: + - `nodeInstanceId` matches source + - `importedFromSpaceId` matches source space + - `nodeTypeId` is mapped to local ID + - `publishedToGroups` removed + - Other frontmatter preserved + +--- + +## 8. Refresh Functionality + +### 8.1 Refresh Single File +**Test Case**: Refresh one imported file +- **Prerequisites**: + - File exists with `importedFromSpaceId` and `nodeInstanceId` + - Source content updated in database +- **Steps**: + 1. Open file in Discourse Context view + 2. Click "🔄 Refresh" button +- **Expected**: + - File content updated from database + - File title updated if changed + - Success notice displayed + - Frontmatter preserved + +### 8.2 Refresh All Files +**Test Case**: Refresh all imported files via command +- **Prerequisites**: + - Multiple imported files exist + - Some have updates in database +- **Steps**: + 1. Run "Fetch latest content from imported nodes" command + 2. Wait for completion +- **Expected**: + - All files refreshed + - Success/failed counts displayed + - Errors logged for failed files + - Notice shows summary + +### 8.3 Refresh File with Missing Metadata +**Test Case**: Refresh file missing `importedFromSpaceId` or `nodeInstanceId` +- **Prerequisites**: + - File exists but missing required frontmatter +- **Steps**: Attempt refresh +- **Expected**: + - Error returned: "File is not an imported file" + - No database queries attempted + - File not modified + +### 8.4 Refresh File Not in Database +**Test Case**: Refresh file where source node deleted from database +- **Prerequisites**: + - File exists locally + - Source node no longer in database +- **Steps**: Attempt refresh +- **Expected**: + - Error: "Could not fetch latest content" + - File not modified + - Error logged + +### 8.5 Refresh During Import +**Test Case**: Refresh file while import is in progress +- **Prerequisites**: + - Import operation running +- **Steps**: + 1. Start import + 2. Immediately try to refresh a file being imported +- **Expected**: + - Operations don't conflict + - Both complete successfully + - Or refresh waits/queues appropriately + +--- + +## 9. UI/UX Scenarios + +### 9.1 Modal Loading States +**Test Case**: Verify loading states in ImportNodesModal +- **Steps**: + 1. Open import modal + 2. Observe loading state + 3. Wait for nodes to load +- **Expected**: + - Loading indicator shown + - Nodes appear when ready + - No flickering or empty states + +### 9.2 Empty Groups +**Test Case**: User has groups but no published nodes +- **Prerequisites**: + - User is member of groups + - Groups have no published nodes +- **Steps**: Open import modal +- **Expected**: + - Empty state message displayed + - No errors thrown + - Modal can be closed + +### 9.3 No Groups +**Test Case**: User is not member of any groups +- **Prerequisites**: + - User has no group memberships +- **Steps**: Open import modal +- **Expected**: + - Empty state or error message + - Clear indication of why no nodes available + +### 9.4 Large Node Lists +**Test Case**: Import modal with 100+ nodes +- **Prerequisites**: + - Groups have 100+ published nodes +- **Steps**: + 1. Open import modal + 2. Scroll through list + 3. Select multiple nodes +- **Expected**: + - List renders efficiently + - Scrolling is smooth + - Selection works correctly + - Import completes in reasonable time + +### 9.5 Filtering/Search (if implemented) +**Test Case**: Search/filter nodes in import modal +- **Steps**: + 1. Open import modal + 2. Use search/filter if available +- **Expected**: + - Filtering works correctly + - Results update in real-time + - Selection state preserved + +### 9.6 Selection State Persistence +**Test Case**: Selection state during loading/refreshing +- **Steps**: + 1. Select nodes + 2. Trigger refresh/reload +- **Expected**: + - Selection preserved or cleared appropriately + - No state corruption + +--- + +## 10. File System Edge Cases + +### 10.1 Invalid File Names +**Test Case**: Import node with invalid characters in title +- **Prerequisites**: + - Source title: `File/With\Invalid:Chars<>|?*` +- **Steps**: Import node +- **Expected**: + - File name sanitized + - Invalid characters removed/replaced + - File created successfully + +### 10.2 Very Long File Names +**Test Case**: Import node with very long title (>255 chars) +- **Prerequisites**: + - Source title > 255 characters +- **Steps**: Import node +- **Expected**: + - File name truncated appropriately + - No filesystem errors + - File created successfully + +### 10.3 Reserved File Names +**Test Case**: Import node with reserved filename (e.g., "CON", "PRN" on Windows) +- **Prerequisites**: + - Source title is reserved name +- **Steps**: Import node +- **Expected**: + - File name sanitized/renamed + - No filesystem errors + +### 10.4 Folder Creation Permissions +**Test Case**: Import when import/ folder can't be created +- **Prerequisites**: + - Insufficient permissions or disk full +- **Steps**: Attempt import +- **Expected**: + - Error caught and displayed + - Clear error message + - Partial imports handled gracefully + +### 10.5 Disk Space Full +**Test Case**: Import when disk is full +- **Prerequisites**: + - Vault disk is full +- **Steps**: Attempt import +- **Expected**: + - Error caught + - Clear error message + - No partial/corrupted files + +--- + +## 11. Sync Integration + +### 11.1 Imported Files Not Synced +**Test Case**: Verify imported files don't sync back to database +- **Prerequisites**: + - File imported with `importedFromSpaceId` +- **Steps**: + 1. Modify imported file + 2. Wait for sync +- **Expected**: + - File changes not synced to database + - File not in sync queue + - `shouldSyncFile` returns false + +### 11.2 Imported File Without importedFromSpaceId +**Test Case**: Edge case - file missing `importedFromSpaceId` but should be imported +- **Prerequisites**: + - File in import/ folder but missing frontmatter +- **Steps**: Modify file +- **Expected**: + - File not synced (if in import/ folder) + - Or synced if not marked as imported + +### 11.3 Refresh Updates File - No Sync +**Test Case**: Refresh updates file, verify it doesn't trigger sync +- **Steps**: + 1. Refresh imported file + 2. Check sync queue +- **Expected**: + - File updated + - File not queued for sync + - `importedFromSpaceId` preserved + +--- + +## 12. Error Handling and Recovery + +### 12.1 Partial Import Failure +**Test Case**: Some nodes import successfully, others fail +- **Prerequisites**: + - 5 nodes selected + - 2 nodes have database errors +- **Steps**: Import all 5 +- **Expected**: + - 3 nodes imported successfully + - 2 nodes fail with errors + - Success/failed counts accurate + - Error messages for failed nodes + +### 12.2 Import Interruption +**Test Case**: Import interrupted (app closed, network lost) +- **Steps**: + 1. Start large import + 2. Interrupt mid-process +- **Expected**: + - Already imported files remain + - No corrupted files + - Can resume/retry remaining nodes + +### 12.3 Concurrent Imports +**Test Case**: Multiple import operations simultaneously +- **Steps**: + 1. Open two import modals + 2. Import from both +- **Expected**: + - Operations don't conflict + - Both complete successfully + - Or second operation waits/queues + +### 12.4 Error Messages Clarity +**Test Case**: Verify error messages are user-friendly +- **Steps**: Trigger various error conditions +- **Expected**: + - Error messages clear and actionable + - Technical details in console, user-friendly in UI + - No stack traces shown to user + +--- + +## 13. Performance Tests + +### 13.1 Large Batch Import +**Test Case**: Import 50+ nodes +- **Steps**: Import 50 nodes +- **Expected**: + - Completes in reasonable time (< 2 minutes) + - Progress indicator shows progress + - No memory leaks + - UI remains responsive + +### 13.2 DataCore Performance +**Test Case**: Compare performance with/without DataCore +- **Steps**: + 1. Import with DataCore enabled + 2. Import with DataCore disabled +- **Expected**: + - DataCore version significantly faster + - Both complete successfully + - No functional differences + +### 13.3 Refresh All Performance +**Test Case**: Refresh 100+ imported files +- **Steps**: Refresh all imported files +- **Expected**: + - Completes in reasonable time + - Progress shown + - No timeout errors + +--- + +## 14. Data Integrity + +### 14.1 NodeInstanceId Uniqueness +**Test Case**: Verify nodeInstanceId + importedFromSpaceId uniqueness +- **Prerequisites**: + - Two files with same combination (shouldn't happen) +- **Steps**: Import/refresh +- **Expected**: + - Only one file updated/created + - Warning logged if duplicates found + +### 14.2 Frontmatter Consistency +**Test Case**: Verify frontmatter consistency after import/refresh +- **Steps**: + 1. Import node + 2. Check frontmatter + 3. Refresh node + 4. Check frontmatter again +- **Expected**: + - `nodeInstanceId` never changes + - `importedFromSpaceId` never changes + - `nodeTypeId` mapped correctly and consistently + +### 14.3 Content Integrity +**Test Case**: Verify imported content matches source +- **Steps**: + 1. Import node + 2. Compare file content with database +- **Expected**: + - Content matches exactly + - No data loss or corruption + - Encoding handled correctly (UTF-8) + +--- + +## 15. Regression Tests + +### 15.1 Re-import After Delete +**Test Case**: Delete imported file, then re-import +- **Steps**: + 1. Import node + 2. Delete file + 3. Re-import same node +- **Expected**: + - File recreated successfully + - Same location and content + - No errors + +### 15.2 Import After Local Modification +**Test Case**: Import node, modify locally, then refresh +- **Steps**: + 1. Import node + 2. Manually edit file content + 3. Refresh file +- **Expected**: + - Local changes overwritten with database content + - Frontmatter preserved correctly + +### 15.3 Multiple Refresh Cycles +**Test Case**: Refresh same file multiple times +- **Steps**: + 1. Import node + 2. Refresh 5 times in succession +- **Expected**: + - Each refresh succeeds + - No errors accumulate + - File state consistent + +--- + +## Test Execution Checklist + +### Pre-Test Setup +- [ ] Create test spaces with various node types +- [ ] Create test groups and publish nodes +- [ ] Set up test vault with/without DataCore +- [ ] Prepare test data (nodes with various characteristics) +- [ ] Document expected database state + +### Test Execution +- [ ] Execute all test cases systematically +- [ ] Document actual results vs expected +- [ ] Capture screenshots for UI tests +- [ ] Log console errors/warnings +- [ ] Measure performance metrics + +### Post-Test Validation +- [ ] Verify no files left in inconsistent state +- [ ] Check database for any unintended changes +- [ ] Verify all imported files have correct metadata +- [ ] Clean up test files +- [ ] Document any bugs found + +--- + +## Priority Levels + +**P0 (Critical - Must Test)**: +- 1.1, 1.2, 1.3 (Basic import) +- 2.1, 2.2 (Node type mapping) +- 3.1, 3.2 (File updates) +- 6.1, 6.2 (Database errors) +- 8.1, 8.2 (Refresh functionality) +- 11.1 (Sync integration) + +**P1 (High - Should Test)**: +- 2.3, 2.4, 2.5 (Node type edge cases) +- 4.1, 4.2 (Space name handling) +- 5.1, 5.2 (DataCore) +- 7.1, 7.2 (Frontmatter) +- 9.1, 9.2 (UI states) +- 12.1, 12.2 (Error handling) + +**P2 (Medium - Nice to Test)**: +- 3.3, 3.4 (File conflicts) +- 4.3, 4.4 (Space name edge cases) +- 5.3, 5.4 (DataCore edge cases) +- 10.1-10.5 (File system) +- 13.1-13.3 (Performance) + +**P3 (Low - Optional)**: +- 9.3-9.6 (Advanced UI) +- 12.3, 12.4 (Advanced error handling) +- 14.1-14.3 (Data integrity) +- 15.1-15.3 (Regression) + +--- + +## Notes + +- Test in both development and production-like environments +- Test with various Obsidian versions if applicable +- Test with different vault sizes (small, medium, large) +- Test with different network conditions (fast, slow, intermittent) +- Consider automated testing for critical paths +- Maintain test data sets for consistent testing diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index 9bea1f099..27b5afce7 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -47,7 +47,6 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { return; } - // Get user's groups const groups = await getAvailableGroups(client); if (groups.length === 0) { new Notice("You are not a member of any groups"); @@ -57,29 +56,23 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { const groupIds = groups.map((g) => g.group_id); - // Get published nodes for these groups const publishedNodes = await getPublishedNodesForGroups({ client, groupIds, currentSpaceId: context.spaceId, }); - // Get local nodeInstanceIds to filter out existing nodes const localNodeInstanceIds = await getLocalNodeInstanceIds(plugin); - const localIdsSet = new Set(localNodeInstanceIds); // Filter out nodes that already exist locally const importableNodes = publishedNodes.filter( - (node) => !localIdsSet.has(node.source_local_id), + (node) => !localNodeInstanceIds.has(node.source_local_id), ); - // Get space names const uniqueSpaceIds = [ ...new Set(importableNodes.map((n) => n.space_id)), ]; const spaceNames = await getSpaceNames(client, uniqueSpaceIds); - console.log("spaceNames", spaceNames); - // Group nodes by group and space const grouped: Map = new Map(); for (const node of importableNodes) { @@ -156,7 +149,13 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { setImportProgress({ current: 0, total: selectedNodes.length }); try { - const result = await importSelectedNodes({ plugin, selectedNodes }); + const result = await importSelectedNodes({ + plugin, + selectedNodes, + onProgress: (current, total) => { + setImportProgress({ current, total }); + }, + }); if (result.failed > 0) { new Notice( diff --git a/apps/obsidian/src/services/QueryEngine.ts b/apps/obsidian/src/services/QueryEngine.ts index 3b9b43b52..1f88add96 100644 --- a/apps/obsidian/src/services/QueryEngine.ts +++ b/apps/obsidian/src/services/QueryEngine.ts @@ -292,6 +292,7 @@ export class QueryEngine { /** * Find an existing imported file by nodeInstanceId and importedFromSpaceId + * Uses DataCore when available; falls back to vault iteration otherwise * Returns the file if found, null otherwise */ findExistingImportedFile = ( @@ -300,7 +301,8 @@ export class QueryEngine { ): TFile | null => { if (this.dc) { try { - const dcQuery = `@page and nodeInstanceId = "${nodeInstanceId}" and importedFromSpaceId = ${importedFromSpaceId}`; + const safeId = nodeInstanceId.replace(/"/g, '\\"'); + const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceId = ${importedFromSpaceId}`; const results = this.dc.query(dcQuery); for (const page of results) { @@ -311,10 +313,21 @@ export class QueryEngine { } } } - return null; } catch (error) { console.warn("Error querying DataCore for imported file:", error); - return null; + } + } + + // Fallback: DataCore absent, query failed, or indexed field mismatch + const allFiles = this.app.vault.getMarkdownFiles(); + for (const f of allFiles) { + const fm = this.app.metadataCache.getFileCache(f)?.frontmatter; + if ( + fm?.nodeInstanceId === nodeInstanceId && + (fm.importedFromSpaceId === importedFromSpaceId || + fm.importedFromSpaceId === String(importedFromSpaceId)) + ) { + return f; } } return null; diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 526548a7f..0f6b6e5cd 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -67,12 +67,18 @@ export const getPublishedNodesForGroups = async ({ } } - // Fetch Content with variant "direct" for all these nodes - const spaceIdSourceIdPairs = Array.from(nodeKeys).map((key) => { + // Group nodes by space_id for batched queries + const nodesBySpace = new Map>(); + for (const key of nodeKeys) { const [spaceId, sourceLocalId] = key.split(":"); - return { spaceId: parseInt(spaceId || "0", 10), sourceLocalId: sourceLocalId || "" }; - }); + const spaceIdNum = parseInt(spaceId || "0", 10); + if (!nodesBySpace.has(spaceIdNum)) { + nodesBySpace.set(spaceIdNum, new Set()); + } + nodesBySpace.get(spaceIdNum)!.add(sourceLocalId || ""); + } + // Fetch Content with variant "direct" for all nodes, batched by space const nodes: Array<{ source_local_id: string; space_id: number; @@ -80,26 +86,43 @@ export const getPublishedNodesForGroups = async ({ account_uid: string; }> = []; - for (const { spaceId, sourceLocalId } of spaceIdSourceIdPairs) { - const { data: contentData } = await client + for (const [spaceId, sourceLocalIds] of nodesBySpace.entries()) { + const sourceLocalIdsArray = Array.from(sourceLocalIds); + if (sourceLocalIdsArray.length === 0) { + continue; + } + + // Single query for all nodes in this space + const { data: contentDataArray } = await client .from("Content") - .select("text") - .eq("source_local_id", sourceLocalId) + .select("source_local_id, text") .eq("space_id", spaceId) .eq("variant", "direct") - .maybeSingle(); + .in("source_local_id", sourceLocalIdsArray); - if (contentData?.text) { - const relevantRAs = resourceAccessData.filter( - (ra) => - ra.source_local_id === sourceLocalId && ra.space_id === spaceId, - ); + if (!contentDataArray || contentDataArray.length === 0) { + continue; + } - for (const ra of relevantRAs) { + // Create a map for quick lookup + const contentMap = new Map(); + for (const content of contentDataArray) { + if (content.source_local_id && content.text) { + contentMap.set(content.source_local_id, content.text); + } + } + + // Match content with ResourceAccess entries + for (const ra of resourceAccessData) { + if ( + ra.space_id === spaceId && + ra.source_local_id && + contentMap.has(ra.source_local_id) + ) { nodes.push({ - source_local_id: sourceLocalId, + source_local_id: ra.source_local_id, space_id: spaceId, - text: contentData.text, + text: contentMap.get(ra.source_local_id)!, account_uid: ra.account_uid, }); } @@ -120,10 +143,7 @@ export const getLocalNodeInstanceIds = async ( const frontmatter = cache?.frontmatter; if (frontmatter?.nodeInstanceId) { - const nodeInstanceId = frontmatter.nodeInstanceId as string; - if (nodeInstanceId) { - nodeInstanceIds.add(nodeInstanceId); - } + nodeInstanceIds.add(frontmatter.nodeInstanceId as string); } } @@ -269,7 +289,7 @@ const parseFrontmatter = (content: string): { for (let i = 0; i < lines.length; i++) { const line = lines[i]; if (!line) continue; - + const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; @@ -280,34 +300,50 @@ const parseFrontmatter = (content: string): { const valueStr = trimmed.slice(colonIndex + 1).trim(); let value: unknown = valueStr; - // Handle array values (simple parsing for publishedToGroups) - if (valueStr.startsWith("-")) { + // Handle array values: inline (valueStr starts with "-") or block (valueStr empty, next lines start with "-") + const isInlineArrayStart = valueStr.startsWith("-"); + const isEmptyValue = !valueStr || valueStr.trim() === ""; + + if (isInlineArrayStart || isEmptyValue) { const arrayValues: string[] = []; - let currentLine: string | undefined = trimmed; let nextLineIndex = i + 1; - // Collect array items - while (currentLine && currentLine.trim().startsWith("-")) { - const itemValue = currentLine.slice(1).trim(); + // First item on same line (inline): "key: - item" + if (isInlineArrayStart) { + const firstItem = valueStr.slice(1).trim(); + if (firstItem) arrayValues.push(firstItem); + } + + // Collect array items from subsequent lines that trim-start with "-" + let currentLine: string | undefined = + nextLineIndex < lines.length ? lines[nextLineIndex] : undefined; + while (currentLine != null && currentLine.trim().startsWith("-")) { + const itemValue = currentLine.trim().slice(1).trim(); if (itemValue) arrayValues.push(itemValue); - if (nextLineIndex < lines.length) { - currentLine = lines[nextLineIndex]; - nextLineIndex++; - } else { - break; - } + nextLineIndex++; + currentLine = + nextLineIndex < lines.length ? lines[nextLineIndex] : undefined; } - value = arrayValues.length > 0 ? arrayValues : [trimmed.slice(1).trim()]; + + value = + arrayValues.length > 0 + ? arrayValues + : isInlineArrayStart + ? [valueStr.slice(1).trim()] + : []; + frontmatter[key] = value; + i = nextLineIndex - 1; // skip consumed lines + continue; + } + + // Scalar value: remove quotes if present + if ( + (valueStr.startsWith('"') && valueStr.endsWith('"')) || + (valueStr.startsWith("'") && valueStr.endsWith("'")) + ) { + value = valueStr.slice(1, -1); } else { - // Remove quotes if present - if ( - (valueStr.startsWith('"') && valueStr.endsWith('"')) || - (valueStr.startsWith("'") && valueStr.endsWith("'")) - ) { - value = valueStr.slice(1, -1); - } else { - value = valueStr; - } + value = valueStr; } frontmatter[key] = value; @@ -417,8 +453,18 @@ const processFileContent = async ({ rawContent: string; filePath: string; }): Promise<{ file: TFile; error?: string }> => { + // 1. Create or update the file with the fetched content first + let file: TFile | null = plugin.app.vault.getFileByPath(filePath); + if (!file) { + file = await plugin.app.vault.create(filePath, rawContent); + } else { + await plugin.app.vault.modify(file, rawContent); + } + + // 2. Parse frontmatter from rawContent (metadataCache is updated async and is + // often empty immediately after create/modify), then map nodeTypeId and update frontmatter. const { frontmatter } = parseFrontmatter(rawContent); - const sourceNodeTypeId = frontmatter.nodeTypeId; + const sourceNodeTypeId = frontmatter.nodeTypeId as string | undefined; let mappedNodeTypeId: string | undefined; if (sourceNodeTypeId && typeof sourceNodeTypeId === "string") { @@ -430,12 +476,6 @@ const processFileContent = async ({ }); } - let file: TFile | null = plugin.app.vault.getFileByPath(filePath); - if (!file) { - file = await plugin.app.vault.create(filePath, rawContent); - } else { - await plugin.app.vault.modify(file, rawContent); - } await plugin.app.fileManager.processFrontMatter(file, (fm) => { if (mappedNodeTypeId !== undefined) { (fm as Record).nodeTypeId = mappedNodeTypeId; @@ -449,9 +489,11 @@ const processFileContent = async ({ export const importSelectedNodes = async ({ plugin, selectedNodes, + onProgress, }: { plugin: DiscourseGraphPlugin; selectedNodes: ImportableNode[]; + onProgress?: (current: number, total: number) => void; }): Promise<{ success: number; failed: number }> => { const client = await getLoggedInClient(plugin); if (!client) { @@ -467,6 +509,8 @@ export const importSelectedNodes = async ({ let successCount = 0; let failedCount = 0; + let processedCount = 0; + const totalNodes = selectedNodes.length; // Group nodes by space to create folders efficiently const nodesBySpace = new Map(); @@ -483,7 +527,8 @@ export const importSelectedNodes = async ({ const importFolderPath = `import/${sanitizeFileName(spaceName)}`; // Ensure the folder exists - const folderExists = await plugin.app.vault.adapter.exists(importFolderPath); + const folderExists = + await plugin.app.vault.adapter.exists(importFolderPath); if (!folderExists) { await plugin.app.vault.createFolder(importFolderPath); } @@ -497,6 +542,8 @@ export const importSelectedNodes = async ({ node.spaceId, ); + console.log("existingFile", existingFile); + // Fetch the file name (direct variant) and content (full variant) const fileName = await fetchNodeContent({ client, @@ -510,6 +557,8 @@ export const importSelectedNodes = async ({ `No direct variant found for node ${node.nodeInstanceId}`, ); failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); continue; } @@ -523,6 +572,8 @@ export const importSelectedNodes = async ({ if (content === null) { console.warn(`No full variant found for node ${node.nodeInstanceId}`); failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); continue; } @@ -561,6 +612,8 @@ export const importSelectedNodes = async ({ result.error, ); failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); continue; } @@ -578,12 +631,13 @@ export const importSelectedNodes = async ({ } successCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); } catch (error) { - console.error( - `Error importing node ${node.nodeInstanceId}:`, - error, - ); + console.error(`Error importing node ${node.nodeInstanceId}:`, error); failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); } } } @@ -609,16 +663,35 @@ export const refreshImportedFile = async ({ throw new Error("Cannot get Supabase client"); } const cache = plugin.app.metadataCache.getFileCache(file); - const frontmatter = cache?.frontmatter; - const spaceName = await getSpaceName(supabaseClient, frontmatter?.importedFromSpaceId as number); - const result = await importSelectedNodes({ plugin, selectedNodes: [{ - nodeInstanceId: frontmatter?.nodeInstanceId as string, - title: file.basename, - spaceId: frontmatter?.importedFromSpaceId as number, - spaceName: spaceName, - groupId: frontmatter?.publishedToGroups?.[0] as string, - selected: false, - }] }); + const frontmatter = cache?.frontmatter as Record; + if ( + !frontmatter.importedFromSpaceId || + !frontmatter.nodeInstanceId || + !frontmatter.publishedToGroups + ) { + return { + success: false, + error: + "Missing frontmatter: importedFromSpaceId or nodeInstanceId or publishedToGroups", + }; + } + const spaceName = await getSpaceName( + supabaseClient, + frontmatter.importedFromSpaceId as number, + ); + const result = await importSelectedNodes({ + plugin, + selectedNodes: [ + { + nodeInstanceId: frontmatter.nodeInstanceId as string, + title: file.basename, + spaceId: frontmatter.importedFromSpaceId as number, + spaceName: spaceName, + groupId: (frontmatter.publishedToGroups as string[])[0] ?? "", + selected: false, + }, + ], + }); return { success: result.success > 0, error: result.failed > 0 ? "Failed to refresh imported file" : undefined }; }; From 58e2099c890abb5d6d425930663c9784155cf27f Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Mon, 26 Jan 2026 17:54:53 -0500 Subject: [PATCH 05/33] address PR comments --- apps/obsidian/TEST_PLAN_IMPORT.md | 789 ----------------------- apps/obsidian/src/utils/importNodes.ts | 30 +- packages/database/scripts/createGroup.ts | 176 +++++ 3 files changed, 189 insertions(+), 806 deletions(-) delete mode 100644 apps/obsidian/TEST_PLAN_IMPORT.md create mode 100644 packages/database/scripts/createGroup.ts diff --git a/apps/obsidian/TEST_PLAN_IMPORT.md b/apps/obsidian/TEST_PLAN_IMPORT.md deleted file mode 100644 index eb7266c80..000000000 --- a/apps/obsidian/TEST_PLAN_IMPORT.md +++ /dev/null @@ -1,789 +0,0 @@ -# Import Functionality Test Plan - -## Overview -This document outlines comprehensive test cases for the node import and refresh functionality in the Obsidian Discourse Graph plugin. - ---- - -## 1. Basic Import Functionality - -### 1.1 Happy Path - Single Node Import -**Test Case**: Import a single node from another space -- **Prerequisites**: - - User is member of at least one group - - Group has published nodes - - Node is not already imported -- **Steps**: - 1. Open "Import nodes from another space" command - 2. Select one node from the list - 3. Click "Import" -- **Expected**: - - File created in `import/[spaceName]/[nodeTitle].md` - - File contains correct frontmatter with `nodeInstanceId` and `importedFromSpaceId` - - File content matches database "full" variant - - File title matches database "direct" variant - - Success notice displayed - -### 1.2 Multiple Nodes Import -**Test Case**: Import multiple nodes from same space -- **Steps**: - 1. Open import modal - 2. Select 3-5 nodes from same space - 3. Click "Import" -- **Expected**: - - All files created in same `import/[spaceName]/` folder - - All files have correct metadata - - Success count matches number of selected nodes - -### 1.3 Multiple Nodes from Different Spaces -**Test Case**: Import nodes from multiple spaces simultaneously -- **Steps**: - 1. Select nodes from 2-3 different spaces - 2. Click "Import" -- **Expected**: - - Files organized in separate folders: `import/[space1]/`, `import/[space2]/`, etc. - - Each file has correct `importedFromSpaceId` matching its source space - -### 1.4 Import All Nodes from Group -**Test Case**: Select and import all available nodes -- **Steps**: - 1. Use "Select All" if available - 2. Import all nodes -- **Expected**: - - All nodes imported successfully - - No duplicates created - - Performance is acceptable (no timeout) - ---- - -## 2. Node Type Mapping - -### 2.1 Matching Local Node Type -**Test Case**: Import node with nodeTypeId that matches local node type by name -- **Prerequisites**: - - Source space has node type "Question" with ID `node_abc123` - - Local vault has node type "Question" with ID `node_xyz789` -- **Steps**: Import a node of type "Question" -- **Expected**: - - Imported file has `nodeTypeId: node_xyz789` (local ID) - - Not `node_abc123` (source ID) - - File displays correctly with local node type formatting - -### 2.2 Non-Matching Node Type - Auto-Create -**Test Case**: Import node with nodeTypeId that doesn't exist locally -- **Prerequisites**: - - Source space has node type "Claim" with specific format/color - - Local vault doesn't have "Claim" node type -- **Steps**: Import a "Claim" node -- **Expected**: - - New local node type "Claim" is created - - New node type has same name, format, color, tag, template, keyImage as source - - New node type appears in settings - - Imported file uses the newly created node type ID - -### 2.3 Node Type with Missing Schema Data -**Test Case**: Import node where source schema can't be fetched -- **Prerequisites**: - - Node has `nodeTypeId` but schema not published/accessible -- **Steps**: Import the node -- **Expected**: - - Import proceeds with original `nodeTypeId` (no mapping) - - Or creates node type with fallback values - - Error logged but import doesn't fail completely - -### 2.4 Node Type with Complex Literal Content -**Test Case**: Import node type with nested `literal_content` structure -- **Prerequisites**: - - Schema has `literal_content` with nested `source_data` object -- **Steps**: Import node of this type -- **Expected**: - - Node type created correctly with all nested properties extracted - - Format, color, tag, template, keyImage all preserved - -### 2.5 Node Type with Flat Literal Content -**Test Case**: Import node type with flat `literal_content` structure -- **Prerequisites**: - - Schema has flat `literal_content` (no `source_data` nesting) -- **Steps**: Import node of this type -- **Expected**: - - Node type created correctly from flat structure - - All properties extracted properly - ---- - -## 3. File Existence and Updates - -### 3.1 Import Existing File - Update -**Test Case**: Import node that already exists (same nodeInstanceId + importedFromSpaceId) -- **Prerequisites**: - - File already exists with matching `nodeInstanceId` and `importedFromSpaceId` -- **Steps**: Import the same node again -- **Expected**: - - Existing file is updated (not duplicated) - - File content refreshed from database - - File remains in same location - - No duplicate files created - -### 3.2 Import with Title Change -**Test Case**: Refresh imported file where title changed in source -- **Prerequisites**: - - File exists: `import/Space1/Old Title.md` - - Source title changed to "New Title" -- **Steps**: Refresh the file -- **Expected**: - - File renamed to `import/Space1/New Title.md` - - Content updated - - Frontmatter preserved (`nodeInstanceId`, `importedFromSpaceId`) - -### 3.3 Import with Path Conflict (Different nodeInstanceId) -**Test Case**: Import node where target path exists but different nodeInstanceId -- **Prerequisites**: - - File exists: `import/Space1/Title.md` with different `nodeInstanceId` - - Importing node with same title but different `nodeInstanceId` -- **Steps**: Import the node -- **Expected**: - - New file created as `import/Space1/Title (1).md` - - Both files coexist correctly - - Each has correct `nodeInstanceId` - -### 3.4 File in Wrong Location -**Test Case**: File exists but in wrong folder (e.g., root instead of import/) -- **Prerequisites**: - - File exists at root: `Title.md` with matching `nodeInstanceId` + `importedFromSpaceId` -- **Steps**: Import/refresh the node -- **Expected**: - - File moved to correct `import/[spaceName]/` folder - - Or file updated in place (depending on implementation) - ---- - -## 4. Space Name Handling - -### 4.1 Space Name with Special Characters -**Test Case**: Import from space with special characters in name -- **Prerequisites**: - - Space name: "Test Space / With-Special_Chars" -- **Steps**: Import nodes from this space -- **Expected**: - - Folder name sanitized: `import/Test Space With-Special_Chars/` - - Files created successfully - - No filesystem errors - -### 4.2 Space Name Fetch Failure -**Test Case**: Space name can't be fetched (RLS/permission issue) -- **Prerequisites**: - - User has access to nodes but not space metadata -- **Steps**: Import nodes from this space -- **Expected**: - - `ensureSpaceNames` retries and handles gracefully - - Uses fallback: `import/space-[id]/` or throws descriptive error - - Import proceeds if possible - -### 4.3 Space Name with Very Long Name -**Test Case**: Import from space with very long name -- **Prerequisites**: - - Space name > 200 characters -- **Steps**: Import nodes -- **Expected**: - - Folder name truncated or handled appropriately - - No filesystem path length errors - -### 4.4 Multiple Spaces with Similar Names -**Test Case**: Import from spaces with similar/duplicate names -- **Prerequisites**: - - Two spaces both named "Test Space" -- **Steps**: Import from both -- **Expected**: - - Files organized correctly (may need space ID in folder name) - - No conflicts or overwrites - ---- - -## 5. DataCore Integration - -### 5.1 DataCore Available -**Test Case**: Find existing file with DataCore enabled -- **Prerequisites**: - - DataCore plugin installed and enabled - - File exists with matching `nodeInstanceId` + `importedFromSpaceId` -- **Steps**: Import/refresh node -- **Expected**: - - `findExistingImportedFile` uses DataCore query - - Fast lookup (no full vault scan) - - File found correctly - -### 5.2 DataCore Unavailable -**Test Case**: Find existing file without DataCore -- **Prerequisites**: - - DataCore plugin not installed or disabled -- **Steps**: Import/refresh node -- **Expected**: - - Falls back gracefully (returns null) - - Import still works (creates new file) - - No errors thrown - -### 5.3 DataCore Query Error -**Test Case**: DataCore query throws error -- **Prerequisites**: - - DataCore installed but query fails -- **Steps**: Import/refresh node -- **Expected**: - - Error caught and logged - - Falls back gracefully - - Import continues - -### 5.4 DataCore Returns Multiple Matches -**Test Case**: DataCore query returns multiple files (shouldn't happen but test) -- **Prerequisites**: - - Multiple files with same `nodeInstanceId` + `importedFromSpaceId` (data inconsistency) -- **Steps**: Find existing file -- **Expected**: - - Returns first match - - Warning logged about multiple matches - - Import proceeds with first file - ---- - -## 6. Database and Network Errors - -### 6.1 Missing Direct Variant -**Test Case**: Node exists but "direct" variant missing -- **Prerequisites**: - - Node in database without "direct" variant content -- **Steps**: Import node -- **Expected**: - - Import fails for this node - - Error message: "No direct variant found" - - Other nodes still import successfully - - Failed count incremented - -### 6.2 Missing Full Variant -**Test Case**: Node exists but "full" variant missing -- **Prerequisites**: - - Node in database without "full" variant content -- **Steps**: Import node -- **Expected**: - - Import fails for this node - - Error message: "No full variant found" - - Other nodes still import successfully - -### 6.3 Database Connection Error -**Test Case**: Database connection fails during import -- **Prerequisites**: - - Network disconnected or database unavailable -- **Steps**: Attempt import -- **Expected**: - - Error caught and displayed to user - - Partial imports rolled back or marked as failed - - Clear error message - -### 6.4 RLS Policy Blocking Access -**Test Case**: User doesn't have permission to access node content -- **Prerequisites**: - - Node exists but RLS policy blocks access -- **Steps**: Attempt import -- **Expected**: - - Query returns null/empty - - Import fails for this node - - Error logged appropriately - - Other accessible nodes still import - -### 6.5 Schema Not Published -**Test Case**: Node type schema not published to group -- **Prerequisites**: - - Node exists but its schema not accessible via ResourceAccess -- **Steps**: Import node -- **Expected**: - - Node type mapping fails gracefully - - Uses original `nodeTypeId` or creates with fallback - - Import still succeeds - ---- - -## 7. Frontmatter Handling - -### 7.1 File with Only Frontmatter (No Body) -**Test Case**: Import node with only YAML frontmatter, no body content -- **Prerequisites**: - - Source content: `---\nnodeTypeId: ...\n---\n` (no body) -- **Steps**: Import node -- **Expected**: - - File created correctly - - Frontmatter parsed properly - - Empty body handled gracefully - -### 7.2 Frontmatter with Special Characters -**Test Case**: Import node with special characters in frontmatter values -- **Prerequisites**: - - Frontmatter contains quotes, colons, newlines in values -- **Steps**: Import node -- **Expected**: - - Frontmatter preserved correctly - - No parsing errors - - Special characters escaped properly - -### 7.3 Frontmatter with Arrays -**Test Case**: Import node with array values in frontmatter -- **Prerequisites**: - - Frontmatter contains `publishedToGroups: [id1, id2]` -- **Steps**: Import node -- **Expected**: - - Arrays preserved in frontmatter - - `publishedToGroups` removed (as per implementation) - - Other arrays handled correctly - -### 7.4 Missing nodeInstanceId in Source -**Test Case**: Source content missing nodeInstanceId in frontmatter -- **Prerequisites**: - - Database content has frontmatter without `nodeInstanceId` -- **Steps**: Import node -- **Expected**: - - `nodeInstanceId` added from import process - - File created with correct `nodeInstanceId` - -### 7.5 Frontmatter Update After Import -**Test Case**: Verify frontmatter is correctly set after import -- **Steps**: - 1. Import node - 2. Check file frontmatter -- **Expected**: - - `nodeInstanceId` matches source - - `importedFromSpaceId` matches source space - - `nodeTypeId` is mapped to local ID - - `publishedToGroups` removed - - Other frontmatter preserved - ---- - -## 8. Refresh Functionality - -### 8.1 Refresh Single File -**Test Case**: Refresh one imported file -- **Prerequisites**: - - File exists with `importedFromSpaceId` and `nodeInstanceId` - - Source content updated in database -- **Steps**: - 1. Open file in Discourse Context view - 2. Click "🔄 Refresh" button -- **Expected**: - - File content updated from database - - File title updated if changed - - Success notice displayed - - Frontmatter preserved - -### 8.2 Refresh All Files -**Test Case**: Refresh all imported files via command -- **Prerequisites**: - - Multiple imported files exist - - Some have updates in database -- **Steps**: - 1. Run "Fetch latest content from imported nodes" command - 2. Wait for completion -- **Expected**: - - All files refreshed - - Success/failed counts displayed - - Errors logged for failed files - - Notice shows summary - -### 8.3 Refresh File with Missing Metadata -**Test Case**: Refresh file missing `importedFromSpaceId` or `nodeInstanceId` -- **Prerequisites**: - - File exists but missing required frontmatter -- **Steps**: Attempt refresh -- **Expected**: - - Error returned: "File is not an imported file" - - No database queries attempted - - File not modified - -### 8.4 Refresh File Not in Database -**Test Case**: Refresh file where source node deleted from database -- **Prerequisites**: - - File exists locally - - Source node no longer in database -- **Steps**: Attempt refresh -- **Expected**: - - Error: "Could not fetch latest content" - - File not modified - - Error logged - -### 8.5 Refresh During Import -**Test Case**: Refresh file while import is in progress -- **Prerequisites**: - - Import operation running -- **Steps**: - 1. Start import - 2. Immediately try to refresh a file being imported -- **Expected**: - - Operations don't conflict - - Both complete successfully - - Or refresh waits/queues appropriately - ---- - -## 9. UI/UX Scenarios - -### 9.1 Modal Loading States -**Test Case**: Verify loading states in ImportNodesModal -- **Steps**: - 1. Open import modal - 2. Observe loading state - 3. Wait for nodes to load -- **Expected**: - - Loading indicator shown - - Nodes appear when ready - - No flickering or empty states - -### 9.2 Empty Groups -**Test Case**: User has groups but no published nodes -- **Prerequisites**: - - User is member of groups - - Groups have no published nodes -- **Steps**: Open import modal -- **Expected**: - - Empty state message displayed - - No errors thrown - - Modal can be closed - -### 9.3 No Groups -**Test Case**: User is not member of any groups -- **Prerequisites**: - - User has no group memberships -- **Steps**: Open import modal -- **Expected**: - - Empty state or error message - - Clear indication of why no nodes available - -### 9.4 Large Node Lists -**Test Case**: Import modal with 100+ nodes -- **Prerequisites**: - - Groups have 100+ published nodes -- **Steps**: - 1. Open import modal - 2. Scroll through list - 3. Select multiple nodes -- **Expected**: - - List renders efficiently - - Scrolling is smooth - - Selection works correctly - - Import completes in reasonable time - -### 9.5 Filtering/Search (if implemented) -**Test Case**: Search/filter nodes in import modal -- **Steps**: - 1. Open import modal - 2. Use search/filter if available -- **Expected**: - - Filtering works correctly - - Results update in real-time - - Selection state preserved - -### 9.6 Selection State Persistence -**Test Case**: Selection state during loading/refreshing -- **Steps**: - 1. Select nodes - 2. Trigger refresh/reload -- **Expected**: - - Selection preserved or cleared appropriately - - No state corruption - ---- - -## 10. File System Edge Cases - -### 10.1 Invalid File Names -**Test Case**: Import node with invalid characters in title -- **Prerequisites**: - - Source title: `File/With\Invalid:Chars<>|?*` -- **Steps**: Import node -- **Expected**: - - File name sanitized - - Invalid characters removed/replaced - - File created successfully - -### 10.2 Very Long File Names -**Test Case**: Import node with very long title (>255 chars) -- **Prerequisites**: - - Source title > 255 characters -- **Steps**: Import node -- **Expected**: - - File name truncated appropriately - - No filesystem errors - - File created successfully - -### 10.3 Reserved File Names -**Test Case**: Import node with reserved filename (e.g., "CON", "PRN" on Windows) -- **Prerequisites**: - - Source title is reserved name -- **Steps**: Import node -- **Expected**: - - File name sanitized/renamed - - No filesystem errors - -### 10.4 Folder Creation Permissions -**Test Case**: Import when import/ folder can't be created -- **Prerequisites**: - - Insufficient permissions or disk full -- **Steps**: Attempt import -- **Expected**: - - Error caught and displayed - - Clear error message - - Partial imports handled gracefully - -### 10.5 Disk Space Full -**Test Case**: Import when disk is full -- **Prerequisites**: - - Vault disk is full -- **Steps**: Attempt import -- **Expected**: - - Error caught - - Clear error message - - No partial/corrupted files - ---- - -## 11. Sync Integration - -### 11.1 Imported Files Not Synced -**Test Case**: Verify imported files don't sync back to database -- **Prerequisites**: - - File imported with `importedFromSpaceId` -- **Steps**: - 1. Modify imported file - 2. Wait for sync -- **Expected**: - - File changes not synced to database - - File not in sync queue - - `shouldSyncFile` returns false - -### 11.2 Imported File Without importedFromSpaceId -**Test Case**: Edge case - file missing `importedFromSpaceId` but should be imported -- **Prerequisites**: - - File in import/ folder but missing frontmatter -- **Steps**: Modify file -- **Expected**: - - File not synced (if in import/ folder) - - Or synced if not marked as imported - -### 11.3 Refresh Updates File - No Sync -**Test Case**: Refresh updates file, verify it doesn't trigger sync -- **Steps**: - 1. Refresh imported file - 2. Check sync queue -- **Expected**: - - File updated - - File not queued for sync - - `importedFromSpaceId` preserved - ---- - -## 12. Error Handling and Recovery - -### 12.1 Partial Import Failure -**Test Case**: Some nodes import successfully, others fail -- **Prerequisites**: - - 5 nodes selected - - 2 nodes have database errors -- **Steps**: Import all 5 -- **Expected**: - - 3 nodes imported successfully - - 2 nodes fail with errors - - Success/failed counts accurate - - Error messages for failed nodes - -### 12.2 Import Interruption -**Test Case**: Import interrupted (app closed, network lost) -- **Steps**: - 1. Start large import - 2. Interrupt mid-process -- **Expected**: - - Already imported files remain - - No corrupted files - - Can resume/retry remaining nodes - -### 12.3 Concurrent Imports -**Test Case**: Multiple import operations simultaneously -- **Steps**: - 1. Open two import modals - 2. Import from both -- **Expected**: - - Operations don't conflict - - Both complete successfully - - Or second operation waits/queues - -### 12.4 Error Messages Clarity -**Test Case**: Verify error messages are user-friendly -- **Steps**: Trigger various error conditions -- **Expected**: - - Error messages clear and actionable - - Technical details in console, user-friendly in UI - - No stack traces shown to user - ---- - -## 13. Performance Tests - -### 13.1 Large Batch Import -**Test Case**: Import 50+ nodes -- **Steps**: Import 50 nodes -- **Expected**: - - Completes in reasonable time (< 2 minutes) - - Progress indicator shows progress - - No memory leaks - - UI remains responsive - -### 13.2 DataCore Performance -**Test Case**: Compare performance with/without DataCore -- **Steps**: - 1. Import with DataCore enabled - 2. Import with DataCore disabled -- **Expected**: - - DataCore version significantly faster - - Both complete successfully - - No functional differences - -### 13.3 Refresh All Performance -**Test Case**: Refresh 100+ imported files -- **Steps**: Refresh all imported files -- **Expected**: - - Completes in reasonable time - - Progress shown - - No timeout errors - ---- - -## 14. Data Integrity - -### 14.1 NodeInstanceId Uniqueness -**Test Case**: Verify nodeInstanceId + importedFromSpaceId uniqueness -- **Prerequisites**: - - Two files with same combination (shouldn't happen) -- **Steps**: Import/refresh -- **Expected**: - - Only one file updated/created - - Warning logged if duplicates found - -### 14.2 Frontmatter Consistency -**Test Case**: Verify frontmatter consistency after import/refresh -- **Steps**: - 1. Import node - 2. Check frontmatter - 3. Refresh node - 4. Check frontmatter again -- **Expected**: - - `nodeInstanceId` never changes - - `importedFromSpaceId` never changes - - `nodeTypeId` mapped correctly and consistently - -### 14.3 Content Integrity -**Test Case**: Verify imported content matches source -- **Steps**: - 1. Import node - 2. Compare file content with database -- **Expected**: - - Content matches exactly - - No data loss or corruption - - Encoding handled correctly (UTF-8) - ---- - -## 15. Regression Tests - -### 15.1 Re-import After Delete -**Test Case**: Delete imported file, then re-import -- **Steps**: - 1. Import node - 2. Delete file - 3. Re-import same node -- **Expected**: - - File recreated successfully - - Same location and content - - No errors - -### 15.2 Import After Local Modification -**Test Case**: Import node, modify locally, then refresh -- **Steps**: - 1. Import node - 2. Manually edit file content - 3. Refresh file -- **Expected**: - - Local changes overwritten with database content - - Frontmatter preserved correctly - -### 15.3 Multiple Refresh Cycles -**Test Case**: Refresh same file multiple times -- **Steps**: - 1. Import node - 2. Refresh 5 times in succession -- **Expected**: - - Each refresh succeeds - - No errors accumulate - - File state consistent - ---- - -## Test Execution Checklist - -### Pre-Test Setup -- [ ] Create test spaces with various node types -- [ ] Create test groups and publish nodes -- [ ] Set up test vault with/without DataCore -- [ ] Prepare test data (nodes with various characteristics) -- [ ] Document expected database state - -### Test Execution -- [ ] Execute all test cases systematically -- [ ] Document actual results vs expected -- [ ] Capture screenshots for UI tests -- [ ] Log console errors/warnings -- [ ] Measure performance metrics - -### Post-Test Validation -- [ ] Verify no files left in inconsistent state -- [ ] Check database for any unintended changes -- [ ] Verify all imported files have correct metadata -- [ ] Clean up test files -- [ ] Document any bugs found - ---- - -## Priority Levels - -**P0 (Critical - Must Test)**: -- 1.1, 1.2, 1.3 (Basic import) -- 2.1, 2.2 (Node type mapping) -- 3.1, 3.2 (File updates) -- 6.1, 6.2 (Database errors) -- 8.1, 8.2 (Refresh functionality) -- 11.1 (Sync integration) - -**P1 (High - Should Test)**: -- 2.3, 2.4, 2.5 (Node type edge cases) -- 4.1, 4.2 (Space name handling) -- 5.1, 5.2 (DataCore) -- 7.1, 7.2 (Frontmatter) -- 9.1, 9.2 (UI states) -- 12.1, 12.2 (Error handling) - -**P2 (Medium - Nice to Test)**: -- 3.3, 3.4 (File conflicts) -- 4.3, 4.4 (Space name edge cases) -- 5.3, 5.4 (DataCore edge cases) -- 10.1-10.5 (File system) -- 13.1-13.3 (Performance) - -**P3 (Low - Optional)**: -- 9.3-9.6 (Advanced UI) -- 12.3, 12.4 (Advanced error handling) -- 14.1-14.3 (Data integrity) -- 15.1-15.3 (Regression) - ---- - -## Notes - -- Test in both development and production-like environments -- Test with various Obsidian versions if applicable -- Test with different vault sizes (small, medium, large) -- Test with different network conditions (fast, slow, intermittent) -- Consider automated testing for critical paths -- Maintain test data sets for consistent testing diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 0f6b6e5cd..55f732820 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -4,7 +4,6 @@ import type { DGSupabaseClient } from "@repo/database/lib/client"; import type DiscourseGraphPlugin from "~/index"; import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; import type { DiscourseNode, ImportableNode } from "~/types"; -import generateUid from "~/utils/generateUid"; import { QueryEngine } from "~/services/QueryEngine"; export const getAvailableGroups = async ( @@ -132,9 +131,9 @@ export const getPublishedNodesForGroups = async ({ return nodes; }; -export const getLocalNodeInstanceIds = async ( +export const getLocalNodeInstanceIds = ( plugin: DiscourseGraphPlugin, -): Promise> => { +): Set => { const allFiles = plugin.app.vault.getMarkdownFiles(); const nodeInstanceIds = new Set(); @@ -263,23 +262,21 @@ type ParsedFrontmatter = { [key: string]: unknown; }; -const parseFrontmatter = (content: string): { +const parseFrontmatter = ( + content: string, +): { frontmatter: ParsedFrontmatter; body: string; } => { - // Updated regex to handle files with only frontmatter (no body content) - // The body is optional - it can be empty or not exist at all // Pattern: ---\n(frontmatter)\n---\n(body - optional) const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*(?:\n([\s\S]*))?$/; const match = content.match(frontmatterRegex); if (!match || !match[1]) { - // No frontmatter, return empty frontmatter and full content as body return { frontmatter: {}, body: content }; } const frontmatterText = match[1]; - // Body is optional - if there's no body content, match[2] will be undefined const body = match[2] ?? ""; // Parse YAML-like frontmatter (simple parser for key: value pairs) @@ -423,14 +420,18 @@ const mapNodeTypeIdToLocal = async ({ schemaName, ); + const now = new Date().getTime(); + const newNodeType: DiscourseNode = { - id: generateUid("node"), + id: sourceNodeTypeId, name: parsed.name, format: parsed.format, color: parsed.color, tag: parsed.tag, template: parsed.template, keyImage: parsed.keyImage, + created: now, + modified: now, }; plugin.settings.nodeTypes = [ ...plugin.settings.nodeTypes, @@ -464,7 +465,7 @@ const processFileContent = async ({ // 2. Parse frontmatter from rawContent (metadataCache is updated async and is // often empty immediately after create/modify), then map nodeTypeId and update frontmatter. const { frontmatter } = parseFrontmatter(rawContent); - const sourceNodeTypeId = frontmatter.nodeTypeId as string | undefined; + const sourceNodeTypeId = frontmatter.nodeTypeId; let mappedNodeTypeId: string | undefined; if (sourceNodeTypeId && typeof sourceNodeTypeId === "string") { @@ -664,15 +665,10 @@ export const refreshImportedFile = async ({ } const cache = plugin.app.metadataCache.getFileCache(file); const frontmatter = cache?.frontmatter as Record; - if ( - !frontmatter.importedFromSpaceId || - !frontmatter.nodeInstanceId || - !frontmatter.publishedToGroups - ) { + if (!frontmatter.importedFromSpaceId || !frontmatter.nodeInstanceId) { return { success: false, - error: - "Missing frontmatter: importedFromSpaceId or nodeInstanceId or publishedToGroups", + error: "Missing frontmatter: importedFromSpaceId or nodeInstanceId", }; } const spaceName = await getSpaceName( diff --git a/packages/database/scripts/createGroup.ts b/packages/database/scripts/createGroup.ts new file mode 100644 index 000000000..1d3fba1a8 --- /dev/null +++ b/packages/database/scripts/createGroup.ts @@ -0,0 +1,176 @@ +/** + * Create a group and/or add member UUIDs to group_membership. + * + * Requires: SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY in .env (or environment) + * + * Usage: + * pnpm run create-group-and-members [uuid1] [uuid2] ... + * + * - If the group already exists (by name: {name}@groups.discoursegraphs.com): adds the + * given UUIDs to group_membership. If a UUID is already in the group, it is ignored. + * - If the group does not exist: creates it and adds the UUIDs (first one becomes admin). + * - With an existing group, memberUids can be empty (no-op). + * - When creating a new group, at least one member UUID is required. + */ + +import { createClient } from "@supabase/supabase-js"; +import dotenv from "dotenv"; +import type { Database } from "@repo/database/dbTypes"; + +dotenv.config(); + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const isUuid = (s: string): boolean => UUID_RE.test(s); + +const parseArgs = (): { groupName: string; memberUids: string[] } => { + const argv = process.argv.slice(2); + const groupName = argv[0]; + const memberUids = argv.slice(1); + + if (!groupName) { + console.error( + "Usage: create-group-and-members [uuid1] [uuid2] ...", + ); + process.exit(1); + } + + const invalid = memberUids.filter((u) => !isUuid(u)); + if (invalid.length > 0) { + console.error("Invalid UUID(s):", invalid.join(", ")); + process.exit(1); + } + + return { groupName, memberUids }; +}; + +const groupEmail = (name: string): string => + `${name}@groups.discoursegraphs.com`; + +const findExistingGroup = async ( + supabase: ReturnType>, + groupName: string, +): Promise<{ id: string } | null> => { + const email = groupEmail(groupName); + let page = 1; + const perPage = 1000; + + while (true) { + const { data, error } = await supabase.auth.admin.listUsers({ + page, + perPage, + }); + + if (error) { + console.error("Failed to list users:", error.message); + process.exit(1); + } + + const users = data?.users ?? []; + const found = users.find((u) => u.email === email); + if (found) return { id: found.id }; + + if (users.length < perPage) return null; + page += 1; + } +}; + +const main = async (): Promise => { + const { groupName, memberUids } = parseArgs(); + + const url = process.env.SUPABASE_URL; + const serviceKey = process.env.SUPABASE_SERVICE_ROLE_KEY; + if (!url || !serviceKey) { + console.error( + "Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY in .env or environment", + ); + process.exit(1); + } + + const supabase = createClient(url, serviceKey, { + auth: { persistSession: false }, + }); + + const existing = await findExistingGroup(supabase, groupName); + console.log("existing", existing); + let resolvedGroupId: string; + let isNewGroup: boolean; + + if (existing) { + resolvedGroupId = existing.id; + isNewGroup = false; + console.log("Using existing group:", resolvedGroupId, `(${groupName})`); + } else { + if (memberUids.length === 0) { + console.error( + "When creating a new group, provide at least one member UUID.", + ); + process.exit(1); + } + + const email = groupEmail(groupName); + const password = crypto.randomUUID(); + + const { data, error } = await supabase.auth.admin.createUser({ + email, + password, + role: "anon", + user_metadata: { group: true }, + email_confirm: false, + }); + + if (error) { + if ((error as { code?: string }).code === "email_exists") { + console.error("A group with this name already exists:", email); + process.exit(1); + } + console.error("Failed to create group user:", error.message); + process.exit(1); + } + + if (!data.user) { + console.error("Failed to create group user: no user returned"); + process.exit(1); + } + + resolvedGroupId = data.user.id; + isNewGroup = true; + console.log("Created group:", resolvedGroupId, `(${groupName})`); + } + + let added = 0; + let skipped = 0; + + for (let i = 0; i < memberUids.length; i++) { + const member_id = memberUids[i]!; + const admin = isNewGroup && i === 0; + + const { error } = await supabase.from("group_membership").insert({ + group_id: resolvedGroupId, + member_id, + admin, + }); + + if (error) { + if (error.code === "23505") { + skipped += 1; + } else { + console.error("Failed to insert member", member_id, ":", error.message); + process.exit(1); + } + } else { + added += 1; + } + } + + console.log("Added", added, "member(s) to group_membership."); + if (skipped > 0) { + console.log("Skipped", skipped, "member(s) already in the group."); + } + if (isNewGroup && added > 0) { + console.log("Admin:", memberUids[0]); + } +}; + +main(); From 5ad8ac810fe9ff48b93c9a4bd8f11b3066fcee39 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Mon, 26 Jan 2026 17:55:44 -0500 Subject: [PATCH 06/33] remove local test file --- packages/database/scripts/createGroup.ts | 176 ----------------------- 1 file changed, 176 deletions(-) delete mode 100644 packages/database/scripts/createGroup.ts diff --git a/packages/database/scripts/createGroup.ts b/packages/database/scripts/createGroup.ts deleted file mode 100644 index 1d3fba1a8..000000000 --- a/packages/database/scripts/createGroup.ts +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Create a group and/or add member UUIDs to group_membership. - * - * Requires: SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY in .env (or environment) - * - * Usage: - * pnpm run create-group-and-members [uuid1] [uuid2] ... - * - * - If the group already exists (by name: {name}@groups.discoursegraphs.com): adds the - * given UUIDs to group_membership. If a UUID is already in the group, it is ignored. - * - If the group does not exist: creates it and adds the UUIDs (first one becomes admin). - * - With an existing group, memberUids can be empty (no-op). - * - When creating a new group, at least one member UUID is required. - */ - -import { createClient } from "@supabase/supabase-js"; -import dotenv from "dotenv"; -import type { Database } from "@repo/database/dbTypes"; - -dotenv.config(); - -const UUID_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -const isUuid = (s: string): boolean => UUID_RE.test(s); - -const parseArgs = (): { groupName: string; memberUids: string[] } => { - const argv = process.argv.slice(2); - const groupName = argv[0]; - const memberUids = argv.slice(1); - - if (!groupName) { - console.error( - "Usage: create-group-and-members [uuid1] [uuid2] ...", - ); - process.exit(1); - } - - const invalid = memberUids.filter((u) => !isUuid(u)); - if (invalid.length > 0) { - console.error("Invalid UUID(s):", invalid.join(", ")); - process.exit(1); - } - - return { groupName, memberUids }; -}; - -const groupEmail = (name: string): string => - `${name}@groups.discoursegraphs.com`; - -const findExistingGroup = async ( - supabase: ReturnType>, - groupName: string, -): Promise<{ id: string } | null> => { - const email = groupEmail(groupName); - let page = 1; - const perPage = 1000; - - while (true) { - const { data, error } = await supabase.auth.admin.listUsers({ - page, - perPage, - }); - - if (error) { - console.error("Failed to list users:", error.message); - process.exit(1); - } - - const users = data?.users ?? []; - const found = users.find((u) => u.email === email); - if (found) return { id: found.id }; - - if (users.length < perPage) return null; - page += 1; - } -}; - -const main = async (): Promise => { - const { groupName, memberUids } = parseArgs(); - - const url = process.env.SUPABASE_URL; - const serviceKey = process.env.SUPABASE_SERVICE_ROLE_KEY; - if (!url || !serviceKey) { - console.error( - "Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY in .env or environment", - ); - process.exit(1); - } - - const supabase = createClient(url, serviceKey, { - auth: { persistSession: false }, - }); - - const existing = await findExistingGroup(supabase, groupName); - console.log("existing", existing); - let resolvedGroupId: string; - let isNewGroup: boolean; - - if (existing) { - resolvedGroupId = existing.id; - isNewGroup = false; - console.log("Using existing group:", resolvedGroupId, `(${groupName})`); - } else { - if (memberUids.length === 0) { - console.error( - "When creating a new group, provide at least one member UUID.", - ); - process.exit(1); - } - - const email = groupEmail(groupName); - const password = crypto.randomUUID(); - - const { data, error } = await supabase.auth.admin.createUser({ - email, - password, - role: "anon", - user_metadata: { group: true }, - email_confirm: false, - }); - - if (error) { - if ((error as { code?: string }).code === "email_exists") { - console.error("A group with this name already exists:", email); - process.exit(1); - } - console.error("Failed to create group user:", error.message); - process.exit(1); - } - - if (!data.user) { - console.error("Failed to create group user: no user returned"); - process.exit(1); - } - - resolvedGroupId = data.user.id; - isNewGroup = true; - console.log("Created group:", resolvedGroupId, `(${groupName})`); - } - - let added = 0; - let skipped = 0; - - for (let i = 0; i < memberUids.length; i++) { - const member_id = memberUids[i]!; - const admin = isNewGroup && i === 0; - - const { error } = await supabase.from("group_membership").insert({ - group_id: resolvedGroupId, - member_id, - admin, - }); - - if (error) { - if (error.code === "23505") { - skipped += 1; - } else { - console.error("Failed to insert member", member_id, ":", error.message); - process.exit(1); - } - } else { - added += 1; - } - } - - console.log("Added", added, "member(s) to group_membership."); - if (skipped > 0) { - console.log("Skipped", skipped, "member(s) already in the group."); - } - if (isNewGroup && added > 0) { - console.log("Admin:", memberUids[0]); - } -}; - -main(); From 29334624283429edc452caf58ecf746b222aa2da Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Tue, 27 Jan 2026 14:12:08 -0500 Subject: [PATCH 07/33] curr progress --- apps/obsidian/src/utils/importNodes.ts | 475 ++++++++++++++++++++++++- 1 file changed, 472 insertions(+), 3 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 55f732820..43961beae 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { TFile } from "obsidian"; +import { App, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import type DiscourseGraphPlugin from "~/index"; import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; @@ -246,6 +246,432 @@ export const fetchNodeMetadata = async ({ }; }; +const fetchFileReferences = async ({ + client, + spaceId, + nodeInstanceId, +}: { + client: DGSupabaseClient; + spaceId: number; + nodeInstanceId: string; +}): Promise< + Array<{ + filepath: string; + filehash: string; + created: string; + last_modified: string; + }> +> => { + const { data, error } = await client + .from("FileReference") + .select("filepath, filehash, created, last_modified") + .eq("space_id", spaceId) + .eq("source_local_id", nodeInstanceId); + + if (error) { + console.error("Error fetching file references:", error); + return []; + } + + return data || []; +}; + +const downloadFileFromStorage = async ({ + client, + filehash, +}: { + client: DGSupabaseClient; + filehash: string; +}): Promise => { + try { + const { data, error } = await client.storage + .from("assets") + .download(filehash); + + if (error) { + console.warn(`Error downloading file ${filehash}:`, error); + return null; + } + + if (!data) { + console.warn(`No data returned for file ${filehash}`); + return null; + } + + return await data.arrayBuffer(); + } catch (error) { + console.error(`Exception downloading file ${filehash}:`, error); + return null; + } +}; + +const extractFileName = (filepath: string): { name: string; ext: string } => { + // Handle paths like "attachments/image.png" or "folder/subfolder/file.jpg" + // Extract just the filename with extension + const parts = filepath.split("/"); + const fileName = parts[parts.length - 1] || filepath; + + // Split filename and extension + const lastDotIndex = fileName.lastIndexOf("."); + if (lastDotIndex === -1 || lastDotIndex === fileName.length - 1) { + // No extension or extension is empty + return { name: fileName, ext: "" }; + } + + const name = fileName.slice(0, lastDotIndex); + const ext = fileName.slice(lastDotIndex + 1); + + return { name, ext }; +}; + +const updateMarkdownAssetLinks = ({ + content, + oldPathToNewPath, + targetFile, + app, +}: { + content: string; + oldPathToNewPath: Map; + targetFile: TFile; + app: App; +}): string => { + if (oldPathToNewPath.size === 0) { + return content; + } + + // Create a set of all new paths for quick lookup + const newPaths = new Set(oldPathToNewPath.values()); + + // Create a map of old paths to new files for quick lookup + const oldPathToNewFile = new Map(); + for (const [oldPath, newPath] of oldPathToNewPath.entries()) { + const newFile = app.metadataCache.getFirstLinkpathDest( + newPath, + targetFile.path, + ); + if (newFile) { + oldPathToNewFile.set(oldPath, newFile); + } + } + + let updatedContent = content; + + // Helper to check if a link path matches an old path + const matchesOldPath = (linkPath: string, oldPath: string): boolean => { + // Exact match + if (linkPath === oldPath) return true; + + // Match by filename (handles relative paths) + const oldFileName = extractFileName(oldPath); + const linkFileName = extractFileName(linkPath); + if ( + oldFileName.name === linkFileName.name && + oldFileName.ext === linkFileName.ext + ) { + return true; + } + + // Match if linkPath ends with oldPath or vice versa (handles relative vs absolute) + if (linkPath.endsWith(oldPath) || oldPath.endsWith(linkPath)) { + return true; + } + + return false; + }; + + // Helper to find file for a link path, checking if it's one of our imported assets + const findImportedAssetFile = (linkPath: string): TFile | null => { + // Try to resolve the link + const resolvedFile = app.metadataCache.getFirstLinkpathDest( + linkPath, + targetFile.path, + ); + + if (resolvedFile && newPaths.has(resolvedFile.path)) { + // This file is one of our imported assets + return resolvedFile; + } + + // Also check if the resolved file is in an assets folder (user may have renamed it) + if (resolvedFile && resolvedFile.path.includes("/assets/")) { + // Check if any of our new files match this one (by checking if path is similar) + for (const newPath of newPaths) { + const newFile = app.metadataCache.getFirstLinkpathDest( + newPath, + targetFile.path, + ); + if (newFile && newFile.path === resolvedFile.path) { + return resolvedFile; + } + } + } + + return null; + }; + + // Match wiki links: [[path]] or [[path|alias]] + const wikiLinkRegex = /\[\[([^\]]+)\]\]/g; + updatedContent = updatedContent.replace(wikiLinkRegex, (match, linkContent) => { + // Extract path and optional alias + const [linkPath, alias] = linkContent.split("|").map((s: string) => s.trim()); + + // Skip external URLs + if (linkPath.startsWith("http://") || linkPath.startsWith("https://")) { + return match; + } + + // First, try to find if this link resolves to one of our imported assets + const importedAssetFile = findImportedAssetFile(linkPath); + if (importedAssetFile) { + const linkText = app.metadataCache.fileToLinktext( + importedAssetFile, + targetFile.path, + ); + if (alias) { + return `[[${linkText}|${alias}]]`; + } + return `[[${linkText}]]`; + } + + // Fallback: Find matching old path + for (const [oldPath, newFile] of oldPathToNewFile.entries()) { + if (matchesOldPath(linkPath, oldPath)) { + const linkText = app.metadataCache.fileToLinktext(newFile, targetFile.path); + if (alias) { + return `[[${linkText}|${alias}]]`; + } + return `[[${linkText}]]`; + } + } + + return match; + }); + + // Match markdown image links: ![alt](path) or ![alt](path "title") + const markdownImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; + updatedContent = updatedContent.replace( + markdownImageRegex, + (match, alt, linkPath) => { + // Remove optional title from linkPath: "path" or "path title" + const cleanPath = linkPath.replace(/\s+"[^"]*"$/, "").trim(); + + // Skip external URLs + if (cleanPath.startsWith("http://") || cleanPath.startsWith("https://")) { + return match; + } + + // First, try to find if this link resolves to one of our imported assets + const importedAssetFile = findImportedAssetFile(cleanPath); + if (importedAssetFile) { + const linkText = app.metadataCache.fileToLinktext( + importedAssetFile, + targetFile.path, + ); + return `![${alt}](${linkText})`; + } + + // Fallback: Find matching old path + for (const [oldPath, newFile] of oldPathToNewFile.entries()) { + if (matchesOldPath(cleanPath, oldPath)) { + const linkText = app.metadataCache.fileToLinktext(newFile, targetFile.path); + return `![${alt}](${linkText})`; + } + } + + return match; + }, + ); + + return updatedContent; +}; + + +const importAssetsForNode = async ({ + plugin, + client, + spaceId, + nodeInstanceId, + spaceName, + targetMarkdownFile, +}: { + plugin: DiscourseGraphPlugin; + client: DGSupabaseClient; + spaceId: number; + nodeInstanceId: string; + spaceName: string; + targetMarkdownFile: TFile; +}): Promise<{ + success: boolean; + pathMapping: Map; // old path -> new path + errors: string[]; +}> => { + const pathMapping = new Map(); + const errors: string[] = []; + + // Fetch FileReference records for the node + const fileReferences = await fetchFileReferences({ + client, + spaceId, + nodeInstanceId, + }); + + if (fileReferences.length === 0) { + return { success: true, pathMapping, errors }; + } + + const importFolderPath = `import/${sanitizeFileName(spaceName)}`; + const assetsFolderPath = `${importFolderPath}/assets`; + + // Ensure assets folder exists + const assetsFolderExists = + await plugin.app.vault.adapter.exists(assetsFolderPath); + if (!assetsFolderExists) { + await plugin.app.vault.createFolder(assetsFolderPath); + } + + // Get existing asset mappings from frontmatter + const cache = plugin.app.metadataCache.getFileCache(targetMarkdownFile); + const frontmatter = (cache?.frontmatter as Record) || {}; + const importedAssetsRaw = frontmatter.importedAssets; + const importedAssets: Record = + importedAssetsRaw && typeof importedAssetsRaw === "object" && !Array.isArray(importedAssetsRaw) + ? (importedAssetsRaw as Record) + : {}; + // importedAssets format: { filehash: vaultPath } + + // Process each file reference + for (const fileRef of fileReferences) { + try { + const { filepath, filehash } = fileRef; + const { name, ext } = extractFileName(filepath); + + // Check if we already have a file for this hash + let existingAssetPath: string | undefined = importedAssets[filehash]; + let existingFile: TFile | null = null; + + if (existingAssetPath) { + // Check if the file still exists at the stored path + const file = plugin.app.vault.getAbstractFileByPath(existingAssetPath); + if (file && file instanceof TFile) { + existingFile = file; + } else { + // File was moved/renamed - search for it in the assets folder + // Try to find a file with the same name in the assets folder + const { name: fileName, ext: fileExt } = extractFileName(existingAssetPath); + const searchFileName = `${fileName}${fileExt ? `.${fileExt}` : ""}`; + + // Search all files in the assets folder + const allFiles = plugin.app.vault.getFiles(); + for (const vaultFile of allFiles) { + if ( + vaultFile instanceof TFile && + vaultFile.path.startsWith(assetsFolderPath) && + vaultFile.basename === fileName && + vaultFile.extension === fileExt + ) { + // Found a file with matching name in assets folder - likely the same file + existingFile = vaultFile; + existingAssetPath = vaultFile.path; + // Update frontmatter with new path + await plugin.app.fileManager.processFrontMatter( + targetMarkdownFile, + (fm) => { + const assetsRaw = (fm as Record).importedAssets; + const assets: Record = + assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) + ? (assetsRaw as Record) + : {}; + assets[filehash] = vaultFile.path; + (fm as Record).importedAssets = assets; + }, + ); + break; + } + } + } + } + + // If we found an existing file, reuse it + if (existingFile) { + pathMapping.set(filepath, existingFile.path); + console.log(`Reusing existing asset: ${filehash} -> ${existingFile.path}`); + continue; + } + + // No existing file found, need to download + // Determine target path + const sanitizedName = sanitizeFileName(name); + const sanitizedExt = ext ? `.${ext}` : ""; + const sanitizedFileName = `${sanitizedName}${sanitizedExt}`; + let targetPath = `${assetsFolderPath}/${sanitizedFileName}`; + + // Check if file already exists at target path (avoid duplicates) + if (await plugin.app.vault.adapter.exists(targetPath)) { + // File exists at expected path, reuse it + const file = plugin.app.vault.getAbstractFileByPath(targetPath); + if (file && file instanceof TFile) { + pathMapping.set(filepath, targetPath); + // Update frontmatter to track this mapping + await plugin.app.fileManager.processFrontMatter( + targetMarkdownFile, + (fm) => { + const assetsRaw = (fm as Record).importedAssets; + const assets: Record = + assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) + ? (assetsRaw as Record) + : {}; + assets[filehash] = targetPath; + (fm as Record).importedAssets = assets; + }, + ); + console.log(`Reusing existing file at path: ${targetPath}`); + continue; + } + } + + // File doesn't exist, download it + const fileContent = await downloadFileFromStorage({ + client, + filehash, + }); + + if (!fileContent) { + errors.push(`Failed to download file: ${filepath}`); + console.warn(`Failed to download file ${filepath} (hash: ${filehash})`); + continue; + } + + // Save file to vault + await plugin.app.vault.createBinary(targetPath, fileContent); + + // Update frontmatter to track this mapping + await plugin.app.fileManager.processFrontMatter(targetMarkdownFile, (fm) => { + const assetsRaw = (fm as Record).importedAssets; + const assets: Record = + assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) + ? (assetsRaw as Record) + : {}; + assets[filehash] = targetPath; + (fm as Record).importedAssets = assets; + }); + + // Track path mapping + pathMapping.set(filepath, targetPath); + console.log(`Imported asset: ${filepath} -> ${targetPath}`); + } catch (error) { + const errorMsg = `Error importing asset ${fileRef.filepath}: ${error}`; + errors.push(errorMsg); + console.error(errorMsg, error); + } + } + + return { + success: errors.length === 0 || pathMapping.size > 0, + pathMapping, + errors, + }; +}; + const sanitizeFileName = (fileName: string): string => { // Remove invalid characters for file names @@ -526,14 +952,22 @@ export const importSelectedNodes = async ({ for (const [spaceId, nodes] of nodesBySpace.entries()) { const spaceName = await getSpaceName(client, spaceId); const importFolderPath = `import/${sanitizeFileName(spaceName)}`; + const assetsFolderPath = `${importFolderPath}/assets`; - // Ensure the folder exists + // Ensure the import folder exists const folderExists = await plugin.app.vault.adapter.exists(importFolderPath); if (!folderExists) { await plugin.app.vault.createFolder(importFolderPath); } + // Ensure the assets folder exists + const assetsFolderExists = + await plugin.app.vault.adapter.exists(assetsFolderPath); + if (!assetsFolderExists) { + await plugin.app.vault.createFolder(assetsFolderPath); + } + // Process each node in this space for (const node of nodes) { try { @@ -618,8 +1052,43 @@ export const importSelectedNodes = async ({ continue; } - // If title changed and file exists, rename it to match the new title const processedFile = result.file; + + // Import assets for this node + const assetImportResult = await importAssetsForNode({ + plugin, + client, + spaceId: node.spaceId, + nodeInstanceId: node.nodeInstanceId, + spaceName, + targetMarkdownFile: processedFile, + }); + + // Update markdown content with new asset paths if assets were imported + if (assetImportResult.pathMapping.size > 0) { + const currentContent = await plugin.app.vault.read(processedFile); + const updatedContent = updateMarkdownAssetLinks({ + content: currentContent, + oldPathToNewPath: assetImportResult.pathMapping, + targetFile: processedFile, + app: plugin.app, + }); + + // Only update if content changed + if (updatedContent !== currentContent) { + await plugin.app.vault.modify(processedFile, updatedContent); + } + } + + // Log asset import errors if any + if (assetImportResult.errors.length > 0) { + console.warn( + `Some assets failed to import for node ${node.nodeInstanceId}:`, + assetImportResult.errors, + ); + } + + // If title changed and file exists, rename it to match the new title if (existingFile && processedFile.basename !== sanitizedFileName) { const newPath = `${importFolderPath}/${sanitizedFileName}.md`; let targetPath = newPath; From 546939788b0c1c9e6dcfca145eceb66bb30f76a0 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 5 Feb 2026 10:45:36 -0500 Subject: [PATCH 08/33] revert what is handled in #721 --- apps/obsidian/src/utils/conceptConversion.ts | 146 +----------------- .../src/utils/syncDgNodesToSupabase.ts | 58 +------ apps/roam/src/utils/conceptConversion.ts | 56 ++----- 3 files changed, 22 insertions(+), 238 deletions(-) diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index 1dffaf7e1..74e8853b6 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -1,13 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { TFile } from "obsidian"; -import type DiscourseGraphPlugin from "~/index"; -import type { - DiscourseNode, - DiscourseRelation, - DiscourseRelationType, -} from "~/types"; +import type { DiscourseNode } from "~/types"; import type { SupabaseContext } from "./supabaseContext"; -import type { DiscourseNodeInVault } from "./syncDgNodesToSupabase"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase"; import type { Json } from "@repo/database/dbTypes"; @@ -58,150 +52,21 @@ export const discourseNodeSchemaToLocalConcept = ({ }; }; -const STANDARD_ROLES = ["source", "destination"]; - -export const discourseRelationTypeToLocalConcept = ({ - context, - relationType, - accountLocalId, -}: { - context: SupabaseContext; - relationType: DiscourseRelationType; - accountLocalId: string; -}): LocalConceptDataInput => { - const { id, label, complement, created, modified, ...otherData } = - relationType; - return { - space_id: context.spaceId, - name: label, - source_local_id: id, - is_schema: true, - author_local_id: accountLocalId, - created: new Date(created).toISOString(), - last_modified: new Date(modified).toISOString(), - literal_content: { - label, - complement, - source_data: otherData, - } as unknown as Json, - }; -}; - -export const discourseRelationSchemaToLocalConcept = ({ - context, - relation, - accountLocalId, - nodeTypesById, - relationTypesById, -}: { - context: SupabaseContext; - relation: DiscourseRelation; - accountLocalId: string; - nodeTypesById: Record; - relationTypesById: Record; -}): LocalConceptDataInput => { - const { id, relationshipTypeId, sourceId, destinationId, created, modified } = - relation; - const sourceName = nodeTypesById[sourceId]?.name ?? sourceId; - const destinationName = nodeTypesById[destinationId]?.name ?? destinationId; - const relationType = relationTypesById[relationshipTypeId]; - if (!relationType) - throw new Error(`missing relation type ${relationshipTypeId}`); - const { label, complement } = relationType; - - return { - space_id: context.spaceId, - name: `${sourceName} -${label}-> ${destinationName}`, - source_local_id: id, - is_schema: true, - author_local_id: accountLocalId, - created: new Date(created).toISOString(), - last_modified: new Date(modified).toISOString(), - literal_content: { - roles: STANDARD_ROLES, - label, - complement, - }, - local_reference_content: { - relation_type: relationshipTypeId, - source: sourceId, - destination: destinationId, - }, - }; -}; - /** * Convert discourse node instance (file) to LocalConceptDataInput */ -export const discourseNodeInstanceToLocalConcepts = ({ - plugin, - allNodesByName, +export const discourseNodeInstanceToLocalConcept = ({ context, nodeData, accountLocalId, }: { - plugin: DiscourseGraphPlugin; - allNodesByName: Record; context: SupabaseContext; nodeData: ObsidianDiscourseNodeData; accountLocalId: string; -}): LocalConceptDataInput[] => { +}): LocalConceptDataInput => { const extraData = getNodeExtraData(nodeData.file, accountLocalId); const { nodeInstanceId, nodeTypeId, ...otherData } = nodeData.frontmatter; - const response: LocalConceptDataInput[] = []; - for (const relType of plugin.settings.relationTypes) { - const rels = otherData[relType.id]; - if (rels) { - delete otherData[relType.id]; - const triples = plugin.settings.discourseRelations.filter( - (r) => r.relationshipTypeId === relType.id && r.sourceId === nodeTypeId, - ); - if (!triples.length) { - // we're probably the target. - continue; - } - const tripleIdByDestType = Object.fromEntries( - triples.map((rel) => [rel.destinationId, rel.id]), - ); - for (let rel of rels as string[]) { - if (rel.startsWith("[[") && rel.endsWith("]]")) - rel = rel.substring(2, rel.length - 2); - if (rel.endsWith(".md")) rel = rel.substring(0, rel.length - 3); - const target = allNodesByName[rel]; - if (!target) { - console.error(`Could not find node name ${rel}`); - continue; - } - const targetTypeId = target.frontmatter.nodeTypeId as string; - const targetInstanceId = target.frontmatter.nodeInstanceId as string; - const relSchemaId = tripleIdByDestType[targetTypeId]; - if (relSchemaId === undefined) { - console.error( - `Found a relation of type ${relType.id} between ${nodeData.file.path} and ${rel} but no relation fits`, - ); - continue; - } - const compositeInstanceId = [ - relSchemaId, - nodeInstanceId as string, - targetInstanceId, - ].join(":"); - response.push({ - space_id: context.spaceId, - name: `[[${nodeData.file.basename}]] -${relType.label}-> [[${target.file.basename}]]`, - source_local_id: compositeInstanceId, - schema_represented_by_local_id: relSchemaId, - is_schema: false, - local_reference_content: { - source: nodeInstanceId as string, - destination: targetInstanceId, - }, - ...extraData, - }); - } - } - } - response.push({ + return { space_id: context.spaceId, name: nodeData.file.path, source_local_id: nodeInstanceId as string, @@ -212,8 +77,7 @@ export const discourseNodeInstanceToLocalConcepts = ({ source_data: otherData as unknown as Json, }, ...extraData, - }); - return response; + }; }; export const relatedConcepts = (concept: LocalConceptDataInput): string[] => { diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index fd0746024..4a7bc70cd 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -12,10 +12,8 @@ import { default as DiscourseGraphPlugin } from "~/index"; import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings"; import { orderConceptsByDependency, - discourseNodeInstanceToLocalConcepts, + discourseNodeInstanceToLocalConcept, discourseNodeSchemaToLocalConcept, - discourseRelationSchemaToLocalConcept, - discourseRelationTypeToLocalConcept, } from "./conceptConversion"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; @@ -178,7 +176,7 @@ const getLastSchemaSyncTime = async ( return new Date((data?.last_modified || DEFAULT_TIME) + "Z"); }; -export type DiscourseNodeInVault = { +type DiscourseNodeInVault = { file: TFile; frontmatter: Record; nodeTypeId: string; @@ -204,7 +202,7 @@ const mergeChangeTypes = ( * Step 1: Collect all discourse nodes from the vault * Filters markdown files that have nodeTypeId in frontmatter */ -export const collectDiscourseNodesFromVault = async ( +const collectDiscourseNodesFromVault = async ( plugin: DiscourseGraphPlugin, ): Promise => { const allFiles = plugin.app.vault.getMarkdownFiles(); @@ -512,27 +510,13 @@ const convertDgToSupabaseConcepts = async ({ context: SupabaseContext; accountLocalId: string; plugin: DiscourseGraphPlugin; - convertRelations?: boolean; }): Promise => { const lastSchemaSync = ( await getLastSchemaSyncTime(supabaseClient, context.spaceId) ).getTime(); - const nodeTypes = plugin.settings.nodeTypes ?? []; - const newNodeTypes = nodeTypes.filter((n) => n.modified > lastSchemaSync); - const relationTypes = (plugin.settings.relationTypes ?? []).filter( + const newNodeTypes = (plugin.settings.nodeTypes ?? []).filter( (n) => n.modified > lastSchemaSync, ); - const discourseRelations = (plugin.settings.discourseRelations ?? []).filter( - (n) => n.modified > lastSchemaSync, - ); - const allNodes = await collectDiscourseNodesFromVault(plugin); - const allNodesByName = Object.fromEntries( - allNodes.map((n) => [n.file.basename, n]), - ); - - const nodeTypesById = Object.fromEntries( - nodeTypes.map((nodeType) => [nodeType.id, nodeType]), - ); const nodesTypesToLocalConcepts = newNodeTypes.map((nodeType) => discourseNodeSchemaToLocalConcept({ @@ -542,44 +526,16 @@ const convertDgToSupabaseConcepts = async ({ }), ); - const relationTypesById = Object.fromEntries( - relationTypes.map((relationType) => [relationType.id, relationType]), - ); - - const relationTypesToLocalConcepts = relationTypes.map((relationType) => - discourseRelationTypeToLocalConcept({ + const nodeInstanceToLocalConcepts = nodesSince.map((node) => + discourseNodeInstanceToLocalConcept({ context, - relationType, + nodeData: node, accountLocalId, }), ); - const discourseRelationsToLocalConcepts = discourseRelations.map((relation) => - discourseRelationSchemaToLocalConcept({ - context, - relation, - accountLocalId, - nodeTypesById, - relationTypesById, - }), - ); - - const nodeInstanceToLocalConcepts = nodesSince - .map((node) => { - return discourseNodeInstanceToLocalConcepts({ - plugin, - allNodesByName, - context, - nodeData: node, - accountLocalId, - }); - }) - .flat(); - const conceptsToUpsert: LocalConceptDataInput[] = [ ...nodesTypesToLocalConcepts, - ...relationTypesToLocalConcepts, - ...discourseRelationsToLocalConcepts, ...nodeInstanceToLocalConcepts, ]; diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index df1ddfbde..e257e1880 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -1,13 +1,9 @@ -import { InputTextNode } from "roamjs-components/types"; -import getBlockProps from "./getBlockProps"; import { DiscourseNode } from "./getDiscourseNodes"; import getDiscourseRelations from "./getDiscourseRelations"; import type { DiscourseRelation } from "./getDiscourseRelations"; import type { SupabaseContext } from "~/utils/supabaseContext"; -import { DISCOURSE_GRAPH_PROP_NAME } from "~/utils/createReifiedBlock"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; -import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageUid"; const getNodeExtraData = ( node_uid: string, @@ -55,39 +51,18 @@ const getNodeExtraData = ( }; }; -const indent = (s: string): string => - s - .split("\n") - .map((l) => " " + l) - .join("\n") + "\n"; - -const templateToText = (template: InputTextNode[]): string => - template - .filter((itn) => !itn.text.startsWith("{{")) - .map( - (itn) => - `* ${itn.text}\n${itn.children?.length ? indent(templateToText(itn.children)) : ""}`, - ) - .join(""); - export const discourseNodeSchemaToLocalConcept = ( context: SupabaseContext, node: DiscourseNode, ): LocalConceptDataInput => { const titleParts = node.text.split("/"); - const result: LocalConceptDataInput = { + return { space_id: context.spaceId, - name: node.text, + name: titleParts[titleParts.length - 1], source_local_id: node.type, is_schema: true, ...getNodeExtraData(node.type), }; - if (node.template !== undefined) - result.literal_content = { - label: titleParts[titleParts.length - 1], - template: templateToText(node.template), - }; - return result; }; export const discourseNodeBlockToLocalConcept = ( @@ -112,7 +87,7 @@ export const discourseNodeBlockToLocalConcept = ( }; }; -const STANDARD_ROLES = ["source", "destination"]; +const STANDARD_ROLES = ["source", "target"]; export const discourseRelationSchemaToLocalConcept = ( context: SupabaseContext, @@ -122,7 +97,7 @@ export const discourseRelationSchemaToLocalConcept = ( space_id: context.spaceId, source_local_id: relation.id, // Not using the label directly, because it is not unique and name should be unique - name: getPageTitleByPageUid(relation.id), + name: `${relation.id}-${relation.label}`, is_schema: true, local_reference_content: Object.fromEntries( Object.entries(relation).filter(([key, v]) => @@ -141,21 +116,9 @@ export const discourseRelationSchemaToLocalConcept = ( export const discourseRelationDataToLocalConcept = ( context: SupabaseContext, - relationUid: string, + relationSchemaUid: string, + relationNodes: { [role: string]: string }, ): LocalConceptDataInput => { - // assuming reified - const relationProps = getBlockProps(relationUid); - const relationSchemaData = relationProps[DISCOURSE_GRAPH_PROP_NAME] as Record< - string, - string - >; - if (!relationSchemaData) { - throw new Error(`Missing relation data for ${relationUid}`); - } - const relationSchemaUid = relationSchemaData.hasSchema; - if (!relationSchemaUid) { - throw new Error(`Missing relation schema uid for ${relationUid}`); - } const roamRelation = getDiscourseRelations().find( (r) => r.id === relationSchemaUid, ); @@ -169,7 +132,7 @@ export const discourseRelationDataToLocalConcept = ( const roles = (litContent["roles"] as string[] | undefined) || STANDARD_ROLES; const casting: { [role: string]: string } = Object.fromEntries( roles - .map((role) => [role, relationSchemaData[role + "Uid"]]) + .map((role) => [role, relationNodes[role]]) .filter(([, uid]) => uid !== undefined), ); if (Object.keys(casting).length === 0) { @@ -189,13 +152,14 @@ export const discourseRelationDataToLocalConcept = ( Math.max(...nodeData.map((nd) => new Date(nd.created).getTime())), ).toISOString(); const author_local_id: string = nodeData[0].author_uid; // take any one; again until I get the relation object + const source_local_id = casting["target"] || Object.values(casting)[0]; // This one is tricky. Prefer the target for now. return { space_id: context.spaceId, - source_local_id: relationUid, + source_local_id, author_local_id, created, last_modified, - name: relationUid, + name: `${relationSchemaUid}-${Object.values(casting).join("-")}`, is_schema: false, schema_represented_by_local_id: relationSchemaUid, local_reference_content: casting, From 5bf1e0b2d898ed3371d0077b03c39121b05b7688 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 5 Feb 2026 14:00:33 -0500 Subject: [PATCH 09/33] Use SpaceURI rather than spaceId in frontmatter --- .../src/components/DiscourseContextView.tsx | 2 +- .../src/components/ImportNodesModal.tsx | 98 ++++---- apps/obsidian/src/index.ts | 2 +- apps/obsidian/src/services/QueryEngine.ts | 9 +- apps/obsidian/src/utils/importNodes.ts | 238 ++++++++++++------ .../src/utils/syncDgNodesToSupabase.ts | 4 +- 6 files changed, 222 insertions(+), 131 deletions(-) diff --git a/apps/obsidian/src/components/DiscourseContextView.tsx b/apps/obsidian/src/components/DiscourseContextView.tsx index bbf9f7e99..555beebb3 100644 --- a/apps/obsidian/src/components/DiscourseContextView.tsx +++ b/apps/obsidian/src/components/DiscourseContextView.tsx @@ -73,7 +73,7 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { return
Unknown node type: {frontmatter.nodeTypeId}
; } - const isImported = !!frontmatter.importedFromSpaceId; + const isImported = !!frontmatter.importedFromSpaceUri; return ( <> diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index 27b5afce7..b86baa3f7 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -7,7 +7,7 @@ import { getAvailableGroups, getPublishedNodesForGroups, getLocalNodeInstanceIds, - getSpaceNames, + getSpaceNameFromIds, importSelectedNodes, } from "~/utils/importNodes"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; @@ -21,9 +21,7 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { const [step, setStep] = useState<"loading" | "select" | "importing">( "loading", ); - const [groupsWithNodes, setGroupsWithNodes] = useState( - [], - ); + const [groupsWithNodes, setGroupsWithNodes] = useState([]); const [isLoading, setIsLoading] = useState(true); const [importProgress, setImportProgress] = useState({ current: 0, @@ -72,7 +70,7 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { const uniqueSpaceIds = [ ...new Set(importableNodes.map((n) => n.space_id)), ]; - const spaceNames = await getSpaceNames(client, uniqueSpaceIds); + const spaceNames = await getSpaceNameFromIds(client, uniqueSpaceIds); const grouped: Map = new Map(); for (const node of importableNodes) { @@ -120,16 +118,13 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { return { ...group, nodes: group.nodes.map((node, idx) => - idx === nodeIndex - ? { ...node, selected: !node.selected } - : node, + idx === nodeIndex ? { ...node, selected: !node.selected } : node, ), }; }), ); }; - const handleImport = async () => { const selectedNodes: ImportableNode[] = []; for (const group of groupsWithNodes) { @@ -163,10 +158,7 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { 5000, ); } else { - new Notice( - `Successfully imported ${result.success} node(s)`, - 3000, - ); + new Notice(`Successfully imported ${result.success} node(s)`, 3000); } onClose(); @@ -182,7 +174,9 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { const renderLoadingStep = () => (

Loading importable nodes...

-
Fetching groups and published nodes
+
+ Fetching groups and published nodes +
); @@ -192,15 +186,21 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { 0, ); const selectedCount = groupsWithNodes.reduce( - (sum, group) => - sum + group.nodes.filter((n) => n.selected).length, + (sum, group) => sum + group.nodes.filter((n) => n.selected).length, 0, ); // Group nodes by space for better organization const nodesBySpace = new Map< number, - { spaceName: string; nodes: Array<{ node: ImportableNode; groupId: string; nodeIndex: number }> } + { + spaceName: string; + nodes: Array<{ + node: ImportableNode; + groupId: string; + nodeIndex: number; + }>; + } >(); for (const group of groupsWithNodes) { @@ -257,40 +257,42 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => {
- {Array.from(nodesBySpace.entries()).map(([spaceId, { spaceName, nodes }]) => { - return ( -
-
- 📂 - - {spaceName} - - - ({nodes.length} node{nodes.length !== 1 ? "s" : ""}) - -
+ {Array.from(nodesBySpace.entries()).map( + ([spaceId, { spaceName, nodes }]) => { + return ( +
+
+ 📂 + + {spaceName} + + + ({nodes.length} node{nodes.length !== 1 ? "s" : ""}) + +
- {nodes.map(({ node, groupId, nodeIndex }) => ( -
- handleNodeToggle(groupId, nodeIndex)} - className="mr-3 mt-1 flex-shrink-0" - /> -
-
- {node.title} + {nodes.map(({ node, groupId, nodeIndex }) => ( +
+ handleNodeToggle(groupId, nodeIndex)} + className="mr-3 mt-1 flex-shrink-0" + /> +
+
+ {node.title} +
-
- ))} -
- ); - })} + ))} +
+ ); + }, + )}
diff --git a/apps/obsidian/src/index.ts b/apps/obsidian/src/index.ts index fe473b9c4..1707ab6b0 100644 --- a/apps/obsidian/src/index.ts +++ b/apps/obsidian/src/index.ts @@ -275,7 +275,7 @@ export default class DiscourseGraphPlugin extends Plugin { keysToHide.push( ...[ "nodeTypeId", - "importedFromSpaceId", + "importedFromSpaceUri", "nodeInstanceId", "publishedToGroups", ], diff --git a/apps/obsidian/src/services/QueryEngine.ts b/apps/obsidian/src/services/QueryEngine.ts index 1f88add96..1a82d3eb2 100644 --- a/apps/obsidian/src/services/QueryEngine.ts +++ b/apps/obsidian/src/services/QueryEngine.ts @@ -291,18 +291,18 @@ export class QueryEngine { } /** - * Find an existing imported file by nodeInstanceId and importedFromSpaceId + * Find an existing imported file by nodeInstanceId and importedFromSpaceUri * Uses DataCore when available; falls back to vault iteration otherwise * Returns the file if found, null otherwise */ findExistingImportedFile = ( nodeInstanceId: string, - importedFromSpaceId: number, + importedFromSpaceUri: string, ): TFile | null => { if (this.dc) { try { const safeId = nodeInstanceId.replace(/"/g, '\\"'); - const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceId = ${importedFromSpaceId}`; + const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceUri = ${importedFromSpaceUri}`; const results = this.dc.query(dcQuery); for (const page of results) { @@ -324,8 +324,7 @@ export class QueryEngine { const fm = this.app.metadataCache.getFileCache(f)?.frontmatter; if ( fm?.nodeInstanceId === nodeInstanceId && - (fm.importedFromSpaceId === importedFromSpaceId || - fm.importedFromSpaceId === String(importedFromSpaceId)) + fm.importedFromSpaceUri === importedFromSpaceUri ) { return f; } diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 43961beae..e537f4221 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -149,7 +149,7 @@ export const getLocalNodeInstanceIds = ( return nodeInstanceIds; }; -export const getSpaceName = async ( +export const getSpaceNameFromId = async ( client: DGSupabaseClient, spaceId: number, ): Promise => { @@ -167,7 +167,25 @@ export const getSpaceName = async ( return data.name; }; -export const getSpaceNames = async ( +export const getSpaceNameIdFromUri = async ( + client: DGSupabaseClient, + spaceUri: string, +): Promise<{ spaceName: string; spaceId: number }> => { + const { data, error } = await client + .from("Space") + .select("name, id") + .eq("url", spaceUri) + .maybeSingle(); + + if (error || !data) { + console.error("Error fetching space name:", error); + return { spaceName: "", spaceId: -1 }; + } + + return { spaceName: data.name, spaceId: data.id }; +}; + +export const getSpaceNameFromIds = async ( client: DGSupabaseClient, spaceIds: number[], ): Promise> => { @@ -193,6 +211,32 @@ export const getSpaceNames = async ( return spaceMap; }; +export const getSpaceUris = async ( + client: DGSupabaseClient, + spaceIds: number[], +): Promise> => { + if (spaceIds.length === 0) { + return new Map(); + } + + const { data, error } = await client + .from("Space") + .select("id, url") + .in("id", spaceIds); + + if (error) { + console.error("Error fetching space urls:", error); + return new Map(); + } + + const spaceMap = new Map(); + (data || []).forEach((space) => { + spaceMap.set(space.id, space.url); + }); + + return spaceMap; +}; + export const fetchNodeContent = async ({ client, spaceId, @@ -242,7 +286,9 @@ export const fetchNodeMetadata = async ({ .maybeSingle(); return { - nodeTypeId: (conceptData?.literal_content as unknown as { nodeTypeId?: string })?.nodeTypeId || undefined, + nodeTypeId: + (conceptData?.literal_content as unknown as { nodeTypeId?: string }) + ?.nodeTypeId || undefined, }; }; @@ -411,41 +457,49 @@ const updateMarkdownAssetLinks = ({ // Match wiki links: [[path]] or [[path|alias]] const wikiLinkRegex = /\[\[([^\]]+)\]\]/g; - updatedContent = updatedContent.replace(wikiLinkRegex, (match, linkContent) => { - // Extract path and optional alias - const [linkPath, alias] = linkContent.split("|").map((s: string) => s.trim()); - - // Skip external URLs - if (linkPath.startsWith("http://") || linkPath.startsWith("https://")) { - return match; - } + updatedContent = updatedContent.replace( + wikiLinkRegex, + (match, linkContent) => { + // Extract path and optional alias + const [linkPath, alias] = linkContent + .split("|") + .map((s: string) => s.trim()); - // First, try to find if this link resolves to one of our imported assets - const importedAssetFile = findImportedAssetFile(linkPath); - if (importedAssetFile) { - const linkText = app.metadataCache.fileToLinktext( - importedAssetFile, - targetFile.path, - ); - if (alias) { - return `[[${linkText}|${alias}]]`; + // Skip external URLs + if (linkPath.startsWith("http://") || linkPath.startsWith("https://")) { + return match; } - return `[[${linkText}]]`; - } - // Fallback: Find matching old path - for (const [oldPath, newFile] of oldPathToNewFile.entries()) { - if (matchesOldPath(linkPath, oldPath)) { - const linkText = app.metadataCache.fileToLinktext(newFile, targetFile.path); + // First, try to find if this link resolves to one of our imported assets + const importedAssetFile = findImportedAssetFile(linkPath); + if (importedAssetFile) { + const linkText = app.metadataCache.fileToLinktext( + importedAssetFile, + targetFile.path, + ); if (alias) { return `[[${linkText}|${alias}]]`; } return `[[${linkText}]]`; } - } - return match; - }); + // Fallback: Find matching old path + for (const [oldPath, newFile] of oldPathToNewFile.entries()) { + if (matchesOldPath(linkPath, oldPath)) { + const linkText = app.metadataCache.fileToLinktext( + newFile, + targetFile.path, + ); + if (alias) { + return `[[${linkText}|${alias}]]`; + } + return `[[${linkText}]]`; + } + } + + return match; + }, + ); // Match markdown image links: ![alt](path) or ![alt](path "title") const markdownImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; @@ -473,7 +527,10 @@ const updateMarkdownAssetLinks = ({ // Fallback: Find matching old path for (const [oldPath, newFile] of oldPathToNewFile.entries()) { if (matchesOldPath(cleanPath, oldPath)) { - const linkText = app.metadataCache.fileToLinktext(newFile, targetFile.path); + const linkText = app.metadataCache.fileToLinktext( + newFile, + targetFile.path, + ); return `![${alt}](${linkText})`; } } @@ -485,7 +542,6 @@ const updateMarkdownAssetLinks = ({ return updatedContent; }; - const importAssetsForNode = async ({ plugin, client, @@ -534,7 +590,9 @@ const importAssetsForNode = async ({ const frontmatter = (cache?.frontmatter as Record) || {}; const importedAssetsRaw = frontmatter.importedAssets; const importedAssets: Record = - importedAssetsRaw && typeof importedAssetsRaw === "object" && !Array.isArray(importedAssetsRaw) + importedAssetsRaw && + typeof importedAssetsRaw === "object" && + !Array.isArray(importedAssetsRaw) ? (importedAssetsRaw as Record) : {}; // importedAssets format: { filehash: vaultPath } @@ -557,7 +615,8 @@ const importAssetsForNode = async ({ } else { // File was moved/renamed - search for it in the assets folder // Try to find a file with the same name in the assets folder - const { name: fileName, ext: fileExt } = extractFileName(existingAssetPath); + const { name: fileName, ext: fileExt } = + extractFileName(existingAssetPath); const searchFileName = `${fileName}${fileExt ? `.${fileExt}` : ""}`; // Search all files in the assets folder @@ -576,9 +635,12 @@ const importAssetsForNode = async ({ await plugin.app.fileManager.processFrontMatter( targetMarkdownFile, (fm) => { - const assetsRaw = (fm as Record).importedAssets; + const assetsRaw = (fm as Record) + .importedAssets; const assets: Record = - assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) + assetsRaw && + typeof assetsRaw === "object" && + !Array.isArray(assetsRaw) ? (assetsRaw as Record) : {}; assets[filehash] = vaultFile.path; @@ -594,7 +656,9 @@ const importAssetsForNode = async ({ // If we found an existing file, reuse it if (existingFile) { pathMapping.set(filepath, existingFile.path); - console.log(`Reusing existing asset: ${filehash} -> ${existingFile.path}`); + console.log( + `Reusing existing asset: ${filehash} -> ${existingFile.path}`, + ); continue; } @@ -603,7 +667,7 @@ const importAssetsForNode = async ({ const sanitizedName = sanitizeFileName(name); const sanitizedExt = ext ? `.${ext}` : ""; const sanitizedFileName = `${sanitizedName}${sanitizedExt}`; - let targetPath = `${assetsFolderPath}/${sanitizedFileName}`; + const targetPath = `${assetsFolderPath}/${sanitizedFileName}`; // Check if file already exists at target path (avoid duplicates) if (await plugin.app.vault.adapter.exists(targetPath)) { @@ -617,7 +681,9 @@ const importAssetsForNode = async ({ (fm) => { const assetsRaw = (fm as Record).importedAssets; const assets: Record = - assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) + assetsRaw && + typeof assetsRaw === "object" && + !Array.isArray(assetsRaw) ? (assetsRaw as Record) : {}; assets[filehash] = targetPath; @@ -645,15 +711,20 @@ const importAssetsForNode = async ({ await plugin.app.vault.createBinary(targetPath, fileContent); // Update frontmatter to track this mapping - await plugin.app.fileManager.processFrontMatter(targetMarkdownFile, (fm) => { - const assetsRaw = (fm as Record).importedAssets; - const assets: Record = - assetsRaw && typeof assetsRaw === "object" && !Array.isArray(assetsRaw) - ? (assetsRaw as Record) - : {}; - assets[filehash] = targetPath; - (fm as Record).importedAssets = assets; - }); + await plugin.app.fileManager.processFrontMatter( + targetMarkdownFile, + (fm) => { + const assetsRaw = (fm as Record).importedAssets; + const assets: Record = + assetsRaw && + typeof assetsRaw === "object" && + !Array.isArray(assetsRaw) + ? (assetsRaw as Record) + : {}; + assets[filehash] = targetPath; + (fm as Record).importedAssets = assets; + }, + ); // Track path mapping pathMapping.set(filepath, targetPath); @@ -672,7 +743,6 @@ const importAssetsForNode = async ({ }; }; - const sanitizeFileName = (fileName: string): string => { // Remove invalid characters for file names return fileName @@ -783,7 +853,10 @@ const parseFrontmatter = ( const parseSchemaLiteralContent = ( literalContent: unknown, fallbackName: string, -): Pick => { +): Pick< + DiscourseNode, + "name" | "format" | "color" | "tag" | "template" | "keyImage" +> => { const obj = typeof literalContent === "string" ? (JSON.parse(literalContent) as Record) @@ -793,15 +866,15 @@ const parseSchemaLiteralContent = ( const formatFromSchema = (src.format as string) || (obj.format as string) || ""; const format = - formatFromSchema || - `${name.slice(0, 3).toUpperCase()} - {content}`; + formatFromSchema || `${name.slice(0, 3).toUpperCase()} - {content}`; return { name, format, color: (src.color as string) || (obj.color as string) || undefined, tag: (src.tag as string) || (obj.tag as string) || undefined, template: (obj.template as string) || undefined, - keyImage: (src.keyImage as boolean) ?? (obj.keyImage as boolean) ?? undefined, + keyImage: + (src.keyImage as boolean) ?? (obj.keyImage as boolean) ?? undefined, }; }; @@ -859,10 +932,7 @@ const mapNodeTypeIdToLocal = async ({ created: now, modified: now, }; - plugin.settings.nodeTypes = [ - ...plugin.settings.nodeTypes, - newNodeType, - ]; + plugin.settings.nodeTypes = [...plugin.settings.nodeTypes, newNodeType]; await plugin.saveSettings(); return newNodeType.id; }; @@ -871,12 +941,14 @@ const processFileContent = async ({ plugin, client, sourceSpaceId, + sourceSpaceUri, rawContent, filePath, }: { plugin: DiscourseGraphPlugin; client: DGSupabaseClient; sourceSpaceId: number; + sourceSpaceUri: string; rawContent: string; filePath: string; }): Promise<{ file: TFile; error?: string }> => { @@ -907,7 +979,7 @@ const processFileContent = async ({ if (mappedNodeTypeId !== undefined) { (fm as Record).nodeTypeId = mappedNodeTypeId; } - (fm as Record).importedFromSpaceId = sourceSpaceId; + (fm as Record).importedFromSpaceUri = sourceSpaceUri; }); return { file }; @@ -948,11 +1020,18 @@ export const importSelectedNodes = async ({ nodesBySpace.get(node.spaceId)!.push(node); } + const spaceUris = await getSpaceUris(client, [...nodesBySpace.keys()]); + // Process each space for (const [spaceId, nodes] of nodesBySpace.entries()) { - const spaceName = await getSpaceName(client, spaceId); + const spaceName = await getSpaceNameFromId(client, spaceId); const importFolderPath = `import/${sanitizeFileName(spaceName)}`; const assetsFolderPath = `${importFolderPath}/assets`; + const spaceUri = spaceUris.get(spaceId); + if (!spaceUri) { + console.warn(`Missing URI for space ${spaceId}`); + continue; + } // Ensure the import folder exists const folderExists = @@ -971,10 +1050,10 @@ export const importSelectedNodes = async ({ // Process each node in this space for (const node of nodes) { try { - // Check if file already exists by nodeInstanceId + importedFromSpaceId + // Check if file already exists by nodeInstanceId + importedFromSpaceUri const existingFile = queryEngine.findExistingImportedFile( node.nodeInstanceId, - node.spaceId, + spaceUri, ); console.log("existingFile", existingFile); @@ -982,7 +1061,7 @@ export const importSelectedNodes = async ({ // Fetch the file name (direct variant) and content (full variant) const fileName = await fetchNodeContent({ client, - spaceId: node.spaceId, + spaceId, nodeInstanceId: node.nodeInstanceId, variant: "direct", }); @@ -999,7 +1078,7 @@ export const importSelectedNodes = async ({ const content = await fetchNodeContent({ client, - spaceId: node.spaceId, + spaceId, nodeInstanceId: node.nodeInstanceId, variant: "full", }); @@ -1036,7 +1115,8 @@ export const importSelectedNodes = async ({ const result = await processFileContent({ plugin, client, - sourceSpaceId: node.spaceId, + sourceSpaceId: spaceId, + sourceSpaceUri: spaceUri, rawContent: content, filePath: finalFilePath, }); @@ -1058,7 +1138,7 @@ export const importSelectedNodes = async ({ const assetImportResult = await importAssetsForNode({ plugin, client, - spaceId: node.spaceId, + spaceId, nodeInstanceId: node.nodeInstanceId, spaceName, targetMarkdownFile: processedFile, @@ -1128,36 +1208,42 @@ export const refreshImportedFile = async ({ file: TFile; client?: DGSupabaseClient; }): Promise<{ success: boolean; error?: string }> => { - const supabaseClient = client || await getLoggedInClient(plugin); + const supabaseClient = client || (await getLoggedInClient(plugin)); if (!supabaseClient) { throw new Error("Cannot get Supabase client"); } const cache = plugin.app.metadataCache.getFileCache(file); const frontmatter = cache?.frontmatter as Record; - if (!frontmatter.importedFromSpaceId || !frontmatter.nodeInstanceId) { + if (!frontmatter.importedFromSpaceUri || !frontmatter.nodeInstanceId) { return { success: false, - error: "Missing frontmatter: importedFromSpaceId or nodeInstanceId", + error: "Missing frontmatter: importedFromSpaceUri or nodeInstanceId", }; } - const spaceName = await getSpaceName( + const { spaceName, spaceId } = await getSpaceNameIdFromUri( supabaseClient, - frontmatter.importedFromSpaceId as number, + frontmatter.importedFromSpaceUri as string, ); + if (spaceId === -1) { + return { success: false, error: "Could not get the space Id" }; + } const result = await importSelectedNodes({ plugin, selectedNodes: [ { nodeInstanceId: frontmatter.nodeInstanceId as string, title: file.basename, - spaceId: frontmatter.importedFromSpaceId as number, - spaceName: spaceName, + spaceId, + spaceName, groupId: (frontmatter.publishedToGroups as string[])[0] ?? "", selected: false, }, ], }); - return { success: result.success > 0, error: result.failed > 0 ? "Failed to refresh imported file" : undefined }; + return { + success: result.success > 0, + error: result.failed > 0 ? "Failed to refresh imported file" : undefined, + }; }; /** @@ -1165,7 +1251,11 @@ export const refreshImportedFile = async ({ */ export const refreshAllImportedFiles = async ( plugin: DiscourseGraphPlugin, -): Promise<{ success: number; failed: number; errors: Array<{ file: string; error: string }> }> => { +): Promise<{ + success: number; + failed: number; + errors: Array<{ file: string; error: string }>; +}> => { const allFiles = plugin.app.vault.getMarkdownFiles(); const importedFiles: TFile[] = []; const client = await getLoggedInClient(plugin); @@ -1176,7 +1266,7 @@ export const refreshAllImportedFiles = async ( for (const file of allFiles) { const cache = plugin.app.metadataCache.getFileCache(file); const frontmatter = cache?.frontmatter; - if (frontmatter?.importedFromSpaceId && frontmatter?.nodeInstanceId) { + if (frontmatter?.importedFromSpaceUri && frontmatter?.nodeInstanceId) { importedFiles.push(file); } } diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 4a7bc70cd..5c8d3280c 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -217,7 +217,7 @@ const collectDiscourseNodesFromVault = async ( continue; } - if (frontmatter.importedFromSpaceId) { + if (frontmatter.importedFromSpaceUri) { continue; } @@ -631,7 +631,7 @@ const collectDiscourseNodesFromPaths = async ( continue; } - if (frontmatter.importedFromSpaceId) { + if (frontmatter.importedFromSpaceUri) { console.debug(`Skipping imported file: ${filePath}`); continue; } From 4c6d8908173420ef484750ae3ff8948fbe9135ef Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Fri, 6 Feb 2026 23:08:33 -0500 Subject: [PATCH 10/33] address PR comments --- apps/obsidian/package.json | 3 +- .../src/components/ImportNodesModal.tsx | 7 +- apps/obsidian/src/services/QueryEngine.ts | 2 +- apps/obsidian/src/utils/conceptConversion.ts | 17 +- apps/obsidian/src/utils/fileChangeListener.ts | 2 +- apps/obsidian/src/utils/importNodes.ts | 290 ++++++++---------- .../src/utils/syncDgNodesToSupabase.ts | 14 +- pnpm-lock.yaml | 96 ++---- 8 files changed, 177 insertions(+), 254 deletions(-) diff --git a/apps/obsidian/package.json b/apps/obsidian/package.json index 99e457466..70537f7fc 100644 --- a/apps/obsidian/package.json +++ b/apps/obsidian/package.json @@ -19,10 +19,10 @@ "@octokit/core": "^6.1.2", "@repo/eslint-config": "workspace:*", "@repo/typescript-config": "workspace:*", + "@types/mime-types": "3.0.1", "@types/node": "^20", "@types/react": "catalog:obsidian", "@types/react-dom": "catalog:obsidian", - "@types/mime-types": "3.0.1", "autoprefixer": "^10.4.21", "builtin-modules": "3.3.0", "dotenv": "^16.4.5", @@ -43,6 +43,7 @@ "@repo/utils": "workspace:*", "@supabase/supabase-js": "catalog:", "date-fns": "^4.1.0", + "gray-matter": "^4.0.3", "mime-types": "^3.0.1", "nanoid": "^4.0.2", "react": "catalog:obsidian", diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index b86baa3f7..8537a5d2d 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -60,7 +60,7 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { currentSpaceId: context.spaceId, }); - const localNodeInstanceIds = await getLocalNodeInstanceIds(plugin); + const localNodeInstanceIds = getLocalNodeInstanceIds(plugin); // Filter out nodes that already exist locally const importableNodes = publishedNodes.filter( @@ -223,8 +223,9 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => {

Select Nodes to Import

- {totalNodes} importable node(s) found. Select which nodes to import - into your vault. + {totalNodes > 0 + ? `${totalNodes} importable node(s) found. Select which nodes to import into your vault.` + : "No importable nodes found."}

diff --git a/apps/obsidian/src/services/QueryEngine.ts b/apps/obsidian/src/services/QueryEngine.ts index 1a82d3eb2..abcc52871 100644 --- a/apps/obsidian/src/services/QueryEngine.ts +++ b/apps/obsidian/src/services/QueryEngine.ts @@ -302,7 +302,7 @@ export class QueryEngine { if (this.dc) { try { const safeId = nodeInstanceId.replace(/"/g, '\\"'); - const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceUri = ${importedFromSpaceUri}`; + const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceUri = "${importedFromSpaceUri.replace(/"/g, '\\"')}"`; const results = this.dc.query(dcQuery); for (const page of results) { diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index 74e8853b6..161ec1a3f 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -37,7 +37,7 @@ export const discourseNodeSchemaToLocalConcept = ({ node; return { space_id: context.spaceId, - name: name, + name: id ? `${name} (${id})` : name, source_local_id: id, is_schema: true, author_local_id: accountLocalId, @@ -92,7 +92,11 @@ export const relatedConcepts = (concept: LocalConceptDataInput): string[] => { }; /** - * Recursively order concepts by dependency + * Recursively order concepts by dependency so that dependents (e.g. instances) + * come after their dependencies (e.g. schemas). When we look up a related + * concept by id in `remainder`, we use the same id that appears in + * schema_represented_by_local_id or local_reference_content — so that id + * must equal some concept's source_local_id or it is reported as "missing". */ const orderConceptsRec = ( ordered: LocalConceptDataInput[], @@ -119,6 +123,13 @@ const orderConceptsRec = ( return missing; }; +/** + * Order concepts so dependencies (schemas) are before dependents (instances). + * Assumes every concept has source_local_id; concepts without it are excluded + * from the map (same as Roam). A node type is "missing" when an instance + * references schema_represented_by_local_id = X but no concept in the input + * has source_local_id === X (e.g. schema not included, or id vs nodeTypeId mismatch). + */ export const orderConceptsByDependency = ( concepts: LocalConceptDataInput[], ): { ordered: LocalConceptDataInput[]; missing: string[] } => { @@ -126,7 +137,7 @@ export const orderConceptsByDependency = ( const conceptById: { [key: string]: LocalConceptDataInput } = Object.fromEntries( concepts - .filter((c) => c.source_local_id) + .filter((c) => c.source_local_id != null && c.source_local_id !== "") .map((c) => [c.source_local_id!, c]), ); const ordered: LocalConceptDataInput[] = []; diff --git a/apps/obsidian/src/utils/fileChangeListener.ts b/apps/obsidian/src/utils/fileChangeListener.ts index 36197b331..c81dd923f 100644 --- a/apps/obsidian/src/utils/fileChangeListener.ts +++ b/apps/obsidian/src/utils/fileChangeListener.ts @@ -98,7 +98,7 @@ export class FileChangeListener { return false; } - if (frontmatter?.importedFromSpaceId) { + if (frontmatter?.importedFromSpaceUri) { return false; } diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index e537f4221..606625e3f 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import matter from "gray-matter"; import { App, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import type DiscourseGraphPlugin from "~/index"; @@ -58,23 +59,16 @@ export const getPublishedNodesForGroups = async ({ return []; } - // Group by space_id and source_local_id to fetch content efficiently - const nodeKeys = new Set(); - for (const ra of resourceAccessData) { - if (ra.source_local_id && ra.space_id) { - nodeKeys.add(`${ra.space_id}:${ra.source_local_id}`); - } - } - - // Group nodes by space_id for batched queries + // Group unique (space_id, source_local_id) by space_id for batched queries. + // Build nodesBySpace directly so source_local_id can contain any character (e.g. colons). const nodesBySpace = new Map>(); - for (const key of nodeKeys) { - const [spaceId, sourceLocalId] = key.split(":"); - const spaceIdNum = parseInt(spaceId || "0", 10); - if (!nodesBySpace.has(spaceIdNum)) { - nodesBySpace.set(spaceIdNum, new Set()); + for (const ra of resourceAccessData) { + if (ra.source_local_id && ra.space_id != null) { + if (!nodesBySpace.has(ra.space_id)) { + nodesBySpace.set(ra.space_id, new Set()); + } + nodesBySpace.get(ra.space_id)!.add(ra.source_local_id); } - nodesBySpace.get(spaceIdNum)!.add(sourceLocalId || ""); } // Fetch Content with variant "direct" for all nodes, batched by space @@ -267,28 +261,41 @@ export const fetchNodeContent = async ({ return data.text; }; -export const fetchNodeMetadata = async ({ +export const fetchNodeContentWithMetadata = async ({ client, spaceId, nodeInstanceId, + variant, }: { client: DGSupabaseClient; spaceId: number; nodeInstanceId: string; -}): Promise<{ nodeTypeId?: string }> => { - // Try to get nodeTypeId from Concept table - const { data: conceptData } = await client - .from("Concept") - .select("literal_content") + variant: "direct" | "full"; +}): Promise<{ + content: string; + createdAt: string; + modifiedAt: string; +} | null> => { + const { data, error } = await client + .from("Content") + .select("text, created, last_modified") .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) - .eq("is_schema", false) + .eq("variant", variant) .maybeSingle(); + if (error || !data) { + console.error( + `Error fetching node content with metadata (${variant}):`, + error || "No data", + ); + return null; + } + return { - nodeTypeId: - (conceptData?.literal_content as unknown as { nodeTypeId?: string }) - ?.nodeTypeId || undefined, + content: data.text, + createdAt: data.created ?? new Date(0).toISOString(), + modifiedAt: data.last_modified ?? new Date(0).toISOString(), }; }; @@ -402,28 +409,15 @@ const updateMarkdownAssetLinks = ({ let updatedContent = content; - // Helper to check if a link path matches an old path - const matchesOldPath = (linkPath: string, oldPath: string): boolean => { - // Exact match - if (linkPath === oldPath) return true; - - // Match by filename (handles relative paths) - const oldFileName = extractFileName(oldPath); - const linkFileName = extractFileName(linkPath); - if ( - oldFileName.name === linkFileName.name && - oldFileName.ext === linkFileName.ext - ) { - return true; - } - - // Match if linkPath ends with oldPath or vice versa (handles relative vs absolute) - if (linkPath.endsWith(oldPath) || oldPath.endsWith(linkPath)) { - return true; - } + // Normalize path for comparison: strip leading "./", collapse repeated slashes. + // We match only by full path (exact after normalizing), not by filename alone, + // so that different paths with the same name (e.g. experiment1/result.jpg vs + // experiment2/result.jpg) are never treated as the same asset. + const normalizePathForMatch = (p: string): string => + p.replace(/^\.\//, "").replace(/\/+/g, "/").trim(); - return false; - }; + const matchesOldPath = (linkPath: string, oldPath: string): boolean => + normalizePathForMatch(linkPath) === normalizePathForMatch(oldPath); // Helper to find file for a link path, checking if it's one of our imported assets const findImportedAssetFile = (linkPath: string): TFile | null => { @@ -653,45 +647,61 @@ const importAssetsForNode = async ({ } } - // If we found an existing file, reuse it + let overwritePath: string | undefined; if (existingFile) { - pathMapping.set(filepath, existingFile.path); - console.log( - `Reusing existing asset: ${filehash} -> ${existingFile.path}`, - ); - continue; + const refLastModifiedMs = fileRef.last_modified + ? new Date(fileRef.last_modified + "Z").getTime() + : 0; + const localModifiedAfterRef = + refLastModifiedMs > 0 && existingFile.stat.mtime > refLastModifiedMs; + if (!localModifiedAfterRef) { + pathMapping.set(filepath, existingFile.path); + console.log( + `Reusing existing asset: ${filehash} -> ${existingFile.path}`, + ); + continue; + } + overwritePath = existingFile.path; } - // No existing file found, need to download - // Determine target path + // Determine target path (new file or overwrite of modified local file) const sanitizedName = sanitizeFileName(name); const sanitizedExt = ext ? `.${ext}` : ""; const sanitizedFileName = `${sanitizedName}${sanitizedExt}`; - const targetPath = `${assetsFolderPath}/${sanitizedFileName}`; + const targetPath = + overwritePath ?? `${assetsFolderPath}/${sanitizedFileName}`; - // Check if file already exists at target path (avoid duplicates) + // If local mtime is newer than fileRef.last_modified, overwrite with DB version. if (await plugin.app.vault.adapter.exists(targetPath)) { - // File exists at expected path, reuse it const file = plugin.app.vault.getAbstractFileByPath(targetPath); if (file && file instanceof TFile) { - pathMapping.set(filepath, targetPath); - // Update frontmatter to track this mapping - await plugin.app.fileManager.processFrontMatter( - targetMarkdownFile, - (fm) => { - const assetsRaw = (fm as Record).importedAssets; - const assets: Record = - assetsRaw && - typeof assetsRaw === "object" && - !Array.isArray(assetsRaw) - ? (assetsRaw as Record) - : {}; - assets[filehash] = targetPath; - (fm as Record).importedAssets = assets; - }, - ); - console.log(`Reusing existing file at path: ${targetPath}`); - continue; + const localMtimeMs = file.stat.mtime; + const refLastModifiedMs = fileRef.last_modified + ? new Date(fileRef.last_modified + "Z").getTime() + : 0; + const localModifiedAfterRef = + refLastModifiedMs > 0 && localMtimeMs > refLastModifiedMs; + if (!localModifiedAfterRef) { + pathMapping.set(filepath, targetPath); + await plugin.app.fileManager.processFrontMatter( + targetMarkdownFile, + (fm) => { + const assetsRaw = (fm as Record) + .importedAssets; + const assets: Record = + assetsRaw && + typeof assetsRaw === "object" && + !Array.isArray(assetsRaw) + ? (assetsRaw as Record) + : {}; + assets[filehash] = targetPath; + (fm as Record).importedAssets = assets; + }, + ); + console.log(`Reusing existing file at path: ${targetPath}`); + continue; + } + // Local file was modified since fileRef's last_modified; overwrite with DB version } } @@ -760,89 +770,12 @@ type ParsedFrontmatter = { const parseFrontmatter = ( content: string, -): { - frontmatter: ParsedFrontmatter; - body: string; -} => { - // Pattern: ---\n(frontmatter)\n---\n(body - optional) - const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*(?:\n([\s\S]*))?$/; - const match = content.match(frontmatterRegex); - - if (!match || !match[1]) { - return { frontmatter: {}, body: content }; - } - - const frontmatterText = match[1]; - const body = match[2] ?? ""; - - // Parse YAML-like frontmatter (simple parser for key: value pairs) - const frontmatter: ParsedFrontmatter = {}; - const lines = frontmatterText.split("\n"); - - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (!line) continue; - - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith("#")) continue; - - const colonIndex = trimmed.indexOf(":"); - if (colonIndex === -1) continue; - - const key = trimmed.slice(0, colonIndex).trim(); - const valueStr = trimmed.slice(colonIndex + 1).trim(); - let value: unknown = valueStr; - - // Handle array values: inline (valueStr starts with "-") or block (valueStr empty, next lines start with "-") - const isInlineArrayStart = valueStr.startsWith("-"); - const isEmptyValue = !valueStr || valueStr.trim() === ""; - - if (isInlineArrayStart || isEmptyValue) { - const arrayValues: string[] = []; - let nextLineIndex = i + 1; - - // First item on same line (inline): "key: - item" - if (isInlineArrayStart) { - const firstItem = valueStr.slice(1).trim(); - if (firstItem) arrayValues.push(firstItem); - } - - // Collect array items from subsequent lines that trim-start with "-" - let currentLine: string | undefined = - nextLineIndex < lines.length ? lines[nextLineIndex] : undefined; - while (currentLine != null && currentLine.trim().startsWith("-")) { - const itemValue = currentLine.trim().slice(1).trim(); - if (itemValue) arrayValues.push(itemValue); - nextLineIndex++; - currentLine = - nextLineIndex < lines.length ? lines[nextLineIndex] : undefined; - } - - value = - arrayValues.length > 0 - ? arrayValues - : isInlineArrayStart - ? [valueStr.slice(1).trim()] - : []; - frontmatter[key] = value; - i = nextLineIndex - 1; // skip consumed lines - continue; - } - - // Scalar value: remove quotes if present - if ( - (valueStr.startsWith('"') && valueStr.endsWith('"')) || - (valueStr.startsWith("'") && valueStr.endsWith("'")) - ) { - value = valueStr.slice(1, -1); - } else { - value = valueStr; - } - - frontmatter[key] = value; - } - - return { frontmatter, body }; +): { frontmatter: ParsedFrontmatter; body: string } => { + const { data, content: body } = matter(content); + return { + frontmatter: (data ?? {}) as ParsedFrontmatter, + body: body ?? "", + }; }; /** @@ -904,11 +837,18 @@ const mapNodeTypeIdToLocal = async ({ const schemaName = schemaData.name; - // Find a local nodeType with the same name (use plugin.settings so we see newly created types) + // Prefer match by node type ID (imported type may already exist locally with same id) + const matchById = plugin.settings.nodeTypes.find( + (nt) => nt.id === sourceNodeTypeId, + ); + if (matchById) { + return matchById.id; + } + + // Fall back to match by name const matchingLocalNodeType = plugin.settings.nodeTypes.find( (nt) => nt.name === schemaName, ); - if (matchingLocalNodeType) { return matchingLocalNodeType.id; } @@ -944,6 +884,8 @@ const processFileContent = async ({ sourceSpaceUri, rawContent, filePath, + importedCreatedAt, + importedModifiedAt, }: { plugin: DiscourseGraphPlugin; client: DGSupabaseClient; @@ -951,6 +893,8 @@ const processFileContent = async ({ sourceSpaceUri: string; rawContent: string; filePath: string; + importedCreatedAt?: string; + importedModifiedAt?: string; }): Promise<{ file: TFile; error?: string }> => { // 1. Create or update the file with the fetched content first let file: TFile | null = plugin.app.vault.getFileByPath(filePath); @@ -961,7 +905,7 @@ const processFileContent = async ({ } // 2. Parse frontmatter from rawContent (metadataCache is updated async and is - // often empty immediately after create/modify), then map nodeTypeId and update frontmatter. + // often empty immediately after create/modify), then map nodeTypeId and update frontmatter. const { frontmatter } = parseFrontmatter(rawContent); const sourceNodeTypeId = frontmatter.nodeTypeId; @@ -976,10 +920,17 @@ const processFileContent = async ({ } await plugin.app.fileManager.processFrontMatter(file, (fm) => { + const record = fm as Record; if (mappedNodeTypeId !== undefined) { - (fm as Record).nodeTypeId = mappedNodeTypeId; + record.nodeTypeId = mappedNodeTypeId; + } + record.importedFromSpaceUri = sourceSpaceUri; + if (importedCreatedAt !== undefined) { + record.importedCreatedAt = importedCreatedAt; + } + if (importedModifiedAt !== undefined) { + record.importedModifiedAt = importedModifiedAt; } - (fm as Record).importedFromSpaceUri = sourceSpaceUri; }); return { file }; @@ -1076,14 +1027,14 @@ export const importSelectedNodes = async ({ continue; } - const content = await fetchNodeContent({ + const contentWithMeta = await fetchNodeContentWithMetadata({ client, spaceId, nodeInstanceId: node.nodeInstanceId, variant: "full", }); - if (content === null) { + if (contentWithMeta === null) { console.warn(`No full variant found for node ${node.nodeInstanceId}`); failedCount++; processedCount++; @@ -1091,6 +1042,8 @@ export const importSelectedNodes = async ({ continue; } + const { content, createdAt, modifiedAt } = contentWithMeta; + // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); let finalFilePath: string; @@ -1110,7 +1063,7 @@ export const importSelectedNodes = async ({ } } - // Process the file content (maps nodeTypeId, handles frontmatter) + // Process the file content (maps nodeTypeId, handles frontmatter, stores import timestamps) // This updates existing file or creates new one const result = await processFileContent({ plugin, @@ -1119,6 +1072,8 @@ export const importSelectedNodes = async ({ sourceSpaceUri: spaceUri, rawContent: content, filePath: finalFilePath, + importedCreatedAt: createdAt, + importedModifiedAt: modifiedAt, }); if (result.error) { @@ -1235,7 +1190,8 @@ export const refreshImportedFile = async ({ title: file.basename, spaceId, spaceName, - groupId: (frontmatter.publishedToGroups as string[])[0] ?? "", + groupId: + (frontmatter.publishedToGroups as string[] | undefined)?.[0] ?? "", selected: false, }, ], diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 5c8d3280c..d47344d64 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -514,11 +514,15 @@ const convertDgToSupabaseConcepts = async ({ const lastSchemaSync = ( await getLastSchemaSyncTime(supabaseClient, context.spaceId) ).getTime(); - const newNodeTypes = (plugin.settings.nodeTypes ?? []).filter( - (n) => n.modified > lastSchemaSync, + const schemaIdsReferencedByInstances = new Set( + nodesSince.map((n) => n.nodeTypeId), + ); + const nodeTypesToUpsert = (plugin.settings.nodeTypes ?? []).filter( + (n) => + n.modified > lastSchemaSync || schemaIdsReferencedByInstances.has(n.id), ); - const nodesTypesToLocalConcepts = newNodeTypes.map((nodeType) => + const nodesTypesToLocalConcepts = nodeTypesToUpsert.map((nodeType) => discourseNodeSchemaToLocalConcept({ context, node: nodeType, @@ -539,11 +543,13 @@ const convertDgToSupabaseConcepts = async ({ ...nodeInstanceToLocalConcepts, ]; + console.log("conceptsToUpsert", conceptsToUpsert); + if (conceptsToUpsert.length > 0) { const { ordered } = orderConceptsByDependency(conceptsToUpsert); const { error } = await supabaseClient.rpc("upsert_concepts", { - data: ordered as Json, + data: conceptsToUpsert as Json, v_space_id: context.spaceId, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a715ce10d..4f7134235 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,6 +115,9 @@ importers: date-fns: specifier: ^4.1.0 version: 4.1.0 + gray-matter: + specifier: ^4.0.3 + version: 4.0.3 mime-types: specifier: ^3.0.1 version: 3.0.2 @@ -310,7 +313,7 @@ importers: version: 3.5.2(react@18.2.0) roamjs-components: specifier: 0.86.4 - version: 0.86.4(ad0016961dcdb2b46ced8f4d10229b4e) + version: 0.86.4(5b78e88b35a4681c3fec2bd59ab560ac) tldraw: specifier: 2.4.6 version: 2.4.6(patch_hash=56e196052862c9a58a11b43e5e121384cd1d6548416afa0f16e9fbfbf0e4080d)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -362,7 +365,7 @@ importers: version: 0.17.14 tailwindcss: specifier: ^3.4.17 - version: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) + version: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)) tsx: specifier: ^4.19.2 version: 4.20.5 @@ -601,10 +604,10 @@ importers: version: link:../typescript-config tailwindcss: specifier: ^3.4.1 - version: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) + version: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)) tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3))) + version: 1.0.7(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4))) packages/types: {} @@ -6173,11 +6176,13 @@ packages: glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@11.0.3: resolution: {integrity: sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==} engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@13.0.0: @@ -6186,7 +6191,7 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} @@ -6931,7 +6936,7 @@ packages: lodash.isequal@4.5.0: resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} - deprecated: This package is deprecated. Use require("node:util").isDeepStrictEqual instead. + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. lodash.isequalwith@4.4.0: resolution: {integrity: sha512-dcZON0IalGBpRmJBmMkaoV7d3I80R2O+FrzsZyHdNSFrANq/cgDqKQNmAHE8UEj4+QYWwwhkQOVdLHiAopzlsQ==} @@ -7364,7 +7369,7 @@ packages: node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} - deprecated: Use your platform"s native DOMException instead + deprecated: Use your platform's native DOMException instead node-fetch@2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} @@ -8859,12 +8864,12 @@ packages: tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting i@izs.me + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tar@7.5.2: resolution: {integrity: sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==} engines: {node: '>=18'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting i@izs.me + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tar@7.5.7: resolution: {integrity: sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==} @@ -12959,22 +12964,22 @@ snapshots: '@rushstack/eslint-patch@1.12.0': {} - '@samepage/scripts@0.74.5(@aws-sdk/client-lambda@3.882.0)(@aws-sdk/client-s3@3.882.0)(@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)))(archiver@5.3.2)(axios@0.27.2(debug@4.4.1))(debug@4.4.1)(dotenv@16.6.1)(esbuild@0.17.14)(patch-package@6.5.1)(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)))(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3))(zod@3.25.76)': + '@samepage/scripts@0.74.5(@aws-sdk/client-lambda@3.882.0)(@aws-sdk/client-s3@3.882.0)(@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)))(archiver@5.3.2)(axios@0.27.2(debug@4.4.1))(debug@4.4.1)(dotenv@16.6.1)(esbuild@0.17.14)(patch-package@6.5.1)(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)))(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4))(zod@3.25.76)': dependencies: '@aws-sdk/client-lambda': 3.882.0 '@aws-sdk/client-s3': 3.882.0 - '@samepage/testing': 0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) + '@samepage/testing': 0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)) archiver: 5.3.2 axios: 0.27.2(debug@4.4.1) debug: 4.4.1(supports-color@8.1.1) dotenv: 16.6.1 esbuild: 0.17.14 patch-package: 6.5.1 - tailwindcss: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) - ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.9.3) + tailwindcss: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)) + ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.5.4) zod: 3.25.76 - '@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3))': + '@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4))': dependencies: '@playwright/test': 1.29.0 '@testing-library/react': 16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -12984,7 +12989,7 @@ snapshots: debug: 4.4.1(supports-color@8.1.1) dotenv: 16.6.1 jsdom: 20.0.3 - ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.9.3) + ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.5.4) '@selderee/plugin-htmlparser2@0.11.0': dependencies: @@ -18222,14 +18227,6 @@ snapshots: postcss: 8.5.6 ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.5.4) - postcss-load-config@4.0.2(postcss@8.5.6)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)): - dependencies: - lilconfig: 3.1.3 - yaml: 2.8.1 - optionalDependencies: - postcss: 8.5.6 - ts-node: 10.9.2(@types/node@20.19.13)(typescript@5.9.3) - postcss-nested@6.2.0(postcss@8.5.6): dependencies: postcss: 8.5.6 @@ -18986,12 +18983,12 @@ snapshots: dependencies: glob: 7.2.3 - roamjs-components@0.86.4(ad0016961dcdb2b46ced8f4d10229b4e): + roamjs-components@0.86.4(5b78e88b35a4681c3fec2bd59ab560ac): dependencies: '@blueprintjs/core': 3.50.4(patch_hash=51c5847e0a73a1be0cc263036ff64d8fada46f3b65831ed938dbca5eecf3edc0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@blueprintjs/datetime': 3.23.14(react-dom@18.2.0(react@18.2.0))(react@18.2.0) '@blueprintjs/select': 3.19.1(patch_hash=5b2821b0bf7274e9b64d7824648c596b9e73c61f421d699a6d4c494f12f62355)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) - '@samepage/scripts': 0.74.5(@aws-sdk/client-lambda@3.882.0)(@aws-sdk/client-s3@3.882.0)(@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)))(archiver@5.3.2)(axios@0.27.2(debug@4.4.1))(debug@4.4.1)(dotenv@16.6.1)(esbuild@0.17.14)(patch-package@6.5.1)(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)))(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3))(zod@3.25.76) + '@samepage/scripts': 0.74.5(@aws-sdk/client-lambda@3.882.0)(@aws-sdk/client-s3@3.882.0)(@samepage/testing@0.74.5(@playwright/test@1.29.0)(@testing-library/react@16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@18.2.17)(@types/react@18.2.21)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1))(@types/jsdom@20.0.1)(c8@7.14.0)(debug@4.4.1)(dotenv@16.6.1)(jsdom@20.0.3)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)))(archiver@5.3.2)(axios@0.27.2(debug@4.4.1))(debug@4.4.1)(dotenv@16.6.1)(esbuild@0.17.14)(patch-package@6.5.1)(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)))(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4))(zod@3.25.76) '@types/crypto-js': 4.1.1 '@types/cytoscape': 3.21.9 '@types/file-saver': 2.0.5 @@ -19589,10 +19586,6 @@ snapshots: dependencies: tailwindcss: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)) - tailwindcss-animate@1.0.7(tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3))): - dependencies: - tailwindcss: 3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) - tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.5.4)): dependencies: '@alloc/quick-lru': 5.2.0 @@ -19620,33 +19613,6 @@ snapshots: transitivePeerDependencies: - ts-node - tailwindcss@3.4.17(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)): - dependencies: - '@alloc/quick-lru': 5.2.0 - arg: 5.0.2 - chokidar: 3.6.0 - didyoumean: 1.2.2 - dlv: 1.1.3 - fast-glob: 3.3.3 - glob-parent: 6.0.2 - is-glob: 4.0.3 - jiti: 1.21.7 - lilconfig: 3.1.3 - micromatch: 4.0.8 - normalize-path: 3.0.0 - object-hash: 3.0.0 - picocolors: 1.1.1 - postcss: 8.5.6 - postcss-import: 15.1.0(postcss@8.5.6) - postcss-js: 4.0.1(postcss@8.5.6) - postcss-load-config: 4.0.2(postcss@8.5.6)(ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3)) - postcss-nested: 6.2.0(postcss@8.5.6) - postcss-selector-parser: 6.1.2 - resolve: 1.22.10 - sucrase: 3.35.0 - transitivePeerDependencies: - - ts-node - tar-stream@2.2.0: dependencies: bl: 4.1.0 @@ -19893,24 +19859,6 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - ts-node@10.9.2(@types/node@20.19.13)(typescript@5.9.3): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.11 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 20.19.13 - acorn: 8.15.0 - acorn-walk: 8.3.4 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.9.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - ts-toolbelt@6.15.5: {} ts-toolbelt@9.6.0: {} From 5967490bfca5051b7f9d43bf8656a81c72a796b1 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 8 Feb 2026 00:46:45 -0500 Subject: [PATCH 11/33] address PR comments --- .../src/components/DiscourseContextView.tsx | 14 ++ .../src/components/ImportNodesModal.tsx | 8 +- apps/obsidian/src/types.ts | 3 + apps/obsidian/src/utils/conceptConversion.ts | 3 +- apps/obsidian/src/utils/importNodes.ts | 236 +++++++++--------- .../src/utils/syncDgNodesToSupabase.ts | 14 +- packages/database/scripts/createGroup.ts | 176 +++++++++++++ 7 files changed, 319 insertions(+), 135 deletions(-) create mode 100644 packages/database/scripts/createGroup.ts diff --git a/apps/obsidian/src/components/DiscourseContextView.tsx b/apps/obsidian/src/components/DiscourseContextView.tsx index 555beebb3..f7b5f537b 100644 --- a/apps/obsidian/src/components/DiscourseContextView.tsx +++ b/apps/obsidian/src/components/DiscourseContextView.tsx @@ -74,6 +74,13 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { } const isImported = !!frontmatter.importedFromSpaceUri; + const sourceDates = + isImported && activeFile?.stat + ? { + createdAt: new Date(activeFile.stat.ctime).toLocaleString(), + modifiedAt: new Date(activeFile.stat.mtime).toLocaleString(), + } + : null; return ( <> @@ -106,6 +113,13 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { {extractContentFromTitle(nodeType.format, activeFile.basename)}
)} + + {isImported && sourceDates && ( +
+
Created in source: {sourceDates.createdAt}
+
Last modified in source: {sourceDates.modifiedAt}
+
+ )}
diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index 8537a5d2d..30596efdc 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -74,11 +74,11 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { const grouped: Map = new Map(); for (const node of importableNodes) { - const groupId = node.account_uid; + const groupId = String(node.space_id); if (!grouped.has(groupId)) { grouped.set(groupId, { groupId, - groupName: `Group ${groupId.slice(0, 8)}...`, + groupName: spaceNames.get(node.space_id) ?? `Space ${node.space_id}`, nodes: [], }); } @@ -89,8 +89,10 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { title: node.text, spaceId: node.space_id, spaceName: spaceNames.get(node.space_id) ?? `Space ${node.space_id}`, - groupId: node.account_uid, + groupId, selected: false, + createdAt: node.createdAt, + modifiedAt: node.modifiedAt, }); } diff --git a/apps/obsidian/src/types.ts b/apps/obsidian/src/types.ts index b0a0925b1..54af16730 100644 --- a/apps/obsidian/src/types.ts +++ b/apps/obsidian/src/types.ts @@ -68,6 +68,9 @@ export type ImportableNode = { spaceName: string; groupId: string; selected: boolean; + /** From source Content (latest last_modified across variants). Set when loaded from getPublishedNodesForGroups. */ + createdAt?: string; + modifiedAt?: string; }; export type GroupWithNodes = { diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index 161ec1a3f..e1737d898 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -5,6 +5,7 @@ import type { SupabaseContext } from "./supabaseContext"; import type { LocalConceptDataInput } from "@repo/database/inputTypes"; import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase"; import type { Json } from "@repo/database/dbTypes"; +import DiscourseGraphPlugin from ".."; /** * Get extra data (author, timestamps) from file metadata @@ -37,7 +38,7 @@ export const discourseNodeSchemaToLocalConcept = ({ node; return { space_id: context.spaceId, - name: id ? `${name} (${id})` : name, + name: `${name}`, source_local_id: id, is_schema: true, author_local_id: accountLocalId, diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 606625e3f..83ad57d1c 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -36,90 +36,75 @@ export const getPublishedNodesForGroups = async ({ source_local_id: string; space_id: number; text: string; - account_uid: string; + createdAt: string; + modifiedAt: string; }> > => { if (groupIds.length === 0) { return []; } - // First get all ResourceAccess entries for these groups - const { data: resourceAccessData, error: raError } = await client - .from("ResourceAccess") - .select("source_local_id, space_id, account_uid") - .in("account_uid", groupIds) + // Query my_contents (RLS applied); exclude current space. Get both variants so we can use + // the latest last_modified per node and prefer "direct" for text (title). + const { data, error } = await client + .from("my_contents") + .select("source_local_id, space_id, text, created, last_modified, variant") .neq("space_id", currentSpaceId); - if (raError) { - console.error("Error fetching resource access:", raError); - throw new Error(`Failed to fetch resource access: ${raError.message}`); + if (error) { + console.error("Error fetching published nodes:", error); + throw new Error(`Failed to fetch published nodes: ${error.message}`); } - if (!resourceAccessData || resourceAccessData.length === 0) { + if (!data || data.length === 0) { return []; } - // Group unique (space_id, source_local_id) by space_id for batched queries. - // Build nodesBySpace directly so source_local_id can contain any character (e.g. colons). - const nodesBySpace = new Map>(); - for (const ra of resourceAccessData) { - if (ra.source_local_id && ra.space_id != null) { - if (!nodesBySpace.has(ra.space_id)) { - nodesBySpace.set(ra.space_id, new Set()); - } - nodesBySpace.get(ra.space_id)!.add(ra.source_local_id); - } + type Row = { + source_local_id: string | null; + space_id: number | null; + text: string | null; + created: string | null; + last_modified: string | null; + variant: string | null; + }; + + const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; + const groups = new Map(); + for (const row of data as Row[]) { + if (row.source_local_id == null || row.space_id == null) continue; + const k = key(row); + if (!groups.has(k)) groups.set(k, []); + groups.get(k)!.push(row); } - // Fetch Content with variant "direct" for all nodes, batched by space const nodes: Array<{ source_local_id: string; space_id: number; text: string; - account_uid: string; + createdAt: string; + modifiedAt: string; }> = []; - for (const [spaceId, sourceLocalIds] of nodesBySpace.entries()) { - const sourceLocalIdsArray = Array.from(sourceLocalIds); - if (sourceLocalIdsArray.length === 0) { - continue; - } - - // Single query for all nodes in this space - const { data: contentDataArray } = await client - .from("Content") - .select("source_local_id, text") - .eq("space_id", spaceId) - .eq("variant", "direct") - .in("source_local_id", sourceLocalIdsArray); - - if (!contentDataArray || contentDataArray.length === 0) { - continue; - } - - // Create a map for quick lookup - const contentMap = new Map(); - for (const content of contentDataArray) { - if (content.source_local_id && content.text) { - contentMap.set(content.source_local_id, content.text); - } - } - - // Match content with ResourceAccess entries - for (const ra of resourceAccessData) { - if ( - ra.space_id === spaceId && - ra.source_local_id && - contentMap.has(ra.source_local_id) - ) { - nodes.push({ - source_local_id: ra.source_local_id, - space_id: spaceId, - text: contentMap.get(ra.source_local_id)!, - account_uid: ra.account_uid, - }); - } - } + for (const rows of groups.values()) { + const withDate = rows.filter( + (r) => r.last_modified != null && r.text != null, + ); + if (withDate.length === 0) continue; + const latest = withDate.reduce((a, b) => + (a.last_modified ?? "") >= (b.last_modified ?? "") ? a : b, + ); + const direct = rows.find((r) => r.variant === "direct"); + const text = direct?.text ?? latest.text ?? ""; + const createdAt = latest.created ?? new Date(0).toISOString(); + const modifiedAt = latest.last_modified ?? new Date(0).toISOString(); + nodes.push({ + source_local_id: latest.source_local_id!, + space_id: latest.space_id!, + text, + createdAt, + modifiedAt, + }); } return nodes; @@ -243,14 +228,14 @@ export const fetchNodeContent = async ({ variant: "direct" | "full"; }): Promise => { const { data, error } = await client - .from("Content") + .from("my_contents") .select("text") .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) .maybeSingle(); - if (error || !data) { + if (error || !data || data.text == null) { console.error( `Error fetching node content (${variant}):`, error || "No data", @@ -277,14 +262,14 @@ export const fetchNodeContentWithMetadata = async ({ modifiedAt: string; } | null> => { const { data, error } = await client - .from("Content") + .from("my_contents") .select("text, created, last_modified") .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) .maybeSingle(); - if (error || !data) { + if (error || !data || data.text == null) { console.error( `Error fetching node content with metadata (${variant}):`, error || "No data", @@ -299,6 +284,37 @@ export const fetchNodeContentWithMetadata = async ({ }; }; +/** + * Fetches created/last_modified from the source space Content (my_contents) for an imported node. + * Used by the discourse context view to show "last modified in original vault". + */ +export const getSourceContentDates = async ({ + plugin, + nodeInstanceId, + spaceUri, +}: { + plugin: DiscourseGraphPlugin; + nodeInstanceId: string; + spaceUri: string; +}): Promise<{ createdAt: string; modifiedAt: string } | null> => { + const client = await getLoggedInClient(plugin); + if (!client) return null; + const { spaceId } = await getSpaceNameIdFromUri(client, spaceUri); + if (spaceId < 0) return null; + const { data, error } = await client + .from("my_contents") + .select("created, last_modified") + .eq("source_local_id", nodeInstanceId) + .eq("space_id", spaceId) + .eq("variant", "direct") + .maybeSingle(); + if (error || !data) return null; + return { + createdAt: data.created ?? new Date(0).toISOString(), + modifiedAt: data.last_modified ?? new Date(0).toISOString(), + }; +}; + const fetchFileReferences = async ({ client, spaceId, @@ -598,7 +614,7 @@ const importAssetsForNode = async ({ const { name, ext } = extractFileName(filepath); // Check if we already have a file for this hash - let existingAssetPath: string | undefined = importedAssets[filehash]; + const existingAssetPath: string | undefined = importedAssets[filehash]; let existingFile: TFile | null = null; if (existingAssetPath) { @@ -606,44 +622,6 @@ const importAssetsForNode = async ({ const file = plugin.app.vault.getAbstractFileByPath(existingAssetPath); if (file && file instanceof TFile) { existingFile = file; - } else { - // File was moved/renamed - search for it in the assets folder - // Try to find a file with the same name in the assets folder - const { name: fileName, ext: fileExt } = - extractFileName(existingAssetPath); - const searchFileName = `${fileName}${fileExt ? `.${fileExt}` : ""}`; - - // Search all files in the assets folder - const allFiles = plugin.app.vault.getFiles(); - for (const vaultFile of allFiles) { - if ( - vaultFile instanceof TFile && - vaultFile.path.startsWith(assetsFolderPath) && - vaultFile.basename === fileName && - vaultFile.extension === fileExt - ) { - // Found a file with matching name in assets folder - likely the same file - existingFile = vaultFile; - existingAssetPath = vaultFile.path; - // Update frontmatter with new path - await plugin.app.fileManager.processFrontMatter( - targetMarkdownFile, - (fm) => { - const assetsRaw = (fm as Record) - .importedAssets; - const assets: Record = - assetsRaw && - typeof assetsRaw === "object" && - !Array.isArray(assetsRaw) - ? (assetsRaw as Record) - : {}; - assets[filehash] = vaultFile.path; - (fm as Record).importedAssets = assets; - }, - ); - break; - } - } } } @@ -718,7 +696,19 @@ const importAssetsForNode = async ({ } // Save file to vault - await plugin.app.vault.createBinary(targetPath, fileContent); + const existingFileForOverwrite = + plugin.app.vault.getAbstractFileByPath(targetPath); + if ( + existingFileForOverwrite && + existingFileForOverwrite instanceof TFile + ) { + await plugin.app.vault.modifyBinary( + existingFileForOverwrite, + fileContent, + ); + } else { + await plugin.app.vault.createBinary(targetPath, fileContent); + } // Update frontmatter to track this mapping await plugin.app.fileManager.processFrontMatter( @@ -822,9 +812,9 @@ const mapNodeTypeIdToLocal = async ({ sourceSpaceId: number; sourceNodeTypeId: string; }): Promise => { - // Find the schema in the source space with this nodeTypeId + // Find the schema in the source space with this nodeTypeId (my_concepts applies RLS) const { data: schemaData } = await client - .from("Concept") + .from("my_concepts") .select("name, literal_content") .eq("space_id", sourceSpaceId) .eq("is_schema", true) @@ -896,12 +886,20 @@ const processFileContent = async ({ importedCreatedAt?: string; importedModifiedAt?: string; }): Promise<{ file: TFile; error?: string }> => { - // 1. Create or update the file with the fetched content first + // 1. Create or update the file with the fetched content first. + // On create, set file metadata (ctime/mtime) to original vault dates via vault adapter. let file: TFile | null = plugin.app.vault.getFileByPath(filePath); + const stat = + importedCreatedAt !== undefined && importedModifiedAt !== undefined + ? { + ctime: new Date(importedCreatedAt).getTime(), + mtime: new Date(importedModifiedAt).getTime(), + } + : undefined; if (!file) { - file = await plugin.app.vault.create(filePath, rawContent); + file = await plugin.app.vault.create(filePath, rawContent, stat); } else { - await plugin.app.vault.modify(file, rawContent); + await plugin.app.vault.modify(file, rawContent, stat); } // 2. Parse frontmatter from rawContent (metadataCache is updated async and is @@ -925,12 +923,6 @@ const processFileContent = async ({ record.nodeTypeId = mappedNodeTypeId; } record.importedFromSpaceUri = sourceSpaceUri; - if (importedCreatedAt !== undefined) { - record.importedCreatedAt = importedCreatedAt; - } - if (importedModifiedAt !== undefined) { - record.importedModifiedAt = importedModifiedAt; - } }); return { file }; @@ -1042,7 +1034,9 @@ export const importSelectedNodes = async ({ continue; } - const { content, createdAt, modifiedAt } = contentWithMeta; + const { content } = contentWithMeta; + const createdAt = node.createdAt ?? contentWithMeta.createdAt; + const modifiedAt = node.modifiedAt ?? contentWithMeta.modifiedAt; // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); @@ -1168,8 +1162,8 @@ export const refreshImportedFile = async ({ throw new Error("Cannot get Supabase client"); } const cache = plugin.app.metadataCache.getFileCache(file); - const frontmatter = cache?.frontmatter as Record; - if (!frontmatter.importedFromSpaceUri || !frontmatter.nodeInstanceId) { + const frontmatter = cache?.frontmatter as Record | undefined; + if (!frontmatter?.importedFromSpaceUri || !frontmatter?.nodeInstanceId) { return { success: false, error: "Missing frontmatter: importedFromSpaceUri or nodeInstanceId", diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index d47344d64..5c8d3280c 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -514,15 +514,11 @@ const convertDgToSupabaseConcepts = async ({ const lastSchemaSync = ( await getLastSchemaSyncTime(supabaseClient, context.spaceId) ).getTime(); - const schemaIdsReferencedByInstances = new Set( - nodesSince.map((n) => n.nodeTypeId), - ); - const nodeTypesToUpsert = (plugin.settings.nodeTypes ?? []).filter( - (n) => - n.modified > lastSchemaSync || schemaIdsReferencedByInstances.has(n.id), + const newNodeTypes = (plugin.settings.nodeTypes ?? []).filter( + (n) => n.modified > lastSchemaSync, ); - const nodesTypesToLocalConcepts = nodeTypesToUpsert.map((nodeType) => + const nodesTypesToLocalConcepts = newNodeTypes.map((nodeType) => discourseNodeSchemaToLocalConcept({ context, node: nodeType, @@ -543,13 +539,11 @@ const convertDgToSupabaseConcepts = async ({ ...nodeInstanceToLocalConcepts, ]; - console.log("conceptsToUpsert", conceptsToUpsert); - if (conceptsToUpsert.length > 0) { const { ordered } = orderConceptsByDependency(conceptsToUpsert); const { error } = await supabaseClient.rpc("upsert_concepts", { - data: conceptsToUpsert as Json, + data: ordered as Json, v_space_id: context.spaceId, }); diff --git a/packages/database/scripts/createGroup.ts b/packages/database/scripts/createGroup.ts new file mode 100644 index 000000000..1d3fba1a8 --- /dev/null +++ b/packages/database/scripts/createGroup.ts @@ -0,0 +1,176 @@ +/** + * Create a group and/or add member UUIDs to group_membership. + * + * Requires: SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY in .env (or environment) + * + * Usage: + * pnpm run create-group-and-members [uuid1] [uuid2] ... + * + * - If the group already exists (by name: {name}@groups.discoursegraphs.com): adds the + * given UUIDs to group_membership. If a UUID is already in the group, it is ignored. + * - If the group does not exist: creates it and adds the UUIDs (first one becomes admin). + * - With an existing group, memberUids can be empty (no-op). + * - When creating a new group, at least one member UUID is required. + */ + +import { createClient } from "@supabase/supabase-js"; +import dotenv from "dotenv"; +import type { Database } from "@repo/database/dbTypes"; + +dotenv.config(); + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const isUuid = (s: string): boolean => UUID_RE.test(s); + +const parseArgs = (): { groupName: string; memberUids: string[] } => { + const argv = process.argv.slice(2); + const groupName = argv[0]; + const memberUids = argv.slice(1); + + if (!groupName) { + console.error( + "Usage: create-group-and-members [uuid1] [uuid2] ...", + ); + process.exit(1); + } + + const invalid = memberUids.filter((u) => !isUuid(u)); + if (invalid.length > 0) { + console.error("Invalid UUID(s):", invalid.join(", ")); + process.exit(1); + } + + return { groupName, memberUids }; +}; + +const groupEmail = (name: string): string => + `${name}@groups.discoursegraphs.com`; + +const findExistingGroup = async ( + supabase: ReturnType>, + groupName: string, +): Promise<{ id: string } | null> => { + const email = groupEmail(groupName); + let page = 1; + const perPage = 1000; + + while (true) { + const { data, error } = await supabase.auth.admin.listUsers({ + page, + perPage, + }); + + if (error) { + console.error("Failed to list users:", error.message); + process.exit(1); + } + + const users = data?.users ?? []; + const found = users.find((u) => u.email === email); + if (found) return { id: found.id }; + + if (users.length < perPage) return null; + page += 1; + } +}; + +const main = async (): Promise => { + const { groupName, memberUids } = parseArgs(); + + const url = process.env.SUPABASE_URL; + const serviceKey = process.env.SUPABASE_SERVICE_ROLE_KEY; + if (!url || !serviceKey) { + console.error( + "Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY in .env or environment", + ); + process.exit(1); + } + + const supabase = createClient(url, serviceKey, { + auth: { persistSession: false }, + }); + + const existing = await findExistingGroup(supabase, groupName); + console.log("existing", existing); + let resolvedGroupId: string; + let isNewGroup: boolean; + + if (existing) { + resolvedGroupId = existing.id; + isNewGroup = false; + console.log("Using existing group:", resolvedGroupId, `(${groupName})`); + } else { + if (memberUids.length === 0) { + console.error( + "When creating a new group, provide at least one member UUID.", + ); + process.exit(1); + } + + const email = groupEmail(groupName); + const password = crypto.randomUUID(); + + const { data, error } = await supabase.auth.admin.createUser({ + email, + password, + role: "anon", + user_metadata: { group: true }, + email_confirm: false, + }); + + if (error) { + if ((error as { code?: string }).code === "email_exists") { + console.error("A group with this name already exists:", email); + process.exit(1); + } + console.error("Failed to create group user:", error.message); + process.exit(1); + } + + if (!data.user) { + console.error("Failed to create group user: no user returned"); + process.exit(1); + } + + resolvedGroupId = data.user.id; + isNewGroup = true; + console.log("Created group:", resolvedGroupId, `(${groupName})`); + } + + let added = 0; + let skipped = 0; + + for (let i = 0; i < memberUids.length; i++) { + const member_id = memberUids[i]!; + const admin = isNewGroup && i === 0; + + const { error } = await supabase.from("group_membership").insert({ + group_id: resolvedGroupId, + member_id, + admin, + }); + + if (error) { + if (error.code === "23505") { + skipped += 1; + } else { + console.error("Failed to insert member", member_id, ":", error.message); + process.exit(1); + } + } else { + added += 1; + } + } + + console.log("Added", added, "member(s) to group_membership."); + if (skipped > 0) { + console.log("Skipped", skipped, "member(s) already in the group."); + } + if (isNewGroup && added > 0) { + console.log("Admin:", memberUids[0]); + } +}; + +main(); From 9a25b5bec821d149ffe75ac66b4321dc1576bb0b Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 8 Feb 2026 00:53:08 -0500 Subject: [PATCH 12/33] cleanup --- packages/database/scripts/createGroup.ts | 176 ----------------------- 1 file changed, 176 deletions(-) delete mode 100644 packages/database/scripts/createGroup.ts diff --git a/packages/database/scripts/createGroup.ts b/packages/database/scripts/createGroup.ts deleted file mode 100644 index 1d3fba1a8..000000000 --- a/packages/database/scripts/createGroup.ts +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Create a group and/or add member UUIDs to group_membership. - * - * Requires: SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY in .env (or environment) - * - * Usage: - * pnpm run create-group-and-members [uuid1] [uuid2] ... - * - * - If the group already exists (by name: {name}@groups.discoursegraphs.com): adds the - * given UUIDs to group_membership. If a UUID is already in the group, it is ignored. - * - If the group does not exist: creates it and adds the UUIDs (first one becomes admin). - * - With an existing group, memberUids can be empty (no-op). - * - When creating a new group, at least one member UUID is required. - */ - -import { createClient } from "@supabase/supabase-js"; -import dotenv from "dotenv"; -import type { Database } from "@repo/database/dbTypes"; - -dotenv.config(); - -const UUID_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -const isUuid = (s: string): boolean => UUID_RE.test(s); - -const parseArgs = (): { groupName: string; memberUids: string[] } => { - const argv = process.argv.slice(2); - const groupName = argv[0]; - const memberUids = argv.slice(1); - - if (!groupName) { - console.error( - "Usage: create-group-and-members [uuid1] [uuid2] ...", - ); - process.exit(1); - } - - const invalid = memberUids.filter((u) => !isUuid(u)); - if (invalid.length > 0) { - console.error("Invalid UUID(s):", invalid.join(", ")); - process.exit(1); - } - - return { groupName, memberUids }; -}; - -const groupEmail = (name: string): string => - `${name}@groups.discoursegraphs.com`; - -const findExistingGroup = async ( - supabase: ReturnType>, - groupName: string, -): Promise<{ id: string } | null> => { - const email = groupEmail(groupName); - let page = 1; - const perPage = 1000; - - while (true) { - const { data, error } = await supabase.auth.admin.listUsers({ - page, - perPage, - }); - - if (error) { - console.error("Failed to list users:", error.message); - process.exit(1); - } - - const users = data?.users ?? []; - const found = users.find((u) => u.email === email); - if (found) return { id: found.id }; - - if (users.length < perPage) return null; - page += 1; - } -}; - -const main = async (): Promise => { - const { groupName, memberUids } = parseArgs(); - - const url = process.env.SUPABASE_URL; - const serviceKey = process.env.SUPABASE_SERVICE_ROLE_KEY; - if (!url || !serviceKey) { - console.error( - "Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY in .env or environment", - ); - process.exit(1); - } - - const supabase = createClient(url, serviceKey, { - auth: { persistSession: false }, - }); - - const existing = await findExistingGroup(supabase, groupName); - console.log("existing", existing); - let resolvedGroupId: string; - let isNewGroup: boolean; - - if (existing) { - resolvedGroupId = existing.id; - isNewGroup = false; - console.log("Using existing group:", resolvedGroupId, `(${groupName})`); - } else { - if (memberUids.length === 0) { - console.error( - "When creating a new group, provide at least one member UUID.", - ); - process.exit(1); - } - - const email = groupEmail(groupName); - const password = crypto.randomUUID(); - - const { data, error } = await supabase.auth.admin.createUser({ - email, - password, - role: "anon", - user_metadata: { group: true }, - email_confirm: false, - }); - - if (error) { - if ((error as { code?: string }).code === "email_exists") { - console.error("A group with this name already exists:", email); - process.exit(1); - } - console.error("Failed to create group user:", error.message); - process.exit(1); - } - - if (!data.user) { - console.error("Failed to create group user: no user returned"); - process.exit(1); - } - - resolvedGroupId = data.user.id; - isNewGroup = true; - console.log("Created group:", resolvedGroupId, `(${groupName})`); - } - - let added = 0; - let skipped = 0; - - for (let i = 0; i < memberUids.length; i++) { - const member_id = memberUids[i]!; - const admin = isNewGroup && i === 0; - - const { error } = await supabase.from("group_membership").insert({ - group_id: resolvedGroupId, - member_id, - admin, - }); - - if (error) { - if (error.code === "23505") { - skipped += 1; - } else { - console.error("Failed to insert member", member_id, ":", error.message); - process.exit(1); - } - } else { - added += 1; - } - } - - console.log("Added", added, "member(s) to group_membership."); - if (skipped > 0) { - console.log("Skipped", skipped, "member(s) already in the group."); - } - if (isNewGroup && added > 0) { - console.log("Admin:", memberUids[0]); - } -}; - -main(); From 9a56c72687b54171ac5912b9538e37add8abfa36 Mon Sep 17 00:00:00 2001 From: Trang Doan <44855874+trangdoan982@users.noreply.github.com> Date: Sat, 7 Feb 2026 22:03:30 -0800 Subject: [PATCH 13/33] Update apps/obsidian/src/utils/importNodes.ts Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/obsidian/src/utils/importNodes.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 83ad57d1c..78ebaa634 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -973,9 +973,15 @@ export const importSelectedNodes = async ({ const spaceUri = spaceUris.get(spaceId); if (!spaceUri) { console.warn(`Missing URI for space ${spaceId}`); + for (const _node of nodes) { + failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); + } continue; } + // Ensure the import folder exists const folderExists = await plugin.app.vault.adapter.exists(importFolderPath); From 649bb02af5ae9a0179e648c347d9ea12f5e67ed0 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 10:30:38 -0500 Subject: [PATCH 14/33] normalize on using numbers for dates --- .../src/components/ImportNodesModal.tsx | 3 +- apps/obsidian/src/types.ts | 4 +-- apps/obsidian/src/utils/importNodes.ts | 33 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index 30596efdc..24fac137e 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -78,7 +78,8 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { if (!grouped.has(groupId)) { grouped.set(groupId, { groupId, - groupName: spaceNames.get(node.space_id) ?? `Space ${node.space_id}`, + groupName: + spaceNames.get(node.space_id) ?? `Space ${node.space_id}`, nodes: [], }); } diff --git a/apps/obsidian/src/types.ts b/apps/obsidian/src/types.ts index 54af16730..6f61049a3 100644 --- a/apps/obsidian/src/types.ts +++ b/apps/obsidian/src/types.ts @@ -69,8 +69,8 @@ export type ImportableNode = { groupId: string; selected: boolean; /** From source Content (latest last_modified across variants). Set when loaded from getPublishedNodesForGroups. */ - createdAt?: string; - modifiedAt?: string; + createdAt?: number; + modifiedAt?: number; }; export type GroupWithNodes = { diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 78ebaa634..6da436e2b 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -36,8 +36,8 @@ export const getPublishedNodesForGroups = async ({ source_local_id: string; space_id: number; text: string; - createdAt: string; - modifiedAt: string; + createdAt: number; + modifiedAt: number; }> > => { if (groupIds.length === 0) { @@ -64,8 +64,8 @@ export const getPublishedNodesForGroups = async ({ source_local_id: string | null; space_id: number | null; text: string | null; - created: string | null; - last_modified: string | null; + created: number | null; + last_modified: number | null; variant: string | null; }; @@ -82,8 +82,8 @@ export const getPublishedNodesForGroups = async ({ source_local_id: string; space_id: number; text: string; - createdAt: string; - modifiedAt: string; + createdAt: number; + modifiedAt: number; }> = []; for (const rows of groups.values()) { @@ -96,8 +96,8 @@ export const getPublishedNodesForGroups = async ({ ); const direct = rows.find((r) => r.variant === "direct"); const text = direct?.text ?? latest.text ?? ""; - const createdAt = latest.created ?? new Date(0).toISOString(); - const modifiedAt = latest.last_modified ?? new Date(0).toISOString(); + const createdAt = latest.created ?? 0; + const modifiedAt = latest.last_modified ?? 0; nodes.push({ source_local_id: latest.source_local_id!, space_id: latest.space_id!, @@ -258,8 +258,8 @@ export const fetchNodeContentWithMetadata = async ({ variant: "direct" | "full"; }): Promise<{ content: string; - createdAt: string; - modifiedAt: string; + createdAt: number; + modifiedAt: number; } | null> => { const { data, error } = await client .from("my_contents") @@ -279,8 +279,8 @@ export const fetchNodeContentWithMetadata = async ({ return { content: data.text, - createdAt: data.created ?? new Date(0).toISOString(), - modifiedAt: data.last_modified ?? new Date(0).toISOString(), + createdAt: new Date(data.created ?? 0).valueOf(), + modifiedAt: new Date(data.last_modified ?? 0).valueOf(), }; }; @@ -883,8 +883,8 @@ const processFileContent = async ({ sourceSpaceUri: string; rawContent: string; filePath: string; - importedCreatedAt?: string; - importedModifiedAt?: string; + importedCreatedAt?: number; + importedModifiedAt?: number; }): Promise<{ file: TFile; error?: string }> => { // 1. Create or update the file with the fetched content first. // On create, set file metadata (ctime/mtime) to original vault dates via vault adapter. @@ -892,8 +892,8 @@ const processFileContent = async ({ const stat = importedCreatedAt !== undefined && importedModifiedAt !== undefined ? { - ctime: new Date(importedCreatedAt).getTime(), - mtime: new Date(importedModifiedAt).getTime(), + ctime: importedCreatedAt, + mtime: importedModifiedAt, } : undefined; if (!file) { @@ -981,7 +981,6 @@ export const importSelectedNodes = async ({ continue; } - // Ensure the import folder exists const folderExists = await plugin.app.vault.adapter.exists(importFolderPath); From af8dc7814dc9ae5f88a41f7e291113021bef744a Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 10:49:20 -0500 Subject: [PATCH 15/33] correction --- apps/obsidian/src/utils/importNodes.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 6da436e2b..ad644b676 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -64,8 +64,8 @@ export const getPublishedNodesForGroups = async ({ source_local_id: string | null; space_id: number | null; text: string | null; - created: number | null; - last_modified: number | null; + created: string | null; + last_modified: string | null; variant: string | null; }; @@ -96,8 +96,10 @@ export const getPublishedNodesForGroups = async ({ ); const direct = rows.find((r) => r.variant === "direct"); const text = direct?.text ?? latest.text ?? ""; - const createdAt = latest.created ?? 0; - const modifiedAt = latest.last_modified ?? 0; + const createdAt = latest.created ? new Date(latest.created).valueOf() : 0; + const modifiedAt = latest.last_modified + ? new Date(latest.last_modified).valueOf() + : 0; nodes.push({ source_local_id: latest.source_local_id!, space_id: latest.space_id!, From 0b3486685881f59e14d2e6de35788bd7e4e00ab6 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 10:57:40 -0500 Subject: [PATCH 16/33] apply ctime, mtime to assets --- apps/obsidian/src/utils/importNodes.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index ad644b676..eaaac9e49 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -329,8 +329,8 @@ const fetchFileReferences = async ({ Array<{ filepath: string; filehash: string; - created: string; - last_modified: string; + created: number; + last_modified: number; }> > => { const { data, error } = await client @@ -344,7 +344,12 @@ const fetchFileReferences = async ({ return []; } - return data || []; + return data.map(({ filepath, filehash, created, last_modified }) => ({ + filepath, + filehash, + created: created ? new Date(created).valueOf() : 0, + last_modified: last_modified ? new Date(last_modified).valueOf() : 0, + })); }; const downloadFileFromStorage = async ({ @@ -697,6 +702,7 @@ const importAssetsForNode = async ({ continue; } + const options = { mtime: fileRef.last_modified, ctime: fileRef.created }; // Save file to vault const existingFileForOverwrite = plugin.app.vault.getAbstractFileByPath(targetPath); @@ -707,9 +713,10 @@ const importAssetsForNode = async ({ await plugin.app.vault.modifyBinary( existingFileForOverwrite, fileContent, + options, ); } else { - await plugin.app.vault.createBinary(targetPath, fileContent); + await plugin.app.vault.createBinary(targetPath, fileContent, options); } // Update frontmatter to track this mapping @@ -898,6 +905,7 @@ const processFileContent = async ({ mtime: importedModifiedAt, } : undefined; + console.log(stat); if (!file) { file = await plugin.app.vault.create(filePath, rawContent, stat); } else { From 8f8fb5ea3569a3f5973e68b69a28a0a0df0b7d74 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 11:07:48 -0500 Subject: [PATCH 17/33] spurious log --- apps/obsidian/src/utils/importNodes.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index eaaac9e49..3725e1760 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -905,7 +905,6 @@ const processFileContent = async ({ mtime: importedModifiedAt, } : undefined; - console.log(stat); if (!file) { file = await plugin.app.vault.create(filePath, rawContent, stat); } else { From 0ecb29d4acbcea3964b965d916dad40213111f7d Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 12:10:46 -0500 Subject: [PATCH 18/33] Preserve ctime, mtime in processFrontMatter. Also timezone correction --- apps/obsidian/src/utils/importNodes.ts | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 3725e1760..2897a0432 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -96,9 +96,11 @@ export const getPublishedNodesForGroups = async ({ ); const direct = rows.find((r) => r.variant === "direct"); const text = direct?.text ?? latest.text ?? ""; - const createdAt = latest.created ? new Date(latest.created).valueOf() : 0; + const createdAt = latest.created + ? new Date(latest.created + "Z").valueOf() + : 0; const modifiedAt = latest.last_modified - ? new Date(latest.last_modified).valueOf() + ? new Date(latest.last_modified + "Z").valueOf() : 0; nodes.push({ source_local_id: latest.source_local_id!, @@ -347,8 +349,8 @@ const fetchFileReferences = async ({ return data.map(({ filepath, filehash, created, last_modified }) => ({ filepath, filehash, - created: created ? new Date(created).valueOf() : 0, - last_modified: last_modified ? new Date(last_modified).valueOf() : 0, + created: created ? new Date(created + "Z").valueOf() : 0, + last_modified: last_modified ? new Date(last_modified + "Z").valueOf() : 0, })); }; @@ -580,6 +582,10 @@ const importAssetsForNode = async ({ }> => { const pathMapping = new Map(); const errors: string[] = []; + const stat = { + ctime: targetMarkdownFile.stat.ctime, + mtime: targetMarkdownFile.stat.mtime, + }; // Fetch FileReference records for the node const fileReferences = await fetchFileReferences({ @@ -682,6 +688,7 @@ const importAssetsForNode = async ({ assets[filehash] = targetPath; (fm as Record).importedAssets = assets; }, + stat, ); console.log(`Reusing existing file at path: ${targetPath}`); continue; @@ -733,6 +740,7 @@ const importAssetsForNode = async ({ assets[filehash] = targetPath; (fm as Record).importedAssets = assets; }, + stat, ); // Track path mapping @@ -926,13 +934,17 @@ const processFileContent = async ({ }); } - await plugin.app.fileManager.processFrontMatter(file, (fm) => { - const record = fm as Record; - if (mappedNodeTypeId !== undefined) { - record.nodeTypeId = mappedNodeTypeId; - } - record.importedFromSpaceUri = sourceSpaceUri; - }); + await plugin.app.fileManager.processFrontMatter( + file, + (fm) => { + const record = fm as Record; + if (mappedNodeTypeId !== undefined) { + record.nodeTypeId = mappedNodeTypeId; + } + record.importedFromSpaceUri = sourceSpaceUri; + }, + stat, + ); return { file }; }; From d6ccb28d1999300034229a8f73275df22a0b2cd4 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 12:26:54 -0500 Subject: [PATCH 19/33] add lastModified in frontMatter --- apps/obsidian/src/components/DiscourseContextView.tsx | 8 ++++++-- apps/obsidian/src/index.ts | 2 ++ apps/obsidian/src/utils/importNodes.ts | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/obsidian/src/components/DiscourseContextView.tsx b/apps/obsidian/src/components/DiscourseContextView.tsx index f7b5f537b..0cbf8b97f 100644 --- a/apps/obsidian/src/components/DiscourseContextView.tsx +++ b/apps/obsidian/src/components/DiscourseContextView.tsx @@ -74,11 +74,15 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { } const isImported = !!frontmatter.importedFromSpaceUri; + const modifiedAt = + typeof frontmatter.modifiedAt === "number" + ? frontmatter.modifiedAt + : activeFile.stat.mtime; const sourceDates = isImported && activeFile?.stat ? { createdAt: new Date(activeFile.stat.ctime).toLocaleString(), - modifiedAt: new Date(activeFile.stat.mtime).toLocaleString(), + modifiedAt: new Date(modifiedAt).toLocaleString(), } : null; @@ -115,7 +119,7 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { )} {isImported && sourceDates && ( -
+
Created in source: {sourceDates.createdAt}
Last modified in source: {sourceDates.modifiedAt}
diff --git a/apps/obsidian/src/index.ts b/apps/obsidian/src/index.ts index 1707ab6b0..36c65e301 100644 --- a/apps/obsidian/src/index.ts +++ b/apps/obsidian/src/index.ts @@ -278,6 +278,8 @@ export default class DiscourseGraphPlugin extends Plugin { "importedFromSpaceUri", "nodeInstanceId", "publishedToGroups", + "lastModified", + "importedAssets", ], ); keysToHide.push(...this.settings.relationTypes.map((rt) => rt.id)); diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 2897a0432..d0faba5e0 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -942,6 +942,7 @@ const processFileContent = async ({ record.nodeTypeId = mappedNodeTypeId; } record.importedFromSpaceUri = sourceSpaceUri; + record.lastModified = importedModifiedAt; }, stat, ); From cad28446f7963d548f6c6fd00ea45254764427a3 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 8 Feb 2026 13:39:11 -0500 Subject: [PATCH 20/33] make sure import image works --- apps/obsidian/src/utils/importNodes.ts | 30 +++++++------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index d0faba5e0..f1d001945 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -598,15 +598,7 @@ const importAssetsForNode = async ({ return { success: true, pathMapping, errors }; } - const importFolderPath = `import/${sanitizeFileName(spaceName)}`; - const assetsFolderPath = `${importFolderPath}/assets`; - - // Ensure assets folder exists - const assetsFolderExists = - await plugin.app.vault.adapter.exists(assetsFolderPath); - if (!assetsFolderExists) { - await plugin.app.vault.createFolder(assetsFolderPath); - } + const importBasePath = `import/${sanitizeFileName(spaceName)}`; // Get existing asset mappings from frontmatter const cache = plugin.app.metadataCache.getFileCache(targetMarkdownFile); @@ -624,7 +616,6 @@ const importAssetsForNode = async ({ for (const fileRef of fileReferences) { try { const { filepath, filehash } = fileRef; - const { name, ext } = extractFileName(filepath); // Check if we already have a file for this hash const existingAssetPath: string | undefined = importedAssets[filehash]; @@ -655,12 +646,13 @@ const importAssetsForNode = async ({ overwritePath = existingFile.path; } - // Determine target path (new file or overwrite of modified local file) - const sanitizedName = sanitizeFileName(name); - const sanitizedExt = ext ? `.${ext}` : ""; - const sanitizedFileName = `${sanitizedName}${sanitizedExt}`; + // Determine target path: import/{vaultName}/{original asset filepath} + const sanitizedAssetPath = filepath + .split("/") + .map(sanitizeFileName) + .join("/"); const targetPath = - overwritePath ?? `${assetsFolderPath}/${sanitizedFileName}`; + overwritePath ?? `${importBasePath}/${sanitizedAssetPath}`; // If local mtime is newer than fileRef.last_modified, overwrite with DB version. if (await plugin.app.vault.adapter.exists(targetPath)) { @@ -991,7 +983,6 @@ export const importSelectedNodes = async ({ for (const [spaceId, nodes] of nodesBySpace.entries()) { const spaceName = await getSpaceNameFromId(client, spaceId); const importFolderPath = `import/${sanitizeFileName(spaceName)}`; - const assetsFolderPath = `${importFolderPath}/assets`; const spaceUri = spaceUris.get(spaceId); if (!spaceUri) { console.warn(`Missing URI for space ${spaceId}`); @@ -1010,12 +1001,7 @@ export const importSelectedNodes = async ({ await plugin.app.vault.createFolder(importFolderPath); } - // Ensure the assets folder exists - const assetsFolderExists = - await plugin.app.vault.adapter.exists(assetsFolderPath); - if (!assetsFolderExists) { - await plugin.app.vault.createFolder(assetsFolderPath); - } + // Process each node in this space for (const node of nodes) { From b0948519f8c7a0b9d3683f455e82a719a0cb8784 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 8 Feb 2026 13:47:01 -0500 Subject: [PATCH 21/33] add recursive folder creation --- apps/obsidian/src/utils/importNodes.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index f1d001945..c36f5083b 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -654,6 +654,15 @@ const importAssetsForNode = async ({ const targetPath = overwritePath ?? `${importBasePath}/${sanitizedAssetPath}`; + // Ensure all parent folders exist before writing + const pathParts = targetPath.split("/"); + for (let i = 1; i < pathParts.length - 1; i++) { + const folderPath = pathParts.slice(0, i + 1).join("/"); + if (!(await plugin.app.vault.adapter.exists(folderPath))) { + await plugin.app.vault.createFolder(folderPath); + } + } + // If local mtime is newer than fileRef.last_modified, overwrite with DB version. if (await plugin.app.vault.adapter.exists(targetPath)) { const file = plugin.app.vault.getAbstractFileByPath(targetPath); From 16fb68f3a6b3c6546fa76c60db0de6aed8559dc7 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Sun, 8 Feb 2026 14:22:22 -0500 Subject: [PATCH 22/33] modify the fetchNodeContent flow; simplify the pathMapping --- apps/obsidian/src/utils/importNodes.ts | 208 ++++++++++++++++--------- 1 file changed, 136 insertions(+), 72 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index c36f5083b..6c76b4785 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -288,6 +288,68 @@ export const fetchNodeContentWithMetadata = async ({ }; }; +/** + * Fetches both direct (title) and full (body + dates) variants in one query. + * Used by importSelectedNodes to avoid two round-trips to the content table. + */ +const fetchNodeContentForImport = async ({ + client, + spaceId, + nodeInstanceId, +}: { + client: DGSupabaseClient; + spaceId: number; + nodeInstanceId: string; +}): Promise<{ + fileName: string; + content: string; + createdAt: number; + modifiedAt: number; +} | null> => { + const { data, error } = await client + .from("my_contents") + .select("text, created, last_modified, variant") + .eq("source_local_id", nodeInstanceId) + .eq("space_id", spaceId) + .in("variant", ["direct", "full"]); + + if (error) { + console.error("Error fetching node content for import:", error); + return null; + } + + const rows = (data ?? []) as Array<{ + text: string | null; + created: string | null; + last_modified: string | null; + variant: string | null; + }>; + const direct = rows.find((r) => r.variant === "direct"); + const full = rows.find((r) => r.variant === "full"); + + if ( + !direct?.text || + !full?.text || + full.created == null || + full.last_modified == null + ) { + if (!direct?.text) { + console.warn(`No direct variant found for node ${nodeInstanceId}`); + } + if (!full?.text) { + console.warn(`No full variant found for node ${nodeInstanceId}`); + } + return null; + } + + return { + fileName: direct.text, + content: full.text, + createdAt: new Date(full.created).valueOf(), + modifiedAt: new Date(full.last_modified).valueOf(), + }; +}; + /** * Fetches created/last_modified from the source space Content (my_contents) for an imported node. * Used by the discourse context view to show "last modified in original vault". @@ -383,24 +445,11 @@ const downloadFileFromStorage = async ({ } }; -const extractFileName = (filepath: string): { name: string; ext: string } => { - // Handle paths like "attachments/image.png" or "folder/subfolder/file.jpg" - // Extract just the filename with extension - const parts = filepath.split("/"); - const fileName = parts[parts.length - 1] || filepath; - - // Split filename and extension - const lastDotIndex = fileName.lastIndexOf("."); - if (lastDotIndex === -1 || lastDotIndex === fileName.length - 1) { - // No extension or extension is empty - return { name: fileName, ext: "" }; - } - const name = fileName.slice(0, lastDotIndex); - const ext = fileName.slice(lastDotIndex + 1); - return { name, ext }; -}; +/** Normalize path for lookup: strip leading "./", collapse slashes. Shared so pathMapping keys match link paths. */ +const normalizePathForLookup = (p: string): string => + p.replace(/^\.\//, "").replace(/\/+/g, "/").trim(); const updateMarkdownAssetLinks = ({ content, @@ -417,32 +466,52 @@ const updateMarkdownAssetLinks = ({ return content; } - // Create a set of all new paths for quick lookup + // Create a set of all new paths for quick lookup (used by findImportedAssetFile) const newPaths = new Set(oldPathToNewPath.values()); - // Create a map of old paths to new files for quick lookup - const oldPathToNewFile = new Map(); - for (const [oldPath, newPath] of oldPathToNewPath.entries()) { - const newFile = app.metadataCache.getFirstLinkpathDest( - newPath, - targetFile.path, - ); - if (newFile) { - oldPathToNewFile.set(oldPath, newFile); - } - } - let updatedContent = content; - // Normalize path for comparison: strip leading "./", collapse repeated slashes. - // We match only by full path (exact after normalizing), not by filename alone, - // so that different paths with the same name (e.g. experiment1/result.jpg vs - // experiment2/result.jpg) are never treated as the same asset. - const normalizePathForMatch = (p: string): string => - p.replace(/^\.\//, "").replace(/\/+/g, "/").trim(); + const noteDir = targetFile.path.includes("/") + ? targetFile.path.replace(/\/[^/]*$/, "") + : ""; + + // Resolve a path with ".." and "." segments relative to a base directory (vault-relative). + const resolvePathRelativeToBase = ( + baseDir: string, + relativePath: string, + ): string => { + const baseParts = baseDir ? baseDir.split("/").filter(Boolean) : []; + const pathParts = relativePath.replace(/\/+/g, "/").trim().split("/"); + const result = [...baseParts]; + for (const part of pathParts) { + if (part === "..") { + result.pop(); + } else if (part !== "." && part !== "") { + result.push(part); + } + } + return result.join("/"); + }; - const matchesOldPath = (linkPath: string, oldPath: string): boolean => - normalizePathForMatch(linkPath) === normalizePathForMatch(oldPath); + // Canonical form for matching link paths to oldPath (vault-relative, no import prefix). + const getLinkCanonicalForMatch = (linkPath: string): string => { + const resolved = resolvePathRelativeToBase(noteDir, linkPath); + if (resolved.startsWith("import/")) { + const segments = resolved.split("/"); + return segments.length > 2 ? segments.slice(2).join("/") : resolved; + } + return resolved; + }; + + // Look up new path by link as written in content: use canonical form (resolve relative + strip import prefix). + const getNewPathForLink = (linkPath: string): string | undefined => { + const canonical = normalizePathForLookup( + getLinkCanonicalForMatch(linkPath), + ); + const byCanonical = oldPathToNewPath.get(canonical); + if (byCanonical) return byCanonical; + return oldPathToNewPath.get(normalizePathForLookup(linkPath)); + }; // Helper to find file for a link path, checking if it's one of our imported assets const findImportedAssetFile = (linkPath: string): TFile | null => { @@ -502,9 +571,14 @@ const updateMarkdownAssetLinks = ({ return `[[${linkText}]]`; } - // Fallback: Find matching old path - for (const [oldPath, newFile] of oldPathToNewFile.entries()) { - if (matchesOldPath(linkPath, oldPath)) { + // Direct lookup from pathMapping (record built when we downloaded each asset) + const newPath = getNewPathForLink(linkPath); + if (newPath) { + const newFile = app.metadataCache.getFirstLinkpathDest( + newPath, + targetFile.path, + ); + if (newFile) { const linkText = app.metadataCache.fileToLinktext( newFile, targetFile.path, @@ -543,9 +617,14 @@ const updateMarkdownAssetLinks = ({ return `![${alt}](${linkText})`; } - // Fallback: Find matching old path - for (const [oldPath, newFile] of oldPathToNewFile.entries()) { - if (matchesOldPath(cleanPath, oldPath)) { + // Direct lookup from pathMapping (record built when we downloaded each asset) + const newPath = getNewPathForLink(cleanPath); + if (newPath) { + const newFile = app.metadataCache.getFirstLinkpathDest( + newPath, + targetFile.path, + ); + if (newFile) { const linkText = app.metadataCache.fileToLinktext( newFile, targetFile.path, @@ -587,6 +666,11 @@ const importAssetsForNode = async ({ mtime: targetMarkdownFile.stat.mtime, }; + const setPathMapping = (oldPath: string, newPath: string): void => { + pathMapping.set(oldPath, newPath); + pathMapping.set(normalizePathForLookup(oldPath), newPath); + }; + // Fetch FileReference records for the node const fileReferences = await fetchFileReferences({ client, @@ -637,7 +721,7 @@ const importAssetsForNode = async ({ const localModifiedAfterRef = refLastModifiedMs > 0 && existingFile.stat.mtime > refLastModifiedMs; if (!localModifiedAfterRef) { - pathMapping.set(filepath, existingFile.path); + setPathMapping(filepath, existingFile.path); console.log( `Reusing existing asset: ${filehash} -> ${existingFile.path}`, ); @@ -674,7 +758,7 @@ const importAssetsForNode = async ({ const localModifiedAfterRef = refLastModifiedMs > 0 && localMtimeMs > refLastModifiedMs; if (!localModifiedAfterRef) { - pathMapping.set(filepath, targetPath); + setPathMapping(filepath, targetPath); await plugin.app.fileManager.processFrontMatter( targetMarkdownFile, (fm) => { @@ -744,8 +828,8 @@ const importAssetsForNode = async ({ stat, ); - // Track path mapping - pathMapping.set(filepath, targetPath); + // Track path mapping (raw + normalized key so updateMarkdownAssetLinks can lookup by link text) + setPathMapping(filepath, targetPath); console.log(`Imported asset: ${filepath} -> ${targetPath}`); } catch (error) { const errorMsg = `Error importing asset ${fileRef.filepath}: ${error}`; @@ -1023,42 +1107,22 @@ export const importSelectedNodes = async ({ console.log("existingFile", existingFile); - // Fetch the file name (direct variant) and content (full variant) - const fileName = await fetchNodeContent({ - client, - spaceId, - nodeInstanceId: node.nodeInstanceId, - variant: "direct", - }); - - if (!fileName) { - console.warn( - `No direct variant found for node ${node.nodeInstanceId}`, - ); - failedCount++; - processedCount++; - onProgress?.(processedCount, totalNodes); - continue; - } - - const contentWithMeta = await fetchNodeContentWithMetadata({ + const nodeContent = await fetchNodeContentForImport({ client, spaceId, nodeInstanceId: node.nodeInstanceId, - variant: "full", }); - if (contentWithMeta === null) { - console.warn(`No full variant found for node ${node.nodeInstanceId}`); + if (!nodeContent) { failedCount++; processedCount++; onProgress?.(processedCount, totalNodes); continue; } - const { content } = contentWithMeta; - const createdAt = node.createdAt ?? contentWithMeta.createdAt; - const modifiedAt = node.modifiedAt ?? contentWithMeta.modifiedAt; + const { fileName, content, createdAt: contentCreatedAt, modifiedAt: contentModifiedAt } = nodeContent; + const createdAt = node.createdAt ?? contentCreatedAt; + const modifiedAt = node.modifiedAt ?? contentModifiedAt; // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); From 8fb6c26f159e08e7c3f8f242bd548da1ef5c74eb Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 15:44:57 -0500 Subject: [PATCH 23/33] WIP: shuffle the filePath around --- apps/obsidian/src/types.ts | 1 + apps/obsidian/src/utils/importNodes.ts | 67 ++++++++++++++++--- .../upsertNodesAsContentWithEmbeddings.ts | 3 +- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/apps/obsidian/src/types.ts b/apps/obsidian/src/types.ts index 6f61049a3..7c4be2588 100644 --- a/apps/obsidian/src/types.ts +++ b/apps/obsidian/src/types.ts @@ -71,6 +71,7 @@ export type ImportableNode = { /** From source Content (latest last_modified across variants). Set when loaded from getPublishedNodesForGroups. */ createdAt?: number; modifiedAt?: number; + filePath?: string; }; export type GroupWithNodes = { diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 6c76b4785..7ce4a62e8 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import type { Json } from "@repo/database/dbTypes"; import matter from "gray-matter"; import { App, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; @@ -38,6 +39,7 @@ export const getPublishedNodesForGroups = async ({ text: string; createdAt: number; modifiedAt: number; + filePath: string | undefined; }> > => { if (groupIds.length === 0) { @@ -48,7 +50,9 @@ export const getPublishedNodesForGroups = async ({ // the latest last_modified per node and prefer "direct" for text (title). const { data, error } = await client .from("my_contents") - .select("source_local_id, space_id, text, created, last_modified, variant") + .select( + "source_local_id, space_id, text, created, last_modified, variant, metadata", + ) .neq("space_id", currentSpaceId); if (error) { @@ -67,6 +71,7 @@ export const getPublishedNodesForGroups = async ({ created: string | null; last_modified: string | null; variant: string | null; + metadata: Json; }; const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; @@ -84,6 +89,7 @@ export const getPublishedNodesForGroups = async ({ text: string; createdAt: number; modifiedAt: number; + filePath: string | undefined; }> = []; for (const rows of groups.values()) { @@ -102,12 +108,19 @@ export const getPublishedNodesForGroups = async ({ const modifiedAt = latest.last_modified ? new Date(latest.last_modified + "Z").valueOf() : 0; + const filePath: string | undefined = + direct && + typeof direct.metadata === "object" && + typeof (direct.metadata as Record).filePath === "string" + ? (direct.metadata as Record).filePath + : undefined; nodes.push({ source_local_id: latest.source_local_id!, space_id: latest.space_id!, text, createdAt, modifiedAt, + filePath, }); } @@ -305,10 +318,11 @@ const fetchNodeContentForImport = async ({ content: string; createdAt: number; modifiedAt: number; + filePath?: string; } | null> => { const { data, error } = await client .from("my_contents") - .select("text, created, last_modified, variant") + .select("text, created, last_modified, variant, metadata") .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .in("variant", ["direct", "full"]); @@ -323,6 +337,7 @@ const fetchNodeContentForImport = async ({ created: string | null; last_modified: string | null; variant: string | null; + metadata: Json; }>; const direct = rows.find((r) => r.variant === "direct"); const full = rows.find((r) => r.variant === "full"); @@ -342,11 +357,17 @@ const fetchNodeContentForImport = async ({ return null; } + const filePath: string | undefined = + typeof direct.metadata === "object" && + typeof (direct.metadata as Record).filePath === "string" + ? (direct.metadata as Record).filePath + : undefined; return { fileName: direct.text, content: full.text, createdAt: new Date(full.created).valueOf(), modifiedAt: new Date(full.last_modified).valueOf(), + filePath, }; }; @@ -445,8 +466,6 @@ const downloadFileFromStorage = async ({ } }; - - /** Normalize path for lookup: strip leading "./", collapse slashes. Shared so pathMapping keys match link paths. */ const normalizePathForLookup = (p: string): string => p.replace(/^\.\//, "").replace(/\/+/g, "/").trim(); @@ -975,6 +994,7 @@ const processFileContent = async ({ sourceSpaceId, sourceSpaceUri, rawContent, + originalFilePath, filePath, importedCreatedAt, importedModifiedAt, @@ -984,6 +1004,7 @@ const processFileContent = async ({ sourceSpaceId: number; sourceSpaceUri: string; rawContent: string; + originalFilePath?: string; filePath: string; importedCreatedAt?: number; importedModifiedAt?: number; @@ -1094,8 +1115,6 @@ export const importSelectedNodes = async ({ await plugin.app.vault.createFolder(importFolderPath); } - - // Process each node in this space for (const node of nodes) { try { @@ -1120,7 +1139,13 @@ export const importSelectedNodes = async ({ continue; } - const { fileName, content, createdAt: contentCreatedAt, modifiedAt: contentModifiedAt } = nodeContent; + const { + fileName, + content, + createdAt: contentCreatedAt, + modifiedAt: contentModifiedAt, + filePath, + } = nodeContent; const createdAt = node.createdAt ?? contentCreatedAt; const modifiedAt = node.modifiedAt ?? contentModifiedAt; @@ -1151,6 +1176,7 @@ export const importSelectedNodes = async ({ sourceSpaceId: spaceId, sourceSpaceUri: spaceUri, rawContent: content, + originalFilePath: filePath, filePath: finalFilePath, importedCreatedAt: createdAt, importedModifiedAt: modifiedAt, @@ -1255,21 +1281,44 @@ export const refreshImportedFile = async ({ error: "Missing frontmatter: importedFromSpaceUri or nodeInstanceId", }; } + if ( + typeof frontmatter.importedFromSpaceUri !== "string" || + typeof frontmatter.nodeInstanceId !== "string" + ) { + return { + success: false, + error: "Non-string frontmatter: importedFromSpaceUri or nodeInstanceId", + }; + } const { spaceName, spaceId } = await getSpaceNameIdFromUri( supabaseClient, - frontmatter.importedFromSpaceUri as string, + frontmatter.importedFromSpaceUri, ); if (spaceId === -1) { return { success: false, error: "Could not get the space Id" }; } + const metadataResp = await supabaseClient + .from("Content") + .select("metadata") + .eq("space_id", spaceId) + .eq("source_local_id", frontmatter.nodeInstanceId) + .eq("variant", "direct") + .maybeSingle(); + const metadata = metadataResp.data?.metadata; + const filePath: string | undefined = + typeof metadata === "object" && + typeof (metadata as Record).filePath === "string" + ? (metadata as Record).filePath + : undefined; const result = await importSelectedNodes({ plugin, selectedNodes: [ { - nodeInstanceId: frontmatter.nodeInstanceId as string, + nodeInstanceId: frontmatter.nodeInstanceId, title: file.basename, spaceId, spaceName, + filePath, groupId: (frontmatter.publishedToGroups as string[] | undefined)?.[0] ?? "", selected: false, diff --git a/apps/obsidian/src/utils/upsertNodesAsContentWithEmbeddings.ts b/apps/obsidian/src/utils/upsertNodesAsContentWithEmbeddings.ts index 877cdb386..50e3c5f76 100644 --- a/apps/obsidian/src/utils/upsertNodesAsContentWithEmbeddings.ts +++ b/apps/obsidian/src/utils/upsertNodesAsContentWithEmbeddings.ts @@ -54,7 +54,6 @@ const createNodeContentEntries = async ( created: node.created, last_modified: node.last_modified, scale: "document" as const, - metadata: node.frontmatter as Json, }; const entries: LocalContentDataInput[] = []; @@ -65,6 +64,7 @@ const createNodeContentEntries = async ( ...baseEntry, text: node.file.basename, variant: "direct", + metadata: { filePath: node.file.path }, }); } @@ -76,6 +76,7 @@ const createNodeContentEntries = async ( ...baseEntry, text: fullContent, variant: "full", + metadata: node.frontmatter as Json, }); } catch (error) { console.error(`Error reading file content for ${node.file.path}:`, error); From 0b422518f9726baffd9fa1181b225259a3def930 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 8 Feb 2026 15:59:59 -0500 Subject: [PATCH 24/33] send originalPathDepth --- apps/obsidian/src/utils/importNodes.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 7ce4a62e8..1874e08c7 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -475,11 +475,13 @@ const updateMarkdownAssetLinks = ({ oldPathToNewPath, targetFile, app, + originalPathDepth, }: { content: string; oldPathToNewPath: Map; targetFile: TFile; app: App; + originalPathDepth?: number; }): string => { if (oldPathToNewPath.size === 0) { return content; @@ -1148,6 +1150,9 @@ export const importSelectedNodes = async ({ } = nodeContent; const createdAt = node.createdAt ?? contentCreatedAt; const modifiedAt = node.modifiedAt ?? contentModifiedAt; + const originalPathDepth: number | undefined = node.filePath + ? node.filePath.split("/").length - 1 + : undefined; // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); @@ -1213,6 +1218,7 @@ export const importSelectedNodes = async ({ oldPathToNewPath: assetImportResult.pathMapping, targetFile: processedFile, app: plugin.app, + originalPathDepth, }); // Only update if content changed From f7de3e88044251bbc5b45e4fbbcb8d5c0c2295a2 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 9 Feb 2026 12:13:50 -0500 Subject: [PATCH 25/33] actually send path itself --- apps/obsidian/src/utils/importNodes.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 1874e08c7..7c7f04ca4 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -475,13 +475,13 @@ const updateMarkdownAssetLinks = ({ oldPathToNewPath, targetFile, app, - originalPathDepth, + originalPath, }: { content: string; oldPathToNewPath: Map; targetFile: TFile; app: App; - originalPathDepth?: number; + originalPath?: string; }): string => { if (oldPathToNewPath.size === 0) { return content; @@ -1150,9 +1150,7 @@ export const importSelectedNodes = async ({ } = nodeContent; const createdAt = node.createdAt ?? contentCreatedAt; const modifiedAt = node.modifiedAt ?? contentModifiedAt; - const originalPathDepth: number | undefined = node.filePath - ? node.filePath.split("/").length - 1 - : undefined; + const originalPath: string | undefined = node.filePath; // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); @@ -1218,7 +1216,7 @@ export const importSelectedNodes = async ({ oldPathToNewPath: assetImportResult.pathMapping, targetFile: processedFile, app: plugin.app, - originalPathDepth, + originalPath, }); // Only update if content changed From c8c349c4bd39c8214f36498a0e06ab49b0dfe434 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 9 Feb 2026 12:16:33 -0500 Subject: [PATCH 26/33] clarify var name --- apps/obsidian/src/utils/importNodes.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 7c7f04ca4..361d7aaf4 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -475,13 +475,13 @@ const updateMarkdownAssetLinks = ({ oldPathToNewPath, targetFile, app, - originalPath, + originalNodePath, }: { content: string; oldPathToNewPath: Map; targetFile: TFile; app: App; - originalPath?: string; + originalNodePath?: string; }): string => { if (oldPathToNewPath.size === 0) { return content; @@ -1150,7 +1150,7 @@ export const importSelectedNodes = async ({ } = nodeContent; const createdAt = node.createdAt ?? contentCreatedAt; const modifiedAt = node.modifiedAt ?? contentModifiedAt; - const originalPath: string | undefined = node.filePath; + const originalNodePath: string | undefined = node.filePath; // Sanitize file name const sanitizedFileName = sanitizeFileName(fileName); @@ -1216,7 +1216,7 @@ export const importSelectedNodes = async ({ oldPathToNewPath: assetImportResult.pathMapping, targetFile: processedFile, app: plugin.app, - originalPath, + originalNodePath, }); // Only update if content changed From 045c7519515579c373bc6eb1cfaa9d4a47b56d63 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Tue, 10 Feb 2026 11:44:54 -0500 Subject: [PATCH 27/33] make sure fileImport works for all possible settings --- .../src/components/ImportNodesModal.tsx | 1 + apps/obsidian/src/utils/importNodes.ts | 54 ++++++++++++++++--- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index 24fac137e..16f72d78c 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -94,6 +94,7 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { selected: false, createdAt: node.createdAt, modifiedAt: node.modifiedAt, + filePath: node.filePath, }); } diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 361d7aaf4..945e620db 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -524,6 +524,19 @@ const updateMarkdownAssetLinks = ({ return resolved; }; + // Resolve link relative to the source note's directory (for "path from current file" when imported note is flattened). + const getCanonicalFromOriginalNote = ( + linkPath: string, + ): string | undefined => { + if (!originalNodePath) return undefined; + const originalNoteDir = originalNodePath.includes("/") + ? originalNodePath.replace(/\/[^/]*$/, "") + : ""; + return normalizePathForLookup( + resolvePathRelativeToBase(originalNoteDir, linkPath), + ); + }; + // Look up new path by link as written in content: use canonical form (resolve relative + strip import prefix). const getNewPathForLink = (linkPath: string): string | undefined => { const canonical = normalizePathForLookup( @@ -531,7 +544,11 @@ const updateMarkdownAssetLinks = ({ ); const byCanonical = oldPathToNewPath.get(canonical); if (byCanonical) return byCanonical; - return oldPathToNewPath.get(normalizePathForLookup(linkPath)); + const byRaw = oldPathToNewPath.get(normalizePathForLookup(linkPath)); + if (byRaw) return byRaw; + // "Path from current file" in source: link was relative to source note; pathMapping keys are source vault-relative. + const fromOriginal = getCanonicalFromOriginalNote(linkPath); + return fromOriginal ? oldPathToNewPath.get(fromOriginal) : undefined; }; // Helper to find file for a link path, checking if it's one of our imported assets @@ -661,6 +678,20 @@ const updateMarkdownAssetLinks = ({ return updatedContent; }; +/** Path of an asset relative to the note's directory (vault-relative). If asset is not under note dir, returns full path. */ +const getAssetPathRelativeToNote = ( + assetFilePath: string, + originalNodePath: string, +): string => { + const noteDir = originalNodePath.includes("/") + ? originalNodePath.replace(/\/[^/]*$/, "") + : ""; + if (!noteDir || !assetFilePath.startsWith(`${noteDir}/`)) { + return assetFilePath; + } + return assetFilePath.slice(noteDir.length + 1); +}; + const importAssetsForNode = async ({ plugin, client, @@ -668,6 +699,7 @@ const importAssetsForNode = async ({ nodeInstanceId, spaceName, targetMarkdownFile, + originalNodePath, }: { plugin: DiscourseGraphPlugin; client: DGSupabaseClient; @@ -675,6 +707,8 @@ const importAssetsForNode = async ({ nodeInstanceId: string; spaceName: string; targetMarkdownFile: TFile; + /** Source vault path of the note (e.g. from Content metadata filePath). Used to place assets under import/{space}/ relative to note. */ + originalNodePath?: string; }): Promise<{ success: boolean; pathMapping: Map; // old path -> new path @@ -751,11 +785,20 @@ const importAssetsForNode = async ({ overwritePath = existingFile.path; } - // Determine target path: import/{vaultName}/{original asset filepath} - const sanitizedAssetPath = filepath + // Target path: import/{spaceName}/{path relative to note}. If sourceNotePath is set and asset + // is under the note's directory, use that relative path so assets sit under import/{space}/. + console.log("sourceNotePath", originalNodePath); + console.log("filepath", filepath); + const pathForImport = + originalNodePath !== undefined + ? getAssetPathRelativeToNote(filepath, originalNodePath) + : filepath; + console.log("pathForImport", pathForImport); + const sanitizedAssetPath = pathForImport .split("/") .map(sanitizeFileName) .join("/"); + console.log("sanitizedAssetPath", sanitizedAssetPath); const targetPath = overwritePath ?? `${importBasePath}/${sanitizedAssetPath}`; @@ -1126,8 +1169,6 @@ export const importSelectedNodes = async ({ spaceUri, ); - console.log("existingFile", existingFile); - const nodeContent = await fetchNodeContentForImport({ client, spaceId, @@ -1198,7 +1239,7 @@ export const importSelectedNodes = async ({ const processedFile = result.file; - // Import assets for this node + // Import assets for this node (use originalNodePath so assets go under import/{space}/ relative to note) const assetImportResult = await importAssetsForNode({ plugin, client, @@ -1206,6 +1247,7 @@ export const importSelectedNodes = async ({ nodeInstanceId: node.nodeInstanceId, spaceName, targetMarkdownFile: processedFile, + originalNodePath, }); // Update markdown content with new asset paths if assets were imported From e9f96fd8620d1ac5630c169124447e6e83bbd70c Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Tue, 10 Feb 2026 16:43:28 -0500 Subject: [PATCH 28/33] store relative path when rewrite the wikilink instead of default to fileToLinkText --- apps/obsidian/src/utils/importNodes.ts | 38 +++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 945e620db..94a1aaca8 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -496,6 +496,24 @@ const updateMarkdownAssetLinks = ({ ? targetFile.path.replace(/\/[^/]*$/, "") : ""; + /** Path of targetFile relative to the current note, for use in links. Obsidian resolves relative links from the note's directory. */ + const getRelativeLinkPath = (assetPath: string): string => { + const noteParts = noteDir ? noteDir.split("/").filter(Boolean) : []; + const targetParts = assetPath.split("/").filter(Boolean); + let i = 0; + while ( + i < noteParts.length && + i < targetParts.length && + noteParts[i] === targetParts[i] + ) { + i++; + } + const ups = noteParts.length - i; + const down = targetParts.slice(i); + const segments = [...Array(ups).fill(".."), ...down]; + return segments.join("/"); + }; + // Resolve a path with ".." and "." segments relative to a base directory (vault-relative). const resolvePathRelativeToBase = ( baseDir: string, @@ -599,10 +617,7 @@ const updateMarkdownAssetLinks = ({ // First, try to find if this link resolves to one of our imported assets const importedAssetFile = findImportedAssetFile(linkPath); if (importedAssetFile) { - const linkText = app.metadataCache.fileToLinktext( - importedAssetFile, - targetFile.path, - ); + const linkText = getRelativeLinkPath(importedAssetFile.path); if (alias) { return `[[${linkText}|${alias}]]`; } @@ -617,10 +632,7 @@ const updateMarkdownAssetLinks = ({ targetFile.path, ); if (newFile) { - const linkText = app.metadataCache.fileToLinktext( - newFile, - targetFile.path, - ); + const linkText = getRelativeLinkPath(newFile.path); if (alias) { return `[[${linkText}|${alias}]]`; } @@ -648,10 +660,7 @@ const updateMarkdownAssetLinks = ({ // First, try to find if this link resolves to one of our imported assets const importedAssetFile = findImportedAssetFile(cleanPath); if (importedAssetFile) { - const linkText = app.metadataCache.fileToLinktext( - importedAssetFile, - targetFile.path, - ); + const linkText = getRelativeLinkPath(importedAssetFile.path); return `![${alt}](${linkText})`; } @@ -663,10 +672,7 @@ const updateMarkdownAssetLinks = ({ targetFile.path, ); if (newFile) { - const linkText = app.metadataCache.fileToLinktext( - newFile, - targetFile.path, - ); + const linkText = getRelativeLinkPath(newFile.path); return `![${alt}](${linkText})`; } } From 63e57000d69093877829ca3459b2b189cc09523e Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Tue, 10 Feb 2026 16:57:23 -0500 Subject: [PATCH 29/33] cleanup logs --- apps/obsidian/src/utils/importNodes.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 94a1aaca8..282f13850 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -783,9 +783,6 @@ const importAssetsForNode = async ({ refLastModifiedMs > 0 && existingFile.stat.mtime > refLastModifiedMs; if (!localModifiedAfterRef) { setPathMapping(filepath, existingFile.path); - console.log( - `Reusing existing asset: ${filehash} -> ${existingFile.path}`, - ); continue; } overwritePath = existingFile.path; @@ -793,18 +790,14 @@ const importAssetsForNode = async ({ // Target path: import/{spaceName}/{path relative to note}. If sourceNotePath is set and asset // is under the note's directory, use that relative path so assets sit under import/{space}/. - console.log("sourceNotePath", originalNodePath); - console.log("filepath", filepath); const pathForImport = originalNodePath !== undefined ? getAssetPathRelativeToNote(filepath, originalNodePath) : filepath; - console.log("pathForImport", pathForImport); const sanitizedAssetPath = pathForImport .split("/") .map(sanitizeFileName) .join("/"); - console.log("sanitizedAssetPath", sanitizedAssetPath); const targetPath = overwritePath ?? `${importBasePath}/${sanitizedAssetPath}`; @@ -845,7 +838,6 @@ const importAssetsForNode = async ({ }, stat, ); - console.log(`Reusing existing file at path: ${targetPath}`); continue; } // Local file was modified since fileRef's last_modified; overwrite with DB version From 1714873f639879213954f9cb9edb06a2aa4aba86 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Tue, 10 Feb 2026 19:06:39 -0500 Subject: [PATCH 30/33] nit --- apps/obsidian/src/utils/conceptConversion.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/obsidian/src/utils/conceptConversion.ts b/apps/obsidian/src/utils/conceptConversion.ts index e1737d898..1c7c55cc4 100644 --- a/apps/obsidian/src/utils/conceptConversion.ts +++ b/apps/obsidian/src/utils/conceptConversion.ts @@ -38,7 +38,7 @@ export const discourseNodeSchemaToLocalConcept = ({ node; return { space_id: context.spaceId, - name: `${name}`, + name, source_local_id: id, is_schema: true, author_local_id: accountLocalId, From d9296abae62aa304557344a9e9e1088c0dd1bce3 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Tue, 10 Feb 2026 19:34:37 -0500 Subject: [PATCH 31/33] AI corrections --- apps/obsidian/src/components/DiscourseContextView.tsx | 4 ++-- apps/obsidian/src/utils/importNodes.ts | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/obsidian/src/components/DiscourseContextView.tsx b/apps/obsidian/src/components/DiscourseContextView.tsx index 0cbf8b97f..0ed2c6326 100644 --- a/apps/obsidian/src/components/DiscourseContextView.tsx +++ b/apps/obsidian/src/components/DiscourseContextView.tsx @@ -75,8 +75,8 @@ const DiscourseContext = ({ activeFile }: DiscourseContextProps) => { const isImported = !!frontmatter.importedFromSpaceUri; const modifiedAt = - typeof frontmatter.modifiedAt === "number" - ? frontmatter.modifiedAt + typeof frontmatter.lastModified === "number" + ? frontmatter.lastModified : activeFile.stat.mtime; const sourceDates = isImported && activeFile?.stat diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 282f13850..b2784c4a8 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -776,9 +776,7 @@ const importAssetsForNode = async ({ let overwritePath: string | undefined; if (existingFile) { - const refLastModifiedMs = fileRef.last_modified - ? new Date(fileRef.last_modified + "Z").getTime() - : 0; + const refLastModifiedMs = fileRef.last_modified || 0; const localModifiedAfterRef = refLastModifiedMs > 0 && existingFile.stat.mtime > refLastModifiedMs; if (!localModifiedAfterRef) { @@ -815,9 +813,7 @@ const importAssetsForNode = async ({ const file = plugin.app.vault.getAbstractFileByPath(targetPath); if (file && file instanceof TFile) { const localMtimeMs = file.stat.mtime; - const refLastModifiedMs = fileRef.last_modified - ? new Date(fileRef.last_modified + "Z").getTime() - : 0; + const refLastModifiedMs = fileRef.last_modified || 0; const localModifiedAfterRef = refLastModifiedMs > 0 && localMtimeMs > refLastModifiedMs; if (!localModifiedAfterRef) { From 3475762b0754b96999e31c61b12394f0a193d251 Mon Sep 17 00:00:00 2001 From: Trang Doan <44855874+trangdoan982@users.noreply.github.com> Date: Wed, 11 Feb 2026 14:20:42 -0800 Subject: [PATCH 32/33] Update apps/obsidian/src/utils/importNodes.ts Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/obsidian/src/utils/importNodes.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index b2784c4a8..6f76e1651 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -816,7 +816,9 @@ const importAssetsForNode = async ({ const refLastModifiedMs = fileRef.last_modified || 0; const localModifiedAfterRef = refLastModifiedMs > 0 && localMtimeMs > refLastModifiedMs; - if (!localModifiedAfterRef) { + const remoteIsNewer = + refLastModifiedMs > 0 && refLastModifiedMs > localMtimeMs; + if (!localModifiedAfterRef && !remoteIsNewer) { setPathMapping(filepath, targetPath); await plugin.app.fileManager.processFrontMatter( targetMarkdownFile, @@ -836,6 +838,8 @@ const importAssetsForNode = async ({ ); continue; } + // Local file was modified OR remote is newer; overwrite with DB version + } // Local file was modified since fileRef's last_modified; overwrite with DB version } } From 8487eb0640ff6b0d56ad7b192fd19dd63e99f975 Mon Sep 17 00:00:00 2001 From: Trang Doan Date: Wed, 11 Feb 2026 17:21:14 -0500 Subject: [PATCH 33/33] address PR comments --- apps/obsidian/src/services/QueryEngine.ts | 9 +++++++-- apps/obsidian/src/utils/importNodes.ts | 12 ++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/apps/obsidian/src/services/QueryEngine.ts b/apps/obsidian/src/services/QueryEngine.ts index abcc52871..f8502b4a8 100644 --- a/apps/obsidian/src/services/QueryEngine.ts +++ b/apps/obsidian/src/services/QueryEngine.ts @@ -301,8 +301,13 @@ export class QueryEngine { ): TFile | null => { if (this.dc) { try { - const safeId = nodeInstanceId.replace(/"/g, '\\"'); - const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceUri = "${importedFromSpaceUri.replace(/"/g, '\\"')}"`; + const safeId = nodeInstanceId + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"'); + const safeUri = importedFromSpaceUri + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"'); + const dcQuery = `@page and nodeInstanceId = "${safeId}" and importedFromSpaceUri = "${safeUri}"`; const results = this.dc.query(dcQuery); for (const page of results) { diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 6f76e1651..7eb8090c0 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -296,8 +296,12 @@ export const fetchNodeContentWithMetadata = async ({ return { content: data.text, - createdAt: new Date(data.created ?? 0).valueOf(), - modifiedAt: new Date(data.last_modified ?? 0).valueOf(), + createdAt: data.created + ? new Date(data.created + "Z").valueOf() + : 0, + modifiedAt: data.last_modified + ? new Date(data.last_modified + "Z").valueOf() + : 0, }; }; @@ -365,8 +369,8 @@ const fetchNodeContentForImport = async ({ return { fileName: direct.text, content: full.text, - createdAt: new Date(full.created).valueOf(), - modifiedAt: new Date(full.last_modified).valueOf(), + createdAt: new Date(full.created + "Z").valueOf(), + modifiedAt: new Date(full.last_modified + "Z").valueOf(), filePath, }; };