Skip to content

ENG-1242 Cleanup orphan concepts #997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maparent wants to merge 1 commit into
base: main
Choose a base branch
from
+157 −62
Open

ENG-1242 Cleanup orphan concepts #997

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 157 additions & 62 deletions apps/roam/src/utils/cleanupOrphanedNodes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,80 +4,96 @@ import type { DGSupabaseClient } from "@repo/database/lib/client";
import type { Tables } from "@repo/database/dbTypes";
import internalError from "./internalError";

const getAllNodesFromSupabase = async (
const getAllConceptLocalIdsFromSupabase = async (
supabaseClient: DGSupabaseClient,
spaceId: number,
): Promise<string[]> => {
try {
const { data: schemas, error: schemasError } = (await supabaseClient
const { data, error } = await supabaseClient
.from("my_concepts")
.select("id")
.select("source_local_id")
.eq("space_id", spaceId)
.eq("is_schema", true)
.eq("arity", 0)) as PostgrestResponse<Tables<"Concept">>;
.eq("is_schema", false)
.not("source_local_id", "is", null);

if (schemasError) {
if (error) {
internalError({
error: schemasError,
userMessage: "Failed to get all discourse node schemas from Supabase:",
error: error,
userMessage: "Failed to get concepts from Supabase",
type: "cleanup_orphans_get_concepts",
});
return [];
}
return (
data?.map((c) => c.source_local_id).filter((id) => id !== null) || []
);
} catch (error) {
internalError({
error,
userMessage: "Error in getAllConceptLocalIdsFromSupabase:",
});
return [];
}
};

const schemaIds = schemas.map((s: { id: number }) => s.id);
let nodeResult: string[] = [];

if (schemaIds.length > 0) {
const conceptResponse = await supabaseClient
.from("my_concepts")
.select("source_local_id")
.eq("space_id", spaceId)
.eq("is_schema", false)
.in("schema_id", schemaIds)
.not("source_local_id", "is", null);
const getAllDocumentLocalIdsFromSupabase = async (
supabaseClient: DGSupabaseClient,
spaceId: number,
): Promise<string[]> => {
try {
const { error, data } = await supabaseClient
.from("my_documents")
.select("source_local_id")
.eq("space_id", spaceId)
.not("source_local_id", "is", null);

if (conceptResponse.error) {
internalError({
error: conceptResponse.error,
userMessage: "Failed to get concepts from Supabase",
type: "cleanup_orphans_get_concepts",
});
return [];
}
nodeResult =
conceptResponse.data
?.map(
(c: { source_local_id: string | null }) =>
c.source_local_id || null,
)
.filter((id: string | null): id is string => !!id) || [];
if (error) {
internalError({
error: error,
userMessage: "Failed to get document ids from Supabase:",
});
return [];
}

const blockContentResponse = await supabaseClient
return (
data?.map((c) => c.source_local_id).filter((id) => id !== null) || []
);
} catch (error) {
internalError({
error,
userMessage: "Error in getAllDocumentLocalIdsFromSupabase:",
});
return [];
}
};

const getAllContentLocalIdsFromSupabase = async (
supabaseClient: DGSupabaseClient,
spaceId: number,
): Promise<string[]> => {
try {
const { data, error } = await supabaseClient
.from("my_contents")
.select("source_local_id")
.eq("space_id", spaceId)
.eq("scale", "block")
.not("source_local_id", "is", null);
Comment thread
maparent marked this conversation as resolved.

if (blockContentResponse.error) {
if (error) {
internalError({
error: blockContentResponse.error,
error: error,
userMessage: "Failed to get block content from Supabase:",
});
return [];
}

const blockResult =
blockContentResponse.data
?.map((c: { source_local_id: string | null }) => c.source_local_id)
.filter((id: string | null): id is string => !!id) || [];

const result = [...new Set([...nodeResult, ...blockResult])];

return result;
return (
data?.map((c) => c.source_local_id).filter((id) => id !== null) || []
);
} catch (error) {
internalError({ error, userMessage: "Error in getAllNodesFromSupabase:" });
internalError({
error,
userMessage: "Error in getAllContentLocalIdsFromSupabase:",
});
return [];
}
};
Expand Down Expand Up @@ -143,42 +159,88 @@ const getNonExistentRoamUids = (nodeUids: string[]): string[] => {
}
};

const deleteNodesFromSupabase = async (
const deleteConceptsFromSupabase = async (
uids: string[],
spaceId: number,
supabaseClient: DGSupabaseClient,
): Promise<void> => {
try {
if (uids.length === 0) return;

const { error: conceptDeleteError } = await supabaseClient
const { error: error } = await supabaseClient
.from("Concept")
.delete()
.eq("space_id", spaceId)
.in("source_local_id", uids)
.eq("is_schema", false);

if (conceptDeleteError) {
if (error) {
internalError({
error: conceptDeleteError,
error,
userMessage: "Failed to delete concepts from Supabase",
});
}
} catch (error) {
internalError({
error,
userMessage: "Error in deleteConceptsFromSupabase",
});
}
};

const { error: contentDeleteError } = await supabaseClient
const deleteContentFromSupabase = async (
uids: string[],
spaceId: number,
supabaseClient: DGSupabaseClient,
): Promise<void> => {
try {
if (uids.length === 0) return;

const { error } = await supabaseClient
.from("Content")
.delete()
.eq("space_id", spaceId)
.in("source_local_id", uids);

if (contentDeleteError) {
if (error) {
internalError({
error: contentDeleteError,
userMessage: "Failed to delete content from Supabase:",
error,
userMessage: "Failed to delete contents from Supabase",
});
}
} catch (error) {
internalError({ error, userMessage: "Error in deleteNodesFromSupabase" });
internalError({
error,
userMessage: "Error in deleteContentsFromSupabase",
});
}
};

const deleteDocumentsFromSupabase = async (
uids: string[],
spaceId: number,
supabaseClient: DGSupabaseClient,
): Promise<void> => {
try {
if (uids.length === 0) return;

const { error: error } = await supabaseClient
.from("Document")
.delete()
.eq("space_id", spaceId)
.in("source_local_id", uids);

if (error) {
internalError({
error,
userMessage: "Failed to delete Documents from Supabase",
});
}
} catch (error) {
internalError({
error,
userMessage: "Error in deleteDocumentsFromSupabase",
});
}
};

Expand Down Expand Up @@ -320,15 +382,47 @@ export const cleanupOrphanedNodes = async (
context: SupabaseContext,
): Promise<void> => {
try {
const supabaseUids = await getAllNodesFromSupabase(
const conceptUids = await getAllConceptLocalIdsFromSupabase(
supabaseClient,
context.spaceId,
);
if (conceptUids.length > 0) {
const orphanedConceptUids = getNonExistentRoamUids(conceptUids);
if (orphanedConceptUids.length > 0) {
// console.log("Deleting concepts:", orphanedConceptUids);
await deleteConceptsFromSupabase(
orphanedConceptUids,
context.spaceId,
supabaseClient,
);
}
}

const contentUids = await getAllContentLocalIdsFromSupabase(
supabaseClient,
context.spaceId,
);
if (contentUids.length > 0) {
const orphanedContentUids = getNonExistentRoamUids(contentUids);
if (orphanedContentUids.length > 0) {
// console.log("Deleting content:", orphanedContentUids);
await deleteContentFromSupabase(
orphanedContentUids,
context.spaceId,
supabaseClient,
);
}
}
const documentUids = await getAllDocumentLocalIdsFromSupabase(
supabaseClient,
context.spaceId,
);
if (supabaseUids.length > 0) {
const orphanedUids = getNonExistentRoamUids(supabaseUids);
if (orphanedUids.length > 0) {
await deleteNodesFromSupabase(
orphanedUids,
if (documentUids.length > 0) {
const orphanedDocumentUids = getNonExistentRoamUids(documentUids);
if (orphanedDocumentUids.length > 0) {
// console.log("Deleting documents:", orphanedDocumentUids);
await deleteDocumentsFromSupabase(
orphanedDocumentUids,
context.spaceId,
supabaseClient,
);
Expand All @@ -342,6 +436,7 @@ export const cleanupOrphanedNodes = async (
if (supabaseSchemaUids.length > 0) {
const orphanedSchemaUids = getNonExistentRoamUids(supabaseSchemaUids);
if (orphanedSchemaUids.length > 0) {
// console.log("Deleting schemas:", orphanedSchemaUids);
await deleteNodeSchemasFromSupabase(
orphanedSchemaUids,
supabaseClient,
Expand Down
Loading