From 67e3824d94e45fd38b2ca380b062fbabb4a70b2e Mon Sep 17 00:00:00 2001 From: BrutuZ Date: Sat, 24 Jan 2026 15:14:10 -0300 Subject: [PATCH] Update fields from Fribb's anime-lists --- docs/openapi.yaml | 89 ++++++++++++++++----- src/db/db.ts | 23 +++++- src/index.ts | 4 +- src/migrations/20260124120530_schema.ts | 34 ++++++++ src/routes/v2/ids/handler.test.ts | 21 ++++- src/routes/v2/ids/schemas/common.ts | 4 +- src/routes/v2/ids/schemas/json-body.test.ts | 4 +- src/routes/v2/ids/schemas/json-body.ts | 3 +- src/routes/v2/include.test-utils.ts | 7 +- src/routes/v2/include.ts | 2 +- src/routes/v2/special/handler.test.ts | 28 ++++++- src/update.test.ts | 8 +- src/update.ts | 63 +++++++++++---- 13 files changed, 234 insertions(+), 56 deletions(-) create mode 100644 src/migrations/20260124120530_schema.ts diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 3a596d7a2..3c2e8df16 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -115,13 +115,13 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 - notify-moe: + animenewsnetwork: oneOf: - type: 'null' - - type: string - minLength: 1 - maxLength: 50 - example: "-cQb5Fmmg" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 themoviedb: oneOf: - type: 'null' @@ -129,6 +129,13 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 + themoviedb-season: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + example: 1 thetvdb: oneOf: - type: 'null' @@ -136,6 +143,13 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 + thetvdb-season: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + example: 1 myanimelist: oneOf: - type: 'null' @@ -143,6 +157,27 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 + simkl: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 + animecountdown: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 + media: + oneOf: + - type: 'null' + - type: string + minLength: 0 + maxLength: 10 + example: TV nullable_relation: oneOf: @@ -153,15 +188,19 @@ $defs: example: anidb: 1337 anilist: 1337 - anime-planet: spriggan + anime-planet: dororon-enma-kun anisearch: null imdb: tt0164917 kitsu: null livechart: null - notify-moe: "-cQb5Fmmg" themoviedb: null + themoviedb-season: 1 thetvdb: null + thetvdb-season: 1 myanimelist: null + animecountdown: null + animenewsnetwork: null + media: TV oneOf: - $ref: '#/$defs/nullable_relation' - type: array @@ -305,10 +344,11 @@ paths: - anilist - anidb - anime-planet + - animecountdown + - animenewsnetwork - anisearch - kitsu - livechart - - notify-moe - myanimelist - name: id in: query @@ -356,7 +396,6 @@ paths: example: - anilist: 1337 - anidb: 1337 - - notify-moe: -cQb5Fmmg oneOf: - type: object minProperties: 1 @@ -398,13 +437,19 @@ paths: - type: integer minimum: 0 maximum: 50000000 - notify-moe: + myanimelist: oneOf: - type: 'null' - - type: string - minLength: 1 - maxLength: 50 - myanimelist: + - type: integer + minimum: 0 + maximum: 50000000 + animenewsnetwork: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + animecountdown: oneOf: - type: 'null' - type: integer @@ -454,13 +499,19 @@ paths: - type: integer minimum: 0 maximum: 50000000 - notify-moe: + myanimelist: oneOf: - type: 'null' - - type: string - minLength: 1 - maxLength: 50 - myanimelist: + - type: integer + minimum: 0 + maximum: 50000000 + animenewsnetwork: + oneOf: + - type: 'null' + - type: integer + minimum: 0 + maximum: 50000000 + animecountdown: oneOf: - type: 'null' - type: integer diff --git a/src/db/db.ts b/src/db/db.ts index cc63f5b72..fefad39a0 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -15,12 +15,26 @@ export const Source = { IMDB: "imdb", Kitsu: "kitsu", LiveChart: "livechart", - NotifyMoe: "notify-moe", + AnimeNewsNetwork: "animenewsnetwork", TheMovieDB: "themoviedb", + TheMovieDBSeason: "themoviedb-season", TheTVDB: "thetvdb", + TheTVDBSeason: "thetvdb-season", MAL: "myanimelist", + Simkl: "simkl", + AnimeCountdown: "animecountdown", + MediaType: "media", } as const export type SourceValue = (typeof Source)[keyof typeof Source] +export const NonUniqueFields = [ + Source.IMDB, + Source.TheMovieDB, + Source.TheMovieDBSeason, + Source.TheTVDB, + Source.TheTVDBSeason, + Source.Simkl, + Source.MediaType, +] export type Relation = { [Source.AniDB]?: number @@ -30,10 +44,15 @@ export type Relation = { [Source.IMDB]?: `tt${string}` [Source.Kitsu]?: number [Source.LiveChart]?: number - [Source.NotifyMoe]?: string + [Source.AnimeNewsNetwork]?: number [Source.TheMovieDB]?: number [Source.TheTVDB]?: number [Source.MAL]?: number + [Source.TheTVDBSeason]?: number + [Source.TheMovieDBSeason]?: number + [Source.Simkl]?: number + [Source.AnimeCountdown]?: number + [Source.MediaType]?: string } export type OldRelation = Pick diff --git a/src/index.ts b/src/index.ts index b142c3471..478b75884 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { serve } from "h3" import { createApp } from "./app.ts" -import { config } from "./config.ts" +import { config, Environment } from "./config.ts" import { migrator } from "./db/db.ts" import { updateRelations } from "./update.ts" @@ -11,7 +11,7 @@ const { NODE_ENV, PORT } = config const runUpdateScript = async () => updateRelations() -if (NODE_ENV === "production") { +if (NODE_ENV === Environment.Prod) { void runUpdateScript() // eslint-disable-next-line ts/no-misused-promises diff --git a/src/migrations/20260124120530_schema.ts b/src/migrations/20260124120530_schema.ts new file mode 100644 index 000000000..e75dba2c5 --- /dev/null +++ b/src/migrations/20260124120530_schema.ts @@ -0,0 +1,34 @@ +import { type Kysely, sql } from "kysely" + +export async function up(db: Kysely): Promise { + await sql`PRAGMA journal_mode=WAL`.execute(db) + + await db.schema.dropTable("relations").ifExists().execute() + + await db.schema + .createTable("relations") + .ifNotExists() + + // Original columns + .addColumn("anidb", "integer", (col) => col.unique()) + .addColumn("anilist", "integer", (col) => col.unique()) + .addColumn("myanimelist", "integer", (col) => col.unique()) + .addColumn("kitsu", "integer", (col) => col.unique()) + + // v2 columns + .addColumn("anime-planet", "text", (col) => col.unique()) + .addColumn("anisearch", "integer", (col) => col.unique()) + .addColumn("imdb", "text") + .addColumn("livechart", "integer", (col) => col.unique()) + .addColumn("themoviedb", "integer") + .addColumn("thetvdb", "integer") + + // New Columns + .addColumn("themoviedb-season", "integer") + .addColumn("thetvdb-season", "integer") + .addColumn("animenewsnetwork", "integer", (col) => col.unique()) + .addColumn("animecountdown", "integer", (col) => col.unique()) + .addColumn("simkl", "integer") + .addColumn("media", "text") + .execute() +} diff --git a/src/routes/v2/ids/handler.test.ts b/src/routes/v2/ids/handler.test.ts index 1b98e78e1..b749db26b 100644 --- a/src/routes/v2/ids/handler.test.ts +++ b/src/routes/v2/ids/handler.test.ts @@ -16,10 +16,15 @@ const createRelations = async ( imdb: `tt${id++}`, kitsu: id++, livechart: id++, - "notify-moe": `${id++}`, + animenewsnetwork: id++, themoviedb: id++, + "themoviedb-season": id++, thetvdb: id++, + "thetvdb-season": id++, myanimelist: id++, + simkl: id++, + animecountdown: id++, + media: `${id++}`, })) // Insert each relation @@ -80,10 +85,15 @@ describe("query params", () => { imdb: null!, kitsu: null!, livechart: null!, - "notify-moe": null!, + animenewsnetwork: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, + "thetvdb-season": null!, myanimelist: null!, + simkl: null!, + animecountdown: null!, + media: null!, } await db.insertInto("relations").values(relation).execute() @@ -165,10 +175,15 @@ describe("json body", () => { imdb: null!, kitsu: null!, livechart: null!, - "notify-moe": null!, + animenewsnetwork: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, + "thetvdb-season": null!, myanimelist: null!, + simkl: null!, + animecountdown: null!, + media: null!, } await db.insertInto("relations").values(relation).execute() diff --git a/src/routes/v2/ids/schemas/common.ts b/src/routes/v2/ids/schemas/common.ts index ddc18254e..8e43d1530 100644 --- a/src/routes/v2/ids/schemas/common.ts +++ b/src/routes/v2/ids/schemas/common.ts @@ -4,10 +4,12 @@ import * as v from "valibot" export const numberIdSourceSchema = v.picklist([ "anilist", "anidb", + "animecountdown", + "animenewsnetwork", "anisearch", "kitsu", "livechart", "myanimelist", ]) -export const stringIdSourceSchema = v.picklist(["anime-planet", "notify-moe"]) +export const stringIdSourceSchema = v.picklist(["anime-planet"]) diff --git a/src/routes/v2/ids/schemas/json-body.test.ts b/src/routes/v2/ids/schemas/json-body.test.ts index cd3391272..f9956f7a1 100644 --- a/src/routes/v2/ids/schemas/json-body.test.ts +++ b/src/routes/v2/ids/schemas/json-body.test.ts @@ -22,8 +22,9 @@ const okCases = [ anisearch: 1337, kitsu: 1337, livechart: 1337, - "notify-moe": "1337", + animenewsnetwork: 1337, myanimelist: 1337, + animecountdown: 1337, }, true, ], @@ -47,6 +48,7 @@ const badCases = [ [{ imdb: 1337 }, false], [{ themoviedb: 1337 }, false], [{ thetvdb: 1337 }, false], + [{ simkl: 1337 }, false], ] satisfies Cases const mapToSingularArrayInput = (cases: Cases): Cases => diff --git a/src/routes/v2/ids/schemas/json-body.ts b/src/routes/v2/ids/schemas/json-body.ts index e042e7acd..c0c7aa4e1 100644 --- a/src/routes/v2/ids/schemas/json-body.ts +++ b/src/routes/v2/ids/schemas/json-body.ts @@ -11,10 +11,11 @@ export const singularItemInputSchema = v.pipe( anidb: numberIdSchema, anilist: numberIdSchema, "anime-planet": stringIdSchema, + animecountdown: numberIdSchema, + animenewsnetwork: numberIdSchema, anisearch: numberIdSchema, kitsu: numberIdSchema, livechart: numberIdSchema, - "notify-moe": stringIdSchema, myanimelist: numberIdSchema, }), ), diff --git a/src/routes/v2/include.test-utils.ts b/src/routes/v2/include.test-utils.ts index a1c860eb3..60cc53853 100644 --- a/src/routes/v2/include.test-utils.ts +++ b/src/routes/v2/include.test-utils.ts @@ -84,10 +84,15 @@ export const testIncludeQueryParam = ( imdb: null, kitsu: null, livechart: null, - "notify-moe": null, + animenewsnetwork: null, themoviedb: null, + "themoviedb-season": null, thetvdb: null, + "thetvdb-season": null, myanimelist: null, + simkl: null, + animecountdown: null, + media: null, } expectedResult[source] = prefixify(source, 1337) as never diff --git a/src/routes/v2/include.ts b/src/routes/v2/include.ts index 58b21781e..89a8cf906 100644 --- a/src/routes/v2/include.ts +++ b/src/routes/v2/include.ts @@ -8,7 +8,7 @@ export const includeSchema = v.object({ v.string(), v.regex(/^[\-a-z,]+$/, "Invalid `include` query"), v.minLength(1), - v.maxLength(100), + v.maxLength(200), ), ), }) diff --git a/src/routes/v2/special/handler.test.ts b/src/routes/v2/special/handler.test.ts index f3a632434..fb1addfd2 100644 --- a/src/routes/v2/special/handler.test.ts +++ b/src/routes/v2/special/handler.test.ts @@ -17,10 +17,15 @@ const createRelations = async ( imdb: `tt${specialId ?? id++}`, kitsu: id++, livechart: id++, - "notify-moe": `${id++}`, + animenewsnetwork: id++, themoviedb: specialId ?? id++, + "themoviedb-season": specialId ?? id++, thetvdb: specialId ?? id++, + "thetvdb-season": specialId ?? id++, myanimelist: id++, + simkl: id++, + animecountdown: id++, + media: `${specialId ?? id++}`, })) await db.insertInto("relations").values(relations).execute() @@ -71,10 +76,15 @@ describe("imdb", () => { imdb: "tt1337", kitsu: null!, livechart: null!, - "notify-moe": null!, + animenewsnetwork: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, + "thetvdb-season": null!, myanimelist: null!, + simkl: null!, + animecountdown: null!, + media: null!, } await db.insertInto("relations").values(relation).execute() @@ -119,10 +129,15 @@ describe("thetvdb", () => { imdb: null!, kitsu: null!, livechart: null!, - "notify-moe": null!, + animenewsnetwork: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: 1337, + "thetvdb-season": 1, myanimelist: null!, + simkl: null!, + animecountdown: null!, + media: null!, } await db.insertInto("relations").values(relation).execute() @@ -169,10 +184,15 @@ describe("themoviedb", () => { imdb: null!, kitsu: null!, livechart: null!, - "notify-moe": null!, + animenewsnetwork: null!, themoviedb: 1337, + "themoviedb-season": 1, thetvdb: null!, + "thetvdb-season": null!, myanimelist: null!, + simkl: null!, + animecountdown: null!, + media: null!, } await db.insertInto("relations").values(relation).execute() diff --git a/src/update.test.ts b/src/update.test.ts index 0b8c0f7da..595ee1edb 100644 --- a/src/update.test.ts +++ b/src/update.test.ts @@ -29,11 +29,11 @@ afterAll(async () => { }) it("handles bad values", async () => { - server.get("/Fribb/anime-lists/master/anime-list-full.json", { + server.get("/Fribb/anime-lists/master/anime-list-mini.json", { status: 200, body: [ { anidb_id: 1337, themoviedb_id: "unknown" }, - { anidb_id: 1338, thetvdb_id: "unknown" as never }, + { anidb_id: 1338, tvdb_id: "unknown" as never }, { anidb_id: 1339, imdb_id: "tt1337,tt1338,tt1339" }, { anidb_id: 1340, themoviedb_id: "unknown" }, { anidb_id: 1341, themoviedb_id: 1341 }, @@ -92,7 +92,7 @@ it("handles duplicates", async () => { mocker.unmockGlobal() const entries: Relation[] = await fetch( - "https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-full.json", + "https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-mini.json", ) .then(async (r) => r.json()) .then((e) => (e as any[]).map(formatEntry)) @@ -115,7 +115,7 @@ it("handles duplicates", async () => { Source.AniSearch, Source.Kitsu, Source.LiveChart, - Source.NotifyMoe, + Source.AnimeNewsNetwork, Source.MAL, ] diff --git a/src/update.ts b/src/update.ts index b2710ee0b..7ad1380ce 100644 --- a/src/update.ts +++ b/src/update.ts @@ -1,7 +1,15 @@ import xior, { type XiorError } from "xior" import errorRetryPlugin from "xior/plugins/error-retry" -import { db, type Relation, Source, type SourceValue } from "./db/db.ts" +import { config, Environment } from "./config.ts" +import { + db, + migrator, + NonUniqueFields, + type Relation, + Source, + type SourceValue, +} from "./db/db.ts" import { logger } from "./lib/logger.ts" import { updateBasedOnManualRules } from "./manual-rules.ts" @@ -12,6 +20,7 @@ const isXiorError = (response: T | XiorError): response is XiorError => "stack" in (response as XiorError) export type AnimeListsSchema = Array<{ + type?: string anidb_id?: number anilist_id?: number "anime-planet_id"?: string @@ -20,15 +29,21 @@ export type AnimeListsSchema = Array<{ kitsu_id?: number livechart_id?: number mal_id?: number - "notify.moe_id"?: string + animenewsnetwork_id?: number + animecountdown_id?: number + simkl_id?: number themoviedb_id?: number | "unknown" - thetvdb_id?: number + tvdb_id?: number + season?: { + tvdb?: number + tmdb?: number + } }> const fetchDatabase = async (): Promise => { const response = await http .get( - "https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-full.json", + "https://raw.githubusercontent.com/Fribb/anime-lists/master/anime-list-mini.json", ) .catch((error: XiorError) => error) @@ -62,9 +77,8 @@ const handleBadValues = ( // Removes duplicate source-id pairs from the list, except for thetvdb and themoviedb ids export const removeDuplicates = (entries: Relation[]): Relation[] => { - const sources = (Object.values(Source) as SourceValue[]).filter( - (source) => - source !== Source.TheTVDB && source !== Source.TheMovieDB && source !== Source.IMDB, + const sources = (Object.values(Source) as SourceValue[]).filter((source) => + NonUniqueFields.every((field) => field !== source), ) const existing = new Map>( sources.map((name) => [name, new Set()]), @@ -77,13 +91,7 @@ export const removeDuplicates = (entries: Relation[]): Relation[] => { // Ignore nulls if (id == null) continue // Ignore sources with one-to-many relations - if ( - source === Source.TheTVDB || - source === Source.TheMovieDB || - source === Source.IMDB - ) { - continue - } + if (NonUniqueFields.some((field) => field === source)) continue if (existing.get(source)!.has(id)) return false @@ -105,9 +113,14 @@ export const formatEntry = (entry: AnimeListsSchema[number]): Relation => ({ kitsu: handleBadValues(entry.kitsu_id), livechart: handleBadValues(entry.livechart_id), myanimelist: handleBadValues(entry.mal_id), - "notify-moe": handleBadValues(entry["notify.moe_id"]), + animenewsnetwork: handleBadValues(entry.animenewsnetwork_id), + animecountdown: handleBadValues(entry.animecountdown_id), themoviedb: handleBadValues(entry.themoviedb_id), - thetvdb: handleBadValues(entry.thetvdb_id), + "themoviedb-season": handleBadValues(entry.season?.tmdb), + thetvdb: handleBadValues(entry.tvdb_id), + "thetvdb-season": handleBadValues(entry.season?.tvdb), + simkl: handleBadValues(entry.simkl_id), + media: handleBadValues(entry.type), }) export const updateRelations = async () => { @@ -115,7 +128,7 @@ export const updateRelations = async () => { logger.info("Fetching updated Database...") const data = await fetchDatabase() - logger.info("Fetched updated Database.") + logger.info({ total: Number(data?.length) }, "Fetched updated Database.") if (data == null) { logger.error("got no data") @@ -132,6 +145,22 @@ export const updateRelations = async () => { const goodEntries = removeDuplicates(formattedEntries) logger.info({ remaining: goodEntries.length }, `Removed duplicates.`) + if (config.NODE_ENV !== Environment.Prod) { + const { error, results } = await migrator.migrateToLatest() + + results?.forEach((it) => { + logger.info(`Migration ${it.direction} "${it.migrationName}" ...`) + if (it.status === "Success") { + logger.info(`... was executed successfully`) + } else if (it.status === "Error") { + logger.error(`... FAILED!`) + } + }) + if (error || "Error" in (results?.map((x) => x.status) || [])) { + throw new Error(`failed to run 'migrateToLatest' ${error || ""}`) + } + } + logger.info("Updating database...") await db.transaction().execute(async (trx) => { // Delete all existing relations