diff --git a/apps/frontend/.env. b/apps/frontend/.env. new file mode 100644 index 0000000000..29aad1d10a --- /dev/null +++ b/apps/frontend/.env. @@ -0,0 +1,4 @@ +BASE_URL=http://127.0.0.1:8000/v2/ +BROWSER_BASE_URL=http://127.0.0.1:8000/v2/ +PYRO_BASE_URL=https://staging-archon.modrinth.com +PROD_OVERRIDE=true diff --git a/apps/frontend/src/locales/en-US/index.json b/apps/frontend/src/locales/en-US/index.json index 102787cbf4..2b1b7a8c6a 100644 --- a/apps/frontend/src/locales/en-US/index.json +++ b/apps/frontend/src/locales/en-US/index.json @@ -2663,6 +2663,9 @@ "project.settings.permissions.learn-more": { "message": "Learn more" }, + "project.settings.permissions.no-results": { + "message": "No external projects match your search." + }, "project.settings.permissions.search-placeholder": { "message": "Search {count} {count, plural, one {external project} other {external projects}}..." }, diff --git a/apps/frontend/src/pages/[type]/[id]/settings/permissions.vue b/apps/frontend/src/pages/[type]/[id]/settings/permissions.vue index 2d50a45c5d..856a46e7eb 100644 --- a/apps/frontend/src/pages/[type]/[id]/settings/permissions.vue +++ b/apps/frontend/src/pages/[type]/[id]/settings/permissions.vue @@ -1,4 +1,5 @@ - - - - - - - - - - - {{ formatMessage(messages.learnMore) }} - - + + + + - + + + + + {{ formatMessage(messages.learnMore) }} + + + + + + + + + + - + + + + + + + {{ String(attributionError) }} + + diff --git a/apps/frontend/src/pages/[type]/[id]/settings/versions.vue b/apps/frontend/src/pages/[type]/[id]/settings/versions.vue index 7176020d39..7a51759b78 100644 --- a/apps/frontend/src/pages/[type]/[id]/settings/versions.vue +++ b/apps/frontend/src/pages/[type]/[id]/settings/versions.vue @@ -453,7 +453,9 @@ async function deleteVersion() { stopLoading() } -const withheldVersions = computed(() => ['4.0.0']) +const withheldVersions = computed(() => + versions.value.filter((x) => x.files_missing_attribution?.length > 0), +) const messages = defineMessages({ withheldVersionsWarningTitle: { diff --git a/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json b/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json deleted file mode 100644 index 4bc87e73a5..0000000000 --- a/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM payouts_values_notifications WHERE notified = FALSE AND user_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c" -} diff --git a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json b/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json deleted file mode 100644 index 921f7f92d9..0000000000 --- a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n status AS \"status: PayoutStatus\"\n FROM payouts\n ORDER BY id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false - ] - }, - "hash": "1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286" -} diff --git a/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json b/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json deleted file mode 100644 index 3c99ff3fed..0000000000 --- a/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM payouts_values_notifications WHERE notified = FALSE", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4" -} diff --git a/apps/labrinth/.sqlx/query-226ef980e0f95db3487528a950b493cf2f8d5fd7a7681ef73bafe41954ebb351.json b/apps/labrinth/.sqlx/query-226ef980e0f95db3487528a950b493cf2f8d5fd7a7681ef73bafe41954ebb351.json new file mode 100644 index 0000000000..6f94e6817a --- /dev/null +++ b/apps/labrinth/.sqlx/query-226ef980e0f95db3487528a950b493cf2f8d5fd7a7681ef73bafe41954ebb351.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select id as \"id: DBAttributionGroupId\", flame_project_id\n from project_attribution_groups\n where project_id = $1 and flame_project_id is not null\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: DBAttributionGroupId", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flame_project_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "226ef980e0f95db3487528a950b493cf2f8d5fd7a7681ef73bafe41954ebb351" +} diff --git a/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json b/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json new file mode 100644 index 0000000000..bd768f4571 --- /dev/null +++ b/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "sha1", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "status", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "ByteaArray" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be" +} diff --git a/apps/labrinth/.sqlx/query-29d499ba35e5f8909f2537fa3eef32eb36f53aef0f408e0e994791ba6fa4a3ce.json b/apps/labrinth/.sqlx/query-29d499ba35e5f8909f2537fa3eef32eb36f53aef0f408e0e994791ba6fa4a3ce.json new file mode 100644 index 0000000000..01112679e5 --- /dev/null +++ b/apps/labrinth/.sqlx/query-29d499ba35e5f8909f2537fa3eef32eb36f53aef0f408e0e994791ba6fa4a3ce.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\t\tselect id, name, version_number\n\t\t\tfrom versions\n\t\t\twhere id = ANY($1)\n\t\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "version_number", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "29d499ba35e5f8909f2537fa3eef32eb36f53aef0f408e0e994791ba6fa4a3ce" +} diff --git a/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json b/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json new file mode 100644 index 0000000000..2c5ee6ec20 --- /dev/null +++ b/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into project_attribution_files (group_id, name, sha1)\n select $1, unnest($2::text[]), unnest($3::bytea[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "TextArray", + "ByteaArray" + ] + }, + "nullable": [] + }, + "hash": "2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed" +} diff --git a/apps/labrinth/.sqlx/query-43165d0f3bf4673c811bd3395254815b1a66792b56bb91f90e41ea6a82fd7c0e.json b/apps/labrinth/.sqlx/query-43165d0f3bf4673c811bd3395254815b1a66792b56bb91f90e41ea6a82fd7c0e.json new file mode 100644 index 0000000000..0efe6b743d --- /dev/null +++ b/apps/labrinth/.sqlx/query-43165d0f3bf4673c811bd3395254815b1a66792b56bb91f90e41ea6a82fd7c0e.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select distinct f.version_id as \"version_id: DBVersionId\", f.id as \"file_id: DBFileId\"\n from files f\n inner join override_file_sources ofs on ofs.file_id = f.id\n inner join project_attribution_files paf on paf.sha1 = ofs.sha1\n inner join project_attribution_groups pag on pag.id = paf.group_id\n where f.version_id = ANY($1) and pag.attribution is null\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version_id: DBVersionId", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_id: DBFileId", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "43165d0f3bf4673c811bd3395254815b1a66792b56bb91f90e41ea6a82fd7c0e" +} diff --git a/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json b/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json new file mode 100644 index 0000000000..76174d70dd --- /dev/null +++ b/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tupdate project_attribution_files\n\t\tset group_id = $1\n\t\twhere sha1 = $2\n\t\tand group_id in (\n\t\t\tselect id from project_attribution_groups where project_id = $3\n\t\t)\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a" +} diff --git a/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json b/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json new file mode 100644 index 0000000000..e71a423986 --- /dev/null +++ b/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tupdate project_attribution_files\n\t\tset group_id = $1\n\t\twhere sha1 = $2 and group_id = $3\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6" +} diff --git a/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json b/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json new file mode 100644 index 0000000000..297814d5ad --- /dev/null +++ b/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tinsert into project_attribution_groups (id, project_id)\n\t\tvalues ($1, $2)\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622" +} diff --git a/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json b/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json deleted file mode 100644 index b4c2e5a56e..0000000000 --- a/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO payouts_values_notifications (date_available, user_id, notified)\n VALUES ($1, $2, FALSE)\n ON CONFLICT (date_available, user_id) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Timestamptz", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850" -} diff --git a/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json b/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json deleted file mode 100644 index fc7d2ac98d..0000000000 --- a/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO payouts_values (user_id, mod_id, amount, created, date_available)\n VALUES ($1, NULL, $2, NOW(), $3)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Numeric", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8" -} diff --git a/apps/labrinth/.sqlx/query-8577cf2981cf55f612a81901c6948a80eaad3c943fcb4f2c2aa2e93a8422f208.json b/apps/labrinth/.sqlx/query-8577cf2981cf55f612a81901c6948a80eaad3c943fcb4f2c2aa2e93a8422f208.json new file mode 100644 index 0000000000..78e9f9d0fc --- /dev/null +++ b/apps/labrinth/.sqlx/query-8577cf2981cf55f612a81901c6948a80eaad3c943fcb4f2c2aa2e93a8422f208.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into file_attributions (file_id, scanned_at)\n values ($1, now())\n on conflict (file_id) do update set scanned_at = now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "8577cf2981cf55f612a81901c6948a80eaad3c943fcb4f2c2aa2e93a8422f208" +} diff --git a/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json b/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json new file mode 100644 index 0000000000..3da4278f04 --- /dev/null +++ b/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM project_attribution_groups WHERE id=$1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883" +} diff --git a/apps/labrinth/.sqlx/query-90fb557893ea8ea38dbec50efe77f78fb207844c423586330a2731f338cfbcc1.json b/apps/labrinth/.sqlx/query-90fb557893ea8ea38dbec50efe77f78fb207844c423586330a2731f338cfbcc1.json new file mode 100644 index 0000000000..455c991272 --- /dev/null +++ b/apps/labrinth/.sqlx/query-90fb557893ea8ea38dbec50efe77f78fb207844c423586330a2731f338cfbcc1.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tselect\n\t\t\tg.id as \"id: DBAttributionGroupId\",\n\t\t\tg.flame_project_id,\n\t\t\tg.flame_project_title,\n\t\t\tg.attribution,\n\t\t\tg.attributed_at,\n\t\t\tg.attributed_by as \"attributed_by: i64\"\n\t\tfrom project_attribution_groups g\n\t\twhere g.project_id = $1\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: DBAttributionGroupId", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flame_project_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "flame_project_title", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "attribution", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "attributed_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "attributed_by: i64", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true + ] + }, + "hash": "90fb557893ea8ea38dbec50efe77f78fb207844c423586330a2731f338cfbcc1" +} diff --git a/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json b/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json new file mode 100644 index 0000000000..7b1815f835 --- /dev/null +++ b/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tupdate project_attribution_groups\n\t\tset attribution = $1, attributed_at = now(), attributed_by = $3\n\t\twhere id = $2\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5" +} diff --git a/apps/labrinth/.sqlx/query-aebc07eeda6a8be141ef05024039bcbc6ad105026a8a4a9622aa121168a9cb2b.json b/apps/labrinth/.sqlx/query-aebc07eeda6a8be141ef05024039bcbc6ad105026a8a4a9622aa121168a9cb2b.json new file mode 100644 index 0000000000..f5d6a9ecdd --- /dev/null +++ b/apps/labrinth/.sqlx/query-aebc07eeda6a8be141ef05024039bcbc6ad105026a8a4a9622aa121168a9cb2b.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_attributions fa\n inner join files f on f.id = fa.file_id\n inner join versions v on v.id = f.version_id\n where fa.scanned_at is null\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "file_id: DBFileId", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "project_id: DBProjectId", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "aebc07eeda6a8be141ef05024039bcbc6ad105026a8a4a9622aa121168a9cb2b" +} diff --git a/apps/labrinth/.sqlx/query-af521e5bf0a4675cd6555fdd599ea0922f2dc899b95e7d7ad6f0e96903204ab8.json b/apps/labrinth/.sqlx/query-af521e5bf0a4675cd6555fdd599ea0922f2dc899b95e7d7ad6f0e96903204ab8.json new file mode 100644 index 0000000000..796ffe489a --- /dev/null +++ b/apps/labrinth/.sqlx/query-af521e5bf0a4675cd6555fdd599ea0922f2dc899b95e7d7ad6f0e96903204ab8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO file_attributions (file_id)\n VALUES ($1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "af521e5bf0a4675cd6555fdd599ea0922f2dc899b95e7d7ad6f0e96903204ab8" +} diff --git a/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json b/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json new file mode 100644 index 0000000000..e4cde72b9a --- /dev/null +++ b/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into project_attribution_groups (id, project_id)\n values ($1, $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664" +} diff --git a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json b/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json deleted file mode 100644 index 89bd8147dc..0000000000 --- a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status AS \"status: PayoutStatus\" FROM payouts WHERE id = 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3" -} diff --git a/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json b/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json new file mode 100644 index 0000000000..19a77c556c --- /dev/null +++ b/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into project_attribution_files (group_id, name, sha1)\n values ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853" +} diff --git a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json b/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json deleted file mode 100644 index 469c30168a..0000000000 --- a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, $3, $4, $5, 10.0, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02" -} diff --git a/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json b/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json new file mode 100644 index 0000000000..9d4ffdf994 --- /dev/null +++ b/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tselect paf.group_id, paf.name from project_attribution_files paf\n\t\tinner join project_attribution_groups pag on pag.id = paf.group_id\n\t\twhere paf.sha1 = $1 and pag.project_id = $2\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Bytea", + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa" +} diff --git a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json b/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json deleted file mode 100644 index 52e020ebf2..0000000000 --- a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, NULL, $3, $4, 10.00, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606" -} diff --git a/apps/labrinth/.sqlx/query-cf3e62bfc2f847c1e9c10ec16475d1cecfe5ab9b0f22e231d9e4df5f09b4c6f0.json b/apps/labrinth/.sqlx/query-cf3e62bfc2f847c1e9c10ec16475d1cecfe5ab9b0f22e231d9e4df5f09b4c6f0.json new file mode 100644 index 0000000000..087ce396bf --- /dev/null +++ b/apps/labrinth/.sqlx/query-cf3e62bfc2f847c1e9c10ec16475d1cecfe5ab9b0f22e231d9e4df5f09b4c6f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into project_attribution_groups (id, project_id, flame_project_id, flame_project_title)\n values ($1, $2, $3, $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cf3e62bfc2f847c1e9c10ec16475d1cecfe5ab9b0f22e231d9e4df5f09b4c6f0" +} diff --git a/apps/labrinth/.sqlx/query-de0826c6e850dea35325f5a229e24d7c7d604b57d36705549bcb25925680c2bc.json b/apps/labrinth/.sqlx/query-de0826c6e850dea35325f5a229e24d7c7d604b57d36705549bcb25925680c2bc.json new file mode 100644 index 0000000000..476d0c798f --- /dev/null +++ b/apps/labrinth/.sqlx/query-de0826c6e850dea35325f5a229e24d7c7d604b57d36705549bcb25925680c2bc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n update file_attributions\n set scanned_at = now\n from unnest($1::bigint[], $2::timestamptz[]) as u(id, now)\n where file_attributions.file_id = u.id\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "de0826c6e850dea35325f5a229e24d7c7d604b57d36705549bcb25925680c2bc" +} diff --git a/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json b/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json new file mode 100644 index 0000000000..91a7a9c5c8 --- /dev/null +++ b/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select paf.sha1 from project_attribution_files paf\n inner join project_attribution_groups pag on pag.id = paf.group_id\n where pag.project_id = $1 and paf.sha1 = ANY($2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "sha1", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8", + "ByteaArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78" +} diff --git a/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json b/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json new file mode 100644 index 0000000000..0b7cf69e9c --- /dev/null +++ b/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n insert into override_file_sources (sha1, file_id)\n select unnest($1::bytea[]), $2\n on conflict do nothing\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "ByteaArray", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb" +} diff --git a/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json b/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json new file mode 100644 index 0000000000..1b14566eb0 --- /dev/null +++ b/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tdelete from project_attribution_groups g\n\t\twhere not exists (\n\t\t\tselect 1 from project_attribution_files f where f.group_id = g.id\n\t\t)\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529" +} diff --git a/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json b/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json deleted file mode 100644 index d3e3520bcc..0000000000 --- a/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM notifications WHERE user_id = $1 AND body->>'type' = 'payout_available'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41" -} diff --git a/apps/labrinth/AGENTS.md~HEAD b/apps/labrinth/AGENTS.md~HEAD new file mode 100644 index 0000000000..0b73458d53 --- /dev/null +++ b/apps/labrinth/AGENTS.md~HEAD @@ -0,0 +1,34 @@ +# Labrinth + +Labrinth is the backend API service for Modrinth, written in Rust. + +## Code style + +- When writing `sqlx` queries, NEVER use `query` directly. Always prefer using the `query!`, `query_as!`, `query_scalar!` macros. + +## Pre-PR Checks + +When the user refers to "perform[ing] pre-PR checks", do the following: + +- Run `cargo clippy -p labrinth --all-targets` — there must be ZERO warnings, otherwise CI will fail +- DO NOT run tests unless explicitly requested (they take a long time) +- Prepare the sqlx cache: cd into `apps/labrinth` and run `cargo sqlx prepare -- --tests` + - NEVER run `cargo sqlx prepare --workspace` + +## Testing + +- Run `cargo test -p labrinth --all-targets` to test your changes — all tests must pass + +## Local Services + +- Read the root `docker-compose.yml` to see what running services are available while developing +- Use `docker exec` to access these services + +### Clickhouse + +- Access: `docker exec labrinth-clickhouse clickhouse-client` +- Database: `staging_ariadne` + +### Postgres + +- Access: `docker exec labrinth-postgres psql -U labrinth -d labrinth -c ""` diff --git a/apps/labrinth/migrations/20260423114534_project_attribution.sql b/apps/labrinth/migrations/20260423114534_project_attribution.sql new file mode 100644 index 0000000000..240c17c5bd --- /dev/null +++ b/apps/labrinth/migrations/20260423114534_project_attribution.sql @@ -0,0 +1,34 @@ +create table file_attributions ( + file_id bigint primary key references files(id), + -- if a file.. + -- - does not have a row + -- -> was created before attributions system + -- - has a row, but `scanned_at = null` + -- -> still needs to be scanned + -- - has a row, and `scanned_at` is not null + -- -> attributions have been scanned + scanned_at timestamptz +); + +create table project_attribution_groups ( + id bigint primary key, + project_id bigint not null references mods(id), + flame_project_id bigint, + flame_project_title text, + attribution jsonb, + attributed_at timestamptz, + attributed_by bigint references users(id) +); +create index on project_attribution_groups (project_id); + +create table project_attribution_files ( + group_id bigint not null references project_attribution_groups(id), + name text not null, + sha1 bytea not null +); + +create table override_file_sources ( + sha1 bytea not null, + file_id bigint not null references files(id), + primary key (sha1, file_id) +); diff --git a/apps/labrinth/src/auth/checks.rs b/apps/labrinth/src/auth/checks.rs index 329e071042..525be69b81 100644 --- a/apps/labrinth/src/auth/checks.rs +++ b/apps/labrinth/src/auth/checks.rs @@ -5,7 +5,9 @@ use crate::database::models::version_item::VersionQueryResult; use crate::database::models::{DBCollection, DBOrganization, DBTeamMember}; use crate::database::redis::RedisPool; use crate::database::{DBProject, DBVersion, models}; +use crate::models::ids::FileId; use crate::models::users::User; +use crate::queue::attribution_scan::get_files_missing_attribution; use crate::routes::ApiError; use futures::TryStreamExt; use itertools::Itertools; @@ -204,7 +206,27 @@ pub async fn filter_visible_versions( ) .await?; versions.retain(|x| filtered_version_ids.contains(&x.inner.id)); - Ok(versions.into_iter().map(|x| x.into()).collect()) + + let version_ids: Vec<_> = versions.iter().map(|v| v.inner.id).collect(); + let missing = + get_files_missing_attribution(pool, &version_ids).await.unwrap_or_default(); + + Ok(versions + .into_iter() + .map(|v| { + let files_missing = missing + .get(&v.inner.id) + .map(|ids| { + ids.iter() + .map(|id| FileId(id.0 as u64)) + .collect::>() + }) + .unwrap_or_default(); + let mut version = crate::models::projects::Version::from(v); + version.files_missing_attribution = files_missing; + version + }) + .collect()) } impl ValidateAuthorized for models::DBOAuthClient { diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index b44bdfe6cd..09a006b8a8 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -1,7 +1,9 @@ use crate::database; use crate::database::PgPool; use crate::database::redis::RedisPool; +use crate::file_hosting::FileHost; use crate::queue::analytics::cache::cache_analytics; +use crate::queue::attribution_scan::scan_all_file_override_attributions; use crate::queue::billing::{index_billing, index_subscriptions}; use crate::queue::email::EmailQueue; use crate::queue::payouts::{ @@ -34,6 +36,10 @@ pub enum BackgroundTask { /// Attempts to ping Minecraft Java servers as if we were a client, to /// collect info on if they're online, game version, description, etc. PingMinecraftJavaServers, + /// Finds files of versions which have not been scanned for attributions + /// yet, extracts them to find file overrides, and finds any overrides which + /// require attribution from the creator. + ScanAttributions, } impl BackgroundTask { @@ -44,6 +50,7 @@ impl BackgroundTask { ro_pool: PgPool, redis_pool: RedisPool, search_backend: web::Data, + file_host: web::Data, clickhouse: clickhouse::Client, stripe_client: stripe::Client, anrok_client: anrok::Client, @@ -90,6 +97,14 @@ impl BackgroundTask { PingMinecraftJavaServers => { ping_minecraft_java_servers(pool, redis_pool, clickhouse).await } + ScanAttributions => { + scan_all_file_override_attributions( + &pool, + &redis_pool, + &**file_host, + ) + .await + } } } } diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index 646d5e4cc2..cb7fe97ba4 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -1,12 +1,12 @@ use super::DatabaseError; use crate::database::PgTransaction; use crate::models::ids::{ - AffiliateCodeId, ChargeId, CollectionId, FileId, ImageId, NotificationId, - OAuthAccessTokenId, OAuthClientAuthorizationId, OAuthClientId, - OAuthRedirectUriId, OrganizationId, PatId, PayoutId, ProductId, - ProductPriceId, ProjectId, ReportId, SessionId, SharedInstanceId, - SharedInstanceVersionId, TeamId, TeamMemberId, ThreadId, ThreadMessageId, - UserSubscriptionId, VersionId, + AffiliateCodeId, AttributionGroupId, ChargeId, CollectionId, FileId, + ImageId, NotificationId, OAuthAccessTokenId, OAuthClientAuthorizationId, + OAuthClientId, OAuthRedirectUriId, OrganizationId, PatId, PayoutId, + ProductId, ProductPriceId, ProjectId, ReportId, SessionId, + SharedInstanceId, SharedInstanceVersionId, TeamId, TeamMemberId, ThreadId, + ThreadMessageId, UserSubscriptionId, VersionId, }; use ariadne::ids::base62_impl::to_base62; use ariadne::ids::{UserId, random_base62_rng, random_base62_rng_range}; @@ -168,6 +168,10 @@ db_id_interface!( CollectionId, generator: generate_collection_id @ "collections", ); +db_id_interface!( + AttributionGroupId, + generator: generate_attribution_group_id @ "project_attribution_groups", +); db_id_interface!( FileId, generator: generate_file_id @ "files", diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index 904f799e62..9fa9091a9b 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -6,6 +6,7 @@ use super::{DBUser, ids::*}; use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::database::{PgTransaction, models}; +use crate::file_hosting::FileHost; use crate::models::exp; use crate::models::ids::ProjectId; use crate::models::projects::{ @@ -187,6 +188,8 @@ impl ProjectBuilder { pub async fn insert( self, transaction: &mut PgTransaction<'_>, + redis: &RedisPool, + file_host: &dyn FileHost, http: &reqwest::Client, ) -> Result { let project_struct = DBProject { @@ -235,7 +238,7 @@ impl ProjectBuilder { for mut version in self.initial_versions { version.project_id = self.project_id; - version.insert(&mut *transaction, http).await?; + version.insert(transaction, redis, file_host, http).await?; } LinkUrl::insert_many_projects( diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index 6ffaf90c7f..1a6533968a 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -6,8 +6,11 @@ use crate::database::models::loader_fields::{ QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField, }; use crate::database::redis::RedisPool; +use crate::file_hosting::FileHost; use crate::models::exp; + use crate::models::projects::{FileType, VersionStatus}; +use crate::queue::attribution_scan::scan_file_override_attributions; use crate::routes::internal::delphi::DelphiRunParameters; use chrono::{DateTime, Utc}; use dashmap::{DashMap, DashSet}; @@ -17,6 +20,7 @@ use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashMap; use std::iter; +use tracing::error; pub const VERSIONS_NAMESPACE: &str = "versions"; const VERSION_FILES_NAMESPACE: &str = "versions_files"; @@ -134,7 +138,10 @@ impl VersionFileBuilder { pub async fn insert( self, version_id: DBVersionId, + project_id: DBProjectId, transaction: &mut PgTransaction<'_>, + redis: &RedisPool, + file_host: &dyn FileHost, http: &reqwest::Client, ) -> Result { let file_id = generate_file_id(&mut *transaction).await?; @@ -169,6 +176,16 @@ impl VersionFileBuilder { .await?; } + sqlx::query!( + " + INSERT INTO file_attributions (file_id) + VALUES ($1) + ", + file_id as DBFileId, + ) + .execute(&mut *transaction) + .await?; + if let Err(err) = crate::routes::internal::delphi::run( &mut *transaction, DelphiRunParameters { @@ -178,7 +195,20 @@ impl VersionFileBuilder { ) .await { - tracing::error!("Error submitting new file to Delphi: {err}"); + error!("Error submitting new file to Delphi: {err:?}"); + } + + if let Err(err) = scan_file_override_attributions( + &mut *transaction, + redis, + file_host, + project_id, + file_id, + &self.url, + ) + .await + { + error!("Error scanning new file {file_id:?}: {err:?}"); } Ok(file_id) @@ -195,6 +225,8 @@ impl VersionBuilder { pub async fn insert( self, transaction: &mut PgTransaction<'_>, + redis: &RedisPool, + file_host: &dyn FileHost, http: &reqwest::Client, ) -> Result { let version = DBVersion { @@ -236,7 +268,15 @@ impl VersionBuilder { } = self; for file in files { - file.insert(version_id, transaction, http).await?; + file.insert( + version_id, + self.project_id, + transaction, + redis, + file_host, + http, + ) + .await?; } DependencyBuilder::insert_many( @@ -862,14 +902,14 @@ impl DBVersion { }) } - pub async fn get_files_from_hash<'a, 'b, E>( + pub async fn get_files_from_hash<'a, E>( algorithm: String, hashes: &[String], executor: E, redis: &RedisPool, ) -> Result, DatabaseError> where - E: crate::database::Executor<'a, Database = sqlx::Postgres> + Copy, + E: crate::database::Executor<'a, Database = sqlx::Postgres>, { let val = redis.get_cached_keys( VERSION_FILES_NAMESPACE, diff --git a/apps/labrinth/src/file_hosting/mock.rs b/apps/labrinth/src/file_hosting/mock.rs index 3e414bd393..3e04433b76 100644 --- a/apps/labrinth/src/file_hosting/mock.rs +++ b/apps/labrinth/src/file_hosting/mock.rs @@ -72,6 +72,18 @@ impl FileHost for MockHost { file_name: file_name.to_string(), }) } + + async fn read_file( + &self, + file_name: &str, + file_publicity: FileHostPublicity, + ) -> Result { + let file_name = urlencoding::decode(file_name) + .map_err(|_| FileHostingError::InvalidFilename)?; + let path = get_file_path(&file_name, file_publicity); + let data = std::fs::read(&path)?; + Ok(Bytes::from(data)) + } } fn get_file_path( diff --git a/apps/labrinth/src/file_hosting/mod.rs b/apps/labrinth/src/file_hosting/mod.rs index 667f4cb21e..35114a76c7 100644 --- a/apps/labrinth/src/file_hosting/mod.rs +++ b/apps/labrinth/src/file_hosting/mod.rs @@ -45,7 +45,7 @@ pub enum FileHostPublicity { } #[async_trait] -pub trait FileHost { +pub trait FileHost: Send + Sync { async fn upload_file( &self, content_type: &str, @@ -65,6 +65,12 @@ pub trait FileHost { file_name: &str, file_publicity: FileHostPublicity, ) -> Result; + + async fn read_file( + &self, + file_name: &str, + file_publicity: FileHostPublicity, + ) -> Result; } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/apps/labrinth/src/file_hosting/s3_host.rs b/apps/labrinth/src/file_hosting/s3_host.rs index 56bd0ef45c..558e4d5086 100644 --- a/apps/labrinth/src/file_hosting/s3_host.rs +++ b/apps/labrinth/src/file_hosting/s3_host.rs @@ -169,4 +169,28 @@ impl FileHost for S3Host { file_name: file_name.to_string(), }) } + + async fn read_file( + &self, + file_name: &str, + file_publicity: FileHostPublicity, + ) -> Result { + let bucket = self.get_bucket(file_publicity); + + let response = bucket + .client + .get_object() + .bucket(bucket.name.as_str()) + .key(file_name) + .send() + .await + .map_err(|e| s3_error("reading file", e))?; + + Ok(response + .body + .collect() + .await + .map_err(|e| s3_error("reading file body", e))? + .into_bytes()) + } } diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 15363ed09b..d1c4b30fa4 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -57,7 +57,7 @@ pub struct LabrinthConfig { pub ro_pool: ReadOnlyPgPool, pub redis_pool: RedisPool, pub clickhouse: Client, - pub file_host: Arc, + pub file_host: web::Data, pub scheduler: Arc, pub ip_salt: Pepper, pub search_backend: web::Data, @@ -82,7 +82,7 @@ pub fn app_setup( redis_pool: RedisPool, search_backend: actix_web::web::Data, clickhouse: &mut Client, - file_host: Arc, + file_host: web::Data, stripe_client: stripe::Client, anrok_client: anrok::Client, email_queue: EmailQueue, @@ -339,7 +339,7 @@ pub fn app_config( .app_data(web::Data::new(labrinth_config.redis_pool.clone())) .app_data(web::Data::new(labrinth_config.pool.clone())) .app_data(web::Data::new(labrinth_config.ro_pool.clone())) - .app_data(web::Data::new(labrinth_config.file_host.clone())) + .app_data(labrinth_config.file_host.clone()) .app_data(labrinth_config.search_backend.clone()) .app_data(web::Data::new(labrinth_config.gotenberg_client.clone())) .app_data(labrinth_config.http_client.clone()) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index f24a2fb79d..feba395e1a 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -2,14 +2,14 @@ use actix_web::dev::Service; use actix_web::middleware::from_fn; -use actix_web::{App, HttpServer}; +use actix_web::{App, HttpServer, web}; use actix_web_prom::PrometheusMetricsBuilder; use clap::Parser; use labrinth::background_task::BackgroundTask; use labrinth::database::redis::RedisPool; use labrinth::env::ENV; -use labrinth::file_hosting::{FileHostKind, S3BucketConfig, S3Host}; +use labrinth::file_hosting::{FileHost, FileHostKind, S3BucketConfig, S3Host}; use labrinth::queue::email::EmailQueue; use labrinth::search; use labrinth::util::anrok; @@ -111,44 +111,38 @@ async fn app() -> std::io::Result<()> { let redis_pool = RedisPool::new(""); let storage_backend = ENV.STORAGE_BACKEND; - let file_host: Arc = - match storage_backend { - FileHostKind::S3 => { - let not_empty = |v: &str| -> String { - assert!(!v.is_empty(), "S3 env var is empty"); - v.to_string() - }; - - Arc::new( - S3Host::new( - S3BucketConfig { - name: not_empty(&ENV.S3_PUBLIC_BUCKET_NAME), - uses_path_style: ENV - .S3_PUBLIC_USES_PATH_STYLE_BUCKET, - region: not_empty(&ENV.S3_PUBLIC_REGION), - url: not_empty(&ENV.S3_PUBLIC_URL), - access_token: not_empty( - &ENV.S3_PUBLIC_ACCESS_TOKEN, - ), - secret: not_empty(&ENV.S3_PUBLIC_SECRET), - }, - S3BucketConfig { - name: not_empty(&ENV.S3_PRIVATE_BUCKET_NAME), - uses_path_style: ENV - .S3_PRIVATE_USES_PATH_STYLE_BUCKET, - region: not_empty(&ENV.S3_PRIVATE_REGION), - url: not_empty(&ENV.S3_PRIVATE_URL), - access_token: not_empty( - &ENV.S3_PRIVATE_ACCESS_TOKEN, - ), - secret: not_empty(&ENV.S3_PRIVATE_SECRET), - }, - ) - .unwrap(), + let file_host: Arc = match storage_backend { + FileHostKind::S3 => { + let not_empty = |v: &str| -> String { + assert!(!v.is_empty(), "S3 env var is empty"); + v.to_string() + }; + + Arc::new( + S3Host::new( + S3BucketConfig { + name: not_empty(&ENV.S3_PUBLIC_BUCKET_NAME), + uses_path_style: ENV.S3_PUBLIC_USES_PATH_STYLE_BUCKET, + region: not_empty(&ENV.S3_PUBLIC_REGION), + url: not_empty(&ENV.S3_PUBLIC_URL), + access_token: not_empty(&ENV.S3_PUBLIC_ACCESS_TOKEN), + secret: not_empty(&ENV.S3_PUBLIC_SECRET), + }, + S3BucketConfig { + name: not_empty(&ENV.S3_PRIVATE_BUCKET_NAME), + uses_path_style: ENV.S3_PRIVATE_USES_PATH_STYLE_BUCKET, + region: not_empty(&ENV.S3_PRIVATE_REGION), + url: not_empty(&ENV.S3_PRIVATE_URL), + access_token: not_empty(&ENV.S3_PRIVATE_ACCESS_TOKEN), + secret: not_empty(&ENV.S3_PRIVATE_SECRET), + }, ) - } - FileHostKind::Local => Arc::new(file_hosting::MockHost::new()), - }; + .unwrap(), + ) + } + FileHostKind::Local => Arc::new(file_hosting::MockHost::new()), + }; + let file_host = web::Data::::from(file_host); info!("Initializing clickhouse connection"); let mut clickhouse = clickhouse::init_client().await.unwrap(); @@ -174,6 +168,7 @@ async fn app() -> std::io::Result<()> { ro_pool.into_inner(), redis_pool, search_backend, + file_host, clickhouse, stripe_client, anrok_client.clone(), diff --git a/apps/labrinth/src/models/v3/ids.rs b/apps/labrinth/src/models/v3/ids.rs index 7cb162ec27..f283eb199f 100644 --- a/apps/labrinth/src/models/v3/ids.rs +++ b/apps/labrinth/src/models/v3/ids.rs @@ -1,5 +1,6 @@ use ariadne::ids::base62_id; +base62_id!(AttributionGroupId); base62_id!(ChargeId); base62_id!(CollectionId); base62_id!(FileId); diff --git a/apps/labrinth/src/models/v3/projects.rs b/apps/labrinth/src/models/v3/projects.rs index 92abe3fddb..85a7e26bc1 100644 --- a/apps/labrinth/src/models/v3/projects.rs +++ b/apps/labrinth/src/models/v3/projects.rs @@ -681,6 +681,9 @@ pub struct Version { /// A list of files available for download for this version. pub files: Vec, + /// Files in this version that contain override files not yet attributed. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub files_missing_attribution: Vec, /// A list of projects that this version depends on. pub dependencies: Vec, @@ -768,6 +771,7 @@ impl From for Version { .map(|vf| (vf.field_name, vf.value.serialize_internal())) .collect(), components: data.components, + files_missing_attribution: Vec::new(), } } } diff --git a/apps/labrinth/src/queue/attribution_scan.rs b/apps/labrinth/src/queue/attribution_scan.rs new file mode 100644 index 0000000000..a34bf37907 --- /dev/null +++ b/apps/labrinth/src/queue/attribution_scan.rs @@ -0,0 +1,721 @@ +use std::collections::HashMap; +use std::io::{Cursor, Read}; + +use chrono::Utc; +use eyre::{Result, eyre}; +use hex::ToHex; +use sha1::Digest; +use tokio::task::spawn_blocking; +use tracing::{Instrument, info, info_span, warn}; +use zip::ZipArchive; + +use crate::database::models::ids::{ + DBAttributionGroupId, DBProjectId, DBVersionId, generate_attribution_group_id, +}; +use crate::database::models::moderation_external_item::ExternalLicense; +use crate::database::models::{DBFileId, DBUserId, DBVersion}; +use crate::database::{PgPool, PgTransaction, redis::RedisPool}; +use crate::env::ENV; +use crate::file_hosting::{FileHost, FileHostPublicity}; +use crate::models::ids::FileId; +use crate::queue::moderation::{ + ApprovalType, FingerprintResponse, FlameProject, FlameResponse, +}; +use crate::util::error::Context; +use crate::util::http::HTTP_CLIENT; + +pub async fn scan_all_file_override_attributions( + db: &PgPool, + redis: &RedisPool, + file_host: &dyn FileHost, +) -> Result<()> { + let mut txn = db.begin().await.wrap_err("beginning transaction")?; + + let files_to_scan = sqlx::query!( + r#" + select + fa.file_id as "file_id: DBFileId", + f.url, + v.mod_id as "project_id: DBProjectId" + from file_attributions fa + inner join files f on f.id = fa.file_id + inner join versions v on v.id = f.version_id + where fa.scanned_at is null + "# + ) + .fetch_all(&mut txn) + .await + .wrap_err("fetching files to scan")?; + + info!("Found {} files to scan", files_to_scan.len()); + + let mut scanned_ids = Vec::new(); + + for row in files_to_scan { + let human_file_id = FileId::from(row.file_id); + let span = info_span!("scan", file_id = %human_file_id); + async { + info!("Scanning file"); + + let file_id = row.file_id; + + let overrides = extract_override_files_from_storage( + file_host, file_id, &row.url, + ) + .await + .wrap_err_with(|| { + eyre!("extracting overrides for file {file_id:?}") + })?; + + if overrides.is_empty() { + info!("Found no overrides"); + } else { + info!("Found {} overrides", overrides.len()); + + let resolved = resolve_overrides(&overrides, redis, &mut txn) + .await + .wrap_err_with(|| { + eyre!("resolving overrides for file {file_id:?}") + })?; + info!("Resolved: {resolved:#?}"); + + persist_attribution_results( + row.project_id, + file_id, + &overrides, + &resolved, + &mut txn, + ) + .await + .wrap_err_with(|| { + eyre!("persisting attribution results for file {file_id:?}") + })?; + } + + scanned_ids.push(file_id.0); + eyre::Ok(()) + } + .instrument(span) + .await?; + } + + if !scanned_ids.is_empty() { + let now = Utc::now(); + sqlx::query!( + " + update file_attributions + set scanned_at = now + from unnest($1::bigint[], $2::timestamptz[]) as u(id, now) + where file_attributions.file_id = u.id + ", + &scanned_ids, + &vec![now; scanned_ids.len()], + ) + .execute(&mut txn) + .await + .wrap_err("marking files as scanned")?; + } + + info!("Marked {} files as scanned", scanned_ids.len()); + + txn.commit().await.wrap_err("committing transaction")?; + + Ok(()) +} + +pub async fn scan_file_override_attributions( + txn: &mut PgTransaction<'_>, + redis: &RedisPool, + file_host: &dyn FileHost, + project_id: DBProjectId, + file_id: DBFileId, + file_url: &str, +) -> Result<()> { + let overrides = + extract_override_files_from_storage(file_host, file_id, file_url) + .await + .wrap_err_with(|| { + eyre!("extracting overrides for file {file_id:?}") + })?; + + if !overrides.is_empty() { + let resolved = resolve_overrides(&overrides, redis, txn) + .await + .wrap_err_with(|| { + eyre!("resolving overrides for file {file_id:?}") + })?; + + persist_attribution_results(project_id, file_id, &overrides, &resolved, txn) + .await + .wrap_err_with(|| { + eyre!("persisting attribution results for file {file_id:?}") + })?; + } + + sqlx::query!( + " + insert into file_attributions (file_id, scanned_at) + values ($1, now()) + on conflict (file_id) do update set scanned_at = now() + ", + file_id.0, + ) + .execute(&mut *txn) + .await + .wrap_err("marking file as scanned")?; + + Ok(()) +} + +async fn extract_override_files_from_storage( + file_host: &dyn FileHost, + file_id: DBFileId, + file_url: &str, +) -> Result> { + let key = file_url + .strip_prefix(&ENV.CDN_URL) + .unwrap_or(file_url) + .trim_start_matches('/'); + + let file_data = file_host + .read_file(key, FileHostPublicity::Public) + .await + .wrap_err_with(|| { + eyre!("reading file {file_id:?} from storage at {key}") + })?; + + spawn_blocking(move || extract_override_files(&file_data)) + .await + .wrap_err("extracting override files")? + .wrap_err("extracting override files") +} + +#[derive(Debug)] +pub struct OverrideFile { + pub path: String, + pub sha1: String, + pub murmur2: u32, +} + +#[derive(Debug)] +pub enum OverrideResolution { + OnModrinth, + ExternalLicense(ApprovalType), + Flame { + project_id: u32, + title: String, + url: String, + }, + Unknown, +} + +const OVERRIDE_PREFIXES: &[&str] = &[ + "overrides/mods", + "client-overrides/mods", + "server-overrides/mods", + "overrides/shaderpacks", + "client-overrides/shaderpacks", + "overrides/resourcepacks", + "client-overrides/resourcepacks", +]; + +fn extract_override_files(data: &[u8]) -> Result> { + let reader = Cursor::new(data); + let mut zip = + ZipArchive::new(reader).wrap_err("creating zip archive reader")?; + + let mut files = Vec::new(); + + for i in 0..zip.len() { + let mut file = zip + .by_index(i) + .wrap_err_with(|| eyre!("reading file {i}"))?; + let name = file.name().to_string(); + + if !OVERRIDE_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + { + continue; + } + + if name.matches('/').count() > 2 + || name.ends_with(".txt") + || name.ends_with(".rpo") + { + continue; + } + + let mut contents = Vec::new(); + file.read_to_end(&mut contents)?; + + let sha1 = sha1::Sha1::digest(&contents).encode_hex::(); + let murmur = hash_flame_murmur32(contents); + + files.push(OverrideFile { + sha1, + murmur2: murmur, + path: name, + }); + } + + Ok(files) +} + +async fn persist_attribution_results( + project_id: DBProjectId, + file_id: DBFileId, + overrides: &[OverrideFile], + resolved: &HashMap, + txn: &mut PgTransaction<'_>, +) -> Result<()> { + let all_sha1s: Vec> = overrides + .iter() + .map(|f| f.sha1.as_bytes().to_vec()) + .collect(); + + let already_persisted: Vec> = sqlx::query_scalar!( + " + select paf.sha1 from project_attribution_files paf + inner join project_attribution_groups pag on pag.id = paf.group_id + where pag.project_id = $1 and paf.sha1 = ANY($2) + ", + project_id as DBProjectId, + &all_sha1s, + ) + .fetch_all(&mut *txn) + .await + .wrap_err("checking existing attribution files")?; + + let mut flame_groups: HashMap, Option)> = + HashMap::new(); + let mut unknown_files: Vec<&OverrideFile> = Vec::new(); + + for file in overrides { + if already_persisted + .iter() + .any(|s| s.as_slice() == file.sha1.as_bytes()) + { + continue; + } + + match resolved.get(&file.sha1) { + Some(OverrideResolution::OnModrinth) => continue, + Some(OverrideResolution::ExternalLicense(approval)) => { + if approval.approved() { + continue; + } + unknown_files.push(file); + } + Some(OverrideResolution::Flame { + project_id: fp_id, + title, + .. + }) => { + let entry = flame_groups.entry(*fp_id).or_default(); + entry.0.push(file); + if entry.1.is_none() { + entry.1 = Some(title.clone()); + } + } + Some(OverrideResolution::Unknown) | None => { + unknown_files.push(file); + } + } + } + + let existing_flame_groups = sqlx::query!( + r#" + select id as "id: DBAttributionGroupId", flame_project_id + from project_attribution_groups + where project_id = $1 and flame_project_id is not null + "#, + project_id as DBProjectId, + ) + .fetch_all(&mut *txn) + .await + .wrap_err("fetching existing flame attribution groups")?; + + for (flame_project_id, (files, title)) in &flame_groups { + let existing = existing_flame_groups + .iter() + .find(|g| g.flame_project_id == Some(*flame_project_id as i64)); + + let group_id = if let Some(group) = existing { + group.id + } else { + let id = generate_attribution_group_id(&mut *txn).await?; + sqlx::query!( + " + insert into project_attribution_groups (id, project_id, flame_project_id, flame_project_title) + values ($1, $2, $3, $4) + ", + id as DBAttributionGroupId, + project_id as DBProjectId, + *flame_project_id as i64, + title.as_deref().unwrap_or_default(), + ) + .execute(&mut *txn) + .await + .wrap_err("inserting attribution group")?; + id + }; + + let names: Vec = files.iter().map(|f| f.path.clone()).collect(); + let sha1s: Vec> = + files.iter().map(|f| f.sha1.as_bytes().to_vec()).collect(); + + sqlx::query!( + " + insert into project_attribution_files (group_id, name, sha1) + select $1, unnest($2::text[]), unnest($3::bytea[]) + ", + group_id as DBAttributionGroupId, + &names, + &sha1s, + ) + .execute(&mut *txn) + .await + .wrap_err("inserting attribution files")?; + } + + for file in &unknown_files { + let group_id = generate_attribution_group_id(&mut *txn).await?; + sqlx::query!( + " + insert into project_attribution_groups (id, project_id) + values ($1, $2) + ", + group_id as DBAttributionGroupId, + project_id as DBProjectId, + ) + .execute(&mut *txn) + .await + .wrap_err("inserting unknown attribution group")?; + + sqlx::query!( + " + insert into project_attribution_files (group_id, name, sha1) + values ($1, $2, $3) + ", + group_id as DBAttributionGroupId, + &file.path, + &file.sha1.as_bytes().to_vec() as &[u8], + ) + .execute(&mut *txn) + .await + .wrap_err("inserting unknown attribution file")?; + } + + if !all_sha1s.is_empty() { + sqlx::query!( + " + insert into override_file_sources (sha1, file_id) + select unnest($1::bytea[]), $2 + on conflict do nothing + ", + &all_sha1s, + file_id as DBFileId, + ) + .execute(&mut *txn) + .await + .wrap_err("inserting override file sources")?; + } + + Ok(()) +} + +async fn resolve_overrides( + overrides: &[OverrideFile], + redis: &RedisPool, + txn: &mut PgTransaction<'_>, +) -> Result> { + let mut results: HashMap = HashMap::new(); + let mut remaining: Vec = (0..overrides.len()).collect(); + + if overrides.is_empty() { + return Ok(results); + } + + let hashes: Vec = + overrides.iter().map(|x| x.sha1.clone()).collect(); + let files = DBVersion::get_files_from_hash( + "sha1".to_string(), + &hashes, + &mut *txn, + redis, + ) + .await + .wrap_err("fetching files on platform by hash")?; + + let version_ids: Vec<_> = files.iter().map(|x| x.version_id).collect(); + let versions_data = DBVersion::get_many(&version_ids, &mut *txn, redis) + .await + .wrap_err("fetching versions")?; + + for file in &files { + if !versions_data.iter().any(|v| v.inner.id == file.version_id) { + continue; + } + + if let Some(hash) = file.hashes.get("sha1") + && let Some(pos) = + remaining.iter().position(|i| overrides[*i].sha1 == *hash) + { + let idx = remaining.remove(pos); + results.insert( + overrides[idx].sha1.clone(), + OverrideResolution::OnModrinth, + ); + } + } + + if remaining.is_empty() { + return Ok(results); + } + + let rows = sqlx::query!( + " + SELECT encode(mef.sha1, 'escape') sha1, mel.status status + FROM moderation_external_files mef + INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id + WHERE mef.sha1 = ANY($1) + ", + &remaining + .iter() + .map(|i| overrides[*i].sha1.as_bytes().to_vec()) + .collect::>() + ) + .fetch_all(&mut *txn) + .await + .wrap_err("fetching external file licenses")?; + + for row in rows { + if let Some(sha1) = row.sha1 + && let Some(pos) = + remaining.iter().position(|i| overrides[*i].sha1 == sha1) + { + let idx = remaining.remove(pos); + results.insert( + overrides[idx].sha1.clone(), + OverrideResolution::ExternalLicense( + ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + ), + ); + } + } + + if remaining.is_empty() { + return Ok(results); + } + + let fingerprints: Vec = + remaining.iter().map(|i| overrides[*i].murmur2).collect(); + let res = HTTP_CLIENT + .post(format!("{}/v1/fingerprints", ENV.FLAME_ANVIL_URL)) + .json(&serde_json::json!({ + "fingerprints": fingerprints + })) + .send() + .await; + + if let Err(e) = &res { + warn!("Flame fingerprint request failed: {e}"); + } + + if let Ok(res) = res { + let body = res + .text() + .await + .wrap_err("reading Flame fingerprint response")?; + + let flame_files: Vec<_> = + serde_json::from_str::>(&body) + .ok() + .map(|x| { + x.data + .exact_matches + .into_iter() + .map(|m| m.file) + .collect::>() + }) + .unwrap_or_default(); + + let mut flame_matches: Vec<(String, u32)> = Vec::new(); + for flame_file in &flame_files { + if let Some(hash) = flame_file + .hashes + .iter() + .find(|x| x.algo == 1) + .map(|x| x.value.clone()) + { + flame_matches.push((hash, flame_file.mod_id)); + } + } + + let rows = sqlx::query!( + " + SELECT mel.id, mel.flame_project_id, mel.status status + FROM moderation_external_licenses mel + WHERE mel.flame_project_id = ANY($1) + ", + &flame_matches.iter().map(|x| x.1 as i32).collect::>() + ) + .fetch_all(&mut *txn) + .await + .wrap_err("fetching Flame project licenses")?; + + let mut insert_hashes = Vec::new(); + let mut insert_filenames = Vec::new(); + let mut insert_ids = Vec::new(); + + for row in &rows { + if let Some((curse_idx, (hash, _))) = flame_matches + .iter() + .enumerate() + .find(|(_, x)| Some(x.1 as i32) == row.flame_project_id) + && let Some(remaining_pos) = + remaining.iter().position(|i| overrides[*i].sha1 == *hash) + { + let idx = remaining.remove(remaining_pos); + + results.insert( + overrides[idx].sha1.clone(), + OverrideResolution::ExternalLicense( + ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + ), + ); + + insert_hashes.push(overrides[idx].sha1.as_bytes().to_vec()); + insert_filenames.push(Some(overrides[idx].path.clone())); + insert_ids.push(row.id); + + flame_matches.remove(curse_idx); + } + } + + if !insert_hashes.is_empty() { + ExternalLicense::insert_files( + &mut *txn, + &insert_hashes, + &insert_filenames, + &insert_ids, + DBUserId(0), + ) + .await + .wrap_err("inserting external license files")?; + } + + if !flame_matches.is_empty() { + let flame_projects_res = HTTP_CLIENT + .post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL)) + .json(&serde_json::json!({ + "modIds": flame_matches.iter().map(|x| x.1).collect::>() + })) + .send() + .await; + + let flame_projects = match flame_projects_res { + Ok(res) => res + .text() + .await + .ok() + .and_then(|t| { + serde_json::from_str::< + FlameResponse>, + >(&t) + .ok() + }) + .map(|x| x.data) + .unwrap_or_default(), + Err(e) => { + warn!("Flame projects request failed: {e}"); + Vec::new() + } + }; + + for (sha1, flame_project_id) in &flame_matches { + if let Some(pos) = + remaining.iter().position(|i| overrides[*i].sha1 == *sha1) + { + let idx = remaining.remove(pos); + + let project = flame_projects + .iter() + .find(|p| p.id == *flame_project_id); + + results.insert( + overrides[idx].sha1.clone(), + OverrideResolution::Flame { + project_id: *flame_project_id, + title: project + .map(|p| p.name.clone()) + .unwrap_or_else(|| { + format!("Flame project {flame_project_id}") + }), + url: project + .map(|p| p.links.website_url.clone()) + .unwrap_or_default(), + }, + ); + } + } + } + } + + for idx in remaining { + results + .insert(overrides[idx].sha1.clone(), OverrideResolution::Unknown); + } + + Ok(results) +} + +fn hash_flame_murmur32(input: Vec) -> u32 { + murmur2::murmur2( + &input + .into_iter() + .filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32) + .collect::>(), + 1, + ) +} + +pub async fn get_files_missing_attribution<'a, E>( + exec: E, + version_ids: &[DBVersionId], +) -> Result>> +where + E: sqlx::Executor<'a, Database = sqlx::Postgres>, +{ + if version_ids.is_empty() { + return Ok(std::collections::HashMap::new()); + } + + let rows = sqlx::query!( + r#" + select distinct f.version_id as "version_id: DBVersionId", f.id as "file_id: DBFileId" + from files f + inner join override_file_sources ofs on ofs.file_id = f.id + inner join project_attribution_files paf on paf.sha1 = ofs.sha1 + inner join project_attribution_groups pag on pag.id = paf.group_id + where f.version_id = ANY($1) and pag.attribution is null + "#, + &version_ids.iter().map(|v| v.0).collect::>(), + ) + .fetch_all(exec) + .await + .wrap_err("fetching files missing attribution")?; + + let mut result = std::collections::HashMap::new(); + for row in rows { + result + .entry(row.version_id) + .or_insert_with(Vec::new) + .push(row.file_id); + } + + Ok(result) +} diff --git a/apps/labrinth/src/queue/mod.rs b/apps/labrinth/src/queue/mod.rs index 666670dc0b..2779242ca0 100644 --- a/apps/labrinth/src/queue/mod.rs +++ b/apps/labrinth/src/queue/mod.rs @@ -1,4 +1,5 @@ pub mod analytics; +pub mod attribution_scan; pub mod billing; pub mod email; pub mod moderation; diff --git a/apps/labrinth/src/queue/moderation.rs b/apps/labrinth/src/queue/moderation.rs index 7d852f4eb3..ab41c33f83 100644 --- a/apps/labrinth/src/queue/moderation.rs +++ b/apps/labrinth/src/queue/moderation.rs @@ -570,7 +570,7 @@ impl AutomatedModerationQueue { Vec::new() } else { let res = client - .post(format!("{}v1/mods", ENV.FLAME_ANVIL_URL)) + .post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL)) .json(&serde_json::json!({ "modIds": flame_files.iter().map(|x| x.1).collect::>() })) @@ -823,7 +823,7 @@ pub enum ApprovalType { } impl ApprovalType { - fn approved(&self) -> bool { + pub fn approved(&self) -> bool { match self { ApprovalType::Yes => true, ApprovalType::WithAttributionAndSource => true, diff --git a/apps/labrinth/src/routes/internal/attribution.rs b/apps/labrinth/src/routes/internal/attribution.rs new file mode 100644 index 0000000000..3ca9c70540 --- /dev/null +++ b/apps/labrinth/src/routes/internal/attribution.rs @@ -0,0 +1,353 @@ +use actix_web::{HttpRequest, get, patch, post, web}; +use serde::{Deserialize, Serialize}; +use sqlx::Row; + +use crate::auth::get_user_from_headers; +use crate::database::models::ids::{ + DBAttributionGroupId, DBProjectId, generate_attribution_group_id, +}; +use crate::database::redis::RedisPool; +use crate::database::PgPool; +use crate::models::ids::{ProjectId, VersionId}; +use crate::models::pats::Scopes; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; + +pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) { + cfg.service( + utoipa_actix_web::scope("/attribution") + .service(list) + .service(update_group) + .service(assign) + .service(split), + ); +} + +#[derive(Serialize)] +struct AttributionGroupResponse { + id: crate::models::ids::AttributionGroupId, + flame_project_id: Option, + flame_project_title: Option, + attribution: Option, + attributed_at: Option>, + attributed_by: Option, + files: Vec, + versions: Vec, +} + +#[derive(Clone, Serialize)] +struct VersionInfo { + id: VersionId, + name: String, + version_number: String, + date_created: chrono::DateTime, +} + +#[derive(Serialize)] +struct AttributionFileResponse { + name: String, + sha1: String, + versions: Vec, +} + +#[utoipa::path] +#[get("{project_id}")] +async fn list( + pool: web::Data, + path: web::Path, +) -> Result>, ApiError> { + let project_id: DBProjectId = path.into_inner().into(); + + let groups = sqlx::query!( + r#" + select + g.id as "id: DBAttributionGroupId", + g.flame_project_id, + g.flame_project_title, + g.attribution, + g.attributed_at, + g.attributed_by as "attributed_by: i64" + from project_attribution_groups g + where g.project_id = $1 + "#, + project_id as DBProjectId, + ) + .fetch_all(pool.as_ref()) + .await?; + + let group_ids: Vec = groups.iter().map(|g| g.id.0).collect(); + + let files = if group_ids.is_empty() { + Vec::new() + } else { + sqlx::query( + " + select paf.group_id, paf.name, encode(paf.sha1, 'escape') as sha1, + array_agg(distinct f.version_id) as version_ids + from project_attribution_files paf + left join override_file_sources ofs on ofs.sha1 = paf.sha1 + left join files f on f.id = ofs.file_id + where paf.group_id = ANY($1) + group by paf.group_id, paf.name, paf.sha1 + ", + ) + .bind(&group_ids) + .fetch_all(pool.as_ref()) + .await? + }; + + let mut all_version_ids: Vec = files + .iter() + .filter_map(|f| f.get::>, _>("version_ids")) + .flatten() + .collect(); + all_version_ids.sort_unstable(); + all_version_ids.dedup(); + + let version_infos = if all_version_ids.is_empty() { + Vec::new() + } else { + let rows = sqlx::query!( + " + select id, name, version_number, date_published + from versions + where id = ANY($1) + order by date_published desc + ", + &all_version_ids, + ) + .fetch_all(pool.as_ref()) + .await?; + rows.into_iter() + .map(|v| VersionInfo { + id: VersionId(v.id as u64), + name: v.name, + version_number: v.version_number, + date_created: v.date_published, + }) + .collect() + }; + let version_order = version_infos + .iter() + .enumerate() + .map(|(index, version)| (version.id, index)) + .collect::>(); + + let mut result = Vec::new(); + for group in groups { + let group_files: Vec = files + .iter() + .filter(|f| f.get::("group_id") == group.id.0) + .map(|f| AttributionFileResponse { + name: f.get("name"), + sha1: f.get("sha1"), + versions: { + let mut versions: Vec<_> = f + .get::>, _>("version_ids") + .unwrap_or_default() + .into_iter() + .map(|id| VersionId(id as u64)) + .collect(); + versions.sort_by_key(|id| { + version_order.get(id).copied().unwrap_or(usize::MAX) + }); + versions + }, + }) + .collect(); + + result.push(AttributionGroupResponse { + id: group.id.into(), + flame_project_id: group.flame_project_id, + flame_project_title: group.flame_project_title, + attribution: group.attribution, + attributed_at: group.attributed_at, + attributed_by: group.attributed_by.map(|id| ariadne::ids::UserId(id as u64)), + files: group_files, + versions: version_infos.clone(), + }); + } + + Ok(web::Json(result)) +} + +#[derive(Deserialize, utoipa::ToSchema)] +struct UpdateGroupBody { + attribution: serde_json::Value, +} + +#[utoipa::path] +#[patch("group/{group_id}")] +async fn update_group( + req: HttpRequest, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, + path: web::Path, + web::Json(body): web::Json, +) -> Result<(), ApiError> { + let group_id = path.into_inner(); + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::VERSION_WRITE, + ) + .await? + .1; + + let result = sqlx::query!( + " + update project_attribution_groups + set attribution = $1, attributed_at = now(), attributed_by = $3 + where id = $2 + ", + &body.attribution, + group_id, + user.id.0 as i64, + ) + .execute(pool.as_ref()) + .await?; + + if result.rows_affected() == 0 { + return Err(ApiError::NotFound); + } + + Ok(()) +} + +#[derive(Deserialize, utoipa::ToSchema)] +struct AssignBody { + sha1: String, + target_group_id: i64, + project_id: ProjectId, +} + +#[utoipa::path] +#[post("assign")] +async fn assign( + pool: web::Data, + web::Json(body): web::Json, +) -> Result<(), ApiError> { + let sha1_bytes = hex_to_bytes(&body.sha1).ok_or_else(|| { + ApiError::InvalidInput("invalid sha1 hex string".to_string()) + })?; + let project_id: DBProjectId = body.project_id.into(); + + let result = sqlx::query!( + " + update project_attribution_files + set group_id = $1 + where sha1 = $2 + and group_id in ( + select id from project_attribution_groups where project_id = $3 + ) + ", + body.target_group_id, + &sha1_bytes, + project_id as DBProjectId, + ) + .execute(pool.as_ref()) + .await?; + + if result.rows_affected() == 0 { + return Err(ApiError::NotFound); + } + + cleanup_empty_groups(pool.as_ref()).await?; + + Ok(()) +} + +#[derive(Deserialize, utoipa::ToSchema)] +struct SplitBody { + sha1: String, + project_id: ProjectId, +} + +#[utoipa::path] +#[post("split")] +async fn split( + pool: web::Data, + web::Json(body): web::Json, +) -> Result<(), ApiError> { + let sha1_bytes = hex_to_bytes(&body.sha1).ok_or_else(|| { + ApiError::InvalidInput("invalid sha1 hex string".to_string()) + })?; + let project_id: DBProjectId = body.project_id.into(); + + let existing = sqlx::query!( + " + select paf.group_id, paf.name from project_attribution_files paf + inner join project_attribution_groups pag on pag.id = paf.group_id + where paf.sha1 = $1 and pag.project_id = $2 + ", + &sha1_bytes, + project_id as DBProjectId, + ) + .fetch_optional(pool.as_ref()) + .await?; + + let Some(existing) = existing else { + return Err(ApiError::NotFound); + }; + + let mut txn = pool.begin().await?; + + let new_group_id = generate_attribution_group_id(&mut txn).await?; + + sqlx::query!( + " + insert into project_attribution_groups (id, project_id) + values ($1, $2) + ", + new_group_id as DBAttributionGroupId, + project_id as DBProjectId, + ) + .execute(&mut txn) + .await?; + + sqlx::query!( + " + update project_attribution_files + set group_id = $1 + where sha1 = $2 and group_id = $3 + ", + new_group_id as DBAttributionGroupId, + &sha1_bytes, + existing.group_id, + ) + .execute(&mut txn) + .await?; + + txn.commit().await?; + + cleanup_empty_groups(pool.as_ref()).await?; + + Ok(()) +} + +async fn cleanup_empty_groups(pool: &PgPool) -> Result<(), ApiError> { + sqlx::query!( + " + delete from project_attribution_groups g + where not exists ( + select 1 from project_attribution_files f where f.group_id = g.id + ) + ", + ) + .execute(pool) + .await?; + Ok(()) +} + +fn hex_to_bytes(hex: &str) -> Option> { + if !hex.len().is_multiple_of(2) { + return None; + } + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).ok()) + .collect() +} diff --git a/apps/labrinth/src/routes/internal/flows.rs b/apps/labrinth/src/routes/internal/flows.rs index 49c81fdaa0..825ebfba3d 100644 --- a/apps/labrinth/src/routes/internal/flows.rs +++ b/apps/labrinth/src/routes/internal/flows.rs @@ -38,7 +38,6 @@ use reqwest::header::AUTHORIZATION; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::str::FromStr; -use std::sync::Arc; use tracing::info; use validator::Validate; use zxcvbn::Score; @@ -83,7 +82,7 @@ impl TempUser { provider: AuthProvider, transaction: &mut PgTransaction<'_>, client: &PgPool, - file_host: &Arc, + file_host: &dyn FileHost, redis: &RedisPool, ) -> Result { if let Some(email) = &self.email @@ -150,7 +149,7 @@ impl TempUser { ext, Some(96), Some(1.0), - &**file_host, + file_host, ) .await; @@ -1161,7 +1160,7 @@ pub async fn auth_callback( req: HttpRequest, Query(query): Query>, client: Data, - file_host: Data>, + file_host: Data, redis: Data, ) -> Result { let state_string = query @@ -1319,7 +1318,7 @@ pub async fn auth_callback( provider, &mut transaction, &client, - &file_host, + &**file_host, &redis, ) .await? diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index eac882c339..94d612d73b 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -1,5 +1,6 @@ pub mod admin; pub mod affiliate; +pub mod attribution; pub mod billing; pub mod delphi; pub mod external_notifications; @@ -99,5 +100,10 @@ pub fn utoipa_config( utoipa_actix_web::scope("/_internal/server-ping") .wrap(default_cors()) .configure(server_ping::config), + ) + .service( + utoipa_actix_web::scope("/_internal/attribution") + .wrap(default_cors()) + .configure(attribution::config), ); } diff --git a/apps/labrinth/src/routes/v2/project_creation.rs b/apps/labrinth/src/routes/v2/project_creation.rs index dd14c32bdd..aa8e491229 100644 --- a/apps/labrinth/src/routes/v2/project_creation.rs +++ b/apps/labrinth/src/routes/v2/project_creation.rs @@ -20,7 +20,6 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use std::collections::HashMap; -use std::sync::Arc; use validator::Validate; use super::version_creation::InitialVersionData; @@ -158,7 +157,7 @@ pub async fn project_create( payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: Data, http: Data, ) -> Result { diff --git a/apps/labrinth/src/routes/v2/projects.rs b/apps/labrinth/src/routes/v2/projects.rs index ab24386b8e..46aa924162 100644 --- a/apps/labrinth/src/routes/v2/projects.rs +++ b/apps/labrinth/src/routes/v2/projects.rs @@ -18,7 +18,6 @@ use crate::search::{SearchBackend, SearchRequest}; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::sync::Arc; use validator::Validate; pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) { @@ -924,7 +923,7 @@ pub async fn project_icon_edit( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -964,7 +963,7 @@ pub async fn delete_project_icon( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { // Returns NoContent, so no need to convert @@ -1055,7 +1054,7 @@ pub async fn add_gallery_item( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -1198,7 +1197,7 @@ pub async fn delete_gallery_item( web::Query(item): web::Query, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { // Returns NoContent, so no need to convert diff --git a/apps/labrinth/src/routes/v2/threads.rs b/apps/labrinth/src/routes/v2/threads.rs index 51145e709e..e04a369cc5 100644 --- a/apps/labrinth/src/routes/v2/threads.rs +++ b/apps/labrinth/src/routes/v2/threads.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use crate::database::PgPool; use crate::database::redis::RedisPool; use crate::file_hosting::FileHost; @@ -169,7 +167,7 @@ pub async fn message_delete( pool: web::Data, redis: web::Data, session_queue: web::Data, - file_host: web::Data>, + file_host: web::Data, ) -> Result { // Returns NoContent, so we don't need to convert the response v3::threads::message_delete( diff --git a/apps/labrinth/src/routes/v2/users.rs b/apps/labrinth/src/routes/v2/users.rs index 10de37dbfa..67e24aa57f 100644 --- a/apps/labrinth/src/routes/v2/users.rs +++ b/apps/labrinth/src/routes/v2/users.rs @@ -11,7 +11,6 @@ use crate::queue::session::AuthQueue; use crate::routes::{ApiError, v2_reroute, v3}; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use validator::Validate; pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) { @@ -295,7 +294,7 @@ pub async fn user_icon_edit( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -335,7 +334,7 @@ pub async fn user_icon_delete( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { // Returns NoContent, so we don't need to convert to V2 diff --git a/apps/labrinth/src/routes/v2/version_creation.rs b/apps/labrinth/src/routes/v2/version_creation.rs index 3a21183ba4..bd37a55a96 100644 --- a/apps/labrinth/src/routes/v2/version_creation.rs +++ b/apps/labrinth/src/routes/v2/version_creation.rs @@ -21,7 +21,6 @@ use actix_web::{HttpRequest, HttpResponse, post, web}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::collections::HashMap; -use std::sync::Arc; use validator::Validate; pub fn default_requested_status() -> VersionStatus { @@ -99,7 +98,7 @@ pub async fn version_create( payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: Data, moderation_queue: Data, http: Data, @@ -327,7 +326,7 @@ pub async fn upload_file_to_version( payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: web::Data, http: web::Data, ) -> Result { diff --git a/apps/labrinth/src/routes/v3/collections.rs b/apps/labrinth/src/routes/v3/collections.rs index cbf75477a0..605f857065 100644 --- a/apps/labrinth/src/routes/v3/collections.rs +++ b/apps/labrinth/src/routes/v3/collections.rs @@ -25,7 +25,6 @@ use chrono::Utc; use eyre::eyre; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use validator::Validate; pub fn config(cfg: &mut web::ServiceConfig) { @@ -390,7 +389,7 @@ pub async fn collection_icon_edit( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, mut payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -423,7 +422,7 @@ pub async fn collection_icon_edit( collection_item.icon_url, collection_item.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; @@ -442,7 +441,7 @@ pub async fn collection_icon_edit( &ext.ext, Some(96), Some(1.0), - &***file_host, + &**file_host, ) .await?; @@ -474,7 +473,7 @@ pub async fn delete_collection_icon( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { let user = get_user_from_headers( @@ -505,7 +504,7 @@ pub async fn delete_collection_icon( collection_item.icon_url, collection_item.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; let mut transaction = pool.begin().await?; diff --git a/apps/labrinth/src/routes/v3/images.rs b/apps/labrinth/src/routes/v3/images.rs index efeb59a838..d00e03946c 100644 --- a/apps/labrinth/src/routes/v3/images.rs +++ b/apps/labrinth/src/routes/v3/images.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use super::threads::is_authorized_thread; use crate::auth::checks::{is_team_member_project, is_team_member_version}; use crate::auth::get_user_from_headers; @@ -41,7 +39,7 @@ pub struct ImageUpload { pub async fn images_add( req: HttpRequest, web::Query(data): web::Query, - file_host: web::Data>, + file_host: web::Data, mut payload: web::Payload, pool: web::Data, redis: web::Data, @@ -191,7 +189,7 @@ pub async fn images_add( &data.ext, None, None, - &***file_host, + &**file_host, ) .await?; diff --git a/apps/labrinth/src/routes/v3/oauth_clients.rs b/apps/labrinth/src/routes/v3/oauth_clients.rs index adcb4e477e..7c16ea141d 100644 --- a/apps/labrinth/src/routes/v3/oauth_clients.rs +++ b/apps/labrinth/src/routes/v3/oauth_clients.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, fmt::Display, sync::Arc}; +use std::{collections::HashSet, fmt::Display}; use super::ApiError; use crate::database::{PgPool, PgTransaction}; @@ -354,7 +354,7 @@ pub async fn oauth_client_icon_edit( client_id: web::Path, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, mut payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -382,7 +382,7 @@ pub async fn oauth_client_icon_edit( client.icon_url.clone(), client.raw_icon_url.clone(), FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; @@ -399,7 +399,7 @@ pub async fn oauth_client_icon_edit( &ext.ext, Some(96), Some(1.0), - &***file_host, + &**file_host, ) .await?; @@ -424,7 +424,7 @@ pub async fn oauth_client_icon_delete( client_id: web::Path, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { let user = get_user_from_headers( @@ -450,7 +450,7 @@ pub async fn oauth_client_icon_delete( client.icon_url.clone(), client.raw_icon_url.clone(), FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; diff --git a/apps/labrinth/src/routes/v3/organizations.rs b/apps/labrinth/src/routes/v3/organizations.rs index 890c6a4e42..80b30aba18 100644 --- a/apps/labrinth/src/routes/v3/organizations.rs +++ b/apps/labrinth/src/routes/v3/organizations.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::sync::Arc; use super::ApiError; use crate::auth::checks::is_visible_organization; @@ -1057,7 +1056,7 @@ pub async fn organization_icon_edit( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, mut payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -1108,7 +1107,7 @@ pub async fn organization_icon_edit( organization_item.icon_url, organization_item.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; @@ -1127,7 +1126,7 @@ pub async fn organization_icon_edit( &ext.ext, Some(96), Some(1.0), - &***file_host, + &**file_host, ) .await?; @@ -1163,7 +1162,7 @@ pub async fn delete_organization_icon( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { let user = get_user_from_headers( @@ -1213,7 +1212,7 @@ pub async fn delete_organization_icon( organization_item.icon_url, organization_item.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; diff --git a/apps/labrinth/src/routes/v3/project_creation.rs b/apps/labrinth/src/routes/v3/project_creation.rs index 42f1e4624a..32b487b48d 100644 --- a/apps/labrinth/src/routes/v3/project_creation.rs +++ b/apps/labrinth/src/routes/v3/project_creation.rs @@ -40,7 +40,6 @@ use itertools::Itertools; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::sync::Arc; use thiserror::Error; use validator::Validate; @@ -290,7 +289,7 @@ pub async fn project_create( payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: Data, http: Data, ) -> Result { @@ -311,7 +310,7 @@ pub async fn project_create_internal( mut payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: Data, http: Data, ) -> Result { @@ -325,7 +324,7 @@ pub async fn project_create_internal( req, &mut payload, &mut transaction, - &***file_host, + &**file_host, &mut uploaded_files, &client, &redis, @@ -336,7 +335,7 @@ pub async fn project_create_internal( .await; if result.is_err() { - let undo_result = undo_uploads(&***file_host, &uploaded_files).await; + let undo_result = undo_uploads(&**file_host, &uploaded_files).await; let rollback_result = transaction.rollback().await; undo_result?; @@ -360,7 +359,7 @@ pub async fn project_create_with_id( mut payload: Multipart, client: Data, redis: Data, - file_host: Data>, + file_host: Data, session_queue: Data, http: Data, path: web::Path<(ProjectId,)>, @@ -374,7 +373,7 @@ pub async fn project_create_with_id( req, &mut payload, &mut transaction, - &***file_host, + &**file_host, &mut uploaded_files, &client, &redis, @@ -385,7 +384,7 @@ pub async fn project_create_with_id( .await; if result.is_err() { - let undo_result = undo_uploads(&***file_host, &uploaded_files).await; + let undo_result = undo_uploads(&**file_host, &uploaded_files).await; let rollback_result = transaction.rollback().await; undo_result?; @@ -907,7 +906,7 @@ async fn project_create_inner( let now = Utc::now(); let id = project_builder_actual - .insert(&mut *transaction, http) + .insert(&mut *transaction, redis, file_host, http) .await?; DBUser::clear_project_cache(&[current_user.id.into()], redis).await?; diff --git a/apps/labrinth/src/routes/v3/project_creation/new.rs b/apps/labrinth/src/routes/v3/project_creation/new.rs index cbfd33971f..976c6d612a 100644 --- a/apps/labrinth/src/routes/v3/project_creation/new.rs +++ b/apps/labrinth/src/routes/v3/project_creation/new.rs @@ -16,6 +16,7 @@ use crate::{ }, redis::RedisPool, }, + file_hosting::FileHost, models::{ exp::{self, ProjectComponentKind, component::ComponentRelationError}, ids::ProjectId, @@ -117,6 +118,7 @@ pub async fn create( req: HttpRequest, db: web::Data, redis: web::Data, + file_host: web::Data, session_queue: web::Data, http: web::Data, web::Json(create): web::Json, @@ -305,13 +307,13 @@ pub async fn create( }; project_builder - .insert(&mut txn, &http) + .insert(&mut txn, &redis, &**file_host, &http) .await .wrap_internal_err("failed to insert project")?; if let Some(version_builder) = version_builder { version_builder - .insert(&mut txn, &http) + .insert(&mut txn, &redis, &**file_host, &http) .await .wrap_internal_err("failed to insert initial version")?; } diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index a40f933057..831fdb423a 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -1,6 +1,5 @@ use std::any::type_name; use std::collections::HashMap; -use std::sync::Arc; use crate::auth::checks::{filter_visible_versions, is_visible_project}; use crate::auth::{filter_visible_projects, get_user_from_headers}; @@ -1694,7 +1693,7 @@ async fn project_icon_edit( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -1717,7 +1716,7 @@ pub async fn project_icon_edit_internal( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, mut payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -1775,7 +1774,7 @@ pub async fn project_icon_edit_internal( project_item.inner.icon_url, project_item.inner.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; @@ -1794,7 +1793,7 @@ pub async fn project_icon_edit_internal( &ext.ext, Some(96), Some(1.0), - &***file_host, + &**file_host, ) .await?; @@ -1833,7 +1832,7 @@ async fn delete_project_icon( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { delete_project_icon_internal( @@ -1852,7 +1851,7 @@ pub async fn delete_project_icon_internal( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, session_queue: web::Data, ) -> Result { let user = get_user_from_headers( @@ -1908,7 +1907,7 @@ pub async fn delete_project_icon_internal( project_item.inner.icon_url, project_item.inner.raw_icon_url, FileHostPublicity::Public, - &***file_host, + &**file_host, ) .await?; @@ -1957,7 +1956,7 @@ pub async fn add_gallery_item( info: web::Path<(String,)>, pool: web::Data, redis: web::Data, - file_host: web::Data>, + file_host: web::Data, payload: web::Payload, session_queue: web::Data, ) -> Result { @@ -1982,7 +1981,7 @@ pub async fn add_gallery_item_internal( info: web::Path<(String,)>, pool: web::Data, redis: web::Data
+ {{ String(attributionError) }} +