diff --git a/apps/frontend/.env. b/apps/frontend/.env.
new file mode 100644
index 0000000000..29aad1d10a
--- /dev/null
+++ b/apps/frontend/.env.
@@ -0,0 +1,4 @@
+BASE_URL=http://127.0.0.1:8000/v2/
+BROWSER_BASE_URL=http://127.0.0.1:8000/v2/
+PYRO_BASE_URL=https://staging-archon.modrinth.com
+PROD_OVERRIDE=true
diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue
index 5f316b31d3..b24e659ee1 100644
--- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue
+++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue
@@ -24,7 +24,7 @@
-
+
+
+
+
+
+
+ {{ attributionError }}
+
+
+
diff --git a/apps/frontend/src/pages/[type]/[id]/settings/versions.vue b/apps/frontend/src/pages/[type]/[id]/settings/versions.vue
index 4d1e11ebca..40e9d1068b 100644
--- a/apps/frontend/src/pages/[type]/[id]/settings/versions.vue
+++ b/apps/frontend/src/pages/[type]/[id]/settings/versions.vue
@@ -16,7 +16,7 @@
/>
>'kind' = 'no_permission')\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "version_id: DBVersionId",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "file_id: DBFileId",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 2,
+ "name": "flame_project",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8Array"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ true
+ ]
+ },
+ "hash": "074ec090856894ae86834e3c5202e653c2a418c27d6a8fdde10f0b727ab9fd89"
+}
diff --git a/apps/labrinth/.sqlx/query-0b06b60b7169a3f9ee66c7ec26f94982619e95a78ecb2d3ffa55774b098e0531.json b/apps/labrinth/.sqlx/query-0b06b60b7169a3f9ee66c7ec26f94982619e95a78ecb2d3ffa55774b098e0531.json
new file mode 100644
index 0000000000..ab2e4da34a
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-0b06b60b7169a3f9ee66c7ec26f94982619e95a78ecb2d3ffa55774b098e0531.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n select\n fa.file_id as \"file_id: DBFileId\",\n f.url,\n v.mod_id as \"project_id: DBProjectId\"\n from file_scans fa\n inner join files f on f.id = fa.file_id\n inner join versions v on v.id = f.version_id\n where fa.attributions_scanned_at is null\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "file_id: DBFileId",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "url",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 2,
+ "name": "project_id: DBProjectId",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "0b06b60b7169a3f9ee66c7ec26f94982619e95a78ecb2d3ffa55774b098e0531"
+}
diff --git a/apps/labrinth/.sqlx/query-1d96df0ac64801641f9af796d482d2964c1acf2d03e15b8f1397340d0c994af6.json b/apps/labrinth/.sqlx/query-1d96df0ac64801641f9af796d482d2964c1acf2d03e15b8f1397340d0c994af6.json
new file mode 100644
index 0000000000..25146596a0
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-1d96df0ac64801641f9af796d482d2964c1acf2d03e15b8f1397340d0c994af6.json
@@ -0,0 +1,46 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tselect\n\t\t\tg.id as \"id: DBAttributionGroupId\",\n\t\t\tg.flame_project,\n\t\t\tg.attribution,\n\t\t\tg.attributed_at,\n\t\t\tg.attributed_by as \"attributed_by: i64\"\n\t\tfrom project_attribution_groups g\n\t\twhere g.project_id = $1\n\t\t",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id: DBAttributionGroupId",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "flame_project",
+ "type_info": "Jsonb"
+ },
+ {
+ "ordinal": 2,
+ "name": "attribution",
+ "type_info": "Jsonb"
+ },
+ {
+ "ordinal": 3,
+ "name": "attributed_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 4,
+ "name": "attributed_by: i64",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ true,
+ true,
+ true,
+ true
+ ]
+ },
+ "hash": "1d96df0ac64801641f9af796d482d2964c1acf2d03e15b8f1397340d0c994af6"
+}
diff --git a/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json b/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json
new file mode 100644
index 0000000000..bd768f4571
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "sha1",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "status",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "ByteaArray"
+ ]
+ },
+ "nullable": [
+ null,
+ false
+ ]
+ },
+ "hash": "2980f27376a9d26f2dc815f63522525b456acc00eb636111afd93326af9a18be"
+}
diff --git a/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json b/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json
new file mode 100644
index 0000000000..2c5ee6ec20
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into project_attribution_files (group_id, name, sha1)\n select $1, unnest($2::text[]), unnest($3::bytea[])\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "TextArray",
+ "ByteaArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "2e6bb84f45c0f8ba7bb9b09bf3def4322f727c7af8bd4be49dc4d7c487b925ed"
+}
diff --git a/apps/labrinth/.sqlx/query-424b975139004d2854d9365ef950984dd1a65e333bb80924b5883df5e7d3cad2.json b/apps/labrinth/.sqlx/query-424b975139004d2854d9365ef950984dd1a65e333bb80924b5883df5e7d3cad2.json
new file mode 100644
index 0000000000..c427ada77e
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-424b975139004d2854d9365ef950984dd1a65e333bb80924b5883df5e7d3cad2.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n select id as \"id: DBAttributionGroupId\", flame_project\n from project_attribution_groups\n where project_id = $1 and flame_project is not null\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id: DBAttributionGroupId",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "flame_project",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ true
+ ]
+ },
+ "hash": "424b975139004d2854d9365ef950984dd1a65e333bb80924b5883df5e7d3cad2"
+}
diff --git a/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json b/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json
new file mode 100644
index 0000000000..76174d70dd
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tupdate project_attribution_files\n\t\tset group_id = $1\n\t\twhere sha1 = $2\n\t\tand group_id in (\n\t\t\tselect id from project_attribution_groups where project_id = $3\n\t\t)\n\t\t",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Bytea",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "4fb69c674dca723b6b2cb7bea07d51feb43b2714c1cd6a2987a6fe1f10e2579a"
+}
diff --git a/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json b/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json
new file mode 100644
index 0000000000..e71a423986
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tupdate project_attribution_files\n\t\tset group_id = $1\n\t\twhere sha1 = $2 and group_id = $3\n\t\t",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Bytea",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "5f7b9d628d3ff0addc12cfaab68d7cbcca08a89aedeb15ce5926ed87ed8a12c6"
+}
diff --git a/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json b/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json
new file mode 100644
index 0000000000..297814d5ad
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tinsert into project_attribution_groups (id, project_id)\n\t\tvalues ($1, $2)\n\t\t",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "6320df0491556027ee64fd031797b252c2145ecf179e9ea9a8cedbdc8b7ed622"
+}
diff --git a/apps/labrinth/.sqlx/query-64c6abc464f2df37372875817ceb67b0e33786fe3dd27c052311356d43b1d601.json b/apps/labrinth/.sqlx/query-64c6abc464f2df37372875817ceb67b0e33786fe3dd27c052311356d43b1d601.json
new file mode 100644
index 0000000000..ee0f3478d5
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-64c6abc464f2df37372875817ceb67b0e33786fe3dd27c052311356d43b1d601.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO file_scans (file_id)\n VALUES ($1)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "64c6abc464f2df37372875817ceb67b0e33786fe3dd27c052311356d43b1d601"
+}
diff --git a/apps/labrinth/.sqlx/query-80751dbca09a9dcb469a30fe5b8225e520baf50750a78875f582349610439a55.json b/apps/labrinth/.sqlx/query-80751dbca09a9dcb469a30fe5b8225e520baf50750a78875f582349610439a55.json
new file mode 100644
index 0000000000..b8fc60acc8
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-80751dbca09a9dcb469a30fe5b8225e520baf50750a78875f582349610439a55.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into file_scans (file_id, attributions_scanned_at)\n values ($1, now())\n on conflict (file_id) do update set attributions_scanned_at = now()\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "80751dbca09a9dcb469a30fe5b8225e520baf50750a78875f582349610439a55"
+}
diff --git a/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json b/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json
new file mode 100644
index 0000000000..3da4278f04
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT EXISTS(SELECT 1 FROM project_attribution_groups WHERE id=$1)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "exists",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "8ffcb0c2bd82eaf64b143745adab359db6fcc448d6306292a7b345277a075883"
+}
diff --git a/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json b/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json
new file mode 100644
index 0000000000..7b1815f835
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tupdate project_attribution_groups\n\t\tset attribution = $1, attributed_at = now(), attributed_by = $3\n\t\twhere id = $2\n\t\t",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Jsonb",
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "94920d66b27503c84744d41cb2b44dd213bb461596046488eaab0b62b0fe83c5"
+}
diff --git a/apps/labrinth/.sqlx/query-968904f577c2c696c6222e19cc145bce0e845f2ab9f8629b0789a4861bddb4dc.json b/apps/labrinth/.sqlx/query-968904f577c2c696c6222e19cc145bce0e845f2ab9f8629b0789a4861bddb4dc.json
new file mode 100644
index 0000000000..1a588c429d
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-968904f577c2c696c6222e19cc145bce0e845f2ab9f8629b0789a4861bddb4dc.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n update file_scans\n set attributions_scanned_at = now\n from unnest($1::bigint[], $2::timestamptz[]) as u(id, now)\n where file_scans.file_id = u.id\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8Array",
+ "TimestamptzArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "968904f577c2c696c6222e19cc145bce0e845f2ab9f8629b0789a4861bddb4dc"
+}
diff --git a/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json b/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json
new file mode 100644
index 0000000000..e4cde72b9a
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into project_attribution_groups (id, project_id)\n values ($1, $2)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "b471add519874a6364b7a9cdbfa3a8e6f5a8a1ac9a7bacbce378f7995eb48664"
+}
diff --git a/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json b/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json
new file mode 100644
index 0000000000..19a77c556c
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into project_attribution_files (group_id, name, sha1)\n values ($1, $2, $3)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Text",
+ "Bytea"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c39e1f0d820101ac1b19a3849254b03259f38f3de9ddca32e6189a7e015dd853"
+}
diff --git a/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json b/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json
new file mode 100644
index 0000000000..9d4ffdf994
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tselect paf.group_id, paf.name from project_attribution_files paf\n\t\tinner join project_attribution_groups pag on pag.id = paf.group_id\n\t\twhere paf.sha1 = $1 and pag.project_id = $2\n\t\t",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "group_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Bytea",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "ce9b408d416b4782ad501e14cb1ff006964818e81855bb299b269ffaa4c3f7fa"
+}
diff --git a/apps/labrinth/.sqlx/query-dac21bfc23fe361a6133e69892224351aefa03f14efec1be31a21441419e19b9.json b/apps/labrinth/.sqlx/query-dac21bfc23fe361a6133e69892224351aefa03f14efec1be31a21441419e19b9.json
new file mode 100644
index 0000000000..8565cb6fbd
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-dac21bfc23fe361a6133e69892224351aefa03f14efec1be31a21441419e19b9.json
@@ -0,0 +1,40 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\t\tselect id, name, version_number, date_published\n\t\t\tfrom versions\n\t\t\twhere id = ANY($1)\n\t\t\torder by date_published desc\n\t\t\t",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 2,
+ "name": "version_number",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 3,
+ "name": "date_published",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8Array"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "dac21bfc23fe361a6133e69892224351aefa03f14efec1be31a21441419e19b9"
+}
diff --git a/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json b/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json
new file mode 100644
index 0000000000..91a7a9c5c8
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n select paf.sha1 from project_attribution_files paf\n inner join project_attribution_groups pag on pag.id = paf.group_id\n where pag.project_id = $1 and paf.sha1 = ANY($2)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "sha1",
+ "type_info": "Bytea"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "ByteaArray"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "e4b58fe6e5d19ded48ab9e457b4e96fb51c8f74a7532bb9ab2e7a945b95bbe78"
+}
diff --git a/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json b/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json
new file mode 100644
index 0000000000..0b7cf69e9c
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into override_file_sources (sha1, file_id)\n select unnest($1::bytea[]), $2\n on conflict do nothing\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "ByteaArray",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "e7a0481729efa9cba6140effcdddff1a076eb7269d5b22860db3e3d1a651f6eb"
+}
diff --git a/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json b/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json
new file mode 100644
index 0000000000..1b14566eb0
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n\t\tdelete from project_attribution_groups g\n\t\twhere not exists (\n\t\t\tselect 1 from project_attribution_files f where f.group_id = g.id\n\t\t)\n\t\t",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "ed027c5571c34ff4fd1ac50e96f257b0055c9497d2699927b34829490e7a2529"
+}
diff --git a/apps/labrinth/.sqlx/query-f9131eb55490b96851ac3760e3199d2fd2dbb3d212cb2b8687dda8ba0fad2a80.json b/apps/labrinth/.sqlx/query-f9131eb55490b96851ac3760e3199d2fd2dbb3d212cb2b8687dda8ba0fad2a80.json
new file mode 100644
index 0000000000..0ccd35ecd7
--- /dev/null
+++ b/apps/labrinth/.sqlx/query-f9131eb55490b96851ac3760e3199d2fd2dbb3d212cb2b8687dda8ba0fad2a80.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n insert into project_attribution_groups (id, project_id, flame_project)\n values ($1, $2, $3)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "Jsonb"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "f9131eb55490b96851ac3760e3199d2fd2dbb3d212cb2b8687dda8ba0fad2a80"
+}
diff --git a/apps/labrinth/AGENTS.md~HEAD b/apps/labrinth/AGENTS.md~HEAD
new file mode 100644
index 0000000000..0b73458d53
--- /dev/null
+++ b/apps/labrinth/AGENTS.md~HEAD
@@ -0,0 +1,34 @@
+# Labrinth
+
+Labrinth is the backend API service for Modrinth, written in Rust.
+
+## Code style
+
+- When writing `sqlx` queries, NEVER use `query` directly. Always prefer using the `query!`, `query_as!`, `query_scalar!` macros.
+
+## Pre-PR Checks
+
+When the user refers to "perform[ing] pre-PR checks", do the following:
+
+- Run `cargo clippy -p labrinth --all-targets` — there must be ZERO warnings, otherwise CI will fail
+- DO NOT run tests unless explicitly requested (they take a long time)
+- Prepare the sqlx cache: cd into `apps/labrinth` and run `cargo sqlx prepare -- --tests`
+ - NEVER run `cargo sqlx prepare --workspace`
+
+## Testing
+
+- Run `cargo test -p labrinth --all-targets` to test your changes — all tests must pass
+
+## Local Services
+
+- Read the root `docker-compose.yml` to see what running services are available while developing
+- Use `docker exec` to access these services
+
+### Clickhouse
+
+- Access: `docker exec labrinth-clickhouse clickhouse-client`
+- Database: `staging_ariadne`
+
+### Postgres
+
+- Access: `docker exec labrinth-postgres psql -U labrinth -d labrinth -c ""`
diff --git a/apps/labrinth/migrations/20260423114534_project_attribution.sql b/apps/labrinth/migrations/20260423114534_project_attribution.sql
new file mode 100644
index 0000000000..805d85d870
--- /dev/null
+++ b/apps/labrinth/migrations/20260423114534_project_attribution.sql
@@ -0,0 +1,33 @@
+create table file_scans (
+ file_id bigint primary key references files(id),
+ -- if a file..
+ -- - does not have a row
+ -- -> was created before attributions system
+ -- - has a row, but `attributions_scanned_at = null`
+ -- -> still needs to be scanned
+ -- - has a row, and `attributions_scanned_at` is not null
+ -- -> attributions have been scanned
+ attributions_scanned_at timestamptz
+);
+
+create table project_attribution_groups (
+ id bigint primary key,
+ project_id bigint not null references mods(id),
+ flame_project jsonb,
+ attribution jsonb,
+ attributed_at timestamptz,
+ attributed_by bigint references users(id)
+);
+create index on project_attribution_groups (project_id);
+
+create table project_attribution_files (
+ group_id bigint not null references project_attribution_groups(id),
+ name text not null,
+ sha1 bytea not null
+);
+
+create table override_file_sources (
+ sha1 bytea not null,
+ file_id bigint not null references files(id),
+ primary key (sha1, file_id)
+);
diff --git a/apps/labrinth/src/auth/checks.rs b/apps/labrinth/src/auth/checks.rs
index 329e071042..39509cd536 100644
--- a/apps/labrinth/src/auth/checks.rs
+++ b/apps/labrinth/src/auth/checks.rs
@@ -5,7 +5,10 @@ use crate::database::models::version_item::VersionQueryResult;
use crate::database::models::{DBCollection, DBOrganization, DBTeamMember};
use crate::database::redis::RedisPool;
use crate::database::{DBProject, DBVersion, models};
+use crate::models::ids::FileId;
+use crate::models::projects::{MissingAttributionFile, OverrideSource};
use crate::models::users::User;
+use crate::queue::file_scan::get_files_missing_attribution;
use crate::routes::ApiError;
use futures::TryStreamExt;
use itertools::Itertools;
@@ -204,7 +207,40 @@ pub async fn filter_visible_versions(
)
.await?;
versions.retain(|x| filtered_version_ids.contains(&x.inner.id));
- Ok(versions.into_iter().map(|x| x.into()).collect())
+
+ let version_ids: Vec<_> = versions.iter().map(|v| v.inner.id).collect();
+ let missing = get_files_missing_attribution(pool, &version_ids)
+ .await
+ .unwrap_or_default();
+
+ Ok(versions
+ .into_iter()
+ .map(|v| {
+ let files_missing = missing
+ .get(&v.inner.id)
+ .map(|entries| {
+ entries
+ .iter()
+ .map(|(id, fp)| MissingAttributionFile {
+ id: FileId(id.0 as u64),
+ override_source: fp
+ .as_ref()
+ .map(|p| OverrideSource::Flame {
+ id: p.id,
+ title: p.title.clone(),
+ url: p.url.clone(),
+ icon_url: p.icon_url.clone(),
+ })
+ .or(Some(OverrideSource::Unknown)),
+ })
+ .collect::>()
+ })
+ .unwrap_or_default();
+ let mut version = crate::models::projects::Version::from(v);
+ version.files_missing_attribution = files_missing;
+ version
+ })
+ .collect())
}
impl ValidateAuthorized for models::DBOAuthClient {
@@ -258,13 +294,20 @@ pub async fn filter_visible_version_ids(
filter_enlisted_version_ids(versions.clone(), user_option, pool, redis)
.await?;
+ let version_ids: Vec<_> = versions.iter().map(|v| v.id).collect();
+ let withheld_versions = get_files_missing_attribution(pool, &version_ids)
+ .await
+ .unwrap_or_default();
+
// Return versions that are not hidden, we are a mod of, or we are enlisted on the team of
for version in versions {
+ let is_withheld = withheld_versions.contains_key(&version.id);
// We can see the version if:
- // - it's not hidden and we can see the project
+ // - it's not hidden and we can see the project and it's not withheld for attribution
// - we are a mod
// - we are enlisted on the team of the mod
if (!version.status.is_hidden()
+ && !is_withheld
&& visible_project_ids.contains(&version.project_id))
|| user_option.as_ref().is_some_and(|x| x.role.is_mod())
|| enlisted_version_ids.contains(&version.id)
diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs
index b44bdfe6cd..1c8c0ecefc 100644
--- a/apps/labrinth/src/background_task.rs
+++ b/apps/labrinth/src/background_task.rs
@@ -1,9 +1,11 @@
use crate::database;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
+use crate::file_hosting::FileHost;
use crate::queue::analytics::cache::cache_analytics;
use crate::queue::billing::{index_billing, index_subscriptions};
use crate::queue::email::EmailQueue;
+use crate::queue::file_scan::scan_all_files;
use crate::queue::payouts::{
PayoutsQueue, index_payouts_notifications,
insert_bank_balances_and_webhook, process_affiliate_payouts,
@@ -34,6 +36,10 @@ pub enum BackgroundTask {
/// Attempts to ping Minecraft Java servers as if we were a client, to
/// collect info on if they're online, game version, description, etc.
PingMinecraftJavaServers,
+ /// Finds files of versions which have not been scanned for attributions
+ /// yet, extracts them to find file overrides, and finds any overrides which
+ /// require attribution from the creator.
+ ScanFiles,
}
impl BackgroundTask {
@@ -44,6 +50,7 @@ impl BackgroundTask {
ro_pool: PgPool,
redis_pool: RedisPool,
search_backend: web::Data,
+ file_host: web::Data,
clickhouse: clickhouse::Client,
stripe_client: stripe::Client,
anrok_client: anrok::Client,
@@ -90,6 +97,7 @@ impl BackgroundTask {
PingMinecraftJavaServers => {
ping_minecraft_java_servers(pool, redis_pool, clickhouse).await
}
+ ScanFiles => scan_all_files(&pool, &redis_pool, &**file_host).await,
}
}
}
diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs
index 646d5e4cc2..cb7fe97ba4 100644
--- a/apps/labrinth/src/database/models/ids.rs
+++ b/apps/labrinth/src/database/models/ids.rs
@@ -1,12 +1,12 @@
use super::DatabaseError;
use crate::database::PgTransaction;
use crate::models::ids::{
- AffiliateCodeId, ChargeId, CollectionId, FileId, ImageId, NotificationId,
- OAuthAccessTokenId, OAuthClientAuthorizationId, OAuthClientId,
- OAuthRedirectUriId, OrganizationId, PatId, PayoutId, ProductId,
- ProductPriceId, ProjectId, ReportId, SessionId, SharedInstanceId,
- SharedInstanceVersionId, TeamId, TeamMemberId, ThreadId, ThreadMessageId,
- UserSubscriptionId, VersionId,
+ AffiliateCodeId, AttributionGroupId, ChargeId, CollectionId, FileId,
+ ImageId, NotificationId, OAuthAccessTokenId, OAuthClientAuthorizationId,
+ OAuthClientId, OAuthRedirectUriId, OrganizationId, PatId, PayoutId,
+ ProductId, ProductPriceId, ProjectId, ReportId, SessionId,
+ SharedInstanceId, SharedInstanceVersionId, TeamId, TeamMemberId, ThreadId,
+ ThreadMessageId, UserSubscriptionId, VersionId,
};
use ariadne::ids::base62_impl::to_base62;
use ariadne::ids::{UserId, random_base62_rng, random_base62_rng_range};
@@ -168,6 +168,10 @@ db_id_interface!(
CollectionId,
generator: generate_collection_id @ "collections",
);
+db_id_interface!(
+ AttributionGroupId,
+ generator: generate_attribution_group_id @ "project_attribution_groups",
+);
db_id_interface!(
FileId,
generator: generate_file_id @ "files",
diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs
index 904f799e62..9fa9091a9b 100644
--- a/apps/labrinth/src/database/models/project_item.rs
+++ b/apps/labrinth/src/database/models/project_item.rs
@@ -6,6 +6,7 @@ use super::{DBUser, ids::*};
use crate::database::models::DatabaseError;
use crate::database::redis::RedisPool;
use crate::database::{PgTransaction, models};
+use crate::file_hosting::FileHost;
use crate::models::exp;
use crate::models::ids::ProjectId;
use crate::models::projects::{
@@ -187,6 +188,8 @@ impl ProjectBuilder {
pub async fn insert(
self,
transaction: &mut PgTransaction<'_>,
+ redis: &RedisPool,
+ file_host: &dyn FileHost,
http: &reqwest::Client,
) -> Result {
let project_struct = DBProject {
@@ -235,7 +238,7 @@ impl ProjectBuilder {
for mut version in self.initial_versions {
version.project_id = self.project_id;
- version.insert(&mut *transaction, http).await?;
+ version.insert(transaction, redis, file_host, http).await?;
}
LinkUrl::insert_many_projects(
diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs
index 6ffaf90c7f..264971fc40 100644
--- a/apps/labrinth/src/database/models/version_item.rs
+++ b/apps/labrinth/src/database/models/version_item.rs
@@ -6,8 +6,11 @@ use crate::database::models::loader_fields::{
QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
};
use crate::database::redis::RedisPool;
+use crate::file_hosting::FileHost;
use crate::models::exp;
+
use crate::models::projects::{FileType, VersionStatus};
+use crate::queue::file_scan::scan_file;
use crate::routes::internal::delphi::DelphiRunParameters;
use chrono::{DateTime, Utc};
use dashmap::{DashMap, DashSet};
@@ -17,6 +20,7 @@ use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::collections::HashMap;
use std::iter;
+use tracing::error;
pub const VERSIONS_NAMESPACE: &str = "versions";
const VERSION_FILES_NAMESPACE: &str = "versions_files";
@@ -134,7 +138,10 @@ impl VersionFileBuilder {
pub async fn insert(
self,
version_id: DBVersionId,
+ project_id: DBProjectId,
transaction: &mut PgTransaction<'_>,
+ redis: &RedisPool,
+ file_host: &dyn FileHost,
http: &reqwest::Client,
) -> Result {
let file_id = generate_file_id(&mut *transaction).await?;
@@ -169,6 +176,16 @@ impl VersionFileBuilder {
.await?;
}
+ sqlx::query!(
+ "
+ INSERT INTO file_scans (file_id)
+ VALUES ($1)
+ ",
+ file_id as DBFileId,
+ )
+ .execute(&mut *transaction)
+ .await?;
+
if let Err(err) = crate::routes::internal::delphi::run(
&mut *transaction,
DelphiRunParameters {
@@ -178,7 +195,20 @@ impl VersionFileBuilder {
)
.await
{
- tracing::error!("Error submitting new file to Delphi: {err}");
+ error!("Error submitting new file to Delphi: {err:?}");
+ }
+
+ if let Err(err) = scan_file(
+ &mut *transaction,
+ redis,
+ file_host,
+ project_id,
+ file_id,
+ &self.url,
+ )
+ .await
+ {
+ error!("Error scanning new file {file_id:?}: {err:?}");
}
Ok(file_id)
@@ -195,6 +225,8 @@ impl VersionBuilder {
pub async fn insert(
self,
transaction: &mut PgTransaction<'_>,
+ redis: &RedisPool,
+ file_host: &dyn FileHost,
http: &reqwest::Client,
) -> Result {
let version = DBVersion {
@@ -236,7 +268,15 @@ impl VersionBuilder {
} = self;
for file in files {
- file.insert(version_id, transaction, http).await?;
+ file.insert(
+ version_id,
+ self.project_id,
+ transaction,
+ redis,
+ file_host,
+ http,
+ )
+ .await?;
}
DependencyBuilder::insert_many(
@@ -862,14 +902,14 @@ impl DBVersion {
})
}
- pub async fn get_files_from_hash<'a, 'b, E>(
+ pub async fn get_files_from_hash<'a, E>(
algorithm: String,
hashes: &[String],
executor: E,
redis: &RedisPool,
) -> Result, DatabaseError>
where
- E: crate::database::Executor<'a, Database = sqlx::Postgres> + Copy,
+ E: crate::database::Executor<'a, Database = sqlx::Postgres>,
{
let val = redis.get_cached_keys(
VERSION_FILES_NAMESPACE,
diff --git a/apps/labrinth/src/file_hosting/mock.rs b/apps/labrinth/src/file_hosting/mock.rs
index 3e414bd393..3e04433b76 100644
--- a/apps/labrinth/src/file_hosting/mock.rs
+++ b/apps/labrinth/src/file_hosting/mock.rs
@@ -72,6 +72,18 @@ impl FileHost for MockHost {
file_name: file_name.to_string(),
})
}
+
+ async fn read_file(
+ &self,
+ file_name: &str,
+ file_publicity: FileHostPublicity,
+ ) -> Result {
+ let file_name = urlencoding::decode(file_name)
+ .map_err(|_| FileHostingError::InvalidFilename)?;
+ let path = get_file_path(&file_name, file_publicity);
+ let data = std::fs::read(&path)?;
+ Ok(Bytes::from(data))
+ }
}
fn get_file_path(
diff --git a/apps/labrinth/src/file_hosting/mod.rs b/apps/labrinth/src/file_hosting/mod.rs
index 667f4cb21e..35114a76c7 100644
--- a/apps/labrinth/src/file_hosting/mod.rs
+++ b/apps/labrinth/src/file_hosting/mod.rs
@@ -45,7 +45,7 @@ pub enum FileHostPublicity {
}
#[async_trait]
-pub trait FileHost {
+pub trait FileHost: Send + Sync {
async fn upload_file(
&self,
content_type: &str,
@@ -65,6 +65,12 @@ pub trait FileHost {
file_name: &str,
file_publicity: FileHostPublicity,
) -> Result;
+
+ async fn read_file(
+ &self,
+ file_name: &str,
+ file_publicity: FileHostPublicity,
+ ) -> Result;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
diff --git a/apps/labrinth/src/file_hosting/s3_host.rs b/apps/labrinth/src/file_hosting/s3_host.rs
index 56bd0ef45c..558e4d5086 100644
--- a/apps/labrinth/src/file_hosting/s3_host.rs
+++ b/apps/labrinth/src/file_hosting/s3_host.rs
@@ -169,4 +169,28 @@ impl FileHost for S3Host {
file_name: file_name.to_string(),
})
}
+
+ async fn read_file(
+ &self,
+ file_name: &str,
+ file_publicity: FileHostPublicity,
+ ) -> Result {
+ let bucket = self.get_bucket(file_publicity);
+
+ let response = bucket
+ .client
+ .get_object()
+ .bucket(bucket.name.as_str())
+ .key(file_name)
+ .send()
+ .await
+ .map_err(|e| s3_error("reading file", e))?;
+
+ Ok(response
+ .body
+ .collect()
+ .await
+ .map_err(|e| s3_error("reading file body", e))?
+ .into_bytes())
+ }
}
diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs
index 15363ed09b..d1c4b30fa4 100644
--- a/apps/labrinth/src/lib.rs
+++ b/apps/labrinth/src/lib.rs
@@ -57,7 +57,7 @@ pub struct LabrinthConfig {
pub ro_pool: ReadOnlyPgPool,
pub redis_pool: RedisPool,
pub clickhouse: Client,
- pub file_host: Arc,
+ pub file_host: web::Data,
pub scheduler: Arc,
pub ip_salt: Pepper,
pub search_backend: web::Data,
@@ -82,7 +82,7 @@ pub fn app_setup(
redis_pool: RedisPool,
search_backend: actix_web::web::Data,
clickhouse: &mut Client,
- file_host: Arc,
+ file_host: web::Data,
stripe_client: stripe::Client,
anrok_client: anrok::Client,
email_queue: EmailQueue,
@@ -339,7 +339,7 @@ pub fn app_config(
.app_data(web::Data::new(labrinth_config.redis_pool.clone()))
.app_data(web::Data::new(labrinth_config.pool.clone()))
.app_data(web::Data::new(labrinth_config.ro_pool.clone()))
- .app_data(web::Data::new(labrinth_config.file_host.clone()))
+ .app_data(labrinth_config.file_host.clone())
.app_data(labrinth_config.search_backend.clone())
.app_data(web::Data::new(labrinth_config.gotenberg_client.clone()))
.app_data(labrinth_config.http_client.clone())
diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs
index f24a2fb79d..feba395e1a 100644
--- a/apps/labrinth/src/main.rs
+++ b/apps/labrinth/src/main.rs
@@ -2,14 +2,14 @@
use actix_web::dev::Service;
use actix_web::middleware::from_fn;
-use actix_web::{App, HttpServer};
+use actix_web::{App, HttpServer, web};
use actix_web_prom::PrometheusMetricsBuilder;
use clap::Parser;
use labrinth::background_task::BackgroundTask;
use labrinth::database::redis::RedisPool;
use labrinth::env::ENV;
-use labrinth::file_hosting::{FileHostKind, S3BucketConfig, S3Host};
+use labrinth::file_hosting::{FileHost, FileHostKind, S3BucketConfig, S3Host};
use labrinth::queue::email::EmailQueue;
use labrinth::search;
use labrinth::util::anrok;
@@ -111,44 +111,38 @@ async fn app() -> std::io::Result<()> {
let redis_pool = RedisPool::new("");
let storage_backend = ENV.STORAGE_BACKEND;
- let file_host: Arc =
- match storage_backend {
- FileHostKind::S3 => {
- let not_empty = |v: &str| -> String {
- assert!(!v.is_empty(), "S3 env var is empty");
- v.to_string()
- };
-
- Arc::new(
- S3Host::new(
- S3BucketConfig {
- name: not_empty(&ENV.S3_PUBLIC_BUCKET_NAME),
- uses_path_style: ENV
- .S3_PUBLIC_USES_PATH_STYLE_BUCKET,
- region: not_empty(&ENV.S3_PUBLIC_REGION),
- url: not_empty(&ENV.S3_PUBLIC_URL),
- access_token: not_empty(
- &ENV.S3_PUBLIC_ACCESS_TOKEN,
- ),
- secret: not_empty(&ENV.S3_PUBLIC_SECRET),
- },
- S3BucketConfig {
- name: not_empty(&ENV.S3_PRIVATE_BUCKET_NAME),
- uses_path_style: ENV
- .S3_PRIVATE_USES_PATH_STYLE_BUCKET,
- region: not_empty(&ENV.S3_PRIVATE_REGION),
- url: not_empty(&ENV.S3_PRIVATE_URL),
- access_token: not_empty(
- &ENV.S3_PRIVATE_ACCESS_TOKEN,
- ),
- secret: not_empty(&ENV.S3_PRIVATE_SECRET),
- },
- )
- .unwrap(),
+ let file_host: Arc = match storage_backend {
+ FileHostKind::S3 => {
+ let not_empty = |v: &str| -> String {
+ assert!(!v.is_empty(), "S3 env var is empty");
+ v.to_string()
+ };
+
+ Arc::new(
+ S3Host::new(
+ S3BucketConfig {
+ name: not_empty(&ENV.S3_PUBLIC_BUCKET_NAME),
+ uses_path_style: ENV.S3_PUBLIC_USES_PATH_STYLE_BUCKET,
+ region: not_empty(&ENV.S3_PUBLIC_REGION),
+ url: not_empty(&ENV.S3_PUBLIC_URL),
+ access_token: not_empty(&ENV.S3_PUBLIC_ACCESS_TOKEN),
+ secret: not_empty(&ENV.S3_PUBLIC_SECRET),
+ },
+ S3BucketConfig {
+ name: not_empty(&ENV.S3_PRIVATE_BUCKET_NAME),
+ uses_path_style: ENV.S3_PRIVATE_USES_PATH_STYLE_BUCKET,
+ region: not_empty(&ENV.S3_PRIVATE_REGION),
+ url: not_empty(&ENV.S3_PRIVATE_URL),
+ access_token: not_empty(&ENV.S3_PRIVATE_ACCESS_TOKEN),
+ secret: not_empty(&ENV.S3_PRIVATE_SECRET),
+ },
)
- }
- FileHostKind::Local => Arc::new(file_hosting::MockHost::new()),
- };
+ .unwrap(),
+ )
+ }
+ FileHostKind::Local => Arc::new(file_hosting::MockHost::new()),
+ };
+ let file_host = web::Data::::from(file_host);
info!("Initializing clickhouse connection");
let mut clickhouse = clickhouse::init_client().await.unwrap();
@@ -174,6 +168,7 @@ async fn app() -> std::io::Result<()> {
ro_pool.into_inner(),
redis_pool,
search_backend,
+ file_host,
clickhouse,
stripe_client,
anrok_client.clone(),
diff --git a/apps/labrinth/src/models/v3/ids.rs b/apps/labrinth/src/models/v3/ids.rs
index 7cb162ec27..f283eb199f 100644
--- a/apps/labrinth/src/models/v3/ids.rs
+++ b/apps/labrinth/src/models/v3/ids.rs
@@ -1,5 +1,6 @@
use ariadne::ids::base62_id;
+base62_id!(AttributionGroupId);
base62_id!(ChargeId);
base62_id!(CollectionId);
base62_id!(FileId);
diff --git a/apps/labrinth/src/models/v3/projects.rs b/apps/labrinth/src/models/v3/projects.rs
index 92abe3fddb..e1b26852f4 100644
--- a/apps/labrinth/src/models/v3/projects.rs
+++ b/apps/labrinth/src/models/v3/projects.rs
@@ -645,6 +645,54 @@ impl SideTypesMigrationReviewStatus {
}
}
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
+pub struct MissingAttributionFile {
+ pub id: FileId,
+ pub override_source: Option,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum OverrideSource {
+ Flame {
+ id: u32,
+ title: String,
+ url: String,
+ icon_url: String,
+ },
+ Unknown,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
+#[serde(untagged)]
+pub enum AttributionLicense {
+ Spdx(String),
+ Custom { name: String },
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum AttributionResolutionKind {
+ License {
+ license: AttributionLicense,
+ link_to_work: url::Url,
+ },
+ MyProject {
+ license: AttributionLicense,
+ },
+ SpecialPermissions {
+ link_to_work: url::Url,
+ },
+ NoPermission,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
+pub struct AttributionResolution {
+ pub kind: AttributionResolutionKind,
+ pub notes: String,
+ pub image_urls: Vec,
+}
+
/// A specific version of a project
#[derive(Debug, Serialize, Deserialize, Clone, utoipa::ToSchema)]
pub struct Version {
@@ -681,6 +729,9 @@ pub struct Version {
/// A list of files available for download for this version.
pub files: Vec,
+ /// Files in this version that contain override files not yet attributed.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub files_missing_attribution: Vec,
/// A list of projects that this version depends on.
pub dependencies: Vec,
@@ -768,6 +819,7 @@ impl From for Version {
.map(|vf| (vf.field_name, vf.value.serialize_internal()))
.collect(),
components: data.components,
+ files_missing_attribution: Vec::new(),
}
}
}
diff --git a/apps/labrinth/src/queue/file_scan.rs b/apps/labrinth/src/queue/file_scan.rs
new file mode 100644
index 0000000000..48d28aa7a6
--- /dev/null
+++ b/apps/labrinth/src/queue/file_scan.rs
@@ -0,0 +1,785 @@
+use std::collections::HashMap;
+use std::io::{Cursor, Read};
+
+use chrono::Utc;
+use eyre::{Result, eyre};
+use hex::ToHex;
+use sha1::Digest;
+use tokio::task::spawn_blocking;
+use tracing::{Instrument, info, info_span, warn};
+use zip::ZipArchive;
+
+use crate::database::models::ids::{
+ DBAttributionGroupId, DBProjectId, DBVersionId,
+ generate_attribution_group_id,
+};
+use crate::database::models::moderation_external_item::ExternalLicense;
+use crate::database::models::{DBFileId, DBUserId, DBVersion};
+use crate::database::{PgPool, PgTransaction, redis::RedisPool};
+use crate::env::ENV;
+use crate::file_hosting::{FileHost, FileHostPublicity};
+use crate::models::ids::FileId;
+use crate::queue::moderation::{
+ ApprovalType, FingerprintResponse, FlameProject, FlameResponse,
+};
+use crate::util::error::Context;
+use crate::util::http::HTTP_CLIENT;
+
+pub async fn scan_all_files(
+ db: &PgPool,
+ redis: &RedisPool,
+ file_host: &dyn FileHost,
+) -> Result<()> {
+ let mut txn = db.begin().await.wrap_err("beginning transaction")?;
+
+ let files_to_scan = sqlx::query!(
+ r#"
+ select
+ fa.file_id as "file_id: DBFileId",
+ f.url,
+ v.mod_id as "project_id: DBProjectId"
+ from file_scans fa
+ inner join files f on f.id = fa.file_id
+ inner join versions v on v.id = f.version_id
+ where fa.attributions_scanned_at is null
+ "#
+ )
+ .fetch_all(&mut txn)
+ .await
+ .wrap_err("fetching files to scan")?;
+
+ info!("Found {} files to scan", files_to_scan.len());
+
+ let mut scanned_ids = Vec::new();
+
+ for row in files_to_scan {
+ let human_file_id = FileId::from(row.file_id);
+ let span = info_span!("scan", file_id = %human_file_id);
+ async {
+ info!("Scanning file");
+
+ let file_id = row.file_id;
+
+ let overrides = extract_override_files_from_storage(
+ file_host, file_id, &row.url,
+ )
+ .await
+ .wrap_err_with(|| {
+ eyre!("extracting overrides for file {file_id:?}")
+ })?;
+
+ if overrides.is_empty() {
+ info!("Found no overrides");
+ } else {
+ info!("Found {} overrides", overrides.len());
+
+ let resolved = resolve_overrides(&overrides, redis, &mut txn)
+ .await
+ .wrap_err_with(|| {
+ eyre!("resolving overrides for file {file_id:?}")
+ })?;
+ info!("Resolved: {resolved:#?}");
+
+ persist_attribution_results(
+ row.project_id,
+ file_id,
+ &overrides,
+ &resolved,
+ &mut txn,
+ )
+ .await
+ .wrap_err_with(|| {
+ eyre!("persisting attribution results for file {file_id:?}")
+ })?;
+ }
+
+ scanned_ids.push(file_id.0);
+ eyre::Ok(())
+ }
+ .instrument(span)
+ .await?;
+ }
+
+ if !scanned_ids.is_empty() {
+ let now = Utc::now();
+ sqlx::query!(
+ "
+ update file_scans
+ set attributions_scanned_at = now
+ from unnest($1::bigint[], $2::timestamptz[]) as u(id, now)
+ where file_scans.file_id = u.id
+ ",
+ &scanned_ids,
+ &vec![now; scanned_ids.len()],
+ )
+ .execute(&mut txn)
+ .await
+ .wrap_err("marking files as scanned")?;
+ }
+
+ info!("Marked {} files as scanned", scanned_ids.len());
+
+ txn.commit().await.wrap_err("committing transaction")?;
+
+ Ok(())
+}
+
+pub async fn scan_file(
+ txn: &mut PgTransaction<'_>,
+ redis: &RedisPool,
+ file_host: &dyn FileHost,
+ project_id: DBProjectId,
+ file_id: DBFileId,
+ file_url: &str,
+) -> Result<()> {
+ let overrides =
+ extract_override_files_from_storage(file_host, file_id, file_url)
+ .await
+ .wrap_err_with(|| {
+ eyre!("extracting overrides for file {file_id:?}")
+ })?;
+
+ if !overrides.is_empty() {
+ let resolved = resolve_overrides(&overrides, redis, txn)
+ .await
+ .wrap_err_with(|| {
+ eyre!("resolving overrides for file {file_id:?}")
+ })?;
+
+ persist_attribution_results(
+ project_id, file_id, &overrides, &resolved, txn,
+ )
+ .await
+ .wrap_err_with(|| {
+ eyre!("persisting attribution results for file {file_id:?}")
+ })?;
+ }
+
+ sqlx::query!(
+ "
+ insert into file_scans (file_id, attributions_scanned_at)
+ values ($1, now())
+ on conflict (file_id) do update set attributions_scanned_at = now()
+ ",
+ file_id.0,
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("marking file as scanned")?;
+
+ Ok(())
+}
+
+pub async fn scan_override_files(
+ file_host: &dyn FileHost,
+ file_id: DBFileId,
+ file_url: &str,
+) -> Result> {
+ extract_override_files_from_storage(file_host, file_id, file_url)
+ .await
+ .wrap_err_with(|| eyre!("extracting overrides for file {file_id:?}"))
+}
+
+async fn extract_override_files_from_storage(
+ file_host: &dyn FileHost,
+ file_id: DBFileId,
+ file_url: &str,
+) -> Result> {
+ let key = file_url
+ .strip_prefix(&ENV.CDN_URL)
+ .unwrap_or(file_url)
+ .trim_start_matches('/');
+
+ let file_data = file_host
+ .read_file(key, FileHostPublicity::Public)
+ .await
+ .wrap_err_with(|| {
+ eyre!("reading file {file_id:?} from storage at {key}")
+ })?;
+
+ spawn_blocking(move || extract_override_files(&file_data))
+ .await
+ .wrap_err("extracting override files")?
+ .wrap_err("extracting override files")
+}
+
+#[derive(Debug)]
+pub struct OverrideFile {
+ pub path: String,
+ pub sha1: String,
+ pub murmur2: u32,
+}
+
+#[derive(Debug)]
+pub enum OverrideResolution {
+ OnModrinth,
+ ExternalLicense(ApprovalType),
+ Flame {
+ project_id: u32,
+ title: String,
+ url: String,
+ icon_url: String,
+ },
+ Unknown,
+}
+
+const OVERRIDE_PREFIXES: &[&str] = &[
+ "overrides/mods",
+ "client-overrides/mods",
+ "server-overrides/mods",
+ "overrides/shaderpacks",
+ "client-overrides/shaderpacks",
+ "overrides/resourcepacks",
+ "client-overrides/resourcepacks",
+];
+
+fn extract_override_files(data: &[u8]) -> Result> {
+ let reader = Cursor::new(data);
+ let mut zip =
+ ZipArchive::new(reader).wrap_err("creating zip archive reader")?;
+
+ let mut files = Vec::new();
+
+ for i in 0..zip.len() {
+ let mut file = zip
+ .by_index(i)
+ .wrap_err_with(|| eyre!("reading file {i}"))?;
+ let name = file.name().to_string();
+
+ if !OVERRIDE_PREFIXES
+ .iter()
+ .any(|prefix| name.starts_with(prefix))
+ {
+ continue;
+ }
+
+ if name.matches('/').count() > 2
+ || name.ends_with(".txt")
+ || name.ends_with(".rpo")
+ {
+ continue;
+ }
+
+ let mut contents = Vec::new();
+ file.read_to_end(&mut contents)?;
+
+ let sha1 = sha1::Sha1::digest(&contents).encode_hex::();
+ let murmur = hash_flame_murmur32(contents);
+
+ files.push(OverrideFile {
+ sha1,
+ murmur2: murmur,
+ path: name,
+ });
+ }
+
+ Ok(files)
+}
+
+async fn persist_attribution_results(
+ project_id: DBProjectId,
+ file_id: DBFileId,
+ overrides: &[OverrideFile],
+ resolved: &HashMap,
+ txn: &mut PgTransaction<'_>,
+) -> Result<()> {
+ let all_sha1s: Vec> = overrides
+ .iter()
+ .map(|f| f.sha1.as_bytes().to_vec())
+ .collect();
+
+ let already_persisted: Vec> = sqlx::query_scalar!(
+ "
+ select paf.sha1 from project_attribution_files paf
+ inner join project_attribution_groups pag on pag.id = paf.group_id
+ where pag.project_id = $1 and paf.sha1 = ANY($2)
+ ",
+ project_id as DBProjectId,
+ &all_sha1s,
+ )
+ .fetch_all(&mut *txn)
+ .await
+ .wrap_err("checking existing attribution files")?;
+
+ let mut flame_groups: HashMap<
+ u32,
+ (Vec<&OverrideFile>, Option<&OverrideResolution>),
+ > = HashMap::new();
+ let mut unknown_files: Vec<&OverrideFile> = Vec::new();
+
+ for file in overrides {
+ if already_persisted
+ .iter()
+ .any(|s| s.as_slice() == file.sha1.as_bytes())
+ {
+ continue;
+ }
+
+ match resolved.get(&file.sha1) {
+ Some(OverrideResolution::OnModrinth) => continue,
+ Some(OverrideResolution::ExternalLicense(approval)) => {
+ if approval.approved() {
+ continue;
+ }
+ unknown_files.push(file);
+ }
+ Some(
+ res @ OverrideResolution::Flame {
+ project_id: fp_id, ..
+ },
+ ) => {
+ let entry = flame_groups.entry(*fp_id).or_default();
+ entry.0.push(file);
+ if entry.1.is_none() {
+ entry.1 = Some(res);
+ }
+ }
+ Some(OverrideResolution::Unknown) | None => {
+ unknown_files.push(file);
+ }
+ }
+ }
+
+ let existing_flame_groups = sqlx::query!(
+ r#"
+ select id as "id: DBAttributionGroupId", flame_project
+ from project_attribution_groups
+ where project_id = $1 and flame_project is not null
+ "#,
+ project_id as DBProjectId,
+ )
+ .fetch_all(&mut *txn)
+ .await
+ .wrap_err("fetching existing flame attribution groups")?;
+
+ for (flame_project_id, (files, resolution)) in &flame_groups {
+ let existing = existing_flame_groups.iter().find(|g| {
+ g.flame_project
+ .as_ref()
+ .and_then(|fp| {
+ serde_json::from_value::<
+ crate::routes::internal::attribution::FlameProject,
+ >(fp.clone())
+ .ok()
+ })
+ .is_some_and(|fp| fp.id == *flame_project_id)
+ });
+
+ let group_id = if let Some(group) = existing {
+ group.id
+ } else {
+ let fp = resolution
+ .and_then(|r| {
+ if let OverrideResolution::Flame {
+ project_id,
+ title,
+ url,
+ icon_url,
+ } = r
+ {
+ Some(
+ serde_json::to_value(
+ crate::routes::internal::attribution::FlameProject {
+ id: *project_id,
+ title: title.clone(),
+ url: url.clone(),
+ icon_url: icon_url.clone(),
+ },
+ )
+ .ok(),
+ )
+ } else {
+ None
+ }
+ })
+ .flatten();
+
+ let id = generate_attribution_group_id(&mut *txn).await?;
+ sqlx::query!(
+ "
+ insert into project_attribution_groups (id, project_id, flame_project)
+ values ($1, $2, $3)
+ ",
+ id as DBAttributionGroupId,
+ project_id as DBProjectId,
+ fp,
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("inserting attribution group")?;
+ id
+ };
+
+ let names: Vec = files.iter().map(|f| f.path.clone()).collect();
+ let sha1s: Vec> =
+ files.iter().map(|f| f.sha1.as_bytes().to_vec()).collect();
+
+ sqlx::query!(
+ "
+ insert into project_attribution_files (group_id, name, sha1)
+ select $1, unnest($2::text[]), unnest($3::bytea[])
+ ",
+ group_id as DBAttributionGroupId,
+ &names,
+ &sha1s,
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("inserting attribution files")?;
+ }
+
+ for file in &unknown_files {
+ let group_id = generate_attribution_group_id(&mut *txn).await?;
+ sqlx::query!(
+ "
+ insert into project_attribution_groups (id, project_id)
+ values ($1, $2)
+ ",
+ group_id as DBAttributionGroupId,
+ project_id as DBProjectId,
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("inserting unknown attribution group")?;
+
+ sqlx::query!(
+ "
+ insert into project_attribution_files (group_id, name, sha1)
+ values ($1, $2, $3)
+ ",
+ group_id as DBAttributionGroupId,
+ &file.path,
+ &file.sha1.as_bytes().to_vec() as &[u8],
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("inserting unknown attribution file")?;
+ }
+
+ if !all_sha1s.is_empty() {
+ sqlx::query!(
+ "
+ insert into override_file_sources (sha1, file_id)
+ select unnest($1::bytea[]), $2
+ on conflict do nothing
+ ",
+ &all_sha1s,
+ file_id as DBFileId,
+ )
+ .execute(&mut *txn)
+ .await
+ .wrap_err("inserting override file sources")?;
+ }
+
+ Ok(())
+}
+
+async fn resolve_overrides(
+ overrides: &[OverrideFile],
+ redis: &RedisPool,
+ txn: &mut PgTransaction<'_>,
+) -> Result> {
+ let mut results: HashMap = HashMap::new();
+ let mut remaining: Vec = (0..overrides.len()).collect();
+
+ if overrides.is_empty() {
+ return Ok(results);
+ }
+
+ let hashes: Vec =
+ overrides.iter().map(|x| x.sha1.clone()).collect();
+ let files = DBVersion::get_files_from_hash(
+ "sha1".to_string(),
+ &hashes,
+ &mut *txn,
+ redis,
+ )
+ .await
+ .wrap_err("fetching files on platform by hash")?;
+
+ let version_ids: Vec<_> = files.iter().map(|x| x.version_id).collect();
+ let versions_data = DBVersion::get_many(&version_ids, &mut *txn, redis)
+ .await
+ .wrap_err("fetching versions")?;
+
+ for file in &files {
+ if !versions_data.iter().any(|v| v.inner.id == file.version_id) {
+ continue;
+ }
+
+ if let Some(hash) = file.hashes.get("sha1")
+ && let Some(pos) =
+ remaining.iter().position(|i| overrides[*i].sha1 == *hash)
+ {
+ let idx = remaining.remove(pos);
+ results.insert(
+ overrides[idx].sha1.clone(),
+ OverrideResolution::OnModrinth,
+ );
+ }
+ }
+
+ if remaining.is_empty() {
+ return Ok(results);
+ }
+
+ let rows = sqlx::query!(
+ "
+ SELECT encode(mef.sha1, 'escape') sha1, mel.status status
+ FROM moderation_external_files mef
+ INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
+ WHERE mef.sha1 = ANY($1)
+ ",
+ &remaining
+ .iter()
+ .map(|i| overrides[*i].sha1.as_bytes().to_vec())
+ .collect::>()
+ )
+ .fetch_all(&mut *txn)
+ .await
+ .wrap_err("fetching external file licenses")?;
+
+ for row in rows {
+ if let Some(sha1) = row.sha1
+ && let Some(pos) =
+ remaining.iter().position(|i| overrides[*i].sha1 == sha1)
+ {
+ let idx = remaining.remove(pos);
+ results.insert(
+ overrides[idx].sha1.clone(),
+ OverrideResolution::ExternalLicense(
+ ApprovalType::from_string(&row.status)
+ .unwrap_or(ApprovalType::Unidentified),
+ ),
+ );
+ }
+ }
+
+ if remaining.is_empty() {
+ return Ok(results);
+ }
+
+ let fingerprints: Vec =
+ remaining.iter().map(|i| overrides[*i].murmur2).collect();
+ let res = HTTP_CLIENT
+ .post(format!("{}/v1/fingerprints", ENV.FLAME_ANVIL_URL))
+ .json(&serde_json::json!({
+ "fingerprints": fingerprints
+ }))
+ .send()
+ .await;
+
+ if let Err(e) = &res {
+ warn!("Flame fingerprint request failed: {e}");
+ }
+
+ if let Ok(res) = res {
+ let body = res
+ .text()
+ .await
+ .wrap_err("reading Flame fingerprint response")?;
+
+ let flame_files: Vec<_> =
+ serde_json::from_str::>(&body)
+ .ok()
+ .map(|x| {
+ x.data
+ .exact_matches
+ .into_iter()
+ .map(|m| m.file)
+ .collect::>()
+ })
+ .unwrap_or_default();
+
+ let mut flame_matches: Vec<(String, u32)> = Vec::new();
+ for flame_file in &flame_files {
+ if let Some(hash) = flame_file
+ .hashes
+ .iter()
+ .find(|x| x.algo == 1)
+ .map(|x| x.value.clone())
+ {
+ flame_matches.push((hash, flame_file.mod_id));
+ }
+ }
+
+ let rows = sqlx::query!(
+ "
+ SELECT mel.id, mel.flame_project_id, mel.status status
+ FROM moderation_external_licenses mel
+ WHERE mel.flame_project_id = ANY($1)
+ ",
+ &flame_matches.iter().map(|x| x.1 as i32).collect::>()
+ )
+ .fetch_all(&mut *txn)
+ .await
+ .wrap_err("fetching Flame project licenses")?;
+
+ let mut insert_hashes = Vec::new();
+ let mut insert_filenames = Vec::new();
+ let mut insert_ids = Vec::new();
+
+ for row in &rows {
+ if let Some((curse_idx, (hash, _))) = flame_matches
+ .iter()
+ .enumerate()
+ .find(|(_, x)| Some(x.1 as i32) == row.flame_project_id)
+ && let Some(remaining_pos) =
+ remaining.iter().position(|i| overrides[*i].sha1 == *hash)
+ {
+ let idx = remaining.remove(remaining_pos);
+
+ results.insert(
+ overrides[idx].sha1.clone(),
+ OverrideResolution::ExternalLicense(
+ ApprovalType::from_string(&row.status)
+ .unwrap_or(ApprovalType::Unidentified),
+ ),
+ );
+
+ insert_hashes.push(overrides[idx].sha1.as_bytes().to_vec());
+ insert_filenames.push(Some(overrides[idx].path.clone()));
+ insert_ids.push(row.id);
+
+ flame_matches.remove(curse_idx);
+ }
+ }
+
+ if !insert_hashes.is_empty() {
+ ExternalLicense::insert_files(
+ &mut *txn,
+ &insert_hashes,
+ &insert_filenames,
+ &insert_ids,
+ DBUserId(0),
+ )
+ .await
+ .wrap_err("inserting external license files")?;
+ }
+
+ if !flame_matches.is_empty() {
+ let flame_projects_res = HTTP_CLIENT
+ .post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL))
+ .json(&serde_json::json!({
+ "modIds": flame_matches.iter().map(|x| x.1).collect::>()
+ }))
+ .send()
+ .await;
+
+ let flame_projects = match flame_projects_res {
+ Ok(res) => res
+ .text()
+ .await
+ .ok()
+ .and_then(|t| {
+ serde_json::from_str::<
+ FlameResponse>,
+ >(&t)
+ .ok()
+ })
+ .map(|x| x.data)
+ .unwrap_or_default(),
+ Err(e) => {
+ warn!("Flame projects request failed: {e}");
+ Vec::new()
+ }
+ };
+
+ for (sha1, flame_project_id) in &flame_matches {
+ if let Some(pos) =
+ remaining.iter().position(|i| overrides[*i].sha1 == *sha1)
+ {
+ let idx = remaining.remove(pos);
+
+ let project = flame_projects
+ .iter()
+ .find(|p| p.id == *flame_project_id);
+
+ results.insert(
+ overrides[idx].sha1.clone(),
+ OverrideResolution::Flame {
+ project_id: *flame_project_id,
+ title: project
+ .map(|p| p.name.clone())
+ .unwrap_or_else(|| {
+ format!("Flame project {flame_project_id}")
+ }),
+ url: project
+ .map(|p| p.links.website_url.clone())
+ .unwrap_or_default(),
+ icon_url: project
+ .map(|p| p.logo.thumbnail_url.clone())
+ .unwrap_or_default(),
+ },
+ );
+ }
+ }
+ }
+ }
+
+ for idx in remaining {
+ results
+ .insert(overrides[idx].sha1.clone(), OverrideResolution::Unknown);
+ }
+
+ Ok(results)
+}
+
+fn hash_flame_murmur32(input: Vec) -> u32 {
+ murmur2::murmur2(
+ &input
+ .into_iter()
+ .filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32)
+ .collect::>(),
+ 1,
+ )
+}
+
+pub async fn get_files_missing_attribution<'a, E>(
+ exec: E,
+ version_ids: &[DBVersionId],
+) -> Result<
+ std::collections::HashMap<
+ DBVersionId,
+ Vec<(
+ DBFileId,
+ Option,
+ )>,
+ >,
+>
+where
+ E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+{
+ if version_ids.is_empty() {
+ return Ok(std::collections::HashMap::new());
+ }
+
+ let rows = sqlx::query!(
+ r#"
+ select distinct f.version_id as "version_id: DBVersionId", f.id as "file_id: DBFileId",
+ pag.flame_project
+ from files f
+ inner join override_file_sources ofs on ofs.file_id = f.id
+ inner join project_attribution_files paf on paf.sha1 = ofs.sha1
+ inner join project_attribution_groups pag on pag.id = paf.group_id
+ where f.version_id = ANY($1) and (pag.attribution is null or pag.attribution->>'kind' = 'no_permission')
+ "#,
+ &version_ids.iter().map(|v| v.0).collect::>(),
+ )
+ .fetch_all(exec)
+ .await
+ .wrap_err("fetching files missing attribution")?;
+
+ let mut result = std::collections::HashMap::new();
+ for row in rows {
+ let flame_project = row
+ .flame_project
+ .and_then(|v| serde_json::from_value(v).ok());
+ result
+ .entry(row.version_id)
+ .or_insert_with(Vec::new)
+ .push((row.file_id, flame_project));
+ }
+
+ Ok(result)
+}
diff --git a/apps/labrinth/src/queue/mod.rs b/apps/labrinth/src/queue/mod.rs
index 666670dc0b..0d7dcb273b 100644
--- a/apps/labrinth/src/queue/mod.rs
+++ b/apps/labrinth/src/queue/mod.rs
@@ -1,6 +1,7 @@
pub mod analytics;
pub mod billing;
pub mod email;
+pub mod file_scan;
pub mod moderation;
pub mod payouts;
pub mod server_ping;
diff --git a/apps/labrinth/src/queue/moderation.rs b/apps/labrinth/src/queue/moderation.rs
index 7d852f4eb3..1914f80359 100644
--- a/apps/labrinth/src/queue/moderation.rs
+++ b/apps/labrinth/src/queue/moderation.rs
@@ -570,7 +570,7 @@ impl AutomatedModerationQueue {
Vec::new()
} else {
let res = client
- .post(format!("{}v1/mods", ENV.FLAME_ANVIL_URL))
+ .post(format!("{}/v1/mods", ENV.FLAME_ANVIL_URL))
.json(&serde_json::json!({
"modIds": flame_files.iter().map(|x| x.1).collect::>()
}))
@@ -823,7 +823,7 @@ pub enum ApprovalType {
}
impl ApprovalType {
- fn approved(&self) -> bool {
+ pub fn approved(&self) -> bool {
match self {
ApprovalType::Yes => true,
ApprovalType::WithAttributionAndSource => true,
@@ -901,6 +901,13 @@ pub struct FlameProject {
pub name: String,
pub slug: String,
pub links: FlameLinks,
+ pub logo: FlameLogo,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FlameLogo {
+ pub thumbnail_url: String,
}
#[derive(Deserialize, Serialize)]
diff --git a/apps/labrinth/src/routes/internal/attribution.rs b/apps/labrinth/src/routes/internal/attribution.rs
new file mode 100644
index 0000000000..920dabb190
--- /dev/null
+++ b/apps/labrinth/src/routes/internal/attribution.rs
@@ -0,0 +1,364 @@
+use actix_web::{HttpRequest, get, patch, post, web};
+use serde::{Deserialize, Serialize};
+use sqlx::Row;
+
+use crate::auth::get_user_from_headers;
+use crate::database::PgPool;
+use crate::database::models::ids::{
+ DBAttributionGroupId, DBProjectId, generate_attribution_group_id,
+};
+use crate::database::redis::RedisPool;
+use crate::models::ids::{ProjectId, VersionId};
+use crate::models::pats::Scopes;
+use crate::queue::session::AuthQueue;
+use crate::routes::ApiError;
+
+pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
+ cfg.service(
+ utoipa_actix_web::scope("/attribution")
+ .service(list)
+ .service(update_group)
+ .service(assign)
+ .service(split),
+ );
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct FlameProject {
+ pub id: u32,
+ pub title: String,
+ pub url: String,
+ pub icon_url: String,
+}
+
+#[derive(Serialize)]
+struct AttributionGroupResponse {
+ id: crate::models::ids::AttributionGroupId,
+ flame_project: Option,
+ attribution: Option,
+ attributed_at: Option>,
+ attributed_by: Option,
+ files: Vec,
+ versions: Vec,
+}
+
+#[derive(Clone, Serialize)]
+struct VersionInfo {
+ id: VersionId,
+ name: String,
+ version_number: String,
+ date_created: chrono::DateTime,
+}
+
+#[derive(Serialize)]
+struct AttributionFileResponse {
+ name: String,
+ sha1: String,
+ versions: Vec,
+}
+
+#[utoipa::path]
+#[get("{project_id}")]
+async fn list(
+ pool: web::Data,
+ path: web::Path,
+) -> Result>, ApiError> {
+ let project_id: DBProjectId = path.into_inner().into();
+
+ let groups = sqlx::query!(
+ r#"
+ select
+ g.id as "id: DBAttributionGroupId",
+ g.flame_project,
+ g.attribution,
+ g.attributed_at,
+ g.attributed_by as "attributed_by: i64"
+ from project_attribution_groups g
+ where g.project_id = $1
+ "#,
+ project_id as DBProjectId,
+ )
+ .fetch_all(pool.as_ref())
+ .await?;
+
+ let group_ids: Vec = groups.iter().map(|g| g.id.0).collect();
+
+ let files = if group_ids.is_empty() {
+ Vec::new()
+ } else {
+ sqlx::query(
+ "
+ select paf.group_id, paf.name, encode(paf.sha1, 'hex') as sha1,
+ array_agg(distinct f.version_id) as version_ids
+ from project_attribution_files paf
+ left join override_file_sources ofs on ofs.sha1 = paf.sha1
+ left join files f on f.id = ofs.file_id
+ where paf.group_id = ANY($1)
+ group by paf.group_id, paf.name, paf.sha1
+ ",
+ )
+ .bind(&group_ids)
+ .fetch_all(pool.as_ref())
+ .await?
+ };
+
+ let mut all_version_ids: Vec = files
+ .iter()
+ .filter_map(|f| f.get::