Add import translation action

app/blueprints/admin/actions.py

@@ -26,7 +26,8 @@
 	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, Thread
 from app.tasks.emails import send_pending_digests
 from app.tasks.forumtasks import import_topic_list, check_all_forum_accounts
-from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support
+from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support, \
+	import_languages
 from app.utils import add_notification, get_system_user
 
 actions = {}
@@ -321,7 +322,7 @@ def check_releases():
 @action("DANGER: Check latest release of all packages (postReleaseCheckUpdate)")
 def reimport_packages():
 	tasks = []
-	for package in Package.query.filter(Package.state != PackageState.DELETED).all():
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
 		release = package.releases.first()
 		if release:
 			tasks.append(check_zip_release.s(release.id, release.file_path))
@@ -335,6 +336,22 @@ def reimport_packages():
 	return redirect(url_for("todo.view_editor"))
 
 
+@action("DANGER: Import translations")
+def reimport_translations():
+	tasks = []
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
+		release = package.releases.first()
+		if release:
+			tasks.append(import_languages.s(release.id, release.file_path))
+
+	result = group(tasks).apply_async()
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+
 @action("DANGER: Import screenshots from Git")
 def import_screenshots():
 	packages = Package.query \

app/tasks/importtasks.py

@@ -164,37 +164,7 @@ def get_meta_packages(names):
 			db.session.add(Dependency(package, meta=meta, optional=True))
 
 		# Read translations
-		allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
-		allowed_languages.discard("en")
-		conn = db.session.connection()
-
-		for language in tree.get_supported_languages():
-			if language not in allowed_languages:
-				continue
-
-			values = {
-				"package_id": package.id,
-				"language_id": language,
-			}
-			stmt = insert(PackageTranslation).values(**values)
-			stmt = stmt.on_conflict_do_nothing(
-				index_elements=[PackageTranslation.package_id, PackageTranslation.language_id],
-			)
-			conn.execute(stmt)
-
-		raw_translations = tree.get_translations(tree.get("textdomain", tree.name))
-		for raw_translation in raw_translations:
-			if raw_translation.language not in allowed_languages:
-				continue
-
-			to_update = {
-				"title": raw_translation.entries.get(tree.get("title", package.title)),
-				"short_desc": raw_translation.entries.get(tree.get("description", package.short_desc)),
-			}
-
-			PackageTranslation.query \
-				.filter_by(package_id=package.id, language_id=raw_translation.language) \
-				.update(to_update)
+		update_translations(package, tree)
 
 		# Update min/max
 		if tree.meta.get("min_minetest_version"):
@@ -258,6 +228,39 @@ def get_meta_packages(names):
 		raise TaskError(str(err))
 
 
+def update_translations(package: Package, tree: PackageTreeNode):
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	allowed_languages.discard("en")
+	conn = db.session.connection()
+	for language in tree.get_supported_languages():
+		if language not in allowed_languages:
+			continue
+
+		values = {
+			"package_id": package.id,
+			"language_id": language,
+		}
+		stmt = insert(PackageTranslation).values(**values)
+		stmt = stmt.on_conflict_do_nothing(
+			index_elements=[PackageTranslation.package_id, PackageTranslation.language_id],
+		)
+		conn.execute(stmt)
+
+	raw_translations = tree.get_translations(tree.get("textdomain", tree.name))
+	for raw_translation in raw_translations:
+		if raw_translation.language not in allowed_languages:
+			continue
+
+		to_update = {
+			"title": raw_translation.entries.get(tree.get("title", package.title)),
+			"short_desc": raw_translation.entries.get(tree.get("description", package.short_desc)),
+		}
+
+		PackageTranslation.query \
+			.filter_by(package_id=package.id, language_id=raw_translation.language) \
+			.update(to_update)
+
+
 @celery.task(bind=True)
 def check_zip_release(self, id, path):
 	release = PackageRelease.query.get(id)
@@ -277,6 +280,37 @@ def check_zip_release(self, id, path):
 		db.session.commit()
 
 
+@celery.task(bind=True)
+def import_languages(self, id, path):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
+
+	with get_temp_dir() as temp:
+		with ZipFile(path, 'r') as zip_ref:
+			zip_ref.extractall(temp)
+
+		try:
+			tree: PackageTreeNode = build_tree(temp,
+					expected_type=ContentType[release.package.type.name],
+					author=release.package.author.username,
+					name=release.package.name,
+					strict=False)
+			update_translations(release.package, tree)
+			db.session.commit()
+		except (MinetestCheckError, TaskError, LogicError) as err:
+			db.session.rollback()
+
+			task_url = url_for('tasks.check', id=self.request.id)
+			msg = f"{err}\n\n[View Release]({release.get_edit_url()}) | [View Task]({task_url})"
+			post_bot_message(release.package, f"Release {release.title} validation failed", msg)
+			db.session.commit()
+
+			raise TaskError(str(err))
+
+
 @celery.task(bind=True)
 def make_vcs_release(self, id, branch):
 	release = PackageRelease.query.get(id)

app/tasks/minetestcheck/__init__.py

@@ -56,10 +56,10 @@ def validate_same(self, other):
 from .tree import PackageTreeNode, get_base_dir
 
 
-def build_tree(path, expected_type=None, author=None, repo=None, name=None):
+def build_tree(path, expected_type=None, author=None, repo=None, name=None, strict: bool = True):
 	path = get_base_dir(path)
 
-	root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
+	root = PackageTreeNode(path, "/", author=author, repo=repo, name=name, strict=strict)
 	assert root
 
 	if expected_type:

app/tasks/minetestcheck/tree.py

@@ -89,16 +89,20 @@ class PackageTreeNode:
 	meta: dict
 	children: list
 	type: ContentType
+	strict: bool
 
 	def __init__(self, base_dir: str, relative: str,
-			author: Optional[str] = None, repo: Optional[str] = None, name: Optional[str] = None):
+			author: Optional[str] = None,
+			repo: Optional[str] = None, name: Optional[str] = None,
+			strict: bool = True):
 		self.baseDir = base_dir
 		self.relative = relative
 		self.author = author
 		self.name = name
 		self.repo = repo
 		self.meta = {}
 		self.children = []
+		self.strict = strict
 
 		# Detect type
 		self.type = detect_type(base_dir)
@@ -169,7 +173,7 @@ def _read_meta(self):
 			except IOError:
 				pass
 
-			if "release" in result:
+			if self.strict and "release" in result:
 				raise MinetestCheckError("{} should not contain 'release' key, as this is for use by ContentDB only.".format(meta_file_rel))
 
 		# description.txt
@@ -251,7 +255,7 @@ def _add_children_from_mod_dir(self, subdir):
 		for entry in next(os.walk(dir))[1]:
 			path = os.path.join(dir, entry)
 			if not entry.startswith('.') and os.path.isdir(path):
-				child = PackageTreeNode(path, relative + entry + "/", name=entry)
+				child = PackageTreeNode(path, relative + entry + "/", name=entry, strict=self.strict)
 				if not child.type.is_mod_like():
 					raise MinetestCheckError("Expecting mod or modpack, found {} at {} inside {}" \
 						.format(child.type.value, child.relative, self.type.value))