Add end-to-end test case; more robust parsing

Author: huzecong

lchelper/crawler.py

@@ -171,6 +171,6 @@ def get_problems(contest_url: str, site: str, cookie_path: str) -> List[Problem]
         log(f"Parsed problem ({idx + 1}/{len(problem_paths)}): {problem_name}")
 
     browser.quit()
-    log("All problems successfully parsed", "success")
+    log("All problems successfully crawled", "success")
 
     return parsed_problems

lchelper/parser.py

@@ -71,19 +71,24 @@ def parse_problem(problem: Problem, site: str = "leetcode") -> Union[ProblemSign
     r"""Parse the problem given the raw contents crawled from the web.
     """
 
-    def find_example_section(s: str, cur_tag: str, next_tag: str, colon: str = ":") -> str:
+    def find_example_section(s: str, cur_tag: str, next_tag: str, colon: str = ":", ignore_error: bool = False) -> str:
         r"""Find the part in the example that is between two tags. If ``next_tag`` does not exist, then find the part
         until the end.
         """
         start_pos = s.find(cur_tag)
         if start_pos == -1:
-            raise ValueError
-        start_pos += len(cur_tag)
-        if s[start_pos] == colon:
-            start_pos += 1
+            if not ignore_error:
+                raise ValueError
+            start_pos = 0
+        else:
+            start_pos += len(cur_tag)
+            if s[start_pos] == colon:
+                start_pos += 1
         end_pos = s.find(next_tag, start_pos)
         if end_pos == -1:
-            return s[start_pos:].strip()
+            if not ignore_error:
+                raise ValueError
+            end_pos = len(s)
         return s[start_pos:end_pos].strip()
 
     # Parse function signature from code.
@@ -98,8 +103,8 @@ def find_example_section(s: str, cur_tag: str, next_tag: str, colon: str = ":")
                 input_str = find_example_section(example, "输入", "输出", "：")
                 output_str = find_example_section(example, "输出", "解释", "：")
             except ValueError:
-                input_str = find_example_section(example, "Input", "Output")
-                output_str = find_example_section(example, "Output", "Explanation")
+                input_str = find_example_section(example, "Input", "Output", ignore_error=True)
+                output_str = find_example_section(example, "Output", "Explanation", ignore_error=True)
 
             functions, input_str = parse_value(input_str)
             arg_vals, input_str = parse_value(input_str)
@@ -130,8 +135,8 @@ def find_example_section(s: str, cur_tag: str, next_tag: str, colon: str = ":")
                 input_str = find_example_section(example, "输入", "输出", "：")
                 output_str = find_example_section(example, "输出", "解释", "：")
             except ValueError:
-                input_str = find_example_section(example, "Input", "Output")
-                output_str = find_example_section(example, "Output", "Explanation")
+                input_str = find_example_section(example, "Input", "Output", ignore_error=True)
+                output_str = find_example_section(example, "Output", "Explanation", ignore_error=True)
 
             input_vals = {}
             for idx, (_, name) in enumerate(func_signature.arguments):

test.py

@@ -1,19 +1,52 @@
 import unittest
-from typing import Union
+from typing import Union, Dict, Optional, List
 
-from lchelper.parser import parse_problem
+import lchelper.codegen
 from lchelper.common import FunctionSignature, Example, ProblemSignature, Interaction, \
     InteractiveProblemSignature, Problem
 
 
+class EndToEndTest(unittest.TestCase):
+    def _test_problem_set(self, url: str, site: str = "leetcode", ignore_problems: Optional[List[int]] = None):
+        available_users = [user for user in lchelper.get_users() if user.site == site]
+        assert len(available_users) > 0, f"User cookie from site \"{site}\" required for end-to-end tests"
+        user = available_users[0]
+        problems = lchelper.get_problems(url, site, lchelper.get_cookie_path(user.username, user.site))
+        codegen: Dict[str, lchelper.codegen.CodeGen] = {
+            lang: codegen_klass()
+            for lang, codegen_klass in lchelper.LANGUAGES.items()
+        }
+
+        ignore_problems = ignore_problems or []
+        for idx, problem in enumerate(problems):
+            if idx in ignore_problems:
+                continue
+            problem_signature = lchelper.parse_problem(problem, site)
+            for lang, gen in codegen.items():
+                _, _ = gen.generate_code(problem, problem_signature)
+
+    def test_contests(self):
+        contests = [
+            ("weekly-contest-183", []),
+            ("weekly-contest-182", []),
+            ("weekly-contest-181", []),
+            ("weekly-contest-180", []),
+            ("weekly-contest-163", []),
+            ("biweekly-contest-14", []),
+        ]
+        for contest, ignore_problems in contests:
+            url = f"https://leetcode.com/contest/{contest}"
+            self._test_problem_set(url, ignore_problems=ignore_problems)
+
+
 class ParseTest(unittest.TestCase):
     def _function_equal(self, parsed_function: FunctionSignature, function: FunctionSignature):
         assert parsed_function.return_type == function.return_type
         assert parsed_function.name == function.name
         assert parsed_function.arguments == function.arguments
 
     def _test_parse_problem(self, problem: Problem, signature: Union[ProblemSignature, InteractiveProblemSignature]):
-        parsed_signature = parse_problem(problem)
+        parsed_signature = lchelper.parse_problem(problem)
         assert type(parsed_signature) is type(signature)
         if isinstance(signature, InteractiveProblemSignature):
             assert parsed_signature.class_name == signature.class_name
@@ -141,7 +174,7 @@ def test_parse_problem_2(self):
             class_name="FindElements",
             functions=[
                 FunctionSignature(
-                    return_type="", name="FindElements",
+                    return_type="FindElements", name="FindElements",
                     arguments=[("TreeNode*", "root")]),
                 FunctionSignature(
                     return_type="bool", name="find",