Merge pull request #41 from pengyunie/39-improvements-to-project

39 improvements to project

seutil/ds/graph_common.py

@@ -1,5 +1,5 @@
 import abc
-from typing import List, Optional
+from typing import Iterable, List, Optional
 
 import igraph
 
@@ -165,3 +165,9 @@ def __str__(self):
 
     def __repr__(self):
         return self.g.__repr__()
+
+    def nodes(self) -> Iterable[Node]:
+        return [Node(v, self) for v in self.g.vs]
+
+    def edges(self) -> Iterable[Edge]:
+        return [Edge(e, self) for e in self.g.es]

seutil/project.py

@@ -1,13 +1,15 @@
 import dataclasses
+import re
 import sys
 import traceback
 from pathlib import Path
 from types import TracebackType
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
+import warnings
 
 from tqdm import tqdm
 
-from . import bash, io, log, ds
+from . import bash, ds, io, log
 
 logger = log.get_logger(__name__, log.INFO)
 
@@ -55,17 +57,21 @@ def dir(self) -> Path:
         self.require_cloned()
         return self._dir
 
+    re_github_url = re.compile(
+        r"^((https://)?github\.com/|git@github\.com:)(?P<repo>[^/\n\r]+)/(?P<name>[^/\n\r]+?)(\.git)?$"
+    )
+
     @classmethod
-    def from_github_url(
-        cls,
-        url: str,
-        revision: Optional[str] = None,
-        default_branch: Optional[str] = None,
-    ) -> "Project":
+    def from_github_url(cls, url: str) -> "Project":
         """
-        TODO: Creates a Project from a GitHub URL.
+        Creates a Project from a GitHub URL.
         """
-        raise NotImplementedError()
+        match = cls.re_github_url.fullmatch(url)
+        if match is None:
+            raise ValueError(f"Invalid GitHub URL: {url}")
+        return cls(
+            full_name=f"{match.group('repo')}_{match.group('name')}", clone_url=url
+        )
 
     def clone(
         self, downloads_dir: Path, name: Optional[str] = None, exists: str = "ignore"
@@ -205,7 +211,7 @@ def for_each_revision(
         self,
         func: Callable[["Project", str], Any],
         revisions: Iterable[str],
-        is_auto_checkout: bool = True,
+        auto_checkout: bool = True,
         errors: str = "collate",
         pbar: Optional[tqdm] = None,
     ) -> List[Any]:
@@ -214,7 +220,7 @@ def for_each_revision(
 
         :param func: the function to run, which takes a Project object and a string revision.
         :param revisions: the revisions to run the function on.
-        :param is_auto_checkout: if set to False, will not automatically checkout each revision (for example, if func does that itself, or if func does not need the project to be checked out at the given revision).
+        :param auto_checkout: if set to False, will not automatically checkout each revision (for example, if func does that itself, or if func does not need the project to be checked out at the given revision).
         :param errors: what to do if the function throws errors:
             * ignore: do nothing, not even say a word.
             * warning: make a log.warning for each error.
@@ -235,21 +241,21 @@ def for_each_revision(
                 pbar.set_description(f"Revision {revision}")
 
             try:
-                if is_auto_checkout:
+                if auto_checkout:
                     self.checkout(revision, True)
-                with io.cd(self.checkout_dir):
+                with io.cd(self.dir):
                     results.append(func(self, revision))
             except KeyboardInterrupt:
                 raise
             except:
                 if errors == "ignore":
                     pass
                 elif errors == "warning":
-                    logger.warning(
+                    warnings.warn(
                         f"Project {self.full_name}: error at revision {revision}: {traceback.format_exc()}"
                     )
                 elif errors == "collate":
-                    logger.warning(
+                    warnings.warn(
                         f"Project {self.full_name}: error at revision {revision}: {traceback.format_exc()}"
                     )
                     errored_revisions.append(revision)

tests/test_project.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from pathlib import Path
 
@@ -223,3 +224,140 @@ def test_get_revisions_lattice(local_gitgame_repo: su.project.Project, tmp_path:
     lattice = local_gitgame_repo.get_revisions_lattice()
     assert lattice.ncount() == 6
     assert lattice.ecount() == 5
+
+
+def test_from_github_url():
+    for url in [
+        "https://github.com/pengyunie/seutil",
+        "https://github.com/pengyunie/seutil.git",
+        "git@github.com:pengyunie/seutil.git",
+    ]:
+        proj = su.project.Project.from_github_url(url)
+        assert proj.clone_url == url
+        assert proj.full_name == "pengyunie_seutil"
+
+
+def test_from_github_url_invalid():
+    with pytest.raises(ValueError):
+        su.project.Project.from_github_url("http://github.com/pengyunie/seutil")
+
+
+def test_for_each_revision(local_gitgame_repo: su.project.Project, tmp_path: Path):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+
+    def action(p: su.project.Project, r: str):
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    ret = local_gitgame_repo.for_each_revision(action, revisions)
+    assert [x[0] for x in ret] == revisions
+    assert [Path(x[1]) for x in ret] == [local_gitgame_repo.dir] * len(revisions)
+
+
+def test_for_each_revision_no_auto_checkout(
+    local_gitgame_repo: su.project.Project, tmp_path: Path
+):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+    local_gitgame_repo.checkout(revisions[0])
+
+    def action(p: su.project.Project, r: str):
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    ret = local_gitgame_repo.for_each_revision(action, revisions, auto_checkout=False)
+    assert [x[0] for x in ret] == [revisions[0]] * len(revisions)
+    assert [Path(x[1]) for x in ret] == [local_gitgame_repo.dir] * len(revisions)
+
+
+def test_for_each_revision_errors_ignore(
+    local_gitgame_repo: su.project.Project, tmp_path: Path
+):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+
+    def action(p: su.project.Project, r: str):
+        nonlocal revisions
+        if r == revisions[0]:
+            raise Exception("error")
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    ret = local_gitgame_repo.for_each_revision(action, revisions, errors="ignore")
+    assert [x[0] for x in ret] == revisions[1:]
+    assert [Path(x[1]) for x in ret] == [local_gitgame_repo.dir] * (len(revisions) - 1)
+
+
+def test_for_each_revision_errors_warning(
+    local_gitgame_repo: su.project.Project,
+    tmp_path: Path,
+    recwarn: pytest.WarningsRecorder,
+):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+
+    def action(p: su.project.Project, r: str):
+        nonlocal revisions
+        if r == revisions[0]:
+            raise Exception("error")
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    recwarn.clear()
+    ret = local_gitgame_repo.for_each_revision(action, revisions, errors="warning")
+    assert len(recwarn.list) == 1
+    assert [x[0] for x in ret] == revisions[1:]
+    assert [Path(x[1]) for x in ret] == [local_gitgame_repo.dir] * (len(revisions) - 1)
+
+
+def test_for_each_revision_errors_collate(
+    local_gitgame_repo: su.project.Project,
+    tmp_path: Path,
+    recwarn: pytest.WarningsRecorder,
+):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+    assert len(revisions) >= 2
+
+    def action(p: su.project.Project, r: str):
+        nonlocal revisions
+        if r == revisions[0] or r == revisions[-1]:
+            raise Exception("error")
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    recwarn.clear()
+    with pytest.raises(su.project.CollatedErrors) as exc_info:
+        local_gitgame_repo.for_each_revision(action, revisions, errors="collate")
+
+    assert exc_info.value.contexts == [revisions[0], revisions[-1]]
+    assert len(recwarn.list) == 2
+
+
+def test_for_each_revision_errors_error(
+    local_gitgame_repo: su.project.Project, tmp_path: Path
+):
+    local_gitgame_repo.clone(tmp_path)
+
+    lattice = local_gitgame_repo.get_revisions_lattice()
+    revisions = [n["revision"] for n in lattice.nodes()]
+    assert len(revisions) >= 2
+
+    def action(p: su.project.Project, r: str):
+        nonlocal revisions
+        if r == revisions[0]:
+            pass
+        else:
+            raise Exception(r)
+        return (p.get_cur_revision(), su.bash.run("pwd").stdout.strip())
+
+    with pytest.raises(Exception) as exc_info:
+        local_gitgame_repo.for_each_revision(action, revisions, errors="error")
+
+    assert exc_info.value.args[0] == revisions[1]