From 55d6ae0f290463943e19ec77fec38301fe093cc9 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Wed, 24 Jul 2024 17:15:39 +0200
Subject: [PATCH 1/6] refactor: rename sync_repos into repo_synchronizer
---
git_hg_sync/__main__.py | 2 +-
.../{sync_repos.py => repo_synchronizer.py} | 0
...test_sync_repo.py => test_repo_synchronizer.py} | 14 +++++++-------
3 files changed, 8 insertions(+), 8 deletions(-)
rename git_hg_sync/{sync_repos.py => repo_synchronizer.py} (100%)
rename tests/{test_sync_repo.py => test_repo_synchronizer.py} (81%)
diff --git a/git_hg_sync/__main__.py b/git_hg_sync/__main__.py
index 6197915..0dc3af1 100644
--- a/git_hg_sync/__main__.py
+++ b/git_hg_sync/__main__.py
@@ -7,7 +7,7 @@
from git_hg_sync import config
from git_hg_sync.pulse_consumer import Worker
-from git_hg_sync.sync_repos import RepoSynchronyzer
+from git_hg_sync.repo_synchronizer import RepoSynchronyzer
HERE = Path(__file__).parent
diff --git a/git_hg_sync/sync_repos.py b/git_hg_sync/repo_synchronizer.py
similarity index 100%
rename from git_hg_sync/sync_repos.py
rename to git_hg_sync/repo_synchronizer.py
diff --git a/tests/test_sync_repo.py b/tests/test_repo_synchronizer.py
similarity index 81%
rename from tests/test_sync_repo.py
rename to tests/test_repo_synchronizer.py
index 18bdf2b..1253f15 100644
--- a/tests/test_sync_repo.py
+++ b/tests/test_repo_synchronizer.py
@@ -3,7 +3,7 @@
import mozlog
import pytest
-from git_hg_sync import __main__, sync_repos
+from git_hg_sync import __main__, repo_synchronizer
HERE = Path(__file__).parent
@@ -61,21 +61,21 @@ def setup_module():
def test_parse_entity():
- syncrepos = sync_repos.RepoSynchronyzer(None)
+ syncrepos = repo_synchronizer.RepoSynchronyzer(None)
push_entity = syncrepos.parse_entity(raw_push_entity)
- assert isinstance(push_entity, sync_repos.Push)
+ assert isinstance(push_entity, repo_synchronizer.Push)
tag_entity = syncrepos.parse_entity(raw_tag_entity)
- assert isinstance(tag_entity, sync_repos.Tag)
+ assert isinstance(tag_entity, repo_synchronizer.Tag)
def test_sync_process_with_bad_type():
- syncrepos = sync_repos.RepoSynchronyzer(None)
- with pytest.raises(sync_repos.EntityTypeError):
+ syncrepos = repo_synchronizer.RepoSynchronyzer(None)
+ with pytest.raises(repo_synchronizer.EntityTypeError):
syncrepos.sync({"type": "badType"})
def test_sync_process_with_bad_repo(repos_config):
- syncrepos = sync_repos.RepoSynchronyzer(repos_config=repos_config)
+ syncrepos = repo_synchronizer.RepoSynchronyzer(repos_config=repos_config)
with pytest.raises(AssertionError) as e:
syncrepos.sync(raw_push_entity)
assert str(e.value) == f"clone {repos_config['repo_url']['clone']} doesn't exists"
From fb8a4db3ddda7a3b3f4f283c3ac5abe203309fb5 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Wed, 24 Jul 2024 17:34:40 +0200
Subject: [PATCH 2/6] refactor: rename Worker class/module to PulseWorker
---
git_hg_sync/__main__.py | 4 ++--
git_hg_sync/{pulse_consumer.py => pulse_worker.py} | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
rename git_hg_sync/{pulse_consumer.py => pulse_worker.py} (95%)
diff --git a/git_hg_sync/__main__.py b/git_hg_sync/__main__.py
index 0dc3af1..a27b4c9 100644
--- a/git_hg_sync/__main__.py
+++ b/git_hg_sync/__main__.py
@@ -6,7 +6,7 @@
from mozlog import commandline
from git_hg_sync import config
-from git_hg_sync.pulse_consumer import Worker
+from git_hg_sync.pulse_worker import PulseWorker
from git_hg_sync.repo_synchronizer import RepoSynchronyzer
HERE = Path(__file__).parent
@@ -52,7 +52,7 @@ def main():
repo_synchronyzer = RepoSynchronyzer(repos_config=repos_config)
with connection as conn:
logger.info(f"connected to {conn.host}")
- worker = Worker(conn, queue, repo_synchronyzer=repo_synchronyzer)
+ worker = PulseWorker(conn, queue, repo_synchronyzer=repo_synchronyzer)
worker.run()
diff --git a/git_hg_sync/pulse_consumer.py b/git_hg_sync/pulse_worker.py
similarity index 95%
rename from git_hg_sync/pulse_consumer.py
rename to git_hg_sync/pulse_worker.py
index 1d6a42b..6b2ecc8 100644
--- a/git_hg_sync/pulse_consumer.py
+++ b/git_hg_sync/pulse_worker.py
@@ -4,7 +4,7 @@
logger = get_proxy_logger("pluse_consumer")
-class Worker(ConsumerMixin):
+class PulseWorker(ConsumerMixin):
def __init__(self, connection, queue, *, repo_synchronyzer):
self.connection = connection
From 7eaefd0f63be7dc0989019802683f60225cdcd53 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Wed, 24 Jul 2024 17:35:10 +0200
Subject: [PATCH 3/6] refactor: deduplicate type information from messages
Use match/case when possible.
---
git_hg_sync/repo_synchronizer.py | 26 ++++++++---------
tests/test_repo_synchronizer.py | 50 +++++++++++++++++---------------
2 files changed, 39 insertions(+), 37 deletions(-)
diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py
index 06f0a82..a496028 100644
--- a/git_hg_sync/repo_synchronizer.py
+++ b/git_hg_sync/repo_synchronizer.py
@@ -13,7 +13,6 @@ class EntityTypeError(Exception):
@dataclass
class Push:
- type: str
repo_url: str
heads: list[str]
commits: list[str]
@@ -25,7 +24,6 @@ class Push:
@dataclass
class Tag:
- type: str
repo_url: str
tag: str
commit: str
@@ -42,13 +40,14 @@ def __init__(self, repos_config):
def parse_entity(self, raw_entity):
logger.debug(f"parse_entity: {raw_entity}")
- if raw_entity["type"] == "push":
- entity = Push(**raw_entity)
- elif raw_entity["type"] == "tag":
- entity = Tag(**raw_entity)
- else:
- raise EntityTypeError(f"unsupported type {raw_entity['type']}")
- return entity
+ message_type = raw_entity.pop("type")
+ match message_type:
+ case "push":
+ return Push(**raw_entity)
+ case "tag":
+ return Tag(**raw_entity)
+ case _:
+ raise EntityTypeError(f"unsupported type {message_type}")
def get_remote(self, repo, remote_url):
"""
@@ -70,10 +69,11 @@ def handle_commits(self, entity, clone_dir, remote_url, remote_target):
remote = self.get_remote(repo, remote_url)
# fetch new commits
remote.fetch()
- if entity.type == "push":
- remote.pull("branches/default/tip")
- elif entity.type == "tag":
- pass # TODO
+ match entity:
+ case Push():
+ remote.pull("branches/default/tip")
+ case _:
+ pass # TODO
# push on good repo/branch
remote = repo.remote(remote_target)
remote.push()
diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py
index 1253f15..32735c5 100644
--- a/tests/test_repo_synchronizer.py
+++ b/tests/test_repo_synchronizer.py
@@ -31,28 +31,30 @@ def repos_config():
}
}
+def raw_push_entity():
+ return {
+ "type": "push",
+ "repo_url": "repo_url",
+ "heads": ["head"],
+ "commits": ["commit"],
+ "time": 0,
+ "pushid": 0,
+ "user": "user",
+ "push_json_url": "push_json_url",
+ }
+
-raw_push_entity = {
- "type": "push",
- "repo_url": "repo_url",
- "heads": ["head"],
- "commits": ["commit"],
- "time": 0,
- "pushid": 0,
- "user": "user",
- "push_json_url": "push_json_url",
-}
-
-raw_tag_entity = {
- "type": "tag",
- "repo_url": "repo_url",
- "tag": "tag",
- "commit": "commit",
- "time": 0,
- "pushid": 0,
- "user": "user",
- "push_json_url": "push_json_url",
-}
+def raw_tag_entity():
+ return {
+ "type": "tag",
+ "repo_url": "repo_url",
+ "tag": "tag",
+ "commit": "commit",
+ "time": 0,
+ "pushid": 0,
+ "user": "user",
+ "push_json_url": "push_json_url",
+ }
def setup_module():
@@ -62,9 +64,9 @@ def setup_module():
def test_parse_entity():
syncrepos = repo_synchronizer.RepoSynchronyzer(None)
- push_entity = syncrepos.parse_entity(raw_push_entity)
+ push_entity = syncrepos.parse_entity(raw_push_entity())
assert isinstance(push_entity, repo_synchronizer.Push)
- tag_entity = syncrepos.parse_entity(raw_tag_entity)
+ tag_entity = syncrepos.parse_entity(raw_tag_entity())
assert isinstance(tag_entity, repo_synchronizer.Tag)
@@ -77,7 +79,7 @@ def test_sync_process_with_bad_type():
def test_sync_process_with_bad_repo(repos_config):
syncrepos = repo_synchronizer.RepoSynchronyzer(repos_config=repos_config)
with pytest.raises(AssertionError) as e:
- syncrepos.sync(raw_push_entity)
+ syncrepos.sync(raw_push_entity())
assert str(e.value) == f"clone {repos_config['repo_url']['clone']} doesn't exists"
From 6ffab615a0a3ce741ccfd2b450f549d9d1423eb1 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Thu, 25 Jul 2024 01:31:11 +0200
Subject: [PATCH 4/6] refactor(tests): share mozlog setup through a autouse
fixture in conftest.py
---
tests/conftest.py | 10 ++++++++++
tests/test_repo_synchronizer.py | 6 ------
2 files changed, 10 insertions(+), 6 deletions(-)
create mode 100644 tests/conftest.py
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..f48c0a8
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,10 @@
+import pytest
+import mozlog
+
+@pytest.fixture(autouse=True)
+def mozlog_logging():
+ logger = mozlog.structuredlog.StructuredLogger("tests")
+ mozlog.structuredlog.set_default_logger(logger)
+
+
+
diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py
index 32735c5..d1f0512 100644
--- a/tests/test_repo_synchronizer.py
+++ b/tests/test_repo_synchronizer.py
@@ -1,6 +1,5 @@
from pathlib import Path
-import mozlog
import pytest
from git_hg_sync import __main__, repo_synchronizer
@@ -57,11 +56,6 @@ def raw_tag_entity():
}
-def setup_module():
- logger = mozlog.structuredlog.StructuredLogger("tests")
- mozlog.structuredlog.set_default_logger(logger)
-
-
def test_parse_entity():
syncrepos = repo_synchronizer.RepoSynchronyzer(None)
push_entity = syncrepos.parse_entity(raw_push_entity())
From 6b81ddf2fbbae9aa81af2e3d9997ba46e0838334 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Thu, 25 Jul 2024 01:33:03 +0200
Subject: [PATCH 5/6] refactor: move json message parsing to pulse_worker
---
git_hg_sync/pulse_worker.py | 21 ++++++++++++-
git_hg_sync/repo_synchronizer.py | 18 +----------
tests/test_pulse_worker.py | 40 ++++++++++++++++++++++++
tests/test_repo_synchronizer.py | 53 ++++----------------------------
4 files changed, 67 insertions(+), 65 deletions(-)
create mode 100644 tests/test_pulse_worker.py
diff --git a/git_hg_sync/pulse_worker.py b/git_hg_sync/pulse_worker.py
index 6b2ecc8..47c8319 100644
--- a/git_hg_sync/pulse_worker.py
+++ b/git_hg_sync/pulse_worker.py
@@ -1,9 +1,15 @@
from kombu.mixins import ConsumerMixin
from mozlog import get_proxy_logger
+from git_hg_sync.repo_synchronizer import Push, Tag
+
logger = get_proxy_logger("pluse_consumer")
+class EntityTypeError(Exception):
+ pass
+
+
class PulseWorker(ConsumerMixin):
def __init__(self, connection, queue, *, repo_synchronyzer):
@@ -11,6 +17,18 @@ def __init__(self, connection, queue, *, repo_synchronyzer):
self.task_queue = queue
self.repo_synchronyzer = repo_synchronyzer
+ @staticmethod
+ def parse_entity(raw_entity):
+ logger.debug(f"parse_entity: {raw_entity}")
+ message_type = raw_entity.pop("type")
+ match message_type:
+ case "push":
+ return Push(**raw_entity)
+ case "tag":
+ return Tag(**raw_entity)
+ case _:
+ raise EntityTypeError(f"unsupported type {message_type}")
+
def get_consumers(self, Consumer, channel):
consumer = Consumer(
self.task_queue, auto_declare=False, callbacks=[self.on_task]
@@ -21,4 +39,5 @@ def on_task(self, body, message):
logger.info(f"Received message: {body}")
message.ack()
raw_entity = body["payload"]
- self.repo_synchronyzer.sync(raw_entity)
+ parsed_message = PulseWorker.parse_entity(raw_entity)
+ self.repo_synchronyzer.sync(parsed_message)
diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py
index a496028..82b8883 100644
--- a/git_hg_sync/repo_synchronizer.py
+++ b/git_hg_sync/repo_synchronizer.py
@@ -7,10 +7,6 @@
logger = get_proxy_logger("sync_repo")
-class EntityTypeError(Exception):
- pass
-
-
@dataclass
class Push:
repo_url: str
@@ -38,17 +34,6 @@ class RepoSynchronyzer:
def __init__(self, repos_config):
self._repos_config = repos_config
- def parse_entity(self, raw_entity):
- logger.debug(f"parse_entity: {raw_entity}")
- message_type = raw_entity.pop("type")
- match message_type:
- case "push":
- return Push(**raw_entity)
- case "tag":
- return Tag(**raw_entity)
- case _:
- raise EntityTypeError(f"unsupported type {message_type}")
-
def get_remote(self, repo, remote_url):
"""
get the repo if it exists, create it otherwise
@@ -79,8 +64,7 @@ def handle_commits(self, entity, clone_dir, remote_url, remote_target):
remote.push()
logger.info(f"Done for entity {entity.pushid}")
- def sync(self, raw_entity):
- entity = self.parse_entity(raw_entity)
+ def sync(self, entity: Push | Tag) -> None:
repo_config = self._repos_config.get(entity.repo_url)
if not repo_config:
logger.warning(f"repo {entity.repo_url} is not supported yet")
diff --git a/tests/test_pulse_worker.py b/tests/test_pulse_worker.py
new file mode 100644
index 0000000..cb02c5c
--- /dev/null
+++ b/tests/test_pulse_worker.py
@@ -0,0 +1,40 @@
+import pytest
+
+from git_hg_sync.pulse_worker import PulseWorker, EntityTypeError
+from git_hg_sync.repo_synchronizer import Push, Tag
+
+def raw_push_entity():
+ return {
+ "type": "push",
+ "repo_url": "repo_url",
+ "heads": ["head"],
+ "commits": ["commit"],
+ "time": 0,
+ "pushid": 0,
+ "user": "user",
+ "push_json_url": "push_json_url",
+ }
+
+
+def raw_tag_entity():
+ return {
+ "type": "tag",
+ "repo_url": "repo_url",
+ "tag": "tag",
+ "commit": "commit",
+ "time": 0,
+ "pushid": 0,
+ "user": "user",
+ "push_json_url": "push_json_url",
+ }
+
+def test_parse_entity_valid():
+ push_entity = PulseWorker.parse_entity(raw_push_entity())
+ assert isinstance(push_entity, Push)
+ tag_entity = PulseWorker.parse_entity(raw_tag_entity())
+ assert isinstance(tag_entity, Tag)
+
+
+def test_parse_invalid_type():
+ with pytest.raises(EntityTypeError):
+ PulseWorker.parse_entity({"type": "unknown"})
diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py
index d1f0512..693627f 100644
--- a/tests/test_repo_synchronizer.py
+++ b/tests/test_repo_synchronizer.py
@@ -2,10 +2,8 @@
import pytest
-from git_hg_sync import __main__, repo_synchronizer
-
-HERE = Path(__file__).parent
-
+from git_hg_sync.__main__ import get_connection, get_queue
+from git_hg_sync.repo_synchronizer import RepoSynchronyzer, Push
@pytest.fixture
def pulse_config():
@@ -30,56 +28,17 @@ def repos_config():
}
}
-def raw_push_entity():
- return {
- "type": "push",
- "repo_url": "repo_url",
- "heads": ["head"],
- "commits": ["commit"],
- "time": 0,
- "pushid": 0,
- "user": "user",
- "push_json_url": "push_json_url",
- }
-
-
-def raw_tag_entity():
- return {
- "type": "tag",
- "repo_url": "repo_url",
- "tag": "tag",
- "commit": "commit",
- "time": 0,
- "pushid": 0,
- "user": "user",
- "push_json_url": "push_json_url",
- }
-
-
-def test_parse_entity():
- syncrepos = repo_synchronizer.RepoSynchronyzer(None)
- push_entity = syncrepos.parse_entity(raw_push_entity())
- assert isinstance(push_entity, repo_synchronizer.Push)
- tag_entity = syncrepos.parse_entity(raw_tag_entity())
- assert isinstance(tag_entity, repo_synchronizer.Tag)
-
-
-def test_sync_process_with_bad_type():
- syncrepos = repo_synchronizer.RepoSynchronyzer(None)
- with pytest.raises(repo_synchronizer.EntityTypeError):
- syncrepos.sync({"type": "badType"})
-
def test_sync_process_with_bad_repo(repos_config):
- syncrepos = repo_synchronizer.RepoSynchronyzer(repos_config=repos_config)
+ syncrepos = RepoSynchronyzer(repos_config=repos_config)
with pytest.raises(AssertionError) as e:
- syncrepos.sync(raw_push_entity())
+ syncrepos.sync(Push(repo_url="repo_url", heads=["head"], commits=["commits"], time=0, pushid=0, user="user", push_json_url="push_json_url"))
assert str(e.value) == f"clone {repos_config['repo_url']['clone']} doesn't exists"
def test_get_connection_and_queue(pulse_config):
- connection = __main__.get_connection(pulse_config)
- queue = __main__.get_queue(pulse_config)
+ connection = get_connection(pulse_config)
+ queue = get_queue(pulse_config)
assert connection.userid == pulse_config["userid"]
assert connection.host == f"{pulse_config['host']}:{pulse_config['port']}"
assert queue.name == pulse_config["queue"]
From 6774a1e3306e3a398461e0c37f0e7e46244ee4e8 Mon Sep 17 00:00:00 2001
From: Frank Bessou <frank.bessou@logilab.fr>
Date: Thu, 25 Jul 2024 01:58:57 +0200
Subject: [PATCH 6/6] style: run black
---
tests/conftest.py | 4 +---
tests/test_pulse_worker.py | 4 +++-
tests/test_repo_synchronizer.py | 13 ++++++++++++-
3 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/tests/conftest.py b/tests/conftest.py
index f48c0a8..347e4ed 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,10 +1,8 @@
import pytest
import mozlog
+
@pytest.fixture(autouse=True)
def mozlog_logging():
logger = mozlog.structuredlog.StructuredLogger("tests")
mozlog.structuredlog.set_default_logger(logger)
-
-
-
diff --git a/tests/test_pulse_worker.py b/tests/test_pulse_worker.py
index cb02c5c..b5d464b 100644
--- a/tests/test_pulse_worker.py
+++ b/tests/test_pulse_worker.py
@@ -1,7 +1,8 @@
import pytest
from git_hg_sync.pulse_worker import PulseWorker, EntityTypeError
-from git_hg_sync.repo_synchronizer import Push, Tag
+from git_hg_sync.repo_synchronizer import Push, Tag
+
def raw_push_entity():
return {
@@ -28,6 +29,7 @@ def raw_tag_entity():
"push_json_url": "push_json_url",
}
+
def test_parse_entity_valid():
push_entity = PulseWorker.parse_entity(raw_push_entity())
assert isinstance(push_entity, Push)
diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py
index 693627f..16c03dd 100644
--- a/tests/test_repo_synchronizer.py
+++ b/tests/test_repo_synchronizer.py
@@ -5,6 +5,7 @@
from git_hg_sync.__main__ import get_connection, get_queue
from git_hg_sync.repo_synchronizer import RepoSynchronyzer, Push
+
@pytest.fixture
def pulse_config():
return {
@@ -32,7 +33,17 @@ def repos_config():
def test_sync_process_with_bad_repo(repos_config):
syncrepos = RepoSynchronyzer(repos_config=repos_config)
with pytest.raises(AssertionError) as e:
- syncrepos.sync(Push(repo_url="repo_url", heads=["head"], commits=["commits"], time=0, pushid=0, user="user", push_json_url="push_json_url"))
+ syncrepos.sync(
+ Push(
+ repo_url="repo_url",
+ heads=["head"],
+ commits=["commits"],
+ time=0,
+ pushid=0,
+ user="user",
+ push_json_url="push_json_url",
+ )
+ )
assert str(e.value) == f"clone {repos_config['repo_url']['clone']} doesn't exists"