diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 375583334..c02f45558 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -73,6 +73,9 @@ jobs: rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true + - name: add hpk42 key to staging server + run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys' + - name: run deploy-chatmail offline tests run: pytest --pyargs cmdeploy diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2f5bb6d..bd17c8505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- Remove echobot from relays + ([#753](https://github.com/chatmail/relay/pull/753)) + - Add robots.txt to exclude all web crawlers ([#732](https://github.com/chatmail/relay/pull/732)) diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index dd6bb11b2..7b4954b63 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chatmaild" -version = "0.2" +version = "0.3" dependencies = [ "aiosmtpd", "iniconfig", @@ -25,7 +25,6 @@ where = ['src'] doveauth = "chatmaild.doveauth:main" chatmail-metadata = "chatmaild.metadata:main" filtermail = "chatmaild.filtermail:main" -echobot = "chatmaild.echo:main" chatmail-metrics = "chatmaild.metrics:main" chatmail-expire = "chatmaild.expire:main" chatmail-fsreport = "chatmaild.fsreport:main" diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index c8762af8c..309553e4c 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -4,8 +4,6 @@ from chatmaild.user import User -echobot_password_path = Path("/run/echobot/password") - def read_config(inipath): assert Path(inipath).exists(), inipath @@ -72,10 +70,7 @@ def get_user(self, addr) -> User: raise ValueError(f"invalid address {addr!r}") maildir = self.mailboxes_dir.joinpath(addr) - if addr.startswith("echo@"): - password_path = echobot_password_path - else: - password_path = maildir.joinpath("password") + password_path = maildir.joinpath("password") return User(maildir, addr, password_path, uid="vmail", gid="vmail") diff --git a/chatmaild/src/chatmaild/doveauth.py b/chatmaild/src/chatmaild/doveauth.py index e6292a362..5914735fa 100644 --- a/chatmaild/src/chatmaild/doveauth.py +++ b/chatmaild/src/chatmaild/doveauth.py @@ -40,10 +40,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool: return False localpart, domain = parts - if localpart == "echo": - # echobot account should not be created in the database - return False - if ( len(localpart) > config.username_max_length or len(localpart) < config.username_min_length diff --git a/chatmaild/src/chatmaild/echo.py b/chatmaild/src/chatmaild/echo.py deleted file mode 100644 index c31701fdb..000000000 --- a/chatmaild/src/chatmaild/echo.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -"""Advanced echo bot example. - -it will echo back any message that has non-empty text and also supports the /help command. -""" - -import logging -import os -import subprocess -import sys -from pathlib import Path - -from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events - -from chatmaild.config import echobot_password_path, read_config -from chatmaild.doveauth import encrypt_password -from chatmaild.newemail import create_newemail_dict - -hooks = events.HookCollection() - - -@hooks.on(events.RawEvent) -def log_event(event): - if event.kind == EventType.INFO: - logging.info(event.msg) - elif event.kind == EventType.WARNING: - logging.warning(event.msg) - - -@hooks.on(events.RawEvent(EventType.ERROR)) -def log_error(event): - logging.error("%s", event.msg) - - -@hooks.on(events.MemberListChanged) -def on_memberlist_changed(event): - logging.info( - "member %s was %s", event.member, "added" if event.member_added else "removed" - ) - - -@hooks.on(events.GroupImageChanged) -def on_group_image_changed(event): - logging.info("group image %s", "deleted" if event.image_deleted else "changed") - - -@hooks.on(events.GroupNameChanged) -def on_group_name_changed(event): - logging.info(f"group name changed, old name: {event.old_name}") - - -@hooks.on(events.NewMessage(func=lambda e: not e.command)) -def echo(event): - snapshot = event.message_snapshot - if snapshot.is_info: - # Ignore info messages - return - if snapshot.text or snapshot.file: - snapshot.chat.send_message(text=snapshot.text, file=snapshot.file) - - -@hooks.on(events.NewMessage(command="/help")) -def help_command(event): - snapshot = event.message_snapshot - snapshot.chat.send_text("Send me any message and I will echo it back") - - -def main(): - logging.basicConfig(level=logging.INFO) - path = os.environ.get("PATH") - venv_path = sys.argv[0].strip("echobot") - os.environ["PATH"] = path + ":" + venv_path - with Rpc() as rpc: - deltachat = DeltaChat(rpc) - system_info = deltachat.get_system_info() - logging.info(f"Running deltachat core {system_info.deltachat_core_version}") - - accounts = deltachat.get_all_accounts() - account = accounts[0] if accounts else deltachat.add_account() - - bot = Bot(account, hooks) - - config = read_config(sys.argv[1]) - addr = "echo@" + config.mail_domain - - # Create password file - if bot.is_configured(): - password = bot.account.get_config("mail_pw") - else: - password = create_newemail_dict(config)["password"] - - echobot_password_path.write_text(encrypt_password(password)) - # Give the user which doveauth runs as access to the password file. - subprocess.check_call( - ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path], - ) - - if not bot.is_configured(): - bot.configure(addr, password) - - # write invite link to working directory - invitelink = bot.account.get_qr_code() - Path("invite-link.txt").write_text(invitelink) - - bot.run_forever() - - -if __name__ == "__main__": - main() diff --git a/chatmaild/src/chatmaild/tests/test_lastlogin.py b/chatmaild/src/chatmaild/tests/test_lastlogin.py index 44e304f59..a7e9e348e 100644 --- a/chatmaild/src/chatmaild/tests/test_lastlogin.py +++ b/chatmaild/src/chatmaild/tests/test_lastlogin.py @@ -36,29 +36,3 @@ def test_handle_dovecot_request_last_login(testaddr, example_config): res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions) assert res == "O\n" assert len(dictproxy_transactions) == 0 - - -def test_handle_dovecot_request_last_login_echobot(example_config): - dictproxy = LastLoginDictProxy(config=example_config) - - authproxy = AuthDictProxy(config=example_config) - testaddr = f"echo@{example_config.mail_domain}" - authproxy.lookup_passdb(testaddr, "ignore") - user = dictproxy.config.get_user(testaddr) - - transactions = {} - - # set last-login info for user - tx = "1111" - msg = f"B{tx}\t{testaddr}" - res = dictproxy.handle_dovecot_request(msg, transactions) - assert not res - assert transactions == {tx: dict(addr=testaddr, res="O\n")} - - timestamp = int(time.time()) - msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}" - res = dictproxy.handle_dovecot_request(msg, transactions) - assert not res - assert len(transactions) == 1 - read_timestamp = user.get_last_login_timestamp() - assert read_timestamp is None diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index 6e5f7c7ef..a6344e4d6 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -109,15 +109,6 @@ def run_cmd(args, out): try: retcode = out.check_call(cmd, env=env) if retcode == 0: - if not args.disable_mail: - print("\nYou can try out the relay by talking to this echo bot: ") - sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose) - print( - sshexec( - call=remote.rshell.shell, - kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"), - ) - ) out.green("Deploy completed, call `cmdeploy dns` next.") elif not remote_data["acme_account_url"]: out.red("Deploy completed but letsencrypt not configured") diff --git a/cmdeploy/src/cmdeploy/deployers.py b/cmdeploy/src/cmdeploy/deployers.py index 35aaca186..4686e4c11 100644 --- a/cmdeploy/src/cmdeploy/deployers.py +++ b/cmdeploy/src/cmdeploy/deployers.py @@ -270,6 +270,14 @@ def install(self): path="/var/log/journal/", present=False, ) + # remove echobot if it is still running + if host.get_fact(SystemdEnabled).get("echobot.service"): + systemd.service( + name="Disable echobot.service", + service="echobot.service", + running=False, + enabled=False, + ) def check_config(config): @@ -404,30 +412,6 @@ def activate(self): self.need_restart = False -class EchobotDeployer(Deployer): - # - # This deployer depends on the dovecot and postfix deployers because - # it needs to base its decision of whether to restart the service on - # whether those two services were restarted. - # - def __init__(self, mail_domain): - self.mail_domain = mail_domain - self.units = ["echobot"] - - def install(self): - apt.packages( - # required for setfacl for echobot - name="Install acl", - packages="acl", - ) - - def configure(self): - configure_remote_units(self.mail_domain, self.units) - - def activate(self): - activate_remote_units(self.units) - - class ChatmailVenvDeployer(Deployer): def __init__(self, config): self.config = config @@ -590,7 +574,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: PostfixDeployer(config, disable_mail), FcgiwrapDeployer(), NginxDeployer(config), - EchobotDeployer(mail_domain), MtailDeployer(config.mtail_address), GithashDeployer(), ] diff --git a/cmdeploy/src/cmdeploy/service/echobot.service.f b/cmdeploy/src/cmdeploy/service/echobot.service.f deleted file mode 100644 index 5117b3940..000000000 --- a/cmdeploy/src/cmdeploy/service/echobot.service.f +++ /dev/null @@ -1,67 +0,0 @@ -[Unit] -Description=Chatmail echo bot for testing it works - -[Service] -ExecStart={execpath} {config_path} -Environment="PATH={remote_venv_dir}:$PATH" -Restart=always -RestartSec=30 - -User=echobot -Group=echobot - -# Create /var/lib/echobot -StateDirectory=echobot - -# Create /run/echobot -# -# echobot stores /run/echobot/password -# with a password there, which doveauth then reads. -RuntimeDirectory=echobot - -WorkingDirectory=/var/lib/echobot - -# Apply security restrictions suggested by -# systemd-analyze security echobot.service -CapabilityBoundingSet= -LockPersonality=true -MemoryDenyWriteExecute=true -NoNewPrivileges=true -PrivateDevices=true -PrivateMounts=true -PrivateTmp=true - -# We need to know about doveauth user to give it access to /run/echobot/password -PrivateUsers=false - -ProtectClock=true -ProtectControlGroups=true -ProtectHostname=true -ProtectKernelLogs=true -ProtectKernelModules=true -ProtectKernelTunables=true -ProtectProc=noaccess - -# Should be "strict", but we currently write /accounts folder in a protected path -ProtectSystem=full - -RemoveIPC=true -RestrictAddressFamilies=AF_INET AF_INET6 -RestrictNamespaces=true -RestrictRealtime=true -RestrictSUIDSGID=true -SystemCallArchitectures=native -SystemCallFilter=~@clock -SystemCallFilter=~@cpu-emulation -SystemCallFilter=~@debug -SystemCallFilter=~@module -SystemCallFilter=~@mount -SystemCallFilter=~@obsolete -SystemCallFilter=~@raw-io -SystemCallFilter=~@reboot -SystemCallFilter=~@resources -SystemCallFilter=~@swap -UMask=0077 - -[Install] -WantedBy=multi-user.target diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 4dc6b6ef5..2b55f6bf3 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -81,7 +81,6 @@ def test_status_cmd(chatmail_config, capsys, request): "chatmail-metadata", "doveauth", "dovecot", - "echobot", "fcgiwrap", "filtermail-incoming", "filtermail", diff --git a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py index ca6815f43..e45acec9b 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py @@ -160,22 +160,3 @@ def test_hide_senders_ip_address(cmfactory): user2.direct_imap.select_folder("Inbox") msg = user2.direct_imap.get_all_messages()[0] assert public_ip not in msg.obj.as_string() - - -def test_echobot(cmfactory, chatmail_config, lp, sshdomain): - ac = cmfactory.get_online_accounts(1)[0] - - # establish contact with echobot - sshexec = SSHExec(sshdomain) - command = "cat /var/lib/echobot/invite-link.txt" - echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command)) - chat = ac.qr_setup_contact(echo_invite_link) - ac._evtracker.wait_securejoin_joiner_progress(1000) - - # send message and check it gets replied back - lp.sec("Send message to echobot") - text = "hi, I hope you text me back" - chat.send_text(text) - lp.sec("Wait for reply from echobot") - reply = ac._evtracker.wait_next_incoming_message() - assert reply.text == text diff --git a/cmdeploy/src/cmdeploy/www.py b/cmdeploy/src/cmdeploy/www.py index aa4371336..e0b3a3c8a 100644 --- a/cmdeploy/src/cmdeploy/www.py +++ b/cmdeploy/src/cmdeploy/www.py @@ -166,7 +166,7 @@ def main(): build_webpages(src_path, build_dir, config) print(f"[{changenum}] regenerated web pages at: {index_path}") print(f"URL: file://{index_path.resolve()}\n\n") - + time.sleep(debounce_time) # simple debounce diff --git a/doc/source/migrate.rst b/doc/source/migrate.rst index 4086744f9..d88f363fc 100644 --- a/doc/source/migrate.rst +++ b/doc/source/migrate.rst @@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended. or receive messages until the migration is completed. 2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``, - ``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to + ``/etc/dkimkeys``, and ``/var/spool/postfix`` to the new site. Login to the old site while forwarding your SSH agent so you can copy directly from the old to the new site with your SSH key: @@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended. :: ssh -A root@13.37.13.37 - tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /" + tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /" - This transfers all addresses, the TLS certificate, DKIM keys (so DKIM - DNS record remains valid), and the echobot’s password so it continues - to function. It also preserves the Postfix mail spool so any messages + This transfers all addresses, the TLS certificate, + and DKIM keys (so DKIM DNS record remains valid). + It also preserves the Postfix mail spool so any messages pending delivery will still be delivered. 3. Install chatmail on the new machine: @@ -58,7 +58,6 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended. chown root: -R /var/lib/acme chown opendkim: -R /etc/dkimkeys chown vmail: -R /home/vmail/mail - chown echobot: -R /run/echobot 5. Now, update DNS entries. diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 09bcbcad7..e878db804 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -109,10 +109,6 @@ short overview of ``chatmaild`` services: is contacted by Dovecot when a user logs in and stores the date of the login. -- `echobot `_ - is a small bot for test purposes. It simply echoes back messages from - users. - - `metrics `_ collects some metrics and displays them at ``https://example.org/metrics``. @@ -276,8 +272,8 @@ by OpenDKIM screen policy script before validating the signatures. This corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``). If there is no valid DKIM signature on the incoming email, the sender receives a “5.7.1 No valid DKIM signature found” error. -After validating the DKIM signature, -the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc. +After validating the DKIM signature, +the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc. Note that chatmail relays