extract backend in a proper pip package

.github/workflows/main.yml

@@ -26,8 +26,12 @@ jobs:
       matrix:
         platform: [ubuntu-latest, macos-latest]  #, windows-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
-        # vvv just an example of excluding stuff from matrix
-        # exclude: [{platform: macos-latest, python-version: '3.6'}]
+        exclude: [
+          # 3.8 and 3.9 aren't available on latest macos M1 runners
+          # see https://github.com/actions/setup-python/issues/808
+          {platform: macos-latest, python-version: '3.8'},
+          {platform: macos-latest, python-version: '3.9'},
+        ]
 
     runs-on: ${{ matrix.platform }}
 
@@ -57,6 +61,37 @@ jobs:
         path: .coverage.mypy/
 
 
+  pypi:
+    runs-on: ubuntu-latest
+    needs: [build] # add all other jobs here
+
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: 'release to test pypi'
+      # always deploy merged master to test pypi
+      if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
+      env:
+        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
+      run: pip3 install --user --upgrade build twine && .ci/release --test
+
+    - name: 'release to pypi'
+      # always deploy tags to release pypi
+      # NOTE: release tags are guarded by on: push: tags on the top
+      if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      run: pip3 install --user --upgrade build twine && .ci/release
+
   ###
   build_extension:
     env:
@@ -93,3 +128,4 @@ jobs:
       with:
         name : '${{ env.name }}-firefox-release-latest.zip'
         path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'
+

README.md

@@ -9,12 +9,18 @@ possibly selected text, additional comments or tags and adds it into your [Org M
 
 # Running
 In the simplest setup, the server runs locally, and you can use 'localhost' version of the extension. If you have to work on a computer where you can't run python scripts,
-or your target capture file is just not there, you can selfhost the server part elsewhere and use the 'any host' version. Don't forget to set the endpoint in extension settings!
+or your target capture file is just not there, you can selfhost the server part elsewhere. Don't forget to set the endpoint in extension settings!
 
-1. Install server counterpart as systemd service (to autostart it): `server/setup --path /path/to/your/capture.org [--port <custom port>] [--template <custom org-capture template>]`.
+## Setup
+- clone the repository and cd into the checkout directory
+- install `grasp_backend` package: `pip3 install --user git+https://github.com/karlicoss/grasp.git`
+- install systemd/launchd service to autorun grasp
+
+  `python3 -m grasp_backend setup --path /path/to/your/capture.org [--port <custom port>] [--template <custom org-capture template>]`
+
+  Or alternatively, just run it directly if you don't want to autostart `python3 -m grasp_backend serve --path /path/to/your/capture.org [--port <custom port>] [--template <custom org-capture template>]`
 
-    Or alternatively, just run it directly if you don't want to autostart it: `server/grasp_server.py --path /path/to/your/capture.org [--port <custom_port>] [--template <custom org-capture template>]`.
-2. Install chrome extension and configure hotkeys
+- install chrome extension and configure hotkeys
 
 That's it! If you're using custom port make sure it's the same as in the extension settings (default is `12212`).
 
@@ -23,7 +29,7 @@ That's it! If you're using custom port make sure it's the same as in the extensi
 [Here](https://github.com/karlicoss/grasp/blob/af24c991579986cec73695daa8318e7831049305/server/org_tools.py#L91-L109) you can find some references for the `--template` syntax.
 
 If you are looking for more flexible formatting that's not supported by template syntax, see [config.py.example](misc/config.py.example).
-You can modify it to your liking and pass as `--config` to `grasp_server/setup` scripts.
+You can modify it to your liking and pass as `--config` to `grasp_backend setup` command.
 
 # Motivation
 Why use org-capture? Well, it's hard to explain, maybe some other time... However, if you do know you want to use it instead of/alongside your browser bookmarks, by default

conftest.py

@@ -0,0 +1,38 @@
+# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly
+# without it, pytest can't discover the package root for some reason
+# also see https://github.com/karlicoss/pytest_namespace_pkgs for more
+
+import pathlib
+from typing import Optional
+
+import _pytest.main
+import _pytest.pathlib
+
+# we consider all dirs in repo/ to be namespace packages
+root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src'
+assert root_dir.exists(), root_dir
+
+# TODO assert it contains package name?? maybe get it via setuptools..
+
+namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()]
+
+# resolve_package_path is called from _pytest.pathlib.import_path
+# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem
+resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
+def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+    result = path  # search from the test file upwards
+    for parent in result.parents:
+        if str(parent) in namespace_pkg_dirs:
+            return parent
+    raise RuntimeError("Couldn't determine path for ", path)
+_pytest.pathlib.resolve_package_path = resolve_package_path
+
+
+# without patching, the orig function returns just a package name for some reason
+# (I think it's used as a sort of fallback)
+# so we need to point it at the absolute path properly
+# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
+search_pypath_orig = _pytest.main.search_pypath
+def search_pypath(module_name: str) -> str:
+    return str(root_dir)
+_pytest.main.search_pypath = search_pypath

pyproject.toml

@@ -0,0 +1,50 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+[project]
+dynamic = ["version"]  # version is managed by setuptools_scm
+name = "grasp_backend"
+dependencies = []
+requires-python = ">=3.8"
+
+## these need to be set if you're planning to upload to pypi
+# description = "TODO"
+license = {file = "LICENSE.md"}
+authors = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+]
+maintainers = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+]
+# keywords = []
+# # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers
+# classifiers = [
+# ]
+
+
+[project.urls]
+Homepage = "https://github.com/karlicoss/grasp"
+##
+
+
+[project.optional-dependencies]
+testing = [
+    "pytest",
+    "mypy",
+    "lxml",  # for mypy html coverage
+    "ruff",
+
+    "requests",
+
+    # end2end tests:
+    "selenium",
+    "loguru",
+    "click",
+]
+
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+version_scheme = "python-simplified-semver"
+local_scheme = "dirty-tag"

server/grasp_server.py

@@ -1,164 +1,16 @@
 #!/usr/bin/env python3
-import argparse
-from http.server import BaseHTTPRequestHandler, HTTPServer
-import json
-import os
-import logging
-from pathlib import Path
-import re
-from typing import List, Optional, Dict, Any
-
-from org_tools import as_org, empty, DEFAULT_TEMPLATE, Config
-
-CAPTURE_PATH_VAR     = 'GRASP_CAPTURE_PATH'
-CAPTURE_TEMPLATE_VAR = 'GRASP_CAPTURE_TEMPLATE'
-CAPTURE_CONFIG_VAR   = 'GRASP_CAPTURE_CONFIG'
-
-
-def get_logger():
-    return logging.getLogger('grasp-server')
-
-
-def append_org(
-        path: Path,
-        org: str
-):
-    logger = get_logger()
-    # TODO perhaps should be an error?...
-    if not path.exists():
-        logger.warning("path %s didn't exist!", path)
-    # https://stackoverflow.com/a/13232181
-    if len(org.encode('utf8')) > 4096:
-        logger.warning("writing out %s might be non-atomic", org)
-    with path.open('a') as fo:
-        fo.write(org)
-
-
-from functools import lru_cache
-@lru_cache(1)
-def capture_config() -> Optional[Config]:
-    cvar = os.environ.get(CAPTURE_CONFIG_VAR)
-    if cvar is None:
-        return None
-
-    globs: Dict[str, Any] = {}
-    exec(Path(cvar).read_text(), globs)
-    ConfigClass = globs['Config']
-    return ConfigClass()
-
-
-def capture(
-        url: str,
-        title,
-        selection,
-        comment,
-        tag_str,
-):
-    logger = get_logger()
-    # protect strings against None
-    def safe(s: Optional[str]) -> str:
-        if s is None:
-            return ''
-        else:
-            return s
-    capture_path = Path(os.environ[CAPTURE_PATH_VAR]).expanduser()
-    org_template = os.environ[CAPTURE_TEMPLATE_VAR]
-    config = capture_config()
-    logger.info('capturing %s to %s', (url, title, selection, comment, tag_str), capture_path)
-
-    url = safe(url)
-    title = safe(title)
-    selection = safe(selection)
-    comment = safe(comment)
-    tag_str = safe(tag_str)
-
-    tags: List[str] = []
-    if not empty(tag_str):
-        tags = re.split(r'[\s,]', tag_str)
-        tags = [t for t in tags if not empty(t)] # just in case
+import warnings
+warnings.warn("This way of running grasp is deprecated! Please refer to readme and install it as a pip package")
 
-    org = as_org(
-        url=url,
-        title=title,
-        selection=selection,
-        comment=comment,
-        tags=tags,
-        org_template=org_template,
-        config=config,
-    )
-    append_org(
-        path=capture_path,
-        org=org,
-    )
-
-    response = {
-        'path': str(capture_path),
-        'status': 'ok',
-    }
-    return json.dumps(response).encode('utf8')
-
-
-class GraspRequestHandler(BaseHTTPRequestHandler):
-    def handle_POST(self):
-        logger = get_logger()
-
-        content_length = int(self.headers['Content-Length'])
-        post_data = self.rfile.read(content_length)
-        payload = json.loads(post_data.decode('utf8'))
-        logger.info("incoming request %s", payload)
-        res = capture(**payload)
-        self.send_response(200)
-        self.send_header('Content-Type', 'application/json')
-        self.end_headers()
-        self.wfile.write(res)
-
-    def respond_error(self, message: str):
-        self.send_response(500)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(message.encode('utf8'))
-
-    def do_POST(self):
-        logger = get_logger()
-        try:
-            self.handle_POST()
-        except Exception as e:
-            logger.error("Error during processing")
-            logger.exception(e)
-            self.respond_error(message=str(e))
-
-
-def run(port: str, capture_path: str, template: str, config: Optional[Path]):
-    logger = get_logger()
-    logger.info("Using template %s", template)
-
-    # not sure if there is a simpler way to communicate with the server...
-    os.environ[CAPTURE_PATH_VAR] = capture_path
-    os.environ[CAPTURE_TEMPLATE_VAR] = template
-    if config is not None:
-        os.environ[CAPTURE_CONFIG_VAR] = str(config)
-    httpd = HTTPServer(('', int(port)), GraspRequestHandler)
-    logger.info(f"Starting httpd on port {port}")
-    httpd.serve_forever()
-
-
-def setup_parser(p):
-    p.add_argument('--port', type=str, default='12212', help='Port for communicating with extension')
-    p.add_argument('--path', type=str, default='~/capture.org', help='File to capture into')
-    p.add_argument('--template', type=str, default=DEFAULT_TEMPLATE, help=f"""
-    {as_org.__doc__}
-    """)
-    abspath = lambda p: str(Path(p).absolute())
-    p.add_argument('--config', type=abspath, required=False, help='Optional dynamic config')
-
-
-def main():
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
+from pathlib import Path
 
-    p = argparse.ArgumentParser('grasp server', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=100))
-    setup_parser(p)
-    args = p.parse_args()
-    run(args.port, args.path, args.template, args.config)
+SRC_DIR = Path(__file__).absolute().parent.parent / 'src'
+assert SRC_DIR.exists(), SRC_DIR
 
-if __name__ == '__main__':
-    main()
+import os
+import sys
+os.chdir(SRC_DIR)
+os.execvp(
+    sys.executable,
+    [sys.executable, '-m', 'grasp_backend', 'serve', *sys.argv[1:]]
+)