From e093e6965ab841c5d635c62ef3b988e920fc6a43 Mon Sep 17 00:00:00 2001 From: matttriano Date: Sun, 9 Feb 2025 00:14:48 -0600 Subject: [PATCH 1/4] Adds rss feed and comments out a print statement that printed unparsable characters that the RSS feed functionality couldn't render. --- _quarto.yml | 16 +++- .../install_code_security_audit.ipynb | 74 ++++--------------- .../setting_up_tailscale.ipynb | 20 ++--- 3 files changed, 39 insertions(+), 71 deletions(-) diff --git a/_quarto.yml b/_quarto.yml index be86df1..0444179 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -4,12 +4,26 @@ project: website: title: "Matt Triano" site-url: https://www.matttriano.dev - description: "Experiments, walkthroughs past-me would have liked, and gifts to future me" + description: "Notes and how-tos on data engineering, analysis, python, networking, experiments with hot new things, and more" navbar: right: - about.qmd - icon: github href: https://github.com/MattTriano + - icon: rss + href: index.xml +feed: + posts: + limit: 20 + feed-url: /index.xml + escape: true + items: + - title: true + - date: true + - description: + html: true + html-type: application/xhtml + length: 2000 format: html: theme: darkly diff --git a/posts/019_setup_tailscale_for_ssh/install_code_security_audit.ipynb b/posts/019_setup_tailscale_for_ssh/install_code_security_audit.ipynb index 4df94f8..b911899 100644 --- a/posts/019_setup_tailscale_for_ssh/install_code_security_audit.ipynb +++ b/posts/019_setup_tailscale_for_ssh/install_code_security_audit.ipynb @@ -47,7 +47,7 @@ "id": "639b5105-cae7-470d-902b-e7bfebe7219f", "metadata": {}, "source": [ - "### Tailscale Client Installation on a Ubuntu/Debian Linux Machine {.linux_install}\n", + "### Tailscale Client Installation on a Ubuntu/Debian Linux Machine\n", "\n", "```console\n", "curl -fsSL https://tailscale.com/install.sh | sh\n", @@ -80,7 +80,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Lines in install script: 626\n" + "Lines in install script: 627\n" ] } ], @@ -493,7 +493,7 @@ "\t\t\t\tVERSION=\"\" # rolling release\n", "\t\t\t\tPACKAGETYPE=\"pacman\"\n", "\t\t\t\t;;\n", - "\t\t\tmanjaro|manjaro-arm)\n", + "\t\t\tmanjaro|manjaro-arm|biglinux)\n", "\t\t\t\tOS=\"manjaro\"\n", "\t\t\t\tVERSION=\"\" # rolling release\n", "\t\t\t\tPACKAGETYPE=\"pacman\"\n", @@ -829,7 +829,8 @@ "\t\t\t;;\n", "\t\tfreebsd)\n", "\t\t\tif [ \"$VERSION\" != \"12\" ] && \\\n", - "\t\t\t [ \"$VERSION\" != \"13\" ]\n", + "\t\t\t [ \"$VERSION\" != \"13\" ] && \\\n", + "\t\t\t [ \"$VERSION\" != \"14\" ]\n", "\t\t\tthen\n", "\t\t\t\tOS_UNSUPPORTED=1\n", "\t\t\tfi\n", @@ -882,8 +883,7 @@ "\t\telse\n", "\t\t\techo \"No /etc/os-release\"\n", "\t\tfi\n", - "\t\texit 1\n", - "\tfi\n" + "\t\texit 1\n" ] } ], @@ -916,6 +916,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "\n", "\t# Step 3: work out if we can run privileged commands, and if so,\n", "\t# how.\n", "\tCAN_ROOT=\n", @@ -934,8 +935,7 @@ "\t\techo \"This installer needs to run commands as root.\"\n", "\t\techo \"We tried looking for 'sudo' and 'doas', but couldn't find them.\"\n", "\t\techo \"Either re-run this script as root, or set up sudo/doas.\"\n", - "\t\texit 1\n", - "\tfi\n" + "\t\texit 1\n" ] } ], @@ -1038,6 +1038,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "\n", "\t# Step 4: run the installation.\n", "\tOSVERSION=\"$OS\"\n", "\t[ \"$VERSION\" != \"\" ] && OSVERSION=\"$OSVERSION $VERSION\"\n", @@ -1142,7 +1143,7 @@ "\t\t\t;;\n", "\t\tpkg)\n", "\t\t\tset -x\n", - "\t\t\t$SUDO pkg install tailscale\n", + "\t\t\t$SUDO pkg install --yes tailscale\n", "\t\t\t$SUDO service tailscaled enable\n", "\t\t\t$SUDO service tailscaled start\n", "\t\t\tset +x\n", @@ -1180,8 +1181,7 @@ "\t\t*)\n", "\t\t\techo \"unexpected: unknown package type $PACKAGETYPE\"\n", "\t\t\texit 1\n", - "\t\t\t;;\n", - "\tesac\n" + "\t\t\t;;\n" ] } ], @@ -1263,54 +1263,7 @@ "\n", "\n", "Modern keyring gpg key (compressed, binary format)\n", - "Pʌ�g\u0006�|\u0004Bb���e-�A�d��<�\u0005�\u001c", - "�\u001afCJͳpc/���4���0� ��N<\u0012�Q��Fk&��\u001f���j�=,�70�� ����%�P��\u0006*T1�\u001f�\u0000^.C_�w/v\u0018x��#\"�7� `�k�?�F�^CA�.�\u0016����\u0003\u001c", - "�^�\\4�Dm\u000f��a��\u001f�'y��^�ɫo׍�z;\t\u0010^ꐦ�\u0001��Hȵ�嚍�\u0000s_�R�m#(҆��Q�W\u0013N�7U���\u0015'pP��<�\u0010�|\u0001#\"�aiU�\u0001�\u0011d�ݕ{\u001c", - "���\u0018\u0003Sλ(�[H���i�}x*ӟ6)�kL��\u0005tă�L5���M��\u0011�F7�3�#����^a!)�m� =\u0019!��\\jR��e�G�w7Lj���.�b��^��w�\n", - "$�j\"\\�\n", - "�h�%\u0014�nj�����H?\f", - "��u���*㚳���zߕC�\n", - "-=g2���\u0007�&/}���ňlvt�\u0004tِ����Ӗ��{�� J=\u0000\u0011\u0001\u0000\u0001�DTailscale Inc. (Package repository signing key) �\u0002N\u0004\u0013\u00008\u0016!\u0004%�����8!�<\n", - "yE��2�Xh\u0005\u0002^T��\u0002\u001b\u0003\u0005\u000b", - "\u0007\u0002\u0006\u0015\n", - "\u000b", - "\u0002\u0004\u0016\u0002\u0003\u0001\u0002\u001e", - "\u0001\u0002\u0017�\u0000\n", - "\u0004^T��\u0001\u0010\u0000�H��x]�X��\u0011���v�|��d'$UK'b6�\u0003�x�\f", - "/z�\u001be�z$.Rl2�M͒\u0013����\u0004\u0010�*���\u000b", - "N\u001d", - "��\u0017#�G�rGST�\u00057�.�\u0005��\u0014�ɥ\u0013���5�X9��4\u000f�\u0007\u0013k3ل��o���a7V�\f", - "�Jdݢ���գ\u0000jIZ>@d!�\u001d", - "�k�kU���ܫBBa�F$u�3\u001b��]H~!~a�a��xm�+{�s���j��CU��D�\u00191�=���r��\u0002�\f", - "��k����\u001e", - "}4�Ԃ\u0002�P\"k���\u0002\n", - "!#&\u0013Q�Y\"C�`\u0016�\u00067�Va�H�\u0012�K�����}��8x%��P:�ȰT�U���\"�f��x���b����2Pr�U\u0006�\u001456x(��Ĺ�k�X��(�!I�AO\u0007�Y��\u001d", - "����\u0004Jʀ�\"�3���\n", - "$L�P\u0006�\u0004\u000e0�_��ܸ��>A\n", - "�\u0002�w�g\u000b", - "��H�۷�jK$��\u0014��f�2kb��@�T�X�l\u001a\u0006�Ņ�i%{�o�y�\u0019%�@\u0014�q٘\u001f!ϸ�+��v\u0000�5X,\u0007Ť\n", - "j�\t�%��X�g��\u0002�;]_ֹ1\u000e1e�L�g�ه��J�\u0019A�2F����� h '��T\u0000\u0010�j�O�/Y�V�<��\u0002�\u00157��Q�\u001e", - "\u0000}'#\u0000\u0011\u0001\u0000\u0001�\u00026\u0004\u0018\u0000 \u0016!\u0004%�����8!�<\n", - "yE��2�Xh\u0005\u0002^T��\u0002\u001b\f", - "\u0000\n", - "�\u0014�\tMx�G Date: Mon, 10 Feb 2025 20:18:22 -0600 Subject: [PATCH 2/4] Adds test to check against rss-breaking conditions. --- tests/test_rss_feed.py | 179 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tests/test_rss_feed.py diff --git a/tests/test_rss_feed.py b/tests/test_rss_feed.py new file mode 100644 index 0000000..39f6bf3 --- /dev/null +++ b/tests/test_rss_feed.py @@ -0,0 +1,179 @@ +import xml.etree.ElementTree as ET +import requests +import subprocess +import sys +from pathlib import Path +from typing import Tuple + + +class RSSValidationError(Exception): + """Custom exception for RSS feed validation errors""" + + def __init__(self, message="RSS validation failed", details=None): + self.message = message + self.details = details or {} + super().__init__(self.message) + + def __str__(self): + if self.details: + return f"{self.message}: {self.details}" + return self.message + + +class CDATAError(RSSValidationError): + """Custom exception for CDATA-related errors""" + + def __init__(self, message="CDATA validation failed", details=None): + super().__init__(message, details) + + +class SpecialCharacterError(RSSValidationError): + """Custom exception for special character errors""" + + def __init__(self, message="Special character validation failed", details=None): + super().__init__(message, details) + + +class MissingRSSElementError(RSSValidationError): + """Custom exception for missing RSS element errors""" + + def __init__(self, message="Required RSS element check failed", details=None): + super().__init__(message, details) + + +def get_project_root() -> Path: + """Get the absolute path to the project root directory""" + return Path(__file__).parent.parent + + +def render_blog() -> None: + result = subprocess.run( + ["quarto", "render"], cwd=get_project_root(), capture_output=True, text=True + ) + if result.returncode != 0: + raise Exception(f"Unexpected error: {result.stderr}") + + +def check_rss_for_required_elements() -> bool: + try: + rss_path = get_project_root().joinpath("_site", "index.xml") + if not rss_path.exists(): + print(f"RSS feed not found at {rss_path}") + return False + tree = ET.parse(rss_path) + root = tree.getroot() + channel = root.find("channel") + if channel is None: + raise MissingRSSElementError("No channel element found in RSS feed") + required_elements = ["title", "link", "description"] + missing_elements = [] + for element in required_elements: + if channel.find(element) is None: + missing_elements.append(element) + if len(missing_elements) > 0: + raise MissingRSSElementError( + "Required elements missing from RSS feed", {"missing_elements": missing_elements} + ) + items = channel.findall("item") + if not items: + raise MissingRSSElementError("Required items missing from RSS feed") + for item in items: + missing_elements = [] + for element in ["title", "link"]: + if item.find(element) is None: + missing_items.append(element) + if len(missing_elements) > 0: + raise MissingRSSElementError( + "Required item-elements missing from RSS feed", + {"item": item, "missing_elements": missing_elements}, + ) + print("RSS feed validation successful!") + return True + except ET.ParseError as e: + print(f"XML parsing error: {e}") + return False + + +def check_cdata_sections(xml_content: str) -> bool: + """Check for properly closed CDATA sections""" + open_cdata = xml_content.count("") + if open_cdata != close_cdata: + raise RSSValidationError( + "CDATA sections mismatch", + { + "opens": open_cdata, + "closes": close_cdata, + "difference": abs(open_cdata - close_cdata), + }, + ) + return True + + +class BinaryContentError(RSSValidationError): + """Exception for binary or non-printable characters in content""" + + pass + + +def find_binary_characters(xml_content: str) -> list[Tuple[int, str, str]]: + """Find binary and non-printable characters in xml_content. + + Returns: + List of tuples containing (position, character, hex representation) + """ + binary_chars = [] + for i, char in enumerate(xml_content): + if ord(char) < 32 and char not in "\n\r\t": + binary_chars.append((i, char, f"\\u{ord(char):04x}")) # Unicode escape sequence + elif ord(char) == 0xFFFF or ord(char) == 0xFFFE: # Unicode BOM + binary_chars.append((i, char, f"\\u{ord(char):04x}")) + elif char == "\ufffd": # Unicode replacement character + binary_chars.append((i, char, "\\uFFFD (Unicode replacement character)")) + return binary_chars + + +def check_for_binary_content(xml_content: str) -> bool: + """Check for binary or non-printable characters in XML content""" + binary_chars = find_binary_characters(xml_content) + if binary_chars: + locations = [] + for pos, char, hex_repr in binary_chars: + start = max(0, pos - 20) + end = min(len(xml_content), pos + 20) + context = xml_content[start:end].replace("\n", "\\n") + locations.append( + {"position": pos, "character": hex_repr, "context": f"...{context}..."} + ) + raise BinaryContentError( + "Binary or non-printable characters found in XML content", + {"count": len(binary_chars), "locations": locations}, + ) + return True + + +def validate_rss_feed() -> bool: + """Build the site and validate the RSS feed""" + + render_blog() + try: + required_elements_present = check_rss_for_required_elements() + rss_path = get_project_root().joinpath("_site", "index.xml") + with open(rss_path, "r") as f: + content = f.read() + cdata_sections_valid = check_cdata_sections(content) + no_binary_content_found = check_for_binary_content(content) + return required_elements_present and cdata_sections_valid and no_binary_content_found + except Exception as e: + print(f"Unexpected error: {e}") + return False + + +def test_rss_feed(): + """Pytest function to test RSS feed""" + assert validate_rss_feed() + + +if __name__ == "__main__": + rss_feed_valid = validate_rss_feed() + sys.exit(0 if rss_feed_valid else 1) From 6c89de2989b314d0431f296838d5659f84734030 Mon Sep 17 00:00:00 2001 From: matttriano Date: Mon, 10 Feb 2025 20:29:58 -0600 Subject: [PATCH 3/4] Adds pre-commit hook and github-action to test rss. --- .githooks/pre-commit | 12 ++++++++++++ .github/workflows/test-rss.yml | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100755 .githooks/pre-commit create mode 100644 .github/workflows/test-rss.yml diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..fb02782 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,12 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "Testing RSS feed..." +cd "$PROJECT_ROOT" && python -m tests.test_rss_feed + +if [ $? -ne 0 ]; then + echo "RSS feed validation failed" + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/test-rss.yml b/.github/workflows/test-rss.yml new file mode 100644 index 0000000..640df24 --- /dev/null +++ b/.github/workflows/test-rss.yml @@ -0,0 +1,31 @@ +name: Test RSS Feed + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test-rss: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Test RSS feed + run: python test_rss_feed.py \ No newline at end of file From 0aef0218189845cc3d7a1492a44ec88a04c394f1 Mon Sep 17 00:00:00 2001 From: matttriano Date: Mon, 10 Feb 2025 20:39:53 -0600 Subject: [PATCH 4/4] Fixes erroneous invocation in github-action script. --- .github/workflows/test-rss.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-rss.yml b/.github/workflows/test-rss.yml index 640df24..c74d550 100644 --- a/.github/workflows/test-rss.yml +++ b/.github/workflows/test-rss.yml @@ -28,4 +28,4 @@ jobs: pip install requests - name: Test RSS feed - run: python test_rss_feed.py \ No newline at end of file + run: python -m tests.test_rss_feed \ No newline at end of file