From 4eba81084e06727c9f8c757722b8ba4690765cb8 Mon Sep 17 00:00:00 2001 From: Max Dymond Date: Mon, 4 Mar 2024 14:13:22 +0000 Subject: [PATCH] Initial commit --- .flake8 | 3 + .funcignore | 8 + .gitignore | 531 ++++++++++------------------------------ .vscode/extensions.json | 6 + .vscode/launch.json | 12 + .vscode/settings.json | 9 + .vscode/tasks.json | 27 ++ README.md | 59 ++++- SUPPORT.md | 38 +-- create_resources.sh | 111 +++++++++ function_app.py | 218 +++++++++++++++++ host.json | 15 ++ mypy.ini | 4 + requirements.txt | 10 + rg.bicep | 167 +++++++++++++ rg_add_eventgrid.bicep | 72 ++++++ 16 files changed, 858 insertions(+), 432 deletions(-) create mode 100644 .flake8 create mode 100644 .funcignore create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json create mode 100755 create_resources.sh create mode 100644 function_app.py create mode 100644 host.json create mode 100644 mypy.ini create mode 100644 requirements.txt create mode 100644 rg.bicep create mode 100644 rg_add_eventgrid.bicep diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..8dd399a --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203 diff --git a/.funcignore b/.funcignore new file mode 100644 index 0000000..9966315 --- /dev/null +++ b/.funcignore @@ -0,0 +1,8 @@ +.git* +.vscode +__azurite_db*__.json +__blobstorage__ +__queuestorage__ +local.settings.json +test +.venv \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8a30d25..74fc765 100644 --- a/.gitignore +++ b/.gitignore @@ -1,398 +1,135 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.tlog -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files -*.ncb -*.aps - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) +# Byte-compiled / optimized / DLL files __pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider -*.sln.iml +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don’t work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Azure Functions artifacts +bin +obj +appsettings.json +local.settings.json + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json +.python_packages diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..3f63eb9 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "ms-azuretools.vscode-azurefunctions", + "ms-python.python" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..ea3e0f1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Python Functions", + "type": "python", + "request": "attach", + "port": 9091, + "preLaunchTask": "func: host start" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..60e70c2 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "azureFunctions.deploySubpath": ".", + "azureFunctions.scmDoBuildDuringDeployment": true, + "azureFunctions.pythonVenv": ".venv", + "azureFunctions.projectLanguage": "Python", + "azureFunctions.projectRuntime": "~4", + "debug.internalConsoleOptions": "neverOpen", + "azureFunctions.projectLanguageModel": 2 +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..a450213 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,27 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "func", + "label": "func: host start", + "command": "host start", + "problemMatcher": "$func-python-watch", + "isBackground": true, + "dependsOn": "pip install (functions)" + }, + { + "label": "pip install (functions)", + "type": "shell", + "osx": { + "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" + }, + "windows": { + "command": "${config:azureFunctions.pythonVenv}/Scripts/python -m pip install -r requirements.txt" + }, + "linux": { + "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" + }, + "problemMatcher": [] + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 5cd7cec..0fe2caa 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,53 @@ -# Project +# apt-package-function -> This repo has been populated by an initial template to help get you started. Please -> make sure to update the content to build a great experience for community-building. +Functionality to create a Debian package repository in Azure Blob Storage with +an Azure Function App to keep it up to date. For use with +[apt-transport-blob](https://github.com/microsoft/apt-transport-blob). -As the maintainer of this project, please make a few updates: +# Getting Started -- Improving this README.MD file to provide a great experience -- Updating SUPPORT.MD with content about this project's support experience -- Understanding the security reporting process in SECURITY.MD -- Remove this section from the README +To create a new Debian package repository with an Azure Function App, run + +```bash +./create_resources.sh +``` + +with the name of the desired resource group. The scripting will autogenerate a +package repository name for you - `debianrepo` followed by a unique string to +differentiate it across Azure. + +If you wish to control the suffix used, you can pass the `-s` parameter: + +```bash +./create_resources.sh -s +``` +which will attempt to create a storage container named `debianrepo`. + +By default all resources are created in the `uksouth` location - this can be +overridden by passing the `-l` parameter: + +```bash +./create_resources.sh -l eastus +``` + +# Design + +The function app works as follows: + +- It is triggered whenever a `.deb` file is uploaded to the monitored blob + storage container + - It can be triggered by both blob storage triggers and by Event Grid triggers +- It iterates over all `.deb` files and looks for a matching `.package` file. +- If that file does not exist, it is created + - The `.deb` file is downloaded and the control information is extracted + - The hash values for the file are calculated (MD5sum, SHA1, SHA256) + - All of this information is added to the `.package` file +- All `.package` files are iterated over, downloaded, and combined into a + single `Package` file, which is then uploaded. + +As the function app works on a Consumption plan it may take up to 10 minutes for +the function app to trigger and regenerate the package information. In practice, +the eventGridTrigger is triggered very quickly. ## Contributing @@ -26,8 +65,8 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio ## Trademarks -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft -trademarks or logos is subject to and must follow +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/SUPPORT.md b/SUPPORT.md index 291d4d4..e9e71a7 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,25 +1,13 @@ -# TODO: The maintainer of this repo has not yet edited this file - -**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? - -- **No CSS support:** Fill out this template with information about how to file issues and get help. -- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. -- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. - -*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* - -# Support - -## How to file issues and get help - -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or -feature request as a new Issue. - -For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE -FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER -CHANNEL. WHERE WILL YOU HELP PEOPLE?**. - -## Microsoft Support Policy - -Support for this **PROJECT or PRODUCT** is limited to the resources listed above. +# Support + +## How to file issues and get help + +This project uses [GitHub Issues](https://github.com/microsoft/apt-package-function/issues) to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please use the [GitHub Discussions](https://github.com/microsoft/apt-package-function/discussions) feature. + +## Microsoft Support Policy + +Support for this project is limited to the resources listed above. diff --git a/create_resources.sh b/create_resources.sh new file mode 100755 index 0000000..a97db85 --- /dev/null +++ b/create_resources.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail + +# This script uses Bicep scripts to create a function app and a storage account, +# then uses the Azure CLI to deploy the function code to that app. + +LOCATION="eastus" + +function usage() +{ + echo "Usage: $0 [-l ] [-s ] " + echo + echo "By default, location is '${LOCATION}'" + echo "A list of location names can be obtained by running 'az account list-locations --query \"[].name\"'" +} + +PARAMETERS="" + +while getopts ":l:s:" opt; do + case "${opt}" in + l) + LOCATION=${OPTARG} + ;; + s) + PARAMETERS="${PARAMETERS} --parameter suffix=${OPTARG}" + ;; + *) + usage + exit 0 + ;; + esac +done +shift $((OPTIND-1)) + +# Takes parameters of the resource group name. +RESOURCE_GROUP_NAME=${1:-} + +if [[ -z ${RESOURCE_GROUP_NAME} ]] +then + echo "Requires a resource group name" + echo + usage + exit 1 +fi + +echo "Ensuring resource group ${RESOURCE_GROUP_NAME} exists" +az group create --name "${RESOURCE_GROUP_NAME}" --location "${LOCATION}" --output none + +# Create the resources +DEPLOYMENT_NAME="${RESOURCE_GROUP_NAME}" +echo "Creating resources in resource group ${RESOURCE_GROUP_NAME}" +az deployment group create \ + --name "${DEPLOYMENT_NAME}" \ + --resource-group "${RESOURCE_GROUP_NAME}" \ + --template-file ./rg.bicep \ + ${PARAMETERS} \ + --output none +echo "Resources created" + +# There's some output in the deployment that we need. +APT_SOURCES=$(az deployment group show -n "${DEPLOYMENT_NAME}" -g "${RESOURCE_GROUP_NAME}" --output tsv --query properties.outputs.apt_sources.value) +FUNCTION_APP_NAME=$(az deployment group show -n "${DEPLOYMENT_NAME}" -g "${RESOURCE_GROUP_NAME}" --output tsv --query properties.outputs.function_app_name.value) +STORAGE_ACCOUNT=$(az deployment group show -n "${DEPLOYMENT_NAME}" -g "${RESOURCE_GROUP_NAME}" --output tsv --query properties.outputs.storage_account.value) +PACKAGE_CONTAINER=$(az deployment group show -n "${DEPLOYMENT_NAME}" -g "${RESOURCE_GROUP_NAME}" --output tsv --query properties.outputs.package_container.value) + +# Zip up the functionapp code +mkdir -p build/ +rm -f build/function_app.zip +zip -r build/function_app.zip host.json requirements.txt function_app.py + +# Deploy the function code +echo "Deploying function app code to ${FUNCTION_APP_NAME}" +az functionapp deployment source config-zip \ + --resource-group "${RESOURCE_GROUP_NAME}" \ + --name "${FUNCTION_APP_NAME}" \ + --src build/function_app.zip \ + --build-remote true \ + --output none +echo "Function app code deployed" + +# Clean up +rm -f build/function_app.zip + +# Now run the second deployment script to create the eventgrid subscription. +# This must be run after the function app is deployed, because the ARM ID of the +# eventGridTrigger function doesn't exist until after deployment. +az deployment group create \ + --name "${DEPLOYMENT_NAME}_eg" \ + --resource-group "${RESOURCE_GROUP_NAME}" \ + --template-file ./rg_add_eventgrid.bicep \ + ${PARAMETERS} \ + --output none + +# Report to the user how to use this repository +echo "The repository has been created!" +echo "You can upload packages to the container '${PACKAGE_CONTAINER}' in the storage account '${STORAGE_ACCOUNT}'." +echo "The function app '${FUNCTION_APP_NAME}' will be triggered by new packages" +echo "in that container and regenerate the repository." +echo +echo "To download packages, you need to have apt-transport-blob installed on your machine." +echo "Next, add this line to /etc/apt/sources.list:" +echo +echo " ${APT_SOURCES}" +echo +echo "Ensure that you have a valid Azure credential, (either by logging in with 'az login' or " +echo "by setting the AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID environment variables)." +echo "That credential must have 'Storage Blob Data Reader' access to the storage account." +echo "Then you can use apt-get update and apt-get install as usual." diff --git a/function_app.py b/function_app.py new file mode 100644 index 0000000..fb26b33 --- /dev/null +++ b/function_app.py @@ -0,0 +1,218 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""A function app to manage a Debian repository in Azure Blob Storage.""" + +import contextlib +import io +import logging +import lzma +import os +import tempfile +from pathlib import Path + +import azure.functions as func +import pydpkg +from azure.storage.blob import ContainerClient + +app = func.FunctionApp() +log = logging.getLogger("apt-package-function") +log.addHandler(logging.NullHandler()) + +# Turn down logging for azure functions +logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel( + logging.WARNING +) + +CONTAINER_NAME = os.environ["BLOB_CONTAINER"] +DEB_CHECK_KEY = "DebLastModified" + + +@contextlib.contextmanager +def temporary_filename(): + """Create a temporary file and return the filename.""" + try: + with tempfile.NamedTemporaryFile(delete=False) as f: + temporary_name = f.name + yield temporary_name + finally: + os.unlink(temporary_name) + + +class PackageBlob: + """A class to manage a Debian package in a storage account.""" + + def __init__(self, container_client: ContainerClient, name: str) -> None: + """Create a PackageBlob object.""" + self.path = Path(name) + + # Get a Blob Client for the given name + self.blob_client = container_client.get_blob_client(name) + self.package_properties = self.blob_client.get_blob_properties() + self.last_modified = str(self.package_properties.last_modified) + + # Create a Blob Client for the metadata file + self.metadata_path = self.path.with_suffix(".package") + self.metadata_blob_client = container_client.get_blob_client( + str(self.metadata_path) + ) + + def check(self) -> None: + """Check the package and metadata file.""" + log.info("Checking package: %s", self.path) + + # Check if the metadata file exists and if it doesn't, create it + if not self.metadata_blob_client.exists(): + log.error("Metadata file missing for: %s", self.path) + self.create_metadata() + return + + # The metadata file exists. First, check the BlobProperties metadata + # to make sure that the LastModified time of the package is the same as + # the LastModified metadata variable on the metadata file. + metadata_properties = self.metadata_blob_client.get_blob_properties() + + if DEB_CHECK_KEY not in metadata_properties.metadata: + log.error("Metadata file missing DebLastModified for: %s", self.path) + self.create_metadata() + return + + if self.last_modified != metadata_properties.metadata[DEB_CHECK_KEY]: + log.error("Metadata file out of date for: %s", self.path) + self.create_metadata() + return + + def create_metadata(self) -> None: + """Create the metadata file for the package.""" + log.info("Creating metadata file for: %s", self.path) + + # Get a temporary filename to work with + with temporary_filename() as temp_filename: + # Download the package to the temporary file + with open(temp_filename, "wb") as f: + stream = self.blob_client.download_blob() + f.write(stream.readall()) + + # Now with the package on disc, load it with pydpkg. + pkg = pydpkg.Dpkg(temp_filename) + + # Construct the metadata file, which is: + # - the data in the control file + # - the filename + # - the MD5sum of the package + # - the SHA1 of the package + # - the SHA256 of the package + # - the size of the package + contents = f"""{pkg.control_str.rstrip()} +Filename: {self.path} +MD5sum: {pkg.md5} +SHA1: {pkg.sha1} +SHA256: {pkg.sha256} +Size: {pkg.filesize} + +""" + # Log the metadata information + log.info("Metadata info for %s: %s", self.path, contents) + + # Upload the metadata information to the metadata file. Make sure + # the DebLastModified metadata variable is set to the LastModified + # time of the package. + self.metadata_blob_client.upload_blob( + contents, + metadata={DEB_CHECK_KEY: self.last_modified}, + overwrite=True, + ) + + +class RepoManager: + """A class which manages a Debian repository in a storage account.""" + + def __init__(self) -> None: + """Create a RepoManager object.""" + self.connection_string = os.environ["AzureWebJobsStorage"] + self.container_client = ContainerClient.from_connection_string( + self.connection_string, CONTAINER_NAME + ) + self.package_file = self.container_client.get_blob_client("Packages") + self.package_file_xz = self.container_client.get_blob_client("Packages.xz") + + def check_metadata(self) -> None: + """Iterate over the packages and check the metadata file.""" + # Get the list of all blobs in the container + blobs = self.container_client.list_blobs() + + # Get all of the Debian packages + for blob in blobs: + if not blob.name.endswith(".deb"): + continue + + # Create a PackageBlob object and check it + pb = PackageBlob(self.container_client, blob.name) + pb.check() + + def create_packages(self) -> None: + """Iterate over all metadata files to create a Packages file.""" + # Get the list of all blobs in the container + blobs = self.container_client.list_blobs() + + # Get all of the metadata files + packages_stream = io.BytesIO() + + for blob in blobs: + if not blob.name.endswith(".package"): + continue + + log.info("Processing metadata file: %s", blob.name) + + # Get the contents of the metadata file + metadata_blob_client = self.container_client.get_blob_client(blob.name) + num_bytes = metadata_blob_client.download_blob().readinto(packages_stream) + log.info("Read %d bytes from %s", num_bytes, blob.name) + + # The stream now contains all of the metadata files. + # Read out as bytes + packages_stream.seek(0) + packages_bytes = packages_stream.read() + + # Upload the data to the Packages file + self.package_file.upload_blob(packages_bytes, overwrite=True) + log.info("Created Packages file") + + # Compress the Packages file using lzma and then upload it to the + # Packages.xz file + compressed_data = lzma.compress(packages_bytes) + self.package_file_xz.upload_blob(compressed_data, overwrite=True) + log.info("Created Packages.xz file") + + +@app.blob_trigger( + arg_name="newfile", + path=f"{CONTAINER_NAME}/{{name}}.deb", + connection="AzureWebJobsStorage", +) +def blob_trigger(newfile: func.InputStream): + """Process a new blob in the container.""" + # Have to use %s for the length because .length is optional + log.info( + "Python blob trigger function processed blob; Name: %s, Blob Size: %s bytes", + newfile.name, + newfile.length, + ) + if not newfile.name or not newfile.name.endswith(".deb"): + log.info("Not a Debian package: %s", newfile.name) + return + + rm = RepoManager() + rm.check_metadata() + rm.create_packages() + log.info("Done processing %s", newfile.name) + + +@app.function_name(name="eventGridTrigger") +@app.event_grid_trigger(arg_name="event") +def eventGridTrigger(event: func.EventGridEvent): + """Process an event grid trigger for a new blob in the container""" + log.info("Processing event %s", event.id) + rm = RepoManager() + rm.check_metadata() + rm.create_packages() + log.info("Done processing event %s", event.id) diff --git a/host.json b/host.json new file mode 100644 index 0000000..f2b7c0d --- /dev/null +++ b/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.0.0, 5.0.0)" + } +} \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..45eaa87 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,4 @@ +[mypy] + +[mypy-pydpkg.*] +ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fa47559 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure-storage-blob +pydpkg diff --git a/rg.bicep b/rg.bicep new file mode 100644 index 0000000..1624389 --- /dev/null +++ b/rg.bicep @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This file creates all the resource group scope resources +targetScope = 'resourceGroup' + +@description('Unique suffix') +param suffix string = uniqueString(resourceGroup().id) + +@description('The location of the resources') +param location string = resourceGroup().location + +@description('The name of the function app to use') +param appName string = 'debfnapp${suffix}' + +// Storage account names must be between 3 and 24 characters, and unique, so +// generate a unique name. +@description('The name of the storage account to use') +param storage_account_name string = 'debianrepo${suffix}' + +// Choose the package container name. This will be passed to the function app. +var package_container_name = 'packages' + +// The version of Python to run with +var python_version = '3.11' + +// The name of the hosting plan, application insights, and function app +var functionAppName = appName +var hostingPlanName = appName +var applicationInsightsName = appName + +// Create a storage account for both package storage and function app storage +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: storage_account_name + location: location + kind: 'StorageV2' + sku: { + name: 'Standard_LRS' + } + properties: { + publicNetworkAccess: 'Enabled' + } +} + +// Create a container for the packages +resource defBlobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { + parent: storageAccount + name: 'default' +} +resource packageContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = { + parent: defBlobServices + name: package_container_name + properties: { + } +} + +// Create a default Packages file if it doesn't exist using a deployment script +resource deploymentScript 'Microsoft.Resources/deploymentScripts@2023-08-01' = { + name: 'createPackagesFile${suffix}' + location: location + kind: 'AzureCLI' + properties: { + azCliVersion: '2.28.0' + retentionInterval: 'PT1H' + environmentVariables: [ + { + name: 'AZURE_STORAGE_ACCOUNT' + value: storageAccount.name + } + { + name: 'AZURE_BLOB_CONTAINER' + value: packageContainer.name + } + { + name: 'AZURE_STORAGE_KEY' + secureValue: storageAccount.listKeys().keys[0].value + } + ] + // This script preserves the Packages file if it exists and creates it + // if it does not. + scriptContent: ''' +az storage blob download -f Packages -c "${AZURE_BLOB_CONTAINER}" -n Packages || echo "No existing file" +touch Packages +az storage blob upload -f Packages -c "${AZURE_BLOB_CONTAINER}" -n Packages + ''' + cleanupPreference: 'OnSuccess' + } +} + +// Create a hosting plan for the function app +resource hostingPlan 'Microsoft.Web/serverfarms@2023-01-01' = { + name: hostingPlanName + location: location + sku: { + name: 'Y1' + tier: 'Dynamic' + } + properties: { + reserved: true + } +} + +// Create application insights +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: applicationInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + Request_Source: 'rest' + } +} + +// Create the function app. +resource functionApp 'Microsoft.Web/sites@2023-01-01' = { + name: functionAppName + location: location + kind: 'functionapp,linux' + properties: { + serverFarmId: hostingPlan.id + siteConfig: { + linuxFxVersion: 'Python|${python_version}' + pythonVersion: python_version + appSettings: [ + { + name: 'AzureWebJobsStorage' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'WEBSITE_CONTENTSHARE' + value: toLower(functionAppName) + } + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~4' + } + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: applicationInsights.properties.InstrumentationKey + } + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: 'python' + } + // Pass the blob container name to the function app - this is the + // container which is monitored for new packages. + { + name: 'BLOB_CONTAINER' + value: packageContainer.name + } + ] + ftpsState: 'FtpsOnly' + minTlsVersion: '1.2' + } + httpsOnly: true + } +} + +// Create the apt sources string for using apt-transport-blob +output apt_sources string = 'deb [trusted=yes] blob://${storageAccount.name}.blob.core.windows.net/${packageContainer.name} /' +output function_app_name string = functionApp.name +output storage_account string = storageAccount.name +output package_container string = packageContainer.name diff --git a/rg_add_eventgrid.bicep b/rg_add_eventgrid.bicep new file mode 100644 index 0000000..7c2f679 --- /dev/null +++ b/rg_add_eventgrid.bicep @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This file adds an event grid subscription to a storage account that triggers +// when a blob is added +targetScope = 'resourceGroup' + +@description('Unique suffix') +param suffix string = uniqueString(resourceGroup().id) + +@description('The location of the resources') +param location string = resourceGroup().location + +@description('The name of the function app to use') +param appName string = 'debfnapp${suffix}' + +@description('The name of the storage account to use') +param storage_account_name string = 'debianrepo${suffix}' + +var package_container_name = 'packages' + +// Define all existing resources +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = { + name: storage_account_name +} +resource defBlobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' existing = { + parent: storageAccount + name: 'default' +} +resource packageContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' existing = { + parent: defBlobServices + name: package_container_name +} +resource functionApp 'Microsoft.Web/sites@2023-01-01' existing = { + name: appName +} + +// Now create an event grid subscription so that when a blob is created, +// it triggers the function app +resource systemTopic 'Microsoft.EventGrid/systemTopics@2021-12-01' = { + name: 'blobscreated${suffix}' + location: location + properties: { + source: storageAccount.id + topicType: 'Microsoft.Storage.StorageAccounts' + } +} + +// Construct the resource ID to use for the event subscription +var functionId = '${functionApp.id}/functions/eventGridTrigger' + +resource eventSubscription 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2023-12-15-preview' = { + parent: systemTopic + name: 'blobscreated${suffix}' + properties: { + destination: { + endpointType: 'AzureFunction' + properties: { + resourceId: functionId + maxEventsPerBatch: 1 + } + } + filter: { + includedEventTypes: [ + 'Microsoft.Storage.BlobCreated' + ] + // Filter in Debian files only + subjectBeginsWith: '/blobServices/default/containers/${packageContainer.name}/' + subjectEndsWith: '.deb' + } + } +}