first pass at updates

azure/Makefile

@@ -4,46 +4,50 @@ export LOCATION?=eastus2
 export MACHINE_NAME?=paas
 export DNS_NAME:=$(MACHINE_NAME)-$(RESOURCE_GROUP)
 export FQDN:=$(DNS_NAME).$(LOCATION).cloudapp.azure.com
+export SSH_PORT:=22
 export ADMIN_USERNAME?=$(notdir $(HOME))
 export TIMESTAMP:=$(shell date "+%Y-%m-%d-%H-%M-%S")
 export STORAGE_SUFFIX:=$(shell date "+%m%d%H")
 export STORAGE_ACCOUNT_NAME:=$(MACHINE_NAME)diag$(STORAGE_SUFFIX)
 export VM_SIZE:=Standard_B2ms
+export OS_DISK_SIZE:=32
+export OS_DISK_SKU:=StandardSSD_LRS
+export VM_IMAGE:=Canonical:0001-com-ubuntu-minimal-jammy:minimal-22_04-lts-gen2:latest
 
 
 # Permanent local overrides
 -include .env
+.DEFAULT_GOAL := help
 
-# dump resource groups
-resources:
+list-resources: ## Dump resource groups
 	az group list --output table
 
-# Dump list of location IDs
-locations:
+list-locations: ## Dump list of location IDs
 	az account list-locations --output table
 
-sizes:
+list-sizes: ## Dump list of VM sizes in the selected location
 	az vm list-sizes --location=$(LOCATION) --output table
 
-images:
-	az vm image list-skus --output table
+list-image-offers: ## Dump list of OS image offers
+	az vm image list-offers --location=$(LOCATION) --publisher Canonical --output table
 
-# Create a resource group and deploy the cluster resources inside it
+list-image-skus: ## Dump list of OS image SKUs
+	az vm image list-skus --location=$(LOCATION) --publisher Canonical --offer 0001-com-ubuntu-server-jammy --output table --output table 
 
-preflight:
+preflight: ## Create the empty resource group
 	az group create \
 		--name $(RESOURCE_GROUP) \
 		--location $(LOCATION) \
 		--output table 
 
-deploy-storage:
+deploy-storage: ## Deploy a storage account for diagnostics and restic backups
 	az storage account create \
 		--name $(STORAGE_ACCOUNT_NAME) \
 		--resource-group $(RESOURCE_GROUP) \
 		--kind StorageV2 \
 		--sku Standard_LRS
 
-deploy-network:
+deploy-network: ## Deploy the network and open network security group ports
 	az network nsg create \
 		--name $(MACHINE_NAME) \
 		--resource-group $(RESOURCE_GROUP) \
@@ -54,7 +58,7 @@ deploy-network:
 		--access Allow \
 		--direction Inbound \
 		--protocol TCP \
-		--destination-port-ranges 22 \
+		--destination-port-ranges $(SSH_PORT) \
 		--nsg-name $(MACHINE_NAME) \
 		--resource-group $(RESOURCE_GROUP) \
 	    --output table
@@ -79,58 +83,59 @@ deploy-network:
 		--resource-group $(RESOURCE_GROUP) \
 		--output table
 
-deploy-compute:
+deploy-compute: ## Deploy the VM only
 	az vm create \
 		--name $(MACHINE_NAME) \
 		--os-disk-name $(MACHINE_NAME) \
-		--os-disk-size-gb 32 \
+		--os-disk-size-gb $(OS_DISK_SIZE) \
+		--storage-sku $(OS_DISK_SKU) \
 		--size $(VM_SIZE) \
 		--admin-username $(ADMIN_USERNAME) \
 		--ssh-key-value @$(HOME)/.ssh/id_rsa.pub \
 		--public-ip-address-dns-name $(DNS_NAME) \
 		--boot-diagnostics-storage $(STORAGE_ACCOUNT_NAME) \
 		--custom-data @cloud-init.yml \
-		--image Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest \
+		--image $(VM_IMAGE) \
 		--resource-group $(RESOURCE_GROUP) \
 		--nsg $(MACHINE_NAME) \
 		--output table \
 		--no-wait
 
-# Destroy the entire resource group and all cluster resources
-destroy-all:
+destroy-all: ## Destroy the entire resource group
 	az group delete \
 		--name $(RESOURCE_GROUP) \
 		--no-wait \
 		--yes
 
-deploy:
+deploy: ## *START HERE* Deploy all the resources
 	make preflight
 	make deploy-storage
 	make deploy-network
 	make deploy-compute
 
-redeploy:
+redeploy: ## Destroy and redeploy [WARNING: this is for testing only!]
 	-make destroy-all
 	while [[ $$(az group list | grep Deleting) =~ "Deleting" ]]; do sleep 30; done
 	make deploy
 
-# View deployment details
-view-deployment:
+view-deployment: ## View deployment details
 	az group deployment operation list \
 		--resource-group $(RESOURCE_GROUP) \
 		--query "[].{OperationID:operationId,Name:properties.targetResource.resourceName,Type:properties.targetResource.resourceType,State:properties.provisioningState,Status:properties.statusCode}" \
 		--output table
 
 # Do not output warnings, do not validate or add remote host keys (useful when doing successive deployments or going through a load balancer)
-ssh:
-	ssh -q -A $(ADMIN_USERNAME)@$(FQDN) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+ssh: ## SSH into the VM (disabling strict host checking, for quick testing)
+	ssh -q -A $(ADMIN_USERNAME)@$(FQDN) -p $(SSH_PORT) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
 
-# List endpoints
-list-endpoints:
+list-endpoints: ## List open endpoints
 	az network public-ip list \
 		--resource-group $(RESOURCE_GROUP) \
 		--query '[].{dnsSettings:dnsSettings.fqdn}' \
 		--output table
 
-set:
+help: ## This help
+	@grep -hE '^[A-Za-z0-9_ \-]*?:.*##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+set: ## Dump current environment variables
 	set

azure/README.md

@@ -21,7 +21,7 @@
 ## Caveats
 
 * This would best be done as an Azure Resource Manager template, but using the CLI alone allows for easier experimentation and provides a more readable example
-* This will deploy everything on Ubuntu LTS 20.04 (which is one of the core distributions we test [`piku`][piku] against).
+* This will deploy everything on Ubuntu LTS 22.04 (which is one of the core distributions we test [`piku`][piku] against).
 
 [az]: http://azure.microsoft.com/
 [azcli]: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli

azure/cloud-init.yml

@@ -1,7 +1,7 @@
 #cloud-config
 
 package_upgrade: true
-# This assumes Ubuntu 20.04 - most Debian 10+ packages are identical, but not all.
+# This assumes Ubuntu 22.04 - most Debian 11 packages are identical, but not all.
 packages:
   - ntp
   - tmux
@@ -31,33 +31,34 @@ packages:
   - libffi-dev 
   - libsasl2-dev
   - libldap2-dev
-  # Clojure (will bring in OpenJDK 11)
-  - leiningen
   # Piku stack, plus helpers
   - git
   - certbot
-  - incron
   - nginx
   - python3-certbot-nginx
   - python3-dev
   - python3-pip
   - python3-click
   - python3-virtualenv
-  - python-is-python3
   - uwsgi
   - uwsgi-plugin-python3
   - uwsgi-plugin-asyncio-python3
   - uwsgi-plugin-gevent-python3
   - uwsgi-plugin-greenlet-python3
   - uwsgi-plugin-tornado-python3
-  - uwsgi-plugin-jwsgi-openjdk-11
+  # Clojure (will bring in OpenJDK 11)
+  # disabled by default to save space
+  #- leiningen
+  #- uwsgi-plugin-jwsgi-openjdk-11
   - uwsgi-plugin-php
   - uwsgi-plugin-lua5.1
   - uwsgi-plugin-lua5.2
   - uwsgi-plugin-luajit
   # Optional services (Ubuntu default packages bind these to localhost, be wary of exposed ports if using other distros)
+  # - mosquitto
   # - redis
   # - mongodb-server
+  # - postgresql
 
 write_files:
   - path: /etc/nginx/sites-available/default
@@ -74,9 +75,43 @@ write_files:
       }
       include /home/piku/.piku/nginx/*.conf;
 
-  - path: /etc/incron.d/paas
+  - path: /etc/uwsgi/apps-enabled/piku.ini
+    content: |
+      [uwsgi]
+      chdir = /home/piku/.piku/uwsgi
+      emperor = /home/piku/.piku/uwsgi-enabled
+      log-maxsize = 1048576
+      logto = /home/piku/.piku/uwsgi/uwsgi.log
+      log-backupname = /home/piku/.piku/uwsgi/uwsgi.old.log
+      socket = /home/piku/.piku/uwsgi/uwsgi.sock
+      uid = piku
+      gid = www-data
+      enable-threads = true
+      threads = 2
+
+  - path: /etc/systemd/system/piku-nginx.path
     content: |
-      /home/piku/.piku/nginx IN_MODIFY,IN_NO_LOOP /bin/systemctl reload nginx
+      [Unit]
+      Description=Monitor .piku/nginx for changes
+
+      [Path]
+      PathChanged=/home/piku/.piku/nginx
+      Unit=piku-nginx.service
+
+      [Install]
+      WantedBy=multi-user.target
+  
+  - path: /etc/systemd/system/piku-nginx.service
+    content: |
+      [Unit]
+      Description=Restarts nginx when .piku/nginx changes.
+
+      [Service]
+      Type=simple
+      ExecStart=/bin/systemctl reload nginx
+
+      [Install]
+      WantedBy=multi-user.target
   
   # If you change the SSH port, don't forget to edit this accordingly
   #- path: /etc/fail2ban/jail.d/defaults-debian.conf
@@ -85,19 +120,19 @@ write_files:
   #    enabled = true
   #    port = 22
 
+  # Ensure we have inotify working (some of my apps require it)
+  - path: /etc/sysctl.conf
+    append: true
+    content: |
+        fs.inotify.max_user_watches=1048576
+
   # Have the Azure agent set up a swapfile for us
   - path: /etc/waagent.conf
     content: |
         ResourceDisk.Format=y
         ResourceDisk.EnableSwap=y
         ResourceDisk.SwapSizeMB=2048
 
-  # Ensure we have inotify working (some of my apps require it)
-  - path: /etc/sysctl.conf
-    append: true
-    content: |
-        fs.inotify.max_user_watches=1048576
-
 runcmd:
   # Adjust to taste
   - timedatectl set-timezone Europe/Lisbon
@@ -106,16 +141,9 @@ runcmd:
   # Now we can add the Piku user and do the remaining setup
   - adduser --disabled-password --gecos 'PaaS access' --ingroup www-data piku
   - su - piku -c "wget https://raw.githubusercontent.com/piku/piku/master/piku.py && python3 ~/piku.py setup"
-  - ln /home/piku/.piku/uwsgi/uwsgi.ini /etc/uwsgi/apps-enabled/piku.ini
-  - su - piku -c "python3 ~/piku.py setup:ssh /tmp/id_rsa.pub"
-  # Remove snaps (primarily useful in smaller VMs, but DO NOT DO THIS IN ORACLE CLOUD since they use snap to install the monitoring agent)
+  - su - piku -c "python ~/piku.py setup:ssh /tmp/id_rsa.pub"
+  # Remove snaps (primarily useful in smaller VMs)
   - sudo apt-get purge snapd -y
-  # Set up swap on Oracle Cloud
-  #- fallocate -l 2G /swapfile
-  #- chmod 600 /swapfile
-  #- mkswap /swapfile
-  #- swapon /swapfile
-  #- echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
   # Install nodesource LTS version
   - curl -sL https://deb.nodesource.com/setup_lts.x | sudo -E bash -
   - apt-get install -y nodejs

baremetal-ubuntu/cloud-init.yml

@@ -61,7 +61,8 @@ packages:
   - uwsgi-plugin-gevent-python3
   - uwsgi-plugin-greenlet-python3
   - uwsgi-plugin-tornado-python3
-  - uwsgi-plugin-jwsgi-openjdk-11
+  # disabled by default to save space
+  #- uwsgi-plugin-jwsgi-openjdk-11
   - uwsgi-plugin-php
   - uwsgi-plugin-lua5.1
   - uwsgi-plugin-lua5.2
@@ -70,9 +71,10 @@ packages:
   # - mosquitto
   # - redis
   # - mongodb-server
+  # - postgresql
 
 write_files:
-  - path: /etc/nginx/sites-available/default
+- path: /etc/nginx/sites-available/default
     content: |
       server {
           listen 80 default_server;
@@ -86,9 +88,43 @@ write_files:
       }
       include /home/piku/.piku/nginx/*.conf;
 
-  - path: /etc/incron.d/paas
+  - path: /etc/uwsgi/apps-enabled/piku.ini
     content: |
-      /home/piku/.piku/nginx IN_MODIFY,IN_NO_LOOP /bin/systemctl reload nginx
+      [uwsgi]
+      chdir = /home/piku/.piku/uwsgi
+      emperor = /home/piku/.piku/uwsgi-enabled
+      log-maxsize = 1048576
+      logto = /home/piku/.piku/uwsgi/uwsgi.log
+      log-backupname = /home/piku/.piku/uwsgi/uwsgi.old.log
+      socket = /home/piku/.piku/uwsgi/uwsgi.sock
+      uid = piku
+      gid = www-data
+      enable-threads = true
+      threads = 2
+
+  - path: /etc/systemd/system/piku-nginx.path
+    content: |
+      [Unit]
+      Description=Monitor .piku/nginx for changes
+
+      [Path]
+      PathChanged=/home/piku/.piku/nginx
+      Unit=piku-nginx.service
+
+      [Install]
+      WantedBy=multi-user.target
+  
+  - path: /etc/systemd/system/piku-nginx.service
+    content: |
+      [Unit]
+      Description=Restarts nginx when .piku/nginx changes.
+
+      [Service]
+      Type=simple
+      ExecStart=/bin/systemctl reload nginx
+
+      [Install]
+      WantedBy=multi-user.target
   
   # If you change the SSH port, don't forget to edit this accordingly
   #- path: /etc/fail2ban/jail.d/defaults-debian.conf