feat: state based queue

closes #7

README.md

@@ -25,8 +25,15 @@ k3d cluster create devcluster \
 
 ```
 docker volume create local_registry
-docker container run -d --name registry.localhost -v local_registry:/var/lib/registry --restart always -p 5000:5000 registry:2
+docker container run -d \
+  --name registry.localhost \
+  -e "REGISTRY_STORAGE_DELETE_ENABLED=true" \
+  -p 5000:5000 \
+  -v local_registry:/var/lib/registry \
+  --restart always \
+  registry:2
 docker network connect k3d-devcluster registry.localhost
+docker login registry.localhost:5000 -u admin -p password
 ```
 
 ### Add Helm Repo

erpnext_feature_board/clients/container_registry.py

@@ -0,0 +1,52 @@
+import json
+import os
+
+import dxf
+import frappe
+from erpnext_feature_board.clients.k8s import get_container_registry
+
+
+def get_registry_client(repo=None):
+	auth = get_container_registry_auth()
+	is_dev_mode = True if frappe.get_conf().get("developer_mode") else False
+
+	dxf_client = dxf.DXF(
+		get_container_registry(),
+		auth=auth,
+		repo=repo,
+		insecure=is_dev_mode,
+		tlsverify=not is_dev_mode,
+	)
+	dxf_client.authenticate(authorization=auth, actions=["*"])
+
+	return dxf_client
+
+
+def get_container_registry_auth():
+	config = {}
+	home = os.environ.get("HOME", "/home/frappe")
+	docker_config = os.path.join(home, ".docker", "config.json")
+
+	with open(docker_config, encoding="utf-8") as f:
+		config = json.load(f)
+
+	return "Basic " + (
+		config.get("auths", {})
+		.get(get_container_registry(), {})
+		.get("auth", "YWRtaW46cGFzc3dvcmQ=")
+	)
+
+
+def delete_image_tag(image_name, tag_name):
+	try:
+		dxf_client = get_registry_client(image_name)
+		res = dxf_client.del_alias(tag_name)
+		return res
+	except Exception as e:
+		out = {
+			"error": e,
+			"params": {"image_name": image_name, "tag_name": tag_name},
+		}
+
+		frappe.log_error(out, "delete_image_tag")
+		return out

erpnext_feature_board/k8s.py → erpnext_feature_board/clients/k8s.py

@@ -182,23 +182,22 @@ def get_job_status(job_name):
 		return out
 
 
-def create_helm_release(improvement_name):
-	improvement = frappe.get_doc("Improvement", improvement_name)
-	site_password = frappe.generate_hash(length=10)
-
-	# Save site password
-	# improvement.site_password = site_password
-	# improvement.save()
-	# frappe.db.commit()
-
+def create_helm_release(improvement_name, site_name, site_password):
 	db_root_user = frappe.get_conf().get("db_root_user", "root")
 	db_root_password = frappe.get_conf().get("db_root_password", "admin")
 	mariadb_host = frappe.get_conf().get(
 		"mariadb_host", "mariadb.mariadb.svc.cluster.local"
 	)
+	redis_queue_host = frappe.get_conf().get(
+		"redis_queue_host", "redis-master.redis.svc.cluster.local:6379/0"
+	)
+	redis_cache_host = frappe.get_conf().get(
+		"redis_cache_host", "redis-master.redis.svc.cluster.local:6379/1"
+	)
+	redis_socketio_host = frappe.get_conf().get(
+		"redis_socketio_host", "redis-master.redis.svc.cluster.local:6379/2"
+	)
 	install_apps = frappe.get_conf().get("install_apps", "erpnext")
-	domain_name = frappe.get_conf().get("domain_name", "test-erpnext.org")
-	site_name = improvement_name.lower() + "." + domain_name
 	load_config()
 	crd = client.CustomObjectsApi()
 	body = {
@@ -281,9 +280,7 @@ def create_helm_release(improvement_name):
 
 
 def update_helm_release(improvement_name):
-	improvement = frappe.get_doc("Improvement", improvement_name)
-
-	migration_timestamp = str(time.time())
+	migration_timestamp = str(round(time.time()))
 	load_config()
 	crd = client.CustomObjectsApi()
 	body = {
@@ -302,7 +299,6 @@ def update_helm_release(improvement_name):
 			"helmreleases",
 			improvement_name.lower(),
 			body,
-			pretty=True,
 		)
 		return to_dict(res)
 	except (ApiException, Exception) as e:
@@ -465,3 +461,31 @@ def delete_job(job_name):
 
 		frappe.log_error(out, "Exception: BatchV1Api->delete_namespaced_job")
 		return out
+
+
+def get_helm_release(improvement_name):
+	load_config()
+	crd = client.CustomObjectsApi()
+	try:
+		res = crd.get_namespaced_custom_object(
+			"helm.fluxcd.io",
+			"v1",
+			get_namespace(),
+			"helmreleases",
+			improvement_name,
+		)
+		return to_dict(res)
+	except (ApiException, Exception) as e:
+		out = {
+			"error": e,
+			"function_name": "get_helm_release",
+			"params": {"improvement_name": improvement_name},
+		}
+		reason = getattr(e, "reason", None)
+		if reason:
+			out["reason"] = reason
+
+		frappe.log_error(
+			out, "Exception: CustomObjectsApi->get_namespaced_custom_object"
+		)
+		return out

erpnext_feature_board/erpnext_feature_board/doctype/improvement/improvement.json

@@ -9,6 +9,7 @@
  "field_order": [
   "repository",
   "naming_series",
+  "deployment_status",
   "cb_improvement",
   "route",
   "sb_pull_request",
@@ -22,7 +23,11 @@
   "source_branch",
   "target_branch",
   "sb_debugging",
-  "raw_data"
+  "raw_data",
+  "sb_site_details",
+  "site_url",
+  "cb_site_details",
+  "site_admin_password"
  ],
  "fields": [
   {
@@ -119,13 +124,41 @@
    "fieldtype": "Data",
    "label": "Target Branch",
    "read_only": 1
+  },
+  {
+   "fieldname": "deployment_status",
+   "fieldtype": "Select",
+   "label": "Deployment Status",
+   "options": "\nReady\nBuild Queued\nBuild Queue Failed\nBuilding\nBuild Complete\nBuild Error\nRelease Queued\nRelease Queue Failed\nRelease Error\nUpgrade Queued\nUpgrade Queued Failed\nRebuilding\nRebuild Error\nRebuild Complete\nUpgrading\nUpgrading Failed\nUpgrade Error\nDelete Queued\nDelete Queue Failed\nRelease Deleted",
+   "read_only": 1
+  },
+  {
+   "fieldname": "sb_site_details",
+   "fieldtype": "Section Break",
+   "label": "Site Details"
+  },
+  {
+   "fieldname": "site_url",
+   "fieldtype": "Data",
+   "label": "Site URL",
+   "read_only": 1
+  },
+  {
+   "fieldname": "cb_site_details",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "site_admin_password",
+   "fieldtype": "Data",
+   "label": "Site Admin Password",
+   "read_only": 1
   }
  ],
  "has_web_view": 1,
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-06-01 13:05:14.104361",
+ "modified": "2021-06-04 22:00:36.575097",
  "modified_by": "Administrator",
  "module": "ERPNext Feature Board",
  "name": "Improvement",

erpnext_feature_board/erpnext_feature_board/doctype/improvement/improvement.py

@@ -1,11 +1,66 @@
 # Copyright (c) 2021, ERPNext Community and contributors
 # For license information, please see license.txt
 
-# import frappe
+import frappe
+from frappe import _
 from frappe.website.website_generator import WebsiteGenerator
 
 
 class Improvement(WebsiteGenerator):
 	# the WebsiteGenerator class overrides the naming_series in the
 	# doctype definition, overriding the override
 	autoname = "naming_series:"
+
+
+@frappe.whitelist(methods=["POST"])
+def queue_deployment(improvement_name):
+	return set_deployment_status(
+		improvement_name,
+		"Build Queued",
+	).as_dict()
+
+
+@frappe.whitelist(methods=["POST"])
+def queue_upgrade(improvement_name):
+	return set_deployment_status(
+		improvement_name,
+		"Upgrade Queued",
+	).as_dict()
+
+
+@frappe.whitelist(methods=["POST"])
+def queue_delete(improvement_name):
+	return set_deployment_status(
+		improvement_name,
+		"Delete Queued",
+	).as_dict()
+
+
+def set_deployment_status(improvement_name, deployment_status):
+	imps_under_process = frappe.get_list(
+		"Improvement",
+		filters={
+			"deployment_status": [
+				"in",
+				[
+					"Build Queued",
+					"Building",
+					"Build Complete",
+					"Release Queued",
+					"Upgrade Queued",
+					"Rebuilding",
+					"Rebuild Complete",
+					"Upgrading",
+					"Delete Queued",
+					"Release Deleted",
+				],
+			],
+		},
+	)
+	if imps_under_process:
+		frappe.throw(_("Improvements already under process"))
+
+	improvement = frappe.get_doc("Improvement", improvement_name)
+	improvement.deployment_status = deployment_status
+	improvement.save()
+	return improvement

erpnext_feature_board/erpnext_feature_board/workspace/feature_board/feature_board.json

@@ -0,0 +1,39 @@
+{
+ "category": "Modules",
+ "charts": [],
+ "creation": "2021-06-04 13:23:22.576857",
+ "developer_mode_only": 0,
+ "disable_user_customization": 0,
+ "docstatus": 0,
+ "doctype": "Workspace",
+ "extends_another_page": 0,
+ "hide_custom": 0,
+ "icon": "",
+ "idx": 0,
+ "is_default": 0,
+ "is_standard": 1,
+ "label": "Feature Board",
+ "links": [],
+ "modified": "2021-06-04 13:23:22.576857",
+ "modified_by": "Administrator",
+ "module": "ERPNext Feature Board",
+ "name": "Feature Board",
+ "owner": "Administrator",
+ "pin_to_bottom": 0,
+ "pin_to_top": 0,
+ "shortcuts": [
+  {
+   "doc_view": "List",
+   "label": "Github Repository",
+   "link_to": "Github Repository",
+   "type": "DocType"
+  },
+  {
+   "doc_view": "List",
+   "label": "Improvement",
+   "link_to": "Improvement",
+   "type": "DocType"
+  }
+ ],
+ "shortcuts_label": "Feature Links"
+}

erpnext_feature_board/hooks.py

@@ -112,7 +112,19 @@
 scheduler_events = {
 	"hourly": [
 		"erpnext_feature_board.erpnext_feature_board.doctype.github_repository.github_repository.sync_repository_improvements"
-	]
+	],
+	"all": [
+		"erpnext_feature_board.tasks.deploy.process_build_queue_improvements",
+		"erpnext_feature_board.tasks.deploy.process_building_improvements",
+		"erpnext_feature_board.tasks.deploy.process_build_complete_improvements",
+		"erpnext_feature_board.tasks.deploy.process_release_queued_improvements",
+		"erpnext_feature_board.tasks.deploy.process_upgrade_queued_improvements",
+		"erpnext_feature_board.tasks.deploy.process_rebuilding_improvements",
+		"erpnext_feature_board.tasks.deploy.process_rebuild_complete_improvements",
+		"erpnext_feature_board.tasks.deploy.process_upgrading_improvements",
+		"erpnext_feature_board.tasks.deploy.process_delete_queued_improvements",
+		"erpnext_feature_board.tasks.deploy.process_release_deleted_improvements",
+	],
 }
 
 # Testing