Examples

CantorApp/Simple/Dataset/annotations/instances_val2017.json

@@ -0,0 +1,47 @@
+{
+ "info": "spytensor created",
+ "license": [
+  "license"
+ ],
+ "images": [
+  {
+   "height": 1840,
+   "width": 4096,
+   "id": 0,
+   "file_name": "5.jpg"
+  }
+ ],
+ "annotations": [
+  {
+   "id": 0,
+   "image_id": 0,
+   "category_id": 1,
+   "segmentation": [
+    [
+     1612.9326424870467,
+     362.0976809291396,
+     2430.975035327367,
+     362.0976809291396,
+     1612.9326424870467,
+     1250.8827010512478,
+     2430.975035327367,
+     1250.8827010512478
+    ]
+   ],
+   "bbox": [
+    1612.9326424870467,
+    362.0976809291396,
+    818.0423928403204,
+    888.7850201221081
+   ],
+   "iscrowd": 0,
+   "area": 1.0
+  }
+ ],
+ "categories": [
+  {
+   "id": 1,
+   "name": "valve"
+  }
+ ]
+}

CantorApp/Simple/Filters/detectron2Trainer.py

@@ -0,0 +1,160 @@
+import argparse,os,torch,json
+import cv2
+import detectron2
+from detectron2.data.datasets import register_coco_instances
+from detectron2.engine import DefaultTrainer, HookBase
+from detectron2.config import get_cfg
+from detectron2.utils.logger import setup_logger
+from detectron2.utils.events import get_event_storage
+
+torch.cuda.empty_cache()
+
+setup_logger()
+
+os.environ['CUDA_LAUNCH_BLOCKING']='1'
+
+def to_int(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+def to_float(value):
+    try:
+        return float(value)
+    except ValueError:
+        return value
+
+def BuildCfg(paras):
+    cfg = get_cfg()
+    cfg.merge_from_file(
+        paras.get('config-file')
+    )
+    cfg.DATASETS.TRAIN = (paras.get('dataset'),)
+    cfg.DATASETS.TEST = ()  # no metrics implemented for this dataset
+    cfg.DATALOADER.NUM_WORKERS = paras.get('num_workers')
+    cfg.MODEL.WEIGHTS = paras.get('weights')
+    cfg.OUTPUT_DIR = paras.get('output_dir')
+    cfg.SOLVER.IMS_PER_BATCH = to_int(paras.get('batch_size'))
+    cfg.SOLVER.BASE_LR = to_float(paras.get('learning_rate'))
+    cfg.SOLVER.MAX_ITER = to_int(paras.get('max_iter'))
+    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = to_int(paras.get('roi_batch_size'))
+    cfg.MODEL.ROI_HEADS.NUM_CLASSES = to_int(paras.get('roi_num_classes')) 
+
+    os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
+    return cfg
+
+def ExtractMetrics(metric_full):
+    keys_to_extract = ['eta_seconds', 'time', 'total_loss', 'lr']
+    selected_metrics = {}
+    iteration_number = None
+    for key in keys_to_extract:
+        if key in metric_full:
+            value, iteration = metric_full[key]
+            selected_metrics[key] = value
+            if iteration_number is None:  # Assign once
+                iteration_number = iteration
+    if iteration_number is not None:
+        selected_metrics['iteration_number'] = iteration_number
+    return selected_metrics
+
+class TrainHook(HookBase):
+    def __init__(self,callback=None):
+        self.callback = callback
+        pass
+
+    def before_train(self):
+        ev=get_event_storage()
+        latest_metric = ev.latest()
+        select_metrics = ExtractMetrics(latest_metric)
+        self.callback('before_train',select_metrics)
+        pass
+
+    def after_train(self):
+        ev=get_event_storage()
+        latest_metric = ev.latest()
+        select_metrics = ExtractMetrics(latest_metric)
+        self.callback('after_train',select_metrics)
+        pass
+
+    def before_step(self):
+        ev=get_event_storage()
+        latest_metric = ev.latest()
+        select_metrics = ExtractMetrics(latest_metric)
+        self.callback('before_step',select_metrics)
+        pass
+
+    def after_step(self):
+        ev=get_event_storage()
+        latest_metric = ev.latest()
+        select_metrics = ExtractMetrics(latest_metric)        
+        self.callback('after_step',select_metrics)
+        pass
+
+def TrainMain(train_config,jsonParameters,resume,callback=None):
+    # get detectron2 root path for config file
+    init_file_path = detectron2.__file__
+    init_dir_path = os.path.dirname(init_file_path)
+    detectron2_root = os.path.dirname(init_dir_path)
+    print("Training Start,detectron2 root:",detectron2_root)
+
+    config_params = json.loads(train_config)
+    dataset_location = "./"
+    output_dir = "./"
+    if 'dataset_location' in config_params:
+        dataset_location = config_params['dataset_location']
+    if 'output_dir' in config_params:
+        output_dir = config_params['output_dir']
+
+    parser = argparse.ArgumentParser(description="Train")
+
+    paras = json.loads(jsonParameters)
+    config_file = "configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"
+    if 'config_file' in paras:
+        config_file = paras['config_file']
+    train_coco_json_filepath = ""
+    if 'train_coco_json_filepath' in paras:
+        train_coco_json_filepath = paras['train_coco_json_filepath']
+
+    config_file_path = os.path.join(detectron2_root, config_file)
+    parser.add_argument("--config_file",default=config_file_path)
+    args = parser.parse_args()
+
+    # config dataset
+    dataset_name ="default"    
+    paras['dataset'] =  dataset_name
+    paras['output_dir'] = output_dir
+    paras['config-file'] = args.config_file
+    train_coco_json_filefullpath = os.path.join(dataset_location,train_coco_json_filepath)
+    register_coco_instances(dataset_name, {}, train_coco_json_filefullpath, dataset_location)
+
+    cfg = BuildCfg(paras)
+    trainer = DefaultTrainer(cfg)
+    trainer.register_hooks([ TrainHook(callback),])
+    trainer.resume_or_load(resume=resume)
+    trainer.train()
+
+if __name__ == "__main__":
+    def callback(event,msg):
+        print("callback:",event,msg)
+    json_params ="""
+        {
+            "train_coco_json_filepath": "../../annotations/instances_train2017.json",
+            "config_file": "configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml",
+            "weights": "detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl",
+            "num_workers": 2,
+            "batch_size": 2,
+            "learning_rate": 0.0025,
+            "max_iter": 2500,
+            "roi_batch_size": 128,
+            "roi_num_classes": 12
+        }
+    """
+    train_config ="""
+        {
+            "output_dir": "D:/Test/output",
+            "dataset_location": "D:/ToGithub/CantorAI/Galaxy/train/coco/images/train2017"
+        }
+    """
+    TrainMain(train_config,json_params,False,callback)
+    pass

CantorApp/Simple/Filters/detectron_filter.x

@@ -0,0 +1,42 @@
+from Galaxy import galaxy
+import detectron_predictor deferred
+
+class Detectron_Filter(galaxy.BaseFilter):
+	inputPin:Pin = None
+	outputPin:Pin = None
+	detectron_ready = False
+	predictor = None
+
+	def Detectron_Filter():
+		this.inputPin = this.NewInputPin()
+		this.inputPin.setputcallback(this.OneFrame)
+		this.outputPin = this.NewOutputPin()
+
+	def OneFrame(frame):
+		cantor = galaxy.cantor
+		if not this.detectron_ready:
+			gpuIndex = cantor.TakeFirstAvaibleResourceIndex("GPU")
+			# gpuIndex =gpuIndex % 2
+			print("detectrton filter,gpuIndex:",gpuIndex)
+			print("Detectron2 is not ready,Loading")
+			detectron_predictor.load()
+			print("Detectron2 loaded")
+			this.predictor = detectron_predictor.Predict(gpuIndex)
+			this.detectron_ready = True
+		img = frame.ToNdarray()
+		detect_result = this.predictor.Inference(img)
+		if detect_result != None:
+			# his.predictor.show(img,"Detected")
+			result_frame = galaxy.DataFrame()
+			# keep same timestamp as input frame
+			result_frame.NodeId_High = frame.NodeId_High
+			result_frame.NodeId_Low = frame.NodeId_Low
+			result_frame.startTime = frame.startTime
+			result_frame.type = "Dect"
+			result_frame.format1 = frame.format1 # w && h
+			result_frame.data = to_bin(detect_result)
+			this.outputPin.put(result_frame)
+			print("OneFrame called with results")
+		else:
+			print("OneFrame called with result is None")
+		cantor.CantorStatHit("Detectron_Filter_FPS")

CantorApp/Simple/Filters/detectron_predictor.py

@@ -0,0 +1,126 @@
+import numpy as np
+import multiprocessing as mp
+import cv2
+import torch
+import os
+from pathlib import Path
+import json
+
+import detectron2
+from detectron2.data import MetadataCatalog
+from detectron2.engine.defaults import DefaultPredictor
+from detectron2.utils.visualizer import ColorMode, Visualizer
+from detectron2.config import get_cfg
+from detectron2.structures import Boxes, Keypoints,RotatedBoxes
+from detectron2.data.detection_utils import read_image
+print("after import detectron2")
+
+def setup_cfg(cuda_device_index):
+    confidence_threshold =0.5
+    detectron_path = Path(os.path.dirname(os.path.realpath(detectron2.__file__)))
+    config_path = detectron_path.parent
+    config_file = os.path.join(config_path, "configs", "COCO-Keypoints","keypoint_rcnn_R_50_FPN_3x.yaml")
+    opts =list(("MODEL.WEIGHTS","detectron2://COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x/137849621/model_final_a6e10b.pkl"))
+    cfg = get_cfg()
+    #the default inference size is 800, seted by this MIN_SIZE_TEST
+    cfg.INPUT.MIN_SIZE_TEST = 1080
+    cfg.INPUT.MAX_SIZE_TEST = 2160
+    cfg.merge_from_file(config_file)
+    cfg.merge_from_list(opts)
+    # cuda device set
+    cfg.MODEL.DEVICE ='cuda:'+str(cuda_device_index)
+    # Set score_threshold for builtin models
+    cfg.MODEL.RETINANET.SCORE_THRESH_TEST = confidence_threshold
+    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = confidence_threshold
+    cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = confidence_threshold
+    cfg.freeze()
+    return cfg
+
+class Predict(object):
+    def __init__(self,cuda_device_index):
+        print("Predict Engine using (cuda) GPU:",cuda_device_index)
+        cfg = setup_cfg(cuda_device_index)
+        self.metadata = MetadataCatalog.get(
+            cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused"
+        )
+        self.cpu_device = torch.device("cpu")
+        self.predictor = DefaultPredictor(cfg)
+        self.predictions = None
+    def LoadImage(self,path):
+        img = read_image(path, format="BGR")
+        return img
+
+    def Inference(self, image):
+        # bgr_image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
+        # print("Inference:",image)
+        res = None
+        self.predictions = self.predictor(image)
+        if "instances" in self.predictions:
+            instances = self.predictions["instances"].to(self.cpu_device)
+            res = self.parse_instance_predictions(instances)
+        return res
+
+    def convert_boxes(self,boxes):
+        if isinstance(boxes, Boxes) or isinstance(boxes, RotatedBoxes):
+            return boxes.tensor.numpy()
+        else:
+            return np.asarray(boxes)
+
+    def convert_keypoints(self,keypoints):
+        if isinstance(keypoints, Keypoints):
+            keypoints = keypoints.tensor
+        keypoints = np.asarray(keypoints)
+        return keypoints
+
+    def parse_instance_predictions(self,predictions):
+        boxes = predictions.pred_boxes if predictions.has("pred_boxes") else None
+        scores = predictions.scores if predictions.has("scores") else None
+        classes = predictions.pred_classes if predictions.has("pred_classes") else None
+        #labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None))
+        keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None
+        boxes = self.convert_boxes(boxes)
+        keypoints =self.convert_keypoints(keypoints)
+        data ={
+            "boxes":boxes.tolist(),
+            "keypoints":keypoints.tolist(),
+            "scores":np.asarray(scores).tolist(),
+            "classes":np.asarray(classes).tolist()
+            }
+        json_data = json.dumps(data, indent=4)
+        return json_data
+
+    def show(self,img1,title):
+        vis_output = None
+        # Convert image from OpenCV BGR format to Matplotlib RGB format.
+        visualizer = Visualizer(img1, self.metadata, instance_mode=ColorMode.IMAGE)
+        if "panoptic_seg" in self.predictions:
+            panoptic_seg, segments_info = self.predictions["panoptic_seg"]
+            vis_output = visualizer.draw_panoptic_seg_predictions(
+                panoptic_seg.to(self.cpu_device), segments_info
+            )
+        else:
+            if "sem_seg" in self.predictions:
+                vis_output = visualizer.draw_sem_seg(
+                    self.predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                )
+            if "instances" in self.predictions:
+                instances = self.predictions["instances"].to(self.cpu_device)
+                vis_output = visualizer.draw_instance_predictions(predictions=instances)
+        if vis_output != None:
+            img_show =vis_output.get_image()[:, :, ::-1]
+        else:
+            img_show = img1
+        cv2.imshow(title,img_show)
+        cv2.waitKey(1)
+
+
+def MakePredict(cuda_device_index):
+    print("MakePredict")
+    return Predict(cuda_device_index)
+
+def DoPredict(predictor,img_path):
+    print("DoPredict")
+    img = predictor.LoadImage(img_path)
+    data =  predictor.Inference(img)
+    # predictor.show(img,"Test")
+    print("After DoPredict")