From 01bf1195b99b98bf6469dd09e61d89111a701efd Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Tue, 30 Sep 2025 18:01:17 -0700 Subject: [PATCH 1/7] visualize some stuff --- viewData.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 viewData.py diff --git a/viewData.py b/viewData.py new file mode 100644 index 0000000..d5235e7 --- /dev/null +++ b/viewData.py @@ -0,0 +1,29 @@ +import json +import base64 +import io +import zlib +import numpy as np +from PIL import Image +import matplotlib.pyplot as plt + +with open('Data/fsoco_segmentation_train/epflrt/ann/amz_00843.png.json') as f: + ann = json.load(f) + +img = Image.open('Data/fsoco_segmentation_train/epflrt/img/amz_00843.png') +fullMask = np.zeros((ann['size']['height'], ann['size']['width']), dtype=np.uint8) + +for obj in ann['objects']: + if obj['geometryType'] == 'bitmap': + bitmapData = base64.b64decode(obj['bitmap']['data']) + + decompressed = zlib.decompress(bitmapData) + mask = Image.open(io.BytesIO(decompressed)).convert('L') + + origin = obj['bitmap']['origin'] + maskArray = np.array(mask) + fullMask[origin[1]:origin[1]+mask.height, origin[0]:origin[0]+mask.width] = np.maximum(fullMask[origin[1]:origin[1]+mask.height, origin[0]:origin[0]+mask.width], maskArray) + +plt.figure(figsize=(15, 8)) +plt.imshow(img) +plt.imshow(fullMask, alpha=0.5) +plt.show() From 9a5dfc6d1f8a4722c04d3e902a652b6d1f7ec12c Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Wed, 8 Oct 2025 14:17:11 -0700 Subject: [PATCH 2/7] git changes --- .gitignore | 3 ++- Data/README.md | 1 + Segmentation/README.md | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Data/README.md create mode 100644 Segmentation/README.md diff --git a/.gitignore b/.gitignore index 37b3e6d..4ea16df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -/Data/ +/Data/fsoco_segmentation_train +/Segmentation/Yolact_minimal diff --git a/Data/README.md b/Data/README.md new file mode 100644 index 0000000..fb383ce --- /dev/null +++ b/Data/README.md @@ -0,0 +1 @@ +Clone FS coco data into here diff --git a/Segmentation/README.md b/Segmentation/README.md new file mode 100644 index 0000000..340244e --- /dev/null +++ b/Segmentation/README.md @@ -0,0 +1,4 @@ +Cloen this repo into this directory to run segmentation: +https://github.com/feiyuhuahuo/Yolact_minimal + +Move the FS From e2c9a284f349f1dc47c018c5946b6f48d775c608 Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Mon, 13 Oct 2025 08:04:53 -0700 Subject: [PATCH 3/7] yolact --- .gitignore | 1 + SampleData/README.md | 2 + .../convertSupervisely2Coco.cpython-312.pyc | Bin 0 -> 7617 bytes Segmentation/coneConfig.py | 41 +++++ Segmentation/convertSupervisely2Coco.py | 143 ++++++++++++++++++ Segmentation/detectCones.py | 95 ++++++++++++ Segmentation/trainCones.py | 65 ++++++++ viewData.py | 4 +- 8 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 SampleData/README.md create mode 100644 Segmentation/__pycache__/convertSupervisely2Coco.cpython-312.pyc create mode 100644 Segmentation/coneConfig.py create mode 100644 Segmentation/convertSupervisely2Coco.py create mode 100644 Segmentation/detectCones.py create mode 100644 Segmentation/trainCones.py diff --git a/.gitignore b/.gitignore index 4ea16df..561cb89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store /Data/fsoco_segmentation_train /Segmentation/Yolact_minimal +/SampleData/driverless.mp4 diff --git a/SampleData/README.md b/SampleData/README.md new file mode 100644 index 0000000..3b958dd --- /dev/null +++ b/SampleData/README.md @@ -0,0 +1,2 @@ +Download the youtube video here: +https://www.youtube.com/watch?v=o5vES5QaeiQ diff --git a/Segmentation/__pycache__/convertSupervisely2Coco.cpython-312.pyc b/Segmentation/__pycache__/convertSupervisely2Coco.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a18da55c4c1f3f3c494016f73e832ead6507f7e GIT binary patch literal 7617 zcmbtZeQZ-#mVaOV{)*#ZCw4*-k^qJ#o+Nyf0A)f#5-6ojphL(4S;z0ivGGTE?}a4R zbL-V?C1)BbPCF8ghAw7RX^iM-f>fy>jWn?8YS>+^WRGF+yk;b%MOv|crGU07TJ7$+ z&wh@Zrm*T>=iTpn?z!jw?z!il{ENY$#UOl1{`-Ww62txt734q_GtWMP%w>$mXgq+8 z6jFQym&(KlA*JL9DW%j1C8dfHC8R_^6;zL?@xmC*hz9D&fHtTb(Lo*cx7diDRzPf^ zl@J?g6~rc{f>vMHj$y2k){K~G?MNkoF($^$RNf#89O2k3K8rhbPf+oM6Bns>-24Qj zs>8E9!ySFwBeRY=DB&aFd-etlzl`xXszkwGBv&XEVgVLe1KAtJUQvoy$nAbaX-vI=ho8C2zl|mTCVlOhOG~Buc%T`#_(B8 zP}1auURdj<^*u4{oQe}7MZ0-7U`;{uvy%*qF!Th0#c33kV{VSwvrE*B?b$^$ zo-oabR3zXV6Sb%moMah}6G_jd4pHq37CR83Nl`n_hJ#~MX% zFX$a$eBKEjP3JxBqxlIIL0D8xhP+Z&RnW~{I2!bd6v~L|@u@)IEtDf>+eKmqzS#~) zJ2-^hhVt-Y-1RQUu$+r_hkQ)H)f@Iq1(^`fxq8C1FXVOgh1uX#z<#lnsU@M$U z-qp+cE-`EX#Nax~cu{{h?+b@qo^S~2`IA#1&`Um!3Cwo%ggxQ5$yw1>kW*RHVQHys zGq9wCkNXy)7`CD_Kh!z0I!E$Y>Qc70EvIwEdRL5<3*otNlFk{MV*QVe=7qpq;7Z5U zgG&d~w%-xISKn6OcXWT*`LO47w&!%tI2`MLq%y|d&zmfd^p=Es?#+j~`mC=0Q{va^ zYwDcN8S8y)GB3=`%_M751DWd9&xgJkzBzo~^hRtTZ?Gh~=guUR$%{*xI1xL%VyH+| zCg{ulMSse?7|fcRQfG1o*XNeyps3s&nl_Xh6QMF|{?-ks7Z3bo2Be|ohA16Ggv(4DXBClsNj>- z-G4ylGA3aD@>Zxr;YK0BqBxKGO2J>QG>dgK?#1DtLhL0=3ZZ<=Mk*1f;Yiuh0S9ew z$1dz$Q>O*xN)zZvMhSsHJ!$F#a)y{G96VH(YS1x^*kEm>r*QlTe@VR-nS(@LhjcWh zv&bU#_~d^eJV8g(sSs7;0XNSxA=U(CQ3qJ_xS|=YnDFbjcxS&C2Sa^Nz^+aE?J=u{s291{VmekH{RclVymcE$PwJ&#Pb-QD| zc}g8O$E)Jq@v5I6OEfQXNppf*bY>~X&yVGGriJ};`#(IGp{m&yn0w>h{|I*d|8m)t zsb>6@S0h>0n-Fk+2|K_>F~~c@RtUt)+BwP#`~3I#$?^+%0|W}VjJv`WK$En>D`Dj< z>~EBW3%Z2wjSvjiMX`Ou@rtODM<+%Kf;8W#4rDyj!=u0ClDj^h{j@1Xr%zqGu-ttuoZGVJcjT?!JG*Ybd9U&I+qu_|XLN66sJ9BB zN5YZ=x}6XdpZzBsegM5(zoCWaFWGaYZlEM-yks8`Wg86&O0M4xV0*cRCzPNPMMg0E z($?23V5PF;GdAl#=bDqE!f%9~1KxKt&yeHZSJIUPepe$6Q9MNCUW2Bbu~^)?Oij zFAD_^1~WDbRWz#uvgGufM`*!zU%F;FzGHjfh?^b`FxLwrspf(9#nB zFs1_XX4LOj*e50X8pq}xRJpR)!1@(RhY_TtE6VvW%%pn-CN{E;i>ulAov<;A1f9I& z@@~P7b_+U?TrcbeSe|rc)BvB6hpVj+1ijq17fTT5+`z5r72Ix!V8KLNZWI-AwBkon zs47t~zk&kVo^?T}kXORi|A^6Mc{IrE1KuGM-{>{)3hEolr-I(Jes7|cW%#oSCWO2) z6j#dpK)vk~1zr7FjZ6VAJ56OfSsEdagdN4tO_k1-1>PO@2$)QCrS!n9FPq;~m|sxA zY>th&_(Zvmhqa5#@+0+B)p|Yk3dth-Xk8F2@@oC{>zbfQ*V1*e)S{bSL4BirGtpJT zrV=Wxpg}ex==mGJKbK$-tnXLR^{Ab0K(Qc~8l}1|SIV>rv^Akx&_!3^%-IAR-7NJ| zAcK`fn%G9Spqgm4P%Wr0;ZSE4)IzmXSHq)=Qwq>ef$HRx-YS*kxi{eCM$yoT+68+I zGz8yLB!vlfS(;G?k1PNw@J>1B-y!GL(zP+Av9&^N5o7rLB;P9cUF#XG14(W}d~PU7 zBgnEwr~~7_R&daj^!6(%nKo!~aMVQ5?PZ+*I|>sF#gZX`&(Bbx!{%ww(`z?_limS( zykv&K*;#AbS%&oD&dT%r;F#CGGe%m&4~$tmA2{7Aqj23l-X_atErqv#$y*?~uD)}V zmvP@F;NRtP7ObguxW%Ul96N*I@Jo>M|2;#V!KQH!)Om=gVj5z|Nw??*c_ONt{yr6a z{$C4J*3+LKDAYS$CnB04!@Jx3TsS1+tf=w?+#JVnEb`faYeIT8i-eDs3{X+Q`}hF( zC*V26z{BSd>O_UyK}Ixi=oyaQNZv3z>m)^`FX;9%oTzh$LgB)TofEai4nBs9DEq=w zA-W-AZQ#mY^$kS^y4XG-9w8bUB1BgM3)gYM$`!$=4S_uy^hT6BB(@`Z?K^DP1MUIv zAT&f&4TXtMA#clgV2Yb~QU!xw;I4?GuuP2YmH6g-5qQN-6r3H3Y2@hu%UN2JX51HG zM$y996KLkQ(EwF=%+GjuPSksuFi4G^J%wB=1(6-4*=p4D2t;6MOH{yt0wNgo(eQpO z3I-MQgHH+yBJL5j zAmyF{m;o;IjA)XOb(B68MqZ+$w5aw#DtWX>*d7sw8lG_ndwnbiJAt+iI_cL$1|?Oo z1MnRRK}UpgFG|%=?gPxg$YPG|M{CeQJE||N8VVAhBp``BiQ1Kj2f)r|wCZ6LokrCp za(AJ+;v?(_lET}}_5+SEHVy<{gO8hqn*$zd;DnlR}Va7AqjUolx0X6I&;<`0E9^$iJit4eJ1mXvd8^iSsgB)*I#*K;LSyZLI@Qdeps=WzYWbT;AsyyJ_#H}`&V@aDlUEB|1-Yx_g(-P(H- znc*|J?vV$bfAtZbF`bPKK=jCHiuLC$Rk7obEj7vQIZM+s?4Y71u0ros+oJ7q?P6_m zBCT59lr!&*D>tr1-axbw@6ElJ9KUaB`E2O7r>~#>?fL8HZ}r`8?aAw_9$Rde_bu+b zGIaIy(&?*bm(HfQE^p7(?ax^b#MNkphNL%nCN=h%B|UtjE^S(#&KM8f;UB1vKdx>0 zbUZzr+qx%LyZ0H^si~Pe9`8xm@}|nf5O^Tle-T}&*pwuady`J^8hTRirW-%&Sytb8 zD}5+q-G67-gNlI_^QMG1$>q#V0187>>f(yke%ZI^OW_{}5+rPzp$TN@sJpstY1>2l zj;wu0dT-9&`Ov;UYu|sX|ISd(J`jI%#aeyYyXd_fSPZ20rjIP|&sh(}`yc7662n>j z=7;*0tiC00t;^Rn=k0aL@ui)qiLBj~uc=M`Y{`*2ovms8TBF!ze2ys$rd1C3y{u(xdix(O9r>!-M?k^1)Rs`?E%uZvYiUPyHQUp+Y)yMY`Pk91 z+_^k+^WdGPI}>+XGw%#%51+|*>|P$fx%1XUwxj=^<=)T!$&q>YZ1(86XCz)biYL^7 z)vDTzr3olKwl$`<=WII@)QZgxj^I5@d!X#tnX|o?pdMLl$@Y)-p>Edtlsad1Cdfw> z)ycYS#kPkPZP|*pynSoFVOze*nQz&iI+Cw#K!5cOUz^lFseF#9&6d?l%v=Xfkc!4d z{b~;mD_rfvF+)}2;@t6H^gs3Du#2mk0A|nM!_gr)9KRHwh_~LSs#hpY{KS>c}$COVdgBmx<2(*vscn^^*M!PiJh{MS<@Rylxw`<3?Q{5sDIQ@5dSl79o(YIm~DeDGIK~Vs8MCNCB<+jXMXG;vlntZ0Nr-f&e@bGMJ|6iaS(7)l!9N$wKaCyxGvxqn?|(qGuc|Q0F3tVFrl7du literal 0 HcmV?d00001 diff --git a/Segmentation/coneConfig.py b/Segmentation/coneConfig.py new file mode 100644 index 0000000..b55eb96 --- /dev/null +++ b/Segmentation/coneConfig.py @@ -0,0 +1,41 @@ +CONE_CLASSES = ('seg_orange_cone', 'seg_yellow_cone', 'seg_large_orange_cone', + 'seg_blue_cone', 'seg_unknown_cone') + +class ConeTrainingConfig: + def __init__(self): + self.classNames = CONE_CLASSES + self.numClasses = len(self.classNames) + 1 + self.continuousId = {(aa + 1): (aa + 1) for aa in range(self.numClasses - 1)} + + self.trainImgs = 'Data/fsoco_segmentation_train/' + self.trainAnn = 'Data/fsoco_segmentation_train/train_coco.json' + self.valImgs = 'Data/fsoco_segmentation_train/' + self.valAnn = 'Data/fsoco_segmentation_train/train_coco.json' + + self.imgSize = 544 + self.trainBs = 8 + self.bsPerGpu = 8 + + self.warmupUntil = 500 + self.lrSteps = (0, 40000, 60000, 70000, 80000) + + self.lr = 0.001 + self.warmupInit = self.lr * 0.1 + + self.posIouThre = 0.5 + self.negIouThre = 0.4 + + self.confAlpha = 1 + self.bboxAlpha = 1.5 + self.maskAlpha = 6.125 + self.semanticAlpha = 1 + + self.masksToTrain = 100 + + self.backboneWeight = 'Yolact_minimal/weights/backbone_res50.pth' + + self.valInterval = 4000 + self.valNum = -1 + + self.scales = [int(self.imgSize / 544 * aa) for aa in (24, 48, 96, 192, 384)] + self.aspectRatios = [1, 1 / 2, 2] diff --git a/Segmentation/convertSupervisely2Coco.py b/Segmentation/convertSupervisely2Coco.py new file mode 100644 index 0000000..bcf98a0 --- /dev/null +++ b/Segmentation/convertSupervisely2Coco.py @@ -0,0 +1,143 @@ +import json +import os +import glob +import base64 +import zlib +import numpy as np +from pathlib import Path +import cv2 +from io import BytesIO + +def decodeSuperviselyBitmap(bitmapData, origin, imgHeight, imgWidth): + data = base64.b64decode(bitmapData['data']) + pngData = zlib.decompress(data) + + maskImg = cv2.imdecode(np.frombuffer(pngData, dtype=np.uint8), cv2.IMREAD_GRAYSCALE) + + if maskImg is None: + return None + + mask = (maskImg > 0).astype(np.uint8) + + fullMask = np.zeros((imgHeight, imgWidth), dtype=np.uint8) + y1, x1 = origin[1], origin[0] + y2, x2 = min(y1 + mask.shape[0], imgHeight), min(x1 + mask.shape[1], imgWidth) + fullMask[y1:y2, x1:x2] = mask[:y2-y1, :x2-x1] + return fullMask + +def maskToRle(mask): + pixels = mask.flatten() + pixels = np.concatenate([[0], pixels, [0]]) + runs = np.where(pixels[1:] != pixels[:-1])[0] + 1 + runs[1::2] -= runs[::2] + return runs.tolist() + +def maskToBbox(mask): + rows = np.any(mask, axis=1) + cols = np.any(mask, axis=0) + if not rows.any() or not cols.any(): + return [0, 0, 0, 0] + ymin, ymax = np.where(rows)[0][[0, -1]] + xmin, xmax = np.where(cols)[0][[0, -1]] + return [int(xmin), int(ymin), int(xmax - xmin + 1), int(ymax - ymin + 1)] + +def convertSupervisely2Coco(dataRoot, outputPath): + metaPath = os.path.join(dataRoot, 'meta.json') + with open(metaPath, 'r') as f: + meta = json.load(f) + + segClasses = [c for c in meta['classes'] if c['shape'] == 'bitmap'] + classIdToCocoId = {c['id']: idx + 1 for idx, c in enumerate(segClasses)} + + cocoData = { + 'images': [], + 'annotations': [], + 'categories': [{'id': idx + 1, 'name': c['title'], 'supercategory': 'cone'} + for idx, c in enumerate(segClasses)] + } + + print(f"Found {len(segClasses)} segmentation classes:") + for cat in cocoData['categories']: + print(f" {cat['id']}: {cat['name']}") + + teamDirs = [d for d in os.listdir(dataRoot) if os.path.isdir(os.path.join(dataRoot, d))] + + imageId = 0 + annotationId = 0 + + for teamDir in sorted(teamDirs): + annDir = os.path.join(dataRoot, teamDir, 'ann') + imgDir = os.path.join(dataRoot, teamDir, 'img') + + if not os.path.exists(annDir) or not os.path.exists(imgDir): + continue + + annFiles = glob.glob(os.path.join(annDir, '*.json')) + print(f"\nProcessing {teamDir}: {len(annFiles)} images", flush=True) + + for idx, annFile in enumerate(sorted(annFiles)): + if idx % 20 == 0: + print(f" {teamDir}: {idx}/{len(annFiles)}", flush=True) + with open(annFile, 'r') as f: + ann = json.load(f) + + imgFilename = os.path.basename(annFile).replace('.json', '') + imgPath = os.path.join(teamDir, 'img', imgFilename) + + cocoData['images'].append({ + 'id': imageId, + 'file_name': imgPath, + 'height': ann['size']['height'], + 'width': ann['size']['width'] + }) + + for obj in ann['objects']: + if obj['geometryType'] != 'bitmap': + continue + + classId = obj['classId'] + if classId not in classIdToCocoId: + continue + + cocoId = classIdToCocoId[classId] + + fullMask = decodeSuperviselyBitmap(obj['bitmap'], obj['bitmap']['origin'], + ann['size']['height'], ann['size']['width']) + + if fullMask is None: + continue + + bbox = maskToBbox(fullMask) + area = int(np.sum(fullMask)) + + if area == 0: + continue + + rle = maskToRle(fullMask) + + cocoData['annotations'].append({ + 'id': annotationId, + 'image_id': imageId, + 'category_id': cocoId, + 'segmentation': [rle], + 'area': area, + 'bbox': bbox, + 'iscrowd': 0 + }) + annotationId += 1 + + imageId += 1 + + print(f"\n{'='*60}") + print(f"Total images: {len(cocoData['images'])}") + print(f"Total annotations: {len(cocoData['annotations'])}") + + with open(outputPath, 'w') as f: + json.dump(cocoData, f) + + print(f"Saved to: {outputPath}") + +if __name__ == '__main__': + dataRoot = 'Data/fsoco_segmentation_train' + outputPath = 'Data/fsoco_segmentation_train/train_coco.json' + convertSupervisely2Coco(dataRoot, outputPath) diff --git a/Segmentation/detectCones.py b/Segmentation/detectCones.py new file mode 100644 index 0000000..d521d76 --- /dev/null +++ b/Segmentation/detectCones.py @@ -0,0 +1,95 @@ +import sys +import os +import argparse + +sys.path.insert(0, 'Yolact_minimal') + +def main(): + parser = argparse.ArgumentParser(description='Detect cones using trained YOLACT model') + parser.add_argument('--weight', type=str, required=True, + help='Path to trained weights (e.g., Yolact_minimal/weights/best_32.5_res50_cone_40000.pth)') + parser.add_argument('--image', type=str, default=None, + help='Path to image or folder of images') + parser.add_argument('--video', type=str, default=None, + help='Path to video file') + parser.add_argument('--img_size', type=int, default=544, + help='Image size for inference') + parser.add_argument('--threshold', type=float, default=0.3, + help='Confidence threshold for detections (0-1)') + parser.add_argument('--hide_mask', action='store_true', + help='Hide segmentation masks (show only boxes)') + parser.add_argument('--hide_bbox', action='store_true', + help='Hide bounding boxes (show only masks)') + parser.add_argument('--hide_score', action='store_true', + help='Hide confidence scores') + parser.add_argument('--real_time', action='store_true', + help='Display video detection in real-time (no saving)') + + args = parser.parse_args() + + if not os.path.exists(args.weight): + print(f"Error: Weight file not found: {args.weight}") + print("\nAvailable weights in Yolact_minimal/weights/:") + if os.path.exists('Yolact_minimal/weights'): + weights = [f for f in os.listdir('Yolact_minimal/weights') if f.endswith('.pth')] + if weights: + for w in weights: + print(f" - {w}") + else: + print(" (No .pth files found)") + return + + if args.image is None and args.video is None: + print("Error: Must specify either --image or --video") + return + + print("="*60) + print("YOLACT Cone Detection") + print("="*60) + print(f"Weight: {args.weight}") + print(f"Image size: {args.img_size}") + print(f"Threshold: {args.threshold}") + if args.image: + print(f"Input: {args.image}") + print(f"Output: Yolact_minimal/results/images/") + elif args.video: + print(f"Input: {args.video}") + if args.real_time: + print(f"Output: Real-time display") + else: + print(f"Output: Yolact_minimal/results/videos/") + print("="*60) + print() + + os.chdir('Yolact_minimal') + + cmd_parts = [ + 'python', 'detect.py', + '--weight', f'../{args.weight}' if not args.weight.startswith('Yolact_minimal') else args.weight.replace('Yolact_minimal/', ''), + '--img_size', str(args.img_size), + '--visual_thre', str(args.threshold) + ] + + if args.image: + img_path = f'../{args.image}' if not args.image.startswith('Yolact_minimal') else args.image.replace('Yolact_minimal/', '') + cmd_parts.extend(['--image', img_path]) + elif args.video: + vid_path = f'../{args.video}' if not args.video.startswith('Yolact_minimal') else args.video.replace('Yolact_minimal/', '') + cmd_parts.extend(['--video', vid_path]) + + if args.hide_mask: + cmd_parts.append('--hide_mask') + if args.hide_bbox: + cmd_parts.append('--hide_bbox') + if args.hide_score: + cmd_parts.append('--hide_score') + if args.real_time: + cmd_parts.append('--real_time') + + cmd = ' '.join(cmd_parts) + print(f"Running: {cmd}\n") + + os.system(cmd) + +if __name__ == '__main__': + main() diff --git a/Segmentation/trainCones.py b/Segmentation/trainCones.py new file mode 100644 index 0000000..c1d4b04 --- /dev/null +++ b/Segmentation/trainCones.py @@ -0,0 +1,65 @@ +import sys +import os + +sys.path.insert(0, 'Yolact_minimal') + +import argparse +from coneConfig import ConeTrainingConfig + +parser = argparse.ArgumentParser(description='Train YOLACT for Cone Segmentation') +parser.add_argument('--cfg', default='res50_cone', help='Config name (will use res50_cone)') +parser.add_argument('--train_bs', type=int, default=8, help='Training batch size') +parser.add_argument('--img_size', type=int, default=544, help='Image size') +parser.add_argument('--resume', default=None, type=str, help='Resume training from checkpoint') +parser.add_argument('--val_interval', type=int, default=4000, help='Validation interval') +parser.add_argument('--val_num', type=int, default=-1, help='Number of validation images') +parser.add_argument('--local_rank', type=int, default=0, help='Local rank for distributed training') +parser.add_argument('--traditional_nms', action='store_true', help='Use traditional NMS') +parser.add_argument('--coco_api', action='store_true', help='Use COCO API for evaluation') + +args = parser.parse_args() + +print("="*60) +print("YOLACT Training for Formula Student Cone Segmentation") +print("="*60) +print(f"Dataset: fsoco_segmentation_train") +print(f"Classes: {len(ConeTrainingConfig().classNames)} cone types") +print(f"Annotations file: {ConeTrainingConfig().trainAnn}") +print(f"Batch size: {args.train_bs}") +print(f"Image size: {args.img_size}") +print("="*60) +print() + +print("To start training, you need to:") +print("1. Download the ResNet50 backbone weights:") +print(" mkdir -p Yolact_minimal/weights") +print(" wget https://download.pytorch.org/models/resnet50-19c8e357.pth -O Yolact_minimal/weights/backbone_res50.pth") +print() +print("2. Modify Yolact_minimal/config.py to add the cone configuration:") +print(" Add this class after the res50_custom class:") +print() +print("class res50_cone(res101_coco):") +print(" def __init__(self, args):") +print(" super().__init__(args)") +print(" self.class_names = ('seg_orange_cone', 'seg_yellow_cone', 'seg_large_orange_cone', 'seg_blue_cone', 'seg_unknown_cone')") +print(" self.num_classes = len(self.class_names) + 1") +print(" self.continuous_id = {(aa + 1): (aa + 1) for aa in range(self.num_classes - 1)}") +print(" if self.mode == 'train':") +print(" self.weight = args.resume if args.resume else 'weights/backbone_res50.pth'") +print(" else:") +print(" self.weight = args.weight") +print() +print(" if self.mode == 'train':") +print(" self.train_imgs = 'Data/fsoco_segmentation_train/'") +print(" self.train_ann = 'Data/fsoco_segmentation_train/train_coco.json'") +print(" self.warmup_until = 500") +print(" self.lr_steps = (0, 40000, 60000, 70000, 80000)") +print() +print(" if self.mode in ('train', 'val'):") +print(" self.val_imgs = 'Data/fsoco_segmentation_train/'") +print(" self.val_ann = 'Data/fsoco_segmentation_train/train_coco.json'") +print() +print("3. Run training with:") +print(" cd Yolact_minimal") +print(" python train.py --cfg res50_cone --train_bs 8 --img_size 544") +print() diff --git a/viewData.py b/viewData.py index d5235e7..87953be 100644 --- a/viewData.py +++ b/viewData.py @@ -6,10 +6,10 @@ from PIL import Image import matplotlib.pyplot as plt -with open('Data/fsoco_segmentation_train/epflrt/ann/amz_00843.png.json') as f: +with open('Data/fsoco_segmentation_train/epflrt/ann/amz_00825.png.json') as f: ann = json.load(f) -img = Image.open('Data/fsoco_segmentation_train/epflrt/img/amz_00843.png') +img = Image.open('Data/fsoco_segmentation_train/epflrt/img/amz_00825.png') fullMask = np.zeros((ann['size']['height'], ann['size']['width']), dtype=np.uint8) for obj in ann['objects']: From 273c8d73ff0f84387e4220791d6ed596334f7883 Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Tue, 28 Oct 2025 13:28:41 -0700 Subject: [PATCH 4/7] mild updates --- .gitignore | 1 + Segmentation/coneConfig.py | 41 --------------- Segmentation/detectCones.py | 102 ++++++++---------------------------- Segmentation/trainCones.py | 92 +++++++++++++------------------- 4 files changed, 59 insertions(+), 177 deletions(-) delete mode 100644 Segmentation/coneConfig.py mode change 100644 => 100755 Segmentation/detectCones.py mode change 100644 => 100755 Segmentation/trainCones.py diff --git a/.gitignore b/.gitignore index 561cb89..bec4f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /Data/fsoco_segmentation_train /Segmentation/Yolact_minimal /SampleData/driverless.mp4 +*.txt diff --git a/Segmentation/coneConfig.py b/Segmentation/coneConfig.py deleted file mode 100644 index b55eb96..0000000 --- a/Segmentation/coneConfig.py +++ /dev/null @@ -1,41 +0,0 @@ -CONE_CLASSES = ('seg_orange_cone', 'seg_yellow_cone', 'seg_large_orange_cone', - 'seg_blue_cone', 'seg_unknown_cone') - -class ConeTrainingConfig: - def __init__(self): - self.classNames = CONE_CLASSES - self.numClasses = len(self.classNames) + 1 - self.continuousId = {(aa + 1): (aa + 1) for aa in range(self.numClasses - 1)} - - self.trainImgs = 'Data/fsoco_segmentation_train/' - self.trainAnn = 'Data/fsoco_segmentation_train/train_coco.json' - self.valImgs = 'Data/fsoco_segmentation_train/' - self.valAnn = 'Data/fsoco_segmentation_train/train_coco.json' - - self.imgSize = 544 - self.trainBs = 8 - self.bsPerGpu = 8 - - self.warmupUntil = 500 - self.lrSteps = (0, 40000, 60000, 70000, 80000) - - self.lr = 0.001 - self.warmupInit = self.lr * 0.1 - - self.posIouThre = 0.5 - self.negIouThre = 0.4 - - self.confAlpha = 1 - self.bboxAlpha = 1.5 - self.maskAlpha = 6.125 - self.semanticAlpha = 1 - - self.masksToTrain = 100 - - self.backboneWeight = 'Yolact_minimal/weights/backbone_res50.pth' - - self.valInterval = 4000 - self.valNum = -1 - - self.scales = [int(self.imgSize / 544 * aa) for aa in (24, 48, 96, 192, 384)] - self.aspectRatios = [1, 1 / 2, 2] diff --git a/Segmentation/detectCones.py b/Segmentation/detectCones.py old mode 100644 new mode 100755 index d521d76..2386f63 --- a/Segmentation/detectCones.py +++ b/Segmentation/detectCones.py @@ -1,95 +1,39 @@ +#!/usr/bin/env python3 import sys import os -import argparse - -sys.path.insert(0, 'Yolact_minimal') +import subprocess def main(): - parser = argparse.ArgumentParser(description='Detect cones using trained YOLACT model') - parser.add_argument('--weight', type=str, required=True, - help='Path to trained weights (e.g., Yolact_minimal/weights/best_32.5_res50_cone_40000.pth)') - parser.add_argument('--image', type=str, default=None, - help='Path to image or folder of images') - parser.add_argument('--video', type=str, default=None, - help='Path to video file') - parser.add_argument('--img_size', type=int, default=544, - help='Image size for inference') - parser.add_argument('--threshold', type=float, default=0.3, - help='Confidence threshold for detections (0-1)') - parser.add_argument('--hide_mask', action='store_true', - help='Hide segmentation masks (show only boxes)') - parser.add_argument('--hide_bbox', action='store_true', - help='Hide bounding boxes (show only masks)') - parser.add_argument('--hide_score', action='store_true', - help='Hide confidence scores') - parser.add_argument('--real_time', action='store_true', - help='Display video detection in real-time (no saving)') - - args = parser.parse_args() + script_dir = os.path.dirname(os.path.abspath(__file__)) + yolact_dir = os.path.join(script_dir, 'yolact_edge') - if not os.path.exists(args.weight): - print(f"Error: Weight file not found: {args.weight}") - print("\nAvailable weights in Yolact_minimal/weights/:") - if os.path.exists('Yolact_minimal/weights'): - weights = [f for f in os.listdir('Yolact_minimal/weights') if f.endswith('.pth')] - if weights: - for w in weights: - print(f" - {w}") - else: - print(" (No .pth files found)") - return + args = sys.argv[1:] - if args.image is None and args.video is None: - print("Error: Must specify either --image or --video") - return + default_args = [ + '--config=yolact_edge_mobilenetv2_cone_config', + '--score_threshold=0.3', + '--top_k=100', + ] - print("="*60) - print("YOLACT Cone Detection") - print("="*60) - print(f"Weight: {args.weight}") - print(f"Image size: {args.img_size}") - print(f"Threshold: {args.threshold}") - if args.image: - print(f"Input: {args.image}") - print(f"Output: Yolact_minimal/results/images/") - elif args.video: - print(f"Input: {args.video}") - if args.real_time: - print(f"Output: Real-time display") - else: - print(f"Output: Yolact_minimal/results/videos/") - print("="*60) - print() + provided_flags = {arg.split('=')[0].lstrip('-') for arg in args if '=' in arg or arg.startswith('--')} - os.chdir('Yolact_minimal') + final_args = [] + for default_arg in default_args: + flag = default_arg.split('=')[0].lstrip('-') + if flag not in provided_flags and not any(arg.startswith('--' + flag) for arg in args): + final_args.append(default_arg) - cmd_parts = [ - 'python', 'detect.py', - '--weight', f'../{args.weight}' if not args.weight.startswith('Yolact_minimal') else args.weight.replace('Yolact_minimal/', ''), - '--img_size', str(args.img_size), - '--visual_thre', str(args.threshold) - ] + final_args.extend(args) - if args.image: - img_path = f'../{args.image}' if not args.image.startswith('Yolact_minimal') else args.image.replace('Yolact_minimal/', '') - cmd_parts.extend(['--image', img_path]) - elif args.video: - vid_path = f'../{args.video}' if not args.video.startswith('Yolact_minimal') else args.video.replace('Yolact_minimal/', '') - cmd_parts.extend(['--video', vid_path]) + cmd = [sys.executable, 'eval.py'] + final_args - if args.hide_mask: - cmd_parts.append('--hide_mask') - if args.hide_bbox: - cmd_parts.append('--hide_bbox') - if args.hide_score: - cmd_parts.append('--hide_score') - if args.real_time: - cmd_parts.append('--real_time') + print(f"Running inference in {yolact_dir}") + print(f"Command: {' '.join(cmd)}\n") - cmd = ' '.join(cmd_parts) - print(f"Running: {cmd}\n") + os.chdir(yolact_dir) - os.system(cmd) + result = subprocess.run(cmd) + sys.exit(result.returncode) if __name__ == '__main__': main() diff --git a/Segmentation/trainCones.py b/Segmentation/trainCones.py old mode 100644 new mode 100755 index c1d4b04..dbbe757 --- a/Segmentation/trainCones.py +++ b/Segmentation/trainCones.py @@ -1,65 +1,43 @@ +#!/usr/bin/env python3 import sys import os +import subprocess -sys.path.insert(0, 'Yolact_minimal') +def main(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + yolact_dir = os.path.join(script_dir, 'yolact_edge') -import argparse -from coneConfig import ConeTrainingConfig + args = sys.argv[1:] -parser = argparse.ArgumentParser(description='Train YOLACT for Cone Segmentation') -parser.add_argument('--cfg', default='res50_cone', help='Config name (will use res50_cone)') -parser.add_argument('--train_bs', type=int, default=8, help='Training batch size') -parser.add_argument('--img_size', type=int, default=544, help='Image size') -parser.add_argument('--resume', default=None, type=str, help='Resume training from checkpoint') -parser.add_argument('--val_interval', type=int, default=4000, help='Validation interval') -parser.add_argument('--val_num', type=int, default=-1, help='Number of validation images') -parser.add_argument('--local_rank', type=int, default=0, help='Local rank for distributed training') -parser.add_argument('--traditional_nms', action='store_true', help='Use traditional NMS') -parser.add_argument('--coco_api', action='store_true', help='Use COCO API for evaluation') + default_args = [ + '--config=yolact_edge_mobilenetv2_cone_config', + '--batch_size=4', + '--save_folder=weights/', + '--validation_epoch=2', + '--save_interval=5000', + '--max_checkpoints=10', + '--num_workers=0', + ] -args = parser.parse_args() + provided_flags = {arg.split('=')[0].lstrip('-') for arg in args if '=' in arg or arg.startswith('--')} -print("="*60) -print("YOLACT Training for Formula Student Cone Segmentation") -print("="*60) -print(f"Dataset: fsoco_segmentation_train") -print(f"Classes: {len(ConeTrainingConfig().classNames)} cone types") -print(f"Annotations file: {ConeTrainingConfig().trainAnn}") -print(f"Batch size: {args.train_bs}") -print(f"Image size: {args.img_size}") -print("="*60) -print() + final_args = [] + for default_arg in default_args: + flag = default_arg.split('=')[0].lstrip('-') + if flag not in provided_flags and not any(arg.startswith('--' + flag) for arg in args): + final_args.append(default_arg) -print("To start training, you need to:") -print("1. Download the ResNet50 backbone weights:") -print(" mkdir -p Yolact_minimal/weights") -print(" wget https://download.pytorch.org/models/resnet50-19c8e357.pth -O Yolact_minimal/weights/backbone_res50.pth") -print() -print("2. Modify Yolact_minimal/config.py to add the cone configuration:") -print(" Add this class after the res50_custom class:") -print() -print("class res50_cone(res101_coco):") -print(" def __init__(self, args):") -print(" super().__init__(args)") -print(" self.class_names = ('seg_orange_cone', 'seg_yellow_cone', 'seg_large_orange_cone', 'seg_blue_cone', 'seg_unknown_cone')") -print(" self.num_classes = len(self.class_names) + 1") -print(" self.continuous_id = {(aa + 1): (aa + 1) for aa in range(self.num_classes - 1)}") -print(" if self.mode == 'train':") -print(" self.weight = args.resume if args.resume else 'weights/backbone_res50.pth'") -print(" else:") -print(" self.weight = args.weight") -print() -print(" if self.mode == 'train':") -print(" self.train_imgs = 'Data/fsoco_segmentation_train/'") -print(" self.train_ann = 'Data/fsoco_segmentation_train/train_coco.json'") -print(" self.warmup_until = 500") -print(" self.lr_steps = (0, 40000, 60000, 70000, 80000)") -print() -print(" if self.mode in ('train', 'val'):") -print(" self.val_imgs = 'Data/fsoco_segmentation_train/'") -print(" self.val_ann = 'Data/fsoco_segmentation_train/train_coco.json'") -print() -print("3. Run training with:") -print(" cd Yolact_minimal") -print(" python train.py --cfg res50_cone --train_bs 8 --img_size 544") -print() + final_args.extend(args) + + cmd = [sys.executable, 'train.py'] + final_args + + print(f"Running training in {yolact_dir}") + print(f"Command: {' '.join(cmd)}\n") + + os.chdir(yolact_dir) + + result = subprocess.run(cmd) + sys.exit(result.returncode) + +if __name__ == '__main__': + main() From b1ba94d7ff37e28b6da8eedd4f2aecd997690db3 Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Tue, 28 Oct 2025 16:12:47 -0700 Subject: [PATCH 5/7] Project descriptions --- Projects/00Template.md | 13 +++++++++++++ Projects/IMUDataProcessing.md | 30 ++++++++++++++++++++++++++++++ Projects/Localization.md | 29 +++++++++++++++++++++++++++++ Projects/PathPlanning.md | 28 ++++++++++++++++++++++++++++ Projects/Yolact.md | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+) create mode 100644 Projects/00Template.md create mode 100644 Projects/IMUDataProcessing.md create mode 100644 Projects/Localization.md create mode 100644 Projects/PathPlanning.md create mode 100644 Projects/Yolact.md diff --git a/Projects/00Template.md b/Projects/00Template.md new file mode 100644 index 0000000..f2e6795 --- /dev/null +++ b/Projects/00Template.md @@ -0,0 +1,13 @@ +# Project Title + +## Active Members + +## Design Review Dates/Timeline + +## Introduction + +## Overview of the Problem + +## Steps in the project + +## Suggested direction \ No newline at end of file diff --git a/Projects/IMUDataProcessing.md b/Projects/IMUDataProcessing.md new file mode 100644 index 0000000..b3d0acc --- /dev/null +++ b/Projects/IMUDataProcessing.md @@ -0,0 +1,30 @@ +# IMU Data Processing + +## Active Members +1. Saahith Veeramaneni + +## Design Review Dates/Timeline +1. Literature review by November 3rd +1. PDR by November 10th +1. Theoretical codebase, Thanksgiving + +## Introduction +Due to the low compute on the car, it is not possible to use multiple +cameras so this project will explore using structure from motion (sfm) +to create binocular views. + +## Overview of the Problem +Having multiple perspectives on a car allows the creation of a +disparity map from an image which can be used to estimate depth if the +exact distance between the cameras is well understood. Thus, we must +explore if we can achieve high enough accuracy from the IMU to use sfm +in the data processing pipeline. + +## Steps in the project +1. Literature review +1. Understanding compute requirements +1. Draft of the code if it is feasible + +## Suggested direction +1. Begin with a long literature review and exploring its use on other + Formula Student teams and in industry diff --git a/Projects/Localization.md b/Projects/Localization.md new file mode 100644 index 0000000..14d79f7 --- /dev/null +++ b/Projects/Localization.md @@ -0,0 +1,29 @@ +# Localization + +## Active Members +1. Parker Costa +1. Hansika Nerusa +1. Chanchal Mukeshsingh + +## Design Review Dates/Timeline +1. PDR by november 4th +1. Creation of an independent working system by end of first quarter + +## Introduction +Localization deals with the creation of both a global and local map +based on camera inputs so that the car can understand how its moving +and its surroundings. + +## Overview of the Problem +By being fed in raw data from the IMU, depth estimation, and cone +segmentation, the project must identify where cones are and create a +global map overtime by processing raw data and estimating where cones +are even if the data is noisy. + +## Steps in the project +1. Literature review and PDR +1. Communication and finalization of input and output data formats +1. Code base developed in a virtual test bench +1. Code review + +## Suggested direction diff --git a/Projects/PathPlanning.md b/Projects/PathPlanning.md new file mode 100644 index 0000000..c117d41 --- /dev/null +++ b/Projects/PathPlanning.md @@ -0,0 +1,28 @@ +# Path Planning + +## Active Members +1. Suhani Agarwal +1. Nathan Yee +1. Aedan Benavides + +## Design Review Dates/Timeline +1. PDR by November 3rd +1. Finished by end of winter quarter + +## Introduction +The project focuses on identifying a path from the map data by +using the cones to find the optimal path to move through the cones. + +## Overview of the Problem +When raw map data is presented it'll be done in the form of a +coordinate system on a 2D plane. To generate a path to plan a route +through, a polygon and direction representation of the path must be identified. + +## Steps in the project +1. Literature review +1. PDR +1. Triangulation methods +1. Data format standardization + +## Suggested direction +- Unknown diff --git a/Projects/Yolact.md b/Projects/Yolact.md new file mode 100644 index 0000000..d030e0d --- /dev/null +++ b/Projects/Yolact.md @@ -0,0 +1,32 @@ +# YOLACT + +## Active Members +1. Abishek Adari +1. Dylan Price + +## Design Review Dates/Timeline +1. Read the paper and PDR November 3rd +1. Implementation and training of YOLACT edge by November 24th +1. Benchmarking and optimization for edge devices, end of quarter + +## Introduction +Identifying cones for processing is a difficult problem and YOLACT +performs semantic segmentation to identify and classify each +individual pixel among their relevant classes. + +## Overview of the Problem +The self driving car must know the bounds it can travel, and the +creation of the bounds must involve identifying where in an image the +cone is. We do this using an implementation of the YOLACTEdge model +which will classify every pixel among all possibilities. We target at +least 30FPS when running alone on a Jetson Orin Nano. + +## Steps in the project +1. Read the papers +1. Base reimplementation of the project +1. Data processing +1. Training +1. Evaluation + +## Suggested direction +Read the YOLACTEdge, Yolact, and YOLACT++ papers. From 9c7a594960d3654b976040b7beebb01ef69c1be0 Mon Sep 17 00:00:00 2001 From: DanielRhee Date: Wed, 29 Oct 2025 18:13:33 -0700 Subject: [PATCH 6/7] Control planning --- Projects/ControlPlanning.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Projects/ControlPlanning.md diff --git a/Projects/ControlPlanning.md b/Projects/ControlPlanning.md new file mode 100644 index 0000000..5586123 --- /dev/null +++ b/Projects/ControlPlanning.md @@ -0,0 +1,30 @@ +# Control Planning + +## Active Members +1. Ray Zou + +## Design Review Dates/Timeline +1. PDR November 3rd + +## Introduction +Control input planning is a vehicle dynamics problem to generate the +optimal set of vehicle dynamics inputs needed to follow a +predetermined path. Other projects, such as path planning will already +generate a path that must be followed. + +## Overview of the Problem +Using and working with the VD and Simulations team, there will be a +program that exists that can use 3 degrees of freedom (brake, +throttle, steering) to estimate the change of position over the next +given amount of time. Using this simulation and a given path, you must +identify what inputs are required to follow the given path. + +## Steps in the project +1. Understand the file format for the given path +1. Understand the capabilities of the vehicle dynamics simulations +1. Literature review over methods to follow a line +1. Design of a control loop +1. Implementation of input selecting for a short period of time +1. Optimization + +## Suggested direction From 09acad97642436759dae3f6fffc6b2b0e8d481c4 Mon Sep 17 00:00:00 2001 From: Daniel Rhee Date: Wed, 29 Oct 2025 18:46:01 -0700 Subject: [PATCH 7/7] Add Jaisree D. RaviKumar to active members list --- Projects/Yolact.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Projects/Yolact.md b/Projects/Yolact.md index d030e0d..b074a3a 100644 --- a/Projects/Yolact.md +++ b/Projects/Yolact.md @@ -3,6 +3,7 @@ ## Active Members 1. Abishek Adari 1. Dylan Price +1.Jaisree D. RaviKumar ## Design Review Dates/Timeline 1. Read the paper and PDR November 3rd