Segment Anything (SAM)

BiologyTools · Aug 9, 2023 · d17f73b · d17f73b
1 parent 2e9676a
commit d17f73b
Show file tree

Hide file tree

Showing 13 changed files with 951 additions and 45 deletions.
diff --git a/BioGTK.csproj b/BioGTK.csproj
@@ -85,6 +85,7 @@
     <None Remove="Glade\Tools.glade" />
     <None Remove="Info.plist" />
     <None Remove="Resources\polyline.png" />
+    <None Remove="SAM\Glade\form.glade" />
   </ItemGroup>
 
   <ItemGroup>
@@ -210,6 +211,9 @@
     <Content Include="Resources\zoom_out.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <EmbeddedResource Include="SAM\Glade\form.glade">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </EmbeddedResource>
   </ItemGroup>
 
   <ItemGroup>
@@ -251,6 +255,8 @@
     <PackageReference Include="Dotnet.Bundle" Version="0.9.13" />
     <PackageReference Include="GtkSharp" Version="3.24.24.95" />
     <PackageReference Include="IKVM" Version="8.6.2" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.15.1" />
     <PackageReference Include="NetVips" Version="2.3.1" />
     <PackageReference Include="NetVips.Native" Version="8.14.3" />
     <PackageReference Include="NetVips.Native.linux-arm64" Version="8.14.3" />

diff --git a/Glade/TabsView.glade b/Glade/TabsView.glade
@@ -238,6 +238,14 @@
                         <property name="use-stock">False</property>
                       </object>
                     </child>
+                    <child>
+                      <object class="GtkImageMenuItem" id="runSAMMenu">
+                        <property name="label">Run SAM</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="use-stock">False</property>
+                      </object>
+                    </child>
                   </object>
                 </child>
               </object>

diff --git a/Glade/TabsView.glade~ b/Glade/TabsView.glade~
@@ -133,7 +133,7 @@
                       </object>
                     </child>
                     <child>
-                      <object class="GtkImageMenuItem" id="savePyramidal">
+                      <object class="GtkImageMenuItem" id="savePyramidalMenu">
                         <property name="label" translatable="yes">Save Pyramidal</property>
                         <property name="visible">True</property>
                         <property name="can-focus">False</property>

diff --git a/SAM/CLIP.cs b/SAM/CLIP.cs
@@ -0,0 +1,117 @@
+using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Windows;
+
+namespace BioGTK
+{
+    /// <summary>
+    /// Image And Text Encoder
+    /// </summary>
+    class CLIP
+    {
+
+        InferenceSession mTxtEncoder;
+        InferenceSession mImgEncoder;
+        public static CLIP theSingleton = null;
+
+        protected CLIP()
+        {
+            Thread thread = new Thread(() =>
+            {
+                this.LoadONNXModel();
+            });
+            thread.Start();       
+        }
+        public static CLIP Instance()
+        {
+            if (null == theSingleton)
+            {
+                theSingleton = new CLIP();
+            }
+            return theSingleton;
+        }
+
+        /// <summary>
+        /// 加载CLIP模型
+        /// </summary>
+        void LoadONNXModel()
+        {
+            if (this.mTxtEncoder != null)
+                this.mTxtEncoder.Dispose();
+
+            if (this.mImgEncoder != null)
+                this.mImgEncoder.Dispose();
+            /*
+            string exePath = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
+            string txtencoder = exePath + @"\textual.onnx";
+            this.mTxtEncoder = new InferenceSession(txtencoder);
+            string imgencoder = exePath + @"\visual.onnx";
+            this.mImgEncoder = new InferenceSession(imgencoder);
+            */
+        }
+
+
+        /// <summary>
+        /// CLIP对文本进行编码
+        /// </summary>
+        public List<float> TxtEncoder(string txt)
+        {
+            //"a diagram", "a dog", "a cat"
+            List<Int64> token = new List<Int64>() {
+              49406,320,22697,49407,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,
+              49406,320,1929,49407,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,
+              49406,320,2368,49407,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0,0,0,0,0,0,0,0,
+              0,0,0,0,0 };
+            var txt_tensor = new DenseTensor<Int64>(token.ToArray(), new[] { 3, 77 });
+            var inputs = new List<NamedOnnxValue>
+            {
+                NamedOnnxValue.CreateFromTensor("input", txt_tensor)
+            };
+
+            var results = this.mTxtEncoder.Run(inputs);
+            var result = results.First().AsTensor<float>().ToArray();
+
+            return new List<float>();
+        }
+        /// <summary>
+        /// CLIP对图像进行编码
+        /// </summary>
+        public List<float> ImgEncoder(float[] img)
+        {
+            var tensor = new DenseTensor<float>(img, new[] { 1, 3, 224, 224 });
+            var inputs = new List<NamedOnnxValue>
+            {
+                NamedOnnxValue.CreateFromTensor("input", tensor)
+            };
+
+            var results = this.mImgEncoder.Run(inputs);
+            var result = results.First().AsTensor<float>().ToArray();
+
+            return new List<float>();
+        }
+    }
+}
diff --git a/SAM/Glade/form.glade b/SAM/Glade/form.glade
@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.40.0 -->
+<interface>
+  <requires lib="gtk+" version="3.20"/>
+  <object class="GtkWindow" id="SAMTool">
+    <property name="name">tabsView</property>
+    <property name="can-focus">False</property>
+    <property name="title" translatable="yes">SAM Tool</property>
+    <property name="resizable">False</property>
+    <property name="default-width">200</property>
+    <child>
+      <object class="GtkBox">
+        <property name="visible">True</property>
+        <property name="can-focus">False</property>
+        <property name="orientation">vertical</property>
+        <child>
+          <object class="GtkMenuBar" id="MainMenu">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <child>
+              <object class="GtkMenuItem" id="fileMenu">
+                <property name="name">fileMenu</property>
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="margin-bottom">1</property>
+                <property name="label" translatable="yes">File</property>
+                <property name="use-underline">True</property>
+                <child type="submenu">
+                  <object class="GtkMenu">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <child>
+                      <object class="GtkImageMenuItem" id="openImagesMenu">
+                        <property name="label">Open Images</property>
+                        <property name="name">openImagesMenu</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="use-stock">False</property>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkImageMenuItem" id="saveSelected">
+                        <property name="label" translatable="yes">Save Selected Tiff</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="use-stock">False</property>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkImageMenuItem" id="setAutoSave">
+                        <property name="label" translatable="yes">Set Auto Save Path</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="use-stock">False</property>
+                      </object>
+                    </child>
+                  </object>
+                </child>
+              </object>
+            </child>
+            <child>
+              <object class="GtkMenuItem" id="ToolsMenu">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="label" translatable="yes">Tools</property>
+                <child type="submenu">
+                  <object class="GtkMenu">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <child>
+                      <object class="GtkCheckMenuItem" id="addMaskMenu">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">Add Mask</property>
+                        <property name="active">True</property>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkCheckMenuItem" id="removeMaskMenu">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">Remove Mask</property>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkCheckMenuItem" id="autoSaveMenu">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">Auto Save</property>
+                        <property name="active">True</property>
+                      </object>
+                    </child>
+                  </object>
+                </child>
+              </object>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">True</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkEntry" id="textBox">
+            <property name="visible">True</property>
+            <property name="can-focus">True</property>
+            <property name="text" translatable="yes">EnterMaskName</property>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+  </object>
+</interface>
diff --git a/SAM/Promotions.cs b/SAM/Promotions.cs
@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BioGTK
+{
+    public enum PromotionType
+    {
+        Point,
+        Box
+    }
+
+    public abstract class Promotion
+    {
+        public abstract float[] GetInput();
+        public abstract float[] GetLable();
+        public PromotionType mType;
+    }
+    /// <summary>
+    /// 提示点
+    /// </summary>
+    public class PointPromotion: Promotion
+    {
+        public PointPromotion(OpType optype)
+        {
+            this.mType = PromotionType.Point;
+            this.m_Optype = optype;
+        }
+        public int X { get; set; }
+        public int Y { get; set; }
+        public override float[] GetInput()
+        {
+            return new float[2] { X ,Y};
+        }
+        public override float[] GetLable()
+        {
+            if (this.m_Optype == OpType.ADD)
+            {
+                return new float[1] { 1 };
+            }
+            else
+            {
+                return new float[1] { 0 };
+            }          
+        }
+
+        public OpType m_Optype;
+    }
+    public enum OpType
+    {
+        ADD,
+        REMOVE
+    }
+    /// <summary>
+    /// 提示框
+    /// </summary>
+    class BoxPromotion : Promotion
+    {
+        public BoxPromotion()
+        {
+            this.mLeftUp = new PointPromotion(OpType.ADD);
+            this.mRightBottom = new PointPromotion(OpType.ADD);
+            this.mType = PromotionType.Box;
+        }
+        public override float[] GetInput()
+        {
+            return new float[4] { this.mLeftUp.X, 
+                this.mLeftUp.Y, 
+                this.mRightBottom.X, 
+                this.mRightBottom.Y };
+        }
+        public override float[] GetLable()
+        {
+            return new float[2] { 2,3 };
+        }
+        public PointPromotion mLeftUp { get; set; }//左上角点
+        public PointPromotion mRightBottom { get; set; }//左上角点
+
+    }
+    /// <summary>
+    /// 提示蒙版
+    /// </summary>
+    class MaskPromotion
+    {
+        public MaskPromotion(int wid,int hei)
+        {
+            this.mWidth = wid;
+            this.mHeight = hei;
+            this.mMask = new float[this.mWidth,this.mHeight];
+        }
+
+        float[,] mMask { get; set; }//蒙版
+        public int mWidth { get; set; }//长度
+        public int mHeight { get; set; }//高度
+    }
+}