[New PowerToy] Create new OCR PowerToy

PowerToys.sln

@@ -415,6 +415,12 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ImageResizerContextMenu", "
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ImageResizerLib", "src\modules\imageresizer\ImageResizerLib\ImageResizerLib.vcxproj", "{18B3DB45-4FFE-4D01-97D6-5223FEEE1853}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PowerOCR", "PowerOCR", "{A50C70A6-2DA0-4027-B90E-B1A40755A8A5}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PowerOCRModuleInterface", "src\modules\PowerOCR\PowerOCRModuleInterface\PowerOCRModuleInterface.vcxproj", "{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PowerOCR", "src\modules\PowerOCR\PowerOCR\PowerOCR.csproj", "{25C91A4E-BA4E-467A-85CD-8B62545BF674}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|ARM64 = Debug|ARM64
@@ -1632,6 +1638,30 @@ Global
 		{18B3DB45-4FFE-4D01-97D6-5223FEEE1853}.Release|x64.Build.0 = Release|x64
 		{18B3DB45-4FFE-4D01-97D6-5223FEEE1853}.Release|x86.ActiveCfg = Release|x64
 		{18B3DB45-4FFE-4D01-97D6-5223FEEE1853}.Release|x86.Build.0 = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|ARM64.ActiveCfg = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|ARM64.Build.0 = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|x64.ActiveCfg = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|x64.Build.0 = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|x86.ActiveCfg = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Debug|x86.Build.0 = Debug|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|ARM64.ActiveCfg = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|ARM64.Build.0 = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|x64.ActiveCfg = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|x64.Build.0 = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|x86.ActiveCfg = Release|x64
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1}.Release|x86.Build.0 = Release|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|ARM64.Build.0 = Debug|ARM64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|x64.ActiveCfg = Debug|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|x64.Build.0 = Debug|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|x86.ActiveCfg = Debug|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Debug|x86.Build.0 = Debug|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|ARM64.ActiveCfg = Release|ARM64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|ARM64.Build.0 = Release|ARM64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|x64.ActiveCfg = Release|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|x64.Build.0 = Release|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|x86.ActiveCfg = Release|x64
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674}.Release|x86.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1770,6 +1800,9 @@ Global
 		{5A1DB2F0-0715-4B3B-98E6-79BC41540045} = {4AFC9975-2456-4C70-94A4-84073C1CED93}
 		{93B72A06-C8BD-484F-A6F7-C9F280B150BF} = {6C7F47CC-2151-44A3-A546-41C70025132C}
 		{18B3DB45-4FFE-4D01-97D6-5223FEEE1853} = {6C7F47CC-2151-44A3-A546-41C70025132C}
+		{A50C70A6-2DA0-4027-B90E-B1A40755A8A5} = {4574FDD0-F61D-4376-98BF-E5A1262C11EC}
+		{6AB6A2D6-F859-4A82-9184-0BD29C9F07D1} = {A50C70A6-2DA0-4027-B90E-B1A40755A8A5}
+		{25C91A4E-BA4E-467A-85CD-8B62545BF674} = {A50C70A6-2DA0-4027-B90E-B1A40755A8A5}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {C3A2F9D1-7930-4EF4-A6FC-7EE0A99821D0}

src/modules/PowerOCR/PowerOCR/App.xaml

@@ -0,0 +1,8 @@
+<Application
+    x:Class="PowerOCR.App"
+    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+    xmlns:local="clr-namespace:PowerOCR"
+    Startup="Application_Startup">
+    <Application.Resources />
+</Application>

src/modules/PowerOCR/PowerOCR/App.xaml.cs

@@ -0,0 +1,15 @@
+using PowerOCR.Utilities;
+using System.Windows;
+
+namespace PowerOCR;
+
+/// <summary>
+/// Interaction logic for App.xaml
+/// </summary>
+public partial class App : Application
+{
+    private void Application_Startup(object sender, StartupEventArgs e)
+    {
+        WindowUtilities.LaunchOCROverlayOnEveryScreen();
+    }
+}

src/modules/PowerOCR/PowerOCR/AssemblyInfo.cs

@@ -0,0 +1,10 @@
+using System.Windows;
+
+[assembly: ThemeInfo(
+    ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
+                                     //(used if a resource is not found in the page,
+                                     // or application resource dictionaries)
+    ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
+                                              //(used if a resource is not found in the page,
+                                              // app, or any theme specific resource dictionaries)
+)]

src/modules/PowerOCR/PowerOCR/ImageMethods.cs

@@ -0,0 +1,237 @@
+namespace PowerOCR;
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Input;
+using System.Windows.Markup;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using Windows.Globalization;
+using Windows.Graphics.Imaging;
+using Windows.Media.Ocr;
+using BitmapDecoder = Windows.Graphics.Imaging.BitmapDecoder;
+
+internal class ImageMethods
+{
+    internal static ImageSource GetWindowBoundsImage(Window passedWindow)
+    {
+        bool isGrabFrame = false;
+
+        DpiScale dpi = VisualTreeHelper.GetDpi(passedWindow);
+        int windowWidth = (int)(passedWindow.ActualWidth * dpi.DpiScaleX);
+        int windowHeight = (int)(passedWindow.ActualHeight * dpi.DpiScaleY);
+
+        System.Windows.Point absPosPoint = passedWindow.GetAbsolutePosition();
+        int thisCorrectedLeft = (int)(absPosPoint.X * dpi.DpiScaleX);
+        int thisCorrectedTop = (int)(absPosPoint.Y * dpi.DpiScaleY);
+
+        if (isGrabFrame == true)
+        {
+            thisCorrectedLeft += (int)(2 * dpi.DpiScaleX);
+            thisCorrectedTop += (int)(26 * dpi.DpiScaleY);
+            windowWidth -= (int)(4 * dpi.DpiScaleX);
+            windowHeight -= (int)(70 * dpi.DpiScaleY);
+        }
+
+        Bitmap bmp = new(windowWidth, windowHeight, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
+        Graphics g = Graphics.FromImage(bmp);
+
+        g.CopyFromScreen(thisCorrectedLeft, thisCorrectedTop, 0, 0, bmp.Size, CopyPixelOperation.SourceCopy);
+        return BitmapToImageSource(bmp);
+    }
+
+    internal static async Task<string> GetRegionsText(Window? passedWindow, Rectangle selectedRegion)
+    {
+        Bitmap bmp = new(selectedRegion.Width, selectedRegion.Height, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
+        Graphics g = Graphics.FromImage(bmp);
+
+        System.Windows.Point absPosPoint = passedWindow == null ? (new()) : passedWindow.GetAbsolutePosition();
+
+        int thisCorrectedLeft = (int)absPosPoint.X + selectedRegion.Left;
+        int thisCorrectedTop = (int)absPosPoint.Y + selectedRegion.Top;
+
+        g.CopyFromScreen(thisCorrectedLeft, thisCorrectedTop, 0, 0, bmp.Size, CopyPixelOperation.SourceCopy);
+        // bmp = PadImage(bmp);
+
+        string? ResultText = await ExtractText(bmp);
+
+        return ResultText != null ? ResultText.Trim() : "";
+    }
+
+    internal static async Task<string> GetClickedWord(Window passedWindow, System.Windows.Point clickedPoint)
+    {
+        DpiScale dpi = VisualTreeHelper.GetDpi(passedWindow);
+        Bitmap bmp = new((int)(passedWindow.ActualWidth * dpi.DpiScaleX), (int)(passedWindow.ActualHeight * dpi.DpiScaleY), System.Drawing.Imaging.PixelFormat.Format32bppArgb);
+        Graphics g = Graphics.FromImage(bmp);
+
+        System.Windows.Point absPosPoint = passedWindow.GetAbsolutePosition();
+        int thisCorrectedLeft = (int)absPosPoint.X;
+        int thisCorrectedTop = (int)absPosPoint.Y;
+
+        g.CopyFromScreen(thisCorrectedLeft, thisCorrectedTop, 0, 0, bmp.Size, CopyPixelOperation.SourceCopy);
+
+        System.Windows.Point adjustedPoint = new(clickedPoint.X, clickedPoint.Y);
+
+        string ResultText = await ExtractText(bmp, adjustedPoint);
+        return ResultText.Trim();
+    }
+
+    public static async Task<string> ExtractText(Bitmap bmp, System.Windows.Point? singlePoint = null)
+    {
+        Language? selectedLanguage = GetOCRLanguage();
+        if (selectedLanguage == null)
+            return "";
+
+        bool isCJKLang = false;
+
+        if (selectedLanguage.LanguageTag.StartsWith("zh", StringComparison.InvariantCultureIgnoreCase) == true)
+            isCJKLang = true;
+        else if (selectedLanguage.LanguageTag.StartsWith("ja", StringComparison.InvariantCultureIgnoreCase) == true)
+            isCJKLang = true;
+        else if (selectedLanguage.LanguageTag.StartsWith("ko", StringComparison.InvariantCultureIgnoreCase) == true)
+            isCJKLang = true;
+
+        XmlLanguage lang = XmlLanguage.GetLanguage(selectedLanguage.LanguageTag);
+        CultureInfo culture = lang.GetEquivalentCulture();
+
+        bool scaleBMP = true;
+
+        if (singlePoint != null
+            || bmp.Width * 1.5 > OcrEngine.MaxImageDimension)
+            scaleBMP = false;
+
+        Bitmap scaledBitmap = scaleBMP ? ScaleBitmapUniform(bmp, 1.5) : ScaleBitmapUniform(bmp, 1.0);
+        StringBuilder text = new();
+
+        await using (MemoryStream memory = new())
+        {
+            scaledBitmap.Save(memory, ImageFormat.Bmp);
+            memory.Position = 0;
+            BitmapDecoder bmpDecoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream());
+            SoftwareBitmap softwareBmp = await bmpDecoder.GetSoftwareBitmapAsync();
+
+            OcrEngine ocrEngine = OcrEngine.TryCreateFromLanguage(selectedLanguage);
+            OcrResult ocrResult = await ocrEngine.RecognizeAsync(softwareBmp);
+
+            if (singlePoint == null)
+                foreach (OcrLine line in ocrResult.Lines) text.AppendLine(line.Text);
+            else
+            {
+                Windows.Foundation.Point fPoint = new(singlePoint.Value.X, singlePoint.Value.Y);
+                foreach (OcrLine ocrLine in ocrResult.Lines)
+                    foreach (OcrWord ocrWord in ocrLine.Words)
+                        if (ocrWord.BoundingRect.Contains(fPoint))
+                            _ = text.Append(ocrWord.Text);
+            }
+        }
+        if (culture.TextInfo.IsRightToLeft)
+        {
+            string[] textListLines = text.ToString().Split(new char[] { '\n', '\r' });
+
+            _ = text.Clear();
+            foreach (string textLine in textListLines)
+            {
+                List<string> wordArray = textLine.Split().ToList();
+                wordArray.Reverse();
+                _ = isCJKLang == true ? text.Append(string.Join("", wordArray)) : text.Append(string.Join(' ', wordArray));
+
+                if (textLine.Length > 0)
+                    _ = text.Append('\n');
+            }
+            return text.ToString();
+        }
+        else
+        {
+            return text.ToString();
+        }
+    }
+
+    public static Bitmap ScaleBitmapUniform(Bitmap passedBitmap, double scale)
+    {
+        using MemoryStream memory = new();
+        passedBitmap.Save(memory, ImageFormat.Bmp);
+        memory.Position = 0;
+        BitmapImage bitmapimage = new();
+        bitmapimage.BeginInit();
+        bitmapimage.StreamSource = memory;
+        bitmapimage.CacheOption = BitmapCacheOption.OnLoad;
+        bitmapimage.EndInit();
+        TransformedBitmap transformedBmp = new();
+        transformedBmp.BeginInit();
+        transformedBmp.Source = bitmapimage;
+        transformedBmp.Transform = new ScaleTransform(scale, scale);
+        transformedBmp.EndInit();
+        return BitmapSourceToBitmap(transformedBmp.Source);
+    }
+
+    public static Bitmap BitmapSourceToBitmap(BitmapSource source)
+    {
+        Bitmap bmp = new(
+          source.PixelWidth,
+          source.PixelHeight,
+          System.Drawing.Imaging.PixelFormat.Format32bppPArgb);
+        BitmapData data = bmp.LockBits(
+          new Rectangle(System.Drawing.Point.Empty, bmp.Size),
+          ImageLockMode.WriteOnly,
+          System.Drawing.Imaging.PixelFormat.Format32bppPArgb);
+        source.CopyPixels(
+          Int32Rect.Empty,
+          data.Scan0,
+          data.Height * data.Stride,
+          data.Stride);
+        bmp.UnlockBits(data);
+        return bmp;
+    }
+
+    internal static BitmapImage BitmapToImageSource(Bitmap bitmap)
+    {
+        using MemoryStream memory = new();
+        bitmap.Save(memory, ImageFormat.Bmp);
+        memory.Position = 0;
+        BitmapImage bitmapimage = new();
+        bitmapimage.BeginInit();
+        bitmapimage.StreamSource = memory;
+        bitmapimage.CacheOption = BitmapCacheOption.OnLoad;
+        bitmapimage.EndInit();
+
+        return bitmapimage;
+    }
+
+    public static Language? GetOCRLanguage()
+    {
+        // use currently selected Language
+        string inputLang = InputLanguageManager.Current.CurrentInputLanguage.Name;
+
+        Language? selectedLanguage = new(inputLang);
+        List<Language> possibleOCRLangs = OcrEngine.AvailableRecognizerLanguages.ToList();
+
+        if (possibleOCRLangs.Count < 1)
+        {
+            MessageBox.Show("No possible OCR languages are installed.", "Text Grab");
+            return null;
+        }
+
+        if (possibleOCRLangs.All(l => l.LanguageTag != selectedLanguage.LanguageTag))
+        {
+            List<Language>? similarLanguages = possibleOCRLangs.Where(
+                la => la.AbbreviatedName == selectedLanguage.AbbreviatedName).ToList();
+
+            if (similarLanguages != null)
+            {
+                selectedLanguage = similarLanguages.Count > 0
+                    ? similarLanguages.FirstOrDefault()
+                    : possibleOCRLangs.FirstOrDefault();
+            }
+        }
+
+        return selectedLanguage;
+    }
+}

src/modules/PowerOCR/PowerOCR/OCROverlay.xaml

@@ -0,0 +1,54 @@
+<Window
+    x:Class="PowerOCR.OCROverlay"
+    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+    xmlns:local="clr-namespace:PowerOCR"
+    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+    Title="PowerOCR"
+    Width="800"
+    Height="450"
+    AllowsTransparency="True"
+    Background="Transparent"
+    Loaded="Window_Loaded"
+    ResizeMode="NoResize"
+    Topmost="True"
+    WindowStyle="None"
+    mc:Ignorable="d">
+    <Grid>
+        <Viewbox>
+            <Image x:Name="BackgroundImage" Stretch="UniformToFill" />
+        </Viewbox>
+        <Canvas
+            Name="RegionClickCanvas"
+            Cursor="Cross"
+            MouseDown="RegionClickCanvas_MouseDown"
+            MouseMove="RegionClickCanvas_MouseMove"
+            MouseUp="RegionClickCanvas_MouseUp">
+            <Canvas.Clip>
+                <CombinedGeometry GeometryCombineMode="Exclude">
+                    <CombinedGeometry.Geometry1>
+                        <RectangleGeometry x:Name="FullWindow" />
+                    </CombinedGeometry.Geometry1>
+                    <CombinedGeometry.Geometry2>
+                        <RectangleGeometry x:Name="clippingGeometry" />
+                    </CombinedGeometry.Geometry2>
+                </CombinedGeometry>
+            </Canvas.Clip>
+            <Canvas.Background>
+                <SolidColorBrush
+                    x:Name="BackgroundBrush"
+                    Opacity="0"
+                    Color="Black" />
+            </Canvas.Background>
+            <Canvas.ContextMenu>
+                <ContextMenu>
+                    <MenuItem
+                        Name="CancelMenuItem"
+                        Click="CancelMenuItem_Click"
+                        Header="Cancel" />
+                </ContextMenu>
+            </Canvas.ContextMenu>
+        </Canvas>
+    </Grid>
+</Window>