diff --git a/src/Translumo.OCR/WindowsOCR/PositionalOcrLine.cs b/src/Translumo.OCR/WindowsOCR/PositionalOcrLine.cs
new file mode 100644
index 00000000..c32cda66
--- /dev/null
+++ b/src/Translumo.OCR/WindowsOCR/PositionalOcrLine.cs
@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+using System.Drawing;
+
+namespace Translumo.OCR.WindowsOCR
+{
+ ///
+ /// A single recognized text line together with its pixel bounding box inside the source image.
+ ///
+ public sealed class PositionalOcrLine
+ {
+ public string Text { get; init; }
+
+ /// Bounding box in source-image pixels.
+ public RectangleF Box { get; init; }
+ }
+
+ ///
+ /// Result of recognizing a captured region: the chosen recognizer language, the recognized
+ /// lines with positions, and the source image size (pixels).
+ ///
+ public sealed class OcrRegionResult
+ {
+ public string LanguageTag { get; init; }
+
+ public IReadOnlyList Lines { get; init; }
+
+ public int ImageWidth { get; init; }
+
+ public int ImageHeight { get; init; }
+ }
+}
diff --git a/src/Translumo.OCR/WindowsOCR/WindowsOcrPositional.cs b/src/Translumo.OCR/WindowsOCR/WindowsOcrPositional.cs
new file mode 100644
index 00000000..fdc0cce0
--- /dev/null
+++ b/src/Translumo.OCR/WindowsOCR/WindowsOcrPositional.cs
@@ -0,0 +1,241 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Windows.Globalization;
+using Windows.Graphics.Imaging;
+using Windows.Media.Ocr;
+using Windows.Storage.Streams;
+
+namespace Translumo.OCR.WindowsOCR
+{
+ ///
+ /// Positional Windows OCR used by the instant image-translation feature.
+ /// Unlike (plain text), this keeps per-line bounding boxes and
+ /// can auto-detect the source language by trying the installed recognizer packs and scoring the
+ /// result by how many characters match each language's writing system (the UWP OCR API does not
+ /// expose per-word confidence, so a script-match heuristic is used instead).
+ ///
+ public static class WindowsOcrPositional
+ {
+ /// Installed Windows OCR recognizers as (BCP-47 tag, human-readable name).
+ public static IReadOnlyList<(string Tag, string DisplayName)> GetInstalledRecognizers()
+ {
+ return OcrEngine.AvailableRecognizerLanguages
+ .Select(l => (l.LanguageTag, l.DisplayName))
+ .ToArray();
+ }
+
+ ///
+ /// Recognizes (encoded bytes). When
+ /// is null the source language is auto-detected across installed recognizer packs.
+ /// Returns null when no OCR recognizer is available at all.
+ ///
+ public static async Task DetectAndRecognizeAsync(byte[] image, string forcedLanguageTag = null)
+ {
+ var (softwareBitmap, width, height) = await DecodeAsync(image).ConfigureAwait(false);
+
+ IReadOnlyList candidates = forcedLanguageTag != null
+ ? new[] { forcedLanguageTag }
+ : GetCandidateLanguageTags();
+
+ if (candidates.Count == 0)
+ {
+ return null;
+ }
+
+ OcrRegionResult best = null;
+ var bestScore = -1;
+ foreach (var tag in candidates)
+ {
+ var engine = OcrEngine.TryCreateFromLanguage(new Language(tag));
+ if (engine == null)
+ {
+ continue;
+ }
+
+ OcrResult ocr = await engine.RecognizeAsync(softwareBitmap).AsTask().ConfigureAwait(false);
+ var lines = ExtractLines(ocr);
+ var score = ScoreForLanguage(ocr.Text, tag);
+
+ if (best == null || score > bestScore)
+ {
+ bestScore = score;
+ best = new OcrRegionResult
+ {
+ LanguageTag = tag,
+ Lines = lines,
+ ImageWidth = width,
+ ImageHeight = height
+ };
+ }
+ }
+
+ return best;
+ }
+
+ private static async Task<(SoftwareBitmap bitmap, int width, int height)> DecodeAsync(byte[] image)
+ {
+ using var memory = new MemoryStream(image);
+ var decoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream()).AsTask().ConfigureAwait(false);
+ var bitmap = await decoder.GetSoftwareBitmapAsync().AsTask().ConfigureAwait(false);
+
+ return (bitmap, (int)decoder.PixelWidth, (int)decoder.PixelHeight);
+ }
+
+ private static List ExtractLines(OcrResult ocr)
+ {
+ var result = new List();
+ foreach (var line in ocr.Lines)
+ {
+ if (line.Words.Count == 0)
+ {
+ continue;
+ }
+
+ double left = double.MaxValue, top = double.MaxValue, right = 0, bottom = 0;
+ foreach (var word in line.Words)
+ {
+ var r = word.BoundingRect;
+ left = Math.Min(left, r.X);
+ top = Math.Min(top, r.Y);
+ right = Math.Max(right, r.X + r.Width);
+ bottom = Math.Max(bottom, r.Y + r.Height);
+ }
+
+ var text = line.Text?.Trim();
+ if (string.IsNullOrEmpty(text))
+ {
+ continue;
+ }
+
+ result.Add(new PositionalOcrLine
+ {
+ Text = text,
+ Box = new System.Drawing.RectangleF(
+ (float)left, (float)top, (float)(right - left), (float)(bottom - top))
+ });
+ }
+
+ return result;
+ }
+
+ ///
+ /// One representative installed recognizer per writing system. CJK scripts (zh/ja/ko) are kept
+ /// separate because they need different packs; the many Latin/Cyrillic packs collapse to one
+ /// representative each to keep the number of OCR passes small.
+ ///
+ private static IReadOnlyList GetCandidateLanguageTags()
+ {
+ var byFamily = new Dictionary();
+ foreach (var lang in OcrEngine.AvailableRecognizerLanguages)
+ {
+ var tag = lang.LanguageTag;
+ var family = ScriptFamily(tag);
+ // Keep the canonical representative when available, otherwise the first seen.
+ if (!byFamily.ContainsKey(family) || IsCanonicalRepresentative(tag, family))
+ {
+ byFamily[family] = tag;
+ }
+ }
+
+ return byFamily.Values.ToArray();
+ }
+
+ private static bool IsCanonicalRepresentative(string tag, string family)
+ {
+ var shortTag = ShortTag(tag);
+ return family switch
+ {
+ "latin" => shortTag == "en",
+ "cyrillic" => shortTag == "ru",
+ "arabic" => shortTag == "ar",
+ "greek" => shortTag == "el",
+ _ => false
+ };
+ }
+
+ private static string ScriptFamily(string tag)
+ {
+ var s = ShortTag(tag);
+ switch (s)
+ {
+ case "zh":
+ return "zh";
+ case "ja":
+ return "ja";
+ case "ko":
+ return "ko";
+ case "ru":
+ case "uk":
+ case "be":
+ case "bg":
+ case "sr":
+ case "mk":
+ return "cyrillic";
+ case "ar":
+ case "fa":
+ case "ur":
+ return "arabic";
+ case "el":
+ return "greek";
+ default:
+ return "latin";
+ }
+ }
+
+ private static string ShortTag(string tag) => tag.Split('-')[0].ToLowerInvariant();
+
+ /// Number of characters in the recognized text that belong to the language's script.
+ private static int ScoreForLanguage(string text, string tag)
+ {
+ if (string.IsNullOrEmpty(text))
+ {
+ return 0;
+ }
+
+ var family = ScriptFamily(tag);
+ var score = 0;
+ foreach (var c in text)
+ {
+ if (MatchesScript(c, family))
+ {
+ score++;
+ }
+ }
+
+ return score;
+ }
+
+ private static bool MatchesScript(char c, string family)
+ {
+ switch (family)
+ {
+ case "zh":
+ return IsCjk(c);
+ case "ja":
+ return IsCjk(c) || IsKana(c);
+ case "ko":
+ return IsHangul(c);
+ case "cyrillic":
+ return c >= 0x0400 && c <= 0x04FF;
+ case "arabic":
+ return (c >= 0x0600 && c <= 0x06FF) || (c >= 0x0750 && c <= 0x077F) ||
+ (c >= 0xFB50 && c <= 0xFDFF) || (c >= 0xFE70 && c <= 0xFEFF);
+ case "greek":
+ return c >= 0x0370 && c <= 0x03FF;
+ default:
+ return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
+ (c >= 0x00C0 && c <= 0x024F);
+ }
+ }
+
+ private static bool IsCjk(char c) => (c >= 0x4E00 && c <= 0x9FFF) || (c >= 0x3400 && c <= 0x4DBF);
+
+ private static bool IsKana(char c) => c >= 0x3040 && c <= 0x30FF;
+
+ private static bool IsHangul(char c) =>
+ (c >= 0xAC00 && c <= 0xD7A3) || (c >= 0x1100 && c <= 0x11FF) || (c >= 0x3130 && c <= 0x318F);
+ }
+}
diff --git a/src/Translumo.Processing/ImageTranslation/ImageTranslationResult.cs b/src/Translumo.Processing/ImageTranslation/ImageTranslationResult.cs
new file mode 100644
index 00000000..43718a27
--- /dev/null
+++ b/src/Translumo.Processing/ImageTranslation/ImageTranslationResult.cs
@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+using System.Drawing;
+
+namespace Translumo.Processing.ImageTranslation
+{
+ /// One OCR line with its translation and pixel bounding box (image-relative).
+ public sealed class TranslatedLine
+ {
+ public string Source { get; init; }
+
+ public string Translation { get; init; }
+
+ public RectangleF Box { get; init; }
+ }
+
+ /// Full result of translating a captured region (Google Lens style).
+ public sealed class ImageTranslationResult
+ {
+ public IReadOnlyList Lines { get; init; } = new List();
+
+ /// BCP-47 tag of the recognizer used (e.g. "en-US"); null when nothing was detected.
+ public string DetectedLanguageTag { get; init; }
+
+ public int ImageWidth { get; init; }
+
+ public int ImageHeight { get; init; }
+
+ public bool HasText => Lines.Count > 0;
+ }
+}
diff --git a/src/Translumo.Processing/ImageTranslation/ImageTranslationService.cs b/src/Translumo.Processing/ImageTranslation/ImageTranslationService.cs
new file mode 100644
index 00000000..0be41e31
--- /dev/null
+++ b/src/Translumo.Processing/ImageTranslation/ImageTranslationService.cs
@@ -0,0 +1,102 @@
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Translumo.Infrastructure.Language;
+using Translumo.OCR.WindowsOCR;
+using Translumo.Translation.Google;
+
+namespace Translumo.Processing.ImageTranslation
+{
+ ///
+ /// Orchestrates the instant image-translation ("Google Lens") flow: positional OCR of a captured
+ /// region (with source-language auto-detect) + per-line translation with source auto-detect.
+ /// The WPF layer captures the region bytes and renders the returned lines over their boxes.
+ ///
+ public sealed class ImageTranslationService
+ {
+ private const int MAX_CONCURRENT_TRANSLATIONS = 3;
+
+ private readonly LanguageService _languageService;
+ private readonly ILogger _logger;
+ private readonly AutoSourceGoogleTranslator _translator = new AutoSourceGoogleTranslator();
+
+ public ImageTranslationService(LanguageService languageService, ILogger logger)
+ {
+ _languageService = languageService;
+ _logger = logger;
+ }
+
+ /// Installed Windows OCR source languages as (BCP-47 tag, display name), for the override dropdown.
+ public IReadOnlyList<(string Tag, string DisplayName)> GetAvailableSourceLanguages()
+ {
+ return WindowsOcrPositional.GetInstalledRecognizers();
+ }
+
+ /// Encoded screenshot bytes of the selected region.
+ /// BCP-47 recognizer tag to force, or null to auto-detect.
+ /// Target translation language.
+ public async Task TranslateRegionAsync(byte[] regionImage, string forcedSourceTag, Languages target)
+ {
+ var ocr = await WindowsOcrPositional.DetectAndRecognizeAsync(regionImage, forcedSourceTag).ConfigureAwait(false);
+ if (ocr == null || ocr.Lines.Count == 0)
+ {
+ return new ImageTranslationResult
+ {
+ DetectedLanguageTag = ocr?.LanguageTag,
+ ImageWidth = ocr?.ImageWidth ?? 0,
+ ImageHeight = ocr?.ImageHeight ?? 0
+ };
+ }
+
+ var targetIso = _languageService.GetLanguageDescriptor(target).IsoCode;
+ var translations = await TranslateLinesAsync(ocr.Lines.Select(l => l.Text), targetIso).ConfigureAwait(false);
+
+ var lines = ocr.Lines
+ .Select(l => new TranslatedLine
+ {
+ Source = l.Text,
+ Translation = translations.TryGetValue(l.Text, out var tr) ? tr : l.Text,
+ Box = l.Box
+ })
+ .ToList();
+
+ return new ImageTranslationResult
+ {
+ Lines = lines,
+ DetectedLanguageTag = ocr.LanguageTag,
+ ImageWidth = ocr.ImageWidth,
+ ImageHeight = ocr.ImageHeight
+ };
+ }
+
+ private async Task> TranslateLinesAsync(IEnumerable sources, string targetIso)
+ {
+ var distinct = sources.Distinct().ToList();
+ var map = new ConcurrentDictionary();
+ using var throttle = new SemaphoreSlim(MAX_CONCURRENT_TRANSLATIONS);
+
+ await Task.WhenAll(distinct.Select(async src =>
+ {
+ await throttle.WaitAsync().ConfigureAwait(false);
+ try
+ {
+ map[src] = await _translator.TranslateAsync(src, targetIso).ConfigureAwait(false);
+ }
+ catch (System.Exception ex)
+ {
+ _logger.LogWarning(ex, "Image line translation failed; keeping source text");
+ map[src] = src;
+ }
+ finally
+ {
+ throttle.Release();
+ }
+ })).ConfigureAwait(false);
+
+ return map;
+ }
+ }
+}
diff --git a/src/Translumo.Translation/Google/AutoSourceGoogleTranslator.cs b/src/Translumo.Translation/Google/AutoSourceGoogleTranslator.cs
new file mode 100644
index 00000000..a2a8af28
--- /dev/null
+++ b/src/Translumo.Translation/Google/AutoSourceGoogleTranslator.cs
@@ -0,0 +1,44 @@
+using System.Net;
+using System.Threading.Tasks;
+using System.Web;
+using Translumo.Infrastructure.Constants;
+using Translumo.Translation.Exceptions;
+using Translumo.Utils.Http;
+
+namespace Translumo.Translation.Google
+{
+ ///
+ /// Lightweight Google translator used by the instant image-translation feature. It always sends
+ /// sl=auto so Google detects the source language server-side, and lets the target language
+ /// be chosen per request. It reuses (proxy-aware reader) and
+ /// without touching the config-bound
+ /// used by continuous translation.
+ ///
+ public sealed class AutoSourceGoogleTranslator
+ {
+ private const string TRANSLATE_URL = "https://translate.google.com/m?hl={0}&sl=auto&tl={0}&ie=UTF-8&prev=_m&q={1}";
+
+ private readonly GoogleContainer _container = new GoogleContainer(isPrimary: true);
+
+ public async Task TranslateAsync(string sourceText, string targetIsoCode)
+ {
+ if (string.IsNullOrWhiteSpace(sourceText))
+ {
+ return sourceText;
+ }
+
+ var url = string.Format(TRANSLATE_URL, targetIsoCode, HttpUtility.UrlEncode(sourceText));
+ var response = await _container.Reader.RequestWebDataAsync(url, HttpMethods.GET, true).ConfigureAwait(false);
+ if (response.IsSuccessful)
+ {
+ var match = RegexStorage.GoogleTranslateResultRegex.Match(response.Body);
+ if (match.Success)
+ {
+ return WebUtility.HtmlDecode(match.Value);
+ }
+ }
+
+ throw new TranslationException($"Unexpected web response: '{response.Body}'");
+ }
+ }
+}
diff --git a/src/Translumo/App.xaml.cs b/src/Translumo/App.xaml.cs
index c3923966..502ec6b2 100644
--- a/src/Translumo/App.xaml.cs
+++ b/src/Translumo/App.xaml.cs
@@ -150,6 +150,8 @@ private void ConfigureServices(ServiceCollection services)
services.AddSingleton();
services.AddSingleton();
+ services.AddSingleton();
+
services.AddTransient();
services.AddTransient();
services.AddTransient();
diff --git a/src/Translumo/HotKeys/HotKeysConfiguration.cs b/src/Translumo/HotKeys/HotKeysConfiguration.cs
index 89b6e7dc..c8387b05 100644
--- a/src/Translumo/HotKeys/HotKeysConfiguration.cs
+++ b/src/Translumo/HotKeys/HotKeysConfiguration.cs
@@ -14,6 +14,7 @@ public class HotKeysConfiguration : BindableBase
TranslationStateKey = new HotKeyInfo(Key.OemTilde, KeyModifier.None),
ShowSelectionAreaKey = new HotKeyInfo(Key.Y, KeyModifier.Alt),
OnceTranslateKey = new HotKeyInfo(Key.F, KeyModifier.Shift),
+ ImageTranslateKey = new HotKeyInfo(Key.D, KeyModifier.Alt),
WindowStyleChangeKey = new HotKeyInfo(Key.T, KeyModifier.Ctrl),
ChatVisibilityGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None),
@@ -22,6 +23,7 @@ public class HotKeysConfiguration : BindableBase
TranslationStateGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None),
ShowSelectionAreaGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None),
OnceTranslateGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None),
+ ImageTranslateGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None),
WindowStyleChangeGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None)
};
@@ -79,6 +81,15 @@ public HotKeyInfo OnceTranslateKey
}
}
+ public HotKeyInfo ImageTranslateKey
+ {
+ get => _imageTranslateKey;
+ set
+ {
+ SetProperty(ref _imageTranslateKey, value);
+ }
+ }
+
public HotKeyInfo WindowStyleChangeKey
{
get => _windowStyleChangeKey;
@@ -144,6 +155,15 @@ public GamepadHotKeyInfo OnceTranslateGamepadKey
}
}
+ public GamepadHotKeyInfo ImageTranslateGamepadKey
+ {
+ get => _imageTranslateGamepadKey;
+ set
+ {
+ SetProperty(ref _imageTranslateGamepadKey, value);
+ }
+ }
+
public GamepadHotKeyInfo WindowStyleChangeGamepadKey
{
get => _windowStyleChangeGamepadKey;
@@ -159,6 +179,7 @@ public GamepadHotKeyInfo WindowStyleChangeGamepadKey
private HotKeyInfo _settingVisibilityKey;
private HotKeyInfo _showSelectionAreaKey;
private HotKeyInfo _onceTranslateKey;
+ private HotKeyInfo _imageTranslateKey = new HotKeyInfo(Key.D, KeyModifier.Alt);
private HotKeyInfo _windowStyleChangeKey = new HotKeyInfo(Key.T, KeyModifier.Ctrl);
private GamepadHotKeyInfo _chatVisibilityGamepadKey;
@@ -167,6 +188,7 @@ public GamepadHotKeyInfo WindowStyleChangeGamepadKey
private GamepadHotKeyInfo _settingVisibilityGamepadKey;
private GamepadHotKeyInfo _showSelctionAreaGamepadKey;
private GamepadHotKeyInfo _onceTranslateGamepadKey;
+ private GamepadHotKeyInfo _imageTranslateGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None);
private GamepadHotKeyInfo _windowStyleChangeGamepadKey = new GamepadHotKeyInfo(GamepadKeyCode.None);
}
}
diff --git a/src/Translumo/HotKeys/HotKeysServiceManager.cs b/src/Translumo/HotKeys/HotKeysServiceManager.cs
index a3ee1a20..9e7a9b1d 100644
--- a/src/Translumo/HotKeys/HotKeysServiceManager.cs
+++ b/src/Translumo/HotKeys/HotKeysServiceManager.cs
@@ -16,6 +16,7 @@ public class HotKeysServiceManager
public event EventHandler SettingVisibilityKeyPressed;
public event EventHandler ShowSelectionAreaKeyPressed;
public event EventHandler OnceTranslateKeyPressed;
+ public event EventHandler ImageTranslateKeyPressed;
public event EventHandler WindowStyleChangeKeyPressed;
public HotKeysConfiguration Configuration { get; }
@@ -41,6 +42,7 @@ public HotKeysServiceManager(HotKeysConfiguration configuration, IControllerInpu
(nameof(configuration.SettingVisibilityKey), nameof(configuration.SettingVisibilityGamepadKey)),
(nameof(configuration.ShowSelectionAreaKey), nameof(configuration.ShowSelectionAreaGamepadKey)),
(nameof(configuration.OnceTranslateKey), nameof(configuration.OnceTranslateGamepadKey)),
+ (nameof(configuration.ImageTranslateKey), nameof(configuration.ImageTranslateGamepadKey)),
(nameof(configuration.WindowStyleChangeKey), nameof(configuration.WindowStyleChangeGamepadKey)),
};
@@ -195,6 +197,11 @@ private void OnOnceTranslatePressed()
OnceTranslateKeyPressed?.Invoke(this, EventArgs.Empty);
}
+ private void OnImageTranslatePressed()
+ {
+ ImageTranslateKeyPressed?.Invoke(this, EventArgs.Empty);
+ }
+
private void OnWindowStyleChangePressed()
{
WindowStyleChangeKeyPressed?.Invoke(this, EventArgs.Empty);
@@ -228,6 +235,10 @@ private IDictionary InitializeHotKeys(HotKeysConfiguration confi
nameof(configuration.OnceTranslateKey), new HotKey(configuration.OnceTranslateKey.Key,
configuration.OnceTranslateKey.KeyModifier, OnOnceTranslatePressed)
},
+ {
+ nameof(configuration.ImageTranslateKey), new HotKey(configuration.ImageTranslateKey.Key,
+ configuration.ImageTranslateKey.KeyModifier, OnImageTranslatePressed)
+ },
{
nameof(configuration.WindowStyleChangeKey), new HotKey(configuration.WindowStyleChangeKey.Key,
configuration.WindowStyleChangeKey.KeyModifier, OnWindowStyleChangePressed)
@@ -245,6 +256,7 @@ private IDictionary InitializeGamepadHotKeys(HotKeysConfi
{ nameof(configuration.SettingVisibilityGamepadKey), new GamepadHotKey(configuration.SettingVisibilityGamepadKey.Key, OnSettingVisibilityPressed) },
{ nameof(configuration.ShowSelectionAreaGamepadKey), new GamepadHotKey(configuration.ShowSelectionAreaGamepadKey.Key, OnShowSelectionAreaPressed) },
{ nameof(configuration.OnceTranslateGamepadKey), new GamepadHotKey(configuration.OnceTranslateGamepadKey.Key, OnOnceTranslatePressed) },
+ { nameof(configuration.ImageTranslateGamepadKey), new GamepadHotKey(configuration.ImageTranslateGamepadKey.Key, OnImageTranslatePressed) },
{ nameof(configuration.WindowStyleChangeGamepadKey), new GamepadHotKey(configuration.WindowStyleChangeGamepadKey.Key, OnWindowStyleChangePressed) }
};
}
diff --git a/src/Translumo/ImageTranslationOverlayWindow.xaml b/src/Translumo/ImageTranslationOverlayWindow.xaml
new file mode 100644
index 00000000..b22877c3
--- /dev/null
+++ b/src/Translumo/ImageTranslationOverlayWindow.xaml
@@ -0,0 +1,92 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Translumo/ImageTranslationOverlayWindow.xaml.cs b/src/Translumo/ImageTranslationOverlayWindow.xaml.cs
new file mode 100644
index 00000000..ae05774b
--- /dev/null
+++ b/src/Translumo/ImageTranslationOverlayWindow.xaml.cs
@@ -0,0 +1,281 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Input;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using Translumo.Infrastructure.Language;
+using Translumo.Processing.ImageTranslation;
+using Point = System.Windows.Point;
+using RectangleF = System.Drawing.RectangleF;
+
+namespace Translumo
+{
+ ///
+ /// Google Lens style overlay: shows a frozen snapshot of the selected region and paints the
+ /// translation of each recognized line over its original position. A floating toolbar lets the
+ /// user override the source recognizer language and pick the target language (both re-run the
+ /// pipeline via the retranslate delegate) and copy all translations.
+ ///
+ public partial class ImageTranslationOverlayWindow : Window
+ {
+ public sealed class TargetLanguageOption
+ {
+ public Languages Value { get; init; }
+ public string Display { get; init; }
+ public override string ToString() => Display;
+ }
+
+ public sealed class SourceLanguageOption
+ {
+ /// BCP-47 recognizer tag, or null for auto-detect.
+ public string Tag { get; init; }
+ public string Display { get; init; }
+ public override string ToString() => Display;
+ }
+
+ private readonly RectangleF _regionScreenPx;
+ private readonly byte[] _regionImage;
+ private readonly IReadOnlyList _targetOptions;
+ private readonly IReadOnlyList _sourceOptions;
+ private readonly Languages _initialTarget;
+ private readonly Func> _retranslate;
+
+ private ImageTranslationResult _result;
+ private Point _dipTopLeft;
+ private Point _dipBottomRight;
+ private Image _snapshot;
+ private bool _suppressEvents;
+
+ public ImageTranslationOverlayWindow(
+ RectangleF regionScreenPx,
+ byte[] regionImage,
+ ImageTranslationResult initialResult,
+ IReadOnlyList targetOptions,
+ IReadOnlyList sourceOptions,
+ Languages initialTarget,
+ Func> retranslate)
+ {
+ InitializeComponent();
+
+ _regionScreenPx = regionScreenPx;
+ _regionImage = regionImage;
+ _result = initialResult;
+ _targetOptions = targetOptions;
+ _sourceOptions = sourceOptions;
+ _initialTarget = initialTarget;
+ _retranslate = retranslate;
+ }
+
+ private void OnLoaded(object sender, RoutedEventArgs e)
+ {
+ _dipTopLeft = PointFromScreen(new Point(_regionScreenPx.X, _regionScreenPx.Y));
+ _dipBottomRight = PointFromScreen(new Point(_regionScreenPx.Right, _regionScreenPx.Bottom));
+
+ AddSnapshot();
+ PopulateCombos();
+ UpdateDetectedLabel();
+ RenderBoxes();
+ }
+
+ private void AddSnapshot()
+ {
+ try
+ {
+ var bitmap = new BitmapImage();
+ using var ms = new MemoryStream(_regionImage);
+ bitmap.BeginInit();
+ bitmap.CacheOption = BitmapCacheOption.OnLoad;
+ bitmap.StreamSource = ms;
+ bitmap.EndInit();
+ bitmap.Freeze();
+
+ _snapshot = new Image
+ {
+ Source = bitmap,
+ Stretch = Stretch.Fill,
+ Width = Math.Max(1, _dipBottomRight.X - _dipTopLeft.X),
+ Height = Math.Max(1, _dipBottomRight.Y - _dipTopLeft.Y)
+ };
+ Canvas.SetLeft(_snapshot, _dipTopLeft.X);
+ Canvas.SetTop(_snapshot, _dipTopLeft.Y);
+ LayerCanvas.Children.Add(_snapshot);
+ }
+ catch
+ {
+ // Snapshot is only a visual backdrop; boxes still render without it.
+ }
+ }
+
+ private void PopulateCombos()
+ {
+ _suppressEvents = true;
+
+ TargetCombo.ItemsSource = _targetOptions;
+ TargetCombo.SelectedItem = _targetOptions.FirstOrDefault(o => o.Value == _initialTarget)
+ ?? _targetOptions.FirstOrDefault();
+
+ var sources = new List
+ {
+ new SourceLanguageOption { Tag = null, Display = "Auto (detect)" }
+ };
+ sources.AddRange(_sourceOptions);
+ SourceCombo.ItemsSource = sources;
+ SourceCombo.SelectedIndex = 0;
+
+ _suppressEvents = false;
+ }
+
+ private (double scaleX, double scaleY) ComputeScale()
+ {
+ var regionW = Math.Max(1, _dipBottomRight.X - _dipTopLeft.X);
+ var regionH = Math.Max(1, _dipBottomRight.Y - _dipTopLeft.Y);
+ var imgW = _result != null && _result.ImageWidth > 0 ? _result.ImageWidth : regionW;
+ var imgH = _result != null && _result.ImageHeight > 0 ? _result.ImageHeight : regionH;
+
+ return (regionW / imgW, regionH / imgH);
+ }
+
+ private void RenderBoxes()
+ {
+ for (var i = LayerCanvas.Children.Count - 1; i >= 0; i--)
+ {
+ if (LayerCanvas.Children[i] is Border b && (string)b.Tag == "line")
+ {
+ LayerCanvas.Children.RemoveAt(i);
+ }
+ }
+
+ if (_result == null || _result.Lines.Count == 0)
+ {
+ return;
+ }
+
+ var (scaleX, scaleY) = ComputeScale();
+ foreach (var line in _result.Lines)
+ {
+ var x = _dipTopLeft.X + line.Box.X * scaleX;
+ var y = _dipTopLeft.Y + line.Box.Y * scaleY;
+ var h = Math.Max(1, line.Box.Height * scaleY);
+
+ var textBlock = new TextBlock
+ {
+ Text = line.Translation,
+ Foreground = new SolidColorBrush(Color.FromRgb(0xF2, 0xF5, 0xF0)),
+ FontSize = Math.Clamp(h * 0.68, 11, 40),
+ TextWrapping = TextWrapping.Wrap
+ };
+
+ var border = new Border
+ {
+ Tag = "line",
+ Background = new SolidColorBrush(Color.FromArgb(0xF0, 0x10, 0x14, 0x18)),
+ CornerRadius = new CornerRadius(3),
+ Padding = new Thickness(4, 1, 4, 1),
+ MaxWidth = Math.Max(40, _dipBottomRight.X - x),
+ Child = textBlock
+ };
+
+ Canvas.SetLeft(border, x);
+ Canvas.SetTop(border, y);
+ LayerCanvas.Children.Add(border);
+ }
+ }
+
+ private void UpdateDetectedLabel()
+ {
+ if (_result == null || !_result.HasText)
+ {
+ DetectedLabel.Text = "No text detected";
+ return;
+ }
+
+ DetectedLabel.Text = string.IsNullOrEmpty(_result.DetectedLanguageTag)
+ ? "Translated"
+ : $"Detected: {_result.DetectedLanguageTag}";
+ }
+
+ private async void SourceCombo_Changed(object sender, SelectionChangedEventArgs e)
+ {
+ if (!_suppressEvents)
+ {
+ await RetranslateAsync();
+ }
+ }
+
+ private async void TargetCombo_Changed(object sender, SelectionChangedEventArgs e)
+ {
+ if (!_suppressEvents)
+ {
+ await RetranslateAsync();
+ }
+ }
+
+ private async Task RetranslateAsync()
+ {
+ var target = (TargetCombo.SelectedItem as TargetLanguageOption)?.Value ?? _initialTarget;
+ var sourceTag = (SourceCombo.SelectedItem as SourceLanguageOption)?.Tag;
+
+ BusyIndicator.Visibility = Visibility.Visible;
+ try
+ {
+ _result = await _retranslate(sourceTag, target);
+ }
+ catch
+ {
+ // Keep the previous result on failure.
+ }
+ finally
+ {
+ BusyIndicator.Visibility = Visibility.Collapsed;
+ }
+
+ UpdateDetectedLabel();
+ RenderBoxes();
+ }
+
+ private void CopyButton_Click(object sender, RoutedEventArgs e)
+ {
+ if (_result == null || !_result.HasText)
+ {
+ return;
+ }
+
+ try
+ {
+ Clipboard.SetText(string.Join(Environment.NewLine, _result.Lines.Select(l => l.Translation)));
+ }
+ catch
+ {
+ // Clipboard can be transiently locked by another process; ignore.
+ }
+ }
+
+ private void CloseButton_Click(object sender, RoutedEventArgs e) => Close();
+
+ private void Toolbar_MouseDown(object sender, MouseButtonEventArgs e) => e.Handled = true;
+
+ private void Root_MouseDown(object sender, MouseButtonEventArgs e)
+ {
+ var pos = e.GetPosition(Root);
+ var insideRegion = pos.X >= _dipTopLeft.X && pos.X <= _dipBottomRight.X
+ && pos.Y >= _dipTopLeft.Y && pos.Y <= _dipBottomRight.Y;
+ if (!insideRegion)
+ {
+ Close();
+ }
+ }
+
+ private void OnKeyDown(object sender, KeyEventArgs e)
+ {
+ if (e.Key == Key.Escape)
+ {
+ Close();
+ }
+ }
+ }
+}
diff --git a/src/Translumo/MVVM/ViewModels/ChatWindowViewModel.cs b/src/Translumo/MVVM/ViewModels/ChatWindowViewModel.cs
index db3f2696..03a92224 100644
--- a/src/Translumo/MVVM/ViewModels/ChatWindowViewModel.cs
+++ b/src/Translumo/MVVM/ViewModels/ChatWindowViewModel.cs
@@ -1,4 +1,7 @@
using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Input;
@@ -9,7 +12,10 @@
using Translumo.Infrastructure;
using Translumo.Infrastructure.Constants;
using Translumo.Infrastructure.Dispatching;
+using Translumo.Infrastructure.Language;
using Translumo.MVVM.Models;
+using Translumo.Processing.ImageTranslation;
+using Translumo.Processing.Interfaces;
using Translumo.Services;
using Translumo.Update;
using Translumo.Utils;
@@ -38,9 +44,16 @@ public bool ChatWindowIsVisible
private readonly ILogger _logger;
private readonly HotKeysServiceManager _hotKeysServiceManager;
private readonly UpdateManager _updateManager;
+ private readonly ImageTranslationService _imageTranslationService;
+ private readonly ICapturerFactory _capturerFactory;
+ private readonly LanguageService _languageService;
- public ChatWindowViewModel(ChatWindowModel model, HotKeysServiceManager hotKeysManager, ChatUITextMediator chatTextMediator, UpdateManager updateManager,
- IActionDispatcher dispatcher, DialogService dialogService, IServiceProvider serviceProvider, ILogger logger)
+ private IScreenCapturer _imageCapturer;
+ private Languages _lastImageTarget = Languages.Vietnamese;
+
+ public ChatWindowViewModel(ChatWindowModel model, HotKeysServiceManager hotKeysManager, ChatUITextMediator chatTextMediator, UpdateManager updateManager,
+ IActionDispatcher dispatcher, DialogService dialogService, IServiceProvider serviceProvider, ImageTranslationService imageTranslationService,
+ ICapturerFactory capturerFactory, LanguageService languageService, ILogger logger)
{
this.Model = model;
this._logger = logger;
@@ -48,6 +61,9 @@ public ChatWindowViewModel(ChatWindowModel model, HotKeysServiceManager hotKeysM
this._serviceProvider = serviceProvider;
this._hotKeysServiceManager = hotKeysManager;
this._updateManager = updateManager;
+ this._imageTranslationService = imageTranslationService;
+ this._capturerFactory = capturerFactory;
+ this._languageService = languageService;
dispatcher.RegisterConsumer(DispatcherActions.PASS_SITE, BrowseSiteHandler);
@@ -57,6 +73,7 @@ public ChatWindowViewModel(ChatWindowModel model, HotKeysServiceManager hotKeysM
hotKeysManager.SettingVisibilityKeyPressed += HotKeysManagerOnSettingVisibilityKeyPressed;
hotKeysManager.ShowSelectionAreaKeyPressed += HotKeysManagerOnShowSelectionAreaKeyPressed;
hotKeysManager.OnceTranslateKeyPressed += HotKeysManagerOnOnceTranslateKeyPressed;
+ hotKeysManager.ImageTranslateKeyPressed += HotKeysManagerOnImageTranslateKeyPressed;
hotKeysManager.WindowStyleChangeKeyPressed += HotKeysManagerOnWindowStyleChangeKeyPressed;
chatTextMediator.TextRaised += ChatTextMediatorOnTextRaised;
chatTextMediator.ClearTextsRaised += ChatTextMediatorOnClearTextsRaised;
@@ -130,8 +147,83 @@ private void HotKeysManagerOnOnceTranslateKeyPressed(object sender, EventArgs e)
}
}
+ private void HotKeysManagerOnImageTranslateKeyPressed(object sender, EventArgs e)
+ {
+ if (_dialogService.WindowIsOpened())
+ {
+ Model.AddChatItem(LocalizationManager.GetValue("Str.Chat.SettingsOpened"), TextTypes.Info);
+
+ return;
+ }
+
+ var result = _dialogService.ShowWindowDialog(out var window);
+ if (result.HasValue && result.Value && window != null && !window.SelectedArea.IsEmpty)
+ {
+ _ = ShowImageTranslationAsync(window.SelectedArea);
+ }
+ }
+
+ private async Task ShowImageTranslationAsync(RectangleF area)
+ {
+ byte[] image;
+ try
+ {
+ image = EnsureImageCapturer().CaptureScreen(area);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to capture region for image translation");
+ Model.AddChatItem($"Failed to capture screen ({ex.Message})", TextTypes.Info);
+
+ return;
+ }
+
+ ImageTranslationResult initial;
+ try
+ {
+ initial = await _imageTranslationService.TranslateRegionAsync(image, null, _lastImageTarget);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Image translation failed");
+ Model.AddChatItem($"Image translation failed: {ex.Message}", TextTypes.Info);
+
+ return;
+ }
+
+ var targetOptions = BuildTargetOptions();
+ var sourceOptions = _imageTranslationService.GetAvailableSourceLanguages()
+ .Select(s => new ImageTranslationOverlayWindow.SourceLanguageOption { Tag = s.Tag, Display = s.DisplayName })
+ .ToList();
+
+ var overlay = new ImageTranslationOverlayWindow(area, image, initial, targetOptions, sourceOptions, _lastImageTarget,
+ (forcedTag, target) =>
+ {
+ _lastImageTarget = target;
+ return _imageTranslationService.TranslateRegionAsync(image, forcedTag, target);
+ });
+ overlay.ShowDialog();
+ }
+
+ private IReadOnlyList BuildTargetOptions()
+ {
+ return _languageService.GetAll(includeTranslationOnly: true)
+ .Select(descriptor => new ImageTranslationOverlayWindow.TargetLanguageOption
+ {
+ Value = descriptor.Language,
+ Display = LocalizationManager.GetValue($"Str.Languages.{descriptor.Language}") ?? descriptor.Language.ToString()
+ })
+ .OrderBy(option => option.Display)
+ .ToList();
+ }
+
+ private IScreenCapturer EnsureImageCapturer()
+ {
+ return _imageCapturer ??= _capturerFactory.CreateCapturer(true);
+ }
+
private void HotKeysManagerOnWindowStyleChangeKeyPressed(object sender, EventArgs e)
- {
+ {
const int WS_EX_TRANSPARENT = 0x00000020;
const int GWL_EXSTYLE = -20;
diff --git a/src/Translumo/MVVM/ViewModels/HotkeysSettingsViewModel.cs b/src/Translumo/MVVM/ViewModels/HotkeysSettingsViewModel.cs
index 0a632406..789e0666 100644
--- a/src/Translumo/MVVM/ViewModels/HotkeysSettingsViewModel.cs
+++ b/src/Translumo/MVVM/ViewModels/HotkeysSettingsViewModel.cs
@@ -54,6 +54,10 @@ public HotkeysSettingsViewModel(HotKeysServiceManager hotKeysServiceManager)
hotKeysServiceManager.GamepadHotkeysEnabled ? _configuration.OnceTranslateGamepadKey : defaultGamepadHotKey,
nameof(_configuration.OnceTranslateKey), nameof(_configuration.OnceTranslateGamepadKey),
LocalizationManager.GetValue("Str.Hotkeys.OnceTranslate", false, OnLocalizedValueChanged, this)),
+ new HotKeyModel(_configuration.ImageTranslateKey,
+ hotKeysServiceManager.GamepadHotkeysEnabled ? _configuration.ImageTranslateGamepadKey : defaultGamepadHotKey,
+ nameof(_configuration.ImageTranslateKey), nameof(_configuration.ImageTranslateGamepadKey),
+ LocalizationManager.GetValue("Str.Hotkeys.ImageTranslate", false, OnLocalizedValueChanged, this)),
new HotKeyModel(_configuration.WindowStyleChangeKey,
hotKeysServiceManager.GamepadHotkeysEnabled ? _configuration.WindowStyleChangeGamepadKey : defaultGamepadHotKey,
nameof(_configuration.WindowStyleChangeKey), nameof(_configuration.WindowStyleChangeGamepadKey),
diff --git a/src/Translumo/Resources/Localization/lang.en-US.xaml b/src/Translumo/Resources/Localization/lang.en-US.xaml
index 3a871d8b..20110a7a 100644
--- a/src/Translumo/Resources/Localization/lang.en-US.xaml
+++ b/src/Translumo/Resources/Localization/lang.en-US.xaml
@@ -40,6 +40,7 @@
On/off translation
Show selected screen capture area
Select area and translate text once
+ Image translation (select area, overlay like Google)
Lock/unlock translation window
Press '{0}' to open the Settings
Press '{0}' to select detection area
diff --git a/src/Translumo/Resources/Localization/lang.ru-RU.xaml b/src/Translumo/Resources/Localization/lang.ru-RU.xaml
index 9f2c782c..8406d734 100644
--- a/src/Translumo/Resources/Localization/lang.ru-RU.xaml
+++ b/src/Translumo/Resources/Localization/lang.ru-RU.xaml
@@ -40,6 +40,7 @@
Включить/выключить перевод
Показать выбранную область перевода
Выбрать область и перевести текст один раз
+ Перевод изображения (выделить область, наложение как в Google)
Заблокировать/разблокировать окно перевода
Нажмите '{0}' чтобы открыть настройки
Нажмите '{0}' чтобы выбрать область захвата текста
diff --git a/src/Translumo/Resources/Localization/lang.zh-CN.xaml b/src/Translumo/Resources/Localization/lang.zh-CN.xaml
index 368daf11..2db243ba 100644
--- a/src/Translumo/Resources/Localization/lang.zh-CN.xaml
+++ b/src/Translumo/Resources/Localization/lang.zh-CN.xaml
@@ -40,6 +40,7 @@
开启/关闭 翻译
显示已选择的翻译区域
选择翻译区域（一次性）
+ 图像翻译（选择区域，像谷歌一样叠加显示）
锁定/解除锁定 翻译墙位置
请按 '{0}' 打开设置页面
请按 '{0}' 选择翻译区域