From e07ebe48156313899ab6f4a06928aac6ef818f0d Mon Sep 17 00:00:00 2001 From: Roy Ben-Shabat Date: Thu, 28 Mar 2019 19:05:42 +0200 Subject: Working on real-time card detection... --- .../Controls/ColorMatrixControl.cs | 55 +++++ .../Controls/IndexedUniformGrid.cs | 47 ++++ .../Graph/WpfGraphController.cs | 36 ++++ .../Images/bottomLeft.bmp | Bin 0 -> 3678 bytes .../Images/bottomRight.bmp | Bin 0 -> 3678 bytes .../Images/topRight.bmp | Bin 0 -> 3678 bytes .../Images/topleft.bmp | Bin 0 -> 3678 bytes .../Tango.MachineStudio.ColorCapture.csproj | 25 +++ .../Themes/Generic.xaml | 40 +++- .../ViewModels/MainViewVM.cs | 113 +++++++++- .../Views/MainView.xaml | 236 ++++++++++++++------- .../Views/MainView.xaml.cs | 8 + 12 files changed, 480 insertions(+), 80 deletions(-) create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/ColorMatrixControl.cs create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/IndexedUniformGrid.cs create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Graph/WpfGraphController.cs create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomLeft.bmp create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomRight.bmp create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topRight.bmp create mode 100644 Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topleft.bmp (limited to 'Software/Visual_Studio/MachineStudio/Modules') diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/ColorMatrixControl.cs b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/ColorMatrixControl.cs new file mode 100644 index 000000000..7ea8681ab --- /dev/null +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/ColorMatrixControl.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Navigation; +using System.Windows.Shapes; + +namespace Tango.MachineStudio.ColorCapture.Controls +{ + public class ColorMatrixControl : Control + { + public ObservableCollection Colors + { + get { return (ObservableCollection)GetValue(ColorsProperty); } + set { SetValue(ColorsProperty, value); } + } + public static readonly DependencyProperty ColorsProperty = + DependencyProperty.Register("Colors", typeof(ObservableCollection), typeof(ColorMatrixControl), new PropertyMetadata(null)); + + public int Columns + { + get { return (int)GetValue(ColumnsProperty); } + set { SetValue(ColumnsProperty, value); } + } + public static readonly DependencyProperty ColumnsProperty = + DependencyProperty.Register("Columns", typeof(int), typeof(ColorMatrixControl), new PropertyMetadata(10)); + + public int Rows + { + get { return (int)GetValue(RowsProperty); } + set { SetValue(RowsProperty, value); } + } + public static readonly DependencyProperty RowsProperty = + DependencyProperty.Register("Rows", typeof(int), typeof(ColorMatrixControl), new PropertyMetadata(11)); + + public ColorMatrixControl() + { + Colors = new ObservableCollection(); + } + + static ColorMatrixControl() + { + DefaultStyleKeyProperty.OverrideMetadata(typeof(ColorMatrixControl), new FrameworkPropertyMetadata(typeof(ColorMatrixControl))); + } + } +} diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/IndexedUniformGrid.cs b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/IndexedUniformGrid.cs new file mode 100644 index 000000000..af34e038d --- /dev/null +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Controls/IndexedUniformGrid.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; + +namespace Tango.MachineStudio.ColorCapture.Controls +{ + public class IndexedUniformGrid : Grid + { + public int Columns + { + get { return (int)GetValue(ColumnsProperty); } + set { SetValue(ColumnsProperty, value); } + } + public static readonly DependencyProperty ColumnsProperty = + DependencyProperty.Register("Columns", typeof(int), typeof(IndexedUniformGrid), new PropertyMetadata(0)); + + public int Rows + { + get { return (int)GetValue(RowsProperty); } + set { SetValue(RowsProperty, value); } + } + public static readonly DependencyProperty RowsProperty = + DependencyProperty.Register("Rows", typeof(int), typeof(IndexedUniformGrid), new PropertyMetadata(0)); + + public IndexedUniformGrid() + { + Loaded += IndexedUniformGrid_Loaded; + } + + private void IndexedUniformGrid_Loaded(object sender, RoutedEventArgs e) + { + for (int i = 0; i < Columns; i++) + { + ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(1, GridUnitType.Star) }); + } + + for (int i = 0; i < Rows; i++) + { + RowDefinitions.Add(new RowDefinition() { Height = new GridLength(1, GridUnitType.Star) }); + } + } + } +} diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Graph/WpfGraphController.cs b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Graph/WpfGraphController.cs new file mode 100644 index 000000000..dd32cde93 --- /dev/null +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Graph/WpfGraphController.cs @@ -0,0 +1,36 @@ +using RealTimeGraphX; +using RealTimeGraphX.DataPoints; +using RealTimeGraphX.Renderers; +using RealTimeGraphX.WPF.DataSeries; +using RealTimeGraphX.WPF.Painters; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Media; + +namespace Tango.MachineStudio.ColorCapture.Graph +{ + public class WpfGraphController : GraphControllerBase + { + public WpfGraphController(int refreshRate = 50) + { + AddDataSeries(new WpfDataSeries() + { + StrokeThickness = 1, + Stroke = Colors.Black, + }); + + var renderer = new GraphScrollingRenderer() + { + RefreshRate = TimeSpan.FromMilliseconds(refreshRate) + }; + + var painter = new WpfScrollingGraphPainter(); + + ConnectOutput(renderer); + renderer.ConnectOutput(painter); + } + } +} diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomLeft.bmp b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomLeft.bmp new file mode 100644 index 000000000..13b33704b Binary files /dev/null and b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomLeft.bmp differ diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomRight.bmp b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomRight.bmp new file mode 100644 index 000000000..d2116fad2 Binary files /dev/null and b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/bottomRight.bmp differ diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topRight.bmp b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topRight.bmp new file mode 100644 index 000000000..f008d454d Binary files /dev/null and b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topRight.bmp differ diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topleft.bmp b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topleft.bmp new file mode 100644 index 000000000..8115cf359 Binary files /dev/null and b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Images/topleft.bmp differ diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Tango.MachineStudio.ColorCapture.csproj b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Tango.MachineStudio.ColorCapture.csproj index 2422cec2a..4d0c0fae5 100644 --- a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Tango.MachineStudio.ColorCapture.csproj +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Tango.MachineStudio.ColorCapture.csproj @@ -88,6 +88,9 @@ GlobalVersionInfo.cs + + + Code @@ -117,6 +120,18 @@ + + {37e4ceab-b54b-451f-b535-04cf7da9c459} + ColorMine + + + {99d233c5-fee7-418e-9c25-d4584cb52e28} + RealTimeGraphX.WPF + + + {6d55a3b8-46d3-493a-a143-aebd2b98d683} + RealTimeGraphX + {f441feee-322a-4943-b566-110e12fd3b72} Tango.BL @@ -145,6 +160,10 @@ {f209fae8-73f9-441b-97f4-0844a0279390} Tango.TCC.BL + + {5d0d4053-cab3-4a4a-929e-37a76483bc22} + Tango.TCC.OpenCV.DLL + {cb0b0aa2-bb24-4bca-a720-45e397684e12} Tango.MachineStudio.Common @@ -156,5 +175,11 @@ + + + + + + \ No newline at end of file diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Themes/Generic.xaml b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Themes/Generic.xaml index 968c541b3..0a2ebbca8 100644 --- a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Themes/Generic.xaml +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Themes/Generic.xaml @@ -1,6 +1,44 @@ - + + diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/ViewModels/MainViewVM.cs b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/ViewModels/MainViewVM.cs index 298b862fb..f21c403de 100644 --- a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/ViewModels/MainViewVM.cs +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/ViewModels/MainViewVM.cs @@ -1,14 +1,20 @@ -using System; +using ColorMine.ColorSpaces; +using ColorMine.ColorSpaces.Comparisons; +using System; using System.Collections.Generic; -using System.Drawing; +using System.Collections.ObjectModel; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Windows.Media; +using System.Windows.Media.Imaging; using Tango.Core.Commands; +using Tango.MachineStudio.ColorCapture.Graph; using Tango.MachineStudio.Common; using Tango.MachineStudio.Common.Notifications; using Tango.MachineStudio.Common.Video; +using Tango.TCC.BL; using Tango.Video.DirectCapture; namespace Tango.MachineStudio.ColorCapture.ViewModels @@ -16,6 +22,8 @@ namespace Tango.MachineStudio.ColorCapture.ViewModels public class MainViewVM : StudioViewModel { private INotificationProvider _notification; + private CardDetector _cardDetector; + private int _sampleCounter; public IVideoCaptureProvider VideoProvider { get; set; } @@ -32,14 +40,74 @@ namespace Tango.MachineStudio.ColorCapture.ViewModels } } + private BitmapSource _detectedSource; + public BitmapSource DetectedSource + { + get { return _detectedSource; } + set { _detectedSource = value; RaisePropertyChangedAuto(); } + } + + private ObservableCollection _colors; + public ObservableCollection Colors + { + get { return _colors; } + set { _colors = value; RaisePropertyChangedAuto(); RaisePropertyChangedAuto(); } + } + + private Color _processedColor; + public Color ProcessedColor + { + get { return _processedColor; } + set { _processedColor = value; RaisePropertyChangedAuto(); } + } + + private Color _capturedColor; + public Color CapturedColor + { + get { return _capturedColor; } + set { _capturedColor = value; RaisePropertyChangedAuto(); } + } + + private double _measureL; + public double MeasureL + { + get { return _measureL; } + set { _measureL = value; RaisePropertyChangedAuto(); } + } + + private double _measureA; + public double MeasureA + { + get { return _measureA; } + set { _measureA = value; RaisePropertyChangedAuto(); } + } + + private double _measureB; + public double MeasureB + { + get { return _measureB; } + set { _measureB = value; RaisePropertyChangedAuto(); } + } + + + public WpfGraphController CaptureDeltaEController { get; set; } + public RelayCommand ToggleCameraCommand { get; set; } - public MainViewVM(IVideoCaptureProvider videoProvider, INotificationProvider notificationProvider) + public MainViewVM() + { + _cardDetector = new CardDetector(); + ToggleCameraCommand = new RelayCommand(ToggleCamera); + CaptureDeltaEController = new WpfGraphController(); + CaptureDeltaEController.Range.AutoY = true; + CaptureDeltaEController.Range.MaximumX = 1000; + } + + public MainViewVM(IVideoCaptureProvider videoProvider, INotificationProvider notificationProvider) : this() { _notification = notificationProvider; VideoProvider = videoProvider; SelectedVideoDevice = videoProvider.AvailableCaptureDevices.FirstOrDefault(); - ToggleCameraCommand = new RelayCommand(ToggleCamera); } private void ToggleCamera() @@ -52,6 +120,7 @@ namespace Tango.MachineStudio.ColorCapture.ViewModels } else { + SelectedVideoDevice.Resolution = new Resolution(1280, 720); SelectedVideoDevice.Start(); } } @@ -71,11 +140,41 @@ namespace Tango.MachineStudio.ColorCapture.ViewModels } } - private void OnVideoFrameReceived(object sender, Video.DirectShow.EventArguments.FrameReceivedEventArgs args) + double deltaE = 0; + private async void OnVideoFrameReceived(object sender, Video.DirectShow.EventArguments.FrameReceivedEventArgs args) { - Bitmap bmp = args.BitmapSource.ConvertFormat(PixelFormats.Rgb24).ToGDIBitmap(); + if (_cardDetector.CanDetect) + { + var result = await _cardDetector.Detect(args.BitmapSource); + + if (result.IsDetected) + { + DetectedSource = result.DetectedBitmap; + + if (Colors == null) + { + Colors = new ObservableCollection(Enumerable.Range(0, result.ColorDetectionOutput.ColorMatrix.Count).Select(x => new Color())); + } + + InvokeUI(() => + { + for (int i = 0; i < result.ColorDetectionOutput.ColorMatrix.Count; i++) + { + var detectedColor = result.ColorDetectionOutput.ColorMatrix[i]; + Colors[i] = Color.FromArgb(255, (byte)detectedColor.R, (byte)detectedColor.G, (byte)detectedColor.B); + } + + CapturedColor = Color.FromArgb(255, (byte)result.ColorDetectionOutput.RawColor.R, (byte)result.ColorDetectionOutput.RawColor.G, (byte)result.ColorDetectionOutput.RawColor.B); + ProcessedColor = Color.FromArgb(255, (byte)result.ColorDetectionOutput.ProcessedColor.R, (byte)result.ColorDetectionOutput.ProcessedColor.G, (byte)result.ColorDetectionOutput.ProcessedColor.B); + }); + + //calculate delta E. + Lab measureLab = new Lab(MeasureL, MeasureA, MeasureB); + deltaE = measureLab.Compare(new Rgb(ProcessedColor.R, ProcessedColor.G, ProcessedColor.B), new CieDe2000Comparison()); + } + } - bmp.Dispose(); + CaptureDeltaEController.PushData(_sampleCounter++, deltaE); } public override void OnApplicationReady() diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml index cd619d0a0..259e3160e 100644 --- a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml @@ -6,11 +6,16 @@ xmlns:materialDesign="http://materialdesigninxaml.net/winfx/xaml/themes" xmlns:converters="clr-namespace:Tango.SharedUI.Converters;assembly=Tango.SharedUI" xmlns:vm="clr-namespace:Tango.MachineStudio.ColorCapture.ViewModels" + xmlns:mahapps="http://metro.mahapps.com/winfx/xaml/controls" + xmlns:graphX="clr-namespace:RealTimeGraphX.WPF.Surfaces;assembly=RealTimeGraphX.WPF" + xmlns:componentsX="clr-namespace:RealTimeGraphX.WPF.Components;assembly=RealTimeGraphX.WPF" + xmlns:controls="clr-namespace:Tango.MachineStudio.ColorCapture.Controls" + xmlns:realtimeGraphX="clr-namespace:RealTimeGraphX.WPF.Surfaces;assembly=RealTimeGraphX.WPF" xmlns:global="clr-namespace:Tango.MachineStudio.ColorCapture" xmlns:local="clr-namespace:Tango.MachineStudio.ColorCapture.Views" mc:Ignorable="d" d:DesignHeight="1080" d:DesignWidth="1920" Background="Transparent" d:DataContext="{d:DesignInstance Type=vm:MainViewVM, IsDesignTimeCreatable=False}" DataContext="{x:Static global:ViewModelLocator.MainViewVM}"> - + @@ -25,83 +30,170 @@ - Selected Capture Device - + Capture Device + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Captured Color + + + + + + + + + + + + + + + + + + Processed Color + + + + + + + + + + + + + + + + + + + + + + + + + + HISTORY + + + + + + + + + + + + + Delta E Reference Point + + L: + + + A: + + + B: + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml.cs b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml.cs index 4f890bfe1..05b5daf7e 100644 --- a/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml.cs +++ b/Software/Visual_Studio/MachineStudio/Modules/Tango.MachineStudio.ColorCapture/Views/MainView.xaml.cs @@ -12,6 +12,7 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Navigation; using System.Windows.Shapes; +using Tango.MachineStudio.ColorCapture.ViewModels; namespace Tango.MachineStudio.ColorCapture.Views { @@ -20,9 +21,16 @@ namespace Tango.MachineStudio.ColorCapture.Views /// public partial class MainView : UserControl { + private MainViewVM _vm; + public MainView() { InitializeComponent(); + Loaded += (_, __) => + { + _vm = DataContext as MainViewVM; + _vm.CaptureDeltaEController.Output.Output.ConnectOutput(Graph); + }; } } } -- cgit v1.3.1