Software/Visual_Studio/Tango.WebRTC/WebRtcClient.cs


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
3using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Serialization;
using System.Text;
using System.Threading.Tasks;
using Tango.Transport.Web;

namespace Tango.PPC.Common.Web
{
    public class LatestVersionResponse : WebResponseMessage
    {
        public String Version { get; set; }
    }
}
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Tango.Core.Threading;
using WebRtc.NET;

namespace Tango.WebRTC
{
    public class WebRtcClient : IDisposable
    {
        private ManagedConductor _conductor;
        private Thread _conductorThread;
        private TurboJpegEncoder _encoder;
        private byte[] _bgrBufflocal;
        private byte[] _imgBuf;
        private GCHandle _bufHandle;
        private IntPtr _imgBufPtr = IntPtr.Zero;
        private bool _isDisposed;
        private Bitmap _sendFrame;
        private TaskCompletionSource<bool> _readyCompletionSource;

        #region Events

        public event EventHandler<NewIceCandidateEventArgs> NewIceCandidate;
        public event EventHandler Ready;
        public event EventHandler<DataMessageReceivedEventArgs<String>> TextMessageReceived;
        public event EventHandler<DataMessageReceivedEventArgs<byte[]>> BinaryMessageReceived;
        public event EventHandler<VideoFrameReceivedEventArgs> FrameReceived;
        public event EventHandler<ErrorEventArgs> Error;
        public event EventHandler Disconnected;

        #endregion

        #region Properties

        private int _frameWidth;
        public int FrameWidth
        {
            get { return _frameWidth; }
            set
            {
                if (IsInitialized)
                {
                    throw new InvalidOperationException("The frame height must be set before calling Init();");
                }

                _frameWidth = value;
            }
        }

        private int _frameHeight;
        public int FrameHeight
        {
            get { return _frameHeight; }
            set
            {
                if (IsInitialized)
                {
                    throw new InvalidOperationException("The frame width must be set before calling Init();");
                }

                _frameHeight = value;
            }
        }

        private int _frameRate;
        public int FrameRate
        {
            get { return _frameRate; }
            set
            {
                if (IsInitialized)
                {
                    throw new InvalidOperationException("The frame rate must be set before calling Init();");
                }

                _frameRate = value;
            }
        }


        private String _dataChannelName;
        public String DataChannelName
        {
            get { return _dataChannelName; }
            set
            {
                if (IsInitialized)
                {
                    throw new InvalidOperationException("The data channel must be set before calling Init();");
                }

                _dataChannelName = value;
            }
        }

        public bool IsInitialized { get; private set; }
        public bool IsReady { get; private set; }

        public Object Tag { get; set; }

        #endregion

        #region Constructors

        public WebRtcClient()
        {
            FrameWidth = 640;
            FrameHeight = 480;
            FrameRate = 5;
            DataChannelName = "DefaultChannelName";
        }

        public WebRtcClient(int frameWidth, int frameHeight, int frameRate) : this()
        {
            FrameWidth = frameWidth;
            FrameHeight = frameHeight;
            FrameRate = frameRate;
        }

        public WebRtcClient(int frameWidth, int frameHeight, int frameRate, String dataChannelName) : this(frameWidth, frameHeight, frameRate)
        {
            DataChannelName = dataChannelName;
        }

        #endregion

        #region Init

        public Task Init()
        {
            if (_isDisposed)
            {
                throw new ObjectDisposedException("This instance was already disposed.");
            }

            if (IsInitialized)
            {
                throw new InvalidOperationException("This instance was already initialized.");
            }

            TaskCompletionSource<bool> completion = new TaskCompletionSource<bool>();

            _conductorThread = new Thread(() =>
            {
                Thread.Sleep(5); //Wait for function to return at least!

                _conductor = new ManagedConductor();
                _encoder = TurboJpegEncoder.CreateEncoder();

                try
                {
                    ManagedConductor.InitializeSSL();

                    //Stun
                    _conductor.AddServerConfig("stun:stun.l.google.com:19302", String.Empty, String.Empty);
                    _conductor.AddServerConfig("stun:stun1.l.google.com:19302", String.Empty, String.Empty);
                    _conductor.AddServerConfig("stun:stun2.l.google.com:19302", String.Empty, String.Empty);
                    _conductor.AddServerConfig("stun:stun3.l.google.com:19302", String.Empty, String.Empty);
                    _conductor.AddServerConfig("stun:stun4.l.google.com:19302", String.Empty, String.Empty);
                    _conductor.AddServerConfig("stun:eu-turn3.xirsys.com", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");

                    //Turn
                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:80?transport=udp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");
                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:3478?transport=udp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");
                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:80?transport=tcp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");
                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:3478?transport=tcp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");

                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:443?transport=tcp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");
                    _conductor.AddServerConfig("turn:eu-turn3.xirsys.com:5349?transport=tcp", "mjyn-kODdallq7iIZN1-eCYHo4GZy36urKu-8GTtdKwcuEUe8i4LjeHVoej-OePwAAAAAF56hb1Sb3liZW4=", "83b30e94-6e1c-11ea-b4c3-72c9c257b255");


                    _conductor.SetAudio(false);
                    _conductor.SetVideoCapturer(FrameWidth, FrameHeight, FrameRate, false);

                    if (!_conductor.InitializePeerConnection())
                    {
                        completion.SetException(new ApplicationException("Error initializing peer connection."));
                        return;
                    }

                    _conductor.CreateDataChannel(DataChannelName);
                    _conductor.OnIceCandidate += _conductor_OnIceCandidate;
                    _conductor.OnDataMessage += _conductor_OnDataMessage;
                    _conductor.OnDataBinaryMessage += _conductor_OnDataBinaryMessage;
                    _conductor.OnError += _conductor_OnError;
                    _conductor.OnFailure += _conductor_OnFailure;
                    _conductor.OnIceStateChanged += _conductor_OnIceStateChanged;

                    unsafe
                    {
                        _conductor.OnRenderRemote += _conductor_OnRenderRemote;
                    }

                    _conductor.ProcessMessages(1000);
                }
                catch (Exception ex)
                {
                    completion.SetException(ex);
                    return;
                }

                IsInitialized = true;

                completion.SetResult(true);

                while (!_isDisposed)
                {
                    _conductor.ProcessMessages(1000);
                    Thread.Sleep(10);
                }

                IsInitialized = false;

                _conductor.OnIceCandidate -= _conductor_OnIceCandidate;
                _conductor.OnDataMessage -= _conductor_OnDataMessage;
                _conductor.OnDataBinaryMessage -= _conductor_OnDataBinaryMessage;
                _conductor.OnError -= _conductor_OnError;
                _conductor.OnFailure -= _conductor_OnFailure;
                _conductor.OnIceStateChanged -= _conductor_OnIceStateChanged;

                unsafe
                {
                    _conductor.OnRenderRemote -= _conductor_OnRenderRemote;
                }

                _conductor.Dispose();

                try
                {
                    if (_sendFrame != null)
                    {
                        _sendFrame.Dispose();
                    }

                }
                catch { }


                try
                {
                    if (_bufHandle != null)
                    {
                        _bufHandle.Free();
                    }
                }
                catch { }
            });

            _conductorThread.SetApartmentState(ApartmentState.STA);
            _conductorThread.IsBackground = true;
            _conductorThread.Start();

            return completion.Task;
        }

        private void _conductor_OnIceStateChanged(IceConnectionStates state)
        {
            if (!_isDisposed)
            {
                if (state == IceConnectionStates.kIceConnectionConnected)
                {
                    OnReady();
                }
                else if (state == IceConnectionStates.kIceConnectionFailed)
                {
                    Disconnected?.Invoke(this, new EventArgs());
                }
            }
        }

        #endregion

        #region WebRTC Event Handlers

        private void _conductor_OnIceCandidate(string sdp_mid, int sdp_mline_index, string sdp)
        {
            NewIceCandidate?.Invoke(this, new NewIceCandidateEventArgs()
            {
                IceCandidate = new IceCandidate()
                {
                    SdpMid = sdp_mid,
                    SdpMLineIndex = sdp_mline_index,
                    Sdp = sdp
                }
            });
        }

        private void _conductor_OnDataMessage(string text)
        {
            OnTextMessageReceived(text);
        }

        private void _conductor_OnDataBinaryMessage(byte[] data)
        {
            OnBinaryMessageReceived(data);
        }

        unsafe private void _conductor_OnRenderRemote(byte* frame_buffer, uint w, uint h)
        {
            if (_isDisposed) return;

            try
            {
                if (_encoder.EncodeI420toBGR24(frame_buffer, w, h, ref _bgrBufflocal, true) == 0)
                {
                    var bufHandle = GCHandle.Alloc(_bgrBufflocal, GCHandleType.Pinned);
                    var bmp = new Bitmap((int)w, (int)h, (int)w * 3, PixelFormat.Format24bppRgb, bufHandle.AddrOfPinnedObject());
                    FrameReceived?.Invoke(this, new VideoFrameReceivedEventArgs()
                    {
                        Bitmap = bmp
                    });
                }
            }
            catch (Exception ex)
            {
                Debug.WriteLine($"Error occurred while receiving the remote video frame.\n{ex.Message}");
            }
        }

        private void _conductor_OnFailure(string error)
        {
            OnError(error);
        }

        private void _conductor_OnError()
        {
            OnError("Unspecified error.");
        }

        #endregion

        #region Public Methods

        public Task<Offer> CreateOffer()
        {
            EnsureInitialized();

            TaskCompletionSource<Offer> completion = new TaskCompletionSource<Offer>();

            ManagedConductor.OnCallbackSdp del = null;

            bool completed = false;

            del = (sdp) =>
            {
                if (!completed)
                {
                    completed = true;
                    _conductor.OnSuccessOffer -= del;
                    completion.SetResult(new Offer() { Sdp = sdp });
                }
            };

            _conductor.OnSuccessOffer += del;

            TimeoutTask.StartNew(() => 
            {
                if (!completed)
                {
                    completed = true;
                    completion.SetException(new TimeoutException("The offer was not created within the given time."));
                }
            }, TimeSpan.FromSeconds(10));

            _conductor.CreateOffer();

            return completion.Task;
        }

        public Task<Answer> CreateAnswer(Offer offer)
        {
            EnsureInitialized();

            TaskCompletionSource<Answer> completion = new TaskCompletionSource<Answer>();

            ManagedConductor.OnCallbackSdp del = null;

            bool completed = false;

            del = (sdp) =>
            {
                if (!completed)
                {
                    completed = true;
                    _conductor.OnSuccessAnswer -= del;
                    completion.SetResult(new Answer() { Sdp = sdp });
                }
            };

            _conductor.OnSuccessAnswer += del;

            TimeoutTask.StartNew(() => 
            {
                if (!completed)
                {
                    completed = true;
                    completion.SetException(new TimeoutException("The answer was not created within the given time."));
                }
            }, TimeSpan.FromSeconds(10));

            _conductor.OnOfferRequest(offer.Sdp);

            return completion.Task;
        }

        public void SetAnswer(Answer answer)
        {
            EnsureInitialized();

            _conductor.OnOfferReply("answer", answer.Sdp);
        }

        public void SendText(String msg)
        {
            EnsureInitialized();

            _conductor.DataChannelSendText(msg);
        }

        public void SendBinary(byte[] data)
        {
            EnsureInitialized();

            _conductor.DataChannelSendData(data);
        }

        public void AddIceCandidate(IceCandidate ice)
        {
            EnsureInitialized();

            ThreadFactory.StartNew(() =>
            {
                _conductor.AddIceCandidate(ice.SdpMid, ice.SdpMLineIndex, ice.Sdp);
            });
        }

        public unsafe void PushFrame(Bitmap bitmap)
        {
            if (_isDisposed) return;

            EnsureInitialized();

            try
            {
                if (_sendFrame == null)
                {
                    _imgBuf = new byte[FrameWidth * 3 * FrameHeight];
                    _bufHandle = GCHandle.Alloc(_imgBuf, GCHandleType.Pinned);
                    _imgBufPtr = _bufHandle.AddrOfPinnedObject();
                    _sendFrame = new Bitmap(FrameWidth, FrameHeight, FrameWidth * 3, PixelFormat.Format24bppRgb, _imgBufPtr);
                }

                using (var g = Graphics.FromImage(_sendFrame))
                {
                    g.DrawImage(bitmap, new Rectangle(0, 0, _sendFrame.Width, _sendFrame.Height));
                }

                byte* firstYuv = _conductor.VideoCapturerI420Buffer();

                int yuvSize = _encoder.EncodeI420((byte*)_imgBufPtr.ToPointer(), FrameWidth, FrameHeight, (int)TJPF.TJPF_BGR, 0, true, firstYuv);

                _conductor.PushFrame();
            }
            catch (Exception ex)
            {
                Debug.WriteLine($"Error occurred while pushing the frame.\n{ex.Message}");
            }
        }

        public Task WaitForReady(TimeSpan? timeout = null)
        {
            if (!IsReady)
            {
                if (timeout != null)
                {
                    TimeoutTask.StartNew(() => 
                    {
                        if (!IsReady)
                        {
                            _readyCompletionSource.SetException(new TimeoutException("The connection was not ready within the given time."));
                            _readyCompletionSource = null;
                        }
                    }, timeout.Value);
                }

                _readyCompletionSource = new TaskCompletionSource<bool>();
                return _readyCompletionSource.Task;
            }
            else
            {
                return Task.FromResult(true);
            }
        }

        public void Dispose()
        {
            if (!_isDisposed)
            {
                _isDisposed = true;
            }
        }

        #endregion

        #region Virtual Methods

        protected virtual void OnTextMessageReceived(String text)
        {
            TextMessageReceived?.Invoke(this, new DataMessageReceivedEventArgs<string>() { Data = text });
        }

        protected virtual void OnBinaryMessageReceived(byte[] data)
        {
            BinaryMessageReceived?.Invoke(this, new DataMessageReceivedEventArgs<byte[]>() { Data = data });
        }

        protected virtual void OnReady()
        {
            if (!IsReady)
            {
                IsReady = true;
                Ready?.Invoke(this, new EventArgs());

                if (_readyCompletionSource != null)
                {
                    _readyCompletionSource.SetResult(true);
                    _readyCompletionSource = null;
                }
            }
        }

        protected virtual void OnError(String error)
        {
            Error?.Invoke(this, new ErrorEventArgs() { Error = error });
        }

        #endregion

        #region Private Methods

        private void EnsureInitialized()
        {
            if (!IsInitialized)
            {
                throw new InvalidOperationException("Invalid operation. The instance was not initialized using Init();");
            }
        }

        #endregion
    }
}