Merge pull request #4518 from QuantConnect/bitfinex-brokerage-updates

Bitfinex Brokerage Updates

Algorithm.Framework/Portfolio/SectorWeightingPortfolioConstructionModel.cs

@@ -71,6 +71,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
 
         /// <summary>
         /// Initialize a new instance of <see cref="SectorWeightingPortfolioConstructionModel"/>
+        /// </summary>
         /// <param name="rebalance">Rebalancing func or if a date rule, timedelta will be converted into func.
         /// For a given algorithm UTC DateTime the func returns the next expected rebalance time
         /// or null if unknown, in which case the function will be called again in the next loop. Returning current time

Algorithm.Python/readme.md

@@ -13,14 +13,14 @@ Before we enable python support, follow the [installation instructions](https://
    3. Click **New**. 
         - Name of the variable: `PYTHONHOME`. 
         - Value of the variable: python installation path.
-4. Install [pandas=0.23.4](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
+4. Install [pandas=0.25.3](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
 5. Install [wrapt=1.11.2](https://pypi.org/project/wrapt/) module.
 6. Reboot computer to ensure changes are propogated.
 
 #### [macOS](https://github.com/QuantConnect/Lean#macos)
 
 1. Use the macOS x86-64 package installer from [Anaconda](https://repo.anaconda.com/archive/Anaconda3-5.2.0-MacOSX-x86_64.pkg) and follow "[Installing on macOS](https://docs.anaconda.com/anaconda/install/mac-os)" instructions from Anaconda documentation page.
-2. Install [pandas=0.23.4](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
+2. Install [pandas=0.25.3](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
 3. Install [wrapt=1.11.2](https://pypi.org/project/wrapt/) module.
 
 *Note:* If you encounter the "System.DllNotFoundException: python3.6m" runtime error when running Python algorithms on macOS:
@@ -46,7 +46,7 @@ rm -rf Miniconda3-4.5.12-Linux-x86_64.sh
 sudo ln -s $HOME/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so
 conda update -y python conda pip
 conda install -y cython=0.29.11
-conda install -y pandas=0.23.4
+conda install -y pandas=0.25.3
 conda install -y wrapt=1.11.2
 ```
 

Brokerages/Bitfinex/BitfinexBrokerage.Messaging.cs

@@ -59,7 +59,7 @@ namespace QuantConnect.Brokerages.Bitfinex
         /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
         /// <param name="priceProvider">The price provider for missing FX conversion rates</param>
         public BitfinexBrokerage(string wssUrl, string restUrl, string apiKey, string apiSecret, IAlgorithm algorithm, IPriceProvider priceProvider)
-            : this(wssUrl, new WebSocketWrapper(), new RestClient(restUrl), apiKey, apiSecret, algorithm, priceProvider)
+            : this(wssUrl, new WebSocketClientWrapper(), new RestClient(restUrl), apiKey, apiSecret, algorithm, priceProvider)
         {
         }
 

Brokerages/Bitfinex/BitfinexSubscriptionManager.cs

@@ -18,6 +18,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Threading;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using QuantConnect.Data.Market;
@@ -39,11 +40,13 @@ namespace QuantConnect.Brokerages.Bitfinex
         /// </remarks>
         private const int MaximumSubscriptionsPerSocket = 30;
 
+        private const int ConnectionTimeout = 30000;
+
         private readonly string _wssUrl;
         private readonly object _locker = new object();
         private readonly BitfinexBrokerage _brokerage;
         private readonly BitfinexSymbolMapper _symbolMapper;
-        private readonly RateGate _connectionRateLimiter = new RateGate(10, TimeSpan.FromMinutes(1));
+        private readonly RateGate _connectionRateLimiter = new RateGate(5, TimeSpan.FromMinutes(1));
         private readonly ConcurrentDictionary<Symbol, List<BitfinexWebSocketWrapper>> _subscriptionsBySymbol = new ConcurrentDictionary<Symbol, List<BitfinexWebSocketWrapper>>();
         private readonly ConcurrentDictionary<BitfinexWebSocketWrapper, List<BitfinexChannel>> _channelsByWebSocket = new ConcurrentDictionary<BitfinexWebSocketWrapper, List<BitfinexChannel>>();
         private readonly ConcurrentDictionary<Symbol, DefaultOrderBook> _orderBooks = new ConcurrentDictionary<Symbol, DefaultOrderBook>();
@@ -190,7 +193,8 @@ namespace QuantConnect.Brokerages.Bitfinex
             webSocket.Initialize(_wssUrl);
             webSocket.Message += OnMessage;
             webSocket.Error += OnError;
-            webSocket.Connect();
+
+            Connect(webSocket);
 
             webSocket.ConnectionHandler.ConnectionLost += OnConnectionLost;
             webSocket.ConnectionHandler.ConnectionRestored += OnConnectionRestored;
@@ -204,6 +208,33 @@ namespace QuantConnect.Brokerages.Bitfinex
             return webSocket;
         }
 
+        private void Connect(IWebSocket webSocket)
+        {
+            var connectedEvent = new ManualResetEvent(false);
+            EventHandler onOpenAction = (s, e) =>
+            {
+                connectedEvent.Set();
+            };
+
+            webSocket.Open += onOpenAction;
+
+            try
+            {
+                webSocket.Connect();
+
+                if (!connectedEvent.WaitOne(ConnectionTimeout))
+                {
+                    throw new Exception("BitfinexSubscriptionManager.Connect(): WebSocket connection timeout.");
+                }
+            }
+            finally
+            {
+                webSocket.Open -= onOpenAction;
+
+                connectedEvent.DisposeSafely();
+            }
+        }
+
         private void OnConnectionLost(object sender, EventArgs e)
         {
             Log.Error("BitfinexSubscriptionManager.OnConnectionLost(): WebSocket connection lost.");
@@ -239,7 +270,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                 return;
             }
 
-            Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): IsOpen:{webSocket.IsOpen} ReadyState:{webSocket.ReadyState} [Id: {connectionHandler.ConnectionId}]");
+            Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): IsOpen:{webSocket.IsOpen} [Id: {connectionHandler.ConnectionId}]");
 
             if (!webSocket.IsOpen)
             {
@@ -249,11 +280,11 @@ namespace QuantConnect.Brokerages.Bitfinex
 
             if (!webSocket.IsOpen)
             {
-                Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): Websocket not open: IsOpen:{webSocket.IsOpen} ReadyState:{webSocket.ReadyState} [Id: {connectionHandler.ConnectionId}]");
+                Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): Websocket not open: IsOpen:{webSocket.IsOpen} [Id: {connectionHandler.ConnectionId}]");
                 return;
             }
 
-            Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): Reconnected: IsOpen:{webSocket.IsOpen} ReadyState:{webSocket.ReadyState} [Id: {connectionHandler.ConnectionId}]");
+            Log.Trace($"BitfinexSubscriptionManager.OnReconnectRequested(): Reconnected: IsOpen:{webSocket.IsOpen} [Id: {connectionHandler.ConnectionId}]");
 
             List<BitfinexChannel> channels;
             lock (_locker)
@@ -338,8 +369,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                         case "ping":
                             return;
                         case "error":
-                            var error = token.ToObject<Messages.ErrorMessage>();
-                            Log.Trace($"BitfinexSubscriptionManager.OnMessage(): {error.Level}: {error.Message}");
+                            Log.Error($"BitfinexSubscriptionManager.OnMessage(): {e.Message}");
                             return;
                         default:
                             Log.Trace($"BitfinexSubscriptionManager.OnMessage(): Unexpected message format: {e.Message}");

Brokerages/Bitfinex/BitfinexSymbolMapper.cs

@@ -81,7 +81,13 @@ namespace QuantConnect.Brokerages.Bitfinex
             "DRNUSD","DRNETH","PNKUSD","PNKETH","DGBUSD","DGBBTC","BSVUSD","BSVBTC","BABUSD","BABBTC",
             "WLOUSD","WLOXLM","VLDUSD","VLDETH","ENJUSD","ENJETH","ONLUSD","ONLETH","RBTUSD","RBTBTC",
             "USTUSD","EUTEUR","EUTUSD","GSDUSD","UDCUSD","TSDUSD","PAXUSD","RIFUSD","RIFBTC","PASUSD",
-            "PASETH","VSYUSD","VSYBTC","ZRXDAI","MKRDAI","OMGDAI"
+            "PASETH","VSYUSD","VSYBTC","ZRXDAI","MKRDAI","OMGDAI","BTTUSD","BTTBTC","BTCUST","ETHUST",
+            "CLOUSD","CLOBTC","IMPUSD","LTCUST","EOSUST","BABUST","SCRUSD","GNOUSD","GENUSD","ATOUSD",
+            "ATOBTC","ATOETH","WBTUSD","XCHUSD","EUSUSD","WBTETH","XCHETH","LEOUSD","LEOBTC","LEOUST",
+            "LEOEOS","LEOETH","ASTUSD","FOAUSD","UFRUSD","ZBTUSD","OKBUSD","USKUSD","GTXUSD","KANUSD",
+            "OKBUST","OKBBTC","USKUST","USKETH","USKBTC","USKEOS","GTXUST","KANUST","AMPUSD","ALGUSD",
+            "ALGBTC","ALGUST","BTCXCH","SWMUSD","TRIUSD","LOOUSD","AMPUST","UOSUSD","UOSBTC","RRBUSD",
+            "RRBUST","DTXUSD","AMPBTC","FTTUSD","FTTUST","PAXUST","UDCUST","TSDUST","CHZUSD","CHZUST",
         };
 
         /// <summary>
@@ -91,7 +97,85 @@ namespace QuantConnect.Brokerages.Bitfinex
         {
             "BCHUSD","BCHBTC","BCHETH",
             "CFIUSD","CFIBTC","CFIETH",
-            "VENUSD","VENBTC","VENETH"
+            "VENUSD","VENBTC","VENETH",
+            "RRTBTC",
+            "QTMETH",
+            "AVTBTC","AVTETH",
+            "QSHBTC","QSHETH",
+            "YYWBTC","YYWETH",
+            "MNAETH",
+            "FUNBTC",
+            "SPKBTC","SPKETH",
+            "RCNBTC","RCNETH",
+            "RLCETH",
+            "AIDBTC","AIDETH",
+            "SNGBTC","SNGETH",
+            "ELFBTC","ELFETH",
+            "AIOETH",
+            "REQBTC","REQETH",
+            "RDNBTC","RDNETH",
+            "LRCETH",
+            "WAXETH",
+            "AGIBTC","AGIETH",
+            "BFTETH",
+            "MTNBTC","MTNETH",
+            "DTHBTC","DTHETH",
+            "MITBTC","MITETH",
+            "STJBTC","STJETH",
+            "XLMJPY",
+            "XVGEUR","XVGJPY","XVGGBP","XVGETH",
+            "BCIUSD","BCIBTC",
+            "KNCETH",
+            "POABTC","POAETH",
+            "LYMBTC","LYMETH",
+            "UTKBTC","UTKETH",
+            "VEEBTC","VEEETH",
+            "DADUSD","DADBTC","DADETH",
+            "ORSBTC","ORSETH",
+            "AUCBTC","AUCETH",
+            "POYBTC","POYETH",
+            "FSNBTC","FSNETH",
+            "CBTBTC","CBTETH",
+            "ZCNBTC","ZCNETH",
+            "SENUSD","SENBTC","SENETH",
+            "NCABTC","NCAETH",
+            "CNDBTC",
+            "CTXBTC","CTXETH",
+            "PAIBTC",
+            "SEEBTC","SEEETH",
+            "ESSBTC","ESSETH",
+            "ATMBTC","ATMETH",
+            "HOTBTC","HOTETH",
+            "DTABTC","DTAETH",
+            "IQXBTC",
+            "WPRBTC","WPRETH",
+            "ZILBTC","ZILETH",
+            "BNTBTC","BNTETH",
+            "ABSETH",
+            "XRAETH",
+            "MANETH",
+            "BBNUSD","BBNETH",
+            "NIOUSD","NIOETH",
+            "VETETH",
+            "UTNETH",
+            "TKNETH",
+            "CNNETH",
+            "BOXETH",
+            "MGOETH",
+            "RTEETH",
+            "YGGETH",
+            "MLNETH",
+            "WTCETH",
+            "CSXETH",
+            "INTETH",
+            "DRNETH",
+            "WLOXLM",
+            "VLDETH",
+            "ENJETH",
+            "ONLETH",
+            "PASETH",
+            "ZRXDAI",
+            "OMGDAI",
         };
 
         /// <summary>

Brokerages/Bitfinex/BitfinexWebSocketWrapper.cs

@@ -20,7 +20,7 @@ namespace QuantConnect.Brokerages.Bitfinex
     /// <summary>
     /// Wrapper class for a Bitfinex websocket connection
     /// </summary>
-    public class BitfinexWebSocketWrapper : WebSocketWrapper
+    public class BitfinexWebSocketWrapper : WebSocketClientWrapper
     {
         /// <summary>
         /// The unique Id for the connection

Brokerages/QuantConnect.Brokerages.csproj

@@ -387,7 +387,6 @@
     <Compile Include="WebSocketError.cs" />
     <Compile Include="WebSocketMessage.cs" />
     <Compile Include="WebSocketClientWrapper.cs" />
-    <Compile Include="WebSocketWrapper.cs" />
     <Compile Include="InteractiveBrokers\Client\AccountDownloadEndEventArgs.cs" />
     <Compile Include="InteractiveBrokers\Client\ActionSide.cs" />
     <Compile Include="InteractiveBrokers\Client\AgentDescription.cs" />

Brokerages/WebSocketWrapper.cs

@@ -1,158 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Configuration;
-using QuantConnect.Logging;
-using WebSocketSharp;
-
-namespace QuantConnect.Brokerages
-{
-    /// <summary>
-    /// Wrapper for WebSocketSharp to enhance testability
-    /// </summary>
-    public class WebSocketWrapper : IWebSocket
-    {
-        private WebSocket _wrapped;
-        private string _url;
-
-        /// <summary>
-        /// Wraps constructor
-        /// </summary>
-        /// <param name="url"></param>
-        public void Initialize(string url)
-        {
-            if (_wrapped != null)
-            {
-                throw new InvalidOperationException("WebSocketWrapper has already been initialized for: " + _url);
-            }
-
-            _url = url;
-            _wrapped = new WebSocket(url)
-            {
-                Log =
-                {
-                    Level = Config.GetBool("websocket-log-trace") ? LogLevel.Trace : LogLevel.Error,
-
-                    // The stack frame number of 3 was derived from the usage of the Logger class in the WebSocketSharp library
-                    Output = (data, file) => { Log.Trace($"{WhoCalledMe.GetMethodName(3)}(): {data.Message}", true); }
-                }
-            };
-
-            // Some API only support SslProtocols Tls12
-            _wrapped.SslConfiguration.EnabledSslProtocols |= System.Security.Authentication.SslProtocols.Tls12;
-
-            _wrapped.OnOpen += (sender, args) => OnOpen();
-            _wrapped.OnMessage += (sender, args) => OnMessage(new WebSocketMessage(args.Data));
-            _wrapped.OnError += (sender, args) => OnError(new WebSocketError(args.Message, args.Exception));
-            _wrapped.OnClose += (sender, args) => OnClose(new WebSocketCloseData(args.Code, args.Reason, args.WasClean));
-        }
-
-        /// <summary>
-        /// Wraps send method
-        /// </summary>
-        /// <param name="data"></param>
-        public void Send(string data)
-        {
-            _wrapped.Send(data);
-        }
-
-        /// <summary>
-        /// Wraps Connect method
-        /// </summary>
-        public void Connect()
-        {
-            if (!IsOpen)
-            {
-                _wrapped.Connect();
-            }
-        }
-
-        /// <summary>
-        /// Wraps Close method
-        /// </summary>
-        public void Close()
-        {
-            _wrapped.Close();
-        }
-
-        /// <summary>
-        /// Wraps IsAlive
-        /// </summary>
-        public bool IsOpen => _wrapped.IsAlive;
-
-        /// <summary>
-        /// Wraps ReadyState
-        /// </summary>
-        public WebSocketState ReadyState => _wrapped.ReadyState;
-
-        /// <summary>
-        /// Wraps message event
-        /// </summary>
-        public event EventHandler<WebSocketMessage> Message;
-
-        /// <summary>
-        /// Wraps error event
-        /// </summary>
-        public event EventHandler<WebSocketError> Error;
-
-        /// <summary>
-        /// Wraps open method
-        /// </summary>
-        public event EventHandler Open;
-
-        /// <summary>
-        /// Wraps close method
-        /// </summary>
-        public event EventHandler<WebSocketCloseData> Closed;
-
-        /// <summary>
-        /// Event invocator for the <see cref="Message"/> event
-        /// </summary>
-        protected virtual void OnMessage(WebSocketMessage e)
-        {
-            //Logging.Log.Trace("WebSocketWrapper.OnMessage(): " + e.Message);
-            Message?.Invoke(this, e);
-        }
-
-        /// <summary>
-        /// Event invocator for the <see cref="Error"/> event
-        /// </summary>
-        /// <param name="e"></param>
-        protected virtual void OnError(WebSocketError e)
-        {
-            Log.Error(e.Exception, $"WebSocketWrapper.OnError(): (IsOpen:{IsOpen}, ReadyState:{_wrapped.ReadyState}): {e.Message}");
-            Error?.Invoke(this, e);
-        }
-
-        /// <summary>
-        /// Event invocator for the <see cref="Open"/> event
-        /// </summary>
-        protected virtual void OnOpen()
-        {
-            Log.Trace($"WebSocketWrapper.OnOpen(): Connection opened (IsOpen:{IsOpen}, ReadyState:{_wrapped.ReadyState}): {_url}");
-            Open?.Invoke(this, EventArgs.Empty);
-        }
-
-        /// <summary>
-        /// Event invocator for the <see cref="Close"/> event
-        /// </summary>
-        protected virtual void OnClose(WebSocketCloseData e)
-        {
-            Log.Trace($"WebSocketWrapper.OnClose(): Connection closed (IsOpen:{IsOpen}, ReadyState:{_wrapped.ReadyState}, Code:{e.Code}, Reason:{e.Reason}, WasClean:{e.WasClean}): {_url}");
-            Closed?.Invoke(this, e);
-        }
-    }
-}

Engine/RealTime/LiveTradingRealTimeHandler.cs

@@ -204,10 +204,12 @@ namespace QuantConnect.Lean.Engine.RealTime
         /// </summary>
         public void Exit()
         {
-            _timeMonitor.DisposeSafely();
-            _timeMonitor = null;
             _realTimeThread.StopSafely(TimeSpan.FromMinutes(5), _cancellationTokenSource);
             _realTimeThread = null;
+            _timeMonitor.DisposeSafely();
+            _timeMonitor = null;
+            _cancellationTokenSource.DisposeSafely();
+            _cancellationTokenSource = null;
         }
 
         /// <summary>

Tests/readme.md

@@ -0,0 +1,34 @@
+QuantConnect Testing
+=============
+
+Before starting any testing, follow the [installation instructions](https://github.com/QuantConnect/Lean#installation-instructions) to get LEAN running C# algorithms in your machine. 
+For any Python related tests please ensure you have followed the setup as described [here](https://github.com/QuantConnect/Lean/tree/master/Algorithm.Python#install-python-36).
+
+If the above installation, build, and initial run was succesful than we can move forward to testing.
+
+
+## Visual Studio:
+
+### Locating Tests
+
+- Open Visual Studios
+- Open Test Explorer ("Test" > "Test Explorer")
+- The list should populate itself as it reads all the tests it found during the build process. If not, press "Run All Tests" and let VS find all of the tests.
+- From here select the tests you would like to run and begin running them.
+
+
+### Failed Test Logs
+
+- On a failed test, check the test for information by clicking on the desired test and selecting "Open Additional Output"
+- This will show the stack trace and where the code failed to meet the testing requirements. 
+
+
+### Common Problems
+
+#### Having .NetFramework issues with testing?
+- Install [NUnit3TestAdapter](https://marketplace.visualstudio.com/items?itemName=NUnitDevelopers.NUnit3TestAdapter) for VS
+
+#### Missing dependencies for Python Algorithm?
+- Use pip or conda to install the module.
+
+