- ago
Below is the example provided for sorting/ranking symbols during PreExecute. I just noticed that when I optimize the strategy it gives different results than when a backtest using the same parameters are used. The test case has no NSF positions and therefore shouldn't be subject to the variability of order execution. If you look at the details below you can see that the position count varies greatly between the backtest and the result of the optimization when using the same parameters. Can you see if you can replicate this on your side?

Settings


Results of Backtest


Optimization Setting


Optimization Results



CODE:
using WealthLab.Backtest; using System; using WealthLab.Core; using WealthLab.Indicators; using System.Drawing; using System.Collections.Generic; namespace WealthScript1 {    public class IndicatorRank : UserStrategyBase    {       //this static list will be available in all instances of this class       private static List<BarHistory> _buys = new List<BarHistory>();       private int _percentOfCandidates = 25;       private int _period = 14;       public IndicatorRank()       {          AddParameter("% of Candidates", ParameterTypes.Int32, 20, 20, 20, 5);          AddParameter("Period", ParameterTypes.Int32, 10, 10, 20, 5);       }       public override void Initialize(BarHistory bars)       {          _percentOfCandidates = Parameters[0].AsInt;          _period = Parameters[1].AsInt;          StartIndex = _period;       }       public override void PreExecute(DateTime dt, List<BarHistory> participants)       {          if (participants.Count < 2)             return;          int candidates = (int)Math.Truncate(participants.Count * _percentOfCandidates / 100d);          //Loop through the BarHistories of the participants, find the index that corresponds to the DateTime dt,          //and save the indicator value to the bars.UserData for sorting          foreach (BarHistory bars in participants)          {             int idx = GetCurrentIndex(bars);             if (idx < _period)             {                bars.UserData = 1.0e10; // change this to a negative value if you change the sort from higher to lower below                continue;             }             //create a formula and assign a value. We'll just use RSI for this example             double rs = RSI.Series(bars.Close, _period)[idx];             bars.UserData = rs;          }          //this Sorts from lower to higher. For higher to lower, just change to .Sort(b, a)          participants.Sort((a, b) => a.UserDataAsDouble.CompareTo(b.UserDataAsDouble));          //Add the participants with the highest formula score to the _buys list          _buys.Clear();          _buys.AddRange(participants.GetRange(0, candidates));       }       public override void Execute(BarHistory bars, int idx)       {          if (!HasOpenPosition(bars, PositionType.Long))          {             // only buy this one if it's in the _buys list             if (_buys.Contains(bars))             {                PlaceTrade(bars, TransactionType.Buy, OrderType.Market);             }          }          else          {             if (!_buys.Contains(bars))                PlaceTrade(bars, TransactionType.Sell, OrderType.Market);          }       }    } }
0
453
Solved
8 Answers

Reply

Bookmark

Sort
- ago
#1
Yes, the reason is that the Optimzier is running in parallel, so by sorting the participants list and using BarHistory UserData (which would be shared across all runs) it's potentially changing things in the middle of another parallel run, leading to inconsistent results. If you use the Exhaustive (Non-Parallel) Optimizer your results should match ...

0
mjj37
- ago
#2
Thanks Dion.
0
mjj37
- ago
#3
So instead of using the bars.UserData property I used a dictionary of <BarHistory, double> to sort the symbols thinking that would solve the issue but it didn't. Is there another way around this or does the optimization routine not synchronize at the PreExecute or PostExecute step when running parallel?
0
- ago
#4
It doesn’t synchronize. I will need to think about it, but for now does the non-parallel optimizer solve this?
0
- ago
#5
QUOTE:
So instead of using the bars.UserData property I used a dictionary of <BarHistory, double> to sort the symbols thinking that would solve the issue but it didn't

Did you use a concurrent collection (which can be accessed by multiple threads at a time)?
0
- ago
#6
I guess you're using a static Dictionary so each instance of the Strategy for each symbol will share it? The trouble is, each parallel run might be hitting that dictionary on different bars, messing up the other runs when they change values on a different bar of processing. Not to mention threading conflicts like Eugene alluded to.

To support optimizations that need to use static variables I think the only way would be to introduce some other Cache object that is within the domain of the current backtest. That way, even if multiple backtests are ocurring in parallel during an optimization, each backtest can have its own data. We can leverage the Backtester class which the Strategy has access to via the Backtester property, and give it a Cache property just like the one that BarHistory/TimeSeries has.

I just tested it in Build 18 and it works for the parallel Exhaustive optimizer ...

CODE:
using WealthLab.Backtest; using System; using WealthLab.Core; using WealthLab.Indicators; using System.Drawing; using System.Collections.Generic; namespace WealthScript1 {    public class IndicatorRank : UserStrategyBase    {       private int _percentOfCandidates = 25;       private int _period = 14;       public IndicatorRank()       {          AddParameter("% of Candidates", ParameterTypes.Int32, 20, 20, 20, 5);          AddParameter("Period", ParameterTypes.Int32, 10, 10, 20, 5);       } public override void BacktestBegin() {          List<BarHistory> buys = new List<BarHistory>();          Backtester.Cache["Buys"] = buys;          Dictionary<string, double> userData = new Dictionary<string, double>();          Backtester.Cache["UserData"] = userData; } public override void Initialize(BarHistory bars)       {          _percentOfCandidates = Parameters[0].AsInt;          _period = Parameters[1].AsInt;          StartIndex = _period;       }       public override void PreExecute(DateTime dt, List<BarHistory> participants)       {          if (participants.Count < 2)             return;          int candidates = (int)Math.Truncate(participants.Count * _percentOfCandidates / 100d);          List<BarHistory> buys = (List<BarHistory>)Backtester.Cache["Buys"];          Dictionary<string, double> userData = (Dictionary<string, Double>)Backtester.Cache["UserData"];          //Loop through the BarHistories of the participants, find the index that corresponds to the DateTime dt,          //and save the indicator value to the bars.UserData for sorting          foreach (BarHistory bars in participants)          {             int idx = GetCurrentIndex(bars);             if (idx < _period)             {                userData[bars.Symbol] = 1.0e10; // change this to a negative value if you change the sort from higher to lower below                continue;             }             //create a formula and assign a value. We'll just use RSI for this example             double rs = RSI.Series(bars.Close, _period)[idx];             userData[bars.Symbol] = rs;          }          //this Sorts from lower to higher. For higher to lower, just change to .Sort(b, a)          List<BarHistory> myList = new List<BarHistory>();          myList.AddRange(participants);          myList.Sort((a, b) =>          {             double valueA = userData[a.Symbol];             double valueB = userData[b.Symbol];             return valueA.CompareTo(valueB);          });          //Add the participants with the highest formula score to the _buys list          buys.Clear();          buys.AddRange(myList.GetRange(0, candidates));       }       public override void Execute(BarHistory bars, int idx)       {          List<BarHistory> buys = (List<BarHistory>)Backtester.Cache["Buys"];          if (!HasOpenPosition(bars, PositionType.Long))          {             // only buy this one if it's in the _buys list             if (buys.Contains(bars))             {                PlaceTrade(bars, TransactionType.Buy, OrderType.Market);             }          }          else          {             if (!buys.Contains(bars))                PlaceTrade(bars, TransactionType.Sell, OrderType.Market);          }       }    } }
3
Best Answer
mjj37
- ago
#7
Thanks guys. That's perfect!
0
mjj37
- ago
#8
Actually concurrent dictionary doesn't work :-(
0

Reply

Bookmark

Sort