freqtrade · glonlas · Jan 7, 2018 · Jan 2, 2018 · Jan 2, 2018 · Jan 2, 2018
diff --git a/docs/backtesting.md b/docs/backtesting.md
@@ -46,6 +46,11 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --refresh-pairs-c
 python3 ./freqtrade/main.py backtesting --realistic-simulation --live
 ```
 
+**Using a different on-disk ticker-data source**
+```bash
+python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101
+```
+
 For help about backtesting usage, please refer to 
 [Backtesting commands](#backtesting-commands).
 
@@ -101,4 +106,4 @@ strategies, your configuration, and the crypto-currency you have set up.
 ## Next step
 Great, your strategy is profitable. What if the bot can give your the
 optimal parameters to use for your strategy?  
-Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md)
+Your next step is to learn [how to find optimal parameters with Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md)
diff --git a/docs/bot-usage.md b/docs/bot-usage.md
@@ -26,6 +26,9 @@ optional arguments:
                         specify configuration file (default: config.json)
   -v, --verbose         be verbose
   --version             show program's version number and exit
+  -dd PATH, --datadir PATH
+                        Path is from where backtesting and hyperopt will load the
+                        ticker data files (default freqdata/tests/testdata).
   --dynamic-whitelist [INT]
                         dynamically generate and update whitelist based on 24h
                         BaseVolume (Default 20 currencies)
@@ -133,4 +136,4 @@ in [misc.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/misc.py#L
 ## Next step
 The optimal strategy of the bot will change with time depending of the
 market trends. The next step is to 
-[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md).
+[optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md).
diff --git a/docs/hyperopt.md b/docs/hyperopt.md
@@ -163,6 +163,11 @@ We strongly recommend to use `screen` to prevent any connection loss.
 python3 ./freqtrade/main.py -c config.json hyperopt
 ```
 
+### Execute hyperopt with different ticker-data source
+If you would like to learn parameters using an alternate ticke-data that
+you have on-disk, use the --datadir PATH option. Default hyperopt will
+use data from directory freqtrade/tests/testdata.
+
 ### Hyperopt with MongoDB
 Hyperopt with MongoDB, is like Hyperopt under steroids. As you saw by
 executing the previous command is the execution takes a long time. 

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
@@ -134,8 +134,8 @@ def get_balances():
     return _API.get_balances()
 
 
-def get_ticker(pair: str) -> dict:
-    return _API.get_ticker(pair)
+def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict:
+    return _API.get_ticker(pair, refresh)
 
 
 @cached(TTLCache(maxsize=100, ttl=30))

diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py
@@ -1,5 +1,5 @@
 import logging
-from typing import List, Dict
+from typing import List, D
6D47
ict, Optional
 
 from bittrex.bittrex import Bittrex as _Bittrex, API_V2_0, API_V1_1
 from requests.exceptions import ContentDecodingError
@@ -38,6 +38,7 @@ def __init__(self, config: dict) -> None:
             calls_per_second=1,
             api_version=API_V2_0,
         )
+        self.cached_ticker = {}
 
     @staticmethod
     def _validate_response(response) -> None:
@@ -95,26 +96,29 @@ def get_balances(self):
             raise OperationalException('{message}'.format(message=data['message']))
         return data['result']
 
-    def get_ticker(self, pair: str) -> dict:
-        data = _API.get_ticker(pair.replace('_', '-'))
-        if not data['success']:
-            Bittrex._validate_response(data)
-            raise OperationalException('{message} params=({pair})'.format(
-                message=data['message'],
-                pair=pair))
-
-        if not data.get('result') \
-                or not data['result'].get('Bid') \
-                or not data['result'].get('Ask') \
-                or not data['result'].get('Last'):
-            raise ContentDecodingError('{message} params=({pair})'.format(
-                message='Got invalid response from bittrex',
-                pair=pair))
-        return {
-            'bid': float(data['result']['Bid']),
-            'ask': float(data['result']['Ask']),
-            'last': float(data['result']['Last']),
-        }
+    def get_ticker(self, pair: str, refresh: Optional[bool] = True
9E81
) -> dict:
+        if refresh or pair not in self.cached_ticker.keys():
+            data = _API.get_ticker(pair.replace('_', '-'))
+            if not data['success']:
+                Bittrex._validate_response(data)
+                raise OperationalException('{message} params=({pair})'.format(
+                    message=data['message'],
+                    pair=pair))
+
+            if not data.get('result') \
+                    or not data['result'].get('Bid') \
+                    or not data['result'].get('Ask') \
+                    or not data['result'].get('Last'):
+                raise ContentDecodingError('{message} params=({pair})'.format(
+                    message='Got invalid response from bittrex',
+                    pair=pair))
+            # Update the pair
+            self.cached_ticker[pair] = {
+                'bid': float(data['result']['Bid']),
+                'ask': float(data['result']['Ask']),
+                'last': float(data['result']['Last']),
+            }
+        return self.cached_ticker[pair]
 
     def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]:
         if tick_interval == 1:

diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py
@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List, Dict
+from typing import List, Dict, Optional
 
 
 class Exchange(ABC):
@@ -62,10 +62,11 @@ def get_balances(self) -> List[dict]:
         """
 
     @abstractmethod
-    def get_ticker(self, pair: str) -> dict:
+    def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict:
         """
         Gets ticker for given pair.
         :param pair: Pair as str, format: BTC_ETC
+        :param refresh: Shall we query a new value or a cached value is enough
         :return: dict, format: {
             'bid': float,
             'ask': float,

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
@@ -125,6 +125,14 @@ def parse_args(args: List[str], description: str):
         action='store_true',
         dest='dry_run_db',
     )
+    parser.add_argument(
+        '-dd', '--datadir',
+        help='path to backtest data (default freqdata/tests/testdata',
+        dest='datadir',
+        default='freqtrade/tests/testdata',
+        type=str,
+        metavar='PATH',
+    )
     parser.add_argument(
         '--dynamic-whitelist',
         help='dynamically generate and update whitelist based on 24h BaseVolume (Default 20 currencies)',  # noqa

diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
@@ -12,12 +12,12 @@
 logger = logging.getLogger(__name__)
 
 
-def load_tickerdata_file(pair, ticker_interval):
+def load_tickerdata_file(datadir, pair, ticker_interval):
     """
     Load a pair from file,
     :return dict OR empty if unsuccesful
     """
-    path = testdata_path()
+    path = make_testdata_path(datadir)
     file = '{abspath}/{pair}-{ticker_interval}.json'.format(
         abspath=path,
         pair=pair,
@@ -33,7 +33,7 @@ def load_tickerdata_file(pair, ticker_interval):
     return pairdata
 
 
-def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None,
+def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None,
          
10000
     refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
@@ -47,16 +47,16 @@ def load_data(ticker_interval: int = 5, pairs: Optional[List[str]] = None,
 
     # If the user force the refresh of pairs
     if refresh_pairs:
-        logger.info('Download data for all pairs and store them in freqtrade/tests/testsdata')
-        download_pairs(_pairs)
+        logger.info('Download data for all pairs and store them in %s', datadir)
+        download_pairs(datadir, _pairs)
 
     for pair in _pairs:
-        pairdata = load_tickerdata_file(pair, ticker_interval)
+        pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
         if not pairdata:
             # download the tickerdata from exchange
-            download_backtesting_testdata(pair=pair, interval=ticker_interval)
+            download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
             # and retry reading the pair
-            pairdata = load_tickerdata_file(pair, ticker_interval)
+            pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
         result[pair] = pairdata
     return result
 
@@ -67,17 +67,18 @@ def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
             for pair, pair_data in tickerdata.items()}
 
 
-def testdata_path() -> str:
+def make_testdata_path(datadir: str) -> str:
     """Return the path where testdata files are stored"""
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tests', 'testdata'))
+    return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                   '..', 'tests', 'testdata'))
 
 
-def download_pairs(pairs: List[str]) -> bool:
+def download_pairs(datadir, pairs: List[str]) -> bool:
     """For each pairs passed in parameters, download 1 and 5 ticker intervals"""
     for pair in pairs:
         try:
             for interval in [1, 5]:
-                download_backtesting_testdata(pair=pair, interval=interval)
+                download_backtesting_testdata(datadir, pair=pair, interval=interval)
         except BaseException:
             logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
                 pair=pair,
@@ -87,15 +88,15 @@ def download_pairs(pairs: List[str]) -> bool:
     return True
 
 
-def download_backtesting_testdata(pair: str, interval: int = 5) -> bool:
+def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool:
     """
     Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
     :param pairs: list of pairs to download
     :return: bool
     """
 
-    path = testdata_path()
+    path = make_testdata_path(datadir)
     logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
         pair=pair,
         interval=interval,

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
@@ -162,7 +162,7 @@ def start(args):
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
-        data = optimize.load_data(pairs=pairs, ticker_interval=args.ticker_interval,
+        data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
                                   refresh_pairs=args.refresh_pairs)
 
         logger.info('Using stake_currency: %s ...', config['stake_currency'])

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
@@ -36,7 +36,7 @@
 EXPECTED_MAX_PROFIT = 3.85
 
 # Configuration and data used by hyperopt
-PROCESSED = optimize.preprocess(optimize.load_data())
+PROCESSED = None  # optimize.preprocess(optimize.load_data())
 OPTIMIZE_CONFIG = hyperopt_optimize_conf()
 
 # Monkey patch config
@@ -224,7 +224,7 @@ def start(args):
     config = load_config(args.config)
     pairs = config['exchange']['pair_whitelist']
     PROCESSED = optimize.preprocess(optimize.load_data(
-        pairs=pairs, ticker_interval=args.ticker_interval))
+        args.datadir, pairs=pairs, ticker_interval=args.ticker_interval))
 
     if args.mongodb:
         logger.info('Using mongodb ...')

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
@@ -140,7 +140,7 @@ def _status(bot: Bot, update: Update) -> None:
             if trade.open_order_id:
                 order = exchange.get_order(trade.open_order_id)
             # calculate profit and send message to user
-            current_rate = exchange.get_ticker(trade.pair)['bid']
+            current_rate = exchange.get_ticker(trade.pair, False)['bid']
             current_profit = trade.calc_profit_percent(current_rate)
             fmt_close_profit = '{:.2f}%'.format(
                 round(trade.close_profit * 100, 2)
@@ -193,7 +193,7 @@ def _status_table(bot: Bot, update: Update) -> None:
         trades_list = []
         for trade in trades:
             # calculate profit and send message to user
-            current_rate = exchange.get_ticker(trade.pair)['bid']
+            current_rate = exchange.get_ticker(trade.pair, False)['bid']
             trades_list.append([
                 trade.id,
                 trade.pair,
@@ -301,7 +301,7 @@ def _profit(bot: Bot, update: Update) -> None:
             profit_closed_percent.append(profit_percent)
         else:
             # Get current rate
-            current_rate = exchange.get_ticker(trade.pair)['bid']
+            current_rate = exchange.get_ticker(trade.pair, False)['bid']
             profit_percent = trade.calc_profit_percent(rate=current_rate)
 
         profit_all_coin.append(trade.calc_profit(rate=Decimal(trade.close_rate or current_rate)))
@@ -579,7 +579,7 @@ def _exec_forcesell(trade: Trade) -> None:
             return
 
     # Get current rate and execute sell
-    current_rate = exchange.get_ticker(trade.pair)['bid']
+    current_rate = exchange.get_ticker(trade.pair, False)['bid']
     from freqtrade.main import execute_sell
     execute_sell(trade, current_rate)