From 6b293cb81e85dfefe36fbf957f77f0a5be5524d4 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 19:55:50 +0000 Subject: [PATCH 1/8] Fix candles help string issue #17 --- duka/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duka/main.py b/duka/main.py index 3a18585..d83f154 100644 --- a/duka/main.py +++ b/duka/main.py @@ -23,7 +23,7 @@ def main(): parser.add_argument('-t', '--thread', type=int, help='number of threads (default 20)', default=5) parser.add_argument('-f', '--folder', type=str, help='destination folder (default .)', default='.') parser.add_argument('-c', '--candle', type=valid_timeframe, - help='use candles instead of ticks. Accepted values 1M 5M 10M 15M 30M 1H 4H', + help='use candles instead of ticks. Accepted values M2 M5 M10 M15 M30 H1 H4', default=TimeFrame.TICK) parser.add_argument('--header', action='store_true', help='include CSV header (default false)', default=False) args = parser.parse_args() From 27b0ee9ed3eb26e3bc4ab3e2c416ed05423bd99f Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 20:15:43 +0000 Subject: [PATCH 2/8] Add local time Fix #18 --- duka/app/app.py | 4 ++-- duka/core/processor.py | 10 ++++++---- duka/main.py | 5 +++-- setup.py | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/duka/app/app.py b/duka/app/app.py index 404c670..410f7a1 100644 --- a/duka/app/app.py +++ b/duka/app/app.py @@ -67,7 +67,7 @@ def name(symbol, timeframe, start, end): return name + ext -def app(symbols, start, end, threads, timeframe, folder, header): +def app(symbols, start, end, threads, timeframe, folder, header, local_time): if start > end: return lock = threading.Lock() @@ -85,7 +85,7 @@ def do_work(symbol, day, csv): star_time = time.time() Logger.info("Fetching day {0}".format(day)) try: - csv.append(day, decompress(day, fetch_day(symbol, day))) + csv.append(day, decompress(day, local_time, fetch_day(symbol, day))) except Exception as e: print("ERROR for {0}, {1} Exception : {2}".format(day, symbol, str(e))) elapsed_time = time.time() - star_time diff --git a/duka/core/processor.py b/duka/core/processor.py index cb8c4e9..b9fddf0 100644 --- a/duka/core/processor.py +++ b/duka/core/processor.py @@ -1,5 +1,6 @@ import struct from datetime import timedelta, datetime +from dateutil import tz from lzma import LZMADecompressor, LZMAError, FORMAT_AUTO from .utils import is_dst @@ -57,16 +58,17 @@ def add_hour(ticks): return ticks -def normalize(day, ticks): +def normalize(day, local_time, ticks): def norm(time, ask, bid, volume_ask, volume_bid): date = datetime(day.year, day.month, day.day) + timedelta(milliseconds=time) - # date.replace(tzinfo=datetime.tzinfo("UTC")) + if local_time: + date.replace(tzinfo=tz.tzlocal()) return date, ask / 100000, bid / 100000, round(volume_ask * 1000000), round(volume_bid * 1000000) return add_hour(list(map(lambda x: norm(*x), ticks))) -def decompress(day, compressed_buffer): +def decompress(day, local_time, compressed_buffer): if compressed_buffer.nbytes == 0: return compressed_buffer - return normalize(day, tokenize(decompress_lzma(compressed_buffer))) + return normalize(day, local_time, tokenize(decompress_lzma(compressed_buffer))) diff --git a/duka/main.py b/duka/main.py index d83f154..af1ae5f 100644 --- a/duka/main.py +++ b/duka/main.py @@ -7,7 +7,7 @@ from duka.core import valid_date, set_up_signals from duka.core.utils import valid_timeframe, TimeFrame -VERSION = '0.2.0' +VERSION = '0.2.1' def main(): @@ -26,6 +26,7 @@ def main(): help='use candles instead of ticks. Accepted values M2 M5 M10 M15 M30 H1 H4', default=TimeFrame.TICK) parser.add_argument('--header', action='store_true', help='include CSV header (default false)', default=False) + parser.add_argument('--local-time', action='store_true', help='use local time (default GMT)', default=False) args = parser.parse_args() if args.startdate is not None: @@ -39,7 +40,7 @@ def main(): end = args.day set_up_signals() - app(args.symbols, start, end, args.thread, args.candle, args.folder, args.header) + app(args.symbols, start, end, args.thread, args.candle, args.folder, args.header, args.local-time) if __name__ == '__main__': diff --git a/setup.py b/setup.py index 430c1aa..dc27222 100644 --- a/setup.py +++ b/setup.py @@ -3,12 +3,12 @@ from setuptools import setup, find_packages NAME = "duka" -VERSION = '0.2.0' +VERSION = '0.2.1' setup( name=NAME, packages=find_packages(), - install_requires=['requests>=2.9.1'], + install_requires=['requests>=2.9.1', 'python-dateutil'], version=VERSION, description='Dukascopy Bank SA historical data downloader', author='Giuseppe Pes', From d5dbe9432b2b9e6fae2ad1d33bbeabedadd20299 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 20:38:50 +0000 Subject: [PATCH 3/8] Fix unit tests --- Makefile | 5 +++++ duka/tests/test_dates_generator.py | 2 +- duka/tests/test_find_sunday.py | 10 +++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 195e7e7..9a3d956 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +.PHONY: test test-register test-release register release + +test: + @python3 -m unittest discover -s ./duka/tests -p "test_*" + test-register: python setup.py register -r pypitest diff --git a/duka/tests/test_dates_generator.py b/duka/tests/test_dates_generator.py index 5276873..8031df9 100644 --- a/duka/tests/test_dates_generator.py +++ b/duka/tests/test_dates_generator.py @@ -1,6 +1,6 @@ import unittest from datetime import date -from ..app.app import days +from duka.app.app import days class TestDateGenerator(unittest.TestCase): diff --git a/duka/tests/test_find_sunday.py b/duka/tests/test_find_sunday.py index c2f67b0..2643d04 100644 --- a/duka/tests/test_find_sunday.py +++ b/duka/tests/test_find_sunday.py @@ -38,21 +38,21 @@ def test_dst_2015(self): self.assertEqual(end.month, 11) def test_is_dst(self): - day = datetime.datetime(2015, 4, 5) + day = datetime.date(2015, 4, 5) self.assertTrue(is_dst(day)) def test_is_not_dst(self): - day = datetime.datetime(2015, 1, 1) + day = datetime.date(2015, 1, 1) self.assertFalse(is_dst(day)) def test_day_change_is_dst(self): - day = datetime.datetime(2015, 3, 8) + day = datetime.date(2015, 3, 8) self.assertTrue(is_dst(day)) def test_day_change_back_is_not_dst(self): - day = datetime.datetime(2015, 11, 1) + day = datetime.date(2015, 11, 1) self.assertFalse(is_dst(day)) def test_is_dst(self): - day = datetime.datetime(2013, 11, 3) + day = datetime.date(2013, 11, 3) self.assertFalse(is_dst(day)) From 9dc8c26d07b2a6b20354a821e805bdc4cda92cc8 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 20:49:24 +0000 Subject: [PATCH 4/8] Added travis --- .travis.yml | 6 ++++++ requirements.txt | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 .travis.yml create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e7728a1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: python +python: + - "3.5" +install: + - pip install -r requirements.txt +script: make test diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c9cbdda --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.9.1 +python-dateutil From 73fc96967cff3165788015a5bd0c94198368fcdb Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 20:59:06 +0000 Subject: [PATCH 5/8] Update Help message --- README.md | 5 ++++- duka/main.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8996ee8..a5dae22 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,15 @@ pip install duka optional arguments: -h show help message and exit + -v show program's version number and exit -d DAY specific day format YYYY-MM-DD (default today) -s STARTDATE start date format YYYY-MM-DD (default today) -e ENDDATE end date format YYYY-MM-DD (default today) - -c CANDLE use candles instead of ticks. Accepted values 1M 5M 10M 15M 30M 1H 4H 1D + -c CANDLE use candles instead of ticks. Accepted values M1 M2 M5 M10 M15 M30 H1 H4 D1 -f FOLDER the dowloaded data will be saved in FOLDER (default '.') -t THREAD number of threads (default 10) + --header include CSV header (default false) + --local-time use local time (default UTC) ``` ## Examples diff --git a/duka/main.py b/duka/main.py index af1ae5f..a9ed990 100644 --- a/duka/main.py +++ b/duka/main.py @@ -23,10 +23,10 @@ def main(): parser.add_argument('-t', '--thread', type=int, help='number of threads (default 20)', default=5) parser.add_argument('-f', '--folder', type=str, help='destination folder (default .)', default='.') parser.add_argument('-c', '--candle', type=valid_timeframe, - help='use candles instead of ticks. Accepted values M2 M5 M10 M15 M30 H1 H4', + help='use candles instead of ticks. Accepted values M1 M2 M5 M10 M15 M30 H1 H4', default=TimeFrame.TICK) parser.add_argument('--header', action='store_true', help='include CSV header (default false)', default=False) - parser.add_argument('--local-time', action='store_true', help='use local time (default GMT)', default=False) + parser.add_argument('--local-time', action='store_true', help='use local time (default UTC)', default=False) args = parser.parse_args() if args.startdate is not None: From b8be65a4f4dd192ca1724cbb76c8c77a22b61087 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 21:08:33 +0000 Subject: [PATCH 6/8] Fix variable name --- README.md | 2 +- duka/main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a5dae22..eb746bb 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ pip install duka -f FOLDER the dowloaded data will be saved in FOLDER (default '.') -t THREAD number of threads (default 10) --header include CSV header (default false) - --local-time use local time (default UTC) + --local-time use local time (default GMT) ``` ## Examples diff --git a/duka/main.py b/duka/main.py index a9ed990..5cde6b7 100644 --- a/duka/main.py +++ b/duka/main.py @@ -26,7 +26,7 @@ def main(): help='use candles instead of ticks. Accepted values M1 M2 M5 M10 M15 M30 H1 H4', default=TimeFrame.TICK) parser.add_argument('--header', action='store_true', help='include CSV header (default false)', default=False) - parser.add_argument('--local-time', action='store_true', help='use local time (default UTC)', default=False) + parser.add_argument('--local-time', action='store_true', help='use local time (default GMT)', default=False) args = parser.parse_args() if args.startdate is not None: @@ -40,7 +40,7 @@ def main(): end = args.day set_up_signals() - app(args.symbols, start, end, args.thread, args.candle, args.folder, args.header, args.local-time) + app(args.symbols, start, end, args.thread, args.candle, args.folder, args.header, args.local_time) if __name__ == '__main__': From b2a20917101bc94a31d21047db130eb92e5b9787 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 21:48:01 +0000 Subject: [PATCH 7/8] Add timezone info to the file name. --- duka/app/app.py | 4 ++-- duka/core/csv_dumper.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/duka/app/app.py b/duka/app/app.py index 410f7a1..5187226 100644 --- a/duka/app/app.py +++ b/duka/app/app.py @@ -52,7 +52,7 @@ def avg(fetch_times): return -1 -def name(symbol, timeframe, start, end): +def name(symbol, timeframe, time, start, end): ext = ".csv" for x in dir(TimeFrame): @@ -98,7 +98,7 @@ def do_work(symbol, day, csv): with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: - files = {symbol: CSVDumper(symbol, timeframe, start, end, folder, header) for symbol in symbols} + files = {symbol: CSVDumper(symbol, timeframe, start, end, folder, header, local_time) for symbol in symbols} for symbol in symbols: for day in days(start, end): diff --git a/duka/core/csv_dumper.py b/duka/core/csv_dumper.py index 08294ae..cbdf226 100644 --- a/duka/core/csv_dumper.py +++ b/duka/core/csv_dumper.py @@ -5,7 +5,7 @@ from .candle import Candle from .utils import TimeFrame, stringify, Logger -TEMPLATE_FILE_NAME = "{}-{}_{:02d}_{:02d}-{}_{:02d}_{:02d}.csv" +TEMPLATE_FILE_NAME = "{}-{}_{:02d}_{:02d}-{}_{:02d}_{:02d}-{}.csv" def format_float(number): @@ -39,7 +39,7 @@ def write_candle(writer, candle): class CSVDumper: - def __init__(self, symbol, timeframe, start, end, folder, header=False): + def __init__(self, symbol, timeframe, start, end, folder, header=False, local_time=False): self.symbol = symbol self.timeframe = timeframe self.start = start @@ -47,6 +47,7 @@ def __init__(self, symbol, timeframe, start, end, folder, header=False): self.folder = folder self.include_header = header self.buffer = {} + self.local_time = local_time def get_header(self): if self.timeframe == TimeFrame.TICK: @@ -75,10 +76,16 @@ def append(self, day, ticks): if self.timeframe != TimeFrame.TICK: self.buffer[day].append(Candle(self.symbol, previous_key, self.timeframe, current_ticks)) + def get_timezone(self): + if self.local_time: + return "local_time" + else: + return "GMT" + def dump(self): file_name = TEMPLATE_FILE_NAME.format(self.symbol, self.start.year, self.start.month, self.start.day, - self.end.year, self.end.month, self.end.day) + self.end.year, self.end.month, self.end.day, self.get_timezone()) Logger.info("Writing {0}".format(file_name)) From 1929f9fc03f4ef736bee3fdee264e919f3ec2543 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Sun, 27 Nov 2016 21:52:09 +0000 Subject: [PATCH 8/8] Removed unused name function --- duka/app/app.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/duka/app/app.py b/duka/app/app.py index 5187226..4336f0e 100644 --- a/duka/app/app.py +++ b/duka/app/app.py @@ -52,21 +52,6 @@ def avg(fetch_times): return -1 -def name(symbol, timeframe, time, start, end): - ext = ".csv" - - for x in dir(TimeFrame): - if getattr(TimeFrame, x) == timeframe: - ts_str = x - - name = symbol + "_" + ts_str + "_" + str(start) - - if start != end: - name += "_" + str(end) - - return name + ext - - def app(symbols, start, end, threads, timeframe, folder, header, local_time): if start > end: return