From e8ec84a7c9060654e51b2143aa7db79e3cbd3c90 Mon Sep 17 00:00:00 2001 From: Ayush Ojha Date: Fri, 30 Jan 2026 23:56:16 -0800 Subject: [PATCH 1/2] feat: add auto lint fix tooling with isort and enhanced pre-commit - Add isort for import sorting with black-compatible profile - Add pre-commit hooks: trailing-whitespace, end-of-file-fixer, check-yaml, debug-statements, check-added-large-files - Add isort check to CI workflow (test_qlib_from_source.yml) - Add 'make fix' target for auto-formatting (black + isort + nbqa) - Add 'make isort' target for import sort checking - Add isort to lint dependencies in pyproject.toml - Configure [tool.isort] and [tool.black] sections in pyproject.toml Refs #2060 --- .github/workflows/test_qlib_from_source.yml | 4 ++++ .pre-commit-config.yaml | 24 ++++++++++++++++++--- Makefile | 15 +++++++++++-- pyproject.toml | 11 ++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index a53c5180552..0075240b1bd 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -58,6 +58,10 @@ jobs: run: | make black + - name: Check import sorting with isort + run: | + make isort + - name: Make html with sphinx # Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04. if: ${{ matrix.os == 'ubuntu-22.04' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15f00414c93..105dcbb0b9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,30 @@ repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + exclude: '\.md$' + - id: end-of-file-fixer + exclude: '\.md$' + - id: check-yaml + - id: debug-statements + - id: check-added-large-files + args: ['--maxkb=500'] + +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/psf/black rev: 23.7.0 hooks: - id: black - args: ["qlib", "-l 120"] + args: ["-l", "120"] + exclude: 'qlib/_version\.py' - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: - - id: flake8 - args: ["--ignore=E501,F541,E266,E402,W503,E731,E203"] + - id: flake8 + args: ["--ignore=E501,F541,E266,E402,W503,E731,E203", "--per-file-ignores=__init__.py:F401,F403"] diff --git a/Makefile b/Makefile index 40c80431875..ed9edd334bb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean deepclean prerequisite dependencies lightgbm rl develop lint docs package test analysis all install dev black pylint flake8 mypy nbqa nbconvert lint build upload docs-gen +.PHONY: clean deepclean prerequisite dependencies lightgbm rl develop lint docs package test analysis all install dev black isort pylint flake8 mypy nbqa nbconvert lint fix build upload docs-gen #You can modify it according to your terminal SHELL := /bin/bash @@ -118,6 +118,10 @@ dev: prerequisite all black: black . -l 120 --check --diff --exclude qlib/_version.py +# Check import sorting with isort. +isort: + isort --check-only --diff qlib scripts + # Check code folder with pylint. # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102 # C0103: invalid-name @@ -190,7 +194,14 @@ nbqa: nbconvert: jupyter nbconvert --to notebook --execute examples/workflow_by_code.ipynb -lint: black pylint flake8 mypy nbqa +# Run all lint checks. +lint: black isort pylint flake8 mypy nbqa + +# Auto-fix formatting issues (black + isort). +fix: + black . -l 120 --exclude qlib/_version.py + isort qlib scripts + nbqa black . -l 120 ######################################################################################## # Package diff --git a/pyproject.toml b/pyproject.toml index 4bfe68727d4..d96217c1a4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ rl = [ # which can cause false-positive --check results and inconsistent notebook formatting. lint = [ "black!=26.1.0", + "isort", "pylint", "mypy<1.5.0", "flake8", @@ -124,3 +125,13 @@ qrun = "qlib.cli.run:run" local_scheme = "no-local-version" version_scheme = "guess-next-dev" write_to = "qlib/_version.py" + +[tool.isort] +profile = "black" +line_length = 120 +known_first_party = ["qlib"] +skip_glob = ["qlib/_version.py"] + +[tool.black] +line-length = 120 +exclude = "qlib/_version.py" From c7cb809c277eb4d64eb49adb322af3cdf17e7964 Mon Sep 17 00:00:00 2001 From: Ayush Ojha Date: Fri, 30 Jan 2026 23:56:23 -0800 Subject: [PATCH 2/2] style: auto-fix import sorting across all Python files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 'isort qlib scripts' to sort all imports according to the isort configuration added in the previous commit. All changes are mechanical — no logic modifications. --- qlib/backtest/backtest.py | 3 +- qlib/backtest/utils.py | 2 +- qlib/cli/data.py | 2 +- qlib/cli/run.py | 2 +- qlib/config.py | 19 ++++---- qlib/contrib/data/data.py | 5 +- qlib/contrib/data/dataset.py | 8 ++-- qlib/contrib/data/handler.py | 6 ++- qlib/contrib/data/highfreq_processor.py | 3 +- qlib/contrib/data/highfreq_provider.py | 15 +++--- qlib/contrib/data/processor.py | 2 +- qlib/contrib/data/utils/sepdf.py | 3 +- qlib/contrib/eva/alpha.py | 8 ++-- qlib/contrib/evaluate.py | 23 +++++---- qlib/contrib/evaluate_portfolio.py | 9 ++-- qlib/contrib/meta/__init__.py | 3 +- qlib/contrib/meta/data_selection/__init__.py | 1 - qlib/contrib/meta/data_selection/dataset.py | 18 +++---- qlib/contrib/meta/data_selection/model.py | 22 ++++----- qlib/contrib/meta/data_selection/net.py | 2 +- qlib/contrib/model/__init__.py | 4 +- qlib/contrib/model/catboost_model.py | 9 ++-- qlib/contrib/model/double_ensemble.py | 8 ++-- qlib/contrib/model/gbdt.py | 13 +++-- qlib/contrib/model/highfreq_gdbt_model.py | 5 +- qlib/contrib/model/linear.py | 12 +++-- qlib/contrib/model/pytorch_adarnn.py | 6 +-- qlib/contrib/model/pytorch_add.py | 7 ++- qlib/contrib/model/pytorch_alstm.py | 17 ++++--- qlib/contrib/model/pytorch_alstm_ts.py | 19 ++++---- qlib/contrib/model/pytorch_gats.py | 20 ++++---- qlib/contrib/model/pytorch_gats_ts.py | 21 ++++----- qlib/contrib/model/pytorch_general_nn.py | 22 ++++----- qlib/contrib/model/pytorch_gru.py | 4 +- qlib/contrib/model/pytorch_gru_ts.py | 17 ++++--- qlib/contrib/model/pytorch_hist.py | 23 ++++----- qlib/contrib/model/pytorch_igmtf.py | 21 ++++----- qlib/contrib/model/pytorch_krnn.py | 15 +++--- qlib/contrib/model/pytorch_localformer.py | 17 ++++--- qlib/contrib/model/pytorch_localformer_ts.py | 15 +++--- qlib/contrib/model/pytorch_lstm.py | 15 +++--- qlib/contrib/model/pytorch_lstm_ts.py | 15 +++--- qlib/contrib/model/pytorch_nn.py | 30 ++++++------ qlib/contrib/model/pytorch_sandwich.py | 15 +++--- qlib/contrib/model/pytorch_sfm.py | 17 ++++--- qlib/contrib/model/pytorch_tabnet.py | 19 ++++---- qlib/contrib/model/pytorch_tcn.py | 17 ++++--- qlib/contrib/model/pytorch_tcn_ts.py | 15 +++--- qlib/contrib/model/pytorch_tcts.py | 13 +++-- qlib/contrib/model/pytorch_tra.py | 14 +++--- qlib/contrib/model/pytorch_transformer.py | 15 +++--- qlib/contrib/model/pytorch_transformer_ts.py | 13 +++-- qlib/contrib/model/xgboost.py | 8 ++-- qlib/contrib/online/manager.py | 6 ++- qlib/contrib/online/online_model.py | 2 + qlib/contrib/online/operator.py | 16 +++---- qlib/contrib/online/user.py | 2 +- qlib/contrib/online/utils.py | 6 ++- qlib/contrib/ops/high_freq.py | 3 +- .../contrib/report/analysis_model/__init__.py | 1 - .../analysis_model_performance.py | 9 ++-- .../report/analysis_position/__init__.py | 5 +- .../analysis_position/cumulative_return.py | 3 +- .../analysis_position/parse_position.py | 1 - .../report/analysis_position/rank_label.py | 2 +- .../report/analysis_position/report.py | 2 +- .../report/analysis_position/risk_analysis.py | 4 +- qlib/contrib/report/data/ana.py | 9 ++-- qlib/contrib/report/data/base.py | 3 +- qlib/contrib/report/graph.py | 8 ++-- qlib/contrib/rolling/__main__.py | 1 + qlib/contrib/rolling/base.py | 2 +- qlib/contrib/rolling/ddgda.py | 2 +- qlib/contrib/strategy/__init__.py | 17 +++---- qlib/contrib/strategy/cost_control.py | 3 +- qlib/contrib/strategy/optimizer/__init__.py | 3 +- .../strategy/optimizer/enhanced_indexing.py | 8 ++-- qlib/contrib/strategy/optimizer/optimizer.py | 3 +- qlib/contrib/strategy/order_generator.py | 7 +-- qlib/contrib/strategy/rule_strategy.py | 17 +++---- qlib/contrib/strategy/signal_strategy.py | 20 ++++---- qlib/contrib/torch.py | 2 +- qlib/contrib/tuner/config.py | 1 + qlib/contrib/tuner/launcher.py | 2 +- qlib/contrib/tuner/pipeline.py | 6 +-- qlib/contrib/tuner/space.py | 1 - qlib/contrib/tuner/tuner.py | 19 ++++---- qlib/contrib/workflow/__init__.py | 4 +- qlib/contrib/workflow/record_temp.py | 8 ++-- qlib/data/__init__.py | 47 +++++++++---------- qlib/data/_libs/expanding.pyx | 3 +- qlib/data/_libs/rolling.pyx | 3 +- qlib/data/base.py | 5 +- qlib/data/cache.py | 29 ++++++------ qlib/data/data.py | 28 +++++------ qlib/data/dataset/__init__.py | 17 +++---- qlib/data/dataset/handler.py | 19 ++++---- qlib/data/dataset/loader.py | 10 ++-- qlib/data/dataset/processor.py | 12 +++-- qlib/data/dataset/storage.py | 9 ++-- qlib/data/dataset/utils.py | 5 +- qlib/data/filter.py | 7 +-- qlib/data/inst_processor.py | 1 + qlib/data/ops.py | 14 +++--- qlib/data/pit.py | 2 + qlib/data/storage/__init__.py | 3 +- qlib/data/storage/file_storage.py | 8 ++-- qlib/data/storage/storage.py | 3 +- qlib/log.py | 4 +- qlib/model/__init__.py | 1 - qlib/model/base.py | 3 +- qlib/model/ens/ensemble.py | 4 +- qlib/model/ens/group.py | 4 +- qlib/model/interpret/base.py | 3 +- qlib/model/meta/__init__.py | 3 +- qlib/model/meta/dataset.py | 4 +- qlib/model/meta/task.py | 1 + qlib/model/riskmodel/__init__.py | 1 - qlib/model/riskmodel/base.py | 3 +- qlib/model/riskmodel/shrink.py | 3 +- qlib/model/riskmodel/structured.py | 3 +- qlib/rl/__init__.py | 2 +- qlib/rl/contrib/naive_config_parser.py | 2 +- qlib/rl/contrib/train_onpolicy.py | 9 ++-- qlib/rl/interpreter.py | 1 + qlib/rl/order_execution/__init__.py | 8 ++-- qlib/rl/order_execution/policy.py | 2 +- qlib/rl/order_execution/simulator_qlib.py | 1 + qlib/rl/order_execution/simulator_simple.py | 5 +- qlib/rl/order_execution/state.py | 1 + qlib/rl/order_execution/strategy.py | 4 +- qlib/rl/trainer/callbacks.py | 2 +- qlib/rl/utils/data_queue.py | 2 +- qlib/rl/utils/env_wrapper.py | 3 +- qlib/strategy/base.py | 2 +- qlib/tests/__init__.py | 18 +++---- qlib/tests/data.py | 12 +++-- qlib/typehint.py | 2 +- qlib/utils/__init__.py | 41 ++++++++-------- qlib/utils/file.py | 4 +- qlib/utils/index_data.py | 2 +- qlib/utils/mod.py | 2 +- qlib/utils/paral.py | 7 ++- qlib/utils/resam.py | 8 ++-- qlib/utils/serial.py | 4 +- qlib/utils/time.py | 5 +- qlib/workflow/__init__.py | 9 ++-- qlib/workflow/exp.py | 7 ++- qlib/workflow/expm.py | 14 +++--- qlib/workflow/online/manager.py | 1 + qlib/workflow/online/strategy.py | 1 + qlib/workflow/online/update.py | 3 +- qlib/workflow/record_temp.py | 18 +++---- qlib/workflow/recorder.py | 19 ++++---- qlib/workflow/task/collect.py | 6 +-- qlib/workflow/task/gen.py | 4 +- qlib/workflow/task/manage.py | 5 +- qlib/workflow/task/utils.py | 14 +++--- scripts/check_dump_bin.py | 12 ++--- scripts/collect_info.py | 8 ++-- .../data_collector/baostock_5min/collector.py | 13 ++--- scripts/data_collector/base.py | 11 +++-- scripts/data_collector/br_index/collector.py | 6 +-- scripts/data_collector/cn_index/collector.py | 14 +++--- .../fill_cn_1min_data/fill_cn_1min_data.py | 7 +-- .../future_trading_date_collector.py | 7 ++- scripts/data_collector/crypto/collector.py | 13 +++-- scripts/data_collector/fund/collector.py | 7 +-- .../future_calendar_collector.py | 7 ++- scripts/data_collector/index.py | 4 +- scripts/data_collector/pit/collector.py | 8 ++-- scripts/data_collector/us_index/collector.py | 17 +++---- scripts/data_collector/utils.py | 20 ++++---- scripts/data_collector/yahoo/collector.py | 24 +++++----- scripts/dump_bin.py | 9 ++-- scripts/dump_pit.py | 9 ++-- scripts/get_data.py | 2 +- 177 files changed, 791 insertions(+), 764 deletions(-) diff --git a/qlib/backtest/backtest.py b/qlib/backtest/backtest.py index 5e5edacafd6..89ae78736cd 100644 --- a/qlib/backtest/backtest.py +++ b/qlib/backtest/backtest.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Dict, TYPE_CHECKING, Generator, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Dict, Generator, Optional, Tuple, Union, cast import pandas as pd @@ -18,7 +18,6 @@ from ..utils.time import Freq - PORT_METRIC = Dict[str, Tuple[pd.DataFrame, dict]] INDICATOR_METRIC = Dict[str, Tuple[pd.DataFrame, Indicator]] diff --git a/qlib/backtest/utils.py b/qlib/backtest/utils.py index 4210c9548a8..c3b99d39f87 100644 --- a/qlib/backtest/utils.py +++ b/qlib/backtest/utils.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import abstractmethod -from typing import Any, Set, Tuple, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Set, Tuple, Union import numpy as np diff --git a/qlib/cli/data.py b/qlib/cli/data.py index 3710816ce08..20aad754f7e 100644 --- a/qlib/cli/data.py +++ b/qlib/cli/data.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. import fire -from qlib.tests.data import GetData +from qlib.tests.data import GetData if __name__ == "__main__": fire.Fire(GetData) diff --git a/qlib/cli/run.py b/qlib/cli/run.py index c2dcdc2ef0c..d4dea3f5123 100644 --- a/qlib/cli/run.py +++ b/qlib/cli/run.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. import logging import os -from pathlib import Path import sys +from pathlib import Path import fire from jinja2 import Template, meta diff --git a/qlib/config.py b/qlib/config.py index 4e5d62564f7..042370f24e2 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -12,17 +12,16 @@ """ from __future__ import annotations -import os -import re import copy import logging -import platform import multiprocessing +import os +import platform +import re from pathlib import Path -from typing import Callable, Optional, Union -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Optional, Union -from qlib.constant import REG_CN, REG_US, REG_TW +from qlib.constant import REG_CN, REG_TW, REG_US if TYPE_CHECKING: from qlib.utils.time import Freq @@ -438,7 +437,7 @@ def set(self, default_conf: str = "client", **kwargs): default_conf : str the default config template chosen by user: "server", "client" """ - from .utils import set_log_with_config, get_module_logger, can_use_cache # pylint: disable=C0415 + from .utils import can_use_cache, get_module_logger, set_log_with_config # pylint: disable=C0415 self.reset() @@ -480,10 +479,10 @@ def set(self, default_conf: str = "client", **kwargs): ) def register(self): - from .utils import init_instance_by_config # pylint: disable=C0415 - from .data.ops import register_all_ops # pylint: disable=C0415 from .data.data import register_all_wrappers # pylint: disable=C0415 - from .workflow import R, QlibRecorder # pylint: disable=C0415 + from .data.ops import register_all_ops # pylint: disable=C0415 + from .utils import init_instance_by_config # pylint: disable=C0415 + from .workflow import QlibRecorder, R # pylint: disable=C0415 from .workflow.utils import experiment_exit_handler # pylint: disable=C0415 register_all_ops(self) diff --git a/qlib/contrib/data/data.py b/qlib/contrib/data/data.py index c153cfb8f6d..591631d7b0c 100644 --- a/qlib/contrib/data/data.py +++ b/qlib/contrib/data/data.py @@ -7,11 +7,12 @@ # - pip fail to computing the right version number!!!! # - Maybe we can solve this problem by poetry +import pandas as pd +import pymongo + # FIXME: So if you want to use arctic-based provider, please install arctic manually # `pip install arctic` may not be enough. from arctic import Arctic -import pandas as pd -import pymongo from qlib.data.data import FeatureProvider diff --git a/qlib/contrib/data/dataset.py b/qlib/contrib/data/dataset.py index 812e2cc7135..528e623163f 100644 --- a/qlib/contrib/data/dataset.py +++ b/qlib/contrib/data/dataset.py @@ -2,15 +2,15 @@ # Licensed under the MIT License. import copy -import torch import warnings + import numpy as np import pandas as pd -from qlib.utils.data import guess_horizon -from qlib.utils import init_instance_by_config +import torch from qlib.data.dataset import DatasetH - +from qlib.utils import init_instance_by_config +from qlib.utils.data import guess_horizon device = "cuda" if torch.cuda.is_available() else "cpu" diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 2fe5258daa7..ba6f7c764d5 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -1,12 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from inspect import getfullargspec + from qlib.contrib.data.loader import Alpha158DL, Alpha360DL + +from ...data.dataset import processor as processor_module from ...data.dataset.handler import DataHandlerLP from ...data.dataset.processor import Processor from ...utils import get_callable_kwargs -from ...data.dataset import processor as processor_module -from inspect import getfullargspec def check_transform_proc(proc_l, fit_start_time, fit_end_time): diff --git a/qlib/contrib/data/highfreq_processor.py b/qlib/contrib/data/highfreq_processor.py index db2a6e39b4a..fe10cf2a6fa 100644 --- a/qlib/contrib/data/highfreq_processor.py +++ b/qlib/contrib/data/highfreq_processor.py @@ -1,10 +1,11 @@ import os +from typing import Dict import numpy as np import pandas as pd + from qlib.data.dataset.processor import Processor from qlib.data.dataset.utils import fetch_df_by_index -from typing import Dict class HighFreqTrans(Processor): diff --git a/qlib/contrib/data/highfreq_provider.py b/qlib/contrib/data/highfreq_provider.py index 611e30d861f..d5397127100 100644 --- a/qlib/contrib/data/highfreq_provider.py +++ b/qlib/contrib/data/highfreq_provider.py @@ -1,18 +1,19 @@ +import datetime import os +import pickle as pkl import time -import datetime from typing import Optional +from joblib import Parallel, delayed + import qlib from qlib import get_module_logger -from qlib.data import D from qlib.config import REG_CN -from qlib.utils import init_instance_by_config -from qlib.data.dataset.handler import DataHandlerLP +from qlib.contrib.ops.high_freq import BFillNan, Cut, Date, DayLast, FFillNan, IsInf, IsNull, Select, get_calendar_day +from qlib.data import D from qlib.data.data import Cal -from qlib.contrib.ops.high_freq import get_calendar_day, DayLast, FFillNan, BFillNan, Date, Select, IsNull, IsInf, Cut -import pickle as pkl -from joblib import Parallel, delayed +from qlib.data.dataset.handler import DataHandlerLP +from qlib.utils import init_instance_by_config class HighFreqProvider: diff --git a/qlib/contrib/data/processor.py b/qlib/contrib/data/processor.py index 4047ed73ae4..833373a3222 100644 --- a/qlib/contrib/data/processor.py +++ b/qlib/contrib/data/processor.py @@ -1,7 +1,7 @@ import numpy as np -from ...log import TimeInspector from ...data.dataset.processor import Processor, get_group_columns +from ...log import TimeInspector class ConfigSectionProcessor(Processor): diff --git a/qlib/contrib/data/utils/sepdf.py b/qlib/contrib/data/utils/sepdf.py index 00e14e1476e..30936a9395c 100644 --- a/qlib/contrib/data/utils/sepdf.py +++ b/qlib/contrib/data/utils/sepdf.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import pandas as pd from typing import Dict, Iterable, Union +import pandas as pd + def align_index(df_dict, join): res = {} diff --git a/qlib/contrib/eva/alpha.py b/qlib/contrib/eva/alpha.py index 2c2f9ea7e18..e129423fcc6 100644 --- a/qlib/contrib/eva/alpha.py +++ b/qlib/contrib/eva/alpha.py @@ -4,12 +4,14 @@ The interface should be redesigned carefully in the future. """ -import pandas as pd from typing import Tuple -from qlib import get_module_logger -from qlib.utils.paral import complex_parallel, DelayedDict + +import pandas as pd from joblib import Parallel, delayed +from qlib import get_module_logger +from qlib.utils.paral import DelayedDict, complex_parallel + def calc_long_short_prec( pred: pd.Series, label: pd.Series, date_col="datetime", quantile: float = 0.2, dropna=False, is_alpha=False diff --git a/qlib/contrib/evaluate.py b/qlib/contrib/evaluate.py index e0bacfca855..42d70a4c2fc 100644 --- a/qlib/contrib/evaluate.py +++ b/qlib/contrib/evaluate.py @@ -1,25 +1,24 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + +import warnings +from typing import Literal, Union import numpy as np import pandas as pd -import warnings -from typing import Union, Literal +from ..backtest import backtest as backtest_func +from ..backtest import executor as _executor +from ..backtest import get_exchange, position +from ..config import C +from ..data import D +from ..data.dataset.utils import get_level_index from ..log import get_module_logger +from ..strategy.base import BaseStrategy from ..utils import get_date_range from ..utils.resam import Freq -from ..strategy.base import BaseStrategy -from ..backtest import get_exchange, position, backtest as backtest_func, executor as _executor - - -from ..data import D -from ..config import C -from ..data.dataset.utils import get_level_index - logger = get_module_logger("Evaluate") diff --git a/qlib/contrib/evaluate_portfolio.py b/qlib/contrib/evaluate_portfolio.py index 0c598e2fa83..9173971b001 100644 --- a/qlib/contrib/evaluate_portfolio.py +++ b/qlib/contrib/evaluate_portfolio.py @@ -2,17 +2,16 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + +from collections import OrderedDict import numpy as np import pandas as pd -from scipy.stats import spearmanr, pearsonr +from scipy.stats import pearsonr, spearmanr from ..data import D -from collections import OrderedDict - def _get_position_value_from_df(evaluate_date, position, close_data_df): """Get position value by existed close data df diff --git a/qlib/contrib/meta/__init__.py b/qlib/contrib/meta/__init__.py index 2425c001e45..a52079e5c8e 100644 --- a/qlib/contrib/meta/__init__.py +++ b/qlib/contrib/meta/__init__.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS - +from .data_selection import MetaDatasetDS, MetaModelDS, MetaTaskDS __all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"] diff --git a/qlib/contrib/meta/data_selection/__init__.py b/qlib/contrib/meta/data_selection/__init__.py index 969e77a932f..a645defaca6 100644 --- a/qlib/contrib/meta/data_selection/__init__.py +++ b/qlib/contrib/meta/data_selection/__init__.py @@ -4,5 +4,4 @@ from .dataset import MetaDatasetDS, MetaTaskDS from .model import MetaModelDS - __all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"] diff --git a/qlib/contrib/meta/data_selection/dataset.py b/qlib/contrib/meta/data_selection/dataset.py index 61efdd63cfb..01b67c4d348 100644 --- a/qlib/contrib/meta/data_selection/dataset.py +++ b/qlib/contrib/meta/data_selection/dataset.py @@ -1,23 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import pandas as pd -import numpy as np from copy import deepcopy +from typing import Dict, List, Text, Tuple, Union + +import numpy as np +import pandas as pd from joblib import Parallel, delayed # pylint: disable=E0401 -from typing import Dict, List, Union, Text, Tuple -from qlib.data.dataset.utils import init_task_handler -from qlib.data.dataset import DatasetH +from tqdm.auto import tqdm + from qlib.contrib.torch import data_to_tensor -from qlib.model.meta.task import MetaTask +from qlib.data.dataset import DatasetH +from qlib.data.dataset.utils import init_task_handler +from qlib.log import get_module_logger from qlib.model.meta.dataset import MetaTaskDataset +from qlib.model.meta.task import MetaTask from qlib.model.trainer import TrainerR -from qlib.log import get_module_logger from qlib.utils import auto_filter_kwargs, get_date_by_shift, init_instance_by_config from qlib.utils.data import deepcopy_basic_type from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator from qlib.workflow.task.utils import TimeAdjuster -from tqdm.auto import tqdm class InternalData: diff --git a/qlib/contrib/meta/data_selection/model.py b/qlib/contrib/meta/data_selection/model.py index ed3ff9397e7..95a2a326519 100644 --- a/qlib/contrib/meta/data_selection/model.py +++ b/qlib/contrib/meta/data_selection/model.py @@ -1,25 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import pandas as pd +import copy +from typing import List, Union + import numpy as np +import pandas as pd import torch -from torch import nn -from torch import optim +from torch import nn, optim from tqdm.auto import tqdm -import copy -from typing import Union, List + +from qlib.contrib.meta.data_selection.net import PredNet +from qlib.data.dataset.weight import Reweighter +from qlib.log import get_module_logger +from qlib.model.meta.task import MetaTask from ....model.meta.dataset import MetaTaskDataset from ....model.meta.model import MetaTaskModel from ....workflow import R -from .utils import ICLoss from .dataset import MetaDatasetDS - -from qlib.log import get_module_logger -from qlib.model.meta.task import MetaTask -from qlib.data.dataset.weight import Reweighter -from qlib.contrib.meta.data_selection.net import PredNet +from .utils import ICLoss logger = get_module_logger("data selection") diff --git a/qlib/contrib/meta/data_selection/net.py b/qlib/contrib/meta/data_selection/net.py index fce19df3e27..40a26ed456a 100644 --- a/qlib/contrib/meta/data_selection/net.py +++ b/qlib/contrib/meta/data_selection/net.py @@ -5,7 +5,7 @@ import torch from torch import nn -from .utils import preds_to_weight_with_clamp, SingleMetaBase +from .utils import SingleMetaBase, preds_to_weight_with_clamp class TimeWeightMeta(SingleMetaBase): diff --git a/qlib/contrib/model/__init__.py b/qlib/contrib/model/__init__.py index 5d4d5f2e690..23b91aa690e 100644 --- a/qlib/contrib/model/__init__.py +++ b/qlib/contrib/model/__init__.py @@ -25,15 +25,15 @@ print("ModuleNotFoundError. LinearModel is skipped(optional: maybe installing scipy and sklearn can fix it).") # import pytorch models try: + from .pytorch_add import ADD from .pytorch_alstm import ALSTM from .pytorch_gats import GATs from .pytorch_gru import GRU from .pytorch_lstm import LSTM from .pytorch_nn import DNNModelPytorch - from .pytorch_tabnet import TabnetModel from .pytorch_sfm import SFM_Model + from .pytorch_tabnet import TabnetModel from .pytorch_tcn import TCN - from .pytorch_add import ADD pytorch_classes = (ALSTM, GATs, GRU, LSTM, DNNModelPytorch, TabnetModel, SFM_Model, TCN, ADD) except ModuleNotFoundError: diff --git a/qlib/contrib/model/catboost_model.py b/qlib/contrib/model/catboost_model.py index 4fc1c6f8934..a5176267d14 100644 --- a/qlib/contrib/model/catboost_model.py +++ b/qlib/contrib/model/catboost_model.py @@ -1,17 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import Text, Union + import numpy as np import pandas as pd -from typing import Text, Union -from catboost import Pool, CatBoost +from catboost import CatBoost, Pool from catboost.utils import get_gpu_device_count -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...model.interpret.base import FeatureInt from ...data.dataset.weight import Reweighter +from ...model.base import Model +from ...model.interpret.base import FeatureInt class CatBoostModel(Model, FeatureInt): diff --git a/qlib/contrib/model/double_ensemble.py b/qlib/contrib/model/double_ensemble.py index 85d4418f4db..50a9495eb16 100644 --- a/qlib/contrib/model/double_ensemble.py +++ b/qlib/contrib/model/double_ensemble.py @@ -1,15 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import Text, Union + import lightgbm as lgb import numpy as np import pandas as pd -from typing import Text, Union -from ...model.base import Model + from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...model.interpret.base import FeatureInt from ...log import get_module_logger +from ...model.base import Model +from ...model.interpret.base import FeatureInt class DEnsembleModel(Model, FeatureInt): diff --git a/qlib/contrib/model/gbdt.py b/qlib/contrib/model/gbdt.py index 22c29cd4997..ae8b878abef 100644 --- a/qlib/contrib/model/gbdt.py +++ b/qlib/contrib/model/gbdt.py @@ -1,16 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import List, Text, Tuple, Union + +import lightgbm as lgb import numpy as np import pandas as pd -import lightgbm as lgb -from typing import List, Text, Tuple, Union -from ...model.base import ModelFT + +from qlib.workflow import R + from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...model.interpret.base import LightGBMFInt from ...data.dataset.weight import Reweighter -from qlib.workflow import R +from ...model.base import ModelFT +from ...model.interpret.base import LightGBMFInt class LGBModel(ModelFT, LightGBMFInt): diff --git a/qlib/contrib/model/highfreq_gdbt_model.py b/qlib/contrib/model/highfreq_gdbt_model.py index ad0641136f2..0ca4f0cb800 100644 --- a/qlib/contrib/model/highfreq_gdbt_model.py +++ b/qlib/contrib/model/highfreq_gdbt_model.py @@ -2,13 +2,14 @@ # Licensed under the MIT License. import warnings + +import lightgbm as lgb import numpy as np import pandas as pd -import lightgbm as lgb -from ...model.base import ModelFT from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...model.base import ModelFT from ...model.interpret.base import LightGBMFInt diff --git a/qlib/contrib/model/linear.py b/qlib/contrib/model/linear.py index 15cdb739e98..29be719083f 100644 --- a/qlib/contrib/model/linear.py +++ b/qlib/contrib/model/linear.py @@ -1,17 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import Text, Union + import numpy as np import pandas as pd -from typing import Text, Union -from qlib.log import get_module_logger -from qlib.data.dataset.weight import Reweighter from scipy.optimize import nnls -from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.linear_model import Lasso, LinearRegression, Ridge + +from qlib.data.dataset.weight import Reweighter +from qlib.log import get_module_logger -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...model.base import Model class LinearModel(Model): diff --git a/qlib/contrib/model/pytorch_adarnn.py b/qlib/contrib/model/pytorch_adarnn.py index c1585a6ac0a..9b60bdb89ba 100644 --- a/qlib/contrib/model/pytorch_adarnn.py +++ b/qlib/contrib/model/pytorch_adarnn.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. -import os -from torch.utils.data import Dataset, DataLoader - import copy +import os from typing import Text, Union import numpy as np @@ -12,6 +10,8 @@ import torch.nn.functional as F import torch.optim as optim from torch.autograd import Function +from torch.utils.data import DataLoader, Dataset + from qlib.contrib.model.pytorch_utils import count_parameters from qlib.data.dataset import DatasetH from qlib.data.dataset.handler import DataHandlerLP diff --git a/qlib/contrib/model/pytorch_add.py b/qlib/contrib/model/pytorch_add.py index c94a03ecc31..9d4583e8154 100644 --- a/qlib/contrib/model/pytorch_add.py +++ b/qlib/contrib/model/pytorch_add.py @@ -1,9 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function - +from __future__ import division, print_function import copy import math @@ -15,6 +13,8 @@ import torch.nn as nn import torch.nn.functional as F import torch.optim as optim +from torch.autograd import Function + from qlib.contrib.model.pytorch_gru import GRUModel from qlib.contrib.model.pytorch_lstm import LSTMModel from qlib.contrib.model.pytorch_utils import count_parameters @@ -23,7 +23,6 @@ from qlib.log import get_module_logger from qlib.model.base import Model from qlib.utils import get_or_create_path -from torch.autograd import Function class ADD(Model): diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py index d1c619ebf41..7fe9c4d484d 100644 --- a/qlib/contrib/model/pytorch_alstm.py +++ b/qlib/contrib/model/pytorch_alstm.py @@ -2,24 +2,23 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class ALSTM(Model): diff --git a/qlib/contrib/model/pytorch_alstm_ts.py b/qlib/contrib/model/pytorch_alstm_ts.py index 95b5cf95d8b..4912073be5e 100644 --- a/qlib/contrib/model/pytorch_alstm_ts.py +++ b/qlib/contrib/model/pytorch_alstm_ts.py @@ -2,27 +2,26 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...model.utils import ConcatDataset from ...data.dataset.weight import Reweighter +from ...log import get_module_logger +from ...model.base import Model +from ...model.utils import ConcatDataset +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class ALSTM(Model): diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py index 16439b3783a..f16165b8b96 100644 --- a/qlib/contrib/model/pytorch_gats.py +++ b/qlib/contrib/model/pytorch_gats.py @@ -2,25 +2,25 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + +import copy +from typing import Text, Union import numpy as np import pandas as pd -from typing import Text, Union -import copy -from ...utils import get_or_create_path -from ...log import get_module_logger import torch import torch.nn as nn import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model +from ...contrib.model.pytorch_gru import GRUModel +from ...contrib.model.pytorch_lstm import LSTMModel from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...contrib.model.pytorch_lstm import LSTMModel -from ...contrib.model.pytorch_gru import GRUModel +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class GATs(Model): diff --git a/qlib/contrib/model/pytorch_gats_ts.py b/qlib/contrib/model/pytorch_gats_ts.py index 09f0ac08b25..5d71e20ccdb 100644 --- a/qlib/contrib/model/pytorch_gats_ts.py +++ b/qlib/contrib/model/pytorch_gats_ts.py @@ -2,25 +2,24 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + +import copy import numpy as np import pandas as pd -import copy -from ...utils import get_or_create_path -from ...log import get_module_logger import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import DataLoader -from torch.utils.data import Sampler +from torch.utils.data import DataLoader, Sampler -from .pytorch_utils import count_parameters -from ...model.base import Model -from ...data.dataset.handler import DataHandlerLP -from ...contrib.model.pytorch_lstm import LSTMModel from ...contrib.model.pytorch_gru import GRUModel +from ...contrib.model.pytorch_lstm import LSTMModel +from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class DailyBatchSampler(Sampler): diff --git a/qlib/contrib/model/pytorch_general_nn.py b/qlib/contrib/model/pytorch_general_nn.py index 503c5a2a50c..57774255fe9 100644 --- a/qlib/contrib/model/pytorch_general_nn.py +++ b/qlib/contrib/model/pytorch_general_nn.py @@ -1,33 +1,29 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function - -from torch.utils.data import DataLoader +from __future__ import division, print_function +import copy +from typing import Union import numpy as np import pandas as pd -from typing import Union -import copy - import torch import torch.optim as optim from torch.optim.lr_scheduler import ReduceLROnPlateau +from torch.utils.data import DataLoader from qlib.data.dataset.weight import Reweighter -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...model.utils import ConcatDataset from ...utils import ( - init_instance_by_config, get_or_create_path, + init_instance_by_config, ) -from ...log import get_module_logger - -from ...model.utils import ConcatDataset +from .pytorch_utils import count_parameters class GeneralPTNN(Model): diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py index 06aa6810b80..e0150e6454d 100755 --- a/qlib/contrib/model/pytorch_gru.py +++ b/qlib/contrib/model/pytorch_gru.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + import copy from typing import Text, Union diff --git a/qlib/contrib/model/pytorch_gru_ts.py b/qlib/contrib/model/pytorch_gru_ts.py index 65da5ac4b40..77c92bfdbde 100755 --- a/qlib/contrib/model/pytorch_gru_ts.py +++ b/qlib/contrib/model/pytorch_gru_ts.py @@ -2,25 +2,24 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset.handler import DataHandlerLP -from ...model.utils import ConcatDataset from ...data.dataset.weight import Reweighter +from ...log import get_module_logger +from ...model.base import Model +from ...model.utils import ConcatDataset +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class GRU(Model): diff --git a/qlib/contrib/model/pytorch_hist.py b/qlib/contrib/model/pytorch_hist.py index 779cde9c859..7ecbaefb51a 100644 --- a/qlib/contrib/model/pytorch_hist.py +++ b/qlib/contrib/model/pytorch_hist.py @@ -2,26 +2,27 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function +import copy import os +import urllib.request +from typing import Text, Union + import numpy as np import pandas as pd -from typing import Text, Union -import urllib.request -import copy -from ...utils import get_or_create_path -from ...log import get_module_logger import torch import torch.nn as nn import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model + +from ...contrib.model.pytorch_gru import GRUModel +from ...contrib.model.pytorch_lstm import LSTMModel from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...contrib.model.pytorch_lstm import LSTMModel -from ...contrib.model.pytorch_gru import GRUModel +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class HIST(Model): diff --git a/qlib/contrib/model/pytorch_igmtf.py b/qlib/contrib/model/pytorch_igmtf.py index 0bddc5a0f5f..0bb7fafe016 100644 --- a/qlib/contrib/model/pytorch_igmtf.py +++ b/qlib/contrib/model/pytorch_igmtf.py @@ -2,26 +2,25 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model +from ...contrib.model.pytorch_gru import GRUModel +from ...contrib.model.pytorch_lstm import LSTMModel from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...contrib.model.pytorch_lstm import LSTMModel -from ...contrib.model.pytorch_gru import GRUModel +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class IGMTF(Model): diff --git a/qlib/contrib/model/pytorch_krnn.py b/qlib/contrib/model/pytorch_krnn.py index d97920b4dc5..661f55de561 100644 --- a/qlib/contrib/model/pytorch_krnn.py +++ b/qlib/contrib/model/pytorch_krnn.py @@ -2,23 +2,22 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path ######################################################################## ######################################################################## diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py index 42851dd6a28..786863b7fa3 100644 --- a/qlib/contrib/model/pytorch_localformer.py +++ b/qlib/contrib/model/pytorch_localformer.py @@ -2,25 +2,24 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy import math -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim +from torch.nn.modules.container import ModuleList -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from torch.nn.modules.container import ModuleList +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path # qrun examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml ” diff --git a/qlib/contrib/model/pytorch_localformer_ts.py b/qlib/contrib/model/pytorch_localformer_ts.py index ae60a399682..63da23f0ec9 100644 --- a/qlib/contrib/model/pytorch_localformer_ts.py +++ b/qlib/contrib/model/pytorch_localformer_ts.py @@ -2,25 +2,24 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy import math -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim +from torch.nn.modules.container import ModuleList from torch.utils.data import DataLoader -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from torch.nn.modules.container import ModuleList +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path class LocalformerModel(Model): diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py index 3ba09097acd..a672c1d2b9c 100755 --- a/qlib/contrib/model/pytorch_lstm.py +++ b/qlib/contrib/model/pytorch_lstm.py @@ -2,23 +2,22 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path class LSTM(Model): diff --git a/qlib/contrib/model/pytorch_lstm_ts.py b/qlib/contrib/model/pytorch_lstm_ts.py index a0fc34d5832..e097472a2f0 100755 --- a/qlib/contrib/model/pytorch_lstm_ts.py +++ b/qlib/contrib/model/pytorch_lstm_ts.py @@ -2,24 +2,23 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader -from ...model.base import Model from ...data.dataset.handler import DataHandlerLP -from ...model.utils import ConcatDataset from ...data.dataset.weight import Reweighter +from ...log import get_module_logger +from ...model.base import Model +from ...model.utils import ConcatDataset +from ...utils import get_or_create_path class LSTM(Model): diff --git a/qlib/contrib/model/pytorch_nn.py b/qlib/contrib/model/pytorch_nn.py index 9f427bd94d7..5cea1b87bc5 100644 --- a/qlib/contrib/model/pytorch_nn.py +++ b/qlib/contrib/model/pytorch_nn.py @@ -2,38 +2,38 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function -from collections import defaultdict +from __future__ import division, print_function -import os import gc -import numpy as np -import pandas as pd -from packaging import version +import os +from collections import defaultdict from typing import Callable, Optional, Text, Union -from sklearn.metrics import roc_auc_score, mean_squared_error +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim +from packaging import version +from sklearn.metrics import mean_squared_error, roc_auc_score +from torch.nn import DataParallel + +from qlib.contrib.meta.data_selection.utils import ICLoss -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP from ...data.dataset.weight import Reweighter +from ...log import get_module_logger +from ...model.base import Model from ...utils import ( auto_filter_kwargs, + get_or_create_path, init_instance_by_config, - unpack_archive_with_buffer, save_multiple_parts_file, - get_or_create_path, + unpack_archive_with_buffer, ) -from ...log import get_module_logger from ...workflow import R -from qlib.contrib.meta.data_selection.utils import ICLoss -from torch.nn import DataParallel +from .pytorch_utils import count_parameters class DNNModelPytorch(Model): diff --git a/qlib/contrib/model/pytorch_sandwich.py b/qlib/contrib/model/pytorch_sandwich.py index 344368143ff..6ebcdea41a0 100644 --- a/qlib/contrib/model/pytorch_sandwich.py +++ b/qlib/contrib/model/pytorch_sandwich.py @@ -2,23 +2,22 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path from .pytorch_krnn import CNNKRNNEncoder diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py index c971f1a58c5..cbc8519029f 100644 --- a/qlib/contrib/model/pytorch_sfm.py +++ b/qlib/contrib/model/pytorch_sfm.py @@ -1,25 +1,24 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.nn.init as init import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class SFM_Model(nn.Module): diff --git a/qlib/contrib/model/pytorch_tabnet.py b/qlib/contrib/model/pytorch_tabnet.py index 3c698edade3..a8398b084d6 100644 --- a/qlib/contrib/model/pytorch_tabnet.py +++ b/qlib/contrib/model/pytorch_tabnet.py @@ -1,25 +1,24 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn -import torch.optim as optim import torch.nn.functional as F +import torch.optim as optim from torch.autograd import Function -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters class TabnetModel(Model): diff --git a/qlib/contrib/model/pytorch_tcn.py b/qlib/contrib/model/pytorch_tcn.py index f6e7e953a00..8105e81a801 100755 --- a/qlib/contrib/model/pytorch_tcn.py +++ b/qlib/contrib/model/pytorch_tcn.py @@ -2,24 +2,23 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters from .tcn import TemporalConvNet diff --git a/qlib/contrib/model/pytorch_tcn_ts.py b/qlib/contrib/model/pytorch_tcn_ts.py index a6cc38885c3..754d1f63aba 100755 --- a/qlib/contrib/model/pytorch_tcn_ts.py +++ b/qlib/contrib/model/pytorch_tcn_ts.py @@ -2,23 +2,22 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader -from .pytorch_utils import count_parameters -from ...model.base import Model from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path +from .pytorch_utils import count_parameters from .tcn import TemporalConvNet diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py index d8736627c2b..e0c448bad31 100644 --- a/qlib/contrib/model/pytorch_tcts.py +++ b/qlib/contrib/model/pytorch_tcts.py @@ -2,23 +2,22 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy import random -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path class TCTS(Model): diff --git a/qlib/contrib/model/pytorch_tra.py b/qlib/contrib/model/pytorch_tra.py index bc9a6aa9779..6058b3e3be8 100644 --- a/qlib/contrib/model/pytorch_tra.py +++ b/qlib/contrib/model/pytorch_tra.py @@ -1,19 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import io -import os import copy -import math +import io import json +import math +import os + +import matplotlib.pyplot as plt import numpy as np import pandas as pd -import matplotlib.pyplot as plt - import torch import torch.nn as nn -import torch.optim as optim import torch.nn.functional as F +import torch.optim as optim try: from torch.utils.tensorboard import SummaryWriter @@ -23,9 +23,9 @@ from tqdm import tqdm from qlib.constant import EPS +from qlib.contrib.data.dataset import MTSDatasetH from qlib.log import get_module_logger from qlib.model.base import Model -from qlib.contrib.data.dataset import MTSDatasetH device = "cuda" if torch.cuda.is_available() else "cpu" diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py index d05b9f4cad1..9c9796377fa 100644 --- a/qlib/contrib/model/pytorch_transformer.py +++ b/qlib/contrib/model/pytorch_transformer.py @@ -2,24 +2,23 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd -from typing import Text, Union import copy import math -from ...utils import get_or_create_path -from ...log import get_module_logger +from typing import Text, Union +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path # qrun examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml ” diff --git a/qlib/contrib/model/pytorch_transformer_ts.py b/qlib/contrib/model/pytorch_transformer_ts.py index 70590e03e5f..d17009e8710 100644 --- a/qlib/contrib/model/pytorch_transformer_ts.py +++ b/qlib/contrib/model/pytorch_transformer_ts.py @@ -2,24 +2,23 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import numpy as np -import pandas as pd import copy import math -from ...utils import get_or_create_path -from ...log import get_module_logger +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader -from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP +from ...log import get_module_logger +from ...model.base import Model +from ...utils import get_or_create_path class TransformerModel(Model): diff --git a/qlib/contrib/model/xgboost.py b/qlib/contrib/model/xgboost.py index 634259aab14..4a945dda264 100755 --- a/qlib/contrib/model/xgboost.py +++ b/qlib/contrib/model/xgboost.py @@ -1,15 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import Text, Union + import numpy as np import pandas as pd import xgboost as xgb -from typing import Text, Union -from ...model.base import Model + from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP -from ...model.interpret.base import FeatureInt from ...data.dataset.weight import Reweighter +from ...model.base import Model +from ...model.interpret.base import FeatureInt class XGBModel(Model, FeatureInt): diff --git a/qlib/contrib/online/manager.py b/qlib/contrib/online/manager.py index 7475bb6fc52..61c7ea4d8e0 100644 --- a/qlib/contrib/online/manager.py +++ b/qlib/contrib/online/manager.py @@ -5,13 +5,15 @@ # flake8: noqa import pathlib -import pandas as pd import shutil + +import pandas as pd from ruamel.yaml import YAML + from ...backtest.account import Account +from ...utils import init_instance_by_config from .user import User from .utils import load_instance, save_instance -from ...utils import init_instance_by_config class UserManager: diff --git a/qlib/contrib/online/online_model.py b/qlib/contrib/online/online_model.py index 2dc9533df88..cd71c2ec9dd 100644 --- a/qlib/contrib/online/online_model.py +++ b/qlib/contrib/online/online_model.py @@ -5,7 +5,9 @@ # flake8: noqa import random + import pandas as pd + from ...data import D from ..model.base import Model diff --git a/qlib/contrib/online/operator.py b/qlib/contrib/online/operator.py index d5c9edd621b..5afd164ca0b 100644 --- a/qlib/contrib/online/operator.py +++ b/qlib/contrib/online/operator.py @@ -4,24 +4,22 @@ # pylint: skip-file # flake8: noqa +import logging +import pathlib + import fire import pandas as pd -import pathlib + import qlib -import logging from ...data import D from ...log import get_module_logger from ...utils import get_pre_trading_date, is_tradable_date -from ..evaluate import risk_analysis from ..backtest.backtest import update_account - +from ..evaluate import risk_analysis +from .executor import SimulatorExecutor, load_order_list, load_score_series, save_order_list, save_score_series from .manager import UserManager -from .utils import prepare -from .utils import create_user_folder -from .executor import load_order_list, save_order_list -from .executor import SimulatorExecutor -from .executor import save_score_series, load_score_series +from .utils import create_user_folder, prepare class Operator: diff --git a/qlib/contrib/online/user.py b/qlib/contrib/online/user.py index fa74831eed6..b0be8003ccf 100644 --- a/qlib/contrib/online/user.py +++ b/qlib/contrib/online/user.py @@ -6,9 +6,9 @@ import logging +from ...data import D from ...log import get_module_logger from ..evaluate import risk_analysis -from ...data import D class User: diff --git a/qlib/contrib/online/utils.py b/qlib/contrib/online/utils.py index dddf7f0d2a0..099e9e3daa8 100644 --- a/qlib/contrib/online/utils.py +++ b/qlib/contrib/online/utils.py @@ -6,13 +6,15 @@ import pathlib import pickle + import pandas as pd from ruamel.yaml import YAML -from ...data import D + +from ...backtest.exchange import Exchange from ...config import C +from ...data import D from ...log import get_module_logger from ...utils import get_next_trading_date -from ...backtest.exchange import Exchange log = get_module_logger("utils") diff --git a/qlib/contrib/ops/high_freq.py b/qlib/contrib/ops/high_freq.py index 51852b66cca..a9156076003 100644 --- a/qlib/contrib/ops/high_freq.py +++ b/qlib/contrib/ops/high_freq.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from datetime import datetime + import numpy as np import pandas as pd -from datetime import datetime from qlib.data.cache import H from qlib.data.data import Cal diff --git a/qlib/contrib/report/analysis_model/__init__.py b/qlib/contrib/report/analysis_model/__init__.py index a53a74b4ef1..c97821142ac 100644 --- a/qlib/contrib/report/analysis_model/__init__.py +++ b/qlib/contrib/report/analysis_model/__init__.py @@ -3,5 +3,4 @@ from .analysis_model_performance import model_performance_graph - __all__ = ["model_performance_graph"] diff --git a/qlib/contrib/report/analysis_model/analysis_model_performance.py b/qlib/contrib/report/analysis_model/analysis_model_performance.py index cac1f1b8eea..8cb2d994b32 100644 --- a/qlib/contrib/report/analysis_model/analysis_model_performance.py +++ b/qlib/contrib/report/analysis_model/analysis_model_performance.py @@ -1,20 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. from functools import partial +from typing import Sequence +import matplotlib.pyplot as plt import pandas as pd - import plotly.graph_objs as go - import statsmodels.api as sm -import matplotlib.pyplot as plt - from scipy import stats -from typing import Sequence from qlib.typehint import Literal -from ..graph import ScatterGraph, SubplotsGraph, BarGraph, HeatmapGraph +from ..graph import BarGraph, HeatmapGraph, ScatterGraph, SubplotsGraph from ..utils import guess_plotly_rangebreaks diff --git a/qlib/contrib/report/analysis_position/__init__.py b/qlib/contrib/report/analysis_position/__init__.py index cfe51a22495..1c15ba02191 100644 --- a/qlib/contrib/report/analysis_position/__init__.py +++ b/qlib/contrib/report/analysis_position/__init__.py @@ -2,10 +2,9 @@ # Licensed under the MIT License. from .cumulative_return import cumulative_return_graph -from .score_ic import score_ic_graph -from .report import report_graph from .rank_label import rank_label_graph +from .report import report_graph from .risk_analysis import risk_analysis_graph - +from .score_ic import score_ic_graph __all__ = ["cumulative_return_graph", "score_ic_graph", "report_graph", "rank_label_graph", "risk_analysis_graph"] diff --git a/qlib/contrib/report/analysis_position/cumulative_return.py b/qlib/contrib/report/analysis_position/cumulative_return.py index 4f325aa1a36..71711f7c368 100644 --- a/qlib/contrib/report/analysis_position/cumulative_return.py +++ b/qlib/contrib/report/analysis_position/cumulative_return.py @@ -7,9 +7,8 @@ import pandas as pd import plotly.graph_objs as go -from ..graph import BaseGraph, SubplotsGraph - from ..analysis_position.parse_position import get_position_data +from ..graph import BaseGraph, SubplotsGraph def _get_cum_return_data_with_position( diff --git a/qlib/contrib/report/analysis_position/parse_position.py b/qlib/contrib/report/analysis_position/parse_position.py index 0f6510e8182..a139bde541d 100644 --- a/qlib/contrib/report/analysis_position/parse_position.py +++ b/qlib/contrib/report/analysis_position/parse_position.py @@ -3,7 +3,6 @@ import pandas as pd - from ....backtest.profit_attribution import get_stock_weight_df diff --git a/qlib/contrib/report/analysis_position/rank_label.py b/qlib/contrib/report/analysis_position/rank_label.py index 3e94d174c43..a410aae283c 100644 --- a/qlib/contrib/report/analysis_position/rank_label.py +++ b/qlib/contrib/report/analysis_position/rank_label.py @@ -7,8 +7,8 @@ import pandas as pd import plotly.graph_objs as go -from ..graph import ScatterGraph from ..analysis_position.parse_position import get_position_data +from ..graph import ScatterGraph def _get_figure_with_position( diff --git a/qlib/contrib/report/analysis_position/report.py b/qlib/contrib/report/analysis_position/report.py index daefb529550..c09a232f209 100644 --- a/qlib/contrib/report/analysis_position/report.py +++ b/qlib/contrib/report/analysis_position/report.py @@ -3,7 +3,7 @@ import pandas as pd -from ..graph import SubplotsGraph, BaseGraph +from ..graph import BaseGraph, SubplotsGraph def _calculate_maximum(df: pd.DataFrame, is_ex: bool = False): diff --git a/qlib/contrib/report/analysis_position/risk_analysis.py b/qlib/contrib/report/analysis_position/risk_analysis.py index c7cb99c7a35..e75a650cd5c 100644 --- a/qlib/contrib/report/analysis_position/risk_analysis.py +++ b/qlib/contrib/report/analysis_position/risk_analysis.py @@ -4,12 +4,10 @@ from typing import Iterable import pandas as pd - import plotly.graph_objs as py from ...evaluate import risk_analysis - -from ..graph import SubplotsGraph, ScatterGraph +from ..graph import ScatterGraph, SubplotsGraph def _get_risk_analysis_data_with_report( diff --git a/qlib/contrib/report/data/ana.py b/qlib/contrib/report/data/ana.py index e93b07612ad..46dac57157e 100644 --- a/qlib/contrib/report/data/ana.py +++ b/qlib/contrib/report/data/ana.py @@ -12,14 +12,15 @@ fa.plot_all(wspace=0.3, sub_figsize=(12, 3), col_n=5) """ -import pandas as pd import numpy as np +import pandas as pd +import seaborn as sns +from loguru import logger + +from qlib.contrib.eva.alpha import pred_autocorr_all from qlib.contrib.report.data.base import FeaAnalyser from qlib.contrib.report.utils import sub_fig_generator from qlib.utils.paral import datetime_groupby_apply -from qlib.contrib.eva.alpha import pred_autocorr_all -from loguru import logger -import seaborn as sns DT_COL_NAME = "datetime" diff --git a/qlib/contrib/report/data/base.py b/qlib/contrib/report/data/base.py index 0861233b6dc..53b46deca53 100644 --- a/qlib/contrib/report/data/base.py +++ b/qlib/contrib/report/data/base.py @@ -8,8 +8,9 @@ """ import pandas as pd -from qlib.log import TimeInspector + from qlib.contrib.report.utils import sub_fig_generator +from qlib.log import TimeInspector class FeaAnalyser: diff --git a/qlib/contrib/report/graph.py b/qlib/contrib/report/graph.py index 387a057a29c..d37f32058a5 100644 --- a/qlib/contrib/report/graph.py +++ b/qlib/contrib/report/graph.py @@ -1,17 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import math import importlib +import math from typing import Iterable import pandas as pd - -import plotly.offline as py import plotly.graph_objs as go - -from plotly.subplots import make_subplots +import plotly.offline as py from plotly.figure_factory import create_distplot +from plotly.subplots import make_subplots class BaseGraph: diff --git a/qlib/contrib/rolling/__main__.py b/qlib/contrib/rolling/__main__.py index 461c0e777eb..07612b9e6c7 100644 --- a/qlib/contrib/rolling/__main__.py +++ b/qlib/contrib/rolling/__main__.py @@ -1,4 +1,5 @@ import fire + from qlib import auto_init from qlib.contrib.rolling.base import Rolling from qlib.utils.mod import find_all_classes diff --git a/qlib/contrib/rolling/base.py b/qlib/contrib/rolling/base.py index 5f17c05623d..9e8cb039ea9 100644 --- a/qlib/contrib/rolling/base.py +++ b/qlib/contrib/rolling/base.py @@ -2,11 +2,11 @@ # Licensed under the MIT License. from copy import deepcopy from pathlib import Path -from ruamel.yaml import YAML from typing import List, Optional, Union import fire import pandas as pd +from ruamel.yaml import YAML from qlib import auto_init from qlib.log import get_module_logger diff --git a/qlib/contrib/rolling/ddgda.py b/qlib/contrib/rolling/ddgda.py index 0fe01d04550..062f3a7893d 100644 --- a/qlib/contrib/rolling/ddgda.py +++ b/qlib/contrib/rolling/ddgda.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from pathlib import Path import pickle +from pathlib import Path from typing import Optional, Union import pandas as pd diff --git a/qlib/contrib/strategy/__init__.py b/qlib/contrib/strategy/__init__.py index b7051f575da..218f8cdfc87 100644 --- a/qlib/contrib/strategy/__init__.py +++ b/qlib/contrib/strategy/__init__.py @@ -2,20 +2,17 @@ # Licensed under the MIT License. -from .signal_strategy import ( - TopkDropoutStrategy, - WeightStrategyBase, - EnhancedIndexingStrategy, -) - +from .cost_control import SoftTopkStrategy from .rule_strategy import ( - TWAPStrategy, SBBStrategyBase, SBBStrategyEMA, + TWAPStrategy, +) +from .signal_strategy import ( + EnhancedIndexingStrategy, + TopkDropoutStrategy, + WeightStrategyBase, ) - -from .cost_control import SoftTopkStrategy - __all__ = [ "TopkDropoutStrategy", diff --git a/qlib/contrib/strategy/cost_control.py b/qlib/contrib/strategy/cost_control.py index ff51f484f54..22deeeab297 100644 --- a/qlib/contrib/strategy/cost_control.py +++ b/qlib/contrib/strategy/cost_control.py @@ -5,9 +5,10 @@ """ +import copy + from .order_generator import OrderGenWInteract from .signal_strategy import WeightStrategyBase -import copy class SoftTopkStrategy(WeightStrategyBase): diff --git a/qlib/contrib/strategy/optimizer/__init__.py b/qlib/contrib/strategy/optimizer/__init__.py index 18b2aacdf66..99d7a13638a 100644 --- a/qlib/contrib/strategy/optimizer/__init__.py +++ b/qlib/contrib/strategy/optimizer/__init__.py @@ -2,8 +2,7 @@ # Licensed under the MIT License. from .base import BaseOptimizer -from .optimizer import PortfolioOptimizer from .enhanced_indexing import EnhancedIndexingOptimizer - +from .optimizer import PortfolioOptimizer __all__ = ["BaseOptimizer", "PortfolioOptimizer", "EnhancedIndexingOptimizer"] diff --git a/qlib/contrib/strategy/optimizer/enhanced_indexing.py b/qlib/contrib/strategy/optimizer/enhanced_indexing.py index 7e42856a2e8..9b2901bd9bf 100644 --- a/qlib/contrib/strategy/optimizer/enhanced_indexing.py +++ b/qlib/contrib/strategy/optimizer/enhanced_indexing.py @@ -1,14 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import numpy as np -import cvxpy as cp +from typing import Any, Dict, List, Optional, Union -from typing import Union, Optional, Dict, Any, List +import cvxpy as cp +import numpy as np from qlib.log import get_module_logger -from .base import BaseOptimizer +from .base import BaseOptimizer logger = get_module_logger("EnhancedIndexingOptimizer") diff --git a/qlib/contrib/strategy/optimizer/optimizer.py b/qlib/contrib/strategy/optimizer/optimizer.py index a5fb7631275..7b2c724c06b 100644 --- a/qlib/contrib/strategy/optimizer/optimizer.py +++ b/qlib/contrib/strategy/optimizer/optimizer.py @@ -3,10 +3,11 @@ import warnings +from typing import Callable, List, Optional, Union + import numpy as np import pandas as pd import scipy.optimize as so -from typing import Optional, Union, Callable, List from .base import BaseOptimizer diff --git a/qlib/contrib/strategy/order_generator.py b/qlib/contrib/strategy/order_generator.py index fe0d048bfdd..aff2af6b487 100644 --- a/qlib/contrib/strategy/order_generator.py +++ b/qlib/contrib/strategy/order_generator.py @@ -4,11 +4,12 @@ """ This order generator is for strategies based on WeightStrategyBase """ -from ...backtest.position import Position -from ...backtest.exchange import Exchange +import copy import pandas as pd -import copy + +from ...backtest.exchange import Exchange +from ...backtest.position import Position class OrderGenerator: diff --git a/qlib/contrib/strategy/rule_strategy.py b/qlib/contrib/strategy/rule_strategy.py index 2cac662f76c..b678cc4f025 100644 --- a/qlib/contrib/strategy/rule_strategy.py +++ b/qlib/contrib/strategy/rule_strategy.py @@ -1,22 +1,23 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from pathlib import Path import warnings +from pathlib import Path +from typing import IO, List, Tuple, Union + import numpy as np import pandas as pd -from typing import IO, List, Tuple, Union -from qlib.data.dataset.utils import convert_index_format +from qlib.backtest.utils import get_start_end_idx +from qlib.data.dataset.utils import convert_index_format from qlib.utils import lazy_sort_index +from qlib.utils.file import get_io_object -from ...utils.resam import resam_ts_data, ts_data_last -from ...data.data import D -from ...strategy.base import BaseStrategy from ...backtest.decision import BaseTradeDecision, Order, TradeDecisionWO, TradeRange from ...backtest.exchange import Exchange, OrderHelper from ...backtest.utils import CommonInfrastructure, LevelInfrastructure -from qlib.utils.file import get_io_object -from qlib.backtest.utils import get_start_end_idx +from ...data.data import D +from ...strategy.base import BaseStrategy +from ...utils.resam import resam_ts_data, ts_data_last class TWAPStrategy(BaseStrategy): diff --git a/qlib/contrib/strategy/signal_strategy.py b/qlib/contrib/strategy/signal_strategy.py index bad19ddfdc9..8c46d42ff03 100644 --- a/qlib/contrib/strategy/signal_strategy.py +++ b/qlib/contrib/strategy/signal_strategy.py @@ -1,25 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import os import copy +import os import warnings +from abc import ABC +from typing import Dict, List, Text, Tuple, Union + import numpy as np import pandas as pd -from typing import Dict, List, Text, Tuple, Union -from abc import ABC - +from qlib.backtest.decision import Order, OrderDir, TradeDecisionWO +from qlib.backtest.position import Position +from qlib.backtest.signal import Signal, create_signal_from +from qlib.contrib.strategy.optimizer import EnhancedIndexingOptimizer +from qlib.contrib.strategy.order_generator import OrderGenerator, OrderGenWOInteract from qlib.data import D from qlib.data.dataset import Dataset +from qlib.log import get_module_logger from qlib.model.base import BaseModel from qlib.strategy.base import BaseStrategy -from qlib.backtest.position import Position -from qlib.backtest.signal import Signal, create_signal_from -from qlib.backtest.decision import Order, OrderDir, TradeDecisionWO -from qlib.log import get_module_logger from qlib.utils import get_pre_trading_date, load_dataset -from qlib.contrib.strategy.order_generator import OrderGenerator, OrderGenWOInteract -from qlib.contrib.strategy.optimizer import EnhancedIndexingOptimizer class BaseSignalStrategy(BaseStrategy, ABC): diff --git a/qlib/contrib/torch.py b/qlib/contrib/torch.py index fdd18e230be..35043f40028 100644 --- a/qlib/contrib/torch.py +++ b/qlib/contrib/torch.py @@ -5,9 +5,9 @@ They are just some tools for convenience It is should not imported into the core part of qlib """ -import torch import numpy as np import pandas as pd +import torch def data_to_tensor(data, device="cpu", raise_error=False): diff --git a/qlib/contrib/tuner/config.py b/qlib/contrib/tuner/config.py index 4cedd3642bb..6e6975e42f7 100644 --- a/qlib/contrib/tuner/config.py +++ b/qlib/contrib/tuner/config.py @@ -6,6 +6,7 @@ import copy import os + from ruamel.yaml import YAML diff --git a/qlib/contrib/tuner/launcher.py b/qlib/contrib/tuner/launcher.py index 352d2ca48d4..a2fd5e8fda9 100644 --- a/qlib/contrib/tuner/launcher.py +++ b/qlib/contrib/tuner/launcher.py @@ -9,11 +9,11 @@ import argparse import importlib import os + import yaml from .config import TunerConfigManager - args_parser = argparse.ArgumentParser(prog="tuner") args_parser.add_argument( "-c", diff --git a/qlib/contrib/tuner/pipeline.py b/qlib/contrib/tuner/pipeline.py index 34977fa55f1..83f4c4130e0 100644 --- a/qlib/contrib/tuner/pipeline.py +++ b/qlib/contrib/tuner/pipeline.py @@ -4,13 +4,13 @@ # pylint: skip-file # flake8: noqa -import os +import importlib import json import logging -import importlib +import os from abc import abstractmethod -from ...log import get_module_logger, TimeInspector +from ...log import TimeInspector, get_module_logger from ...utils import get_module_by_module_path diff --git a/qlib/contrib/tuner/space.py b/qlib/contrib/tuner/space.py index 4959d8dc939..f829baf7e15 100644 --- a/qlib/contrib/tuner/space.py +++ b/qlib/contrib/tuner/space.py @@ -6,7 +6,6 @@ from hyperopt import hp - TopkAmountStrategySpace = { "topk": hp.choice("topk", [30, 35, 40]), "buffer_margin": hp.choice("buffer_margin", [200, 250, 300]), diff --git a/qlib/contrib/tuner/tuner.py b/qlib/contrib/tuner/tuner.py index 9009f5721c4..974361f9e21 100644 --- a/qlib/contrib/tuner/tuner.py +++ b/qlib/contrib/tuner/tuner.py @@ -4,22 +4,21 @@ # pylint: skip-file # flake8: noqa -import os -import yaml -import json import copy -import logging import importlib +import json +import logging +import os import subprocess -import pandas as pd -import numpy as np - from abc import abstractmethod -from ...log import get_module_logger, TimeInspector +import numpy as np +import pandas as pd +import yaml +from hyperopt import STATUS_FAIL, STATUS_OK, fmin, tpe + +from ...log import TimeInspector, get_module_logger from ...utils.pickle_utils import restricted_pickle_load -from hyperopt import fmin, tpe -from hyperopt import STATUS_OK, STATUS_FAIL class Tuner: diff --git a/qlib/contrib/workflow/__init__.py b/qlib/contrib/workflow/__init__.py index 0faf4e5f9d5..a6d1d5bbf26 100644 --- a/qlib/contrib/workflow/__init__.py +++ b/qlib/contrib/workflow/__init__.py @@ -1,7 +1,5 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from .record_temp import MultiSegRecord -from .record_temp import SignalMseRecord - +from .record_temp import MultiSegRecord, SignalMseRecord __all__ = ["MultiSegRecord", "SignalMseRecord"] diff --git a/qlib/contrib/workflow/record_temp.py b/qlib/contrib/workflow/record_temp.py index 8d10b2ab48b..e55e5ca464e 100644 --- a/qlib/contrib/workflow/record_temp.py +++ b/qlib/contrib/workflow/record_temp.py @@ -2,16 +2,16 @@ # Licensed under the MIT License. import logging -import pandas as pd +from typing import Any, Dict, Text + import numpy as np +import pandas as pd from sklearn.metrics import mean_squared_error -from typing import Dict, Text, Any from ...contrib.eva.alpha import calc_ic -from ...workflow.record_temp import RecordTemp -from ...workflow.record_temp import SignalRecord from ...data import dataset as qlib_dataset from ...log import get_module_logger +from ...workflow.record_temp import RecordTemp, SignalRecord logger = get_module_logger("workflow", logging.INFO) diff --git a/qlib/data/__init__.py b/qlib/data/__init__.py index a6bc6df8894..278b8c028d3 100644 --- a/qlib/data/__init__.py +++ b/qlib/data/__init__.py @@ -2,41 +2,38 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function +from .cache import ( + DatasetCache, + DatasetURICache, + DiskDatasetCache, + DiskExpressionCache, + ExpressionCache, + MemoryCalendarCache, + SimpleDatasetCache, +) from .data import ( - D, + BaseProvider, CalendarProvider, - InstrumentProvider, - FeatureProvider, - ExpressionProvider, + ClientCalendarProvider, + ClientDatasetProvider, + ClientInstrumentProvider, + ClientProvider, + D, DatasetProvider, + ExpressionProvider, + FeatureProvider, + InstrumentProvider, LocalCalendarProvider, - LocalInstrumentProvider, + LocalDatasetProvider, + LocalExpressionProvider, LocalFeatureProvider, + LocalInstrumentProvider, LocalPITProvider, - LocalExpressionProvider, - LocalDatasetProvider, - ClientCalendarProvider, - ClientInstrumentProvider, - ClientDatasetProvider, - BaseProvider, LocalProvider, - ClientProvider, ) -from .cache import ( - ExpressionCache, - DatasetCache, - DiskExpressionCache, - DiskDatasetCache, - SimpleDatasetCache, - DatasetURICache, - MemoryCalendarCache, -) - - __all__ = [ "D", "CalendarProvider", diff --git a/qlib/data/_libs/expanding.pyx b/qlib/data/_libs/expanding.pyx index 6c27c07ebee..3bb8d8210f6 100644 --- a/qlib/data/_libs/expanding.pyx +++ b/qlib/data/_libs/expanding.pyx @@ -2,9 +2,10 @@ # cython: boundscheck=False, wraparound=False, cdivision=True cimport cython cimport numpy as np + import numpy as np -from libc.math cimport sqrt, isnan, NAN +from libc.math cimport NAN, isnan, sqrt from libcpp.vector cimport vector diff --git a/qlib/data/_libs/rolling.pyx b/qlib/data/_libs/rolling.pyx index a18679a9913..7a86346bb31 100644 --- a/qlib/data/_libs/rolling.pyx +++ b/qlib/data/_libs/rolling.pyx @@ -2,9 +2,10 @@ # cython: boundscheck=False, wraparound=False, cdivision=True cimport cython cimport numpy as np + import numpy as np -from libc.math cimport sqrt, isnan, NAN +from libc.math cimport NAN, isnan, sqrt from libcpp.deque cimport deque diff --git a/qlib/data/base.py b/qlib/data/base.py index 496ae38ee23..189b76ca594 100644 --- a/qlib/data/base.py +++ b/qlib/data/base.py @@ -2,11 +2,12 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function import abc + import pandas as pd + from ..log import get_module_logger diff --git a/qlib/data/cache.py b/qlib/data/cache.py index fbf6e839db1..dfb9684cf00 100644 --- a/qlib/data/cache.py +++ b/qlib/data/cache.py @@ -2,37 +2,36 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function +import abc +import contextlib import os -import sys +import pickle import stat +import sys import time -import pickle import traceback -import redis_lock -import contextlib -import abc +from collections import OrderedDict from pathlib import Path +from typing import Iterable, Union + import numpy as np import pandas as pd -from typing import Union, Iterable -from collections import OrderedDict +import redis_lock from ..config import C +from ..log import get_module_logger from ..utils import ( - hash_args, get_redis_connection, - read_bin, - parse_field, - remove_fields_space, + hash_args, normalize_cache_fields, normalize_cache_instruments, + parse_field, + read_bin, + remove_fields_space, ) from ..utils.pickle_utils import restricted_pickle_load - -from ..log import get_module_logger from .base import Feature from .ops import Operators # pylint: disable=W0611 # noqa: F401 diff --git a/qlib/data/data.py b/qlib/data/data.py index aba75c0b1ab..d9e84c1cb5a 100644 --- a/qlib/data/data.py +++ b/qlib/data/data.py @@ -2,41 +2,39 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import re import abc +import bisect import copy import queue -import bisect +import re +from typing import List, Optional, Union + import numpy as np import pandas as pd -from typing import List, Union, Optional # For supporting multiprocessing in outer code, joblib is used from joblib import delayed -from .cache import H from ..config import C -from .inst_processor import InstProcessor - from ..log import get_module_logger -from .cache import DiskDatasetCache from ..utils import ( Wrapper, - init_instance_by_config, - register_wrapper, + code_to_fname, get_module_by_module_path, - parse_field, + get_period_list, hash_args, + init_instance_by_config, normalize_cache_fields, - code_to_fname, - time_to_slc_point, + parse_field, read_period_data, - get_period_list, + register_wrapper, + time_to_slc_point, ) from ..utils.paral import ParallelExt +from .cache import DiskDatasetCache, H +from .inst_processor import InstProcessor from .ops import Operators # pylint: disable=W0611 # noqa: F401 diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index a6cace3730f..464f9890caa 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -1,14 +1,15 @@ -from ...utils.serial import Serializable -from typing import Callable, Union, List, Tuple, Dict, Text, Optional -from ...utils import init_instance_by_config, np_ffill, time_to_slc_point -from ...log import get_module_logger -from .handler import DataHandler, DataHandlerLP +import bisect from copy import copy, deepcopy from inspect import getfullargspec -import pandas as pd +from typing import Callable, Dict, List, Optional, Text, Tuple, Union + import numpy as np -import bisect -from ...utils import lazy_sort_index +import pandas as pd + +from ...log import get_module_logger +from ...utils import init_instance_by_config, lazy_sort_index, np_ffill, time_to_slc_point +from ...utils.serial import Serializable +from .handler import DataHandler, DataHandlerLP from .utils import get_level_index diff --git a/qlib/data/dataset/handler.py b/qlib/data/dataset/handler.py index b6ee9579478..33d5fba869c 100644 --- a/qlib/data/dataset/handler.py +++ b/qlib/data/dataset/handler.py @@ -1,24 +1,23 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import warnings + # coding=utf-8 from abc import abstractmethod -import warnings -from typing import Callable, Union, Tuple, List, Iterator, Optional +from typing import Callable, Iterator, List, Optional, Tuple, Union import pandas as pd from qlib.typehint import Literal -from ...log import get_module_logger, TimeInspector -from ...utils import init_instance_by_config -from ...utils.serial import Serializable -from .utils import fetch_df_by_index, fetch_df_by_col -from ...utils import lazy_sort_index -from .loader import DataLoader -from . import processor as processor_module +from ...log import TimeInspector, get_module_logger +from ...utils import init_instance_by_config, lazy_sort_index +from ...utils.serial import Serializable from . import loader as data_loader_module - +from . import processor as processor_module +from .loader import DataLoader +from .utils import fetch_df_by_col, fetch_df_by_index DATA_KEY_TYPE = Literal["raw", "infer", "learn"] diff --git a/qlib/data/dataset/loader.py b/qlib/data/dataset/loader.py index 2f3615a6357..e39e50d70f9 100644 --- a/qlib/data/dataset/loader.py +++ b/qlib/data/dataset/loader.py @@ -2,16 +2,16 @@ # Licensed under the MIT License. import abc -from pathlib import Path import warnings -import pandas as pd +from pathlib import Path +from typing import Dict, List, Tuple, Union -from typing import Tuple, Union, List, Dict +import pandas as pd from qlib.data import D -from qlib.utils import load_dataset, init_instance_by_config, time_to_slc_point -from qlib.utils.pickle_utils import restricted_pickle_load from qlib.log import get_module_logger +from qlib.utils import init_instance_by_config, load_dataset, time_to_slc_point +from qlib.utils.pickle_utils import restricted_pickle_load from qlib.utils.serial import Serializable diff --git a/qlib/data/dataset/processor.py b/qlib/data/dataset/processor.py index d05dbe381c5..724ef46f63a 100644 --- a/qlib/data/dataset/processor.py +++ b/qlib/data/dataset/processor.py @@ -2,17 +2,19 @@ # Licensed under the MIT License. import abc -from typing import Union, Text, Optional +from typing import Optional, Text, Union + import numpy as np import pandas as pd +from qlib.data import D +from qlib.data.inst_processor import InstProcessor from qlib.utils.data import robust_zscore, zscore + from ...constant import EPS -from .utils import fetch_df_by_index -from ...utils.serial import Serializable from ...utils.paral import datetime_groupby_apply -from qlib.data.inst_processor import InstProcessor -from qlib.data import D +from ...utils.serial import Serializable +from .utils import fetch_df_by_index def get_group_columns(df: pd.DataFrame, group: Union[Text, None]): diff --git a/qlib/data/dataset/storage.py b/qlib/data/dataset/storage.py index ca3325a28cf..c11176bac0d 100644 --- a/qlib/data/dataset/storage.py +++ b/qlib/data/dataset/storage.py @@ -1,12 +1,13 @@ from abc import abstractmethod -import pandas as pd +from typing import List, Union + import numpy as np +import pandas as pd -from .handler import DataHandler -from typing import Union, List from qlib.log import get_module_logger -from .utils import get_level_index, fetch_df_by_index, fetch_df_by_col +from .handler import DataHandler +from .utils import fetch_df_by_col, fetch_df_by_index, get_level_index class BaseHandlerStorage: diff --git a/qlib/data/dataset/utils.py b/qlib/data/dataset/utils.py index 688cde99af7..b76c197f9f8 100644 --- a/qlib/data/dataset/utils.py +++ b/qlib/data/dataset/utils.py @@ -1,8 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. from __future__ import annotations + +from typing import TYPE_CHECKING, List, Union + import pandas as pd -from typing import Union, List, TYPE_CHECKING + from qlib.utils import init_instance_by_config if TYPE_CHECKING: diff --git a/qlib/data/filter.py b/qlib/data/filter.py index 246d6baf765..a55424d1020 100644 --- a/qlib/data/filter.py +++ b/qlib/data/filter.py @@ -2,12 +2,13 @@ # Licensed under the MIT License. from __future__ import print_function -from abc import abstractmethod +import abc import re -import pandas as pd +from abc import abstractmethod + import numpy as np -import abc +import pandas as pd from .data import Cal, DatasetD diff --git a/qlib/data/inst_processor.py b/qlib/data/inst_processor.py index e00132777d5..275d5087a00 100644 --- a/qlib/data/inst_processor.py +++ b/qlib/data/inst_processor.py @@ -1,5 +1,6 @@ import abc import json + import pandas as pd diff --git a/qlib/data/ops.py b/qlib/data/ops.py index d9a2ffbb3e3..ac3ef6aa833 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -2,21 +2,21 @@ # Licensed under the MIT License. -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function + +from typing import List, Type, Union import numpy as np import pandas as pd - -from typing import Union, List, Type from scipy.stats import percentileofscore -from .base import Expression, ExpressionOps, Feature, PFeature + from ..log import get_module_logger from ..utils import get_callable_kwargs +from .base import Expression, ExpressionOps, Feature, PFeature try: - from ._libs.rolling import rolling_slope, rolling_rsquare, rolling_resi - from ._libs.expanding import expanding_slope, expanding_rsquare, expanding_resi + from ._libs.expanding import expanding_resi, expanding_rsquare, expanding_slope + from ._libs.rolling import rolling_resi, rolling_rsquare, rolling_slope except ImportError: print( "#### Do not import qlib package in the repository directory in case of importing qlib from . without compiling #####" diff --git a/qlib/data/pit.py b/qlib/data/pit.py index 33d5e0c5cc8..b80dbb3dadd 100644 --- a/qlib/data/pit.py +++ b/qlib/data/pit.py @@ -15,8 +15,10 @@ """ import numpy as np import pandas as pd + from qlib.data.ops import ElemOperator from qlib.log import get_module_logger + from .data import Cal diff --git a/qlib/data/storage/__init__.py b/qlib/data/storage/__init__.py index a77fb0e3e42..7ce7b1c04a6 100644 --- a/qlib/data/storage/__init__.py +++ b/qlib/data/storage/__init__.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from .storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstVT, InstKT - +from .storage import CalendarStorage, CalVT, FeatureStorage, InstKT, InstrumentStorage, InstVT __all__ = ["CalendarStorage", "InstrumentStorage", "FeatureStorage", "CalVT", "InstVT", "InstKT"] diff --git a/qlib/data/storage/file_storage.py b/qlib/data/storage/file_storage.py index 8a100a2d19e..71cd9b342b0 100644 --- a/qlib/data/storage/file_storage.py +++ b/qlib/data/storage/file_storage.py @@ -3,17 +3,17 @@ import struct from pathlib import Path -from typing import Iterable, Union, Dict, Mapping, Tuple, List +from typing import Dict, Iterable, List, Mapping, Tuple, Union import numpy as np import pandas as pd -from qlib.utils.time import Freq -from qlib.utils.resam import resam_calendar from qlib.config import C from qlib.data.cache import H +from qlib.data.storage import CalendarStorage, CalVT, FeatureStorage, InstKT, InstrumentStorage, InstVT from qlib.log import get_module_logger -from qlib.data.storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstKT, InstVT +from qlib.utils.resam import resam_calendar +from qlib.utils.time import Freq logger = get_module_logger("file_storage") diff --git a/qlib/data/storage/storage.py b/qlib/data/storage/storage.py index 2eb7da1de66..a27ffbb0dad 100644 --- a/qlib/data/storage/storage.py +++ b/qlib/data/storage/storage.py @@ -2,10 +2,11 @@ # Licensed under the MIT License. import re -from typing import Iterable, overload, Tuple, List, Text, Union, Dict +from typing import Dict, Iterable, List, Text, Tuple, Union, overload import numpy as np import pandas as pd + from qlib.log import get_module_logger # calendar value type diff --git a/qlib/log.py b/qlib/log.py index f7683d51163..a71ecddbffb 100644 --- a/qlib/log.py +++ b/qlib/log.py @@ -3,11 +3,11 @@ import logging -from typing import Optional, Text, Dict, Any import re +from contextlib import contextmanager from logging import config as logging_config from time import time -from contextlib import contextmanager +from typing import Any, Dict, Optional, Text from .config import C diff --git a/qlib/model/__init__.py b/qlib/model/__init__.py index 35fb5c43aa8..490f28860f2 100644 --- a/qlib/model/__init__.py +++ b/qlib/model/__init__.py @@ -5,5 +5,4 @@ from .base import Model - __all__ = ["Model", "warnings"] diff --git a/qlib/model/base.py b/qlib/model/base.py index 009a3bd1441..0b85fd5c742 100644 --- a/qlib/model/base.py +++ b/qlib/model/base.py @@ -2,9 +2,10 @@ # Licensed under the MIT License. import abc from typing import Text, Union -from ..utils.serial import Serializable + from ..data.dataset import Dataset from ..data.dataset.weight import Reweighter +from ..utils.serial import Serializable class BaseModel(Serializable, metaclass=abc.ABCMeta): diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index 1670a6538ef..0e02bd74b7e 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -6,9 +6,11 @@ """ from typing import Union + import pandas as pd -from qlib.utils import FLATTEN_TUPLE, flatten_dict + from qlib.log import get_module_logger +from qlib.utils import FLATTEN_TUPLE, flatten_dict class Ensemble: diff --git a/qlib/model/ens/group.py b/qlib/model/ens/group.py index ba6f9f8071b..fce6844ee62 100644 --- a/qlib/model/ens/group.py +++ b/qlib/model/ens/group.py @@ -12,10 +12,12 @@ """ -from qlib.model.ens.ensemble import Ensemble, RollingEnsemble from typing import Callable + from joblib import Parallel, delayed +from qlib.model.ens.ensemble import Ensemble, RollingEnsemble + class Group: """Group the objects based on dict""" diff --git a/qlib/model/interpret/base.py b/qlib/model/interpret/base.py index a490d774424..d7a56832c6e 100644 --- a/qlib/model/interpret/base.py +++ b/qlib/model/interpret/base.py @@ -5,9 +5,10 @@ Interfaces to interpret models """ -import pandas as pd from abc import abstractmethod +import pandas as pd + class FeatureInt: """Feature (Int)erpreter""" diff --git a/qlib/model/meta/__init__.py b/qlib/model/meta/__init__.py index 1290014a701..08b26a02773 100644 --- a/qlib/model/meta/__init__.py +++ b/qlib/model/meta/__init__.py @@ -1,8 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from .task import MetaTask from .dataset import MetaTaskDataset - +from .task import MetaTask __all__ = ["MetaTask", "MetaTaskDataset"] diff --git a/qlib/model/meta/dataset.py b/qlib/model/meta/dataset.py index 34a9b949b31..0e733ef5304 100644 --- a/qlib/model/meta/dataset.py +++ b/qlib/model/meta/dataset.py @@ -2,8 +2,10 @@ # Licensed under the MIT License. import abc +from typing import Dict, List, Text, Tuple, Union + from qlib.model.meta.task import MetaTask -from typing import Dict, Union, List, Tuple, Text + from ...utils.serial import Serializable diff --git a/qlib/model/meta/task.py b/qlib/model/meta/task.py index a051acf1466..39b0cd83577 100644 --- a/qlib/model/meta/task.py +++ b/qlib/model/meta/task.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. from qlib.data.dataset import Dataset + from ...utils import init_instance_by_config diff --git a/qlib/model/riskmodel/__init__.py b/qlib/model/riskmodel/__init__.py index f9459d90533..230fdfca082 100644 --- a/qlib/model/riskmodel/__init__.py +++ b/qlib/model/riskmodel/__init__.py @@ -6,7 +6,6 @@ from .shrink import ShrinkCovEstimator from .structured import StructuredCovEstimator - __all__ = [ "RiskModel", "POETCovEstimator", diff --git a/qlib/model/riskmodel/base.py b/qlib/model/riskmodel/base.py index 7afacfe8ff2..9486da11153 100644 --- a/qlib/model/riskmodel/base.py +++ b/qlib/model/riskmodel/base.py @@ -2,9 +2,10 @@ # Licensed under the MIT License. import inspect +from typing import Union + import numpy as np import pandas as pd -from typing import Union from qlib.model.base import BaseModel diff --git a/qlib/model/riskmodel/shrink.py b/qlib/model/riskmodel/shrink.py index c3c0e48ef8b..7599ff076ef 100644 --- a/qlib/model/riskmodel/shrink.py +++ b/qlib/model/riskmodel/shrink.py @@ -1,6 +1,7 @@ -import numpy as np from typing import Union +import numpy as np + from qlib.model.riskmodel import RiskModel diff --git a/qlib/model/riskmodel/structured.py b/qlib/model/riskmodel/structured.py index 71e442536dc..2919a297c9a 100644 --- a/qlib/model/riskmodel/structured.py +++ b/qlib/model/riskmodel/structured.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import numpy as np from typing import Union + +import numpy as np from sklearn.decomposition import PCA, FactorAnalysis from qlib.model.riskmodel import RiskModel diff --git a/qlib/rl/__init__.py b/qlib/rl/__init__.py index a12afc39960..c3655825819 100644 --- a/qlib/rl/__init__.py +++ b/qlib/rl/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from .interpreter import Interpreter, StateInterpreter, ActionInterpreter +from .interpreter import ActionInterpreter, Interpreter, StateInterpreter from .reward import Reward, RewardCombination from .simulator import Simulator diff --git a/qlib/rl/contrib/naive_config_parser.py b/qlib/rl/contrib/naive_config_parser.py index 5608cbd1ef3..328c0d6b86c 100644 --- a/qlib/rl/contrib/naive_config_parser.py +++ b/qlib/rl/contrib/naive_config_parser.py @@ -7,8 +7,8 @@ import sys import tempfile from importlib import import_module -from ruamel.yaml import YAML +from ruamel.yaml import YAML DELETE_KEY = "_delete_" diff --git a/qlib/rl/contrib/train_onpolicy.py b/qlib/rl/contrib/train_onpolicy.py index 83dd924103f..13335fe233f 100644 --- a/qlib/rl/contrib/train_onpolicy.py +++ b/qlib/rl/contrib/train_onpolicy.py @@ -8,12 +8,15 @@ import sys import warnings from pathlib import Path -from ruamel.yaml import YAML -from typing import cast, List, Optional +from typing import List, Optional, cast import numpy as np import pandas as pd import torch +from ruamel.yaml import YAML +from tianshou.policy import BasePolicy +from torch.utils.data import Dataset + from qlib.backtest import Order from qlib.backtest.decision import OrderDir from qlib.constant import ONE_MIN @@ -25,8 +28,6 @@ from qlib.rl.trainer.callbacks import Callback, EarlyStopping, MetricsWriter from qlib.rl.utils.log import CsvWriter from qlib.utils import init_instance_by_config -from tianshou.policy import BasePolicy -from torch.utils.data import Dataset def seed_everything(seed: int) -> None: diff --git a/qlib/rl/interpreter.py b/qlib/rl/interpreter.py index 5c9cc26c4e6..25ad2f1cfea 100644 --- a/qlib/rl/interpreter.py +++ b/qlib/rl/interpreter.py @@ -10,6 +10,7 @@ from gym import spaces from qlib.typehint import final + from .simulator import ActType, StateType ObsType = TypeVar("ObsType") diff --git a/qlib/rl/order_execution/__init__.py b/qlib/rl/order_execution/__init__.py index b985c13317b..4bbba3f4beb 100644 --- a/qlib/rl/order_execution/__init__.py +++ b/qlib/rl/order_execution/__init__.py @@ -7,17 +7,17 @@ """ from .interpreter import ( - FullHistoryStateInterpreter, - CurrentStepStateInterpreter, CategoricalActionInterpreter, + CurrentStepStateInterpreter, + FullHistoryStateInterpreter, TwapRelativeActionInterpreter, ) from .network import Recurrent -from .policy import AllOne, PPO +from .policy import PPO, AllOne from .reward import PAPenaltyReward from .simulator_simple import SingleAssetOrderExecutionSimple from .state import SAOEMetrics, SAOEState -from .strategy import SAOEStateAdapter, SAOEStrategy, ProxySAOEStrategy, SAOEIntStrategy +from .strategy import ProxySAOEStrategy, SAOEIntStrategy, SAOEStateAdapter, SAOEStrategy __all__ = [ "FullHistoryStateInterpreter", diff --git a/qlib/rl/order_execution/policy.py b/qlib/rl/order_execution/policy.py index a46b587aa11..c34d974a88a 100644 --- a/qlib/rl/order_execution/policy.py +++ b/qlib/rl/order_execution/policy.py @@ -12,7 +12,7 @@ import torch.nn as nn from gym.spaces import Discrete from tianshou.data import Batch, ReplayBuffer, to_torch -from tianshou.policy import BasePolicy, PPOPolicy, DQNPolicy +from tianshou.policy import BasePolicy, DQNPolicy, PPOPolicy from qlib.rl.trainer.trainer import Trainer diff --git a/qlib/rl/order_execution/simulator_qlib.py b/qlib/rl/order_execution/simulator_qlib.py index 1417e2ab4a1..51e9c0a0103 100644 --- a/qlib/rl/order_execution/simulator_qlib.py +++ b/qlib/rl/order_execution/simulator_qlib.py @@ -12,6 +12,7 @@ from qlib.backtest.executor import NestedExecutor from qlib.rl.data.integration import init_qlib from qlib.rl.simulator import Simulator + from .state import SAOEState from .strategy import SAOEStateAdapter, SAOEStrategy diff --git a/qlib/rl/order_execution/simulator_simple.py b/qlib/rl/order_execution/simulator_simple.py index 48aa03a1707..662468994d0 100644 --- a/qlib/rl/order_execution/simulator_simple.py +++ b/qlib/rl/order_execution/simulator_simple.py @@ -3,12 +3,12 @@ from __future__ import annotations -from typing import Any, cast, List, Optional +from pathlib import Path +from typing import Any, List, Optional, cast import numpy as np import pandas as pd -from pathlib import Path from qlib.backtest.decision import Order, OrderDir from qlib.constant import EPS, EPS_T, float_or_ndarray from qlib.rl.data.base import BaseIntradayBacktestData @@ -16,6 +16,7 @@ from qlib.rl.data.pickle_styled import load_simple_intraday_backtest_data from qlib.rl.simulator import Simulator from qlib.rl.utils import LogLevel + from .state import SAOEMetrics, SAOEState __all__ = ["SingleAssetOrderExecutionSimple"] diff --git a/qlib/rl/order_execution/state.py b/qlib/rl/order_execution/state.py index 315735eaf84..949f3988e62 100644 --- a/qlib/rl/order_execution/state.py +++ b/qlib/rl/order_execution/state.py @@ -8,6 +8,7 @@ import numpy as np import pandas as pd + from qlib.backtest import Order from qlib.typehint import TypedDict diff --git a/qlib/rl/order_execution/strategy.py b/qlib/rl/order_execution/strategy.py index 7e66a1f0851..eb1eb928896 100644 --- a/qlib/rl/order_execution/strategy.py +++ b/qlib/rl/order_execution/strategy.py @@ -4,10 +4,10 @@ from __future__ import annotations import collections +import warnings from types import GeneratorType -from typing import Any, Callable, cast, Dict, Generator, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union, cast -import warnings import numpy as np import pandas as pd import torch diff --git a/qlib/rl/trainer/callbacks.py b/qlib/rl/trainer/callbacks.py index 9d1bf4ba282..79d389590e7 100644 --- a/qlib/rl/trainer/callbacks.py +++ b/qlib/rl/trainer/callbacks.py @@ -13,7 +13,7 @@ import time from datetime import datetime from pathlib import Path -from typing import Any, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, List import numpy as np import pandas as pd diff --git a/qlib/rl/utils/data_queue.py b/qlib/rl/utils/data_queue.py index 71c2dff65b9..33fdf958052 100644 --- a/qlib/rl/utils/data_queue.py +++ b/qlib/rl/utils/data_queue.py @@ -4,11 +4,11 @@ from __future__ import annotations import multiprocessing -from multiprocessing.sharedctypes import Synchronized import os import threading import time import warnings +from multiprocessing.sharedctypes import Synchronized from queue import Empty from typing import Any, Generator, Generic, Sequence, TypeVar, cast diff --git a/qlib/rl/utils/env_wrapper.py b/qlib/rl/utils/env_wrapper.py index e863b709a13..bf186a3a000 100644 --- a/qlib/rl/utils/env_wrapper.py +++ b/qlib/rl/utils/env_wrapper.py @@ -4,7 +4,7 @@ from __future__ import annotations import weakref -from typing import Any, Callable, cast, Dict, Generic, Iterable, Iterator, Optional, Tuple +from typing import Any, Callable, Dict, Generic, Iterable, Iterator, Optional, Tuple, cast import gym from gym import Space @@ -14,6 +14,7 @@ from qlib.rl.reward import Reward from qlib.rl.simulator import ActType, InitialStateType, Simulator, StateType from qlib.typehint import TypedDict + from .finite_env import generate_nan_observation from .log import LogCollector, LogLevel diff --git a/qlib/strategy/base.py b/qlib/strategy/base.py index a9e138fdbb7..438cba9d853 100644 --- a/qlib/strategy/base.py +++ b/qlib/strategy/base.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod -from typing import Any, Generator, Optional, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Generator, Optional, Union if TYPE_CHECKING: from qlib.backtest.exchange import Exchange diff --git a/qlib/tests/__init__.py b/qlib/tests/__init__.py index f9793cdabde..9c7fb140343 100644 --- a/qlib/tests/__init__.py +++ b/qlib/tests/__init__.py @@ -1,16 +1,18 @@ -from typing import Union, List, Dict, Tuple +import io import unittest -import pandas as pd +from typing import Dict, List, Tuple, Union + import numpy as np -import io +import pandas as pd -from .data import GetData -from .. import init -from ..constant import REG_CN, REG_TW -from qlib.data.filter import NameDFilter from qlib.data import D from qlib.data.data import Cal, DatasetD -from qlib.data.storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstKT, InstVT +from qlib.data.filter import NameDFilter +from qlib.data.storage import CalendarStorage, CalVT, FeatureStorage, InstKT, InstrumentStorage, InstVT + +from .. import init +from ..constant import REG_CN, REG_TW +from .data import GetData class TestAutoData(unittest.TestCase): diff --git a/qlib/tests/data.py b/qlib/tests/data.py index 2fa76855b58..e775f31cb49 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -1,17 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import datetime import os import re -import sys -import qlib import shutil +import sys import zipfile -import requests -import datetime -from tqdm import tqdm from pathlib import Path + +import requests from loguru import logger +from tqdm import tqdm + +import qlib from qlib.utils import exists_qlib_data diff --git a/qlib/typehint.py b/qlib/typehint.py index 5fd6e13c1a4..29dcb22bd52 100644 --- a/qlib/typehint.py +++ b/qlib/typehint.py @@ -4,8 +4,8 @@ """Commonly used types.""" import sys -from typing import Union from pathlib import Path +from typing import Union __all__ = ["Literal", "TypedDict", "final"] diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 2a94ebd555b..5b0cbbda3e5 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -3,36 +3,37 @@ # TODO: this utils covers too much utilities, please seperat it into sub modules -from __future__ import division -from __future__ import print_function +from __future__ import division, print_function -import os -import re -import copy -import json -import redis import bisect -import struct +import collections +import copy +import datetime import difflib -import inspect import hashlib -import datetime -import requests -import collections +import inspect +import json +import os +import re +import struct +from pathlib import Path +from typing import Callable, List, Optional, Union + import numpy as np import pandas as pd -from pathlib import Path -from typing import List, Union, Optional, Callable +import redis +import requests from packaging import version from ruamel.yaml import YAML + +from ..config import C +from ..log import get_module_logger, set_log_with_config from .file import ( get_or_create_path, + get_tmp_file_with_buffer, save_multiple_parts_file, unpack_archive_with_buffer, - get_tmp_file_with_buffer, ) -from ..config import C -from ..log import get_module_logger, set_log_with_config log = get_module_logger("utils") # MultiIndex.is_lexsorted() is a deprecated method in Pandas 1.3.0. @@ -937,12 +938,12 @@ def fname_to_code(fname: str): from .mod import ( - get_module_by_module_path, - split_module_path, + class_casting, get_callable_kwargs, get_cls_kwargs, + get_module_by_module_path, init_instance_by_config, - class_casting, + split_module_path, ) __all__ = [ diff --git a/qlib/utils/file.py b/qlib/utils/file.py index 1e17a574a9d..64251cb36c2 100644 --- a/qlib/utils/file.py +++ b/qlib/utils/file.py @@ -1,12 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import contextlib import os import shutil import tempfile -import contextlib -from typing import Optional, Text, IO, Union from pathlib import Path +from typing import IO, Optional, Text, Union from qlib.log import get_module_logger diff --git a/qlib/utils/index_data.py b/qlib/utils/index_data.py index c707240d098..ac242b9350b 100644 --- a/qlib/utils/index_data.py +++ b/qlib/utils/index_data.py @@ -11,8 +11,8 @@ from __future__ import annotations -from typing import Dict, Tuple, Union, Callable, List import bisect +from typing import Callable, Dict, List, Tuple, Union import numpy as np import pandas as pd diff --git a/qlib/utils/mod.py b/qlib/utils/mod.py index 5cb2ed3f453..06dbf838b38 100644 --- a/qlib/utils/mod.py +++ b/qlib/utils/mod.py @@ -10,10 +10,10 @@ import contextlib import importlib import os -from pathlib import Path import pkgutil import re import sys +from pathlib import Path from types import ModuleType from typing import Any, Dict, List, Tuple, Union from urllib.parse import urlparse diff --git a/qlib/utils/paral.py b/qlib/utils/paral.py index a6177833413..23a9262d303 100644 --- a/qlib/utils/paral.py +++ b/qlib/utils/paral.py @@ -1,18 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import concurrent import threading from functools import partial +from queue import Empty, Queue from threading import Thread from typing import Callable, Text, Union import joblib +import pandas as pd from joblib import Parallel, delayed from joblib._parallel_backends import MultiprocessingBackend -import pandas as pd - -from queue import Empty, Queue -import concurrent from qlib.config import C, QlibConfig diff --git a/qlib/utils/resam.py b/qlib/utils/resam.py index 99aedfcd50c..64902600c8f 100644 --- a/qlib/utils/resam.py +++ b/qlib/utils/resam.py @@ -1,12 +1,12 @@ +from functools import partial +from typing import Callable, Union + import numpy as np import pandas as pd -from functools import partial -from typing import Union, Callable - +from ..config import C from . import lazy_sort_index from .time import Freq, cal_sam_minute -from ..config import C def resam_calendar( diff --git a/qlib/utils/serial.py b/qlib/utils/serial.py index 720dbd79288..87b1fb74663 100644 --- a/qlib/utils/serial.py +++ b/qlib/utils/serial.py @@ -2,9 +2,11 @@ # Licensed under the MIT License. import pickle -import dill from pathlib import Path from typing import Union + +import dill + from ..config import C diff --git a/qlib/utils/time.py b/qlib/utils/time.py index 238b3f0dd5d..ef5d9581f03 100644 --- a/qlib/utils/time.py +++ b/qlib/utils/time.py @@ -4,17 +4,16 @@ Time related utils are compiled in this script """ import bisect -from datetime import datetime, time, date, timedelta -from typing import List, Optional, Tuple, Union import functools import re +from datetime import date, datetime, time, timedelta +from typing import List, Optional, Tuple, Union import pandas as pd from qlib.config import C from qlib.constant import REG_CN, REG_TW, REG_US - CN_TIME = [ datetime.strptime("9:30", "%H:%M"), datetime.strptime("11:30", "%H:%M"), diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index a29e471c04b..805ff9d7ff2 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -15,12 +15,13 @@ """ from contextlib import contextmanager -from typing import Text, Optional, Any, Dict -from .expm import ExpManager -from .exp import Experiment -from .recorder import Recorder +from typing import Any, Dict, Optional, Text + from ..utils import Wrapper from ..utils.exceptions import RecorderInitializationError +from .exp import Experiment +from .expm import ExpManager +from .recorder import Recorder class QlibRecorder: diff --git a/qlib/workflow/exp.py b/qlib/workflow/exp.py index ae165ef1f87..ca549859d4a 100644 --- a/qlib/workflow/exp.py +++ b/qlib/workflow/exp.py @@ -2,12 +2,15 @@ # Licensed under the MIT License. from typing import Dict, List, Union -from qlib.typehint import Literal + import mlflow from mlflow.entities import ViewType from mlflow.exceptions import MlflowException -from .recorder import Recorder, MLflowRecorder + +from qlib.typehint import Literal + from ..log import get_module_logger +from .recorder import MLflowRecorder, Recorder logger = get_module_logger("workflow") diff --git a/qlib/workflow/expm.py b/qlib/workflow/expm.py index 5047ccfb26c..5be8ad748a2 100644 --- a/qlib/workflow/expm.py +++ b/qlib/workflow/expm.py @@ -1,21 +1,21 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import os +from pathlib import Path +from typing import Optional, Text from urllib.parse import urlparse + import mlflow from filelock import FileLock -from mlflow.exceptions import MlflowException, RESOURCE_ALREADY_EXISTS, ErrorCode from mlflow.entities import ViewType -import os -from typing import Optional, Text -from pathlib import Path +from mlflow.exceptions import RESOURCE_ALREADY_EXISTS, ErrorCode, MlflowException -from .exp import MLflowExperiment, Experiment from ..config import C -from .recorder import Recorder from ..log import get_module_logger from ..utils.exceptions import ExpAlreadyExistError - +from .exp import Experiment, MLflowExperiment +from .recorder import Recorder logger = get_module_logger("workflow") diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 09e96d444f2..f32f60accd7 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -88,6 +88,7 @@ from typing import Callable, List, Union import pandas as pd + from qlib import get_module_logger from qlib.data.data import D from qlib.log import set_global_logger_level diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index d545e4bc9a6..39de5704749 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -6,6 +6,7 @@ """ from typing import List, Union + from qlib.log import get_module_logger from qlib.model.ens.group import RollingGroup from qlib.utils import transform_end_date diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 5047a1bd25e..b2e5674315e 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -8,14 +8,15 @@ from typing import Optional import pandas as pd + from qlib import get_module_logger from qlib.data import D from qlib.data.dataset import Dataset, DatasetH, TSDatasetH from qlib.data.dataset.handler import DataHandlerLP from qlib.model import Model from qlib.utils import get_date_by_shift -from qlib.workflow.recorder import Recorder from qlib.workflow.record_temp import SignalRecord +from qlib.workflow.recorder import Recorder class RMDLoader: diff --git a/qlib/workflow/record_temp.py b/qlib/workflow/record_temp.py index 844914d469f..316679541bb 100644 --- a/qlib/workflow/record_temp.py +++ b/qlib/workflow/record_temp.py @@ -3,25 +3,25 @@ import logging import warnings -import pandas as pd +from pprint import pprint +from typing import Dict, List, Optional, Union + import numpy as np +import pandas as pd from tqdm import trange -from pprint import pprint -from typing import Union, List, Optional, Dict from qlib.utils.exceptions import LoadObjectError -from ..contrib.evaluate import risk_analysis, indicator_analysis +from ..backtest import backtest as normal_backtest +from ..contrib.eva.alpha import calc_ic, calc_long_short_prec, calc_long_short_return +from ..contrib.evaluate import indicator_analysis, risk_analysis from ..data.dataset import DatasetH from ..data.dataset.handler import DataHandlerLP -from ..backtest import backtest as normal_backtest from ..log import get_module_logger -from ..utils import fill_placeholder, flatten_dict, class_casting, get_date_by_shift -from ..utils.time import Freq +from ..utils import class_casting, fill_placeholder, flatten_dict, get_date_by_shift from ..utils.data import deepcopy_basic_type from ..utils.exceptions import QlibException -from ..contrib.eva.alpha import calc_ic, calc_long_short_return, calc_long_short_prec - +from ..utils.time import Freq logger = get_module_logger("workflow", logging.INFO) diff --git a/qlib/workflow/recorder.py b/qlib/workflow/recorder.py index 5fd99c0769f..701962bfb70 100644 --- a/qlib/workflow/recorder.py +++ b/qlib/workflow/recorder.py @@ -2,23 +2,24 @@ # Licensed under the MIT License. import os -import sys -from typing import Optional -import mlflow -import shutil import pickle -import tempfile -import subprocess import platform -from pathlib import Path +import shutil +import subprocess +import sys +import tempfile from datetime import datetime +from pathlib import Path +from typing import Optional + +import mlflow +from mlflow.store.artifact.azure_blob_artifact_repo import AzureBlobArtifactRepository -from qlib.utils.serial import Serializable from qlib.utils.exceptions import LoadObjectError from qlib.utils.paral import AsyncCaller +from qlib.utils.serial import Serializable from ..log import TimeInspector, get_module_logger -from mlflow.store.artifact.azure_blob_artifact_repo import AzureBlobArtifactRepository logger = get_module_logger("workflow") # mlflow limits the length of log_param to 500, but this caused errors when using qrun, so we extended the mlflow limit. diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index bedbd96d201..f55b6777fd2 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -6,11 +6,11 @@ """ from collections import defaultdict -from qlib.log import TimeInspector from typing import Callable, Dict, Iterable, List -from qlib.log import get_module_logger -from qlib.utils.serial import Serializable + +from qlib.log import TimeInspector, get_module_logger from qlib.utils.exceptions import LoadObjectError +from qlib.utils.serial import Serializable from qlib.workflow import R from qlib.workflow.exp import Experiment from qlib.workflow.recorder import Recorder diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index f13a4af7c1f..cfc26a36318 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -5,10 +5,12 @@ """ import abc import copy +from typing import Callable, Dict, List, Union + import pandas as pd -from typing import Dict, List, Union, Callable from qlib.utils import transform_end_date + from .utils import TimeAdjuster diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 59815b5e2dc..c25ee39dd1f 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -23,12 +23,13 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument -from qlib import auto_init, get_module_logger from tqdm.cli import tqdm -from .utils import get_mongodb +from qlib import auto_init, get_module_logger + from ...config import C from ...utils.pickle_utils import restricted_pickle_loads +from .utils import get_mongodb class TaskManager: diff --git a/qlib/workflow/task/utils.py b/qlib/workflow/task/utils.py index 4b4a7c06b8e..ecbb27e6ea7 100644 --- a/qlib/workflow/task/utils.py +++ b/qlib/workflow/task/utils.py @@ -6,17 +6,19 @@ import bisect from copy import deepcopy +from pathlib import Path +from typing import Union + import pandas as pd +from pymongo import MongoClient +from pymongo.database import Database + +from qlib.config import C from qlib.data import D +from qlib.log import get_module_logger from qlib.utils import hash_args from qlib.utils.mod import init_instance_by_config from qlib.workflow import R -from qlib.config import C -from qlib.log import get_module_logger -from pymongo import MongoClient -from pymongo.database import Database -from typing import Union -from pathlib import Path def get_mongodb() -> Database: diff --git a/scripts/check_dump_bin.py b/scripts/check_dump_bin.py index 7ae8a26ab0d..fdf2c9fbdbd 100644 --- a/scripts/check_dump_bin.py +++ b/scripts/check_dump_bin.py @@ -1,17 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from pathlib import Path from concurrent.futures import ProcessPoolExecutor +from pathlib import Path -import qlib -from qlib.data import D - -import fire import datacompy +import fire import pandas as pd -from tqdm import tqdm from loguru import logger +from tqdm import tqdm + +import qlib +from qlib.data import D class CheckBin: diff --git a/scripts/collect_info.py b/scripts/collect_info.py index 9e7a6395ef5..cadef7300cc 100644 --- a/scripts/collect_info.py +++ b/scripts/collect_info.py @@ -1,9 +1,11 @@ -import sys import platform -import qlib +import sys +from pathlib import Path + import fire import pkg_resources -from pathlib import Path + +import qlib QLIB_PATH = Path(__file__).absolute().resolve().parent.parent diff --git a/scripts/data_collector/baostock_5min/collector.py b/scripts/data_collector/baostock_5min/collector.py index 0a69beefb82..1ee6cf9ac96 100644 --- a/scripts/data_collector/baostock_5min/collector.py +++ b/scripts/data_collector/baostock_5min/collector.py @@ -2,16 +2,17 @@ # Licensed under the MIT License. -import sys import copy +import sys +from pathlib import Path +from typing import Iterable, List + +import baostock as bs import fire import numpy as np import pandas as pd -import baostock as bs -from tqdm import tqdm -from pathlib import Path from loguru import logger -from typing import Iterable, List +from tqdm import tqdm import qlib from qlib.data import D @@ -20,7 +21,7 @@ sys.path.append(str(CUR_DIR.parent.parent)) from data_collector.base import BaseCollector, BaseNormalize, BaseRun -from data_collector.utils import generate_minutes_calendar_from_daily, calc_adjusted_price +from data_collector.utils import calc_adjusted_price, generate_minutes_calendar_from_daily class BaostockCollectorHS3005min(BaseCollector): diff --git a/scripts/data_collector/base.py b/scripts/data_collector/base.py index 2efc2feadcb..83797111e0b 100644 --- a/scripts/data_collector/base.py +++ b/scripts/data_collector/base.py @@ -3,17 +3,18 @@ import abc -import time import datetime import importlib -from pathlib import Path -from typing import Type, Iterable +import time from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from typing import Iterable, Type import pandas as pd -from tqdm import tqdm -from loguru import logger from joblib import Parallel, delayed +from loguru import logger +from tqdm import tqdm + from qlib.utils import code_to_fname diff --git a/scripts/data_collector/br_index/collector.py b/scripts/data_collector/br_index/collector.py index 04b2f96d9fb..9c40f7baa63 100644 --- a/scripts/data_collector/br_index/collector.py +++ b/scripts/data_collector/br_index/collector.py @@ -1,14 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from functools import partial +import datetime import sys +from functools import partial from pathlib import Path -import datetime import fire import pandas as pd -from tqdm import tqdm from loguru import logger +from tqdm import tqdm CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) diff --git a/scripts/data_collector/cn_index/collector.py b/scripts/data_collector/cn_index/collector.py index fb6914d24a6..6fe9e5ed603 100644 --- a/scripts/data_collector/cn_index/collector.py +++ b/scripts/data_collector/cn_index/collector.py @@ -1,27 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import re import abc +import re import sys from io import BytesIO -from typing import List, Iterable from pathlib import Path +from typing import Iterable, List +import baostock as bs import fire -import requests import pandas as pd -import baostock as bs -from tqdm import tqdm +import requests from loguru import logger +from tqdm import tqdm CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) from data_collector.index import IndexBase -from data_collector.utils import get_calendar_list, get_trading_date_by_shift, deco_retry -from data_collector.utils import get_instruments - +from data_collector.utils import deco_retry, get_calendar_list, get_instruments, get_trading_date_by_shift NEW_COMPANIES_URL = ( "https://oss-ch.csindex.com.cn/static/html/csindex/public/uploads/file/autofile/cons/{index_code}cons.xls" diff --git a/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py b/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py index 0a721298d3c..2aa70611229 100644 --- a/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py +++ b/scripts/data_collector/contrib/fill_cn_1min_data/fill_cn_1min_data.py @@ -2,15 +2,16 @@ # Licensed under the MIT License. import sys -from pathlib import Path from concurrent.futures import ThreadPoolExecutor +from pathlib import Path import fire -import qlib import pandas as pd +from loguru import logger from tqdm import tqdm + +import qlib from qlib.data import D -from loguru import logger CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent.parent)) diff --git a/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py b/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py index 939ba7f6ad0..85ec6bf1317 100644 --- a/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py +++ b/scripts/data_collector/contrib/future_trading_date_collector/future_trading_date_collector.py @@ -2,17 +2,16 @@ # Licensed under the MIT License. import sys -from typing import List from pathlib import Path +from typing import List +# get data from baostock +import baostock as bs import fire import numpy as np import pandas as pd from loguru import logger -# get data from baostock -import baostock as bs - CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent.parent)) diff --git a/scripts/data_collector/crypto/collector.py b/scripts/data_collector/crypto/collector.py index 302b89e200c..d0d90efc662 100644 --- a/scripts/data_collector/crypto/collector.py +++ b/scripts/data_collector/crypto/collector.py @@ -1,24 +1,23 @@ import abc -import sys import datetime +import sys from abc import ABC from pathlib import Path import fire import pandas as pd -from loguru import logger from dateutil.tz import tzlocal +from loguru import logger CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) +import time +from datetime import datetime as dt +from time import mktime + from data_collector.base import BaseCollector, BaseNormalize, BaseRun from data_collector.utils import deco_retry - from pycoingecko import CoinGeckoAPI -from time import mktime -from datetime import datetime as dt -import time - _CG_CRYPTO_SYMBOLS = None diff --git a/scripts/data_collector/fund/collector.py b/scripts/data_collector/fund/collector.py index 937d3931dbb..a5f6cad8be8 100644 --- a/scripts/data_collector/fund/collector.py +++ b/scripts/data_collector/fund/collector.py @@ -2,17 +2,18 @@ # Licensed under the MIT License. import abc -import sys import datetime import json +import sys from abc import ABC from pathlib import Path import fire -import requests import pandas as pd -from loguru import logger +import requests from dateutil.tz import tzlocal +from loguru import logger + from qlib.constant import REG_CN as REGION_CN CUR_DIR = Path(__file__).resolve().parent diff --git a/scripts/data_collector/future_calendar_collector.py b/scripts/data_collector/future_calendar_collector.py index 4dfd24e4b7b..250a4127776 100644 --- a/scripts/data_collector/future_calendar_collector.py +++ b/scripts/data_collector/future_calendar_collector.py @@ -4,14 +4,13 @@ import abc import importlib from pathlib import Path -from typing import Union, Iterable, List +from typing import Iterable, List, Union +# pip install baostock +import baostock as bs import fire import numpy as np import pandas as pd - -# pip install baostock -import baostock as bs from loguru import logger diff --git a/scripts/data_collector/index.py b/scripts/data_collector/index.py index caaaee53af5..b8ad1cf2a2c 100644 --- a/scripts/data_collector/index.py +++ b/scripts/data_collector/index.py @@ -1,11 +1,11 @@ -import sys import abc +import sys from pathlib import Path from typing import List import pandas as pd -from tqdm import tqdm from loguru import logger +from tqdm import tqdm CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent)) diff --git a/scripts/data_collector/pit/collector.py b/scripts/data_collector/pit/collector.py index c34b31348dc..9bc1951f42d 100644 --- a/scripts/data_collector/pit/collector.py +++ b/scripts/data_collector/pit/collector.py @@ -5,18 +5,18 @@ import sys from datetime import datetime from pathlib import Path -from typing import List, Iterable, Optional, Union +from typing import Iterable, List, Optional, Union +import baostock as bs import fire import pandas as pd -import baostock as bs from loguru import logger BASE_DIR = Path(__file__).resolve().parent sys.path.append(str(BASE_DIR.parent.parent)) -from data_collector.base import BaseCollector, BaseRun, BaseNormalize -from data_collector.utils import get_hs_stock_symbols, get_calendar_list +from data_collector.base import BaseCollector, BaseNormalize, BaseRun +from data_collector.utils import get_calendar_list, get_hs_stock_symbols class PitCollector(BaseCollector): diff --git a/scripts/data_collector/us_index/collector.py b/scripts/data_collector/us_index/collector.py index 800560d2e12..59e40d47145 100644 --- a/scripts/data_collector/us_index/collector.py +++ b/scripts/data_collector/us_index/collector.py @@ -2,28 +2,25 @@ # Licensed under the MIT License. import abc -from functools import partial import sys -from pathlib import Path from concurrent.futures import ThreadPoolExecutor -from typing import List +from functools import partial from io import StringIO +from pathlib import Path +from typing import List import fire -import requests import pandas as pd -from tqdm import tqdm -from loguru import logger +import requests from fake_useragent import UserAgent - +from loguru import logger +from tqdm import tqdm CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) from data_collector.index import IndexBase -from data_collector.utils import deco_retry, get_calendar_list, get_trading_date_by_shift -from data_collector.utils import get_instruments - +from data_collector.utils import deco_retry, get_calendar_list, get_instruments, get_trading_date_by_shift WIKI_URL = "https://en.wikipedia.org/wiki" diff --git a/scripts/data_collector/utils.py b/scripts/data_collector/utils.py index 7b0c0576885..b47529efd40 100644 --- a/scripts/data_collector/utils.py +++ b/scripts/data_collector/utils.py @@ -1,25 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import re +import bisect import copy +import functools import importlib -import time -import bisect import pickle -import requests -import functools +import re +import time +from concurrent.futures import ProcessPoolExecutor +from functools import partial from pathlib import Path -from typing import Iterable, Tuple, List +from typing import Iterable, List, Tuple import numpy as np import pandas as pd +import requests +from bs4 import BeautifulSoup from loguru import logger -from yahooquery import Ticker from tqdm import tqdm -from functools import partial -from concurrent.futures import ProcessPoolExecutor -from bs4 import BeautifulSoup +from yahooquery import Ticker HS_SYMBOLS_URL = "http://app.finance.ifeng.com/hq/list.php?type=stock_a&class={s_type}" diff --git a/scripts/data_collector/yahoo/collector.py b/scripts/data_collector/yahoo/collector.py index 82660f1112b..959655c6c63 100644 --- a/scripts/data_collector/yahoo/collector.py +++ b/scripts/data_collector/yahoo/collector.py @@ -2,45 +2,45 @@ # Licensed under the MIT License. import abc -import sys import copy -import time import datetime import importlib -from abc import ABC import multiprocessing +import sys +import time +from abc import ABC from pathlib import Path from typing import Iterable import fire -import requests import numpy as np import pandas as pd +import requests +from dateutil.tz import tzlocal from loguru import logger from yahooquery import Ticker -from dateutil.tz import tzlocal import qlib +from qlib.constant import REG_CN as REGION_CN from qlib.data import D from qlib.tests.data import GetData -from qlib.utils import code_to_fname, fname_to_code, exists_qlib_data -from qlib.constant import REG_CN as REGION_CN +from qlib.utils import code_to_fname, exists_qlib_data, fname_to_code CUR_DIR = Path(__file__).resolve().parent sys.path.append(str(CUR_DIR.parent.parent)) -from dump_bin import DumpDataUpdate from data_collector.base import BaseCollector, BaseNormalize, BaseRun, Normalize from data_collector.utils import ( + calc_adjusted_price, deco_retry, + generate_minutes_calendar_from_daily, + get_br_stock_symbols, get_calendar_list, get_hs_stock_symbols, - get_us_stock_symbols, get_in_stock_symbols, - get_br_stock_symbols, - generate_minutes_calendar_from_daily, - calc_adjusted_price, + get_us_stock_symbols, ) +from dump_bin import DumpDataUpdate INDEX_BENCH_URL = "http://push2his.eastmoney.com/api/qt/stock/kline/get?secid=1.{index_code}&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58&klt=101&fqt=0&beg={begin}&end={end}" diff --git a/scripts/dump_bin.py b/scripts/dump_bin.py index cb8ed72dabb..c529504f9b0 100644 --- a/scripts/dump_bin.py +++ b/scripts/dump_bin.py @@ -4,17 +4,18 @@ import abc import shutil import traceback +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed +from functools import partial from pathlib import Path from typing import Iterable, List, Union -from functools import partial -from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor import fire import numpy as np import pandas as pd -from tqdm import tqdm from loguru import logger -from qlib.utils import fname_to_code, code_to_fname +from tqdm import tqdm + +from qlib.utils import code_to_fname, fname_to_code def read_as_df(file_path: Union[str, Path], **kwargs) -> pd.DataFrame: diff --git a/scripts/dump_pit.py b/scripts/dump_pit.py index 806bbd0cc94..10e394e0c98 100644 --- a/scripts/dump_pit.py +++ b/scripts/dump_pit.py @@ -8,17 +8,18 @@ import shutil import struct +from concurrent.futures import ProcessPoolExecutor +from functools import partial from pathlib import Path from typing import Iterable -from functools import partial -from concurrent.futures import ProcessPoolExecutor import fire import pandas as pd -from tqdm import tqdm from loguru import logger -from qlib.utils import fname_to_code, get_period_offset +from tqdm import tqdm + from qlib.config import C +from qlib.utils import fname_to_code, get_period_offset class DumpPitData: diff --git a/scripts/get_data.py b/scripts/get_data.py index 3710816ce08..20aad754f7e 100644 --- a/scripts/get_data.py +++ b/scripts/get_data.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. import fire -from qlib.tests.data import GetData +from qlib.tests.data import GetData if __name__ == "__main__": fire.Fire(GetData)