From 6100c0e75840cbc2dd44fcee790b88003e2c2e42 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Tue, 13 Jul 2021 12:12:45 +0800 Subject: [PATCH 01/17] Add files via upload Add naive transformer model and a improved transformer model. --- qlib/contrib/model/pytorch_localformer.py | 341 ++++++++++++++++++++++ qlib/contrib/model/pytorch_transformer.py | 312 ++++++++++++++++++++ 2 files changed, 653 insertions(+) create mode 100644 qlib/contrib/model/pytorch_localformer.py create mode 100644 qlib/contrib/model/pytorch_transformer.py diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py new file mode 100644 index 0000000000..f085bd4b21 --- /dev/null +++ b/qlib/contrib/model/pytorch_localformer.py @@ -0,0 +1,341 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +import math +from ...utils import get_or_create_path +from ...log import get_module_logger + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +from .pytorch_utils import count_parameters +from ...model.base import Model +from ...data.dataset import DatasetH, TSDatasetH +from ...data.dataset.handler import DataHandlerLP +from torch.nn.modules.container import ModuleList + +import pdb + +# qrun benchmarks/Transformer/workflow_config_localformer_Alpha158.yaml +# 0.992366, @13, +''' +{'IC': 0.037426503365732174, + 'ICIR': 0.28977883455541603, + 'Rank IC': 0.04659889541774283, + 'Rank ICIR': 0.373569340092482} + +'The following are analysis results of the excess return without cost.' + risk +mean 0.000381 +std 0.004109 +annualized_return 0.096066 +information_ratio 1.472729 +max_drawdown -0.094917 +'The following are analysis results of the excess return with cost.' + risk +mean 0.000213 +std 0.004111 +annualized_return 0.053630 +information_ratio 0.821711 +max_drawdown -0.113694 +''' + + +class LocalformerModel(Model): + def __init__( + self, + d_feat: int = 20, + d_model: int = 64, + batch_size: int = 8192, + nhead: int = 2, + num_layers: int = 2, + dropout: float = 0, + n_epochs=100, + lr=0.0001, + metric="", + early_stop=5, + loss="mse", + optimizer="adam", + reg=1e-3, + n_jobs=10, + GPU=2, + seed=None, + **kwargs + ): + + # set hyper-parameters. + self.d_model = d_model + self.dropout = dropout + self.n_epochs = n_epochs + self.lr = lr + self.reg = reg + self.metric = metric + self.batch_size = batch_size + self.early_stop = early_stop + self.optimizer = optimizer.lower() + self.loss = loss + self.n_jobs = n_jobs + self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") + self.seed = seed + self.logger = get_module_logger("TransformerModel") + print('do we have gpu?{}'.format(torch.cuda.is_available())) + self.logger.info( + "Improved Transformer:" + "\nbatch_size : {}" + "\ndevice : {}".format(self.batch_size, self.device) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.model = Transformer(d_feat, d_model, nhead, num_layers, dropout, self.device) + if optimizer.lower() == "adam": + self.train_optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + elif optimizer.lower() == "gd": + self.train_optimizer = optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + else: + raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) + + self.fitted = False + self.model.to(self.device) + + @property + def use_gpu(self): + return self.device != torch.device("cpu") + + def mse(self, pred, label): + loss = (pred.float() - label.float()) ** 2 + return torch.mean(loss) + + def loss_fn(self, pred, label): + mask = ~torch.isnan(label) + + if self.loss == "mse": + return self.mse(pred[mask], label[mask]) + + raise ValueError("unknown loss `%s`" % self.loss) + + def metric_fn(self, pred, label): + + mask = torch.isfinite(label) + + if self.metric == "" or self.metric == "loss": + return -self.loss_fn(pred[mask], label[mask]) + + raise ValueError("unknown metric `%s`" % self.metric) + + def train_epoch(self, data_loader): + + self.model.train() + + for data in data_loader: + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + + self.train_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) + self.train_optimizer.step() + + def test_epoch(self, data_loader): + + self.model.eval() + + scores = [] + losses = [] + + for data in data_loader: + + feature = data[:, :, 0:-1].to(self.device) + # feature[torch.isnan(feature)] = 0 + label = data[:, -1, -1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + losses.append(loss.item()) + + score = self.metric_fn(pred, label) + scores.append(score.item()) + + return np.mean(losses), np.mean(scores) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + save_path=None, + ): + + dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + + dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader + dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader + + train_loader = DataLoader( + dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True + ) + valid_loader = DataLoader( + dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + ) + + save_path = get_or_create_path(save_path) + + stop_steps = 0 + train_loss = 0 + best_score = -np.inf + best_epoch = 0 + evals_result["train"] = [] + evals_result["valid"] = [] + + # train + self.logger.info("training...") + self.fitted = True + + for step in range(self.n_epochs): + self.logger.info("Epoch%d:", step) + self.logger.info("training...") + self.train_epoch(train_loader) + self.logger.info("evaluating...") + train_loss, train_score = self.test_epoch(train_loader) + val_loss, val_score = self.test_epoch(valid_loader) + self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) + evals_result["train"].append(train_score) + evals_result["valid"].append(val_score) + + if val_score > best_score: + best_score = val_score + stop_steps = 0 + best_epoch = step + best_param = copy.deepcopy(self.model.state_dict()) + else: + stop_steps += 1 + if stop_steps >= self.early_stop: + self.logger.info("early stop") + break + + self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch)) + self.model.load_state_dict(best_param) + torch.save(best_param, save_path) + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) + dl_test.config(fillna_type="ffill+bfill") + test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + self.model.eval() + preds = [] + + for data in test_loader: + feature = data[:, :, 0:-1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()).detach().cpu().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=1000): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + self.register_buffer('pe', pe) + + def forward(self, x): + # [T, N, F] + return x + self.pe[:x.size(0), :] + + +def _get_clones(module, N): + return ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class LocalformerEncoder(nn.Module): + __constants__ = ['norm'] + + def __init__(self, encoder_layer, num_layers, d_model): + super(LocalformerEncoder, self).__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.conv = _get_clones(nn.Conv1d(d_model, d_model, 3, 1, 1), num_layers) + self.num_layers = num_layers + + def forward(self, src, mask): + output = src + out = src + + for i, mod in enumerate(self.layers): + # [T, N, F] --> [N, T, F] --> [N, F, T] + out = output.transpose(1, 0).transpose(2, 1) + out = self.conv[i](out).transpose(2, 1).transpose(1, 0) + + output = mod(output+out, src_mask=mask) + + return output + out + + +class Transformer(nn.Module): + def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, device=None): + super(Transformer, self).__init__() + self.rnn = nn.GRU( + input_size=d_model, + hidden_size=d_model, + num_layers=num_layers, + batch_first=False, + dropout=dropout, + ) + self.feature_layer = nn.Linear(d_feat, d_model) + self.pos_encoder = PositionalEncoding(d_model) + self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dropout=dropout) + self.transformer_encoder = LocalformerEncoder(self.encoder_layer, num_layers=num_layers, d_model=d_model) + self.decoder_layer = nn.Linear(d_model, 1) + self.device = device + self.d_feat = d_feat + + def forward(self, src): + # pdb.set_trace() + # src [N, T, F], [512, 60, 6] + + src = self.feature_layer(src) # [512, 60, 8] + + # src [N, T, F] --> [T, N, F], [60, 512, 8] + src = src.transpose(1, 0) # not batch first + + mask = None + + src = self.pos_encoder(src) + output = self.transformer_encoder(src, mask) # [60, 512, 8] + + output, _ = self.rnn(output) + + # [T, N, F] --> [N, T*F] + output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] + + return output.squeeze() + diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py new file mode 100644 index 0000000000..85582be1f1 --- /dev/null +++ b/qlib/contrib/model/pytorch_transformer.py @@ -0,0 +1,312 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +import math +from ...utils import get_or_create_path +from ...log import get_module_logger + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +from .pytorch_utils import count_parameters +from ...model.base import Model +from ...data.dataset import DatasetH, TSDatasetH +from ...data.dataset.handler import DataHandlerLP + +import pdb + +# qrun benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml +# 0.993681, @11, +''' + 'IC': 0.03186587768611013, + 'ICIR': 0.2556910881045764, + 'Rank IC': 0.04735251936658551, + 'Rank ICIR': 0.388378955424602 + +'The following are analysis results of the excess return without cost.' + risk +mean 0.000309 +std 0.004209 +annualized_return 0.077839 +information_ratio 1.164993 +max_drawdown -0.106215 +'The following are analysis results of the excess return with cost.' + risk +mean 0.000126 +std 0.004209 +annualized_return 0.031707 +information_ratio 0.474567 +max_drawdown -0.131948 +''' + + +class TransformerModel(Model): + def __init__( + self, + d_feat: int = 20, + d_model: int = 64, + batch_size: int = 8192, + nhead: int = 2, + num_layers: int = 2, + dropout: float = 0, + n_epochs=100, + lr=0.0001, + metric="", + early_stop=5, + loss="mse", + optimizer="adam", + reg=1e-3, + n_jobs=10, + GPU=0, + seed=None, + **kwargs + ): + + # set hyper-parameters. + self.d_model = d_model + self.dropout = dropout + self.n_epochs = n_epochs + self.lr = lr + self.reg = reg + self.metric = metric + self.batch_size = batch_size + self.early_stop = early_stop + self.optimizer = optimizer.lower() + self.loss = loss + self.n_jobs = n_jobs + self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") + self.seed = seed + self.logger = get_module_logger("TransformerModel") + print('do we have gpu?{}'.format(torch.cuda.is_available())) + self.logger.info( + "Naive Transformer:" + "\nbatch_size : {}" + "\ndevice : {}".format(self.batch_size, self.device) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.model = Transformer(d_feat, d_model, nhead, num_layers, dropout, self.device) + if optimizer.lower() == "adam": + self.train_optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + elif optimizer.lower() == "gd": + self.train_optimizer = optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + else: + raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) + + self.fitted = False + self.model.to(self.device) + + @property + def use_gpu(self): + return self.device != torch.device("cpu") + + def mse(self, pred, label): + loss = (pred.float() - label.float()) ** 2 + return torch.mean(loss) + + def loss_fn(self, pred, label): + mask = ~torch.isnan(label) + + if self.loss == "mse": + return self.mse(pred[mask], label[mask]) + + raise ValueError("unknown loss `%s`" % self.loss) + + def metric_fn(self, pred, label): + + mask = torch.isfinite(label) + + if self.metric == "" or self.metric == "loss": + return -self.loss_fn(pred[mask], label[mask]) + + raise ValueError("unknown metric `%s`" % self.metric) + + def train_epoch(self, data_loader): + + self.model.train() + + for data in data_loader: + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + + self.train_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) + self.train_optimizer.step() + + def test_epoch(self, data_loader): + + self.model.eval() + + scores = [] + losses = [] + + for data in data_loader: + + feature = data[:, :, 0:-1].to(self.device) + # feature[torch.isnan(feature)] = 0 + label = data[:, -1, -1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + losses.append(loss.item()) + + score = self.metric_fn(pred, label) + scores.append(score.item()) + + return np.mean(losses), np.mean(scores) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + save_path=None, + ): + + dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + + dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader + dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader + + train_loader = DataLoader( + dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True + ) + valid_loader = DataLoader( + dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + ) + + save_path = get_or_create_path(save_path) + + stop_steps = 0 + train_loss = 0 + best_score = -np.inf + best_epoch = 0 + evals_result["train"] = [] + evals_result["valid"] = [] + + # train + self.logger.info("training...") + self.fitted = True + + for step in range(self.n_epochs): + self.logger.info("Epoch%d:", step) + self.logger.info("training...") + self.train_epoch(train_loader) + self.logger.info("evaluating...") + train_loss, train_score = self.test_epoch(train_loader) + val_loss, val_score = self.test_epoch(valid_loader) + self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) + evals_result["train"].append(train_score) + evals_result["valid"].append(val_score) + + if val_score > best_score: + best_score = val_score + stop_steps = 0 + best_epoch = step + best_param = copy.deepcopy(self.model.state_dict()) + else: + stop_steps += 1 + if stop_steps >= self.early_stop: + self.logger.info("early stop") + break + + self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch)) + self.model.load_state_dict(best_param) + torch.save(best_param, save_path) + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) + dl_test.config(fillna_type="ffill+bfill") + test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + self.model.eval() + preds = [] + + for data in test_loader: + feature = data[:, :, 0:-1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()).detach().cpu().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=1000): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + self.register_buffer('pe', pe) + + def forward(self, x): + # [T, N, F] + return x + self.pe[:x.size(0), :] + + +class Transformer(nn.Module): + def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, device=None): + super(Transformer, self).__init__() + self.rnn = nn.GRU( + input_size=d_feat, + hidden_size=d_model, + num_layers=num_layers, + batch_first=True, + dropout=dropout, + ) + self.feature_layer = nn.Linear(d_feat, d_model) + self.pos_encoder = PositionalEncoding(d_model) + self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dropout=dropout) + self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers) + self.decoder_layer = nn.Linear(d_model, 1) + self.device = device + self.d_feat = d_feat + + def forward(self, src): + # pdb.set_trace() + # src [N, T, F], [512, 60, 6] + + # out, _ = self.rnn(src) + src = self.feature_layer(src) # [512, 60, 8] + + # src [N, T, F] --> [T, N, F], [60, 512, 8] + src = src.transpose(1, 0) # not batch first + + mask = None + + src = self.pos_encoder(src) + output = self.transformer_encoder(src, mask) # [60, 512, 8] + + # [T, N, F] --> [N, T*F] + output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] + + return output.squeeze() + From b13c140988e1d0f852048075a20fa646ec68682b Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 15:49:17 +0800 Subject: [PATCH 02/17] Update pytorch_localformer.py Have passed black. --- qlib/contrib/model/pytorch_localformer.py | 43 +++-------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py index f085bd4b21..bdf77b5be4 100644 --- a/qlib/contrib/model/pytorch_localformer.py +++ b/qlib/contrib/model/pytorch_localformer.py @@ -24,32 +24,6 @@ from ...data.dataset.handler import DataHandlerLP from torch.nn.modules.container import ModuleList -import pdb - -# qrun benchmarks/Transformer/workflow_config_localformer_Alpha158.yaml -# 0.992366, @13, -''' -{'IC': 0.037426503365732174, - 'ICIR': 0.28977883455541603, - 'Rank IC': 0.04659889541774283, - 'Rank ICIR': 0.373569340092482} - -'The following are analysis results of the excess return without cost.' - risk -mean 0.000381 -std 0.004109 -annualized_return 0.096066 -information_ratio 1.472729 -max_drawdown -0.094917 -'The following are analysis results of the excess return with cost.' - risk -mean 0.000213 -std 0.004111 -annualized_return 0.053630 -information_ratio 0.821711 -max_drawdown -0.113694 -''' - class LocalformerModel(Model): def __init__( @@ -88,11 +62,8 @@ def __init__( self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") self.seed = seed self.logger = get_module_logger("TransformerModel") - print('do we have gpu?{}'.format(torch.cuda.is_available())) self.logger.info( - "Improved Transformer:" - "\nbatch_size : {}" - "\ndevice : {}".format(self.batch_size, self.device) + "Improved Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device) ) if self.seed is not None: @@ -161,7 +132,6 @@ def test_epoch(self, data_loader): for data in data_loader: feature = data[:, :, 0:-1].to(self.device) - # feature[torch.isnan(feature)] = 0 label = data[:, -1, -1].to(self.device) with torch.no_grad(): @@ -266,11 +236,11 @@ def __init__(self, d_model, max_len=1000): pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) - self.register_buffer('pe', pe) + self.register_buffer("pe", pe) def forward(self, x): # [T, N, F] - return x + self.pe[:x.size(0), :] + return x + self.pe[: x.size(0), :] def _get_clones(module, N): @@ -278,7 +248,7 @@ def _get_clones(module, N): class LocalformerEncoder(nn.Module): - __constants__ = ['norm'] + __constants__ = ["norm"] def __init__(self, encoder_layer, num_layers, d_model): super(LocalformerEncoder, self).__init__() @@ -295,7 +265,7 @@ def forward(self, src, mask): out = output.transpose(1, 0).transpose(2, 1) out = self.conv[i](out).transpose(2, 1).transpose(1, 0) - output = mod(output+out, src_mask=mask) + output = mod(output + out, src_mask=mask) return output + out @@ -319,9 +289,7 @@ def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, devi self.d_feat = d_feat def forward(self, src): - # pdb.set_trace() # src [N, T, F], [512, 60, 6] - src = self.feature_layer(src) # [512, 60, 8] # src [N, T, F] --> [T, N, F], [60, 512, 8] @@ -338,4 +306,3 @@ def forward(self, src): output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] return output.squeeze() - From d6215c98d63d259610ca02956772a79a1889b73e Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 15:50:33 +0800 Subject: [PATCH 03/17] Update pytorch_transformer.py Have passed black --- qlib/contrib/model/pytorch_transformer.py | 49 ++--------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py index 85582be1f1..c53564903a 100644 --- a/qlib/contrib/model/pytorch_transformer.py +++ b/qlib/contrib/model/pytorch_transformer.py @@ -23,32 +23,6 @@ from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP -import pdb - -# qrun benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml -# 0.993681, @11, -''' - 'IC': 0.03186587768611013, - 'ICIR': 0.2556910881045764, - 'Rank IC': 0.04735251936658551, - 'Rank ICIR': 0.388378955424602 - -'The following are analysis results of the excess return without cost.' - risk -mean 0.000309 -std 0.004209 -annualized_return 0.077839 -information_ratio 1.164993 -max_drawdown -0.106215 -'The following are analysis results of the excess return with cost.' - risk -mean 0.000126 -std 0.004209 -annualized_return 0.031707 -information_ratio 0.474567 -max_drawdown -0.131948 -''' - class TransformerModel(Model): def __init__( @@ -87,12 +61,7 @@ def __init__( self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") self.seed = seed self.logger = get_module_logger("TransformerModel") - print('do we have gpu?{}'.format(torch.cuda.is_available())) - self.logger.info( - "Naive Transformer:" - "\nbatch_size : {}" - "\ndevice : {}".format(self.batch_size, self.device) - ) + self.logger.info("Naive Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device)) if self.seed is not None: np.random.seed(self.seed) @@ -160,7 +129,6 @@ def test_epoch(self, data_loader): for data in data_loader: feature = data[:, :, 0:-1].to(self.device) - # feature[torch.isnan(feature)] = 0 label = data[:, -1, -1].to(self.device) with torch.no_grad(): @@ -265,23 +233,16 @@ def __init__(self, d_model, max_len=1000): pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) - self.register_buffer('pe', pe) + self.register_buffer("pe", pe) def forward(self, x): # [T, N, F] - return x + self.pe[:x.size(0), :] + return x + self.pe[: x.size(0), :] class Transformer(nn.Module): def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, device=None): super(Transformer, self).__init__() - self.rnn = nn.GRU( - input_size=d_feat, - hidden_size=d_model, - num_layers=num_layers, - batch_first=True, - dropout=dropout, - ) self.feature_layer = nn.Linear(d_feat, d_model) self.pos_encoder = PositionalEncoding(d_model) self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dropout=dropout) @@ -291,10 +252,7 @@ def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, devi self.d_feat = d_feat def forward(self, src): - # pdb.set_trace() # src [N, T, F], [512, 60, 6] - - # out, _ = self.rnn(src) src = self.feature_layer(src) # [512, 60, 8] # src [N, T, F] --> [T, N, F], [60, 512, 8] @@ -309,4 +267,3 @@ def forward(self, src): output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] return output.squeeze() - From 9eb333b18e042c37133ef7d57f7094d5b10576a9 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 16:11:51 +0800 Subject: [PATCH 04/17] Add files via upload --- .../workflow_config_localformer_Alpha158.yaml | 82 +++++++++++++++++++ .../workflow_config_localformer_Alpha360.yaml | 82 +++++++++++++++++++ .../workflow_config_transformer_Alpha158.yaml | 82 +++++++++++++++++++ .../workflow_config_transformer_Alpha360.yaml | 82 +++++++++++++++++++ 4 files changed, 328 insertions(+) create mode 100644 examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml create mode 100644 examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml create mode 100644 examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml create mode 100644 examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml diff --git a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml new file mode 100644 index 0000000000..98090356ea --- /dev/null +++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml @@ -0,0 +1,82 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: FilterCol + kwargs: + fields_group: feature + col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", + "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", + "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" + ] + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] + +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: LocalformerModel + module_path: qlib.contrib.model.pytorch_localformer + kwargs: + seed: 0 + n_jobs: 20 + dataset: + class: TSDatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha158 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + step_len: 20 + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml new file mode 100644 index 0000000000..9792a23570 --- /dev/null +++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml @@ -0,0 +1,82 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: FilterCol + kwargs: + fields_group: feature + col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", + "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", + "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" + ] + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] + +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: LocalformerModel + module_path: qlib.contrib.model.pytorch_localformer + kwargs: + seed: 0 + n_jobs: 20 + dataset: + class: TSDatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + step_len: 20 + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml new file mode 100644 index 0000000000..e58c205419 --- /dev/null +++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml @@ -0,0 +1,82 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: FilterCol + kwargs: + fields_group: feature + col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", + "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", + "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" + ] + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] + +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TransformerModel + module_path: qlib.contrib.model.pytorch_transformer + kwargs: + seed: 0 + n_jobs: 20 + dataset: + class: TSDatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha158 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + step_len: 20 + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml new file mode 100644 index 0000000000..59d5fa2b95 --- /dev/null +++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml @@ -0,0 +1,82 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: FilterCol + kwargs: + fields_group: feature + col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", + "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", + "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" + ] + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] + +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TransformerModel + module_path: qlib.contrib.model.pytorch_transformer + kwargs: + seed: 0 + n_jobs: 20 + dataset: + class: TSDatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + step_len: 20 + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config From 71a1d1df67b95f154cab7da2f78f47c2fad7417c Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 16:18:35 +0800 Subject: [PATCH 05/17] Add files via upload --- examples/benchmarks/Localformer/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 examples/benchmarks/Localformer/requirements.txt diff --git a/examples/benchmarks/Localformer/requirements.txt b/examples/benchmarks/Localformer/requirements.txt new file mode 100644 index 0000000000..d5b918797e --- /dev/null +++ b/examples/benchmarks/Localformer/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.17.4 +pandas==1.1.2 +torch==1.2.0 \ No newline at end of file From 2610e4b2791e718df1b6771daf21a7bbc0ef15b2 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 16:20:22 +0800 Subject: [PATCH 06/17] Add files via upload From e398dfb41a8eafa354346f475ae701154e0d938c Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Wed, 14 Jul 2021 16:24:01 +0800 Subject: [PATCH 07/17] Add files via upload --- examples/benchmarks/Transformer/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 examples/benchmarks/Transformer/requirements.txt diff --git a/examples/benchmarks/Transformer/requirements.txt b/examples/benchmarks/Transformer/requirements.txt new file mode 100644 index 0000000000..d5b918797e --- /dev/null +++ b/examples/benchmarks/Transformer/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.17.4 +pandas==1.1.2 +torch==1.2.0 \ No newline at end of file From 664e48f6dbb3a3493de243938fc892f6ecd7df78 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Fri, 16 Jul 2021 15:05:32 +0800 Subject: [PATCH 08/17] Update pytorch_localformer.py --- qlib/contrib/model/pytorch_localformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py index bdf77b5be4..683a9bd4f3 100644 --- a/qlib/contrib/model/pytorch_localformer.py +++ b/qlib/contrib/model/pytorch_localformer.py @@ -42,7 +42,7 @@ def __init__( optimizer="adam", reg=1e-3, n_jobs=10, - GPU=2, + GPU=0, seed=None, **kwargs ): From c797b48895fc2456f0faa42935f118cfc5809c5d Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Fri, 16 Jul 2021 18:33:11 +0800 Subject: [PATCH 09/17] Add files via upload --- qlib/contrib/model/pytorch_localformer.py | 100 +++--- qlib/contrib/model/pytorch_localformer_ts.py | 310 +++++++++++++++++++ qlib/contrib/model/pytorch_transformer.py | 96 +++--- qlib/contrib/model/pytorch_transformer_ts.py | 269 ++++++++++++++++ 4 files changed, 700 insertions(+), 75 deletions(-) create mode 100644 qlib/contrib/model/pytorch_localformer_ts.py create mode 100644 qlib/contrib/model/pytorch_transformer_ts.py diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py index 683a9bd4f3..1b722ead2f 100644 --- a/qlib/contrib/model/pytorch_localformer.py +++ b/qlib/contrib/model/pytorch_localformer.py @@ -8,6 +8,7 @@ import os import numpy as np import pandas as pd +from typing import Text, Union import copy import math from ...utils import get_or_create_path @@ -23,6 +24,7 @@ from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP from torch.nn.modules.container import ModuleList +# qrun examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml ” class LocalformerModel(Model): @@ -30,7 +32,7 @@ def __init__( self, d_feat: int = 20, d_model: int = 64, - batch_size: int = 8192, + batch_size: int = 2048, nhead: int = 2, num_layers: int = 2, dropout: float = 0, @@ -62,9 +64,7 @@ def __init__( self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") self.seed = seed self.logger = get_module_logger("TransformerModel") - self.logger.info( - "Improved Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device) - ) + self.logger.info("Naive Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device)) if self.seed is not None: np.random.seed(self.seed) @@ -106,15 +106,25 @@ def metric_fn(self, pred, label): raise ValueError("unknown metric `%s`" % self.metric) - def train_epoch(self, data_loader): + def train_epoch(self, x_train, y_train): + + x_train_values = x_train.values + y_train_values = np.squeeze(y_train.values) self.model.train() - for data in data_loader: - feature = data[:, :, 0:-1].to(self.device) - label = data[:, -1, -1].to(self.device) + indices = np.arange(len(x_train_values)) + np.random.shuffle(indices) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break - pred = self.model(feature.float()) # .float() + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.model(feature) loss = self.loss_fn(pred, label) self.train_optimizer.zero_grad() @@ -122,20 +132,29 @@ def train_epoch(self, data_loader): torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) self.train_optimizer.step() - def test_epoch(self, data_loader): + def test_epoch(self, data_x, data_y): + + # prepare training data + x_values = data_x.values + y_values = np.squeeze(data_y.values) self.model.eval() scores = [] losses = [] - for data in data_loader: + indices = np.arange(len(x_values)) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break - feature = data[:, :, 0:-1].to(self.device) - label = data[:, -1, -1].to(self.device) + feature = torch.from_numpy(x_values[indices[i: i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i: i + self.batch_size]]).float().to(self.device) with torch.no_grad(): - pred = self.model(feature.float()) # .float() + pred = self.model(feature) loss = self.loss_fn(pred, label) losses.append(loss.item()) @@ -151,21 +170,16 @@ def fit( save_path=None, ): - dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) - dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) - - dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader - dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader - - train_loader = DataLoader( - dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True - ) - valid_loader = DataLoader( - dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, ) - save_path = get_or_create_path(save_path) + x_train, y_train = df_train["feature"], df_train["label"] + x_valid, y_valid = df_valid["feature"], df_valid["label"] + save_path = get_or_create_path(save_path) stop_steps = 0 train_loss = 0 best_score = -np.inf @@ -180,10 +194,10 @@ def fit( for step in range(self.n_epochs): self.logger.info("Epoch%d:", step) self.logger.info("training...") - self.train_epoch(train_loader) + self.train_epoch(x_train, y_train) self.logger.info("evaluating...") - train_loss, train_score = self.test_epoch(train_loader) - val_loss, val_score = self.test_epoch(valid_loader) + train_loss, train_score = self.test_epoch(x_train, y_train) + val_loss, val_score = self.test_epoch(x_valid, y_valid) self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) evals_result["train"].append(train_score) evals_result["valid"].append(val_score) @@ -206,25 +220,32 @@ def fit( if self.use_gpu: torch.cuda.empty_cache() - def predict(self, dataset): + def predict(self, dataset: DatasetH, segment: Union[Text, slice] = "test"): if not self.fitted: raise ValueError("model is not fitted yet!") - dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) - dl_test.config(fillna_type="ffill+bfill") - test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + x_test = dataset.prepare(segment, col_set="feature", data_key=DataHandlerLP.DK_I) + index = x_test.index self.model.eval() + x_values = x_test.values + sample_num = x_values.shape[0] preds = [] - for data in test_loader: - feature = data[:, :, 0:-1].to(self.device) + for begin in range(sample_num)[:: self.batch_size]: + + if sample_num - begin < self.batch_size: + end = sample_num + else: + end = begin + self.batch_size + + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) with torch.no_grad(): - pred = self.model(feature.float()).detach().cpu().numpy() + pred = self.model(x_batch).detach().cpu().numpy() preds.append(pred) - return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + return pd.Series(np.concatenate(preds), index=index) class PositionalEncoding(nn.Module): @@ -289,8 +310,9 @@ def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, devi self.d_feat = d_feat def forward(self, src): - # src [N, T, F], [512, 60, 6] - src = self.feature_layer(src) # [512, 60, 8] + # src [N, F*T] --> [N, T, F] + src = src.reshape(len(src), self.d_feat, -1).permute(0, 2, 1) + src = self.feature_layer(src) # src [N, T, F] --> [T, N, F], [60, 512, 8] src = src.transpose(1, 0) # not batch first diff --git a/qlib/contrib/model/pytorch_localformer_ts.py b/qlib/contrib/model/pytorch_localformer_ts.py new file mode 100644 index 0000000000..aa7af84df2 --- /dev/null +++ b/qlib/contrib/model/pytorch_localformer_ts.py @@ -0,0 +1,310 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +import math +from ...utils import get_or_create_path +from ...log import get_module_logger + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +from .pytorch_utils import count_parameters +from ...model.base import Model +from ...data.dataset import DatasetH, TSDatasetH +from ...data.dataset.handler import DataHandlerLP +from torch.nn.modules.container import ModuleList + + +class LocalformerModel(Model): + def __init__( + self, + d_feat: int = 20, + d_model: int = 64, + batch_size: int = 8192, + nhead: int = 2, + num_layers: int = 2, + dropout: float = 0, + n_epochs=100, + lr=0.0001, + metric="", + early_stop=5, + loss="mse", + optimizer="adam", + reg=1e-3, + n_jobs=10, + GPU=2, + seed=None, + **kwargs + ): + + # set hyper-parameters. + self.d_model = d_model + self.dropout = dropout + self.n_epochs = n_epochs + self.lr = lr + self.reg = reg + self.metric = metric + self.batch_size = batch_size + self.early_stop = early_stop + self.optimizer = optimizer.lower() + self.loss = loss + self.n_jobs = n_jobs + self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") + self.seed = seed + self.logger = get_module_logger("TransformerModel") + self.logger.info( + "Improved Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.model = Transformer(d_feat, d_model, nhead, num_layers, dropout, self.device) + if optimizer.lower() == "adam": + self.train_optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + elif optimizer.lower() == "gd": + self.train_optimizer = optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + else: + raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) + + self.fitted = False + self.model.to(self.device) + + @property + def use_gpu(self): + return self.device != torch.device("cpu") + + def mse(self, pred, label): + loss = (pred.float() - label.float()) ** 2 + return torch.mean(loss) + + def loss_fn(self, pred, label): + mask = ~torch.isnan(label) + + if self.loss == "mse": + return self.mse(pred[mask], label[mask]) + + raise ValueError("unknown loss `%s`" % self.loss) + + def metric_fn(self, pred, label): + + mask = torch.isfinite(label) + + if self.metric == "" or self.metric == "loss": + return -self.loss_fn(pred[mask], label[mask]) + + raise ValueError("unknown metric `%s`" % self.metric) + + def train_epoch(self, data_loader): + + self.model.train() + + for data in data_loader: + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + + self.train_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) + self.train_optimizer.step() + + def test_epoch(self, data_loader): + + self.model.eval() + + scores = [] + losses = [] + + for data in data_loader: + + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + losses.append(loss.item()) + + score = self.metric_fn(pred, label) + scores.append(score.item()) + + return np.mean(losses), np.mean(scores) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + save_path=None, + ): + + dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + import pdb + pdb.set_trace() + + dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader + dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader + + train_loader = DataLoader( + dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True + ) + valid_loader = DataLoader( + dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + ) + + save_path = get_or_create_path(save_path) + + stop_steps = 0 + train_loss = 0 + best_score = -np.inf + best_epoch = 0 + evals_result["train"] = [] + evals_result["valid"] = [] + + # train + self.logger.info("training...") + self.fitted = True + + for step in range(self.n_epochs): + self.logger.info("Epoch%d:", step) + self.logger.info("training...") + self.train_epoch(train_loader) + self.logger.info("evaluating...") + train_loss, train_score = self.test_epoch(train_loader) + val_loss, val_score = self.test_epoch(valid_loader) + self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) + evals_result["train"].append(train_score) + evals_result["valid"].append(val_score) + + if val_score > best_score: + best_score = val_score + stop_steps = 0 + best_epoch = step + best_param = copy.deepcopy(self.model.state_dict()) + else: + stop_steps += 1 + if stop_steps >= self.early_stop: + self.logger.info("early stop") + break + + self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch)) + self.model.load_state_dict(best_param) + torch.save(best_param, save_path) + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) + dl_test.config(fillna_type="ffill+bfill") + test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + self.model.eval() + preds = [] + + for data in test_loader: + feature = data[:, :, 0:-1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()).detach().cpu().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=1000): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + self.register_buffer("pe", pe) + + def forward(self, x): + # [T, N, F] + return x + self.pe[: x.size(0), :] + + +def _get_clones(module, N): + return ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class LocalformerEncoder(nn.Module): + __constants__ = ["norm"] + + def __init__(self, encoder_layer, num_layers, d_model): + super(LocalformerEncoder, self).__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.conv = _get_clones(nn.Conv1d(d_model, d_model, 3, 1, 1), num_layers) + self.num_layers = num_layers + + def forward(self, src, mask): + output = src + out = src + + for i, mod in enumerate(self.layers): + # [T, N, F] --> [N, T, F] --> [N, F, T] + out = output.transpose(1, 0).transpose(2, 1) + out = self.conv[i](out).transpose(2, 1).transpose(1, 0) + + output = mod(output + out, src_mask=mask) + + return output + out + + +class Transformer(nn.Module): + def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, device=None): + super(Transformer, self).__init__() + self.rnn = nn.GRU( + input_size=d_model, + hidden_size=d_model, + num_layers=num_layers, + batch_first=False, + dropout=dropout, + ) + self.feature_layer = nn.Linear(d_feat, d_model) + self.pos_encoder = PositionalEncoding(d_model) + self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dropout=dropout) + self.transformer_encoder = LocalformerEncoder(self.encoder_layer, num_layers=num_layers, d_model=d_model) + self.decoder_layer = nn.Linear(d_model, 1) + self.device = device + self.d_feat = d_feat + + def forward(self, src): + # src [N, T, F], [512, 60, 6] + src = self.feature_layer(src) # [512, 60, 8] + + # src [N, T, F] --> [T, N, F], [60, 512, 8] + src = src.transpose(1, 0) # not batch first + + mask = None + + src = self.pos_encoder(src) + output = self.transformer_encoder(src, mask) # [60, 512, 8] + + output, _ = self.rnn(output) + + # [T, N, F] --> [N, T*F] + output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] + + return output.squeeze() diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py index c53564903a..cca7a78719 100644 --- a/qlib/contrib/model/pytorch_transformer.py +++ b/qlib/contrib/model/pytorch_transformer.py @@ -8,6 +8,7 @@ import os import numpy as np import pandas as pd +from typing import Text, Union import copy import math from ...utils import get_or_create_path @@ -22,6 +23,7 @@ from ...model.base import Model from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP +# qrun examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml ” class TransformerModel(Model): @@ -29,7 +31,7 @@ def __init__( self, d_feat: int = 20, d_model: int = 64, - batch_size: int = 8192, + batch_size: int = 2048, nhead: int = 2, num_layers: int = 2, dropout: float = 0, @@ -103,15 +105,25 @@ def metric_fn(self, pred, label): raise ValueError("unknown metric `%s`" % self.metric) - def train_epoch(self, data_loader): + def train_epoch(self, x_train, y_train): + + x_train_values = x_train.values + y_train_values = np.squeeze(y_train.values) self.model.train() - for data in data_loader: - feature = data[:, :, 0:-1].to(self.device) - label = data[:, -1, -1].to(self.device) + indices = np.arange(len(x_train_values)) + np.random.shuffle(indices) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) - pred = self.model(feature.float()) # .float() + pred = self.model(feature) loss = self.loss_fn(pred, label) self.train_optimizer.zero_grad() @@ -119,20 +131,29 @@ def train_epoch(self, data_loader): torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) self.train_optimizer.step() - def test_epoch(self, data_loader): + def test_epoch(self, data_x, data_y): + + # prepare training data + x_values = data_x.values + y_values = np.squeeze(data_y.values) self.model.eval() scores = [] losses = [] - for data in data_loader: + indices = np.arange(len(x_values)) + + for i in range(len(indices))[:: self.batch_size]: - feature = data[:, :, 0:-1].to(self.device) - label = data[:, -1, -1].to(self.device) + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_values[indices[i: i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i: i + self.batch_size]]).float().to(self.device) with torch.no_grad(): - pred = self.model(feature.float()) # .float() + pred = self.model(feature) loss = self.loss_fn(pred, label) losses.append(loss.item()) @@ -148,21 +169,16 @@ def fit( save_path=None, ): - dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) - dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) - - dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader - dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader - - train_loader = DataLoader( - dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True - ) - valid_loader = DataLoader( - dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, ) - save_path = get_or_create_path(save_path) + x_train, y_train = df_train["feature"], df_train["label"] + x_valid, y_valid = df_valid["feature"], df_valid["label"] + save_path = get_or_create_path(save_path) stop_steps = 0 train_loss = 0 best_score = -np.inf @@ -177,10 +193,10 @@ def fit( for step in range(self.n_epochs): self.logger.info("Epoch%d:", step) self.logger.info("training...") - self.train_epoch(train_loader) + self.train_epoch(x_train, y_train) self.logger.info("evaluating...") - train_loss, train_score = self.test_epoch(train_loader) - val_loss, val_score = self.test_epoch(valid_loader) + train_loss, train_score = self.test_epoch(x_train, y_train) + val_loss, val_score = self.test_epoch(x_valid, y_valid) self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) evals_result["train"].append(train_score) evals_result["valid"].append(val_score) @@ -203,25 +219,32 @@ def fit( if self.use_gpu: torch.cuda.empty_cache() - def predict(self, dataset): + def predict(self, dataset: DatasetH, segment: Union[Text, slice] = "test"): if not self.fitted: raise ValueError("model is not fitted yet!") - dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) - dl_test.config(fillna_type="ffill+bfill") - test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + x_test = dataset.prepare(segment, col_set="feature", data_key=DataHandlerLP.DK_I) + index = x_test.index self.model.eval() + x_values = x_test.values + sample_num = x_values.shape[0] preds = [] - for data in test_loader: - feature = data[:, :, 0:-1].to(self.device) + for begin in range(sample_num)[:: self.batch_size]: + + if sample_num - begin < self.batch_size: + end = sample_num + else: + end = begin + self.batch_size + + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) with torch.no_grad(): - pred = self.model(feature.float()).detach().cpu().numpy() + pred = self.model(x_batch).detach().cpu().numpy() preds.append(pred) - return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + return pd.Series(np.concatenate(preds), index=index) class PositionalEncoding(nn.Module): @@ -252,8 +275,9 @@ def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, devi self.d_feat = d_feat def forward(self, src): - # src [N, T, F], [512, 60, 6] - src = self.feature_layer(src) # [512, 60, 8] + # src [N, F*T] --> [N, T, F] + src = src.reshape(len(src), self.d_feat, -1).permute(0, 2, 1) + src = self.feature_layer(src) # src [N, T, F] --> [T, N, F], [60, 512, 8] src = src.transpose(1, 0) # not batch first diff --git a/qlib/contrib/model/pytorch_transformer_ts.py b/qlib/contrib/model/pytorch_transformer_ts.py new file mode 100644 index 0000000000..c53564903a --- /dev/null +++ b/qlib/contrib/model/pytorch_transformer_ts.py @@ -0,0 +1,269 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +import math +from ...utils import get_or_create_path +from ...log import get_module_logger + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +from .pytorch_utils import count_parameters +from ...model.base import Model +from ...data.dataset import DatasetH, TSDatasetH +from ...data.dataset.handler import DataHandlerLP + + +class TransformerModel(Model): + def __init__( + self, + d_feat: int = 20, + d_model: int = 64, + batch_size: int = 8192, + nhead: int = 2, + num_layers: int = 2, + dropout: float = 0, + n_epochs=100, + lr=0.0001, + metric="", + early_stop=5, + loss="mse", + optimizer="adam", + reg=1e-3, + n_jobs=10, + GPU=0, + seed=None, + **kwargs + ): + + # set hyper-parameters. + self.d_model = d_model + self.dropout = dropout + self.n_epochs = n_epochs + self.lr = lr + self.reg = reg + self.metric = metric + self.batch_size = batch_size + self.early_stop = early_stop + self.optimizer = optimizer.lower() + self.loss = loss + self.n_jobs = n_jobs + self.device = torch.device("cuda:%d" % GPU if torch.cuda.is_available() and GPU >= 0 else "cpu") + self.seed = seed + self.logger = get_module_logger("TransformerModel") + self.logger.info("Naive Transformer:" "\nbatch_size : {}" "\ndevice : {}".format(self.batch_size, self.device)) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.model = Transformer(d_feat, d_model, nhead, num_layers, dropout, self.device) + if optimizer.lower() == "adam": + self.train_optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + elif optimizer.lower() == "gd": + self.train_optimizer = optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.reg) + else: + raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) + + self.fitted = False + self.model.to(self.device) + + @property + def use_gpu(self): + return self.device != torch.device("cpu") + + def mse(self, pred, label): + loss = (pred.float() - label.float()) ** 2 + return torch.mean(loss) + + def loss_fn(self, pred, label): + mask = ~torch.isnan(label) + + if self.loss == "mse": + return self.mse(pred[mask], label[mask]) + + raise ValueError("unknown loss `%s`" % self.loss) + + def metric_fn(self, pred, label): + + mask = torch.isfinite(label) + + if self.metric == "" or self.metric == "loss": + return -self.loss_fn(pred[mask], label[mask]) + + raise ValueError("unknown metric `%s`" % self.metric) + + def train_epoch(self, data_loader): + + self.model.train() + + for data in data_loader: + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + + self.train_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.model.parameters(), 3.0) + self.train_optimizer.step() + + def test_epoch(self, data_loader): + + self.model.eval() + + scores = [] + losses = [] + + for data in data_loader: + + feature = data[:, :, 0:-1].to(self.device) + label = data[:, -1, -1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()) # .float() + loss = self.loss_fn(pred, label) + losses.append(loss.item()) + + score = self.metric_fn(pred, label) + scores.append(score.item()) + + return np.mean(losses), np.mean(scores) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + save_path=None, + ): + + dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + + dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader + dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader + + train_loader = DataLoader( + dl_train, batch_size=self.batch_size, shuffle=True, num_workers=self.n_jobs, drop_last=True + ) + valid_loader = DataLoader( + dl_valid, batch_size=self.batch_size, shuffle=False, num_workers=self.n_jobs, drop_last=True + ) + + save_path = get_or_create_path(save_path) + + stop_steps = 0 + train_loss = 0 + best_score = -np.inf + best_epoch = 0 + evals_result["train"] = [] + evals_result["valid"] = [] + + # train + self.logger.info("training...") + self.fitted = True + + for step in range(self.n_epochs): + self.logger.info("Epoch%d:", step) + self.logger.info("training...") + self.train_epoch(train_loader) + self.logger.info("evaluating...") + train_loss, train_score = self.test_epoch(train_loader) + val_loss, val_score = self.test_epoch(valid_loader) + self.logger.info("train %.6f, valid %.6f" % (train_score, val_score)) + evals_result["train"].append(train_score) + evals_result["valid"].append(val_score) + + if val_score > best_score: + best_score = val_score + stop_steps = 0 + best_epoch = step + best_param = copy.deepcopy(self.model.state_dict()) + else: + stop_steps += 1 + if stop_steps >= self.early_stop: + self.logger.info("early stop") + break + + self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch)) + self.model.load_state_dict(best_param) + torch.save(best_param, save_path) + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I) + dl_test.config(fillna_type="ffill+bfill") + test_loader = DataLoader(dl_test, batch_size=self.batch_size, num_workers=self.n_jobs) + self.model.eval() + preds = [] + + for data in test_loader: + feature = data[:, :, 0:-1].to(self.device) + + with torch.no_grad(): + pred = self.model(feature.float()).detach().cpu().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=dl_test.get_index()) + + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, max_len=1000): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + self.register_buffer("pe", pe) + + def forward(self, x): + # [T, N, F] + return x + self.pe[: x.size(0), :] + + +class Transformer(nn.Module): + def __init__(self, d_feat=6, d_model=8, nhead=4, num_layers=2, dropout=0.5, device=None): + super(Transformer, self).__init__() + self.feature_layer = nn.Linear(d_feat, d_model) + self.pos_encoder = PositionalEncoding(d_model) + self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dropout=dropout) + self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers) + self.decoder_layer = nn.Linear(d_model, 1) + self.device = device + self.d_feat = d_feat + + def forward(self, src): + # src [N, T, F], [512, 60, 6] + src = self.feature_layer(src) # [512, 60, 8] + + # src [N, T, F] --> [T, N, F], [60, 512, 8] + src = src.transpose(1, 0) # not batch first + + mask = None + + src = self.pos_encoder(src) + output = self.transformer_encoder(src, mask) # [60, 512, 8] + + # [T, N, F] --> [N, T*F] + output = self.decoder_layer(output.transpose(1, 0)[:, -1, :]) # [512, 1] + + return output.squeeze() From 549a645d58af7eef30e8be2f25846a48788d433e Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Fri, 16 Jul 2021 18:35:00 +0800 Subject: [PATCH 10/17] Add files via upload --- .../workflow_config_transformer_Alpha158.yaml | 2 +- .../workflow_config_transformer_Alpha360.yaml | 17 ++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml index e58c205419..54707386f9 100644 --- a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml +++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml @@ -50,7 +50,7 @@ port_analysis_config: &port_analysis_config task: model: class: TransformerModel - module_path: qlib.contrib.model.pytorch_transformer + module_path: qlib.contrib.model.pytorch_transformer_ts kwargs: seed: 0 n_jobs: 20 diff --git a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml index 59d5fa2b95..e568a1b307 100644 --- a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml +++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml @@ -10,13 +10,6 @@ data_handler_config: &data_handler_config fit_end_time: 2014-12-31 instruments: *market infer_processors: - - class: FilterCol - kwargs: - fields_group: feature - col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", - "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", - "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" - ] - class: RobustZScoreNorm kwargs: fields_group: feature @@ -29,8 +22,7 @@ data_handler_config: &data_handler_config - class: CSRankNorm kwargs: fields_group: label - label: ["Ref($close, -2) / Ref($close, -1) - 1"] - + label: ["Ref($close, -2) / Ref($close, -1) - 1"] port_analysis_config: &port_analysis_config strategy: class: TopkDropoutStrategy @@ -52,10 +44,10 @@ task: class: TransformerModel module_path: qlib.contrib.model.pytorch_transformer kwargs: + d_feat: 6 seed: 0 - n_jobs: 20 dataset: - class: TSDatasetH + class: DatasetH module_path: qlib.data.dataset kwargs: handler: @@ -66,7 +58,6 @@ task: train: [2008-01-01, 2014-12-31] valid: [2015-01-01, 2016-12-31] test: [2017-01-01, 2020-08-01] - step_len: 20 record: - class: SignalRecord module_path: qlib.workflow.record_temp @@ -79,4 +70,4 @@ task: - class: PortAnaRecord module_path: qlib.workflow.record_temp kwargs: - config: *port_analysis_config + config: *port_analysis_config \ No newline at end of file From 65ae97694d8b2ec09963297d9cc2ef440b738473 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Fri, 16 Jul 2021 18:35:50 +0800 Subject: [PATCH 11/17] Add files via upload --- .../workflow_config_localformer_Alpha158.yaml | 2 +- .../workflow_config_localformer_Alpha360.yaml | 17 ++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml index 98090356ea..d7e9673333 100644 --- a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml +++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml @@ -50,7 +50,7 @@ port_analysis_config: &port_analysis_config task: model: class: LocalformerModel - module_path: qlib.contrib.model.pytorch_localformer + module_path: qlib.contrib.model.pytorch_localformer_ts kwargs: seed: 0 n_jobs: 20 diff --git a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml index 9792a23570..1c8489461c 100644 --- a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml +++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml @@ -10,13 +10,6 @@ data_handler_config: &data_handler_config fit_end_time: 2014-12-31 instruments: *market infer_processors: - - class: FilterCol - kwargs: - fields_group: feature - col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", - "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", - "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW" - ] - class: RobustZScoreNorm kwargs: fields_group: feature @@ -29,8 +22,7 @@ data_handler_config: &data_handler_config - class: CSRankNorm kwargs: fields_group: label - label: ["Ref($close, -2) / Ref($close, -1) - 1"] - + label: ["Ref($close, -2) / Ref($close, -1) - 1"] port_analysis_config: &port_analysis_config strategy: class: TopkDropoutStrategy @@ -52,10 +44,10 @@ task: class: LocalformerModel module_path: qlib.contrib.model.pytorch_localformer kwargs: + d_feat: 6 seed: 0 - n_jobs: 20 dataset: - class: TSDatasetH + class: DatasetH module_path: qlib.data.dataset kwargs: handler: @@ -66,7 +58,6 @@ task: train: [2008-01-01, 2014-12-31] valid: [2015-01-01, 2016-12-31] test: [2017-01-01, 2020-08-01] - step_len: 20 record: - class: SignalRecord module_path: qlib.workflow.record_temp @@ -79,4 +70,4 @@ task: - class: PortAnaRecord module_path: qlib.workflow.record_temp kwargs: - config: *port_analysis_config + config: *port_analysis_config \ No newline at end of file From 516a825ddfdbe233e90f4ba842707bfd7a44f57b Mon Sep 17 00:00:00 2001 From: you-n-g Date: Sun, 18 Jul 2021 12:07:56 +0800 Subject: [PATCH 12/17] Update setup.py From 3bc8c96bac458e1df7a27d3978d987df0dd91c3b Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 18 Jul 2021 12:09:57 +0800 Subject: [PATCH 13/17] update `run_all_model` and black format --- examples/run_all_model.py | 44 +++++++++++++++----- qlib/contrib/model/pytorch_localformer.py | 5 ++- qlib/contrib/model/pytorch_localformer_ts.py | 1 + qlib/contrib/model/pytorch_transformer.py | 5 ++- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/examples/run_all_model.py b/examples/run_all_model.py index c79fee004d..1284d8e995 100644 --- a/examples/run_all_model.py +++ b/examples/run_all_model.py @@ -23,7 +23,6 @@ from qlib.workflow import R from qlib.tests.data import GetData - # init qlib provider_uri = "~/.qlib/qlib_data/cn_data" exp_folder_name = "run_all_model_records" @@ -40,6 +39,7 @@ GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN, exp_manager=exp_manager) + # decorator to check the arguments def only_allow_defined_args(function_to_decorate): @functools.wraps(function_to_decorate) @@ -92,7 +92,8 @@ def create_env(): # function to execute the cmd -def execute(cmd): +def execute(cmd, wait_when_err=False): + print("Running CMD:", cmd) with subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True) as p: for line in p.stdout: sys.stdout.write(line.split("\b")[0]) @@ -102,6 +103,8 @@ def execute(cmd): sys.stdout.write("\b" * 10 + "\b".join(line.split("\b")[1:-1])) if p.returncode != 0: + if wait_when_err: + input("Press Enter to Continue") return p.stderr else: return None @@ -184,7 +187,15 @@ def gen_and_save_md_table(metrics, dataset): # function to run the all the models @only_allow_defined_args -def run(times=1, models=None, dataset="Alpha360", exclude=False): +def run( + times=1, + models=None, + dataset="Alpha360", + exclude=False, + qlib_uri: str = "git+https://github.com/microsoft/qlib#egg=pyqlib", + wait_before_rm_env: bool = False, + wait_when_err: bool = False, +): """ Please be aware that this function can only work under Linux. MacOS and Windows will be supported in the future. Any PR to enhance this method is highly welcomed. Besides, this script doesn't support parrallel running the same model @@ -200,6 +211,13 @@ def run(times=1, models=None, dataset="Alpha360", exclude=False): determines whether the model being used is excluded or included. dataset : str determines the dataset to be used for each model. + qlib_uri : str + the uri to install qlib with pip + it could be url on the we or local path + wait_before_rm_env : bool + wait before remove environment. + wait_when_err : bool + wait when errors raised when executing commands Usage: ------- @@ -240,32 +258,36 @@ def run(times=1, models=None, dataset="Alpha360", exclude=False): sys.stderr.write("\n") # install requirements.txt sys.stderr.write("Installing requirements.txt...\n") - execute(f"{python_path} -m pip install -r {req_path}") + execute(f"{python_path} -m pip install -r {req_path}", wait_when_err=wait_when_err) sys.stderr.write("\n") # setup gpu for tft if fn == "TFT": execute( - f"conda install -y --prefix {env_path} anaconda cudatoolkit=10.0 && conda install -y --prefix {env_path} cudnn" + f"conda install -y --prefix {env_path} anaconda cudatoolkit=10.0 && conda install -y --prefix {env_path} cudnn", + wait_when_err=wait_when_err, ) sys.stderr.write("\n") # install qlib sys.stderr.write("Installing qlib...\n") - execute(f"{python_path} -m pip install --upgrade pip") # TODO: FIX ME! - execute(f"{python_path} -m pip install --upgrade cython") # TODO: FIX ME! + execute(f"{python_path} -m pip install --upgrade pip", wait_when_err=wait_when_err) # TODO: FIX ME! + execute(f"{python_path} -m pip install --upgrade cython", wait_when_err=wait_when_err) # TODO: FIX ME! if fn == "TFT": execute( - f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall --ignore-installed PyYAML -e git+https://github.com/microsoft/qlib#egg=pyqlib" + f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall --ignore-installed PyYAML -e {qlib_uri}", + wait_when_err=wait_when_err, ) # TODO: FIX ME! else: execute( - f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall -e git+https://github.com/microsoft/qlib#egg=pyqlib" + f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall -e {qlib_uri}", + wait_when_err=wait_when_err, ) # TODO: FIX ME! sys.stderr.write("\n") # run workflow_by_config for multiple times for i in range(times): sys.stderr.write(f"Running the model: {fn} for iteration {i+1}...\n") errs = execute( - f"{python_path} {env_path / 'src/pyqlib/qlib/workflow/cli.py'} {yaml_path} {fn} {exp_folder_name}" + f"{python_path} {env_path / 'bin' / 'qrun'} {yaml_path} {fn} {exp_folder_name}", + wait_when_err=wait_when_err, ) if errs is not None: _errs = errors.get(fn, {}) @@ -274,6 +296,8 @@ def run(times=1, models=None, dataset="Alpha360", exclude=False): sys.stderr.write("\n") # remove env sys.stderr.write(f"Deleting the environment: {env_path}...\n") + if wait_before_rm_env: + input("Press Enter to Continue") shutil.rmtree(env_path) # getting all results sys.stderr.write(f"Retrieving results...\n") diff --git a/qlib/contrib/model/pytorch_localformer.py b/qlib/contrib/model/pytorch_localformer.py index 1b722ead2f..2ec56067f0 100644 --- a/qlib/contrib/model/pytorch_localformer.py +++ b/qlib/contrib/model/pytorch_localformer.py @@ -24,6 +24,7 @@ from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP from torch.nn.modules.container import ModuleList + # qrun examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml ” @@ -150,8 +151,8 @@ def test_epoch(self, data_x, data_y): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_values[indices[i: i + self.batch_size]]).float().to(self.device) - label = torch.from_numpy(y_values[indices[i: i + self.batch_size]]).float().to(self.device) + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) with torch.no_grad(): pred = self.model(feature) diff --git a/qlib/contrib/model/pytorch_localformer_ts.py b/qlib/contrib/model/pytorch_localformer_ts.py index aa7af84df2..b4999bc9ce 100644 --- a/qlib/contrib/model/pytorch_localformer_ts.py +++ b/qlib/contrib/model/pytorch_localformer_ts.py @@ -154,6 +154,7 @@ def fit( dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) import pdb + pdb.set_trace() dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader diff --git a/qlib/contrib/model/pytorch_transformer.py b/qlib/contrib/model/pytorch_transformer.py index cca7a78719..53ebff3c5a 100644 --- a/qlib/contrib/model/pytorch_transformer.py +++ b/qlib/contrib/model/pytorch_transformer.py @@ -23,6 +23,7 @@ from ...model.base import Model from ...data.dataset import DatasetH, TSDatasetH from ...data.dataset.handler import DataHandlerLP + # qrun examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml ” @@ -149,8 +150,8 @@ def test_epoch(self, data_x, data_y): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_values[indices[i: i + self.batch_size]]).float().to(self.device) - label = torch.from_numpy(y_values[indices[i: i + self.batch_size]]).float().to(self.device) + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) with torch.no_grad(): pred = self.model(feature) From fd9e0051fc830cd3a1b7cecc9e66f58e53aa2340 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Sun, 18 Jul 2021 22:37:41 +0800 Subject: [PATCH 14/17] Update pytorch_localformer_ts.py --- qlib/contrib/model/pytorch_localformer_ts.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qlib/contrib/model/pytorch_localformer_ts.py b/qlib/contrib/model/pytorch_localformer_ts.py index b4999bc9ce..683a9bd4f3 100644 --- a/qlib/contrib/model/pytorch_localformer_ts.py +++ b/qlib/contrib/model/pytorch_localformer_ts.py @@ -42,7 +42,7 @@ def __init__( optimizer="adam", reg=1e-3, n_jobs=10, - GPU=2, + GPU=0, seed=None, **kwargs ): @@ -153,9 +153,6 @@ def fit( dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) - import pdb - - pdb.set_trace() dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader From 36feb177a06c47923821cb805cf280177ac2c5a5 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Tue, 20 Jul 2021 14:55:03 +0800 Subject: [PATCH 15/17] Add performance of two new models Add the performance of transformer and localformer. --- examples/benchmarks/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/benchmarks/README.md b/examples/benchmarks/README.md index 1920a6a3b1..be394566bc 100644 --- a/examples/benchmarks/README.md +++ b/examples/benchmarks/README.md @@ -23,6 +23,8 @@ The numbers shown below demonstrate the performance of the entire `workflow` of | DoubleEnsemble (Chuheng Zhang, et al.) | Alpha360 | 0.0407±0.00| 0.3053±0.00 | 0.0490±0.00 | 0.3840±0.00 | 0.0380±0.02 | 0.5000±0.21 | -0.0984±0.02 | | TabNet (Sercan O. Arik, et al.)| Alpha360 | 0.0192±0.00 | 0.1401±0.00| 0.0291±0.00 | 0.2163±0.00 | -0.0258±0.00 | -0.2961±0.00| -0.1429±0.00 | | TCTS (Xueqing Wu, et al.)| Alpha360 | 0.0485±0.00 | 0.3689±0.04| 0.0586±0.00 | 0.4669±0.02 | 0.0816±0.02 | 1.1572±0.30| -0.0689±0.02 | +| Transformer (Ashish Vaswani, et al.)| Alpha360 | 0.0141±0.00 | 0.0917±0.02| 0.0331±0.00 | 0.2357±0.03 | -0.0259±0.03 | -0.3323±0.43| -0.1763±0.07 | +| Localformer (Juyong Jiang, et al.)| Alpha360 | 0.0408±0.00 | 0.2988±0.03| 0.0538±0.00 | 0.4105±0.02 | 0.0275±0.03 | 0.3464±0.37| -0.1182±0.03 | ## Alpha158 dataset | Model Name | Dataset | IC | ICIR | Rank IC | Rank ICIR | Annualized Return | Information Ratio | Max Drawdown | @@ -39,6 +41,8 @@ The numbers shown below demonstrate the performance of the entire `workflow` of | GATs (Petar Velickovic, et al.) | Alpha158 (with selected 20 features) | 0.0349±0.00 | 0.2511±0.01| 0.0457±0.00 | 0.3537±0.01 | 0.0578±0.02 | 0.8221±0.25| -0.0824±0.02 | | DoubleEnsemble (Chuheng Zhang, et al.) | Alpha158 | 0.0544±0.00 | 0.4338±0.01 | 0.0523±0.00 | 0.4257±0.01 | 0.1253±0.01 | 1.4105±0.14 | -0.0902±0.01 | | TabNet (Sercan O. Arik, et al.)| Alpha158 | 0.0383±0.00 | 0.3414±0.00| 0.0388±0.00 | 0.3460±0.00 | 0.0226±0.00 | 0.2652±0.00| -0.1072±0.00 | +| Transformer (Ashish Vaswani, et al.)| Alpha158 | 0.0274±0.00 | 0.2166±0.04| 0.0409±0.00 | 0.3342±0.04 | 0.0204±0.03 | 0.2888±0.40| -0.1216±0.04 | +| Localformer (Juyong Jiang, et al.)| Alpha158 | 0.0355±0.00 | 0.2747±0.04| 0.0466±0.00 | 0.3762±0.03 | 0.0506±0.02 | 0.7447±0.34| -0.0875±0.02 | - The selected 20 features are based on the feature importance of a lightgbm-based model. - The base model of DoubleEnsemble is LGBM. From 88f20fbee928e5b5e34f676b02dd2475f6ce5fb3 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Tue, 20 Jul 2021 15:00:15 +0800 Subject: [PATCH 16/17] Add two new model in zoo Add transformer and localformer (SLGT) models for time series prediction in finance in the Quant Model Zoo. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1e31915984..8825620e8e 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,8 @@ Here is a list of models built on `Qlib`. - [TabNet based on pytorch (Sercan O. Arik, et al. AAAI 2019)](qlib/contrib/model/pytorch_tabnet.py) - [DoubleEnsemble based on LightGBM (Chuheng Zhang, et al. ICDM 2020)](qlib/contrib/model/double_ensemble.py) - [TCTS based on pytorch (Xueqing Wu, et al. ICML 2021)](qlib/contrib/model/pytorch_tcts.py) +- [Transformer based on pytorch (Ashish Vaswani, et al. NeurIPS 2017)](qlib/contrib/model/pytorch_transformer.py) +- [TCTS based on pytorch (Juyong Jiang, et al.)](qlib/contrib/model/pytorch_localformer.py) Your PR of new Quant models is highly welcomed. From c18b64e96702ace58beeb66151148737a549a901 Mon Sep 17 00:00:00 2001 From: Ying-Tao Luo Date: Tue, 20 Jul 2021 15:00:59 +0800 Subject: [PATCH 17/17] Add two new models in model zoo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8825620e8e..b9bc561950 100644 --- a/README.md +++ b/README.md @@ -292,7 +292,7 @@ Here is a list of models built on `Qlib`. - [DoubleEnsemble based on LightGBM (Chuheng Zhang, et al. ICDM 2020)](qlib/contrib/model/double_ensemble.py) - [TCTS based on pytorch (Xueqing Wu, et al. ICML 2021)](qlib/contrib/model/pytorch_tcts.py) - [Transformer based on pytorch (Ashish Vaswani, et al. NeurIPS 2017)](qlib/contrib/model/pytorch_transformer.py) -- [TCTS based on pytorch (Juyong Jiang, et al.)](qlib/contrib/model/pytorch_localformer.py) +- [Localformer based on pytorch (Juyong Jiang, et al.)](qlib/contrib/model/pytorch_localformer.py) Your PR of new Quant models is highly welcomed.