diff --git a/nbs/common.model_checks.ipynb b/nbs/common.model_checks.ipynb index c93db5794..d618c5c33 100644 --- a/nbs/common.model_checks.ipynb +++ b/nbs/common.model_checks.ipynb @@ -13,16 +13,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "%load_ext autoreload\n", @@ -220,6 +211,29 @@ " except RuntimeError:\n", " raise Exception(f\"{model_class.__name__}: AirPassengers forecast test failed.\")\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| eval: false\n", + "#| hide\n", + "# Run tests in this file. This is a slow test\n", + "import warnings\n", + "import logging\n", + "from neuralforecast.models import RNN, GRU, TCN, LSTM, DeepAR, DilatedRNN, BiTCN, MLP, NBEATS, NBEATSx, NHITS, DLinear, NLinear, TiDE, DeepNPTS, TFT, VanillaTransformer, Informer, Autoformer, FEDformer, TimesNet, iTransformer, KAN, RMoK, StemGNN, TSMixer, TSMixerx, MLPMultivariate, SOFTS, TimeMixer\n", + "\n", + "models = [RNN, GRU, TCN, LSTM, DeepAR, DilatedRNN, BiTCN, MLP, NBEATS, NBEATSx, NHITS, DLinear, NLinear, TiDE, DeepNPTS, TFT, VanillaTransformer, Informer, Autoformer, FEDformer, TimesNet, iTransformer, KAN, RMoK, StemGNN, TSMixer, TSMixerx, MLPMultivariate, SOFTS, TimeMixer]\n", + "\n", + "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n", + "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n", + "with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " for model in models:\n", + " check_model(model, checks=[\"losses\"])" + ] } ], "metadata": { diff --git a/nbs/core.ipynb b/nbs/core.ipynb index 38dc73a49..c4f36ec39 100644 --- a/nbs/core.ipynb +++ b/nbs/core.ipynb @@ -738,13 +738,14 @@ " names: List[str] = []\n", " count_names = {'model': 0}\n", " for model in self.models:\n", - " if add_level and (model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)):\n", - " continue\n", - "\n", " model_name = repr(model)\n", " count_names[model_name] = count_names.get(model_name, -1) + 1\n", " if count_names[model_name] > 0:\n", " model_name += str(count_names[model_name])\n", + "\n", + " if add_level and (model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)):\n", + " continue\n", + "\n", " names.extend(model_name + n for n in model.loss.output_names)\n", " return names\n", "\n", @@ -906,6 +907,7 @@ " raise Exception(\"You must fit the model before predicting.\")\n", " \n", " quantiles_ = None\n", + " level_ = None\n", " has_level = False \n", " if level is not None:\n", " has_level = True\n", @@ -1012,7 +1014,7 @@ " self._scalers_transform(futr_dataset)\n", " dataset = dataset.append(futr_dataset)\n", " \n", - " fcsts, cols = self._generate_forecasts(dataset=dataset, quantiles_=quantiles_, has_level=has_level, **data_kwargs)\n", + " fcsts, cols = self._generate_forecasts(dataset=dataset, uids=uids, quantiles_=quantiles_, level_=level_, has_level=has_level, **data_kwargs)\n", " \n", " if self.scalers_:\n", " indptr = np.append(0, np.full(len(uids), self.h).cumsum())\n", @@ -1028,26 +1030,26 @@ " _warn_id_as_idx()\n", " fcsts_df = fcsts_df.set_index(self.id_col)\n", "\n", - " # add prediction intervals or quantiles to models trained with point loss functions via level argument\n", - " if level is not None or quantiles is not None:\n", - " model_names = self._get_model_names(add_level=True)\n", - " if model_names:\n", - " if self.prediction_intervals is None:\n", - " raise AttributeError(\n", - " \"You have trained one or more models with a point loss function (e.g. MAE, MSE). \"\n", - " \"You then must set `prediction_intervals` during fit to use level or quantiles during predict.\") \n", - " prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n", - "\n", - " fcsts_df = prediction_interval_method(\n", - " fcsts_df,\n", - " self._cs_df,\n", - " model_names=list(model_names),\n", - " level=level_ if level is not None else None,\n", - " cs_n_windows=self.prediction_intervals.n_windows,\n", - " n_series=len(uids),\n", - " horizon=self.h,\n", - " quantiles=quantiles_ if quantiles is not None else None,\n", - " ) \n", + " # # add prediction intervals or quantiles to models trained with point loss functions via level argument\n", + " # if level is not None or quantiles is not None:\n", + " # model_names = self._get_model_names(add_level=True)\n", + " # if model_names:\n", + " # if self.prediction_intervals is None:\n", + " # raise AttributeError(\n", + " # \"You have trained one or more models with a point loss function (e.g. MAE, MSE). \"\n", + " # \"You then must set `prediction_intervals` during fit to use level or quantiles during predict.\") \n", + " # prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n", + "\n", + " # fcsts_df = prediction_interval_method(\n", + " # fcsts_df,\n", + " # self._cs_df,\n", + " # model_names=list(model_names),\n", + " # level=level_ if level is not None else None,\n", + " # cs_n_windows=self.prediction_intervals.n_windows,\n", + " # n_series=len(uids),\n", + " # horizon=self.h,\n", + " # quantiles=quantiles_ if quantiles is not None else None,\n", + " # ) \n", "\n", " return fcsts_df\n", "\n", @@ -1696,7 +1698,7 @@ " dropped = list(set(cv_results.columns) - set(kept))\n", " return ufp.drop_columns(cv_results, dropped) \n", " \n", - " def _generate_forecasts(self, dataset: TimeSeriesDataset, quantiles_: Optional[List[float]] = None, has_level: Optional[bool] = False, **data_kwargs) -> np.array:\n", + " def _generate_forecasts(self, dataset: TimeSeriesDataset, uids: Series, quantiles_: Optional[List[float]] = None, level_: Optional[List[Union[int, float]]] = None, has_level: Optional[bool] = False, **data_kwargs) -> np.array:\n", " fcsts_list: List = []\n", " cols = []\n", " count_names = {'model': 0}\n", @@ -1711,6 +1713,7 @@ " model_name += str(count_names[model_name])\n", "\n", " # Predict for every quantile or level if requested and the loss function supports it\n", + " # case 1: DistributionLoss and MixtureLosses\n", " if quantiles_ is not None and not isinstance(model.loss, IQLoss) and hasattr(model.loss, 'update_quantile') and callable(model.loss.update_quantile):\n", " model_fcsts = model.predict(dataset=dataset, quantiles = quantiles_, **data_kwargs)\n", " fcsts_list.append(model_fcsts) \n", @@ -1725,6 +1728,7 @@ " cols.extend(col_names + [model_name + param_name for param_name in model.loss.param_names])\n", " else:\n", " cols.extend(col_names)\n", + " # case 2: IQLoss\n", " elif quantiles_ is not None and isinstance(model.loss, IQLoss):\n", " col_names = []\n", " for i, quantile in enumerate(quantiles_):\n", @@ -1733,6 +1737,27 @@ " col_name = self._get_column_name(model_name, quantile, has_level)\n", " col_names.extend([col_name]) \n", " cols.extend(col_names)\n", + " # case 3: PointLoss via prediction intervals\n", + " elif quantiles_ is not None and model.loss.outputsize_multiplier == 1:\n", + " if self.prediction_intervals is None:\n", + " raise AttributeError(\n", + " f\"You have trained {model_name} with loss={type(model.loss).__name__}(). \\n\"\n", + " \" You then must set `prediction_intervals` during fit to use level or quantiles during predict.\") \n", + " model_fcsts = model.predict(dataset=dataset, quantiles = quantiles_, **data_kwargs)\n", + " prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n", + " fcsts_with_intervals, out_cols = prediction_interval_method(\n", + " model_fcsts,\n", + " self._cs_df,\n", + " model=model_name,\n", + " level=level_ if has_level else None,\n", + " cs_n_windows=self.prediction_intervals.n_windows,\n", + " n_series=len(uids),\n", + " horizon=self.h,\n", + " quantiles=quantiles_ if not has_level else None,\n", + " ) \n", + " fcsts_list.append(fcsts_with_intervals) \n", + " cols.extend([model_name] + out_cols)\n", + " # base case: quantiles or levels are not supported or provided as arguments\n", " else:\n", " model_fcsts = model.predict(dataset=dataset, **data_kwargs)\n", " fcsts_list.append(model_fcsts)\n", @@ -3530,7 +3555,7 @@ "\n", "nf = NeuralForecast(models=models, freq='M')\n", "nf.fit(AirPassengersPanel_train, prediction_intervals=prediction_intervals)\n", - "# Test default prediction and correct columns\n", + "# Test default prediction\n", "preds = nf.predict(futr_df=AirPassengersPanel_test)\n", "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1-median', 'NHITS1-lo-90',\n", " 'NHITS1-lo-80', 'NHITS1-hi-80', 'NHITS1-hi-90', 'NHITS2_ql0.5', 'LSTM',\n", @@ -3538,26 +3563,26 @@ " 'LSTM1-hi-90', 'LSTM2_ql0.5', 'TSMixer', 'TSMixer1', 'TSMixer1-median',\n", " 'TSMixer1-lo-90', 'TSMixer1-lo-80', 'TSMixer1-hi-80', 'TSMixer1-hi-90',\n", " 'TSMixer2_ql0.5']\n", - "# Test multiple quantile prediction and correct columns\n", + "# Test quantile prediction\n", "preds = nf.predict(futr_df=AirPassengersPanel_test, quantiles=[0.2, 0.3])\n", - "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1_ql0.2', 'NHITS1_ql0.3',\n", - " 'NHITS2_ql0.2', 'NHITS2_ql0.3', 'LSTM', 'LSTM1', 'LSTM1_ql0.2',\n", - " 'LSTM1_ql0.3', 'LSTM2_ql0.2', 'LSTM2_ql0.3', 'TSMixer', 'TSMixer1',\n", - " 'TSMixer1_ql0.2', 'TSMixer1_ql0.3', 'TSMixer2_ql0.2', 'TSMixer2_ql0.3',\n", - " 'NHITS-ql0.2', 'NHITS-ql0.3', 'LSTM-ql0.2', 'LSTM-ql0.3',\n", - " 'TSMixer-ql0.2', 'TSMixer-ql0.3']\n", - "# Test multiple level prediction and correct columns\n", + "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS-ql0.2', 'NHITS-ql0.3', 'NHITS1',\n", + " 'NHITS1_ql0.2', 'NHITS1_ql0.3', 'NHITS2_ql0.2', 'NHITS2_ql0.3', 'LSTM',\n", + " 'LSTM-ql0.2', 'LSTM-ql0.3', 'LSTM1', 'LSTM1_ql0.2', 'LSTM1_ql0.3',\n", + " 'LSTM2_ql0.2', 'LSTM2_ql0.3', 'TSMixer', 'TSMixer-ql0.2',\n", + " 'TSMixer-ql0.3', 'TSMixer1', 'TSMixer1_ql0.2', 'TSMixer1_ql0.3',\n", + " 'TSMixer2_ql0.2', 'TSMixer2_ql0.3']\n", + "# Test level prediction\n", "preds = nf.predict(futr_df=AirPassengersPanel_test, level=[80, 90])\n", - "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1-lo-90', 'NHITS1-lo-80',\n", - " 'NHITS1-hi-80', 'NHITS1-hi-90', 'NHITS2-lo-90', 'NHITS2-lo-80',\n", - " 'NHITS2-hi-80', 'NHITS2-hi-90', 'LSTM', 'LSTM1', 'LSTM1-lo-90',\n", - " 'LSTM1-lo-80', 'LSTM1-hi-80', 'LSTM1-hi-90', 'LSTM2-lo-90',\n", - " 'LSTM2-lo-80', 'LSTM2-hi-80', 'LSTM2-hi-90', 'TSMixer', 'TSMixer1',\n", - " 'TSMixer1-lo-90', 'TSMixer1-lo-80', 'TSMixer1-hi-80', 'TSMixer1-hi-90',\n", - " 'TSMixer2-lo-90', 'TSMixer2-lo-80', 'TSMixer2-hi-80', 'TSMixer2-hi-90',\n", - " 'NHITS-lo-90', 'NHITS-lo-80', 'NHITS-hi-80', 'NHITS-hi-90',\n", - " 'LSTM-lo-90', 'LSTM-lo-80', 'LSTM-hi-80', 'LSTM-hi-90', 'TSMixer-lo-90',\n", - " 'TSMixer-lo-80', 'TSMixer-hi-80', 'TSMixer-hi-90']\n", + "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS-lo-90', 'NHITS-lo-80', 'NHITS-hi-80',\n", + " 'NHITS-hi-90', 'NHITS1', 'NHITS1-lo-90', 'NHITS1-lo-80', 'NHITS1-hi-80',\n", + " 'NHITS1-hi-90', 'NHITS2-lo-90', 'NHITS2-lo-80', 'NHITS2-hi-80',\n", + " 'NHITS2-hi-90', 'LSTM', 'LSTM-lo-90', 'LSTM-lo-80', 'LSTM-hi-80',\n", + " 'LSTM-hi-90', 'LSTM1', 'LSTM1-lo-90', 'LSTM1-lo-80', 'LSTM1-hi-80',\n", + " 'LSTM1-hi-90', 'LSTM2-lo-90', 'LSTM2-lo-80', 'LSTM2-hi-80',\n", + " 'LSTM2-hi-90', 'TSMixer', 'TSMixer-lo-90', 'TSMixer-lo-80',\n", + " 'TSMixer-hi-80', 'TSMixer-hi-90', 'TSMixer1', 'TSMixer1-lo-90',\n", + " 'TSMixer1-lo-80', 'TSMixer1-hi-80', 'TSMixer1-hi-90', 'TSMixer2-lo-90',\n", + " 'TSMixer2-lo-80', 'TSMixer2-hi-80', 'TSMixer2-hi-90']\n", "# Re-Test default prediction - note that they are different from the first test (this is expected)\n", "preds = nf.predict(futr_df=AirPassengersPanel_test)\n", "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1-median', 'NHITS2_ql0.5',\n", diff --git a/nbs/utils.ipynb b/nbs/utils.ipynb index 41123fec0..e8cb8c170 100644 --- a/nbs/utils.ipynb +++ b/nbs/utils.ipynb @@ -47,12 +47,11 @@ "#| export\n", "import random\n", "from itertools import chain\n", - "from typing import List, Union, Optional\n", + "from typing import List, Union, Optional, Tuple\n", "from utilsforecast.compat import DFType\n", "\n", "import numpy as np\n", - "import pandas as pd\n", - "import utilsforecast.processing as ufp" + "import pandas as pd" ] }, { @@ -1302,15 +1301,15 @@ "source": [ "#| export\n", "def add_conformal_distribution_intervals(\n", - " fcst_df: DFType, \n", + " model_fcsts: np.array, \n", " cs_df: DFType,\n", - " model_names: List[str],\n", + " model: str,\n", " cs_n_windows: int,\n", " n_series: int,\n", " horizon: int,\n", " level: Optional[List[Union[int, float]]] = None,\n", " quantiles: Optional[List[float]] = None,\n", - ") -> DFType:\n", + ") -> Tuple[np.array, List[str]]:\n", " \"\"\"\n", " Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`.\n", " `level` should be already sorted. This strategy creates forecasts paths\n", @@ -1318,7 +1317,6 @@ " \"\"\"\n", " assert level is not None or quantiles is not None, \"Either level or quantiles must be provided\"\n", " \n", - " fcst_df = ufp.copy_if_pandas(fcst_df, deep=False)\n", " if quantiles is None and level is not None:\n", " alphas = [100 - lv for lv in level]\n", " cuts = [alpha / 200 for alpha in reversed(alphas)]\n", @@ -1326,29 +1324,28 @@ " elif quantiles is not None:\n", " cuts = quantiles\n", " \n", - " for model in model_names:\n", - " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", - " scores = scores.transpose(1, 0, 2)\n", - " # restrict scores to horizon\n", - " scores = scores[:,:,:horizon]\n", - " mean = fcst_df[model].to_numpy().reshape(1, n_series, -1)\n", - " scores = np.vstack([mean - scores, mean + scores])\n", - " scores_quantiles = np.quantile(\n", - " scores,\n", - " cuts,\n", - " axis=0,\n", - " )\n", - " scores_quantiles = scores_quantiles.reshape(len(cuts), -1).T\n", - " if quantiles is None and level is not None:\n", - " lo_cols = [f\"{model}-lo-{lv}\" for lv in reversed(level)]\n", - " hi_cols = [f\"{model}-hi-{lv}\" for lv in level]\n", - " out_cols = lo_cols + hi_cols\n", - " elif quantiles is not None:\n", - " out_cols = [f\"{model}-ql{q}\" for q in quantiles]\n", + " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", + " scores = scores.transpose(1, 0, 2)\n", + " # restrict scores to horizon\n", + " scores = scores[:,:,:horizon]\n", + " mean = model_fcsts.reshape(1, n_series, -1)\n", + " scores = np.vstack([mean - scores, mean + scores])\n", + " scores_quantiles = np.quantile(\n", + " scores,\n", + " cuts,\n", + " axis=0,\n", + " )\n", + " scores_quantiles = scores_quantiles.reshape(len(cuts), -1).T\n", + " if quantiles is None and level is not None:\n", + " lo_cols = [f\"{model}-lo-{lv}\" for lv in reversed(level)]\n", + " hi_cols = [f\"{model}-hi-{lv}\" for lv in level]\n", + " out_cols = lo_cols + hi_cols\n", + " elif quantiles is not None:\n", + " out_cols = [f\"{model}-ql{q}\" for q in quantiles]\n", "\n", - " fcst_df = ufp.assign_columns(fcst_df, out_cols, scores_quantiles)\n", + " fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles])\n", "\n", - " return fcst_df" + " return fcsts_with_intervals, out_cols" ] }, { @@ -1359,15 +1356,15 @@ "source": [ "#| export\n", "def add_conformal_error_intervals(\n", - " fcst_df: DFType, \n", + " model_fcsts: np.array, \n", " cs_df: DFType, \n", - " model_names: List[str],\n", + " model: str,\n", " cs_n_windows: int,\n", " n_series: int,\n", " horizon: int,\n", " level: Optional[List[Union[int, float]]] = None,\n", " quantiles: Optional[List[float]] = None,\n", - ") -> DFType:\n", + ") -> Tuple[np.array, List[str]]:\n", " \"\"\"\n", " Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`.\n", " `level` should be already sorted. This startegy creates prediction intervals\n", @@ -1375,44 +1372,43 @@ " \"\"\"\n", " assert level is not None or quantiles is not None, \"Either level or quantiles must be provided\"\n", "\n", - " fcst_df = ufp.copy_if_pandas(fcst_df, deep=False)\n", " if quantiles is None and level is not None:\n", " cuts = [lv / 100 for lv in level]\n", " elif quantiles is not None:\n", " cuts = quantiles\n", "\n", - " for model in model_names:\n", - " mean = fcst_df[model].to_numpy().ravel()\n", - " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", - " scores = scores.transpose(1, 0, 2)\n", - " # restrict scores to horizon\n", - " scores = scores[:,:,:horizon]\n", - " scores_quantiles = np.quantile(\n", - " scores,\n", - " cuts,\n", - " axis=0,\n", - " )\n", - " scores_quantiles = scores_quantiles.reshape(len(cuts), -1)\n", - " if quantiles is None and level is not None:\n", - " lo_cols = [f\"{model}-lo-{lv}\" for lv in reversed(level)]\n", - " hi_cols = [f\"{model}-hi-{lv}\" for lv in level]\n", - " out_cols = lo_cols + hi_cols\n", - " scores_quantiles = np.vstack([mean - scores_quantiles[::-1], mean + scores_quantiles]).T\n", - " elif quantiles is not None:\n", - " out_cols = []\n", - " scores_quantiles_ls = []\n", - " for i, q in enumerate(quantiles):\n", - " out_cols.append(f\"{model}-ql{q}\")\n", - " if q < 0.5:\n", - " scores_quantiles_ls.append(mean - scores_quantiles[::-1][i])\n", - " elif q > 0.5:\n", - " scores_quantiles_ls.append(mean + scores_quantiles[i])\n", - " else:\n", - " scores_quantiles_ls.append(mean)\n", - " scores_quantiles = np.vstack(scores_quantiles_ls).T \n", + " mean = model_fcsts.ravel()\n", + " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", + " scores = scores.transpose(1, 0, 2)\n", + " # restrict scores to horizon\n", + " scores = scores[:,:,:horizon]\n", + " scores_quantiles = np.quantile(\n", + " scores,\n", + " cuts,\n", + " axis=0,\n", + " )\n", + " scores_quantiles = scores_quantiles.reshape(len(cuts), -1)\n", + " if quantiles is None and level is not None:\n", + " lo_cols = [f\"{model}-lo-{lv}\" for lv in reversed(level)]\n", + " hi_cols = [f\"{model}-hi-{lv}\" for lv in level]\n", + " out_cols = lo_cols + hi_cols\n", + " scores_quantiles = np.vstack([mean - scores_quantiles[::-1], mean + scores_quantiles]).T\n", + " elif quantiles is not None:\n", + " out_cols = []\n", + " scores_quantiles_ls = []\n", + " for i, q in enumerate(quantiles):\n", + " out_cols.append(f\"{model}-ql{q}\")\n", + " if q < 0.5:\n", + " scores_quantiles_ls.append(mean - scores_quantiles[::-1][i])\n", + " elif q > 0.5:\n", + " scores_quantiles_ls.append(mean + scores_quantiles[i])\n", + " else:\n", + " scores_quantiles_ls.append(mean)\n", + " scores_quantiles = np.vstack(scores_quantiles_ls).T \n", + "\n", + " fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles])\n", "\n", - " fcst_df = ufp.assign_columns(fcst_df, out_cols, scores_quantiles)\n", - " return fcst_df" + " return fcsts_with_intervals, out_cols" ] }, { diff --git a/neuralforecast/core.py b/neuralforecast/core.py index 942f4f7b2..446da5126 100644 --- a/neuralforecast/core.py +++ b/neuralforecast/core.py @@ -687,15 +687,16 @@ def _get_model_names(self, add_level=False) -> List[str]: names: List[str] = [] count_names = {"model": 0} for model in self.models: + model_name = repr(model) + count_names[model_name] = count_names.get(model_name, -1) + 1 + if count_names[model_name] > 0: + model_name += str(count_names[model_name]) + if add_level and ( model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss) ): continue - model_name = repr(model) - count_names[model_name] = count_names.get(model_name, -1) + 1 - if count_names[model_name] > 0: - model_name += str(count_names[model_name]) names.extend(model_name + n for n in model.loss.output_names) return names @@ -865,6 +866,7 @@ def predict( raise Exception("You must fit the model before predicting.") quantiles_ = None + level_ = None has_level = False if level is not None: has_level = True @@ -974,7 +976,12 @@ def predict( dataset = dataset.append(futr_dataset) fcsts, cols = self._generate_forecasts( - dataset=dataset, quantiles_=quantiles_, has_level=has_level, **data_kwargs + dataset=dataset, + uids=uids, + quantiles_=quantiles_, + level_=level_, + has_level=has_level, + **data_kwargs, ) if self.scalers_: @@ -991,29 +998,26 @@ def predict( _warn_id_as_idx() fcsts_df = fcsts_df.set_index(self.id_col) - # add prediction intervals or quantiles to models trained with point loss functions via level argument - if level is not None or quantiles is not None: - model_names = self._get_model_names(add_level=True) - if model_names: - if self.prediction_intervals is None: - raise AttributeError( - "You have trained one or more models with a point loss function (e.g. MAE, MSE). " - "You then must set `prediction_intervals` during fit to use level or quantiles during predict." - ) - prediction_interval_method = get_prediction_interval_method( - self.prediction_intervals.method - ) - - fcsts_df = prediction_interval_method( - fcsts_df, - self._cs_df, - model_names=list(model_names), - level=level_ if level is not None else None, - cs_n_windows=self.prediction_intervals.n_windows, - n_series=len(uids), - horizon=self.h, - quantiles=quantiles_ if quantiles is not None else None, - ) + # # add prediction intervals or quantiles to models trained with point loss functions via level argument + # if level is not None or quantiles is not None: + # model_names = self._get_model_names(add_level=True) + # if model_names: + # if self.prediction_intervals is None: + # raise AttributeError( + # "You have trained one or more models with a point loss function (e.g. MAE, MSE). " + # "You then must set `prediction_intervals` during fit to use level or quantiles during predict.") + # prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method) + + # fcsts_df = prediction_interval_method( + # fcsts_df, + # self._cs_df, + # model_names=list(model_names), + # level=level_ if level is not None else None, + # cs_n_windows=self.prediction_intervals.n_windows, + # n_series=len(uids), + # horizon=self.h, + # quantiles=quantiles_ if quantiles is not None else None, + # ) return fcsts_df @@ -1697,7 +1701,9 @@ def _conformity_scores( def _generate_forecasts( self, dataset: TimeSeriesDataset, + uids: Series, quantiles_: Optional[List[float]] = None, + level_: Optional[List[Union[int, float]]] = None, has_level: Optional[bool] = False, **data_kwargs, ) -> np.array: @@ -1715,6 +1721,7 @@ def _generate_forecasts( model_name += str(count_names[model_name]) # Predict for every quantile or level if requested and the loss function supports it + # case 1: DistributionLoss and MixtureLosses if ( quantiles_ is not None and not isinstance(model.loss, IQLoss) @@ -1742,6 +1749,7 @@ def _generate_forecasts( ) else: cols.extend(col_names) + # case 2: IQLoss elif quantiles_ is not None and isinstance(model.loss, IQLoss): col_names = [] for i, quantile in enumerate(quantiles_): @@ -1752,6 +1760,32 @@ def _generate_forecasts( col_name = self._get_column_name(model_name, quantile, has_level) col_names.extend([col_name]) cols.extend(col_names) + # case 3: PointLoss via prediction intervals + elif quantiles_ is not None and model.loss.outputsize_multiplier == 1: + if self.prediction_intervals is None: + raise AttributeError( + f"You have trained {model_name} with loss={type(model.loss).__name__}(). \n" + " You then must set `prediction_intervals` during fit to use level or quantiles during predict." + ) + model_fcsts = model.predict( + dataset=dataset, quantiles=quantiles_, **data_kwargs + ) + prediction_interval_method = get_prediction_interval_method( + self.prediction_intervals.method + ) + fcsts_with_intervals, out_cols = prediction_interval_method( + model_fcsts, + self._cs_df, + model=model_name, + level=level_ if has_level else None, + cs_n_windows=self.prediction_intervals.n_windows, + n_series=len(uids), + horizon=self.h, + quantiles=quantiles_ if not has_level else None, + ) + fcsts_list.append(fcsts_with_intervals) + cols.extend([model_name] + out_cols) + # base case: quantiles or levels are not supported or provided as arguments else: model_fcsts = model.predict(dataset=dataset, **data_kwargs) fcsts_list.append(model_fcsts) diff --git a/neuralforecast/utils.py b/neuralforecast/utils.py index 3374de04d..ab3ff1d5e 100644 --- a/neuralforecast/utils.py +++ b/neuralforecast/utils.py @@ -11,12 +11,11 @@ # %% ../nbs/utils.ipynb 3 import random from itertools import chain -from typing import List, Union, Optional +from typing import List, Union, Optional, Tuple from utilsforecast.compat import DFType import numpy as np import pandas as pd -import utilsforecast.processing as ufp # %% ../nbs/utils.ipynb 6 def generate_series( @@ -484,15 +483,15 @@ def __repr__(self): # %% ../nbs/utils.ipynb 32 def add_conformal_distribution_intervals( - fcst_df: DFType, + model_fcsts: np.array, cs_df: DFType, - model_names: List[str], + model: str, cs_n_windows: int, n_series: int, horizon: int, level: Optional[List[Union[int, float]]] = None, quantiles: Optional[List[float]] = None, -) -> DFType: +) -> Tuple[np.array, List[str]]: """ Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`. `level` should be already sorted. This strategy creates forecasts paths @@ -502,7 +501,6 @@ def add_conformal_distribution_intervals( level is not None or quantiles is not None ), "Either level or quantiles must be provided" - fcst_df = ufp.copy_if_pandas(fcst_df, deep=False) if quantiles is None and level is not None: alphas = [100 - lv for lv in level] cuts = [alpha / 200 for alpha in reversed(alphas)] @@ -510,41 +508,40 @@ def add_conformal_distribution_intervals( elif quantiles is not None: cuts = quantiles - for model in model_names: - scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) - scores = scores.transpose(1, 0, 2) - # restrict scores to horizon - scores = scores[:, :, :horizon] - mean = fcst_df[model].to_numpy().reshape(1, n_series, -1) - scores = np.vstack([mean - scores, mean + scores]) - scores_quantiles = np.quantile( - scores, - cuts, - axis=0, - ) - scores_quantiles = scores_quantiles.reshape(len(cuts), -1).T - if quantiles is None and level is not None: - lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)] - hi_cols = [f"{model}-hi-{lv}" for lv in level] - out_cols = lo_cols + hi_cols - elif quantiles is not None: - out_cols = [f"{model}-ql{q}" for q in quantiles] + scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) + scores = scores.transpose(1, 0, 2) + # restrict scores to horizon + scores = scores[:, :, :horizon] + mean = model_fcsts.reshape(1, n_series, -1) + scores = np.vstack([mean - scores, mean + scores]) + scores_quantiles = np.quantile( + scores, + cuts, + axis=0, + ) + scores_quantiles = scores_quantiles.reshape(len(cuts), -1).T + if quantiles is None and level is not None: + lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)] + hi_cols = [f"{model}-hi-{lv}" for lv in level] + out_cols = lo_cols + hi_cols + elif quantiles is not None: + out_cols = [f"{model}-ql{q}" for q in quantiles] - fcst_df = ufp.assign_columns(fcst_df, out_cols, scores_quantiles) + fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles]) - return fcst_df + return fcsts_with_intervals, out_cols # %% ../nbs/utils.ipynb 33 def add_conformal_error_intervals( - fcst_df: DFType, + model_fcsts: np.array, cs_df: DFType, - model_names: List[str], + model: str, cs_n_windows: int, n_series: int, horizon: int, level: Optional[List[Union[int, float]]] = None, quantiles: Optional[List[float]] = None, -) -> DFType: +) -> Tuple[np.array, List[str]]: """ Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`. `level` should be already sorted. This startegy creates prediction intervals @@ -554,46 +551,45 @@ def add_conformal_error_intervals( level is not None or quantiles is not None ), "Either level or quantiles must be provided" - fcst_df = ufp.copy_if_pandas(fcst_df, deep=False) if quantiles is None and level is not None: cuts = [lv / 100 for lv in level] elif quantiles is not None: cuts = quantiles - for model in model_names: - mean = fcst_df[model].to_numpy().ravel() - scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) - scores = scores.transpose(1, 0, 2) - # restrict scores to horizon - scores = scores[:, :, :horizon] - scores_quantiles = np.quantile( - scores, - cuts, - axis=0, - ) - scores_quantiles = scores_quantiles.reshape(len(cuts), -1) - if quantiles is None and level is not None: - lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)] - hi_cols = [f"{model}-hi-{lv}" for lv in level] - out_cols = lo_cols + hi_cols - scores_quantiles = np.vstack( - [mean - scores_quantiles[::-1], mean + scores_quantiles] - ).T - elif quantiles is not None: - out_cols = [] - scores_quantiles_ls = [] - for i, q in enumerate(quantiles): - out_cols.append(f"{model}-ql{q}") - if q < 0.5: - scores_quantiles_ls.append(mean - scores_quantiles[::-1][i]) - elif q > 0.5: - scores_quantiles_ls.append(mean + scores_quantiles[i]) - else: - scores_quantiles_ls.append(mean) - scores_quantiles = np.vstack(scores_quantiles_ls).T - - fcst_df = ufp.assign_columns(fcst_df, out_cols, scores_quantiles) - return fcst_df + mean = model_fcsts.ravel() + scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) + scores = scores.transpose(1, 0, 2) + # restrict scores to horizon + scores = scores[:, :, :horizon] + scores_quantiles = np.quantile( + scores, + cuts, + axis=0, + ) + scores_quantiles = scores_quantiles.reshape(len(cuts), -1) + if quantiles is None and level is not None: + lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)] + hi_cols = [f"{model}-hi-{lv}" for lv in level] + out_cols = lo_cols + hi_cols + scores_quantiles = np.vstack( + [mean - scores_quantiles[::-1], mean + scores_quantiles] + ).T + elif quantiles is not None: + out_cols = [] + scores_quantiles_ls = [] + for i, q in enumerate(quantiles): + out_cols.append(f"{model}-ql{q}") + if q < 0.5: + scores_quantiles_ls.append(mean - scores_quantiles[::-1][i]) + elif q > 0.5: + scores_quantiles_ls.append(mean + scores_quantiles[i]) + else: + scores_quantiles_ls.append(mean) + scores_quantiles = np.vstack(scores_quantiles_ls).T + + fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles]) + + return fcsts_with_intervals, out_cols # %% ../nbs/utils.ipynb 34 def get_prediction_interval_method(method: str):