{ "cells": [ { "cell_type": "markdown", "id": "893b0097", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "# ✈️ Air Passengers：AI 预测与滚动评估\n", "\n", "**目标**\n", "- 使用 `air_passengers_with_id.csv`。 \n", "- 统一用 **`from nixtla import NixtlaClient`** 做预测：比较 `seasonal_naive / theta / ets / auto_arima`。 \n", "- 评估指标：**ME / MAE / MSE / RMSE / MPE / MAPE / sMAPE**。 \n", "- **Rolling Window**：对 **T+1 ~ T+5** 的平均误差进行汇总对比。 " ] }, { "cell_type": "markdown", "id": "c5101678", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "#### 导入与初始化" ] }, { "cell_type": "code", "execution_count": 1, "id": "001a0ae2", "metadata": { "editable": true, "slideshow": { "slide_type": "fragment" }, "tags": [] }, "outputs": [], "source": [ "# 0) 导入与初始化\n", "import numpy as np\n", "import pandas as pd\n", "from nixtla import NixtlaClient\n", "from utilsforecast.losses import mae, mse, rmse, mape, smape\n", "from utilsforecast.evaluation import evaluate\n", "\n", "nixtla = NixtlaClient(api_key='YOUR-API-KEY') # 初始化客户端\n", "FREQ = \"MS\" # 月度\n", "MODELS = [\"timegpt-1\", \"timegpt-1-long-horizon\"] # 模型" ] }, { "cell_type": "markdown", "id": "ac66ce8e", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "#### 读取数据" ] }, { "cell_type": "code", "execution_count": 2, "id": "d929a4d4", "metadata": { "editable": true, "slideshow": { "slide_type": "fragment" }, "tags": [] }, "outputs": [], "source": [ "# 1) 读取数据\n", "df = pd.read_csv(\"data/air_passengers_with_id.csv\") #输入你的真实地址\n", "df[\"ds\"] = pd.to_datetime(df[\"ds\"]) #将ds列转换为日期时间格式\n", "df = df.sort_values([\"unique_id\",\"ds\"]).reset_index(drop=True) #按unique_id和ds列排序，并重置索引\n", "UID = df[\"unique_id\"].iloc[0] " ] }, { "cell_type": "markdown", "id": "5b8ae3ee", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "#### 定义指标函数ME、MPE" ] }, { "cell_type": "code", "execution_count": 3, "id": "6f3aa939", "metadata": { "editable": true, "slideshow": { "slide_type": "fragment" }, "tags": [] }, "outputs": [], "source": [ "# 2) 指标函数：ME / MPE（%）\n", "def me(y_true, y_pred): #计算预测值与真实值的平均误差\n", " y_true = np.asarray(y_true, float); y_pred = np.asarray(y_pred, float)\n", " return float(np.mean(y_true - y_pred))\n", "\n", "def mpe(y_true, y_pred): #计算预测值与真实值的百分比误差\n", " y_true = np.asarray(y_true, float); y_pred = np.asarray(y_pred, float)\n", " mask = y_true != 0\n", " return float(np.mean((y_true[mask] - y_pred[mask]) / y_true[mask]) * 100.0) if np.any(mask) else np.nan" ] }, { "cell_type": "markdown", "id": "f1f216ed", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "#### 评估函数" ] }, { "cell_type": "code", "execution_count": 4, "id": "7c9f86eb", "metadata": { "editable": true, "slideshow": { "slide_type": "fragment" }, "tags": [] }, "outputs": [], "source": [ "# 3) 公共小工具\n", "_METRICS = [mae, mse, rmse, mape, smape]\n", "def _eval_df(df_like):\n", " ev = evaluate(df=df_like, metrics=_METRICS, models=[\"TimeGPT\"], \n", " time_col=\"ds\", target_col=\"y\").set_index(\"metric\")[\"TimeGPT\"]\n", " y, yhat = df_like[\"y\"].values, df_like[\"TimeGPT\"].values; return ev, y, yhat" ] }, { "cell_type": "code", "execution_count": null, "id": "5f316fe2", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "outputs": [], "source": [ "# 4) 函数1：单模型的 Hold-out 指标\n", "def holdout_metrics(df, model_name, H_TEST=12):\n", " # 划分训练集和测试集\n", " train, test = df.iloc[:-H_TEST], df.iloc[-H_TEST:]\n", " # 使用 Nixtla 模型进行预测，并与真实测试数据按日期合并\n", " merged = nixtla.forecast(df=train, h=H_TEST, freq=FREQ, \n", " model=model_name).merge(test, on=[\"unique_id\",\"ds\"], how=\"left\")\n", " # 计算误差指标并提取真实值与预测值\n", " ev, y, yhat = _eval_df(merged)\n", " # 汇总结果\n", " return {\"model\":model_name,\"unique_id\":UID,\"ME\":me(y,yhat),\"MPE\":mpe(y,yhat),\n", " \"MAE\":ev[\"mae\"],\"MSE\":ev[\"mse\"],\"RMSE\":ev[\"rmse\"],\"MAPE\":ev[\"mape\"],\"sMAPE\":ev[\"smape\"]}" ] }, { "cell_type": "code", "execution_count": null, "id": "44c6abcd", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "outputs": [], "source": [ "# 5) 函数2：单模型的 Rolling(T+1..T+5) 平均指标\n", "def rolling_metrics(df, model_name, H=5, n_windows=10):\n", " # 生成多次滚动窗口预测结果，并按时间排序\n", " cv = nixtla.cross_validation(df=df, h=H, n_windows=n_windows, freq=FREQ, \n", " model=model_name).sort_values([\"unique_id\",\"cutoff\",\"ds\"]).reset_index(drop=True)\n", " # 给每个窗口的预测样本标上步长 h（第几步预测）\n", " cv[\"h\"] = cv.groupby([\"unique_id\",\"cutoff\"]).cumcount() + 1; rows = []\n", " # 循环计算每个预测步长（T+1..T+H）的平均误差\n", " for h in range(1, H+1):\n", " sub = cv[cv[\"h\"]==h][[\"unique_id\",\"ds\",\"y\",\"TimeGPT\"]]; ev, y, yhat = _eval_df(sub)\n", " # 记录该步长下的各类指标\n", " rows.append({\"model\":model_name,\"unique_id\":UID,\"horizon\":h,\"ME\":me(y,yhat),\"MPE\":mpe(y,yhat),\n", " \"MAE\":ev[\"mae\"],\"MSE\":ev[\"mse\"],\"RMSE\":ev[\"rmse\"],\"MAPE\":ev[\"mape\"],\"sMAPE\":ev[\"smape\"]})\n", " # 汇总所有步长的指标表并返回\n", " return pd.DataFrame(rows).sort_values(\"horizon\").reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 7, "id": "e437b7fa", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "outputs": [], "source": [ "# 6) 函数3：对比两种 TimeGPT 模型\n", "def compare_models(df, model_list=MODELS, H_TEST=12, H=5, n_windows=10):\n", " # Hold-out\n", " hold_rows = [holdout_metrics(df, m, H_TEST) for m in model_list]\n", " holdout_df = pd.DataFrame(hold_rows).sort_values(\"MAE\").round(3)\n", " # Rolling\n", " roll_dfs = [rolling_metrics(df, m, H, n_windows) for m in model_list]\n", " rolling_df = pd.concat(roll_dfs, ignore_index=True).round(3)\n", " return holdout_df, rolling_df" ] }, { "cell_type": "markdown", "id": "5f74c7c1", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "source": [ "#### 对比结果" ] }, { "cell_type": "code", "execution_count": 8, "id": "7c4fe815", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:nixtla.nixtla_client:Validating inputs...\n", "INFO:nixtla.nixtla_client:Preprocessing dataframes...\n", "INFO:nixtla.nixtla_client:Querying model metadata...\n", "INFO:nixtla.nixtla_client:Restricting input...\n", "INFO:nixtla.nixtla_client:Calling Forecast Endpoint...\n", "INFO:nixtla.nixtla_client:Validating inputs...\n", "INFO:nixtla.nixtla_client:Preprocessing dataframes...\n", "INFO:nixtla.nixtla_client:Querying model metadata...\n", "INFO:nixtla.nixtla_client:Restricting input...\n", "INFO:nixtla.nixtla_client:Calling Forecast Endpoint...\n", "INFO:nixtla.nixtla_client:Validating inputs...\n", "INFO:nixtla.nixtla_client:Preprocessing dataframes...\n", "INFO:nixtla.nixtla_client:Restricting input...\n", "INFO:nixtla.nixtla_client:Calling Cross Validation Endpoint...\n", "INFO:nixtla.nixtla_client:Validating inputs...\n", "INFO:nixtla.nixtla_client:Preprocessing dataframes...\n", "INFO:nixtla.nixtla_client:Restricting input...\n", "INFO:nixtla.nixtla_client:Calling Cross Validation Endpoint...\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelunique_idMEMPEMAEMSERMSEMAPEsMAPE
1timegpt-1-long-horizonD13.0150.51811.062199.13214.1110.0230.012
0timegpt-1D14.8030.79012.679213.93614.6270.0270.014
\n", "
" ], "text/plain": [ " model unique_id ME MPE MAE MSE RMSE \\\n", "1 timegpt-1-long-horizon D1 3.015 0.518 11.062 199.132 14.111 \n", "0 timegpt-1 D1 4.803 0.790 12.679 213.936 14.627 \n", "\n", " MAPE sMAPE \n", "1 0.023 0.012 \n", "0 0.027 0.014 " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelunique_idhorizonMEMPEMAEMSERMSEMAPEsMAPE
0timegpt-1D11-6.199-1.62710.340152.83112.3620.0260.013
1timegpt-1D120.795-0.07213.483241.00015.5240.0330.016
2timegpt-1D13-0.001-0.45013.507218.25014.7730.0360.018
3timegpt-1D144.7450.48020.547512.55922.6400.0500.025
4timegpt-1D157.1331.19917.200411.19320.2780.0410.020
5timegpt-1-long-horizonD11-3.530-0.91014.061236.97415.3940.0350.017
6timegpt-1-long-horizonD123.8300.79314.638311.75517.6570.0360.018
7timegpt-1-long-horizonD132.9650.33716.404363.03919.0540.0430.022
8timegpt-1-long-horizonD146.5100.99723.530698.04126.4200.0590.030
9timegpt-1-long-horizonD159.9451.99719.138555.67723.5730.0440.023
\n", "
" ], "text/plain": [ " model unique_id horizon ME MPE MAE MSE \\\n", "0 timegpt-1 D1 1 -6.199 -1.627 10.340 152.831 \n", "1 timegpt-1 D1 2 0.795 -0.072 13.483 241.000 \n", "2 timegpt-1 D1 3 -0.001 -0.450 13.507 218.250 \n", "3 timegpt-1 D1 4 4.745 0.480 20.547 512.559 \n", "4 timegpt-1 D1 5 7.133 1.199 17.200 411.193 \n", "5 timegpt-1-long-horizon D1 1 -3.530 -0.910 14.061 236.974 \n", "6 timegpt-1-long-horizon D1 2 3.830 0.793 14.638 311.755 \n", "7 timegpt-1-long-horizon D1 3 2.965 0.337 16.404 363.039 \n", "8 timegpt-1-long-horizon D1 4 6.510 0.997 23.530 698.041 \n", "9 timegpt-1-long-horizon D1 5 9.945 1.997 19.138 555.677 \n", "\n", " RMSE MAPE sMAPE \n", "0 12.362 0.026 0.013 \n", "1 15.524 0.033 0.016 \n", "2 14.773 0.036 0.018 \n", "3 22.640 0.050 0.025 \n", "4 20.278 0.041 0.020 \n", "5 15.394 0.035 0.017 \n", "6 17.657 0.036 0.018 \n", "7 19.054 0.043 0.022 \n", "8 26.420 0.059 0.030 \n", "9 23.573 0.044 0.023 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# 7) 得到 Hold-out 与 Rolling 的对比结果\n", "holdout_cmp, rolling_cmp = compare_models(df, MODELS, H_TEST=12, H=5, n_windows=10)\n", "display(holdout_cmp) # 各模型在最后12期的总体指标\n", "display(rolling_cmp) # 各模型在 h=1..5 的平均指标" ] }, { "cell_type": "code", "execution_count": 9, "id": "06245157", "metadata": { "editable": true, "slideshow": { "slide_type": "slide" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "📈 Rolling Window 平均误差（T+1 ~ T+5）\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelMEMAEMSERMSEMPEMAPEsMAPE
0timegpt-11.29515.015307.16717.115-0.0940.0370.018
1timegpt-1-long-horizon3.94417.554433.09720.4200.6430.0430.022
\n", "
" ], "text/plain": [ " model ME MAE MSE RMSE MPE MAPE sMAPE\n", "0 timegpt-1 1.295 15.015 307.167 17.115 -0.094 0.037 0.018\n", "1 timegpt-1-long-horizon 3.944 17.554 433.097 20.420 0.643 0.043 0.022" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# 按模型分组取平均\n", "rolling_avg = (\n", " rolling_cmp\n", " .groupby(\"model\")[[\"ME\",\"MAE\",\"MSE\",\"RMSE\",\"MPE\",\"MAPE\",\"sMAPE\"]]\n", " .mean()\n", " .reset_index()\n", " .round(3)\n", ")\n", "print(\"📈 Rolling Window 平均误差（T+1 ~ T+5）\")\n", "display(rolling_avg)" ] } ], "metadata": { "kernelspec": { "display_name": "/usr/bin/python (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" } }, "nbformat": 4, "nbformat_minor": 5 }