Compute cross validation performance metrics on a rolling window

This commit is contained in:
Ben Letham 2018-05-03 19:48:41 -07:00
parent 994db64942
commit 3e59bbdc84
2 changed files with 51 additions and 93 deletions

View file

@ -196,7 +196,7 @@ def prophet_copy(m, cutoff=None):
return m2 return m2
def performance_metrics(df, metrics=None, aggregation='horizon'): def performance_metrics(df, metrics=None, rolling_window=0.05):
"""Compute performance metrics from cross-validation results. """Compute performance metrics from cross-validation results.
Computes a suite of performance metrics on the output of cross-validation. Computes a suite of performance metrics on the output of cross-validation.
@ -209,13 +209,17 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
A subset of these can be specified by passing a list of names as the A subset of these can be specified by passing a list of names as the
`metrics` argument. `metrics` argument.
By default, metrics will be computed for each horizon (ds - cutoff). Metrics are calculated over a rolling window of cross validation
Alternatively, metrics can be computed at the level of individual ds/cutoff predictions, after sorting by horizon. The size of that window (number of
pairs (aggregation='none'), or aggregated over all ds/cutoffs simulated forecast points) is determined by the rolling_window argument,
(aggregation='all'). which specifies a proportion of simulated forecast points to include in
each window. rolling_window=0 will compute it separately for each simulated
forecast point (i.e., 'mse' will actually be squared error with no mean).
The default of rolling_window=0.05 will use 5% of the rows in df in each
window. rolling_window=1 will compute the metric across all simulated forecast
points. The results are set to the right edge of the window.
The output is a dataframe containing the columns corresponding to the level The output is a dataframe containing column 'horizon' along with columns
of aggregation ('horizon', 'ds' and 'cutoff', or none) along with columns
for each of the metrics computed. for each of the metrics computed.
Parameters Parameters
@ -223,22 +227,13 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
df: The dataframe returned by cross_validation. df: The dataframe returned by cross_validation.
metrics: A list of performance metrics to compute. If not provided, will metrics: A list of performance metrics to compute. If not provided, will
use ['mse', 'mae', 'mape', 'coverage']. use ['mse', 'mae', 'mape', 'coverage'].
aggregation: Level of aggregation for computing performance statistics. rolling_window: Proportion of data to use in each rolling window for
Must be 'horizon', 'none', or 'all'. computing the metrics.
Returns Returns
------- -------
Dataframe with a column for each metric, and a combination of columns 'ds', Dataframe with a column for each metric, and column 'horizon'
'cutoff', and 'horizon', depending on the aggregation level.
""" """
# Input validation
valid_aggregations = ['horizon', 'all', 'none']
if aggregation not in valid_aggregations:
raise ValueError(
'Aggregation {} is not valid; must be one of {}'.format(
aggregation, valid_agggregations
)
)
valid_metrics = ['mse', 'mae', 'mape', 'coverage'] valid_metrics = ['mse', 'mae', 'mape', 'coverage']
if metrics is None: if metrics is None:
metrics = valid_metrics metrics = valid_metrics
@ -248,62 +243,56 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
raise ValueError( raise ValueError(
'Valid values for metrics are: {}'.format(valid_metrics) 'Valid values for metrics are: {}'.format(valid_metrics)
) )
# Get function for the metrics we want
metric_fns = {m: eval(m) for m in metrics}
def all_metrics(df_g):
return pd.Series({name: fn(df_g) for name, fn in metric_fns.items()})
# Apply functions to groupby
if aggregation == 'all':
return all_metrics(df)
# else,
df_m = df.copy() df_m = df.copy()
df_m['horizon'] = df_m['ds'] - df_m['cutoff'] df_m['horizon'] = df_m['ds'] - df_m['cutoff']
if aggregation == 'horizon': df_m.sort_values('horizon', inplace=True)
return df_m.groupby('horizon').apply(all_metrics).reset_index() # Window size
# else, w = int(rolling_window * df_m.shape[0])
for name, fn in metric_fns.items(): w = max(w, 1)
df_m[name] = fn(df_m, agg=False) w = min(w, df_m.shape[0])
return df_m cols = ['horizon']
for metric in metrics:
df_m[metric] = eval(metric)(df_m, w)
cols.append(metric)
df_m = df_m[cols]
return df_m.dropna()
def rolling_mean(x, w):
s = np.cumsum(np.insert(x, 0, 0))
prefix = np.empty(w - 1)
prefix.fill(np.nan)
return np.hstack((prefix, (s[w:] - s[:-w]) / float(w))) # right-aligned
# The functions below specify performance metrics for cross-validation results. # The functions below specify performance metrics for cross-validation results.
# Each takes as input the output of cross_validation, and has two modes of # Each takes as input the output of cross_validation, and returns the statistic
# return: if agg=True, returns a float that is the metric aggregated over the # as an array, given a window size for rolling aggregation.
# input. If agg=False, returns results without aggregation (for
# aggregation='none' in performance_metrics).
def mse(df, agg=True): def mse(df, w):
"""Mean squared error """Mean squared error
""" """
se = (df['y'] - df['yhat']) ** 2 se = (df['y'] - df['yhat']) ** 2
if agg: return rolling_mean(se.values, w)
return np.mean(se)
return se
def mae(df, agg=True): def mae(df, w):
"""Mean absolute error """Mean absolute error
""" """
ae = np.abs(df['y'] - df['yhat']) ae = np.abs(df['y'] - df['yhat'])
if agg: return rolling_mean(ae.values, w)
return np.mean(ae)
return ae
def mape(df, agg=True): def mape(df, w):
"""Mean absolute percent error """Mean absolute percent error
""" """
ape = np.abs((df['y'] - df['yhat']) / df['y']) ape = np.abs((df['y'] - df['yhat']) / df['y'])
if agg: return rolling_mean(ape.values, w)
return np.mean(ape)
return ape
def coverage(df, agg=True): def coverage(df, w):
"""Coverage """Coverage
""" """
is_covered = (df['y'] >= df['yhat_lower']) & (df['y'] <= df['yhat_upper']) is_covered = (df['y'] >= df['yhat_lower']) & (df['y'] <= df['yhat_upper'])
if agg: return rolling_mean(is_covered.values, w)
return np.mean(is_covered)
return is_covered

View file

@ -142,52 +142,21 @@ class TestDiagnostics(TestCase):
df_cv = diagnostics.cross_validation( df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='90 days') m, horizon='4 days', period='10 days', initial='90 days')
# Aggregation level none # Aggregation level none
df_none = diagnostics.performance_metrics(df_cv, aggregation='none') df_none = diagnostics.performance_metrics(df_cv, rolling_window=0)
self.assertEqual( self.assertEqual(
set(df_none.columns), set(df_none.columns),
{ {'horizon', 'coverage', 'mae', 'mape', 'mse'},
'y', 'yhat', 'yhat_lower', 'yhat_upper', 'ds', 'cutoff',
'horizon', 'coverage', 'mae', 'mape', 'mse',
},
) )
# Check each metric self.assertEqual(df_none.shape[0], 14)
self.assertEqual( # Aggregation level 0.2
np.abs(df_cv['yhat'][0] - df_cv['y'][0]), df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2)
df_none['mae'][0],
)
self.assertEqual(
np.abs((df_cv['yhat'][0] - df_cv['y'][0]) / df_cv['y'][0]),
df_none['mape'][0],
)
self.assertEqual(
(df_cv['yhat'][0] - df_cv['y'][0]) ** 2,
df_none['mse'][0],
)
self.assertEqual(
(
(df_cv['y'][0] >= df_cv['yhat_lower'][0])
and (df_cv['y'][0] <= df_cv['yhat_upper'][0])
),
df_none['coverage'][0],
)
# Aggregation level horizon (default)
df_horizon = diagnostics.performance_metrics(df_cv)
self.assertEqual(len(df_horizon['horizon'].unique()), 4) self.assertEqual(len(df_horizon['horizon'].unique()), 4)
self.assertEqual( self.assertEqual(df_horizon.shape[0], 13)
set(df_horizon.columns),
{'coverage', 'mse', 'mape', 'mae', 'horizon'},
)
self.assertEqual(df_horizon.shape[0], 4)
# Check aggregation
agg = df_none.groupby('horizon', as_index=False).agg('mean')
for metric in ['mse', 'mape', 'mae', 'horizon']:
self.assertTrue((agg[metric] == df_horizon[metric]).all())
# Aggregation level all # Aggregation level all
df_all = diagnostics.performance_metrics(df_cv, aggregation='all') df_all = diagnostics.performance_metrics(df_cv, rolling_window=1)
self.assertEqual(df_all.shape, (4,)) self.assertEqual(df_all.shape[0], 1)
self.assertEqual(set(df_all.index), {'coverage', 'mse', 'mae', 'mape'})
for metric in ['mse', 'mape', 'mae', 'coverage']: for metric in ['mse', 'mape', 'mae', 'coverage']:
self.assertEqual(df_all[metric], df_all[metric].mean()) self.assertEqual(df_all[metric].values[0], df_none[metric].mean())
# Custom list of metrics # Custom list of metrics
df_horizon = diagnostics.performance_metrics( df_horizon = diagnostics.performance_metrics(
df_cv, metrics=['coverage', 'mse'], df_cv, metrics=['coverage', 'mse'],