mirror of
https://github.com/saymrwulf/prophet.git
synced 2026-11-18 08:53:42 +00:00
Compute cross validation performance metrics on a rolling window
This commit is contained in:
parent
994db64942
commit
3e59bbdc84
2 changed files with 51 additions and 93 deletions
|
|
@ -196,7 +196,7 @@ def prophet_copy(m, cutoff=None):
|
||||||
return m2
|
return m2
|
||||||
|
|
||||||
|
|
||||||
def performance_metrics(df, metrics=None, aggregation='horizon'):
|
def performance_metrics(df, metrics=None, rolling_window=0.05):
|
||||||
"""Compute performance metrics from cross-validation results.
|
"""Compute performance metrics from cross-validation results.
|
||||||
|
|
||||||
Computes a suite of performance metrics on the output of cross-validation.
|
Computes a suite of performance metrics on the output of cross-validation.
|
||||||
|
|
@ -209,13 +209,17 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
|
||||||
A subset of these can be specified by passing a list of names as the
|
A subset of these can be specified by passing a list of names as the
|
||||||
`metrics` argument.
|
`metrics` argument.
|
||||||
|
|
||||||
By default, metrics will be computed for each horizon (ds - cutoff).
|
Metrics are calculated over a rolling window of cross validation
|
||||||
Alternatively, metrics can be computed at the level of individual ds/cutoff
|
predictions, after sorting by horizon. The size of that window (number of
|
||||||
pairs (aggregation='none'), or aggregated over all ds/cutoffs
|
simulated forecast points) is determined by the rolling_window argument,
|
||||||
(aggregation='all').
|
which specifies a proportion of simulated forecast points to include in
|
||||||
|
each window. rolling_window=0 will compute it separately for each simulated
|
||||||
|
forecast point (i.e., 'mse' will actually be squared error with no mean).
|
||||||
|
The default of rolling_window=0.05 will use 5% of the rows in df in each
|
||||||
|
window. rolling_window=1 will compute the metric across all simulated forecast
|
||||||
|
points. The results are set to the right edge of the window.
|
||||||
|
|
||||||
The output is a dataframe containing the columns corresponding to the level
|
The output is a dataframe containing column 'horizon' along with columns
|
||||||
of aggregation ('horizon', 'ds' and 'cutoff', or none) along with columns
|
|
||||||
for each of the metrics computed.
|
for each of the metrics computed.
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
|
|
@ -223,22 +227,13 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
|
||||||
df: The dataframe returned by cross_validation.
|
df: The dataframe returned by cross_validation.
|
||||||
metrics: A list of performance metrics to compute. If not provided, will
|
metrics: A list of performance metrics to compute. If not provided, will
|
||||||
use ['mse', 'mae', 'mape', 'coverage'].
|
use ['mse', 'mae', 'mape', 'coverage'].
|
||||||
aggregation: Level of aggregation for computing performance statistics.
|
rolling_window: Proportion of data to use in each rolling window for
|
||||||
Must be 'horizon', 'none', or 'all'.
|
computing the metrics.
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
Dataframe with a column for each metric, and a combination of columns 'ds',
|
Dataframe with a column for each metric, and column 'horizon'
|
||||||
'cutoff', and 'horizon', depending on the aggregation level.
|
|
||||||
"""
|
"""
|
||||||
# Input validation
|
|
||||||
valid_aggregations = ['horizon', 'all', 'none']
|
|
||||||
if aggregation not in valid_aggregations:
|
|
||||||
raise ValueError(
|
|
||||||
'Aggregation {} is not valid; must be one of {}'.format(
|
|
||||||
aggregation, valid_agggregations
|
|
||||||
)
|
|
||||||
)
|
|
||||||
valid_metrics = ['mse', 'mae', 'mape', 'coverage']
|
valid_metrics = ['mse', 'mae', 'mape', 'coverage']
|
||||||
if metrics is None:
|
if metrics is None:
|
||||||
metrics = valid_metrics
|
metrics = valid_metrics
|
||||||
|
|
@ -248,62 +243,56 @@ def performance_metrics(df, metrics=None, aggregation='horizon'):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
'Valid values for metrics are: {}'.format(valid_metrics)
|
'Valid values for metrics are: {}'.format(valid_metrics)
|
||||||
)
|
)
|
||||||
# Get function for the metrics we want
|
|
||||||
metric_fns = {m: eval(m) for m in metrics}
|
|
||||||
def all_metrics(df_g):
|
|
||||||
return pd.Series({name: fn(df_g) for name, fn in metric_fns.items()})
|
|
||||||
# Apply functions to groupby
|
|
||||||
if aggregation == 'all':
|
|
||||||
return all_metrics(df)
|
|
||||||
# else,
|
|
||||||
df_m = df.copy()
|
df_m = df.copy()
|
||||||
df_m['horizon'] = df_m['ds'] - df_m['cutoff']
|
df_m['horizon'] = df_m['ds'] - df_m['cutoff']
|
||||||
if aggregation == 'horizon':
|
df_m.sort_values('horizon', inplace=True)
|
||||||
return df_m.groupby('horizon').apply(all_metrics).reset_index()
|
# Window size
|
||||||
# else,
|
w = int(rolling_window * df_m.shape[0])
|
||||||
for name, fn in metric_fns.items():
|
w = max(w, 1)
|
||||||
df_m[name] = fn(df_m, agg=False)
|
w = min(w, df_m.shape[0])
|
||||||
return df_m
|
cols = ['horizon']
|
||||||
|
for metric in metrics:
|
||||||
|
df_m[metric] = eval(metric)(df_m, w)
|
||||||
|
cols.append(metric)
|
||||||
|
df_m = df_m[cols]
|
||||||
|
return df_m.dropna()
|
||||||
|
|
||||||
|
|
||||||
|
def rolling_mean(x, w):
|
||||||
|
s = np.cumsum(np.insert(x, 0, 0))
|
||||||
|
prefix = np.empty(w - 1)
|
||||||
|
prefix.fill(np.nan)
|
||||||
|
return np.hstack((prefix, (s[w:] - s[:-w]) / float(w))) # right-aligned
|
||||||
|
|
||||||
|
|
||||||
# The functions below specify performance metrics for cross-validation results.
|
# The functions below specify performance metrics for cross-validation results.
|
||||||
# Each takes as input the output of cross_validation, and has two modes of
|
# Each takes as input the output of cross_validation, and returns the statistic
|
||||||
# return: if agg=True, returns a float that is the metric aggregated over the
|
# as an array, given a window size for rolling aggregation.
|
||||||
# input. If agg=False, returns results without aggregation (for
|
|
||||||
# aggregation='none' in performance_metrics).
|
|
||||||
|
|
||||||
|
|
||||||
def mse(df, agg=True):
|
def mse(df, w):
|
||||||
"""Mean squared error
|
"""Mean squared error
|
||||||
"""
|
"""
|
||||||
se = (df['y'] - df['yhat']) ** 2
|
se = (df['y'] - df['yhat']) ** 2
|
||||||
if agg:
|
return rolling_mean(se.values, w)
|
||||||
return np.mean(se)
|
|
||||||
return se
|
|
||||||
|
|
||||||
|
|
||||||
def mae(df, agg=True):
|
def mae(df, w):
|
||||||
"""Mean absolute error
|
"""Mean absolute error
|
||||||
"""
|
"""
|
||||||
ae = np.abs(df['y'] - df['yhat'])
|
ae = np.abs(df['y'] - df['yhat'])
|
||||||
if agg:
|
return rolling_mean(ae.values, w)
|
||||||
return np.mean(ae)
|
|
||||||
return ae
|
|
||||||
|
|
||||||
|
|
||||||
def mape(df, agg=True):
|
def mape(df, w):
|
||||||
"""Mean absolute percent error
|
"""Mean absolute percent error
|
||||||
"""
|
"""
|
||||||
ape = np.abs((df['y'] - df['yhat']) / df['y'])
|
ape = np.abs((df['y'] - df['yhat']) / df['y'])
|
||||||
if agg:
|
return rolling_mean(ape.values, w)
|
||||||
return np.mean(ape)
|
|
||||||
return ape
|
|
||||||
|
|
||||||
|
|
||||||
def coverage(df, agg=True):
|
def coverage(df, w):
|
||||||
"""Coverage
|
"""Coverage
|
||||||
"""
|
"""
|
||||||
is_covered = (df['y'] >= df['yhat_lower']) & (df['y'] <= df['yhat_upper'])
|
is_covered = (df['y'] >= df['yhat_lower']) & (df['y'] <= df['yhat_upper'])
|
||||||
if agg:
|
return rolling_mean(is_covered.values, w)
|
||||||
return np.mean(is_covered)
|
|
||||||
return is_covered
|
|
||||||
|
|
|
||||||
|
|
@ -142,52 +142,21 @@ class TestDiagnostics(TestCase):
|
||||||
df_cv = diagnostics.cross_validation(
|
df_cv = diagnostics.cross_validation(
|
||||||
m, horizon='4 days', period='10 days', initial='90 days')
|
m, horizon='4 days', period='10 days', initial='90 days')
|
||||||
# Aggregation level none
|
# Aggregation level none
|
||||||
df_none = diagnostics.performance_metrics(df_cv, aggregation='none')
|
df_none = diagnostics.performance_metrics(df_cv, rolling_window=0)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
set(df_none.columns),
|
set(df_none.columns),
|
||||||
{
|
{'horizon', 'coverage', 'mae', 'mape', 'mse'},
|
||||||
'y', 'yhat', 'yhat_lower', 'yhat_upper', 'ds', 'cutoff',
|
|
||||||
'horizon', 'coverage', 'mae', 'mape', 'mse',
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
# Check each metric
|
self.assertEqual(df_none.shape[0], 14)
|
||||||
self.assertEqual(
|
# Aggregation level 0.2
|
||||||
np.abs(df_cv['yhat'][0] - df_cv['y'][0]),
|
df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2)
|
||||||
df_none['mae'][0],
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
np.abs((df_cv['yhat'][0] - df_cv['y'][0]) / df_cv['y'][0]),
|
|
||||||
df_none['mape'][0],
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
(df_cv['yhat'][0] - df_cv['y'][0]) ** 2,
|
|
||||||
df_none['mse'][0],
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
(
|
|
||||||
(df_cv['y'][0] >= df_cv['yhat_lower'][0])
|
|
||||||
and (df_cv['y'][0] <= df_cv['yhat_upper'][0])
|
|
||||||
),
|
|
||||||
df_none['coverage'][0],
|
|
||||||
)
|
|
||||||
# Aggregation level horizon (default)
|
|
||||||
df_horizon = diagnostics.performance_metrics(df_cv)
|
|
||||||
self.assertEqual(len(df_horizon['horizon'].unique()), 4)
|
self.assertEqual(len(df_horizon['horizon'].unique()), 4)
|
||||||
self.assertEqual(
|
self.assertEqual(df_horizon.shape[0], 13)
|
||||||
set(df_horizon.columns),
|
|
||||||
{'coverage', 'mse', 'mape', 'mae', 'horizon'},
|
|
||||||
)
|
|
||||||
self.assertEqual(df_horizon.shape[0], 4)
|
|
||||||
# Check aggregation
|
|
||||||
agg = df_none.groupby('horizon', as_index=False).agg('mean')
|
|
||||||
for metric in ['mse', 'mape', 'mae', 'horizon']:
|
|
||||||
self.assertTrue((agg[metric] == df_horizon[metric]).all())
|
|
||||||
# Aggregation level all
|
# Aggregation level all
|
||||||
df_all = diagnostics.performance_metrics(df_cv, aggregation='all')
|
df_all = diagnostics.performance_metrics(df_cv, rolling_window=1)
|
||||||
self.assertEqual(df_all.shape, (4,))
|
self.assertEqual(df_all.shape[0], 1)
|
||||||
self.assertEqual(set(df_all.index), {'coverage', 'mse', 'mae', 'mape'})
|
|
||||||
for metric in ['mse', 'mape', 'mae', 'coverage']:
|
for metric in ['mse', 'mape', 'mae', 'coverage']:
|
||||||
self.assertEqual(df_all[metric], df_all[metric].mean())
|
self.assertEqual(df_all[metric].values[0], df_none[metric].mean())
|
||||||
# Custom list of metrics
|
# Custom list of metrics
|
||||||
df_horizon = diagnostics.performance_metrics(
|
df_horizon = diagnostics.performance_metrics(
|
||||||
df_cv, metrics=['coverage', 'mse'],
|
df_cv, metrics=['coverage', 'mse'],
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue