Don't use tqdm with multiprocessing starmap

This commit is contained in:
Ben Letham 2020-03-09 12:48:14 -07:00
parent 39c619dbf7
commit 44ceaa8d8b

View file

@ -111,18 +111,18 @@ def cross_validation(model, horizon, period=None, initial=None, multiprocess=Fal
if model.uncertainty_samples: if model.uncertainty_samples:
predict_columns.extend(['yhat_lower', 'yhat_upper']) predict_columns.extend(['yhat_lower', 'yhat_upper'])
predicts = []
cutoffs = generate_cutoffs(df, horizon, initial, period) cutoffs = generate_cutoffs(df, horizon, initial, period)
if multiprocess is True: if multiprocess is True:
with Pool() as pool: with Pool() as pool:
logger.info('Running cross validation in multiprocessing mode') logger.info('Running cross validation in multiprocessing mode')
input_df = ((df, model, cutoff, horizon, predict_columns) for cutoff in tqdm(cutoffs)) input_df = ((df, model, cutoff, horizon, predict_columns) for cutoff in cutoffs)
predicts = pool.starmap(single_cutoff_forecast, input_df) predicts = pool.starmap(single_cutoff_forecast, input_df)
else: else:
if multiprocess is False: predicts = [
for cutoff in tqdm(cutoffs): single_cutoff_forecast(df, model, cutoff, horizon, predict_columns)
predicts.append(single_cutoff_forecast(df, model, cutoff, horizon, predict_columns)) for cutoff in tqdm(cutoffs)
]
# Combine all predicted pd.DataFrame into one pd.DataFrame # Combine all predicted pd.DataFrame into one pd.DataFrame
return pd.concat(predicts, axis=0).reset_index(drop=True) return pd.concat(predicts, axis=0).reset_index(drop=True)