Update holidays csv for R, and fix bad utf-8 parsing

This commit is contained in:
Ben Letham 2020-02-04 15:12:28 -08:00
parent 398d220743
commit f640308395
3 changed files with 30929 additions and 4326 deletions

File diff suppressed because it is too large Load diff

View file

@ -12,6 +12,7 @@ from datetime import date, timedelta
from convertdate.islamic import from_gregorian, to_gregorian
from lunarcalendar import Lunar, Converter
from lunarcalendar.converter import DateNotExist
from holidays import WEEKEND, HolidayBase
from dateutil.easter import easter
@ -604,7 +605,7 @@ class Vietnam(HolidayBase):
for offset in range(-1, 2, 1):
try:
ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 30)).to_date()
except ValueError:
except DateNotExist:
ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 29)).to_date()
if ds.year == year:
self[ds] = name

View file

@ -19,6 +19,23 @@ import holidays as hdays_part1
import fbprophet.hdays as hdays_part2
def utf8_to_ascii(text):
"""Holidays often have utf-8 characters. These are not allowed in R
package data (they generate a NOTE).
TODO: revisit whether we want to do this lossy conversion.
"""
ascii_text = (
unicodedata.normalize('NFD', text)
.encode('ascii', 'ignore')
.decode('ascii')
)
# Check if anything converted
if sum(1 for x in ascii_text if x != ' ') == 0:
return 'FAILED_TO_PARSE'
else:
return ascii_text
def generate_holidays_file():
"""Generate csv file of all possible holiday names, ds,
and countries, year combination
@ -26,8 +43,10 @@ def generate_holidays_file():
years = np.arange(1995, 2045, 1)
all_holidays = []
# class names in holiday packages which are not countries
class_to_exclude = set(['rd', 'datetime', 'date', 'HolidayBase', 'Calendar',
'LunarDate', 'timedelta', 'date'])
class_to_exclude = set([
'rd', 'date', 'Lunar', 'timedelta', 'Calendar', 'Converter', 'HolidayBase',
'DateNotExist',
])
class_list2 = inspect.getmembers(hdays_part2, inspect.isclass)
country_set2 = set(list(zip(*class_list2))[0])
@ -57,17 +76,9 @@ def generate_holidays_file():
generated_holidays['year'] = generated_holidays.ds.apply(lambda x: x.year)
generated_holidays.sort_values(['country', 'ds', 'holiday'], inplace=True)
# The holidays often have utf-8 characters.
# These are not allowed in R package data (they generate a NOTE).
# TODO: revisit whether we want to do this lossy conversion.
def utf8_to_ascii(text):
return (
unicodedata.normalize('NFD', text)
.encode('ascii', 'ignore')
.decode('ascii')
)
# Convert to ASCII, and drop holidays that fail to convert
generated_holidays['holiday'] = generated_holidays['holiday'].apply(utf8_to_ascii)
generated_holidays = generated_holidays[generated_holidays['holiday'] != 'FAILED_TO_PARSE']
generated_holidays.to_csv("../R/data-raw/generated_holidays.csv", index=False)