Update holidays csv for R, and fix bad utf-8 parsing

This commit is contained in:
Ben Letham 2020-02-04 15:12:28 -08:00
parent 398d220743
commit f640308395
3 changed files with 30929 additions and 4326 deletions

File diff suppressed because it is too large Load diff

View file

@ -12,6 +12,7 @@ from datetime import date, timedelta
from convertdate.islamic import from_gregorian, to_gregorian from convertdate.islamic import from_gregorian, to_gregorian
from lunarcalendar import Lunar, Converter from lunarcalendar import Lunar, Converter
from lunarcalendar.converter import DateNotExist
from holidays import WEEKEND, HolidayBase from holidays import WEEKEND, HolidayBase
from dateutil.easter import easter from dateutil.easter import easter
@ -604,7 +605,7 @@ class Vietnam(HolidayBase):
for offset in range(-1, 2, 1): for offset in range(-1, 2, 1):
try: try:
ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 30)).to_date() ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 30)).to_date()
except ValueError: except DateNotExist:
ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 29)).to_date() ds = Converter.Lunar2Solar(Lunar(year - 1 + offset, 12, 29)).to_date()
if ds.year == year: if ds.year == year:
self[ds] = name self[ds] = name

View file

@ -19,6 +19,23 @@ import holidays as hdays_part1
import fbprophet.hdays as hdays_part2 import fbprophet.hdays as hdays_part2
def utf8_to_ascii(text):
"""Holidays often have utf-8 characters. These are not allowed in R
package data (they generate a NOTE).
TODO: revisit whether we want to do this lossy conversion.
"""
ascii_text = (
unicodedata.normalize('NFD', text)
.encode('ascii', 'ignore')
.decode('ascii')
)
# Check if anything converted
if sum(1 for x in ascii_text if x != ' ') == 0:
return 'FAILED_TO_PARSE'
else:
return ascii_text
def generate_holidays_file(): def generate_holidays_file():
"""Generate csv file of all possible holiday names, ds, """Generate csv file of all possible holiday names, ds,
and countries, year combination and countries, year combination
@ -26,8 +43,10 @@ def generate_holidays_file():
years = np.arange(1995, 2045, 1) years = np.arange(1995, 2045, 1)
all_holidays = [] all_holidays = []
# class names in holiday packages which are not countries # class names in holiday packages which are not countries
class_to_exclude = set(['rd', 'datetime', 'date', 'HolidayBase', 'Calendar', class_to_exclude = set([
'LunarDate', 'timedelta', 'date']) 'rd', 'date', 'Lunar', 'timedelta', 'Calendar', 'Converter', 'HolidayBase',
'DateNotExist',
])
class_list2 = inspect.getmembers(hdays_part2, inspect.isclass) class_list2 = inspect.getmembers(hdays_part2, inspect.isclass)
country_set2 = set(list(zip(*class_list2))[0]) country_set2 = set(list(zip(*class_list2))[0])
@ -57,17 +76,9 @@ def generate_holidays_file():
generated_holidays['year'] = generated_holidays.ds.apply(lambda x: x.year) generated_holidays['year'] = generated_holidays.ds.apply(lambda x: x.year)
generated_holidays.sort_values(['country', 'ds', 'holiday'], inplace=True) generated_holidays.sort_values(['country', 'ds', 'holiday'], inplace=True)
# The holidays often have utf-8 characters. # Convert to ASCII, and drop holidays that fail to convert
# These are not allowed in R package data (they generate a NOTE).
# TODO: revisit whether we want to do this lossy conversion.
def utf8_to_ascii(text):
return (
unicodedata.normalize('NFD', text)
.encode('ascii', 'ignore')
.decode('ascii')
)
generated_holidays['holiday'] = generated_holidays['holiday'].apply(utf8_to_ascii) generated_holidays['holiday'] = generated_holidays['holiday'].apply(utf8_to_ascii)
generated_holidays = generated_holidays[generated_holidays['holiday'] != 'FAILED_TO_PARSE']
generated_holidays.to_csv("../R/data-raw/generated_holidays.csv", index=False) generated_holidays.to_csv("../R/data-raw/generated_holidays.csv", index=False)