#!/data_systems/opt/bin/python3 """ delete_superseded_pre_files Delete unneeded preliminary data files from /aim/data/cips re AIMSDS-333 For RAA and PMC files the goal is to delete those files marked preliminary that are superseded by definitive files for the same date. Abbreviations used: PMC -- Polar Mesospheric Cloud RAA -- Rayleigh Albedo Anomaly Created 2020-04 @author: Bill Barrett """ import time import os.path import traceback import re from datetime import datetime, timedelta from season_and_version_utils import SeasonAndVersionUtils from processing_summary_emailer import \ RAA_VERSION_AND_REVISION_FILE, PMC_VERSION_AND_REVISION_FILE # The directory where all current AM CIPS data resides AIM_CIPS_DATA_DIRECTORY = '/aim/data/cips' # The "_prelim" files to be deleted will all match this pattern PRELIM_PATTERN = re.compile(r'.+_prelim\.[ntp].+') # The RAA levels to be checked #RAA_LEVELS = ['2a', '2b'] RAA_LEVELS = [] # don't do RAA cleanup for now # The PMC levels to be checked #PMC_LEVELS = ['2', '3'] PMC_LEVELS = [] # don't do higher-level PMC cleanup for now, because we don't want to release the definitive until a postseason reprocessing # This isn't pretty but it should work PMC_1A_LEVELS = ['1a/mx', '1a/my', '1a/px', '1a/py'] class DeleteSupersededPreFiles: """ Delete unneeded preliminary data files """ def __init__(self, directory_name): """ Verify that the directory path exists and set the dates / seasons to be checked Parameters ---------- directory_name : str the base directory from which to operate Raises: ValueError if the directory path to be checked does not exist """ # Set the directory path to be checked if not os.path.exists(directory_name): raise ValueError('directory does not exist: {}'.format(directory_name)) self.base_dir = directory_name # Get today's date and from that get the sub paths to be checked today = datetime.now() # However we want to look back two weeks in time # in case anything was missed two_weeks_ago = today - timedelta(days=14) today_ym = datetime.strftime(today, '%Y-%m') # The year and day of year are used to determine the PMC seasons # The year and month determine the RAA directory today_ydoy = datetime.strftime(today, '%Y-%j') two_weeks_ago_ym = datetime.strftime(two_weeks_ago, '%Y-%m') two_weeks_ago_ydoy = datetime.strftime(two_weeks_ago, '%Y-%j') today_season = SeasonAndVersionUtils.get_season(today_ydoy) two_weeks_ago_season = SeasonAndVersionUtils.get_season(two_weeks_ago_ydoy) # There are two cases for both the PMC and RAA data # If we are at a season boundary (PMC) or the beginning of a month (RAA), # then we need to look at both the current and the previous # Otherwise we need only the preent one if today_season == two_weeks_ago_season: self.seasons = [ today_season ] else: self.seasons = [ two_weeks_ago_season, today_season ] if today_ym == two_weeks_ago_ym: self.ym = [ today_ym ] else: self.ym = [ two_weeks_ago_ym, today_ym ] def __enter__(self): """ Cycle through the PMC seasons and RAA year months, deleting superseded preliminary files """ # First check the RAA data for unneeded preliminary files # This is done by year and month version, revision = SeasonAndVersionUtils.get_version_and_revision(RAA_VERSION_AND_REVISION_FILE, False) for ym in self.ym: year_month = ym.split('-') base_dir = os.path.join(self.base_dir, 'raa_{}'.format(year_month[0])) for level in RAA_LEVELS: level_dir = os.path.join(base_dir, year_month[1], 'level_{}'.format(level)) self.delete_obs_files(level_dir, version, revision) # Next the check by season for unnecessary preliminary PMC files version, revision = SeasonAndVersionUtils.get_version_and_revision(PMC_VERSION_AND_REVISION_FILE, True) # If the version is missing a leading '0', add it if len(version) < 5: version = '0' + version for season in self.seasons: base_dir = os.path.join(self.base_dir, season) for level in PMC_LEVELS: level_dir = os.path.join(base_dir, 'level_{}'.format(level)) self.delete_obs_files(level_dir, version, revision) for level_1a in PMC_1A_LEVELS: revision = '01' level_dir = os.path.join(base_dir, 'level_{}'.format(level_1a)) self.delete_obs_files(level_dir, version, revision) return self def __exit__(self, exception_type, exception_value, traceback_info): """ Necessary if class has __enter__ True for a normal exit, False if the exception_type is not None """ if exception_type is not None: print(time.strftime("%Y-%m-%d %H:%M:%S") + ' type: {0}, value: {1}'.format(exception_type, exception_value)) traceback.print_exception(exception_type, exception_value, traceback_info) return False # Comment to pass exception through return True def delete_obs_files(self, level_dir, version, revision): """ Parameters ---------- level_dir : str the base directory from which to operate version : str the base directory from which to operate revision : str the base directory from which to operate """ ver_rev_dir = os.path.join(level_dir, 'ver_{}'.format(version), 'rev_{}'.format(revision)) print('\n'+ver_rev_dir) prelim_files = [f for f in os.listdir(ver_rev_dir) if PRELIM_PATTERN.match(f) ] if not prelim_files: return prelim_files.sort() for pfile in prelim_files: prelim_file = os.path.join(ver_rev_dir, pfile) definitive_file = re.sub('_prelim', '', prelim_file) if os.path.exists(definitive_file): try: os.remove(prelim_file) print('Deleted: {}'.format(pfile)) # Report failures, but don't stop the process except OSError as e: print (time.strftime("%Y-%m-%d %H:%M:%S") + ' Error deleting: {0} - {1}'.format(e.filename, e.strerror)) if __name__ == "__main__": """ """ with DeleteSupersededPreFiles(AIM_CIPS_DATA_DIRECTORY) as delete_superseded_pre_files: pass