#!/data_systems/opt/bin/python3 """ create_delta_flat_summaries Created Oct 2019 @author: Bill Barrett """ import os.path import sys import re from collections import namedtuple from operator import attrgetter # The standard AIM CIPS location for the delta flat info files CIPS_DELTA_FLAT_DATA_PATH = '/aim/data/cips/delta_flat_info' # The file pattern the delta flat info files should follow RAW_DELTA_FLAT_FILE_PATTERN = r'(?P(nor|sou)th)_(?P20[012]\d)_delta_flat_info\.txt' # A means of associating a sorting key with the delta flat info filenames FilenameYearSeason = namedtuple('FilenameYearSeason', 'year_season filename') # The orbit lines should all be one to five decimal digits DELTA_FLAT_FILE_ORBIT_PATTERN = re.compile(r'^\d{1,5}$') # The flat file lines are fairly complex DELTA_FLAT_FILE_DATA_PATTERN = re.compile( r'''^(?P[mp][xy]) # CIPS camera \s(?P[0-3]\d\-[A-S][a-z]{2}\-20[012]\d # e.g. 21-Mar-2015 13:09:22.87 \s[0-2]\d(:[0-6]\d){2}(\.\d+)) \s(?P[0-9a-f]{1,4}) # four hex digits \s(?P\w+\.sav)$ # delta flat file name ''', re.VERBOSE) # A data structure that holds the operational id and delta flat file name DeltaFlatStatus = namedtuple('DeltaFlatStatus', 'op_id delta_flat_file') # A combination of an orbit number, start time, and the DeltaFlatStatus tuples # for each of the four cameras (mx. my, px, py) DeltaFlatDatum = namedtuple('DeltaFlatDatum', 'orbit start_time camera_data') class DeltaFlatSummarizer: """ Create season by season summaries from the complete delta flat data files """ def __init__(self, input_delta_flat_directory): """ Initialize the directory path for the input files Parameters ---------- input_delta_flat_directoryy : string the directory where the delta flat info TLE file will be found """ assert os.path.isdir(input_delta_flat_directory), '{} is not a valid directory path'.format( input_delta_flat_directory) self.delta_flat_directory = input_delta_flat_directory def get_ordered_file_list(self): """ Get an ordered list of the available input files. The order that is desired is the by year and season. I.e. the following is the correct order for these items: north_2015_delta_flat_info.txt south_2015_delta_flat_info.txt north_2016_delta_flat_info.txt south_2016_delta_flat_info.txt north_2017_delta_flat_info.txt Returns ------- output_files : list The list of FilenameYearSeason tuples sorted by the 'year_season' key """ output_files = [] for filename in os.listdir(self.delta_flat_directory): filename_match = re.match(RAW_DELTA_FLAT_FILE_PATTERN, filename) if filename_match is not None: output_files.append(FilenameYearSeason( filename_match.group('year') + filename_match.group('season'), filename)) return sorted(output_files, key=attrgetter('year_season')) @staticmethod def read_delta_flat_file(filename): """ Produce file that summarizes the content of the seasonal delta flat info files Parameters ---------- ordered_file_list : list The list of FilenameYearSeason tuples sorted by the 'year_season' key Returns ------- """ delta_flat_change_list = [] orbit = None with open(filename) as file_pointer: while True: line = file_pointer.readline().rstrip() if not line: break if orbit is None: orbit_match = DELTA_FLAT_FILE_ORBIT_PATTERN.match(line) if orbit_match is not None: orbit = line camera_data = {} continue delta_flat_data_match = DELTA_FLAT_FILE_DATA_PATTERN.match(line) if delta_flat_data_match is not None: if orbit is not None: camera_data[delta_flat_data_match.group('camera')] = DeltaFlatStatus( delta_flat_data_match.group('op_id'), delta_flat_data_match.group('delta_flat_file')) if len(camera_data) == 4 and orbit is not None: delta_flat_datum = \ DeltaFlatDatum( orbit, delta_flat_data_match.group('start_time'), camera_data) if delta_flat_change_list: if delta_flat_change_list[-1].camera_data != \ delta_flat_datum.camera_data: delta_flat_change_list.append(delta_flat_datum) orbit = None else: delta_flat_change_list = [delta_flat_datum] return delta_flat_change_list if __name__ == '__main__': """ In the operational directory this is executed from the shell script via the following lines from 'create_cips_vetted_files_metadata_db.sh' Arguments ---------- argv[1] : str a data type in the keys of QUERIES_AND_FILES_TYPES argv[2] : str the name of the database file """ if len(sys.argv) >= 2: RAW_DELTA_FLAT_PATH = sys.argv[1] else: RAW_DELTA_FLAT_PATH = CIPS_DELTA_FLAT_DATA_PATH DELTA_FLAT_SUMMARIZER = DeltaFlatSummarizer(RAW_DELTA_FLAT_PATH) ORDERED_FILE_LIST = DELTA_FLAT_SUMMARIZER.get_ordered_file_list() for delta_flat_file in ORDERED_FILE_LIST: season = delta_flat_file.year_season[4:] year = delta_flat_file.year_season[:4] delta_flat_change_list = DeltaFlatSummarizer.read_delta_flat_file( os.path.join(CIPS_DELTA_FLAT_DATA_PATH, delta_flat_file.filename)) output_file_name = '{0}_{1}_delta_flat_summary.txt'.format(season, year) output_file_path = os.path.join(CIPS_DELTA_FLAT_DATA_PATH, output_file_name) with open(output_file_path, 'w') as output_file: for delta_flat_datum in delta_flat_change_list: print('{0} {1} mx: {2} {3}, my: {4} {5}, px: {6} {7}, py: {8} {9}'. format(delta_flat_datum.orbit, delta_flat_datum.start_time, delta_flat_datum.camera_data['mx'].op_id, delta_flat_datum.camera_data['mx'].delta_flat_file, delta_flat_datum.camera_data['my'].op_id, delta_flat_datum.camera_data['my'].delta_flat_file, delta_flat_datum.camera_data['px'].op_id, delta_flat_datum.camera_data['px'].delta_flat_file, delta_flat_datum.camera_data['py'].op_id, delta_flat_datum.camera_data['py'].delta_flat_file, )) output_file.write('{0} {1} mx: {2} {3}, my: {4} {5}, px: {6} {7}, py: {8} {9}\n'. format(delta_flat_datum.orbit, delta_flat_datum.start_time, delta_flat_datum.camera_data['mx'].op_id, delta_flat_datum.camera_data['mx'].delta_flat_file, delta_flat_datum.camera_data['my'].op_id, delta_flat_datum.camera_data['my'].delta_flat_file, delta_flat_datum.camera_data['px'].op_id, delta_flat_datum.camera_data['px'].delta_flat_file, delta_flat_datum.camera_data['py'].op_id, delta_flat_datum.camera_data['py'].delta_flat_file, ))