#!/data_systems/opt/bin/python3
""" processing_summary_emailer
Send out the daily AIM CIPS processing status report
This is a translation and augmentation of the perl script of the same name.

Abbreviations used:
   PMC -- Polar Mesospheric Cloud
   RAA -- Rayleigh Albedo Anomaly

Created  2019-05

@author: Bill Barrett
"""

from datetime import datetime, timedelta
import re
import socket
import os
from collections import namedtuple

from email_utils import EmailController
from aimpi_shell_execution import AimPiShellExecution
from season_and_version_utils import SeasonAndVersionUtils
from aim_html_formatter import AIMHTMLFormatter, FOUR_SPACES_HTML

# The subject line for the email
SUBJECT = 'CIPS Daily System Status Report'

# The recipients for the email
MAIL_RECIPIENTS = ['james.craft@lasp.colorado.edu', 'adrian.gehr@lasp.colorado.edu']

# The email sender
SENDER = 'aimpi@lasp.colorado.edu'

# The host hame may be returned as either 'aimcips' or 'aimcips.lasp.colorado.edu'
# The only useful part is what precedes 'lasp.colorado.edu'
HOST_PATTERN = re.compile(r'(?P<hostname>aimcips(\d|\-gpu)?)\..+')

# Currently these are the only AIM hosts that can send email
SUPPORTED_HOSTS = ['aimcips', 'aimcips-gpu']

# The current AIM servers and the hosts that they can be accessed from
AIM_SERVERS = {
    'aim': SUPPORTED_HOSTS,
    'aim2': SUPPORTED_HOSTS,
    'aim3': ['aimcips-gpu']
}

# The line to be parsed should look like
# 'lds:/export/aim2  8.0T  5.5T  2.6T  69% /aim2'
DISK_USAGE = re.compile(r'''[\w/:]+      # beginning of line
    (\s+\d+(\.\d+)?[KGT]){2}             # total and used space (ignored)
    \s+(?P<free>\d+(\.\d+)?[KGT])        # free space
    \s+(?P<usage>\d+)%                   # per cent usage of disk
    \s+/(?P<disk>\w+)                    # the disk used
    ''', re.VERBOSE)

# There should be at least 14 orbits worth of data per day
MINIMUM_ORBITS_PER_DAY = 14

# For each level there are a minimum number of files, a regex (regular expression)
# to specify the file type, and part of the message to be printed
MinFilesAndRegex = namedtuple('MinFilesAndRegex', 'minimum_files regex message')

# The file where the PMC version and revision information is kept
PMC_VERSION_AND_REVISION_FILE = '/aim/sds/cips/set_cips_production_vars.pro'

# Levels, minimum number of files, defining regex, and text for PMC data
PMC_MIN_FILES_AND_REGEX_PER_LEVEL = {
    # For level 1A there are files for each camera (mx, px, my, py) for each orbit
    '1a': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY * 4, r'_cam_[mp][xy]_v.+\d\..*nc\.gz', \
                           'data files created'),
    # For level 2 there should be an albedo image file per orbit
    '2': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY, r'_v.+_alb\..*png', 'orbits processed'),
    # For level 3 there is only one daisy per day
    '3': MinFilesAndRegex(1, r'_v.+\d\..*web\.png', 'daisies created')
}

# The file where the RAA version and revision information is kept
RAA_VERSION_AND_REVISION_FILE = \
    '/aim/sds/cips/data_production/level_2CI/proc/default_version_numbers.bash'

# Levels, minimum number of files, defining regex, and text for RAA data
RAA_MIN_FILES_AND_REGEX_PER_LEVEL = {
    # Both levels 2a and 2b are orbit based
    #'2a': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY, r'_v.+_alb\.(preliminary\.)?nc\.gz', 'orbits processed'),
    #'2b': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY, r'_v.+_alb\.(preliminary\.)?nc\.gz', 'orbits processed')
    '2a': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY, r'_v.+_alb_prelim+.?nc\.gz', 'orbits processed'),
    '2b': MinFilesAndRegex(MINIMUM_ORBITS_PER_DAY, r'_v.+_alb_prelim+.?nc\.gz', 'orbits processed')
}

# The pattern used for finding the data sub directories
DIR_PATTERN = re.compile(r'''.+/level_\d[a-z]?  # the data level
    (/[mp][xy])?                                # 1a data includes the camera
    /ver_(?P<version>\d{2}\.\d{2})              # data version
    /rev_(?P<revision>\d{2})                    # data revision
    ''', re.VERBOSE)

# The AIM CIPS root data directory
AIM_CIPS_DATA_ROOT = '/aim/data/cips'

# If it is necessary to use ssh to reach another server
AIM_SSH_USER = 'aimpro'

# PAN (Public Availabilty Notice)
PAN_DIRECTORY = '/aim/pdc/processed_pans/'

# The file name and message to priont for TLE (two line element)
# and spacecraf clock offset
TLE_AND_CLOCK_INFO = {
    '/home/aimsds/tle_incoming/aim_tle.save': 'TLE file',
    '/home/aimsds/aim_clock_incoming/clock_info.txt': 'spacecraft clock info',
    '/aim/pdc/cips_vetted_files_info_l3.txt': 'CIPS vetted files (orbits) list'
}

# There are 86400 seconds per day
SECONDS_PER_DAY = 86400


class AIMProcessingSummary:
    """ Produce the HTML text for an AIM processing summary email
    """

    def __init__(self):
        """ Verify that the host this is run on can send email
        Get the date for which this information is valid
        and the date and season for the definitive data
        """
        # Verify that the host this is run on can send email
        self.hostname = self.verify_host()

        # Get the date for which this information is valid
        self.today = datetime.today()
        self.processing_date = self.today.strftime('%Y-%j')
        # The date for the most recently processed definitive files is six days ago
        definitive_date = datetime.today() - timedelta(days=6)
        self.definitive_date = definitive_date.strftime('%Y-%j')
        # The season for the definitive data
        self.season = SeasonAndVersionUtils.get_season(self.processing_date)
        self.year, self.day_of_year = SeasonAndVersionUtils.validate_year_day_of_year(self.definitive_date)
        self.month = SeasonAndVersionUtils.get_month(self.definitive_date)

    @staticmethod
    def verify_host():
        """ Verify that the host this is an AIM host that  can send email

        Returns
        -------
        hostname : str
            the hostname less qualifiers like '.lasp.colorado.edy'
        """
        # Verify that the host this is run on can send eamil
        full_hostname = socket.gethostbyaddr(socket.gethostname())[0]
        hostname_match = HOST_PATTERN.match(full_hostname)
        assert hostname_match is not None, '{} is not a recognized host'.format(full_hostname)
        hostname = hostname_match.group('hostname')
        assert hostname in SUPPORTED_HOSTS, \
            'email can NOT be sent from {}'.format(hostname)
        return hostname

    def get_header(self):
        """Initialize the HTML body and the header for the AIM processing summary email

        Returns
        -------
        header : str
            the body initialization and header for the email message
        """
        # Set the background color for the email text
        header = '<body style="background-color:Beige;">'
        # Now set the title at the the top of the email
        title = 'CIPS Daily Processing Summary for {}'.format(self.processing_date)
        header += AIMHTMLFormatter.set_html_text_color('blue', False) + \
            AIMHTMLFormatter.html_center_text(AIMHTMLFormatter.html_header(title, 1)) + '</p>'
        header += AIMHTMLFormatter.html_center_text(AIMHTMLFormatter.html_header(
            '/aim/sds/cips/scripts/processing_summary_emailer.py', 4)) + '</p>'
        return header

    def get_disk_space(self):
        """ Get the amount of free disk space on the various aim servers

        Returns
        -------
        text : str
            the disk usage message
        """
        text = AIMHTMLFormatter.set_html_text_color('black', False) + \
            AIMHTMLFormatter.html_header(
                AIMHTMLFormatter.html_underline(
                    'CIPS Disk Status'), 2) + '</p>'
        for disk in sorted(AIM_SERVERS):
            command = 'df -h /{}'.format(disk)
            # It is best if the command can be run locally
            # If not use ssh to run remotely
            if self.hostname in AIM_SERVERS[disk]:
                result_code, output = AimPiShellExecution.tokenize_and_execute_command(command)
            else:
                result_code, output = AimPiShellExecution.execute_ssh_command(
                    AIM_SSH_USER, (AIM_SERVERS[disk])[0], command)
            assert result_code == 0, '"{0}" failed, result code:{1}. output: "{2}"'.format(
                command, result_code, output)

            # Search through the output for the usage statistics
            for line in output.split('\n'):
                usage_match = DISK_USAGE.match(line)
                if usage_match is not None:
                    assert disk == usage_match.group('disk'), \
                        'wanted to get data for {0}, received data from {1}'.format(
                            disk, usage_match.group('disk'))
                    free_space = usage_match.group('free')
                    percent_free = 100 - int(usage_match.group('usage'))
                    text += AIMHTMLFormatter.get_html_ok_not_ok_color(percent_free >= 10) + \
                        '{0} has {1} ({2}%) free</p>'.format(disk, free_space, percent_free)
                    break
        return text

    def get_definitive_status(
            self, header_label, extra_dir, levels_and_regex_dict, version_revision_file, is_pmc):
        """ Get the current definitive processing status

        Parameters
        ----------
        header_label: str
            a label like 'PMC' or 'RAA' for the header for this section

        extra_dir: str
            a string to be added to the path if necessary, '' for PMC, 'raa' for 'RAA'

        levels_and_regex_dict :dictionary
            a dictionary keyed by the data level with regex and other information
            for that level

        version_revision_file : str
            the absolute path for the file for where to find the version and revision info

        is_pmc: bool
            True if for 'PMC' data, false if for 'RAA'


        Returns
        -------
        text : str
            the current PMC processing status message
        """
        title = AIMHTMLFormatter.html_underline('{0} Definitive Processing Status for {1}'.format(
            header_label, self.definitive_date))
        text = AIMHTMLFormatter.set_html_text_color('black', False) + \
            AIMHTMLFormatter.html_header(title, 2) + '</p>'
        aim_data_season_dir = os.path.join(AIM_CIPS_DATA_ROOT, self.season)
        if extra_dir:
            aim_data_season_dir = os.path.join(AIM_CIPS_DATA_ROOT, 'raa_' + str(self.year), self.month)
        # For each level in the data find the directories that match
        # the version and revision, and then look for files that match
        # the file date
        for level, min_files_and_regex in sorted(levels_and_regex_dict.items()):
            level_count = 0
            version, revision = SeasonAndVersionUtils.get_version_and_revision(
                version_revision_file, is_pmc, level=level)
            level_str = 'level_{}'.format(level)
            level_directory = os.path.join(aim_data_season_dir, level_str)
            for search_root, _, _ in os.walk(level_directory):
                dir_match = DIR_PATTERN.match(search_root)
                if dir_match is not None:
                    # Convert the version and revision to numbers to
                    # eliminate potential problems with leading zeros
                    if float(dir_match.group('version')) == float(version) and \
                            int(dir_match.group('revision')) == int(revision):
                        file_pattern = r'.+_' + self.definitive_date + min_files_and_regex.regex
                        files = [f for f in os.listdir(search_root) if re.match(file_pattern, f)]
                        level_count += len(files)
                if level_count > 0 and not is_pmc:
                    break
            message = 'Number of level {0} version {1} revision {2} {3}:{4}{5}'.format(
                level.upper(), version, revision, min_files_and_regex.message,
                FOUR_SPACES_HTML, level_count)
            text += AIMHTMLFormatter.get_html_ok_not_ok_color(
                level_count >= min_files_and_regex.minimum_files) + message
        return text

    def get_tle_and_pan_status(self):
        """ Get the status for the TLE (Two line element) and spacecraft clock offset files
        Print the time of the most recent PAN (Public Availability Notice)file

        Returns
        -------
        text : str
            the TLE, spacecraft clock, vetted files list, and PAN message
        """
        text = AIMHTMLFormatter.set_html_text_color('black', False) + \
            AIMHTMLFormatter.html_header(FOUR_SPACES_HTML + AIMHTMLFormatter.html_underline(
                'TLE, SC Clock, and PAN Status'), 2) + '</p>'

        # The process is identical for getting the time of the last TLE and the
        # time of the last spacecraft clock update
        for filename, value in TLE_AND_CLOCK_INFO.items():
            file_time = os.path.getmtime(filename)
            # Convert difference in seconds per day to an integer number of days
            time_delta = int((self.today.timestamp() - file_time) / SECONDS_PER_DAY)
            file_date = datetime.fromtimestamp(file_time)
            text += AIMHTMLFormatter.get_html_ok_not_ok_color(time_delta <= 3) + \
                '{0} is {1} days old{2}({3})</p>'.format(value, time_delta, FOUR_SPACES_HTML,
                                                         file_date.strftime('%Y/%m/%d %H:%M:%S'))

        # Print a message about the most recent PAN file
        pan_files = [f for f in os.listdir(PAN_DIRECTORY) if f.endswith('png.pan')]
        sorted_pan_files = sorted(pan_files, reverse=True)
        text += AIMHTMLFormatter.set_html_text_color('black') + \
            'The most recent PAN file is:' + \
            AIMHTMLFormatter.html_preserve_formatting(sorted_pan_files[0])
        return text

    def get_summary_file_times(self):
        """ Get the times for the most recent level 3c summary files
        """
        text = AIMHTMLFormatter.set_html_text_color('black', False) + \
            AIMHTMLFormatter.html_header(FOUR_SPACES_HTML + AIMHTMLFormatter.html_underline(
                'Summary Files Status'), 2) + '</p>'
        version, revision = SeasonAndVersionUtils.get_version_and_revision(
            PMC_VERSION_AND_REVISION_FILE, True, level='3')
        summary_files_dir = os.path.join(AIM_CIPS_DATA_ROOT, self.season, 'level_3c',
                                         'ver_0{}'.format(version),  'rev_{}'.format(revision))

        # Because the 3c files are not created until PMC clouds are seen, these files are
        # often not present at the beginning of a season
        if os.path.exists(summary_files_dir):
            files = os.listdir(summary_files_dir)
            if files:
                first_file = os.path.join(summary_files_dir, files[0])
                file_time = os.path.getmtime(first_file)
                # Convert difference in seconds per day to an integer number of seconds
                time_delta = int((self.today.timestamp() - file_time) / SECONDS_PER_DAY)
                file_date = datetime.fromtimestamp(file_time)
                text += AIMHTMLFormatter.get_html_ok_not_ok_color(time_delta <= 3) + \
                    '{0} is {1} days old{2}({3})</p>'.format(
                        files[0], time_delta, FOUR_SPACES_HTML,
                    file_date.strftime('%Y/%m/%d %H:%M:%S'))
            else:
                # directory has been created for the season but no data yet
                text += AIMHTMLFormatter.get_html_ok_not_ok_color(False) + \
                    'no summary files exist for {0} version {1} revision {2} summary files'.format(
                        self.season, version, revision)
        else:
            # too early in the season to even create the directory
            text += AIMHTMLFormatter.get_html_ok_not_ok_color(False) + \
                'directory does not yet exist for {0} version {1} revision {2} summary files' \
                .format(self.season, version, revision)
        return text


if __name__ == "__main__":
    """
    Generate the text and send out the daily AIM CIPS processing status email
    """
    AIM_PROCESSING_SUMMARY = AIMProcessingSummary()
    EMAIL_TEXT = AIM_PROCESSING_SUMMARY.get_header()
    EMAIL_TEXT += AIM_PROCESSING_SUMMARY.get_disk_space()
    EMAIL_TEXT += AIM_PROCESSING_SUMMARY.get_definitive_status(
        'PMC', '', PMC_MIN_FILES_AND_REGEX_PER_LEVEL, PMC_VERSION_AND_REVISION_FILE, True)
    EMAIL_TEXT += AIM_PROCESSING_SUMMARY.get_definitive_status(
        'RAA', 'raa', RAA_MIN_FILES_AND_REGEX_PER_LEVEL, RAA_VERSION_AND_REVISION_FILE, False)
    EMAIL_TEXT += AIM_PROCESSING_SUMMARY.get_tle_and_pan_status()
    EMAIL_TEXT += AIM_PROCESSING_SUMMARY.get_summary_file_times()
    EMAIL_TEXT += AIMHTMLFormatter.html_preserve_formatting(' ')
    EmailController.send_html_email(SUBJECT, SENDER, MAIL_RECIPIENTS, EMAIL_TEXT)