;docformat='rst'
;+
; This is a script that reads in level 2a etc (miscellaneous data),
;   cloud, and catalog files and regenerates the ice water content and albedo plots
;
; :Author:
;    Bill Barrett
;
; :Params:
;   start_day : number
;     the year_day_of_year such as 2018080 to begin processing
;   stop_day : number 
;    the year_day_of_year such as 2018180 to finish  processing
;
; :Keywords:
;   preliminary : boolean
;     if this keyword is set the data will be processed as preliminary
;   output_directory : string
;     an optional path where the generated png files will be written
;
; :Examples:
;   To process data from day 2018080 through day 2018180 as definitive data:
;     process_cips_days_l2, 2018080, 2018180
;   To process data from day 2018125 through 2018180 as preliminary data and write the data to '/tmp'
;     process_cips_days_l2, 2018125, 2018180, /preliminary, output_directory='/tmp'
;-
pro process_cips_days_l2, start_day, stop_day, preliminary=preliminary, output_directory=output_directory

  ; Are the start and stop days valid?
  : Eliminate days before the start of the mission
  if start_day lt 2007144L then begin
    print, get_routine_name() + 'start day before beginning of mission (2007144): ' +  strtrim(string(start_day), 2)
    return
  endif
  current_time=fix(systime(/seconds, /julian, /utc)-0.5, type=3) + 0.5
  current_ymd = long(jd2yd(current_time))
  
  ; Eliminate days in the future
  if start_day ge current_ymd then begin
    print, get_routine_name() + 'start day in the future: '  + strtrim(string(start_day), 2)
    return
  endif
  
  ; Eliminate nonsense queries where the stop is before the start
  if stop_day lt start_day then begin
    print, get_routine_name() + 'stop day before start day, start: ' + strtrim(string(start_day), 2) + $
       ', stop: ' + strtrim(string(stop_day), 2)
    return
  endif
  
  ; Limit the stop day to the present day to avoid queries for data that
  ;   cannot possibly have arrived yet.
  if stop_day gt current_ymd then begin
    stop_day = current_ymd
  endif

  set_cips_production_vars, /force
  
  set_production_flag, flag = (keyword_set(preliminary) ? 0 : 1)
  prestr = keyword_set(preliminary) ? '_prelim' : ''
  
  ci_data_root=getenv('CI_DATA_ROOT')
  if strlen(ci_data_root) le 0 then $
    ci_data_root = !DATA_PRODUCT_PATH
  version_str = string(!VERSION_2, format="(f05.2)")
  
  for year_day = start_day, stop_day do begin
    ; Find out where the input files are and ensure that they exist
    season = get_season(yd2la(year_day))
    data_directory = !DATA_PRODUCT_PATH + '/' + season + '/level_2/ver_0' + !VERSION_2 + '/rev_' + !REVISION_2
    if ~file_test(data_directory) then begin
      print, get_routine_name() + 'input data directory does not exist: ' + data_directory
      break
    endif
    if ~keyword_set(output_directory) then png_directory = data_directory

    year = long(year_day/1000L)
    year_str = string(year, format="(I04)")
    day_of_year = year_day - (1000L * year)
    day_of_year_str = string(day_of_year, format="(I03)")

    ; Get the orbit information for the day being processed
    start_time = yd2usec(year_day)
    stop_time = yd2usec(year_day + 1)
    orbit_info_array = get_orbit_info(start_time, stop_time)
    for orbit_index = 0, n_elements(orbit_info_array) -1 do begin
      orbit_number = orbit_info_array[orbit_index].orbit_number
      orbit_str = string(orbit_number, format="(I05)")

      ; level 2 file name
      root_fn = data_directory + '/cips_sci_2_orbit_' + orbit_str + $
        '_' + year_str + '-' + day_of_year_str + '_v' + version_str + '_r' + !REVISION_2 + '_'

      cat_file=root_fn+'cat'+prestr+'.nc'
      cld_file=root_fn+'cld'+prestr+'.nc'
      etc_file=root_fn+'etc'+prestr+'.nc'

      ; check if this orbit has already been completed
      completion_check=file_test(cat_file + '.gz') && $
        file_test(cld_file + '.gz') && $
        file_test(etc_file + '.gz')
      if ~completion_check then begin
        print, get_routine_name() + 'missing one or more of the required files for: ' + orbit_str + $
          ' ' + year_str + '-' + day_of_year_str
        continue
      endif
      
      etc_data = uncompress_and_expand_struct(etc_file)
      good_alb = where(finite(etc_data.alb_i), ngood)
      if (ngood gt 0) then begin
        retrieval_structure = get_retrieval_structure(!VERSION_2, /cloud)
        cld_data =  uncompress_and_expand_struct(cld_file, structure=retrieval_structure)
        retrieval_structure = get_retrieval_structure(!VERSION_2, /catalog)
        cat_data = uncompress_and_expand_struct(cat_file, structure=retrieval_structure)
        l2a_full_struct=create_struct(cat_data, cld_data, etc_data)
        make_l2_png_images, l2a_full_struct, cat_file + '.gz'
      endif else print, get_routine_name() + 'no good albedo for orbit: ' + orbit_str

      heap_gc
    endfor
  endfor

end