Calibration Smoothing¶

by Josh Dillon, last updated March 29, 2023

This notebook runs calibration smoothing to the gains coming out of file_calibration notebook. It removes any flags founds on by that notebook and replaces them with flags generated from full_day_rfi and full_day_antenna_flagging. It also plots the results for a couple of antennas.

Here's a set of links to skip to particular figures and tables:

• Figure 1: Full-Day Gain Amplitudes Before and After smooth_cal¶

• Figure 2: Full-Day Gain Phases Before and After smooth_cal¶

• Figure 3: Full-Day $\chi^2$ / DoF Waterfall from Redundant-Baseline Calibration¶

• Figure 4: Average $\chi^2$ per Antenna vs. Time and Frequency¶

In [1]:
import time
tstart = time.time()
In [2]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin  # REQUIRED to have the compression plugins available
import numpy as np
import glob
import copy
import warnings
import matplotlib
import matplotlib.pyplot as plt
from hera_cal import io, utils, smooth_cal
from hera_qm.time_series_metrics import true_stretches
%matplotlib inline
from IPython.display import display, HTML

Parse inputs¶

In [3]:
# get files
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = '/users/jsdillon/lustre/H6C/abscal/2459853/zen.2459853.25518.sum.uvh5'
SUM_SUFFIX = os.environ.get("SUM_SUFFIX", 'sum.uvh5')
CAL_SUFFIX = os.environ.get("CAL_SUFFIX", 'sum.omni.calfits')
SMOOTH_CAL_SUFFIX = os.environ.get("SMOOTH_CAL_SUFFIX", 'sum.smooth.calfits')
ANT_FLAG_SUFFIX = os.environ.get("ANT_FLAG_SUFFIX", 'sum.antenna_flags.h5')
RFI_FLAG_SUFFIX = os.environ.get("RFI_FLAG_SUFFIX", 'sum.flag_waterfall.h5')
FREQ_SMOOTHING_SCALE = float(os.environ.get("FREQ_SMOOTHING_SCALE", 10.0)) # MHz
TIME_SMOOTHING_SCALE = float(os.environ.get("TIME_SMOOTHING_SCALE", 6e5)) # seconds
EIGENVAL_CUTOFF = float(os.environ.get("EIGENVAL_CUTOFF", 1e-12))
OUT_YAML_SUFFIX = os.environ.get("OUT_YAML_SUFFIX", '_aposteriori_flags.yaml')
OUT_YAML_DIR = os.environ.get("OUT_YAML_DIR", None)

if OUT_YAML_DIR is None:
    OUT_YAML_DIR = os.path.dirname(SUM_FILE)
out_yaml_file = os.path.join(OUT_YAML_DIR, SUM_FILE.split('.')[-4] + OUT_YAML_SUFFIX)    

for setting in ['SUM_FILE', 'SUM_SUFFIX', 'CAL_SUFFIX', 'SMOOTH_CAL_SUFFIX', 'ANT_FLAG_SUFFIX', 'RFI_FLAG_SUFFIX',
                'FREQ_SMOOTHING_SCALE', 'TIME_SMOOTHING_SCALE', 'EIGENVAL_CUTOFF', 'out_yaml_file']:
        print(f'{setting} = {eval(setting)}')

SUM_FILE = /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/zen.2459869.25299.sum.uvh5
SUM_SUFFIX = sum.uvh5
CAL_SUFFIX = sum.omni.calfits
SMOOTH_CAL_SUFFIX = sum.smooth.calfits
ANT_FLAG_SUFFIX = sum.antenna_flags.h5
RFI_FLAG_SUFFIX = sum.flag_waterfall.h5
FREQ_SMOOTHING_SCALE = 10.0
TIME_SMOOTHING_SCALE = 600000.0
EIGENVAL_CUTOFF = 1e-12
out_yaml_file = /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/2459869_aposteriori_flags.yaml

Load files¶

In [4]:
sum_glob = '.'.join(SUM_FILE.split('.')[:-3]) + '.*.' + SUM_SUFFIX
cal_files_glob = sum_glob.replace(SUM_SUFFIX, CAL_SUFFIX)
cal_files = sorted(glob.glob(cal_files_glob))
print(f'Found {len(cal_files)} *.{CAL_SUFFIX} files starting with {cal_files[0]}.')

Found 1862 *.sum.omni.calfits files starting with /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/zen.2459869.25299.sum.omni.calfits.
In [5]:
rfi_flag_files_glob = sum_glob.replace(SUM_SUFFIX, RFI_FLAG_SUFFIX)
rfi_flag_files = sorted(glob.glob(rfi_flag_files_glob))
print(f'Found {len(rfi_flag_files)} *.{RFI_FLAG_SUFFIX} files starting with {rfi_flag_files[0]}.')

Found 1862 *.sum.flag_waterfall.h5 files starting with /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/zen.2459869.25299.sum.flag_waterfall.h5.
In [6]:
ant_flag_files_glob = sum_glob.replace(SUM_SUFFIX, ANT_FLAG_SUFFIX)
ant_flag_files = sorted(glob.glob(ant_flag_files_glob))
print(f'Found {len(ant_flag_files)} *.{ANT_FLAG_SUFFIX} files starting with {ant_flag_files[0]}.')

Found 1862 *.sum.antenna_flags.h5 files starting with /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/zen.2459869.25299.sum.antenna_flags.h5.
In [7]:
cs = smooth_cal.CalibrationSmoother(cal_files, flag_file_list=(ant_flag_files + rfi_flag_files), ignore_calflags=True,
                                    pick_refant=True, propagate_refant_flags=True, load_chisq=True, load_cspa=True)
for pol in cs.refant:
    print(f'Reference antenna {cs.refant[pol][0]} selected for {pol}.')

Mean of empty slice

Reference antenna 106 selected for Jnn.
Reference antenna 105 selected for Jee.
In [8]:
# duplicate a small number of abscal gains for plotting
antnums = set([ant[0] for ant in cs.ants])
flags_per_antnum = [np.sum(cs.flag_grids[ant, 'Jnn']) + np.sum(cs.flag_grids[ant, 'Jee']) for ant in antnums]
refant_nums = [ant[0] for ant in cs.refant.values()]
candidate_ants = [ant for ant, nflags in zip(antnums, flags_per_antnum) if (ant not in refant_nums) and (nflags <= np.percentile(flags_per_antnum, 25))
                  and not np.all(cs.flag_grids[ant, 'Jee']) and not np.all(cs.flag_grids[ant, 'Jnn'])]
ants_to_plot = [func(candidate_ants) for func in (np.min, np.max)]
abscal_gains = {(ant, pol): np.array(cs.gain_grids[(ant, pol)]) for ant in ants_to_plot for pol in ['Jee', 'Jnn']}

Perform smoothing¶

In [9]:
cs.time_freq_2D_filter(freq_scale=FREQ_SMOOTHING_SCALE, time_scale=TIME_SMOOTHING_SCALE, eigenval_cutoff=EIGENVAL_CUTOFF, 
                       method='DPSS', fit_method='lu_solve', fix_phase_flips=True, flag_phase_flip_ints=True)

No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)

Plot results¶

In [10]:
lst_grid = utils.JD2LST(cs.time_grid) * 12 / np.pi
lst_grid[lst_grid > lst_grid[-1]] -= 24
In [11]:
def amplitude_plot(ant_to_plot):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        # Pick vmax to not saturate 90% of the abscal gains
        vmax = np.max([np.percentile(np.abs(cs.gain_grids[ant_to_plot, pol][~cs.flag_grids[ant_to_plot, pol]]), 99) for pol in ['Jee', 'Jnn']])

        display(HTML(f'<h2>Antenna {ant_to_plot} Amplitude Waterfalls</h2>'))    

        # Plot abscal gain amplitude waterfalls for a single antenna
        fig, axes = plt.subplots(4, 2, figsize=(14,14), gridspec_kw={'height_ratios': [1, 1, .4, .4]})
        for ax, pol in zip(axes[0], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=0, vmax=vmax, extent=extent)
            ax.set_title(f'Smoothcal Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized' )
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)

        # Now flagged plot abscal waterfall    
        for ax, pol in zip(axes[1], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=0, vmax=vmax, extent=extent)
            ax.set_title(f'Abscal Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized' )
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)
            
        # Now plot mean gain spectra 
        for ax, pol in zip(axes[2], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)   
            nflags_spectrum = np.sum(cs.flag_grids[ant], axis=0)
            to_plot = nflags_spectrum <= np.percentile(nflags_spectrum, 75)
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), axis=0)[to_plot], 'r.', label='Abscal')        
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), axis=0)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([0, vmax])
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])    
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('|g| (unitless)')
            ax.set_title(f'Mean Infrequently-Flagged Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized')
            ax.legend(loc='upper left')

        # Now plot mean gain time series
        for ax, pol in zip(axes[3], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            nflags_series = np.sum(cs.flag_grids[ant], axis=1)
            to_plot = nflags_series <= np.percentile(nflags_series, 75)
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), axis=1)[to_plot], 'r.', label='Abscal')        
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), axis=1)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([0, vmax])
            ax.set_xlabel('LST (hours)')
            ax.set_ylabel('|g| (unitless)')
            ax.set_title(f'Mean Infrequently-Flagged Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized')
            ax.set_xticklabels(ax.get_xticks() % 24)
            ax.legend(loc='upper left')

        plt.tight_layout()
        plt.show()
In [12]:
def phase_plot(ant_to_plot):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")    
        display(HTML(f'<h2>Antenna {ant_to_plot} Phase Waterfalls</h2>'))
        fig, axes = plt.subplots(4, 2, figsize=(14,14), gridspec_kw={'height_ratios': [1, 1, .4, .4]})

        # Plot phase waterfalls for a single antenna    
        for ax, pol in zip(axes[0], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=-np.pi, vmax=np.pi, extent=extent)
            ax.set_title(f'Smoothcal Gain Phase of Ant {ant[0]} / Ant {cs.refant[pol][0]}: {pol[-1]}-polarized')
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)

        # Now plot abscal phase waterfall    
        for ax, pol in zip(axes[1], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=-np.pi, vmax=np.pi, extent=extent)
            ax.set_title(f'Abscal Gain Phase of Ant {ant[0]} / Ant {cs.refant[pol][0]}: {pol[-1]}-polarized')
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)
            
        # Now plot median gain spectra 
        for ax, pol in zip(axes[2], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)   
            nflags_spectrum = np.sum(cs.flag_grids[ant], axis=0)
            to_plot = nflags_spectrum <= np.percentile(nflags_spectrum, 75)
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmedian(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), axis=0)[to_plot], 'r.', label='Abscal')        
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmedian(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), axis=0)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([-np.pi, np.pi])
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])    
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel(f'Phase of g$_{{{ant[0]}}}$ / g$_{{{cs.refant[pol][0]}}}$')
            ax.set_title(f'Median Infrequently-Flagged Gain Phase of Ant {ant[0]} / Ant {cs.refant[pol][0]}: {pol[-1]}-polarized')
            ax.legend(loc='upper left')

        # # Now plot median gain time series
        for ax, pol in zip(axes[3], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            nflags_series = np.sum(cs.flag_grids[ant], axis=1)
            to_plot = nflags_series <= np.percentile(nflags_series, 75)
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), axis=1)[to_plot], 'r.', label='Abscal')        
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), axis=1)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([-np.pi, np.pi])    
            ax.set_xlabel('LST (hours)')
            ax.set_ylabel(f'Phase of g$_{{{ant[0]}}}$ / g$_{{{cs.refant[pol][0]}}}$')
            ax.set_title(f'Mean Infrequently-Flagged Gain Phase of Ant {ant[0]} / Ant {cs.refant[pol][0]}: {pol[-1]}-polarized')
            ax.set_xticklabels(ax.get_xticks() % 24)    
            ax.legend(loc='upper left')

        plt.tight_layout()
        plt.show()

Figure 1: Full-Day Gain Amplitudes Before and After smooth_cal¶

Here we plot abscal and smooth_cal gain amplitudes for both of the sample antennas. We also show means across time/frequency, excluding frequencies/times that are frequently flagged.

In [13]:
for ant_to_plot in ants_to_plot:
    amplitude_plot(ant_to_plot)

Antenna 3 Amplitude Waterfalls

Antenna 238 Amplitude Waterfalls

Figure 2: Full-Day Gain Phases Before and After smooth_cal¶

Here we plot abscal and smooth_cal phases relative to each polarization's reference antenna for both of the sample antennas. We also show medians across time/frequency, excluding frequencies/times that are frequently flagged.

In [14]:
for ant_to_plot in ants_to_plot:
    phase_plot(ant_to_plot)

Antenna 3 Phase Waterfalls

Antenna 238 Phase Waterfalls

Examine $\chi^2$¶

In [15]:
def chisq_plot():
    fig, axes = plt.subplots(1, 2, figsize=(14, 10), sharex=True, sharey=True)
    extent = [cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
    for ax, pol in zip(axes, ['Jee', 'Jnn']):

        im = ax.imshow(np.where(cs.flag_grids[cs.refant[pol]], np.nan, cs.chisq_grids[pol]), vmin=1, vmax=5, 
                       aspect='auto', cmap='turbo', interpolation='none', extent=extent)
        ax.set_yticklabels(ax.get_yticks() % 24)
        ax.set_title(f'{pol[1:]}-Polarized $\\chi^2$ / DoF')
        ax.set_xlabel('Frequency (MHz)')

    axes[0].set_ylabel('LST (hours)')
    plt.tight_layout()
    fig.colorbar(im, ax=axes, pad=.07, label='$\\chi^2$ / DoF', orientation='horizontal', extend='both', aspect=50)

Figure 3: Full-Day $\chi^2$ / DoF Waterfall from Redundant-Baseline Calibration¶

Here we plot $\chi^2$ per degree of freedom from redundant-baseline calibration for both polarizations separately. While this plot is a little out of place, as it was not produced by this notebook, it is a convenient place where all the necessary components are readily available. If the array were perfectly redundant and any non-redundancies in the calibrated visibilities were explicable by thermal noise alone, this waterfall should be all 1.

In [16]:
chisq_plot()

FixedFormatter should only be used together with FixedLocator
In [17]:
avg_cspa_vs_time = {ant: np.nanmean(np.where(cs.flag_grids[ant], np.nan, cs.cspa_grids[ant]), axis=1) for ant in cs.ants}
avg_cspa_vs_freq = {ant: np.nanmean(np.where(cs.flag_grids[ant], np.nan, cs.cspa_grids[ant]), axis=0) for ant in cs.ants}

Mean of empty slice
Mean of empty slice
In [18]:
def cspa_vs_time_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(m) for ant, m in avg_cspa_vs_time.items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(m))], 95)
        for ant in avg_cspa_vs_time:
            if ant[1] == pol and not np.all(cs.flag_grids[ant]):
                if np.nanmean(avg_cspa_vs_time[ant]) > detail_cutoff:
                    ax.plot(lst_grid, avg_cspa_vs_time[ant], label=ant, zorder=100)
                else:
                    ax.plot(lst_grid, avg_cspa_vs_time[ant], c='grey', alpha=.2, lw=.5)
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Mean Unflagged $\\chi^2$ per Antenna')
        ax.set_xlabel('LST (hours)')
        ax.set_xticklabels(ax.get_xticks() % 24)

    plt.ylim([1, 5.4])
    plt.tight_layout()
In [19]:
def cspa_vs_freq_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 6), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(m) for ant, m in avg_cspa_vs_freq.items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(m))], 95)
        for ant in avg_cspa_vs_freq:
            if ant[1] == pol and not np.all(cs.flag_grids[ant]):
                if np.nanmean(avg_cspa_vs_freq[ant]) > detail_cutoff:
                    ax.plot(cs.freqs / 1e6, avg_cspa_vs_freq[ant], label=ant, zorder=100)
                else:
                    ax.plot(cs.freqs / 1e6, avg_cspa_vs_freq[ant], c='grey', alpha=.2, lw=.5)
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Mean Unflagged $\\chi^2$ per Antenna')
        ax.set_xlabel('Frequency (MHz)')

    plt.ylim([1, 5.4])
    plt.tight_layout()

Figure 4: Average $\chi^2$ per Antenna vs. Time and Frequency¶

Here we plot $\chi^2$ per antenna from redundant-baseline calibration, separating polarizations and averaging the unflagged pixels in the waterfalls over frequency or time. The worst 5% of antennas are shown in color and highlighted in the legends, the rest are shown in grey.

In [20]:
cspa_vs_time_plot()
cspa_vs_freq_plot()

Mean of empty slice
FixedFormatter should only be used together with FixedLocator

Save Results¶

In [21]:
add_to_history = 'Produced by calibration_smoothing notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65
In [22]:
cs.write_smoothed_cal(output_replace=(CAL_SUFFIX, SMOOTH_CAL_SUFFIX), add_to_history=add_to_history, clobber=True)

Mean of empty slice
In [23]:
# write summary of entirely flagged times/freqs/ants to yaml
all_flagged_times = np.all([np.all(cs.flag_grids[ant], axis=1) for ant in cs.flag_grids], axis=0)
all_flagged_freqs = np.all([np.all(cs.flag_grids[ant], axis=0) for ant in cs.flag_grids], axis=0)
all_flagged_ants = sorted([ant for ant in cs.flag_grids if np.all(cs.flag_grids[ant])])

out_yml_str = 'JD_flags: ' + str([[cs.time_grid[flag_stretch][0] - cs.dt, cs.time_grid[flag_stretch][-1] + cs.dt] 
                                  for flag_stretch in true_stretches(all_flagged_times)])
chan_res = np.median(np.diff(cs.freqs))
out_yml_str += '\n\nfreq_flags: ' + str([[cs.freqs[flag_stretch][0] - chan_res / 2, cs.freqs[flag_stretch][-1] + chan_res / 2] 
                                         for flag_stretch in true_stretches(all_flagged_freqs)])
out_yml_str += '\n\nex_ants: ' + str(all_flagged_ants).replace("'", "").replace('(', '[').replace(')', ']')

print(f'Writing the following to {out_yaml_file}\n' + '-' * (25 + len(out_yaml_file)))
print(out_yml_str)
with open(out_yaml_file, 'w') as outfile:
    outfile.writelines(out_yml_str)

Writing the following to /lustre/aoc/projects/hera/h6c-analysis/IDR2/2459869/2459869_aposteriori_flags.yaml
-----------------------------------------------------------------------------------------------------------
JD_flags: [[2459869.252878312, 2459869.3777008094], [2459869.3779245056, 2459869.378148202], [2459869.37826005, 2459869.378483746], [2459869.378483746, 2459869.3787074424], [2459869.3787074424, 2459869.379154835], [2459869.379378531, 2459869.3797140755], [2459869.3797140755, 2459869.3799377717], [2459869.380161468, 2459869.3804970123], [2459869.3804970123, 2459869.3807207085], [2459869.3808325566, 2459869.381056253], [2459869.381279949, 2459869.3815036453], [2459869.3815036453, 2459869.381951038], [2459869.3823984303, 2459869.3826221265], [2459869.3827339746, 2459869.3835169114], [2459869.3836287595, 2459869.3838524558], [2459869.3844116963, 2459869.3846353926], [2459869.384970937, 2459869.385194633], [2459869.385194633, 2459869.3855301775], [2459869.3856420256, 2459869.385865722], [2459869.386201266, 2459869.3864249624], [2459869.3864249624, 2459869.386760507], [2459869.386760507, 2459869.386984203], [2459869.387096051, 2459869.3873197474], [2459869.3873197474, 2459869.3875434436], [2459869.3876552917, 2459869.387878988], [2459869.3882145323, 2459869.3884382285], [2459869.3885500766, 2459869.388773773], [2459869.388773773, 2459869.389109317], [2459869.3893330134, 2459869.389780406], [2459869.3903396465, 2459869.3905633427], [2459869.3911225833, 2459869.3913462795], [2459869.3915699758, 2459869.391793672], [2459869.391793672, 2459869.3921292163], [2459869.3921292163, 2459869.3923529126], [2459869.3924647607, 2459869.392800305], [2459869.392800305, 2459869.3930240013], [2459869.3931358494, 2459869.3933595456], [2459869.393918786, 2459869.3942543305], [2459869.3949254192, 2459869.3951491155], [2459869.3951491155, 2459869.39548466], [2459869.395708356, 2459869.3959320523], [2459869.3959320523, 2459869.3962675966], [2459869.3963794447, 2459869.396603141], [2459869.396826837, 2459869.3970505334], [2459869.3970505334, 2459869.3972742297], [2459869.3980571665, 2459869.3982808627], [2459869.398728255, 2459869.3989519514], [2459869.3992874958, 2459869.399511192], [2459869.399734888, 2459869.4000704326], [2459869.4000704326, 2459869.400294129], [2459869.4008533694, 2459869.4010770656], [2459869.40141261, 2459869.401636306], [2459869.4017481543, 2459869.4019718505], [2459869.4020836987, 2459869.402419243], [2459869.402419243, 2459869.4026429392], [2459869.4026429392, 2459869.4028666355], [2459869.4028666355, 2459869.4030903317], [2459869.403537724, 2459869.4037614204], [2459869.4037614204, 2459869.4039851166], [2459869.404208813, 2459869.404432509], [2459869.4046562053, 2459869.4048799016], [2459869.4057746865, 2459869.4059983827], [2459869.4070050158, 2459869.407228712], [2459869.407452408, 2459869.4076761045], [2459869.408123497, 2459869.408347193], [2459869.408347193, 2459869.4086827375], [2459869.409018282, 2459869.409241978], [2459869.409353826, 2459869.4095775224], [2459869.4096893705, 2459869.410024915], [2459869.410024915, 2459869.410248611], [2459869.4105841555, 2459869.4108078517], [2459869.4117026366, 2459869.411926333], [2459869.4127092697, 2459869.412932966], [2459869.4133803584, 2459869.4136040546], [2459869.4136040546, 2459869.413939599], [2459869.4155054726, 2459869.415841017], [2459869.416064713, 2459869.4162884094], [2459869.4165121056, 2459869.416735802], [2459869.417630587, 2459869.417854283], [2459869.4181898274, 2459869.4185253717], [2459869.418749068, 2459869.418972764], [2459869.4193083085, 2459869.4195320047], [2459869.419755701, 2459869.4200912453], [2459869.420650486, 2459869.4209860303], [2459869.421545271, 2459869.421768967], [2459869.4222163595, 2459869.4224400558], [2459869.422551904, 2459869.4227756], [2459869.4233348407, 2459869.423558537], [2459869.4238940813, 2459869.4241177775], [2459869.42456517, 2459869.424788866], [2459869.4249007143, 2459869.4252362587], [2459869.4252362587, 2459869.425459955], [2459869.42635474, 2459869.426578436], [2459869.4271376766, 2459869.427361373], [2459869.428479854, 2459869.4287035502], [2459869.4289272465, 2459869.4291509427], [2459869.429262791, 2459869.429486487], [2459869.430269424, 2459869.43049312], [2459869.4308286645, 2459869.4310523607], [2459869.431276057, 2459869.431499753], [2459869.4317234494, 2459869.4319471456], [2459869.43228269, 2459869.432506386], [2459869.433177475, 2459869.433401171], [2459869.4337367155, 2459869.4339604117], [2459869.434295956, 2459869.4345196523], [2459869.435078893, 2459869.435302589], [2459869.4356381334, 2459869.4358618297], [2459869.4365329184, 2459869.4367566146], [2459869.437092159, 2459869.437315855], [2459869.4376513995, 2459869.4378750958], [2459869.4384343363, 2459869.4386580326], [2459869.4387698807, 2459869.438993577], [2459869.4395528175, 2459869.4397765137], [2459869.440112058, 2459869.4403357543], [2459869.4406712987, 2459869.440894995], [2459869.441006843, 2459869.441230539], [2459869.4414542355, 2459869.4416779317], [2459869.44178978, 2459869.442013476], [2459869.4423490204, 2459869.4425727166], [2459869.4425727166, 2459869.442796413], [2459869.443131957, 2459869.4433556534], [2459869.4442504384, 2459869.4444741346], [2459869.444697831, 2459869.444921527], [2459869.4453689195, 2459869.4455926158], [2459869.4462637044, 2459869.4464874007], [2459869.446934793, 2459869.4471584894], [2459869.447941426, 2459869.4481651224], [2459869.4483888187, 2459869.448612515], [2459869.448836211, 2459869.4490599073], [2459869.4491717555, 2459869.4493954517], [2459869.449730996, 2459869.4499546923], [2459869.450513933, 2459869.450737629], [2459869.4510731734, 2459869.4512968697], [2459869.451744262, 2459869.4519679584], [2459869.452415351, 2459869.452639047], [2459869.4531982876, 2459869.453421984], [2459869.4540930726, 2459869.454316769], [2459869.4548760094, 2459869.4550997056], [2459869.455658946, 2459869.4558826424], [2459869.456441883, 2459869.456665579], [2459869.4571129717, 2459869.457336668], [2459869.457336668, 2459869.457560364], [2459869.4576722123, 2459869.4578959085], [2459869.458231453, 2459869.458455149], [2459869.458566997, 2459869.4587906934], [2459869.459126238, 2459869.459349934], [2459869.4599091746, 2459869.460132871], [2459869.460244719, 2459869.460468415], [2459869.460468415, 2459869.4606921114], [2459869.4606921114, 2459869.4609158076], [2459869.4609158076, 2459869.461139504], [2459869.4613632, 2459869.4615868963], [2459869.462034289, 2459869.462257985], [2459869.462369833, 2459869.4625935294], [2459869.4629290737, 2459869.46315277], [2459869.46315277, 2459869.463376466], [2459869.4636001624, 2459869.4638238586], [2459869.464159403, 2459869.464383099], [2459869.4647186436, 2459869.465054188], [2459869.465948973, 2459869.466172669], [2459869.467179302, 2459869.4674029984], [2459869.467962239, 2459869.468185935], [2459869.468968872, 2459869.469192568], [2459869.4696399607, 2459869.469863657], [2459869.4705347456, 2459869.470758442], [2459869.471876923, 2459869.472100619], [2459869.474896822, 2459869.4751205184], [2459869.475679759, 2459869.475903455], [2459869.476686392, 2459869.476910088], [2459869.4780285694, 2459869.4782522656], [2459869.4791470505, 2459869.4793707468], [2459869.4811603166, 2459869.481384013], [2459869.4831735827, 2459869.483397279], [2459869.485186849, 2459869.485410545], [2459869.4889896847, 2459869.489213381], [2459869.492904369, 2459869.493128065], [2459869.49402285, 2459869.494246546], [2459869.4972664453, 2459869.4974901415], [2459869.505543206, 2459869.5061024465], [2459869.514267359, 2459869.514491055], [2459869.5631449856, 2459869.563368682], [2459869.565493796, 2459869.5657174923], [2459869.567507062, 2459869.5677307583], [2459869.569520328, 2459869.5697440244], [2459869.5755601265, 2459869.5757838227], [2459869.6133647896, 2459869.613588486], [2459869.664814923, 2459869.669512544]]

freq_flags: [[49911499.0234375, 50033569.3359375], [62362670.8984375, 62728881.8359375], [69931030.2734375, 70053100.5859375], [87509155.2734375, 108016967.7734375], [112167358.3984375, 112289428.7109375], [112655639.6484375, 112777709.9609375], [113632202.1484375, 113754272.4609375], [116439819.3359375, 116561889.6484375], [124618530.2734375, 125350952.1484375], [136215209.9609375, 136459350.5859375], [136825561.5234375, 137313842.7734375], [137435913.0859375, 137924194.3359375], [141464233.3984375, 141830444.3359375], [142074584.9609375, 142318725.5859375], [143783569.3359375, 144027709.9609375], [145736694.3359375, 145980834.9609375], [147445678.7109375, 147567749.0234375], [149154663.0859375, 149276733.3984375], [154159545.8984375, 154403686.5234375], [170883178.7109375, 171005249.0234375], [175155639.6484375, 175277709.9609375], [181137084.9609375, 181259155.2734375], [183212280.2734375, 183334350.5859375], [187362670.8984375, 187728881.8359375], [189193725.5859375, 189315795.8984375], [191146850.5859375, 191390991.2109375], [197128295.8984375, 197372436.5234375], [198104858.3984375, 198226928.7109375], [199203491.2109375, 199325561.5234375], [201766967.7734375, 201889038.0859375], [208480834.9609375, 208724975.5859375], [212142944.3359375, 212265014.6484375], [220687866.2109375, 220809936.5234375], [223129272.4609375, 223373413.0859375], [227401733.3984375, 227523803.7109375], [229110717.7734375, 229354858.3984375], [231063842.7734375, 231185913.0859375]]

ex_ants: [[4, Jee], [8, Jee], [10, Jee], [10, Jnn], [18, Jee], [18, Jnn], [19, Jee], [19, Jnn], [20, Jee], [20, Jnn], [22, Jee], [27, Jee], [27, Jnn], [28, Jee], [28, Jnn], [32, Jee], [32, Jnn], [33, Jnn], [34, Jee], [43, Jee], [44, Jee], [46, Jnn], [47, Jee], [50, Jee], [50, Jnn], [51, Jee], [51, Jnn], [53, Jee], [53, Jnn], [54, Jee], [54, Jnn], [55, Jnn], [57, Jee], [58, Jee], [58, Jnn], [59, Jnn], [60, Jee], [60, Jnn], [63, Jnn], [68, Jnn], [73, Jee], [73, Jnn], [74, Jee], [74, Jnn], [75, Jnn], [77, Jee], [77, Jnn], [78, Jee], [81, Jee], [81, Jnn], [82, Jee], [82, Jnn], [83, Jee], [83, Jnn], [84, Jnn], [85, Jee], [85, Jnn], [86, Jee], [86, Jnn], [87, Jee], [88, Jee], [88, Jnn], [90, Jee], [90, Jnn], [92, Jee], [92, Jnn], [99, Jee], [99, Jnn], [102, Jee], [102, Jnn], [103, Jee], [103, Jnn], [104, Jnn], [107, Jee], [107, Jnn], [108, Jee], [109, Jnn], [110, Jnn], [111, Jnn], [117, Jee], [117, Jnn], [119, Jnn], [120, Jnn], [121, Jee], [126, Jee], [135, Jnn], [140, Jee], [140, Jnn], [141, Jnn], [142, Jnn], [143, Jee], [145, Jnn], [147, Jee], [148, Jee], [148, Jnn], [149, Jee], [149, Jnn], [150, Jee], [150, Jnn], [151, Jee], [153, Jee], [155, Jee], [156, Jee], [158, Jnn], [161, Jnn], [165, Jee], [166, Jee], [166, Jnn], [167, Jee], [167, Jnn], [169, Jee], [169, Jnn], [170, Jee], [170, Jnn], [173, Jee], [173, Jnn], [176, Jee], [176, Jnn], [177, Jee], [177, Jnn], [178, Jee], [178, Jnn], [179, Jee], [179, Jnn], [180, Jnn], [182, Jee], [183, Jee], [184, Jee], [184, Jnn], [185, Jee], [185, Jnn], [186, Jee], [186, Jnn], [190, Jee], [190, Jnn], [191, Jnn], [192, Jee], [192, Jnn], [193, Jee], [200, Jee], [200, Jnn], [201, Jee], [201, Jnn], [202, Jee], [202, Jnn], [203, Jee], [203, Jnn], [219, Jee], [222, Jee], [320, Jee], [320, Jnn], [321, Jee], [321, Jnn], [322, Jee], [322, Jnn], [323, Jee], [323, Jnn], [324, Jee], [324, Jnn], [325, Jee], [325, Jnn], [329, Jee], [329, Jnn], [333, Jee], [333, Jnn]]

Metadata¶

In [24]:
for repo in ['hera_cal', 'hera_qm', 'hera_filters', 'hera_notebook_templates', 'pyuvdata']:
    exec(f'from {repo} import __version__')
    print(f'{repo}: {__version__}')

hera_cal: 3.2.3
hera_qm: 2.1.1
hera_filters: 0.1.4.dev2+ga4ff591
hera_notebook_templates: 0.1.dev531+gfe314a8
pyuvdata: 2.3.3.dev39+g16031096
In [25]:
print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')

Finished execution in 34.49 minutes.