sot · javierggt · Nov 30, 2020 · Nov 16, 2020 · Nov 16, 2020 · Nov 17, 2020
diff --git a/agasc/scripts/mag_estimate_report.py b/agasc/scripts/mag_estimate_report.py
@@ -7,39 +7,56 @@
 import os
 import argparse
 import numpy as np
+import pathlib
 from astropy import table, time, units as u
-from cxotime import CxoTime
+from cxotime import CxoTime, units
 
 from agasc.supplement.magnitudes import mag_estimate_report
 
 
 def parser():
     parse = argparse.ArgumentParser(description=__doc__)
     parse.add_argument('--start', help='Time to start processing new observations.'
-                                       ' CxoTime-compatible time stamp.')
+                                       ' CxoTime-compatible time stamp.'
+                                       ' Default: now - 90 days')
     parse.add_argument('--stop', help='Time to stop processing new observations.'
-                                      ' CxoTime-compatible time stamp.')
-    parse.add_argument('--out-dir', default=f'./mag_stats_report',
-                       help='Output directory')
-    parse.add_argument('--obs-stats', default=f'mag_stats_obsid.fits',
-                       help='FITS file with mag-stats for all observations')
-    parse.add_argument('--agasc-stats', default=f'mag_stats_agasc.fits',
-                       help='FITS file with mag-stats for all observed AGASC stars')
-    parse.add_argument('--weekly-report', help="Add links to navigate weekly reports",
+                                      ' CxoTime-compatible time stamp.'
+                                      ' Default: now')
+    parse.add_argument('--input-dir', default='$SKA/data/agasc',
+                       help='Directory containing mag-stats files. Default: $SKA/data/agasc')
+    parse.add_argument('--output-dir', default='supplement_reports/suspect',
+                       help='Output directory.'
+                            ' Default: supplement_reports/suspect')
+    parse.add_argument('--obs-stats', default='mag_stats_obsid.fits',
+                       help='FITS file with mag-stats for all observations.'
+                            ' Default: mag_stats_obsid.fits')
+    parse.add_argument('--agasc-stats', default='mag_stats_agasc.fits',
+                       help='FITS file with mag-stats for all observed AGASC stars.'
+                            ' Default: mag_stats_agasc.fits')
+    parse.add_argument('--weekly-report',
+                       help="Add links to navigate weekly reports.",
                        action='store_true', default=False)
     return parse
 
 
 def main():
     args = parser().parse_args()
 
+    args.output_dir = pathlib.Path(os.path.expandvars(args.output_dir))
+    args.input_dir = pathlib.Path(os.path.expandvars(args.input_dir))
+    args.obs_stats = args.input_dir / args.obs_stats
+    args.agasc_stats = args.input_dir / args.agasc_stats
+
     agasc_stats = table.Table.read(args.agasc_stats)
     agasc_stats.convert_bytes
10000
tring_to_unicode()
     obs_stats = table.Table.read(args.obs_stats)
     obs_stats.convert_bytestring_to_unicode()
 
-    args.start = CxoTime(args.start)
     args.stop = CxoTime(args.stop)
+    if args.start is None:
+        args.start = args.stop - 90 * units.day
+    else:
+        args.start = CxoTime(args.start)
 
     t = (obs_stats['mp_starcat_time'])
     ok = (t < args.stop) & (t > args.start) & ~obs_stats['obs_ok']
@@ -54,20 +71,20 @@ def main():
 
     if args.weekly_report:
         t = CxoTime(args.stop)
-        directory = os.path.join(args.out_dir, f'{t.date[:8]}')
+        directory = args.output_dir / t.date[:8]
         week = time.TimeDelta(7*u.day)
         nav_links = {
             'previous': f'../{(t - week).date[:8]}',
             'up': '..',
             'next': f'../{(t + week).date[:8]}'
         }
     else:
-        directory = args.out_dir
+        directory = args.output_dir
         nav_links = None
 
-    msr = mag_estimate_report.MagStatsReport(agasc_stats,
-                                             obs_stats,
-                                             directory=directory)
+    msr = mag_estimate_report.MagEstimateReport(agasc_stats,
+                                                obs_stats,
+                                                directory=directory)
     msr.multi_star_html(sections=sections, tstart=args.start, tstop=args.stop,
                         filename='index.html',
                         nav_links=nav_links)

diff --git a/agasc/scripts/update_mag_supplement.py b/agasc/scripts/update_mag_supplement.py
@@ -7,26 +7,40 @@
 import argparse
 import logging
 from agasc.supplement.magnitudes import update_mag_supplement
+import pathlib
+import os
 
 
 def parser():
     parse = argparse.ArgumentParser(description=__doc__)
     parse.add_argument('--agasc-id-file', help='File containing a list of AGASC IDs, one per line.')
     parse.add_argument('--start',
-                       help='Time to start processing new observations.'
-                            ' CxoTime-compatible time stamp.')
+                       help='Include only stars observed after this time.'
+                            ' CxoTime-compatible time stamp.'
+                            ' Default: now - 14 days.')
     parse.add_argument('--stop',
-                       help='Time to stop processing new observations.'
-                            ' CxoTime-compatible time stamp.')
+                       help='Include only stars observed before this time.'
+                            ' CxoTime-compatible time stamp.'
+                            ' Default: now.')
+    parse.add_argument('--whole-history',
+                       help='Include all star observations.')
     parse.add_argument('--obs-status-override', help='YAML file with observation status.')
     parse.add_argument('--obs', help='Observation ID for status override.')
     parse.add_argument('--agasc-id', help='AGASC ID for status override.')
     parse.add_argument('--status', help='Status to override.')
     parse.add_argument('--comments', help='Comments for status override.', default='')
     parse.add_argument('--email', help='Email to report errors.')
-    parse.add_argument('--report', help='Generate HTML report for the period covered',
+    parse.add_argument('--report',
+                       help='Generate HTML report for the period covered. Default: False',
                        action='store_true', default=False)
-    parse.add_argument('--multi-process', help="Use multi-processing to accelerate run",
+    parse.add_argument('--output-dir',
+                       help='Directory where to place the supplement. Default: .',
+                       default='.')
+    parse.add_argument('--reports-dir',
+                       help='Directory where to place reports.'
+                            ' Default: <output_dir>/supplement_reports/weekly.')
+    parse.add_argument('--multi-process',
+                       help="Use multi-processing to accelerate run.",
                        action='store_true', default=False)
     parse.add_argument('--log-level',
                        default='info',
@@ -38,6 +52,12 @@ def main():
     the_parser = parser()
     args = the_parser.parse_args()
 
+    args.output_dir = pathlib.Path(os.path.expandvars(args.output_dir))
+    if args.reports_dir is None:
+        args.reports_dir = args.output_dir / 'supplement_reports' / 'weekly'
+    else:
+        args.reports_dir = pathlib.Path(os.path.expandvars(args.reports_dir))
+
     logging.basicConfig(level=args.log_level.upper(),
                         format='%(message)s')
 

diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
 import jinja2
 from agasc.supplement.magnitudes import mag_estimate
 from astropy import table
@@ -16,7 +16,7 @@ class MagEstimateReport:
     def __init__(self, agasc_stats, obs_stats, directory='./mag_estimates_reports'):
         self.agasc_stats = agasc_stats
         self.obs_stats = obs_stats
-        self.directory = directory
+        self.directory = Path(directory)
 
     def single_star_html(self, agasc_id, directory,
                          static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources',
@@ -26,8 +26,9 @@ def single_star_html(self, agasc_id, directory,
 
         star_template = jinja2.Template(STAR_REPORT_BOOTSTRAP)
 
-        if not os.path.exists(directory):
-            os.makedirs(directory)
+        directory = Path(directory)
+        if not directory.exists():
+            directory.mkdir(parents=True)
 
         o = self.obs_stats[self.obs_stats['agasc_id'] == agasc_id]
         if len(o) == 0:
@@ -63,14 +64,14 @@ def single_star_html(self, agasc_id, directory,
                          'outside_markers': True
                          })
             args.append({'type': 'flags', 'obsid': obsid})
-        fig = self.plot_set(agasc_id, args=args, filename=os.path.join(directory, f'mag_stats.png'))
+        fig = self.plot_set(agasc_id, args=args, filename=directory / 'mag_stats.png')
         plt.close(fig)
 
-        with open(os.path.join(directory, 'index.html'), 'w') as out:
+        with open(directory / 'index.html', 'w') as out:
             out.write(star_template.render(agasc_stats=s,
                                            obs_stats=o.as_array(),
                                            static_dir=static_dir))
-        return os.path.join(directory, 'index.html')
+        return directory / 'index.html'
 
 
6D4E
    def multi_star_html(self, sections=None, updated_stars=None, fails=(),
                         tstart=None, tstop=None, report_date=None,
@@ -117,62 +118,67 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(),
         # this turns all None into '' in a new list of failures
         fails = [{k: '' if v is None else v for k, v in f.items()} for i, f in enumerate(fails)]
 
+        agasc_stats = self.agasc_stats.copy()
+
         # check how many observations were added in this run, and how many of those are ok
-        new_obs = self.obs_stats[(self.obs_stats['mp_starcat_time'] >= info["tstart"]) &
-                            (self.obs_stats['mp_starcat_time'] <= info["tstop"])]. \
-            group_by('agasc_id')[['agasc_id', 'obsid', 'obs_ok']]. \
-            groups.aggregate(np.count_nonzero)[['agasc_id', 'obsid', 'obs_ok']]
-        new_obs['n_obs_bad_new'] = new_obs['obsid'] - new_obs['obs_ok']
+        new_obs_mask = ((self.obs_stats['mp_starcat_time'] >= info["tstart"]) &
+                        (self.obs_stats['mp_starcat_time'] <= info["tstop"]))
+        if np.any(new_obs_mask):
+            new_obs = self.obs_stats[new_obs_mask]. \
+                group_by('agasc_id')[['agasc_id', 'obsid', 'obs_ok']]. \
+                groups.aggregate(np.count_nonzero)[['agasc_id', 'obsid', 'obs_ok']]
+            new_obs['n_obs_bad_new'] = new_obs['obsid'] - new_obs['obs_ok']
+
+            all_agasc_ids = np.unique(np.concatenate([
+                new_obs['agasc_id'],
+                [f['agasc_id'] for f in fails]
+            ]))
+            agasc_stats = table.join(agasc_stats, new_obs[['agasc_id', 'n_obs_bad_new']],
+                                     keys=['agasc_id'])
+
+            assert np.all(np.in1d(agasc_stats['agasc_id'], all_agasc_ids)
+                          ), 'Not all AGASC IDs are in new obs.'
 
         # add some extra fields
-        agasc_stats = self.agasc_stats.copy()
-        if len(agasc_stats) == 0:
-            return [], []
-        all_agasc_ids = np.unique(np.concatenate([
-            new_obs['agasc_id'],
-            [f['agasc_id'] for f in fails]
-        ]))
-
-        assert np.all(np.in1d(agasc_stats['agasc_id'], all_agasc_ids)), 'Not all AGASC IDs are in new obs.'
-        agasc_stats['n_obs_bad'] = agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok']
-        agasc_stats['flag'] = '          '
         if len(agasc_stats):
-            agasc_stats = table.join(agasc_stats, new_obs[['agasc_id', 'n_obs_bad_new']],
-                                     keys=['agasc_id'])
-        tooltips = {
-            'warning': 'At least one bad observation',
-            'danger': 'At least one new bad observation'
-        }
-        agasc_stats['flag'][:] = ''
-        agasc_stats['flag'][agasc_stats['n_obs_bad'] > 0] = 'warning'
-        agasc_stats['flag'][agasc_stats['n_obs_bad_new'] > 0] = 'danger'
-        agasc_stats['delta'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'])
-        agasc_stats['sigma'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'])/agasc_stats['mag_aca_err']
-        agasc_stats['new'] = True
-        agasc_stats['new'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = False
-        agasc_stats['update_mag_aca'] = np.nan
-        agasc_stats['update_mag_aca_err'] = np.nan
-        agasc_stats['last_obs'] = CxoTime(agasc_stats['last_obs_time']).date
+            if not 'n_obs_bad_new' in agasc_stats.colnames:
+                agasc_stats['n_obs_bad_new'] = 0
+            agasc_stats['n_obs_bad'] = agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok']
+            agasc_stats['flag'] = '          '
+            agasc_stats['flag'][:] = ''
+            agasc_stats['flag'][agasc_stats['n_obs_bad'] > 0] = 'warning'
+            agasc_stats['flag'][agasc_stats['n_obs_bad_new'] > 0] = 'danger'
+            agasc_stats['delta'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'])
+            agasc_stats['sigma'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'])/agasc_stats['mag_aca_err']
+            agasc_stats['new'] = True
+            agasc_stats['new'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = False
+            agasc_stats['update_mag_aca'] = np.nan
+            agasc_stats['update_mag_aca_err'] = np.nan
+            agasc_stats['last_obs'] = CxoTime(agasc_stats['last_obs_time']).date
+
         if len(updated_stars):
             agasc_stats['update_mag_aca'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = \
                 updated_stars['mag_aca']
             agasc_stats['update_mag_aca_err'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] =\
                 updated_stars['mag_aca_err']
+
+        tooltips = {
+            'warning': 'At least one bad observation',
+            'danger': 'At least one new bad observation'
+        }
+
         # make all individual star reports
         star_reports = {}
         for agasc_id in np.atleast_1d(agasc_ids):
             try:
-                dirname = os.path.join(self.directory,
-                                       'stars',
-                                       f'{agasc_id//1e7:03.0f}',
-                                       f'{agasc_id:.0f}')
+                dirname = self.directory / 'stars' / f'{agasc_id//1e7:03.0f}' / f'{agasc_id:.0f}'
                 if make_single_reports:
                     self.single_star_html(
                         agasc_id,
                         directory=dirname,
                         highlight_obs=highlight_obs
                     )
-                if os.path.exists(dirname):
+                if dirname.exists():
                     star_reports[agasc_id] = dirname
             except mag_estimate.MagStatsException:
                 pass
@@ -185,11 +191,11 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(),
                                                    section['stars'])].as_array()
 
         # this is a hack
-        star_reports = {i: os.path.relpath(star_reports[i], self.directory) for i in star_reports}
+        star_reports = {i: str(star_reports[i].relative_to(self.directory)) for i in star_reports}
         # make report
-        if not os.path.exists(self.directory):
-            os.makedirs(self.directory)
-        with open(os.path.join(self.directory, filename), 'w') as out:
+        if not self.directory.exists():
+            self.directory.mkdir(parents=True)
+        with open(self.directory / filename, 'w') as out:
             out.write(run_template.render(info=info,
                                           sections=sections,
                                           failures=fails,