Fix pytests

.github/workflows/python_testing.yml

@@ -21,21 +21,21 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - name: Python 3.11
+            runs-on: ubuntu-latest
+            python-version: '3.11'
+            toxenv: py311
+
           - name: Python 3.10
             runs-on: ubuntu-latest
-            python-version: '3.10'
+            python-version: 3.10
             toxenv: py310
-
+            
           - name: Python 3.9
             runs-on: ubuntu-latest
             python-version: 3.9
             toxenv: py39
 
-          - name: Python 3.8
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: py38
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

tests/test_wrapper.py

@@ -4,127 +4,115 @@
 import shutil
 import urllib.request
 import tarfile
-import pytest
 
 from astropy.io.fits import FITSDiff
 
 from ullyses import wrapper
 from ullyses_utils.ullyses_config import VERSION
-import ullyses_utils
 
-ULLYSES_DATA_LOCATION = 'https://ullyses.stsci.edu/files/'
 
-COS_TARGETS = ['sk-70d16',
-               'n11-els-018']
+AV456 = {'name': 'av456',
+         'url': 'https://stsci.box.com/shared/static/7khk2b23ep9esqmz5t25ymno5yhh9xh1.gz'}
+AV321 = {'name': 'av321',
+         'url': 'https://stsci.box.com/shared/static/ltbnbue3diwb6m1h4gh6t5uiwjt2xi73.gz'}
+HD104237E = {'name': 'hd-104237e',
+             'url': 'https://stsci.box.com/shared/static/irelgbjs7zhdiljksao4su2c6tfhyntx.gz'}
+V_HK_ORI = {'name': 'v-hk-ori',
+            'url': 'https://stsci.box.com/shared/static/az3ytnwohj0t4wnc4m8oqbw9ziozwqx1.gz'}
 
-STIS_TARGETS = ['lh9-34',
-                'pgmw3120']
+RELEASE = 'dr6'
 
-MULTIPLE_TARGETS = ['sk-67d22',
-                    'sk-68d26']
 
 class TestWrapper():
 
-    def test_cos_data(self):
-        for target in COS_TARGETS:
-            self.setup_tree(target)
-            self.run_wrapper('./')
-            report = self.compare_outputs()
-            self.cleanup(target)
-            if report is not None:
-                raise AssertionError(report)
+    def test_av456(self):
+        target = AV456
+        self.setup_tree(target, RELEASE)
+        self.run_wrapper(target['name'])
+        report = self.compare_outputs(target['name'])
+        self.cleanup(target['name'])
+        if report is not None:
+            raise AssertionError(report)
         return
 
-    def test_stis_data(self):
-        for target in STIS_TARGETS:
-            self.setup_tree(target)
-            self.run_wrapper('./')
-            report = self.compare_outputs()
-            self.cleanup(target)
-            if report is not None:
-                raise AssertionError(report)
+    def test_av321(self):
+        target = AV321
+        self.setup_tree(target, RELEASE)
+        self.run_wrapper(target['name'])
+        report = self.compare_outputs(target['name'])
+        self.cleanup(target['name'])
+        if report is not None:
+            raise AssertionError(report)
         return
 
-    def test_fuse_data(self):
-        pass
-
-    def test_multiple_data_types(self):
-        for target in MULTIPLE_TARGETS:
-            self.setup_tree(target)
-            self.run_wrapper('./')
-            report = self.compare_outputs()
-            self.cleanup(target)
-            if report is not None:
-                raise AssertionError(report)
+    def test_hd104237e(self):
+        target = HD104237E
+        self.setup_tree(target, RELEASE)
+        self.run_wrapper(target['name'])
+        report = self.compare_outputs(target['name'])
+        self.cleanup(target['name'])
+        if report is not None:
+            raise AssertionError(report)
         return
 
-    def setup_tree(self, target):
-        if os.path.isdir(target):
-            shutil.rmtree(target)
-        if os.path.isdir('truth'):
-            shutil.rmtree('truth')
-        filename = target + '.tar.gz'
-        fullurl = ULLYSES_DATA_LOCATION + filename
-        print("Retrieving {}".format(fullurl))
-        urllib.request.urlretrieve(fullurl, filename)
-        tarball = tarfile.open(filename)
-        tarball.extractall()
-        os.mkdir('truth')
-        os.chdir(target)
-        hlsps = glob.glob('hlsp_ullyses*')
-        for datafile in hlsps:
-            os.rename(datafile,'../truth/'+datafile)
+    def test_v_hk_ori(self):
+        target = V_HK_ORI
+        self.setup_tree(target, RELEASE)
+        self.run_wrapper(target['name'])
+        report = self.compare_outputs(target['name'])
+        self.cleanup(target['name'])
+        if report is not None:
+            raise AssertionError(report)
         return
 
-    def run_wrapper(self, indir):
+    def setup_tree(self, target, release):
+        target_name = target['name']
+        url = target['url']
+        if os.path.isdir(target_name):
+            shutil.rmtree(target_name)
+        filename = target_name + '.tar.gz'
+        _ = urllib.request.urlretrieve(url, filename)
+        data_tarfile = tarfile.open(filename, mode='r|gz')
+        data_tarfile.extractall()
+        return
+
+    def run_wrapper(self, target):
+        indir = target + '/' + RELEASE + '/input/'
         wrapper.main(indir, outdir=indir, version=VERSION)
         return
 
-    def compare_outputs(self):
+    def compare_outputs(self, target):
         report = None
         # Outputs from current run are in ./, truth files to compare
         # with are in ./truth
-        new_hlsps = glob.glob('hlsp_ullyses*')
         all_ok = True
         fitsdiff_report = ''
-        keywords_to_ignore = ['DATE', 'FITS_SW', 'FILENAME', 'HLSP_VER']
+        keywords_to_ignore = ['DATE', 'FITS_SW', 'FILENAME',
+                              'HLSP_VER', 'S_REGION']
+        new_hlsps = glob.glob(target + '/' + RELEASE + '/input/hlsp_ullyses*')
+        print(new_hlsps)
         for new_product in new_hlsps:
-            truth_filename = self.get_truth_filename(new_product)
-            fdiff = FITSDiff(new_product, truth_filename, ignore_hdus=['provenance'],
-            ignore_keywords=keywords_to_ignore,
-            rtol=1.0e-7)
+            truth_filename = self.get_truth_filename(target, new_product)
+            fdiff = FITSDiff(new_product, truth_filename,
+                             ignore_hdus=['provenance'],
+                             ignore_keywords=keywords_to_ignore,
+                             rtol=1.0e-7)
             fitsdiff_report += fdiff.report()
             if not fdiff.identical and all_ok:
                 all_ok = False
         if not all_ok:
             report = os.linesep + fitsdiff_report
             return report
+        print(fitsdiff_report)
         return None
 
-    def get_truth_filename(self, product):
+    def get_truth_filename(self, target, product):
         # Get the truth filename.  The data release might be different
-        dr_position = product.find('_dr')
-        if dr_position == -1:
-            print('Cannot find dr version in {}'.format(product))
-            return None
-        stop = dr_position + 3
-        filestring = '../truth/' + product[:stop] + '*'
-        truth_filename_list = glob.glob(filestring)
-        if len(truth_filename_list) > 1:
-            print('More than 1 truth filename matches {} != 1'.format(product))
-            for file in truth_filename_list:
-                print(file)
-            print('Returning first instance: {}'.format(truth_filename_list[0]))
-        elif len(truth_filename_list) == 0:
-            print('No truth files match product specification {}'.format(filestring))
-            return None
-        return truth_filename_list[0]
-
+        filename = os.path.basename(product)
+        truth_filename = target + '/' + RELEASE + '/truth/' + filename
         return truth_filename
 
     def cleanup(self, target):
-        os.chdir('..')
         shutil.rmtree(target)
-        shutil.rmtree('truth')
         os.remove(target+'.tar.gz')
         return

ullyses/wrapper.py

@@ -35,7 +35,7 @@ class Ullyses_SegmentList(SegmentList):
     # For ULLYSES this is G230L/2635/NUVC and G230L/2950/NUVC
 #    self.bad_segments = {2635: 'NUVC', 2950: 'NUVC'}
     SegmentList.bad_segments = {2635: 'NUVC', 2950: 'NUVC'}
-    
+
 
     def write(self, filename, overwrite=False, level="", version=""):
 
@@ -200,8 +200,8 @@ def obs_footprint(self):
         return s_region
 
     def get_targname(self):
-        aliases_file = ullyses_utils.__path__[0] + '/data/target_metadata/pd_all_aliases.json'
-        aliases = pd.read_json(aliases_file, orient="split")
+        aliases_file = ullyses_utils.__path__[0] + '/data/target_metadata/ullyses_aliases.csv'
+        aliases = pd.read_csv(aliases_file)
         # These are just preliminary target names, in case we can't find a match
         if len(self.targnames) == 1:
             ull_targname = self.targnames[0]
@@ -493,22 +493,22 @@ def main(indir, outdir, version=VERSION, clobber=False):
             if df.loc[lowind, "gratings"] in ["E140M", "E140H"]:
                 if "G130M" in gratings and "G160M" in gratings:
                     g130mind = df.loc[df["gratings"] == "G130M"].index.values
-                    used = used.append(df.loc[g130mind])
+                    used = pd.concat([used, df.loc[g130mind]])
                     shortestwl = df.loc[g130mind[0], "minwls"]
                     df = df.drop(index=g130mind)
                     g160mind = df.loc[df["gratings"] == "G160M"].index.values
-                    used = used.append(df.loc[g160mind])
+                    used = pd.concat([used, df.loc[g160mind]])
                     maxwl = df.loc[g160mind[0], "maxwls"]
                     df = df.drop(index=g160mind)
                     df = df.drop(index=lowind)
                 else:
                     shortestwl = df.loc[lowind, "minwls"]
-                    used = used.append(df.loc[lowind])
+                    used = pd.concat([used, df.loc[lowind]])
                     maxwl = df.loc[lowind, "maxwls"]
                     df = df.drop(lowind)
             else:
                 shortestwl = df.loc[lowind, "minwls"]
-                used = used.append(df.loc[lowind])
+                used = pd.concat([used, df.loc[lowind]])
                 maxwl = df.loc[lowind, "maxwls"]
                 df = df.drop(lowind)
             while len(df) > 0:
@@ -519,7 +519,7 @@ def main(indir, outdir, version=VERSION, clobber=False):
                 if "G130M" in used.gratings.values and "G160M" in gratings and "G160M" not in used.gratings.values:
                     lowind = df.loc[df["gratings"] == "G160M"].index.values
                     maxwl = df.loc[lowind[0], "maxwls"]
-                    used = used.append(df.loc[lowind])
+                    used = pd.concat([used, df.loc[lowind]])
                     df = df.drop(index=lowind)
                 # Handle case where more than one grating overlaps with bluer data.
                 elif len(lowind) > 1:
@@ -528,19 +528,19 @@ def main(indir, outdir, version=VERSION, clobber=False):
                     biggest = ranges.idxmax()
                     match_grating = df2.loc[biggest, "gratings"]
                     match_ind = df.loc[df["gratings"] == match_grating].index.values
-                    used = used.append(df.loc[match_ind])
+                    used = pd.concat([used, df.loc[match_ind]])
                     maxwl = df.loc[match_ind, "maxwls"].values[0]
                     df = df.drop(index=lowind)
                 # If none overlap, abut with the next closest product.
                 elif len(lowind) == 0:
                     lowind = df["minwls"].idxmin()
-                    used = used.append(df.loc[lowind])
+                    used = pd.concat([used, df.loc[lowind]])
                     maxwl = df.loc[lowind, "maxwls"]
                     df = df.drop(lowind)
                 # This is the easy case- only one mode overlaps with the bluer data.
                 else:
                     maxwl = df.loc[lowind[0], "maxwls"]
-                    used = used.append(df.loc[lowind])
+                    used = pd.concat([used, df.loc[lowind]])
                     df = df.drop(index=lowind)
                 # Check every time if there are any modes that overlap completely
                 # with what has been abutted so far.