Merge pull request #1016 from pypeit/staged

Merges develop into master (1.1.0 tag)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.7dev
+current_version = 1.1.0
 commit = False
 tag = False
 tag_name = {new_version}

.gitignore

@@ -1,5 +1,6 @@
 # Folders#
 .idea/
+.vscode/
 
 # Compiled source #
 ###################

.travis.yml

@@ -34,8 +34,8 @@ env:
         - SKLEARN_VERSION=stable
         - MAIN_CMD='python setup.py'
         - SETUP_CMD='test'
-        - CONDA_DEPENDENCIES='scipy h5py pyyaml matplotlib future numba scikit-learn configobj IPython'
-        - PIP_DEPENDENCIES='extension_helpers'
+        - CONDA_DEPENDENCIES='scipy pyyaml matplotlib future numba scikit-learn configobj IPython'
+        - PIP_DEPENDENCIES='extension_helpers ginga linetools'
         - EVENT_TYPE='pull_request push'
         - TASK='test'
 
@@ -99,26 +99,15 @@ matrix:
         # versions of Python, we can vary Python and Numpy versions at the same
         # time.
 
-        #- os: linux
-        #  env: PYTHON_VERSION=2.7 NUMPY_VERSION=1.13
-        #- os: linux
-        #  env: PYTHON_VERSION=3.3 NUMPY_VERSION=1.8
-        #- os: linux
-        #  env: PYTHON_VERSION=3.4 NUMPY_VERSION=1.9
-        #- os: linux
-        #  env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.10
-        #- os: linux
-        #  env: NUMPY_VERSION=1.14
-
-        # Try numpy 1.16
-        - os: linux
-          env: NUMPY_VERSION=1.16
         # Try numpy 1.17
-        - os: linux
-          env: NUMPY_VERSION=1.17
+        #- os: linux
+        #  env: NUMPY_VERSION=1.17
         # Try numpy 1.18
         - os: linux
           env: NUMPY_VERSION=1.18
+        # Try numpy 1.19
+        - os: linux
+          env: NUMPY_VERSION=1.19
         # Try numpy pre-release
         - os: linux
           env: NUMPY_VERSION=prerelease
@@ -138,8 +127,8 @@ install:
 
     - git clone git://github.com/astropy/ci-helpers.git
     - source ci-helpers/travis/setup_conda_$TRAVIS_OS_NAME.sh
-    - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/linetools/linetools.git; fi
-    - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/profxj/ginga.git; fi
+#    - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/linetools/linetools.git; fi
+#    - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/profxj/ginga.git; fi
     #- conda install numpy
     #- conda install configobj
     #- conda install scikit-learn

CHANGES.rst

@@ -1,26 +1,68 @@
 
-1.0.7dev
---------
- - Fixed a bug fixes a bug in full_template wavelength reidentification for situations where extreme
-   wavelength coverage slits results in reidentification with a purely zero-padded array.
- - Fixed another such bug arising from these zero-padded arrays. 
- - (Hotfix) Deal with chk_calibs test
- - Correct det bug in keck_lris
+1.1.0 (8 Sep 2020)
+------------------
+
+- Fixed a bug for IR reductions for cases where only negative object
+  traces are identified.  These were accidentally being written to the
+  spec1d file.
+- Fixed a bug fixes a bug in full_template wavelength reidentification
+  for situations where extreme wavelength coverage slits results in
+  reidentification with a purely zero-padded array.
+- Fixed a bug fixes a bug in full_template wavelength reidentification
+  for situations where extreme wavelength coverage slits results in
+  reidentification with a purely zero-padded array.
+- Fixed another such bug arising from these zero-padded arrays.
+- (Hotfix) Deal with chk_calibs test
+- Script to generate combined datacubes for IFU data.
+- Changed numpy (> 1.18.0) and scipy (> 1.4.0) version requirements
+- Allow show2d_spec, chk_edges, chk_flats to load older Spec2DObj
+  datamodel versions
+- Implemented a plugin kindly provided by the ginga developers to
+  display images with a secondary wavelength image WCS.
+    - Removes dependency on @profxj's ginga fork, and avoids a bug when
+      using WCS image registration in that fork.
+    - `pypeit/ginga.py` moved to `pypeit/display/display.py` and ginga
+      plugin added to `pypeit/diplay` directory.
+    - ginga plugin registered as an entry point in `setup.py`
+    - Added a script to check that the plugins are all available.
+    - Installation docs updated.  Both `ginga` and `linetools` are now
+      installed via pip.
+- Deprecated `pypeit/debugger.py` and `pypeit/data/settings`
+- Removed h5py as a dependency
+- `linetools` is now listed in `pypeit/requirements.txt` until I can
+  check if it still causes readthedocs to fail...
+- Modify Spec2DObj 2D model for float32 images
+- `pypeit.tracepca.TracePCA` and `pypeit.edgetrace.EdgeTraceSet` now
+  subclass from `pypeit.datamodel.DataContainer`
+- Refactor WaveCalib into a DataContainer
+- Refactor fitting + PypeItFit DataContainer
+- Coadd2D bug fixes
+- Coadd2D without spec1d files
+- Coadd2D offsets
+- Some Coadd2D docs
+- Manual extraction
+- Improve LBT/LUCI
+- Add MMT/MMIRS
+- QL script for Keck/MOSFIRE (beta version)
+- Correct det bug in keck_lris
+- Modifications to allow for flailing LRISr detector
+- Modifications for parse LRIS LAMPS prior to 2010 upgrade
+- Add P200/TripleSpec
 
 1.0.6 (22 Jul 2020)
 -------------------
 
- - (Hotfix) Deal with wavecalib crash
- - Fix class and version check for DataContainer objects.
- - Script to check for calibration files
- - No longer require bias frames as default for DEIMOS
- - Implement grism19 for NOT/ALFOSC
- - Introduced another parameter used to identify box slits, as opposed
-   to erroneous "slits" found by the edge tracing algorithms.  Any slit
-   that has `minimum_slit_length < length < minimum_slit_length_sci` is
-   considered a `BOXSLIT`, any slit with `length < minimum_slit_length`
-   is considered a `SHORTSLIT`; the latter are always ignored.
- - Introduced order matching code into EdgeTraceSet.
+- (Hotfix) Deal with wavecalib crash
+- Fix class and version check for DataContainer objects.
+- Script to check for calibration files
+- No longer require bias frames as default for DEIMOS
+- Implement grism19 for NOT/ALFOSC
+- Introduced another parameter used to identify box slits, as opposed to
+  erroneous "slits" found by the edge tracing algorithms.  Any slit that
+  has `minimum_slit_length < length < minimum_slit_length_sci` is
+  considered a `BOXSLIT`, any slit with `length < minimum_slit_length`
+  is considered a `SHORTSLIT`; the latter are always ignored.
+- Introduced order matching code into EdgeTraceSet.
     - This helps fix an issue for GNIRS_10L caused by the orders
       shifting.
     - Introduces two paramters in `EdgeTraceSetPar` to assist the
@@ -30,54 +72,53 @@
     - Removes the need for `Spectrograph.slit2order` and
       `Spectrograph.order_vec`.  Changes propagated, primarily in
       `wavecalib.py`, `autoid.py`, and `reduce.py`.
- - Adds in Keck/LRISr with the original detector
- - Adds in Keck/LRISb with the FITS format
+- Adds in Keck/LRISr with the original detector
+- Adds in Keck/LRISb with the FITS format
 
 1.0.5 (23 Jun 2020)
 -------------------
 
- - Add median combining code
- - Make biasframes median combine by default
- - Implemented IFU reduction hooks
- - KCWI reduction complete up to spec2D frames
- - Implemented new flatfield DataContainer to separate
-   pixelflat and illumflat
+- Add median combining code
+- Make biasframes median combine by default
+- Implemented IFU reduction hooks
+- KCWI reduction complete up to spec2D frames
+- Implemented new flatfield DataContainer to separate pixelflat and
+  illumflat
 
 1.0.4 (27 May 2020)
 -------------------
 
- - Add a script (pypeit_flux_setup) for creating fluxing,
-   coadd1d and tellfit pypeit files
- - Add telluric fitting script, pypeit_tellfit
-
+- Add a script (pypeit_flux_setup) for creating fluxing, coadd1d and
+  tellfit pypeit files
+- Add telluric fitting script, pypeit_tellfit
 
 1.0.3 (04 May 2020)
 -------------------
 
- - Add illumflat frametype
- - Enable dark image subtraction
- - Refactor of Calibrations (remove cache, add get_dark)
- - Enable calibration-only run
- - Clean up flat, bias handling
- - Make re-use masters the default mode of run_pypeit
- - Require Python 3.7
- - Fixed a bug in NIRES order finding. 
- - Add NOT/ALFOSC
- - Fluxing docs
- - Fix flexure and heliocentric bugs
- - Identify GUI updates
+- Add illumflat frametype
+- Enable dark image subtraction
+- Refactor of Calibrations (remove cache, add get_dark)
+- Enable calibration-only run
+- Clean up flat, bias handling
+- Make re-use masters the default mode of run_pypeit
+- Require Python 3.7
+- Fixed a bug in NIRES order finding. 
+- Add NOT/ALFOSC
+- Fluxing docs
+- Fix flexure and heliocentric bugs
+- Identify GUI updates
 
 1.0.2 (30 Apr 2020)
 -------------------
 
- - Various doc hotfixes
- - wavelength algorithm hotfix, such that they must now generate an
-   entry for every slit, bad or good.
+- Various doc hotfixes
+- wavelength algorithm hotfix, such that they must now generate an entry
+  for every slit, bad or good.
 
 1.0.1 (13 Apr 2020)
 -------------------
 
- - Various hot fixes
+- Various hot fixes
 
 1.0.0 (07 Apr 2020)
 -------------------

MANIFEST.in

@@ -3,12 +3,14 @@ include README.md
 
 include setup.cfg
 
-recursive-include *.pyx *.c *.pxd
+recursive-include pypeit *.pyx *.c *.pxd *.h
 recursive-include docs *
 recursive-include licenses *
 recursive-include cextern *
 recursive-include scripts *
 
+include pypeit/requirements.txt
+
 prune build
 prune docs/_build
 prune docs/api

README.md

@@ -37,6 +37,8 @@ Conduct](https://pypeit.readthedocs.io/en/latest/codeconduct.html).
 * Lick/Kast 
 * Magellan/MagE
 * Magellan/Fire
+* MMT/BinoSpec (270 and 600 tested)
+* MMT/MMIRS (HK_zJ, J_zJ, and K_K tested)
 * MDM/OSMOS
 * Keck/DEIMOS (600ZD, 830G, 1200G)
 * Keck/LRIS 
@@ -50,6 +52,8 @@ Conduct](https://pypeit.readthedocs.io/en/latest/codeconduct.html).
 * VLT/X-Shooter
 * VLT/FORS2  (300I, 300V)
 * WHT/ISIS
+* P200/DBSP (316/7500 on red arm, 600/4000 on blue arm)
+* P200/TripleSpec
 
 # Requirements
 

bin/pypeit_chk_plugins

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+"""
+Check that the installation of python is C enabled.
+"""
+
+import os
+from pypeit.display import required_plugins, plugins_available
+from pypeit import msgs
+
+success, report = plugins_available(return_report=True)
+if not success:
+    msgs.error(report)
+msgs.info('All required plugins found: {0}'.format(', '.join(required_plugins)))
+
+

bin/pypeit_clean

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+#
+# See top-level LICENSE file for Copyright information
+#
+# -*- coding: utf-8 -*-
+
+"""
+This script cleans up files generated by running PypeIt tests
+"""
+
+import os
+import shutil
+from pypeit.tests.tstutils import data_path
+
+def delete_if_exists(fname, f_or_dir):
+    if f_or_dir == 'file':
+        if os.path.isfile(fname):
+            os.remove(fname)
+    if f_or_dir == 'dir':
+        if os.path.isdir(fname):
+            shutil.rmtree(fname)
+
+if __name__ == '__main__':
+    flist = ['fitstbl.fits', 'MasterArc_A_01_22.fits', 'tmp.fits', 'tmp_detector.fits', 'tst_pypeitimage.fits', 'shane_kast_blue.calib', 'shane_kast_blue.sorted', 'sensfunc.fits', 'datacube.fits', 'tst_coadd_files-SPAT0175-DET01.fits']
+    try:
+        from pypeit.tests.test_fluxspec import kast_blue_files
+        f = os.path.basename(kast_blue_files()[1])
+        print(f'adding {data_path(f)} to files to delete')
+        flist.append()
+    except:
+        pass
+    for fname in flist:
+        delete_if_exists(data_path(fname), 'file')
+    for dirname in ['setup_files', 'shane_kast_blue_A', 'keck_lris_blue_A']:
+        delete_if_exists(data_path(dirname), 'dir')
+
+    # TODO: just create new tests/files/out dir and make all tests write to that

bin/pypeit_coadd_datacube

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+"""
+Perform datacube coadds of PypeIt 2D spectra.
+"""
+
+import pypeit.scripts.coadd_datacube as coadd_datacube
+
+if __name__ == '__main__':
+    args = coadd_datacube.parser()
+    coadd_datacube.main(args)

bin/pypeit_ql_keck_mosfire

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+# See top-level LICENSE file for Copyright information
+#
+# -*- coding: utf-8 -*-
+
+
+"""
+This script runs PypeIt
+"""
+
+from pypeit.scripts import ql_keck_mosfire
+import sys
+
+if __name__ == '__main__':
+    args = ql_keck_mosfire.parser()
+    rtval = ql_keck_mosfire.main(args)
+    sys.exit(rtval)