diff --git a/.cz.toml b/.cz.toml deleted file mode 100644 index 4b4c90d..0000000 --- a/.cz.toml +++ /dev/null @@ -1,7 +0,0 @@ -[tool.commitizen] -name = "cz_conventional_commits" -tag_format = "$version" -version_scheme = "semver" -version = "0.1.1" -update_changelog_on_bump = true -major_version_zero = true diff --git a/.github/workflows/bump_version.yml b/.github/workflows/bump_version.yml index 5bba4d2..5f6b4be 100644 --- a/.github/workflows/bump_version.yml +++ b/.github/workflows/bump_version.yml @@ -4,7 +4,8 @@ on: push: branches: # disabling it for now - - _main + - main + - *_rc jobs: bump_version: diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ee11ac..9f21ee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,31 +1,64 @@ -## Unreleased - -### Feat - -- **background_models**: adding a user defined smoothing sigma +## 1.0.1 (2024-05-09) ### Fix -- **gammapy_tools/analysis/rbm.py**: removing hard coded map size values +- **pyproject.toml**: fixing versioning +- **gammapy_tools/make_background/prepare_data.py**: adding overwrite option ### Refactor -- **analysis_notebook**: updating example -- **data_products**: removing plt.show +- **gammapy_tools/__version__.py**: post test update +- **__version__**: switching to __version__ for the version number + +## 1.0.0 (2024-05-07) -## 0.1.3 (2024-02-07) +### Feat + +- **utils/exclusion_finder.py**: adding exclusion finder +- **gammapy_tools/fake_source_coordinates/process.py**: adding mimic_data +- **make_background**: parallel reduction of background and mimic search +- **gammapy_tools/make_background**: implementing a parallel + reduction method to spead up background generation +- **make_background**: implementation of closed N background runs +- **background_models**: adding a user defined smoothing sigma ### Fix +- **pyproject.toml**: changing gammapy version +- **gammapy_tools/make_background/background_models.py**: updating to gammapy1.2 +- **gammapy_tools/utils/exclusion_finder.py**: adding a check for gammacat and hawc +- **gammapy_tools/utils/exclusion_finder.py**: correcting path +- **gammapy_tools/utils/exclusion_finder.py**: correcting imports and file paths +- **fake_source_coordinates/process.py**: adding safety check +- **gammapy_tools/fake_source_coordinates/process.py**: fixing print and removing target +- **process.py**: correcting background +- **make_background**: storing kl_div table +- **background_tools.py**: exposure searching +- **prepare_data.py**: adding more info +- **prepare_data.py**: raise error when no runs are found +- **gammapy_tools/analysis/rbm.py**: removing hard coded map size values +- **Hipparcos_MAG8_1997.dat**: updated hipparcos file to have correct colour column - **analysis**: update the analysis scripts to work with config files ### Refactor +- **gammapy_tools/fake_source_coordinates/process.py**: adding more robust file finder +- **fake_source_coordinates/process.py**: reincluding source when scrambling +- **gammapy_tools/fake_source_coordinates/fake_location.py**: popping exisiting background +- **make_background**: removing ignore warnings +- **make_background.py**: removing additional background and allow previous method +- **templates/config.py**: adding KL_DIV +- **utils**: moving functions to utils +- **analysis_notebook**: updating example +- **data_products**: removing plt.show - **_version.**: bumping version - **analysis_notebook**: migrating analysis notebook - **fake_source_coordinates**: adding __all__ - **pyproject.toml**: changing required to minimum python version +### Perf + +- **gammapy_tools/utils/run_details.py**: Changed the mimic selection to perform a nested search + ## 0.1.2 (2024-01-30) ### Fix diff --git a/Dockerfile b/Dockerfile index 1440947..273d43b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM jupyter/minimal-notebook AS base # Install gammapy -RUN mamba install gcc jupyterlab --yes +RUN mamba install gcc jupyterlab "gammapy==1.2" --yes WORKDIR /gammapy-tools @@ -28,14 +28,17 @@ COPY --from=base /opt/conda /opt/conda WORKDIR /gammapy-tools/tmp_build # RUN gammapy download datasets -ENV GAMMAPY_DATA=/gammapy-tools/gammapy-datasets/1.1/ +ENV GAMMAPY_DATA=/gammapy-tools/gammapy-datasets/1.2/ RUN mkdir -p $GAMMAPY_DATA +WORKDIR /gammapy-tools/ +RUN gammapy download datasets # Add package ADD --chown=1000:100 . /gammapy-tools/tmp_build/gammapy-tools WORKDIR /gammapy-tools/tmp_build/gammapy-tools + # RUN ls -lah RUN pip install . # RUN ./gammapy_tools/Hipparcos_MAG8_1997.dat $GAMMAPY_DATA/catalogs/ diff --git a/docker-compose.yml b/docker-compose.yml index 56e80ff..1716ad6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,5 +11,5 @@ services: - JUPYTER_TOKEN=letmein volumes: # Replace with where your data is located - - /path/to/data:/local_data - - $GAMMAPY_DATA:/gammapy-tools/gammapy-datasets/1.1/ \ No newline at end of file + - /home/obriens/DataAnalysis/Veritas/Gammapy:/local_data + # - $GAMMAPY_DATA:/gammapy-tools/gammapy-datasets/1.1/ \ No newline at end of file diff --git a/docker_build.sh b/docker_build.sh old mode 100644 new mode 100755 diff --git a/gammapy-tools.def b/gammapy-tools.def index 00ebfc9..b6e6e2a 100644 --- a/gammapy-tools.def +++ b/gammapy-tools.def @@ -16,22 +16,27 @@ Stage: build # Port for jupyter lab export JPORT=8000 export GAMMAPY_DATA=/gamma-tools/gammapy-datasets/1.2/ - . "/opt/conda/etc/profile.d/conda.sh" - . /opt/conda/bin/activate - export PATH=/opt/conda/bin/:$PATH - + #. "/opt/conda/etc/profile.d/conda.sh" + #. /opt/conda/bin/activate + #export PATH=/opt/conda/bin/:$PATH + . /opt/gammapy_tools/bin/activate + + %post apt update apt upgrade -y - apt install curl bash git gcc btop emacs -y + apt install curl bash git gcc btop emacs python3 python3-pip python3.12-venv -y # Install mamba and base env - curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - bash Miniforge3-$(uname)-$(uname -m).sh -b -p /opt/conda - . /opt/conda/bin/activate - - mamba install -c conda-forge iminuit cmasher pip papermill matplotlib pip "jupyterlab==4.0.12" notebook ipykernel ipython ipywidgets --yes + #curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + #bash Miniforge3-$(uname)-$(uname -m).sh -b -p /opt/conda + #. /opt/conda/bin/activate + + python3 -m venv /opt/gammapy_tools + . /opt/gammapy_tools/bin/activate + pip install iminuit cmasher pip papermill matplotlib pip "jupyterlab==4.0.12" notebook ipykernel ipython ipywidgets + #mamba install -c conda-forge iminuit cmasher pip papermill matplotlib pip "jupyterlab==4.0.12" notebook ipykernel ipython ipywidgets --yes # Install v2dl3 @@ -41,7 +46,7 @@ Stage: build # Because its an env install of requirements # Note the python version in the latest test throws issues with pytest - grep -A 100 "dependencies:" environment-eventdisplay.yml | grep "-" | grep -v "python" | awk '{print $2}' | xargs mamba install --yes + grep -A 100 "dependencies:" environment-eventdisplay.yml | grep "-" | grep -v "python" | awk '{print $2}' | xargs pip install # root_numpy throws issues too. Only VEGAS uses it mv setup.py _setup.py && grep -v "root_numpy" _setup.py > setup.py && pip install . @@ -59,7 +64,7 @@ Stage: build cd /gamma-tools ; rm -r /gamma-tools/tmp_build - mamba clean -a --yes + #mamba clean -a --yes python -m pip cache purge diff --git a/gammapy_tools/__init__.py b/gammapy_tools/__init__.py index c73820d..e580542 100644 --- a/gammapy_tools/__init__.py +++ b/gammapy_tools/__init__.py @@ -2,10 +2,12 @@ from . import make_background from . import templates from . import fake_source_coordinates - +from .__version__ import __version__ __all__ = ( analysis.__all__ + make_background.__all__ + templates.__all__ + fake_source_coordinates.__all__ + + ["__version__"] ) + diff --git a/gammapy_tools/_version.py b/gammapy_tools/__version__.py similarity index 100% rename from gammapy_tools/_version.py rename to gammapy_tools/__version__.py diff --git a/gammapy_tools/analysis/data_products_LOCAL_4133777.py b/gammapy_tools/analysis/data_products_LOCAL_4133777.py index 7b8f23a..ccf53a4 100644 --- a/gammapy_tools/analysis/data_products_LOCAL_4133777.py +++ b/gammapy_tools/analysis/data_products_LOCAL_4133777.py @@ -1,4 +1,3 @@ -from pathlib import Path import numpy as np import astropy.units as u from astropy.coordinates import Angle, SkyCoord @@ -11,7 +10,6 @@ # %matplotlib inline import matplotlib.pyplot as plt -from IPython.display import display # gammapy imports from gammapy.data import DataStore @@ -35,10 +33,9 @@ SkyModel, ) from gammapy.estimators import LightCurveEstimator -from gammapy.analysis import Analysis, AnalysisConfig -from gammapy.catalog import SourceCatalogGammaCat, SourceCatalog3HWC from gammapy.visualization import plot_spectrum_datasets_off_regions + def make_spectrum_RE(config, plot=True, return_stacked=False): """Make a RE spectrum @@ -67,8 +64,12 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): observations = datastore.get_observations() if config["run_selection"]["pos_from_DL3"]: # get position from DL3 header - hdul = fits.open(config["io"]["out_dir"]+os.listdir(config["io"]["out_dir"])[0]) - source_pos = SkyCoord(hdul[1].header["RA_OBJ"]*u.deg, hdul[1].header["DEC_OBJ"]*u.deg) + hdul = fits.open( + config["io"]["out_dir"] + os.listdir(config["io"]["out_dir"])[0] + ) + source_pos = SkyCoord( + hdul[1].header["RA_OBJ"] * u.deg, hdul[1].header["DEC_OBJ"] * u.deg + ) else: # get position from ra/dec [deg] source_pos = SkyCoord( config["run_selection"]["source_ra"], @@ -148,7 +149,9 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): < 2.0 ) # star_mask &= (star_cat["mag"] + star_cat["colour"]) < 8 - star_mask &= (star_cat["mag"]+star_cat["colour"])< config["sky_map"]["min_star_brightness"] + star_mask &= (star_cat["mag"] + star_cat["colour"]) < config["sky_map"][ + "min_star_brightness" + ] for src in star_cat[star_mask]: exclusion_regions.append( @@ -192,13 +195,15 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): info_table = datasets.info_table(cumulative=True) time = info_table["livetime"].to("h") sig = info_table["sqrt_ts"] - - #plot exclusion regions and reflected regions - if len(exclusion_regions) > 0 and plot: - plt.figure(figsize=(8,8)) + + # plot exclusion regions and reflected regions + if len(exclusion_regions) > 0 and plot: + plt.figure(figsize=(8, 8)) ax = exclusion_mask.plot() - on_region.to_pixel(ax.wcs).plot(ax=ax, edgecolor="magenta",label='ON') - plot_spectrum_datasets_off_regions(ax=ax, datasets=datasets) #add legend=True to plot run numbers associated w/ OFF regions + on_region.to_pixel(ax.wcs).plot(ax=ax, edgecolor="magenta", label="ON") + plot_spectrum_datasets_off_regions( + ax=ax, datasets=datasets + ) # add legend=True to plot run numbers associated w/ OFF regions plt.show() # make spectrum model from user input @@ -272,17 +277,24 @@ def get_flux_lc(config, type="flux"): theta = config["sky_map"]["theta"] datastore = DataStore.from_dir(config["io"]["out_dir"]) - + if config["io"]["from_runlist"]: - observations = datastore.get_observations(obs_id=np.genfromtxt(config["io"]["runlist"],unpack=True),required_irf="all-optional") + observations = datastore.get_observations( + obs_id=np.genfromtxt(config["io"]["runlist"], unpack=True), + required_irf="all-optional", + ) else: observations = datastore.get_observations(required_irf="all-optional") - + amp, idx = config["spectrum"]["params"] - + if config["run_selection"]["pos_from_DL3"]: # get position from DL3 header - hdul = fits.open(config["io"]["out_dir"]+os.listdir(config["io"]["out_dir"])[0]) - source_pos = SkyCoord(hdul[1].header["RA_OBJ"]*u.deg, hdul[1].header["DEC_OBJ"]*u.deg) + hdul = fits.open( + config["io"]["out_dir"] + os.listdir(config["io"]["out_dir"])[0] + ) + source_pos = SkyCoord( + hdul[1].header["RA_OBJ"] * u.deg, hdul[1].header["DEC_OBJ"] * u.deg + ) else: # get position from ra/dec [deg] source_pos = SkyCoord( config["run_selection"]["source_ra"], @@ -290,7 +302,7 @@ def get_flux_lc(config, type="flux"): frame="icrs", unit="deg", ) - + e_min = config["spectrum"]["e_min"] e_max = config["spectrum"]["e_max"] nbin = config["spectrum"]["e_bins"] @@ -315,13 +327,13 @@ def get_flux_lc(config, type="flux"): # exclusion regions exclusion_regions = [] - + exclusion_regions.append( CircleSkyRegion( center=source_pos, radius=config["sky_map"]["on_exclusion_region"] * u.deg ) ) - + if ( len(config["sky_map"]["exclusion_regions"]) > 0 ): # should be a list of CircleSkyRegions @@ -358,7 +370,7 @@ def get_flux_lc(config, type="flux"): "dec": star_data[:, 1], "id": star_data[:, 2], "mag": star_data[:, 3], - "colour": star_data[:,4], + "colour": star_data[:, 4], } ) star_mask = ( @@ -369,8 +381,10 @@ def get_flux_lc(config, type="flux"): < 2.0 ) - star_mask &= (star_cat["mag"]+star_cat["colour"])< config["sky_map"]["min_star_brightness"] - + star_mask &= (star_cat["mag"] + star_cat["colour"]) < config["sky_map"][ + "min_star_brightness" + ] + for src in star_cat[star_mask]: exclusion_regions.append( CircleSkyRegion( @@ -404,7 +418,7 @@ def get_flux_lc(config, type="flux"): if type == "flux": time_intervals = [Time([start, stop])] lc_maker_1d = LightCurveEstimator( - energy_edges=[e_min,e_max] * u.TeV, + energy_edges=[e_min, e_max] * u.TeV, time_intervals=time_intervals, n_sigma_ul=2, reoptimize=False, diff --git a/gammapy_tools/analysis/data_products_LOCAL_4133860.py b/gammapy_tools/analysis/data_products_LOCAL_4133860.py index 7b8f23a..ccf53a4 100644 --- a/gammapy_tools/analysis/data_products_LOCAL_4133860.py +++ b/gammapy_tools/analysis/data_products_LOCAL_4133860.py @@ -1,4 +1,3 @@ -from pathlib import Path import numpy as np import astropy.units as u from astropy.coordinates import Angle, SkyCoord @@ -11,7 +10,6 @@ # %matplotlib inline import matplotlib.pyplot as plt -from IPython.display import display # gammapy imports from gammapy.data import DataStore @@ -35,10 +33,9 @@ SkyModel, ) from gammapy.estimators import LightCurveEstimator -from gammapy.analysis import Analysis, AnalysisConfig -from gammapy.catalog import SourceCatalogGammaCat, SourceCatalog3HWC from gammapy.visualization import plot_spectrum_datasets_off_regions + def make_spectrum_RE(config, plot=True, return_stacked=False): """Make a RE spectrum @@ -67,8 +64,12 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): observations = datastore.get_observations() if config["run_selection"]["pos_from_DL3"]: # get position from DL3 header - hdul = fits.open(config["io"]["out_dir"]+os.listdir(config["io"]["out_dir"])[0]) - source_pos = SkyCoord(hdul[1].header["RA_OBJ"]*u.deg, hdul[1].header["DEC_OBJ"]*u.deg) + hdul = fits.open( + config["io"]["out_dir"] + os.listdir(config["io"]["out_dir"])[0] + ) + source_pos = SkyCoord( + hdul[1].header["RA_OBJ"] * u.deg, hdul[1].header["DEC_OBJ"] * u.deg + ) else: # get position from ra/dec [deg] source_pos = SkyCoord( config["run_selection"]["source_ra"], @@ -148,7 +149,9 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): < 2.0 ) # star_mask &= (star_cat["mag"] + star_cat["colour"]) < 8 - star_mask &= (star_cat["mag"]+star_cat["colour"])< config["sky_map"]["min_star_brightness"] + star_mask &= (star_cat["mag"] + star_cat["colour"]) < config["sky_map"][ + "min_star_brightness" + ] for src in star_cat[star_mask]: exclusion_regions.append( @@ -192,13 +195,15 @@ def make_spectrum_RE(config, plot=True, return_stacked=False): info_table = datasets.info_table(cumulative=True) time = info_table["livetime"].to("h") sig = info_table["sqrt_ts"] - - #plot exclusion regions and reflected regions - if len(exclusion_regions) > 0 and plot: - plt.figure(figsize=(8,8)) + + # plot exclusion regions and reflected regions + if len(exclusion_regions) > 0 and plot: + plt.figure(figsize=(8, 8)) ax = exclusion_mask.plot() - on_region.to_pixel(ax.wcs).plot(ax=ax, edgecolor="magenta",label='ON') - plot_spectrum_datasets_off_regions(ax=ax, datasets=datasets) #add legend=True to plot run numbers associated w/ OFF regions + on_region.to_pixel(ax.wcs).plot(ax=ax, edgecolor="magenta", label="ON") + plot_spectrum_datasets_off_regions( + ax=ax, datasets=datasets + ) # add legend=True to plot run numbers associated w/ OFF regions plt.show() # make spectrum model from user input @@ -272,17 +277,24 @@ def get_flux_lc(config, type="flux"): theta = config["sky_map"]["theta"] datastore = DataStore.from_dir(config["io"]["out_dir"]) - + if config["io"]["from_runlist"]: - observations = datastore.get_observations(obs_id=np.genfromtxt(config["io"]["runlist"],unpack=True),required_irf="all-optional") + observations = datastore.get_observations( + obs_id=np.genfromtxt(config["io"]["runlist"], unpack=True), + required_irf="all-optional", + ) else: observations = datastore.get_observations(required_irf="all-optional") - + amp, idx = config["spectrum"]["params"] - + if config["run_selection"]["pos_from_DL3"]: # get position from DL3 header - hdul = fits.open(config["io"]["out_dir"]+os.listdir(config["io"]["out_dir"])[0]) - source_pos = SkyCoord(hdul[1].header["RA_OBJ"]*u.deg, hdul[1].header["DEC_OBJ"]*u.deg) + hdul = fits.open( + config["io"]["out_dir"] + os.listdir(config["io"]["out_dir"])[0] + ) + source_pos = SkyCoord( + hdul[1].header["RA_OBJ"] * u.deg, hdul[1].header["DEC_OBJ"] * u.deg + ) else: # get position from ra/dec [deg] source_pos = SkyCoord( config["run_selection"]["source_ra"], @@ -290,7 +302,7 @@ def get_flux_lc(config, type="flux"): frame="icrs", unit="deg", ) - + e_min = config["spectrum"]["e_min"] e_max = config["spectrum"]["e_max"] nbin = config["spectrum"]["e_bins"] @@ -315,13 +327,13 @@ def get_flux_lc(config, type="flux"): # exclusion regions exclusion_regions = [] - + exclusion_regions.append( CircleSkyRegion( center=source_pos, radius=config["sky_map"]["on_exclusion_region"] * u.deg ) ) - + if ( len(config["sky_map"]["exclusion_regions"]) > 0 ): # should be a list of CircleSkyRegions @@ -358,7 +370,7 @@ def get_flux_lc(config, type="flux"): "dec": star_data[:, 1], "id": star_data[:, 2], "mag": star_data[:, 3], - "colour": star_data[:,4], + "colour": star_data[:, 4], } ) star_mask = ( @@ -369,8 +381,10 @@ def get_flux_lc(config, type="flux"): < 2.0 ) - star_mask &= (star_cat["mag"]+star_cat["colour"])< config["sky_map"]["min_star_brightness"] - + star_mask &= (star_cat["mag"] + star_cat["colour"]) < config["sky_map"][ + "min_star_brightness" + ] + for src in star_cat[star_mask]: exclusion_regions.append( CircleSkyRegion( @@ -404,7 +418,7 @@ def get_flux_lc(config, type="flux"): if type == "flux": time_intervals = [Time([start, stop])] lc_maker_1d = LightCurveEstimator( - energy_edges=[e_min,e_max] * u.TeV, + energy_edges=[e_min, e_max] * u.TeV, time_intervals=time_intervals, n_sigma_ul=2, reoptimize=False, diff --git a/gammapy_tools/make_background/prepare_data.py b/gammapy_tools/make_background/prepare_data.py index f370025..4e62f0c 100644 --- a/gammapy_tools/make_background/prepare_data.py +++ b/gammapy_tools/make_background/prepare_data.py @@ -6,14 +6,15 @@ from gammapy.data import DataStore -def prepare_dataset(config: dict) -> dict: +def prepare_dataset(config: dict, overwrite: bool = False) -> dict: """Prepare a dataset for analysis. Extract runs of interest from an existing datastore, reporting any missing runs. Parameters ---------- config (dict) - dictionary with config information - + overwrite (bool) - bool to control whether or not to overwrite + the output directory Returns ---------- config (dict) - dictionary with config information @@ -51,7 +52,7 @@ def prepare_dataset(config: dict) -> dict: # Copy try: - data_store.copy_obs(obs_in_db, in_dir) + data_store.copy_obs(obs_in_db, in_dir, overwrite=overwrite) except Exception as e: if len(obs_in_db) == 0: raise RuntimeError( diff --git a/pyproject.toml b/pyproject.toml index 2627c26..ea12c28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,29 +13,17 @@ packages=[ ] [tool.setuptools_scm] -write_to = "gammapy_tools/_version.py" +write_to = "gammapy_tools/__version__.py" [project] name = "gammapy_tools" dynamic = ["version"] -description = "VERITAS (VEGAS and Eventdisplay) to DL3 Converter" +description = "Tools for gammapy analysis for the VERITAS telescope" readme = "README.md" license = { file="LICENSE" } requires-python = ">=3.9" -#dependencies = [ -# "gammapy==1.2", -# "pydantic<=1.10", -# "scipy==1.11.4", -# "astropy==5.3.4", -# "astroquery==0.4.6", -# "matplotlib==3.8.2", -# "multiprocess==0.70.16", -# "numpy==1.26.4", -# #"pyV2DL3==0.5", -# "PyYAML==6.0.1", -# "regions==0.8", -#] + dependencies = [ "gammapy>=1.2", @@ -66,4 +54,13 @@ dependencies = [ [project.scripts] [tool.commitizen] -version = "0.1.3" # This should be your current semver version \ No newline at end of file +name = "cz_conventional_commits" +tag_format = "$version" +version_scheme = "semver" +version_provider = "scm" +update_changelog_on_bump = true +changelog_incremental = true +version_files = [ + "gammapy_tools/__version__.py", + "pyproject.toml:version" +]