diff --git a/pyproject.toml b/pyproject.toml index de302d976..d5bf0df90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,545 @@ simtools-validate-camera-fov = "simtools.applications.validate_camera_fov:main" simtools-validate-optics = "simtools.applications.validate_optics:main" simtools-validate-schema-files = "simtools.applications.validate_schema_files:main" +[tool.pylint.main] +# Explicit list of all pylint options (generated with --generate-toml-config) +# Analyse import fallback blocks. This can be used to support both Python 2 and 3 +# compatible code, which means that the block might have code that exists only in +# one or another interpreter, leading to false positives when analysed. +# analyse-fallback-blocks = + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint in +# a server-like mode. +# clear-cache-post-run = + +# Always return a 0 (non-error) status code, even if lint errors are found. This +# is primarily useful in continuous integration scripts. +# exit-zero = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +# extension-pkg-allow-list = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +# extension-pkg-whitelist = + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +# fail-on = + +# Specify a score threshold under which the program will exit with error. +fail-under = 10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +# from-stdin = + +# Files or directories to be skipped. They should be base names, not paths. +ignore = ["CVS"] + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, it +# can't be used as an escape character. +# ignore-paths = + +# Files or directories matching the regular expression patterns are skipped. The +# regex matches against base names, not paths. The default value ignores Emacs +# file locks +ignore-patterns = ["^\\.#"] + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +# ignored-modules = + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# init-hook = + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 0 + +# Control the amount of potential inferred values when inferring a single object. +# This can help the performance when dealing with large functions or complex, +# nested conditions. +limit-inference-results = 100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +# load-plugins = + +# Pickle collected data for later comparisons. +persistent = true + +# Minimum Python version to use for version dependent checks. Will default to the +# version used to run pylint. +py-version = "3.9" + +# Discover python modules and packages in the file system subtree. +# recursive = + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +# source-roots = + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode = true + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +# unsafe-load-any-extension = + +[tool.pylint.basic] +# Naming style matching correct argument names. +argument-naming-style = "snake_case" + +# Regular expression matching correct argument names. Overrides argument-naming- +# style. If left empty, argument names will be checked with the set naming style. +# argument-rgx = + +# Naming style matching correct attribute names. +attr-naming-style = "snake_case" + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +# attr-rgx = + +# Bad variable names which should always be refused, separated by a comma. +bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +# bad-names-rgxs = + +# Naming style matching correct class attribute names. +class-attribute-naming-style = "any" + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +# class-attribute-rgx = + +# Naming style matching correct class constant names. +class-const-naming-style = "UPPER_CASE" + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +# class-const-rgx = + +# Naming style matching correct class names. +class-naming-style = "PascalCase" + +# Regular expression matching correct class names. Overrides class-naming-style. +# If left empty, class names will be checked with the set naming style. +# class-rgx = + +# Naming style matching correct constant names. +const-naming-style = "UPPER_CASE" + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming style. +# const-rgx = + +# Minimum line length for functions/classes that require docstrings, shorter ones +# are exempt. +docstring-min-length = -1 + +# Naming style matching correct function names. +function-naming-style = "snake_case" + +# Regular expression matching correct function names. Overrides function-naming- +# style. If left empty, function names will be checked with the set naming style. +# function-rgx = + +# Good variable names which should always be accepted, separated by a comma. +good-names = ["e", "i", "j", "k", "x", "y", "n", "f", "r", "ex", "db", "im", "sh", "ax", "ce", "xx", "yy", "zz"] + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +# good-names-rgxs = + +# Include a hint for the correct naming format with invalid-name. +# include-naming-hint = + +# Naming style matching correct inline iteration names. +inlinevar-naming-style = "any" + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +# inlinevar-rgx = + +# Naming style matching correct method names. +method-naming-style = "snake_case" + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +# method-rgx = + +# Naming style matching correct module names. +module-naming-style = "snake_case" + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +# module-rgx = + +# Colon-delimited sets of names that determine each other's naming style when the +# name regexes allow several styles. +# name-group = + +# Regular expression which should only match function or class names that do not +# require a docstring. +no-docstring-rgx = "main" + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. These +# decorators are taken in consideration only for invalid-name. +property-classes = ["abc.abstractproperty"] + +# Regular expression matching correct type alias names. If left empty, type alias +# names will be checked with the set naming style. +# typealias-rgx = + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +# typevar-rgx = + +# Naming style matching correct variable names. +variable-naming-style = "snake_case" + +# Regular expression matching correct variable names. Overrides variable-naming- +# style. If left empty, variable names will be checked with the set naming style. +# variable-rgx = + +[tool.pylint.classes] +# Warn about protected attribute access inside special methods +# check-protected-access-in-special-methods = + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods = ["__init__", "__new__", "setUp", "asyncSetUp", "__post_init__"] + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make", "os._exit"] + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg = ["cls"] + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg = ["mcs"] + +[tool.pylint.design] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +# exclude-too-few-public-methods = + +# List of qualified class names to ignore when counting class parents (see R0901) +# ignored-parents = + +# Maximum number of arguments for function / method. +max-args = 5 + +# Maximum number of attributes for a class (see R0902). +max-attributes = 7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr = 5 + +# Maximum number of branch for function / method body. +max-branches = 12 + +# Maximum number of locals for function / method body. +max-locals = 15 + +# Maximum number of parents for a class (see R0901). +max-parents = 7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods = 20 + +# Maximum number of return / yield for function / method body. +max-returns = 6 + +# Maximum number of statements in function / method body. +max-statements = 50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods = 2 + +[tool.pylint.exceptions] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"] + +[tool.pylint.format] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +# expected-line-ending-format = + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines = "^\\s*(# )??$" + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren = 4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string = " " + +# Maximum number of characters on a single line. +max-line-length = 100 + +# Maximum number of lines in a module. +max-module-lines = 1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +# single-line-class-stmt = + +# Allow the body of an if to be on the same line as the test if there is no else. +# single-line-if-stmt = + +[tool.pylint.imports] +# List of modules that can be imported at any level, not just the top level one. +# allow-any-import-level = + +# Allow explicit reexports by alias from a package __init__. +# allow-reexport-from-package = + +# Allow wildcard imports from modules that define __all__. +# allow-wildcard-with-all = + +# Deprecated modules which should not be used, separated by a comma. +# deprecated-modules = + +# Output a graph (.gv or any supported image format) of external dependencies to +# the given file (report RP0402 must not be disabled). +# ext-import-graph = + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be disabled). +# import-graph = + +# Output a graph (.gv or any supported image format) of internal dependencies to +# the given file (report RP0402 must not be disabled). +# int-import-graph = + +# Force import order to recognize a module as part of the standard compatibility +# libraries. +# known-standard-library = + +# Force import order to recognize a module as part of a third party library. +known-third-party = ["enchant"] + +# Couples of modules and preferred modules, separated by a comma. +# preferred-modules = + +[tool.pylint.logging] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style = "new" + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules = ["logging"] + +[tool.pylint."messages control"] +# Only show warnings with the listed confidence levels. Leave empty to show all. +# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] + +# Disable the message, report, category or checker with the given id(s). You can +# either give multiple identifiers separated by comma (,) or put this option +# multiple times (only on the command line, not in the configuration file where +# it should appear only once). You can also use "--disable=all" to disable +# everything first and then re-enable specific checks. For example, if you want +# to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "invalid-name", "missing-module-docstring", "import-error", "too-many-instance-attributes", "too-many-arguments", "too-many-locals", "logging-fstring-interpolation"] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where it +# should appear only once). See also the "--disable" option for examples. +enable = ["c-extension-no-member"] + +[tool.pylint.method_args] +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] + +[tool.pylint.miscellaneous] +# List of note tags to take in consideration, separated by a comma. +notes = ["FIXME", "XXX", "TODO"] + +# Regular expression of note tags to take in consideration. +# notes-rgx = + +[tool.pylint.refactoring] +# Maximum number of nested blocks for function / method body +max-nested-blocks = 5 + +# Complete name of functions that never returns. When checking for inconsistent- +# return-statements if a never returning function is called then it will be +# considered as an explicit return statement and no message will be printed. +never-returning-functions = ["sys.exit", "argparse.parse_error"] + +[tool.pylint.reports] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each category, +# as well as 'statement' which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))" + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +# msg-template = + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +# output-format = + +# Tells whether to display a full report or only the messages. +# reports = + +# Activate the evaluation score. +score = true + +[tool.pylint.similarities] +# Comments are removed from the similarity computation +ignore-comments = true + +# Docstrings are removed from the similarity computation +ignore-docstrings = true + +# Imports are removed from the similarity computation +# ignore-imports = + +# Signatures are removed from the similarity computation +ignore-signatures = true + +# Minimum lines number of a similarity. +min-similarity-lines = 4 + +[tool.pylint.spelling] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions = 4 + +# Spelling dictionary name. No available dictionaries : You need to install both +# the python package and the system dependency for enchant to work.. +# spelling-dict = + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" + +# List of comma separated words that should not be checked. +# spelling-ignore-words = + +# A path to a file that contains the private dictionary; one word per line. +# spelling-private-dict-file = + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +# spelling-store-unknown-words = + +[tool.pylint.typecheck] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators = ["contextlib.contextmanager"] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# see https://github.com/pylint-dev/pylint/issues/2289 +generated-members = ["gist_heat_r"] + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +# Tells whether to warn about missing members when the owner of the attribute is +# inferred to be None. +ignore-none = true + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference can +# return multiple potential results while evaluating a Python object, but some +# branches might not be evaluated, which results in partial inference. In that +# case, it might be useful to still emit no-member and other checks for the rest +# of the inferred objects. +ignore-on-opaque-inference = true + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes = ["optparse.Values", "thread._local", "_thread._local", "argparse.Namespace"] + +# Show a hint with possible names when a member name was not found. The aspect of +# finding the hint is based on edit distance. +missing-member-hint = true + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance = 1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices = 1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx = ".*[Mm]ixin" + +# List of decorators that change the signature of a decorated function. +# signature-mutators = + +[tool.pylint.variables] +# List of additional names supposed to be defined in builtins. Remember that you +# should avoid defining new builtins when possible. +# additional-builtins = + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables = true + +# List of names allowed to shadow builtins +# allowed-redefined-builtins = + +# List of strings which can identify a callback function by name. A callback name +# must start or end with one of those strings. +callbacks = ["cb_", "_cb"] + +# A regular expression matching the name of dummy variables (i.e. expected to not +# be used). +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" + +# Argument names that match this expression will be ignored. +ignored-argument-names = "_.*|^ignored_|^unused_" + +# Tells whether we should check for unused import in __init__ files. +# init-import = + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + [tool.pytest.ini_options] minversion="6.0" norecursedirs=["build", "docs/_build"] diff --git a/simtools/_dev_version/scm_version.py b/simtools/_dev_version/scm_version.py index 2a7daf063..15876487e 100644 --- a/simtools/_dev_version/scm_version.py +++ b/simtools/_dev_version/scm_version.py @@ -6,5 +6,5 @@ from setuptools_scm import get_version version = get_version(root=pth.join("..", ".."), relative_to=__file__) -except Exception: - raise ImportError("setuptools_scm broken or not installed") +except Exception as exc: + raise ImportError("setuptools_scm broken or not installed") from exc diff --git a/simtools/applications/add_file_to_db.py b/simtools/applications/add_file_to_db.py index d18298ac2..ecc9e3e98 100644 --- a/simtools/applications/add_file_to_db.py +++ b/simtools/applications/add_file_to_db.py @@ -116,7 +116,7 @@ def main(): db = db_handler.DatabaseHandler(mongo_db_config=db_config) - files_to_insert = list() + files_to_insert = [] if args_dict.get("file_name", None) is not None: for file_now in args_dict["file_name"]: if Path(file_now).suffix in db.ALLOWED_FILE_EXTENSIONS: @@ -133,7 +133,7 @@ def main(): plural = "s" if len(files_to_insert) < 1: raise ValueError("No files were provided to upload") - elif len(files_to_insert) == 1: + if len(files_to_insert) == 1: plural = "" else: pass diff --git a/simtools/applications/compare_cumulative_psf.py b/simtools/applications/compare_cumulative_psf.py index da3c86e1f..235d86840 100644 --- a/simtools/applications/compare_cumulative_psf.py +++ b/simtools/applications/compare_cumulative_psf.py @@ -94,6 +94,10 @@ def load_data(datafile): + """ + Load the data file with the measured PSF vs radius [cm]. + + """ d_type = {"names": ("Radius [cm]", "Relative intensity"), "formats": ("f8", "f8")} # test_data_file = io.get_test_data_file('PSFcurve_data_v2.txt') data = np.loadtxt(datafile, dtype=d_type, usecols=(0, 2)) @@ -146,7 +150,7 @@ def main(): # New parameters if args_dict.get("pars", None): - with open(args_dict["pars"]) as file: + with open(args_dict["pars"], encoding="utf-8") as file: new_pars = yaml.safe_load(file) tel_model.change_multiple_parameters(**new_pars) diff --git a/simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py b/simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py index 4bb173c03..9064768ff 100644 --- a/simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py +++ b/simtools/applications/db_development_tools/add_unit_to_parameter_in_db.py @@ -18,7 +18,7 @@ def main(): - config = configurator.Configurator(description=("Add a unit field to a parameter in the DB.")) + config = configurator.Configurator(description="Add a unit field to a parameter in the DB.") args_dict, db_config = config.initialize(db_config=True, telescope_model=True) logger = logging.getLogger() diff --git a/simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py b/simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py index 9e2cc81e3..f2ccd0169 100644 --- a/simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py +++ b/simtools/applications/db_development_tools/mark_non_optics_parameters_non_applicable.py @@ -44,7 +44,7 @@ def main(): db = db_handler.DatabaseHandler(mongo_db_config=db_config) - with open(args_dict["sections"], "r") as stream: + with open(args_dict["sections"], "r", encoding="utf-8") as stream: parameter_catogeries = yaml.safe_load(stream) non_optic_catagories = [ @@ -54,7 +54,7 @@ def main(): "Camera", "Unnecessary", ] - non_optic_parameters = list() + non_optic_parameters = [] for category in non_optic_catagories: for par_now in parameter_catogeries[category]: non_optic_parameters.append(par_now) diff --git a/simtools/applications/derive_mirror_rnda.py b/simtools/applications/derive_mirror_rnda.py index 2505d924a..4a50dad4a 100644 --- a/simtools/applications/derive_mirror_rnda.py +++ b/simtools/applications/derive_mirror_rnda.py @@ -367,9 +367,9 @@ def run(rnda): logger.info(f"Start value for mirror_reflection_random_angle: {rnda_start}") - results_rnda = list() - results_mean = list() - results_sig = list() + results_rnda = [] + results_mean = [] + results_sig = [] if args_dict["no_tuning"]: rnda_opt = rnda_start else: diff --git a/simtools/applications/make_regular_arrays.py b/simtools/applications/make_regular_arrays.py index de732e7f4..31d51f423 100644 --- a/simtools/applications/make_regular_arrays.py +++ b/simtools/applications/make_regular_arrays.py @@ -71,13 +71,13 @@ def main(): # Reading site parameters from DB db = db_handler.DatabaseHandler(mongo_db_config=db_config) - site_pars_db = dict() - layout_center_data = dict() - corsika_telescope_data = dict() + site_pars_db = {} + layout_center_data = {} + corsika_telescope_data = {} for site in ["North", "South"]: site_pars_db[site] = db.get_site_parameters(site=site, model_version="prod5") - layout_center_data[site] = dict() + layout_center_data[site] = {} layout_center_data[site]["center_lat"] = ( float(site_pars_db[site]["ref_lat"]["Value"]) * u.deg ) @@ -88,7 +88,7 @@ def main(): float(site_pars_db[site]["altitude"]["Value"]) * u.m ) layout_center_data[site]["EPSG"] = site_pars_db[site]["EPSG"]["Value"] - corsika_telescope_data[site] = dict() + corsika_telescope_data[site] = {} corsika_telescope_data[site]["corsika_obs_level"] = layout_center_data[site]["center_alt"] corsika_telescope_data[site]["corsika_sphere_center"] = corsika_pars[ "corsika_sphere_center" diff --git a/simtools/applications/plot_simtel_histograms.py b/simtools/applications/plot_simtel_histograms.py index 115064588..02a0bd9ee 100644 --- a/simtools/applications/plot_simtel_histograms.py +++ b/simtools/applications/plot_simtel_histograms.py @@ -60,11 +60,11 @@ def main(): logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) n_lists = len(args_dict["file_lists"]) - simtel_histograms = list() + simtel_histograms = [] for this_list_of_files in args_dict["file_lists"]: # Collecting hist files - histogram_files = list() - with open(this_list_of_files) as file: + histogram_files = [] + with open(this_list_of_files, encoding="utf-8") as file: for line in file: # Removing '\n' from filename, in case it is left there. histogram_files.append(line.replace("\n", "")) diff --git a/simtools/applications/print_array_elements.py b/simtools/applications/print_array_elements.py index d0b2e46de..98f61e68c 100644 --- a/simtools/applications/print_array_elements.py +++ b/simtools/applications/print_array_elements.py @@ -125,7 +125,7 @@ def _parse(label=None, description=None): def main(): label = Path(__file__).stem - args_dict, _ = _parse(label, description=("Print a list of array element positions")) + args_dict, _ = _parse(label, description="Print a list of array element positions") _logger = logging.getLogger() _logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) diff --git a/simtools/applications/production.py b/simtools/applications/production.py index 362ee4715..1e2bd7578 100644 --- a/simtools/applications/production.py +++ b/simtools/applications/production.py @@ -167,16 +167,16 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): """ try: - with open(config_file) as file: + with open(config_file, encoding="utf-8") as file: config_data = yaml.load(file) except FileNotFoundError: logger.error(f"Error loading simulation configuration file from {config_file}") raise - label = config_data.pop("label", dict()) - default_data = config_data.pop("default", dict()) - config_showers = dict() - config_arrays = dict() + label = config_data.pop("label", {}) + default_data = config_data.pop("default", {}) + config_showers = {} + config_arrays = {} for primary, primary_data in config_data.items(): if primary_config is not None and primary != primary_config: @@ -184,8 +184,8 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): this_default = copy(default_data) - config_showers[primary] = copy(this_default.pop("showers", dict())) - config_arrays[primary] = copy(this_default.pop("array", dict())) + config_showers[primary] = copy(this_default.pop("showers", {})) + config_arrays[primary] = copy(this_default.pop("array", {})) # Grabbing common entries for showers and array for key, value in primary_data.items(): @@ -195,12 +195,12 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): config_arrays[primary][key] = value # Grabbing showers entries - for key, value in primary_data.get("showers", dict()).items(): + for key, value in primary_data.get("showers", {}).items(): config_showers[primary][key] = value config_showers[primary]["primary"] = primary # Grabbing array entries - for key, value in primary_data.get("array", dict()).items(): + for key, value in primary_data.get("array", {}).items(): config_arrays[primary][key] = value config_arrays[primary]["primary"] = primary @@ -213,7 +213,7 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): def main(): - args_dict, db_config = _parse(description=("Air shower and array simulations")) + args_dict, db_config = _parse(description="Air shower and array simulations") logger = logging.getLogger() logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) @@ -224,7 +224,7 @@ def main(): if args_dict["label"] is None: args_dict["label"] = label - shower_simulators = dict() + shower_simulators = {} for primary, config_data in shower_configs.items(): shower_simulators[primary] = Simulator( label=label, @@ -242,7 +242,7 @@ def main(): _task_function() if args_dict["array_only"]: - array_simulators = dict() + array_simulators = {} for primary, config_data in array_configs.items(): array_simulators[primary] = Simulator( label=label, diff --git a/simtools/applications/sim_showers_for_trigger_rates.py b/simtools/applications/sim_showers_for_trigger_rates.py index 5890a2df0..50ad4b050 100644 --- a/simtools/applications/sim_showers_for_trigger_rates.py +++ b/simtools/applications/sim_showers_for_trigger_rates.py @@ -177,7 +177,7 @@ def main(): log_file_list = output_dir.joinpath(f"log_files_{args_dict['primary']}.list") def print_list_into_file(list_of_files, file_name): - with open(file_name, "w") as f: + with open(file_name, "w", encoding="utf-8") as f: for line in list_of_files: f.write(line + "\n") diff --git a/simtools/applications/simulate_prod.py b/simtools/applications/simulate_prod.py index b680e439c..2c5859713 100644 --- a/simtools/applications/simulate_prod.py +++ b/simtools/applications/simulate_prod.py @@ -226,7 +226,7 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): """ try: - with open(config_file) as file: + with open(config_file, encoding="utf-8") as file: config_data = yaml.load(file) except FileNotFoundError: logger.error(f"Error loading simulation configuration file from {config_file}") @@ -274,7 +274,7 @@ def _proccess_simulation_config_file(config_file, primary_config, logger): def main(): - args_dict, db_config = _parse(description=("Run simulations for productions")) + args_dict, db_config = _parse(description="Run simulations for productions") logger = logging.getLogger() logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) diff --git a/simtools/applications/tune_psf.py b/simtools/applications/tune_psf.py index 3a17f4e6a..4184cbb2f 100644 --- a/simtools/applications/tune_psf.py +++ b/simtools/applications/tune_psf.py @@ -102,9 +102,17 @@ from simtools.visualization import visualize -def load_data(datafile): +def load_data(data_file): + """ + Load data from file txt file. + + Parameters + ---------- + data_file: str + Name of the data file with the measured cumulative PSF. + """ d_type = {"names": ("Radius [cm]", "Cumulative PSF"), "formats": ("f8", "f8")} - data = np.loadtxt(datafile, dtype=d_type, usecols=(0, 2)) + data = np.loadtxt(data_file, dtype=d_type, usecols=(0, 2)) data["Radius [cm]"] *= 0.1 data["Cumulative PSF"] /= np.max(np.abs(data["Cumulative PSF"])) return data @@ -169,7 +177,7 @@ def main(): # } # tel_model.change_multiple_parameters(**pars_to_change) - all_parameters = list() + all_parameters = [] def add_parameters( mirror_reflection, mirror_align, mirror_reflection_fraction=0.15, mirror_reflection_2=0.035 @@ -178,7 +186,7 @@ def add_parameters( Transform the parameters to the proper format and add a new set of parameters to the all_parameters list. """ - pars = dict() + pars = {} mrra = f"{mirror_reflection:.4f},{mirror_reflection_fraction:.2f},{mirror_reflection_2:.4f}" pars["mirror_reflection_random_angle"] = mrra mar = f"{mirror_align:.4f},28.,0.,0." diff --git a/simtools/camera_efficiency.py b/simtools/camera_efficiency.py index 13da26fd3..7a3332fcb 100644 --- a/simtools/camera_efficiency.py +++ b/simtools/camera_efficiency.py @@ -230,7 +230,7 @@ def analyze(self, export=True, force=False): # Search for at least 5 consecutive numbers to see that we are in the table re_table = re.compile("{0}{0}{0}{0}{0}".format(r"[-+]?[0-9]*\.?[0-9]+\s+")) - with open(self._file_simtel, "r") as file: + with open(self._file_simtel, "r", encoding="utf-8") as file: for line in file: if re_table.match(line): words = line.split() diff --git a/simtools/configuration/configurator.py b/simtools/configuration/configurator.py index f4fbd05f2..a65f8f6b3 100644 --- a/simtools/configuration/configurator.py +++ b/simtools/configuration/configurator.py @@ -242,7 +242,7 @@ def _fill_from_config_file(self, config_file): try: self._logger.debug(f"Reading configuration from {config_file}") - with open(config_file, "r") as stream: + with open(config_file, "r", encoding="utf-8") as stream: _config_dict = yaml.safe_load(stream) if "CTASIMPIPE" in _config_dict: try: diff --git a/simtools/corsika/corsika_config.py b/simtools/corsika/corsika_config.py index e1402a75b..4abb4b893 100644 --- a/simtools/corsika/corsika_config.py +++ b/simtools/corsika/corsika_config.py @@ -143,7 +143,7 @@ def load_corsika_parameters_file(corsika_parameters_file): logger = logging.getLogger(__name__) logger.debug(f"Loading CORSIKA parameters from file {corsika_parameters_file}") - with open(corsika_parameters_file, "r") as f: + with open(corsika_parameters_file, "r", encoding="utf-8") as f: corsika_parameters = yaml.load(f) return corsika_parameters @@ -180,7 +180,7 @@ def set_user_parameters(self, corsika_config_data): """ self._logger.debug("Setting user parameters from corsika_config_data") - self._user_parameters = dict() + self._user_parameters = {} user_pars = self._corsika_parameters["USER_PARAMETERS"] @@ -259,7 +259,7 @@ def _validate_and_convert_argument(self, par_name, par_info, value_args_in): par_unit = gen.copy_as_list(par_info["unit"]) # Checking units and converting them, if needed. - value_args_with_units = list() + value_args_with_units = [] for arg, unit in zip(value_args, par_unit): if unit is None: value_args_with_units.append(arg) @@ -374,7 +374,7 @@ def _get_text_multiple_lines(pars): text += _get_text_single_line(new_pars) return text - with open(self._config_file_path, "w") as file: + with open(self._config_file_path, "w", encoding="utf-8") as file: file.write("\n* [ RUN PARAMETERS ]\n") # Removing AZM entry first _user_pars_temp = copy.copy(self._user_parameters) diff --git a/simtools/corsika/corsika_histograms.py b/simtools/corsika/corsika_histograms.py index ecd81ccbb..146e8d7e9 100644 --- a/simtools/corsika/corsika_histograms.py +++ b/simtools/corsika/corsika_histograms.py @@ -133,7 +133,8 @@ def _initialize_header(self): Initialize the header. """ self.all_run_keys = list( - run_header.run_header_types[np.around(self.corsika_version, 1)].names) + run_header.run_header_types[np.around(self.corsika_version, 1)].names + ) self._header = {} # Get units of the header @@ -168,7 +169,6 @@ def read_event_information(self): """ if self.event_information is None: - with IACTFile(self.input_file) as self.iact_file: self.telescope_positions = np.array(self.iact_file.telescope_positions) self.num_telescopes = np.size(self.telescope_positions, axis=0) @@ -542,7 +542,6 @@ def _fill_histograms(self, photons, rotation_around_z_axis=None, rotation_around for i_tel_info, photons_info in np.array( list(zip(self.telescope_positions, photons)), dtype=object )[self.telescope_indices]: - if rotation_around_z_axis is None or rotation_around_y_axis is None: photon_x, photon_y = photons_info["x"], photons_info["y"] else: @@ -611,7 +610,6 @@ def set_histograms(self, telescope_indices=None, individual_telescopes=None, his event_counter = 0 for event in f: for i_telescope in self.telescope_indices: - if hasattr(event, "photon_bunches"): photons = list(event.photon_bunches.values()) else: @@ -1284,8 +1282,8 @@ def _export_2D_histograms(self): hist_2D_list, x_edges_list, y_edges_list = function() if function_dict["function"] == "get_2D_photon_density_distr": histogram_value_unit = 1 / ( - self._dict_2D_distributions[property_name]["x edges unit"] - * self._dict_2D_distributions[property_name]["y edges unit"] + self._dict_2D_distributions[property_name]["x edges unit"] + * self._dict_2D_distributions[property_name]["y edges unit"] ) else: histogram_value_unit = u.dimensionless_unscaled @@ -1363,9 +1361,7 @@ def fill_ecsv_table(self, hist, x_edges, y_edges, x_label, y_label): ) return table - def export_event_header_1D_histogram( - self, event_header_element, bins=50, hist_range=None - ): + def export_event_header_1D_histogram(self, event_header_element, bins=50, hist_range=None): """ Export to a ecsv file the 1D histogram for the key `event_header_element` from the CORSIKA event header. @@ -1387,7 +1383,8 @@ def export_event_header_1D_histogram( edges *= self.event_information[event_header_element].unit table = self.fill_ecsv_table(hist, edges, None, event_header_element, None) ecsv_file = Path(self.output_path).joinpath( - f"event_1D_histograms_{event_header_element}.ecsv") + f"event_1D_histograms_{event_header_element}.ecsv" + ) self._logger.info(f"Exporting histogram to {ecsv_file}.") table.write(ecsv_file, format="ascii.ecsv", overwrite=True) @@ -1503,7 +1500,6 @@ def event_zenith_angles(self): The zenith angles for each event. """ if self._event_zenith_angles is None: - self._event_zenith_angles = np.around( (self.event_information["zenith"]).to(u.deg), 4, diff --git a/simtools/corsika/corsika_output_visualize.py b/simtools/corsika/corsika_output_visualize.py new file mode 100644 index 000000000..dc5a4d968 --- /dev/null +++ b/simtools/corsika/corsika_output_visualize.py @@ -0,0 +1,345 @@ +import logging + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import colors + +_logger = logging.getLogger(__name__) + + +def _kernel_plot_2D_photons(corsika_output_instance, property_name, log_z=False): + """ + The next functions below are used by the the corsikaOutput class to plot all sort of information + from the Cherenkov photons saved. + + Create the figure of a 2D plot. The parameter `name` indicate which plot. Choices are + "counts", "density", "direction". + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + property_name: string + Name of the quantity. Options are: "counts", "density", "direction", "time_altitude" and + "num_photons_per_telescope". + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + + Returns + ------- + list + List of figures for the given telescopes. + + Raises + ------ + ValueError + if `name` is not allowed. + + """ + x_label = { + "counts": "x (m)", + "density": "x (m)", + "direction": "cos(x)", + "time_altitude": "Time since 1st interaction (ns)", + "num_photons_per_telescope": "Event number", + } + y_label = { + "counts": "y (m)", + "density": "y (m)", + "direction": "cos(y)", + "time_altitude": "Altitude of emission (km)", + "num_photons_per_telescope": "Telescope index", + } + if property_name not in x_label: + msg = f"property_name must be one of {list(x_label.keys())}" + _logger.error(msg) + raise ValueError(msg) + + if property_name == "counts": + hist_values, x_edges, y_edges = corsika_output_instance.get_2D_photon_position_distr( + density=False + ) + elif property_name == "density": + hist_values, x_edges, y_edges = corsika_output_instance.get_2D_photon_position_distr( + density=True + ) + elif property_name == "direction": + hist_values, x_edges, y_edges = corsika_output_instance.get_2D_photon_direction_distr() + elif property_name == "time_altitude": + hist_values, x_edges, y_edges = corsika_output_instance.get_2D_photon_time_altitude() + elif property_name == "num_photons_per_telescope": + hist_values, x_edges, y_edges = corsika_output_instance.get_2D_num_photons_distr() + hist_values, x_edges, y_edges = [hist_values], [x_edges], [y_edges] + + all_figs = [] + for i_hist, _ in enumerate(x_edges): + fig, ax = plt.subplots() + if log_z is True: + norm = colors.LogNorm(vmin=1, vmax=np.amax([np.amax(hist_values[i_hist]), 2])) + else: + norm = None + mesh = ax.pcolormesh(x_edges[i_hist], y_edges[i_hist], hist_values[i_hist], norm=norm) + ax.set_xlabel(x_label[property_name]) + ax.set_ylabel(y_label[property_name]) + ax.set_xlim(np.amin(x_edges[i_hist]), np.amax(x_edges[i_hist])) + ax.set_ylim(np.amin(y_edges[i_hist]), np.amax(y_edges[i_hist])) + ax.set_facecolor("xkcd:black") + fig.colorbar(mesh) + all_figs.append(fig) + if corsika_output_instance.individual_telescopes is False: + fig.savefig(f"histogram_{property_name}_2D_all_tels.png", bbox_inches="tight") + else: + ax.text( + 0.99, + 0.99, + "tel. " + str(i_hist), + ha="right", + va="top", + transform=ax.transAxes, + color="white", + ) + fig.savefig( + f"histogram_{property_name}_2D_tel_" + f"{str(corsika_output_instance.telescope_indices[i_hist])}.png", + bbox_inches="tight", + ) + plt.close() + + return all_figs + + +def plot_2D_counts(corsika_output_instance, log_z=True): + """ + Plot the 2D histogram of the photon positions on the ground. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + """ + return _kernel_plot_2D_photons(corsika_output_instance, "counts", log_z=log_z) + + +def plot_2D_density(corsika_output_instance, log_z=True): + """ + Plot the 2D histogram of the photon density distribution on the ground. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + """ + return _kernel_plot_2D_photons(corsika_output_instance, "density", log_z=log_z) + + +def plot_2D_direction(corsika_output_instance, log_z=True): + """ + Plot the 2D histogram of the incoming direction of photons. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + """ + return _kernel_plot_2D_photons(corsika_output_instance, "direction", log_z=log_z) + + +def plot_2D_time_altitude(corsika_output_instance, log_z=True): + """ + Plot the 2D histogram of the time and altitude where the photon was produced. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + """ + return _kernel_plot_2D_photons(corsika_output_instance, "time_altitude", log_z=log_z) + + +def plot_2D_num_photons_per_telescope(corsika_output_instance, log_z=True): + """ + Plot the 2D histogram of the number of photons per event and per telescope. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_z: bool + if True, the intensity of the color bar is given in logarithmic scale. + """ + return _kernel_plot_2D_photons( + corsika_output_instance, "num_photons_per_telescope", log_z=log_z + ) + + +def _kernel_plot_1D_photons(corsika_output_instance, property_name, log_y=True): + """ + Create the figure of a 1D plot. The parameter `name` indicate which plot. Choices are + "counts", "density", "direction". + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + property_name: string + Name of the quantity. Options are: "wavelength", "counts", "density", "time", "altitude", + "num_photons". + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + + Returns + ------- + list + List of figures for the given telescopes. + + Raises + ------ + ValueError + if `name` is not allowed. + """ + + x_label = { + "wavelength": "Wavelength (nm)", + "counts": "Distance to center (m)", + "density": "Distance to center (m)", + "time": "Time since 1st interaction (ns)", + "altitude": "Altitude of emission (km)", + "num_photons": "Number of photons per event", + } + if property_name not in x_label: + msg = f"results: status must be one of {list(x_label.keys())}" + _logger.error(msg) + raise ValueError(msg) + + if property_name == "wavelength": + hist_values, edges = corsika_output_instance.get_photon_wavelength_distr() + elif property_name == "counts": + hist_values, edges = corsika_output_instance.get_photon_radial_distr(density=False) + elif property_name == "density": + hist_values, edges = corsika_output_instance.get_photon_radial_distr(density=True) + elif property_name == "time": + hist_values, edges = corsika_output_instance.get_photon_time_of_emission_distr() + elif property_name == "altitude": + hist_values, edges = corsika_output_instance.get_photon_altitude_distr() + elif property_name == "num_photons": + hist_values, edges = corsika_output_instance.get_num_photons_distr() + hist_values, edges = [hist_values], [edges] + + all_figs = [] + for i_hist, _ in enumerate(edges): + fig, ax = plt.subplots() + ax.bar( + edges[i_hist][:-1], + hist_values[i_hist], + align="edge", + width=np.abs(np.diff(edges[i_hist])), + ) + ax.set_xlabel(x_label[property_name]) + ax.set_ylabel("Counts") + + if log_y is True: + ax.set_yscale("log") + if corsika_output_instance.individual_telescopes is False: + fig.savefig(f"histogram_{property_name}_tels.png", bbox_inches="tight") + else: + fig.savefig( + f"histogram_{property_name}_tel_" + f"{str(corsika_output_instance.telescope_indices[i_hist])}.png", + bbox_inches="tight", + ) + all_figs.append(fig) + return all_figs + + +def plot_wavelength_distr(corsika_output_instance, log_y=True): + """ + Plots the 1D distribution of the photon wavelengths + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + + """ + return _kernel_plot_1D_photons(corsika_output_instance, "wavelength", log_y=log_y) + + +def plot_counts_distr(corsika_output_instance, log_y=True): + """ + Plots the 1D distribution, i.e. the radial distribution, of the photons on the ground. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + """ + return _kernel_plot_1D_photons(corsika_output_instance, "counts", log_y=log_y) + + +def plot_density_distr(corsika_output_instance, log_y=True): + """ + Plots the photon density distribution on the ground. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + + """ + return _kernel_plot_1D_photons(corsika_output_instance, "density", log_y=log_y) + + +def plot_time_distr(corsika_output_instance, log_y=True): + """ + Plots the distribution times in which the photons were generated in ns. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + """ + return _kernel_plot_1D_photons(corsika_output_instance, "time", log_y=log_y) + + +def plot_altitude_distr(corsika_output_instance, log_y=True): + """ + Plots the distribution of altitude in which the photons were generated in km. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + """ + return _kernel_plot_1D_photons(corsika_output_instance, "altitude", log_y=log_y) + + +def plot_num_photons_distr(corsika_output_instance, log_y=True): + """ + Plots the distribution of the number of Cherenkov photons per event. + + Parameters + ---------- + corsika_output_instance: corsika.corsika_output.corsikaOutput + instance of corsika.corsika_output.corsikaOutput. + log_y: bool + if True, the intensity of the Y axis is given in logarithmic scale. + """ + return _kernel_plot_1D_photons(corsika_output_instance, "num_photons", log_y=log_y) diff --git a/simtools/corsika/corsika_runner.py b/simtools/corsika/corsika_runner.py index 201c062da..225d2ae39 100644 --- a/simtools/corsika/corsika_runner.py +++ b/simtools/corsika/corsika_runner.py @@ -216,7 +216,7 @@ def prepare_run_script(self, use_pfp=True, **kwargs): extra_commands = kwargs["extra_commands"] self._logger.debug(f"Extra commands to be added to the run script {extra_commands}") - with open(script_file_path, "w") as file: + with open(script_file_path, "w", encoding="utf-8") as file: # shebang file.write("#!/usr/bin/env bash\n") @@ -411,7 +411,7 @@ def get_resources(self, run_number=None): _resources = {} _resources["runtime"] = None - with open(sub_log_file, "r") as file: + with open(sub_log_file, "r", encoding="utf-8") as file: for line in reversed(list(file)): if "RUNTIME" in line: _resources["runtime"] = int(line.split()[1]) diff --git a/simtools/corsika_simtel/corsika_simtel_runner.py b/simtools/corsika_simtel/corsika_simtel_runner.py index 6214568a3..d65678653 100644 --- a/simtools/corsika_simtel/corsika_simtel_runner.py +++ b/simtools/corsika_simtel/corsika_simtel_runner.py @@ -39,7 +39,6 @@ class CorsikaSimtelRunner(CorsikaRunner, SimtelRunnerArray): """ def __init__(self, common_args=None, corsika_args=None, simtel_args=None): - CorsikaRunner.__init__(self, use_multipipe=True, **(common_args | corsika_args)) SimtelRunnerArray.__init__(self, **(common_args | simtel_args)) @@ -94,7 +93,7 @@ def export_multipipe_script(self, **kwargs): multipipe_file = Path(self.corsika_config._config_file_path.parent).joinpath( self.corsika_config.get_file_name("multipipe") ) - with open(multipipe_file, "w") as file: + with open(multipipe_file, "w", encoding="utf-8") as file: file.write(f"{run_command}") self._export_multipipe_executable(multipipe_file) @@ -111,7 +110,7 @@ def _export_multipipe_executable(self, multipipe_file): multipipe_executable = Path(self.corsika_config._config_file_path.parent).joinpath( "run_cta_multipipe" ) - with open(multipipe_executable, "w") as file: + with open(multipipe_executable, "w", encoding="utf-8") as file: multipipe_command = Path(self._simtel_source_path).joinpath( "sim_telarray/bin/multipipe_corsika " f"-c {multipipe_file}" @@ -176,10 +175,9 @@ def get_file_name(self, file_type, run_number=None, **kwargs): if file_type in ["output", "log", "histogram"]: return SimtelRunnerArray.get_file_name(self, file_type=file_type, **kwargs) - else: - return CorsikaRunner.get_file_name( - self, file_type=file_type, run_number=run_number, **kwargs - ) + return CorsikaRunner.get_file_name( + self, file_type=file_type, run_number=run_number, **kwargs + ) def get_info_for_file_name(self, run_number): """ diff --git a/simtools/data_model/validate_data.py b/simtools/data_model/validate_data.py index f91f10c5b..d6b870fce 100644 --- a/simtools/data_model/validate_data.py +++ b/simtools/data_model/validate_data.py @@ -221,7 +221,7 @@ def _check_data_for_duplicates(self): self.data_table, keys=_column_with_unique_requirement ) _data_table_unique_for_all_columns = unique(self.data_table, keys=None) - with open(os.devnull, "w") as devnull: + with open(os.devnull, "w", encoding="utf-8") as devnull: if report_diff_values( _data_table_unique_for_key_column, _data_table_unique_for_all_columns, diff --git a/simtools/data_model/validate_schema.py b/simtools/data_model/validate_schema.py index a5323ea23..e02ffcd4f 100644 --- a/simtools/data_model/validate_schema.py +++ b/simtools/data_model/validate_schema.py @@ -85,7 +85,7 @@ def _validate_schema(self, ref_schema, data_dict): _this_data = data_dict[key] else: if self._field_is_optional(value): - self._logger.debug("Optional field %s", key) + self._logger.debug(f"Optional field {key}") continue msg = f"Missing required field '{key}'" raise ValueError(msg) @@ -96,7 +96,7 @@ def _validate_schema(self, ref_schema, data_dict): try: self._validate_data_type(value, key, _this_data) except UnboundLocalError: - self._logger.error("No data for `%s` key", key) + self._logger.error(f"No data for {key} key") raise else: self._validate_schema(value, _this_data) diff --git a/simtools/db_handler.py b/simtools/db_handler.py index 8fb69cdaa..c970f070c 100644 --- a/simtools/db_handler.py +++ b/simtools/db_handler.py @@ -315,7 +315,7 @@ def _get_model_parameters_yaml( _which_tel_labels = [_tel_name_converted] # Selecting version and applicable (if on) - _pars = dict() + _pars = {} for _tel in _which_tel_labels: _all_pars = self._get_all_model_parameters_yaml(_tel) @@ -379,7 +379,7 @@ def _get_model_parameters_mongo_db( _which_tel_labels = [_tel_name_db] # Selecting version and applicable (if on) - _pars = dict() + _pars = {} for _tel in _which_tel_labels: self._logger.debug(f"Getting {_tel} parameters from MongoDB") @@ -449,7 +449,7 @@ def read_mongo_db( """ collection = DatabaseHandler.db_client[db_name][collection_name] - _parameters = dict() + _parameters = {} _model_version = self._convert_version_to_tagged( model_version, DatabaseHandler.DB_CTA_SIMULATION_MODEL @@ -500,7 +500,7 @@ def _get_all_model_parameters_yaml(self, telescope_name_yaml): _file_name_db = f"parValues-{telescope_name_yaml}.yml" _yaml_file = gen.find_file(_file_name_db, self.io_handler.model_path) self._logger.debug(f"Reading DB file {_yaml_file}") - with open(_yaml_file, "r") as stream: + with open(_yaml_file, "r", encoding="utf-8") as stream: _all_pars = yaml.safe_load(stream) return _all_pars @@ -564,10 +564,10 @@ def _get_site_parameters_yaml(self, site, model_version, only_applicable=False): yaml_file = gen.find_file("parValues-Sites.yml", self.io_handler.model_path) self._logger.info(f"Reading DB file {yaml_file}") - with open(yaml_file, "r") as stream: + with open(yaml_file, "r", encoding="utf-8") as stream: _all_pars_versions = yaml.safe_load(stream) - _pars = dict() + _pars = {} for par_name, par_info in _all_pars_versions.items(): if not par_info["Applicable"] and only_applicable: continue @@ -606,7 +606,7 @@ def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_appli _site_validated = names.validate_site_name(site) collection = DatabaseHandler.db_client[db_name].sites - _parameters = dict() + _parameters = {} _model_version = self._convert_version_to_tagged( model_version, DatabaseHandler.DB_CTA_SIMULATION_MODEL @@ -654,7 +654,7 @@ def get_descriptions( collection = DatabaseHandler.db_client[db_name][collection_name] - _parameters = dict() + _parameters = {} empty_query = {} for post in collection.find(empty_query): @@ -691,7 +691,7 @@ def get_reference_data(self, site, model_version, only_applicable=False): _site_validated = names.validate_site_name(site) collection = DatabaseHandler.db_client[DatabaseHandler.DB_REFERENCE_DATA].reference_values - _parameters = dict() + _parameters = {} _model_version = self._convert_version_to_tagged( names.validate_model_version_name(model_version), @@ -861,7 +861,7 @@ def copy_telescope( ) collection = DatabaseHandler.db_client[db_name][collection_name] - db_entries = list() + db_entries = [] _version_to_copy = self._convert_version_to_tagged( version_to_copy, DatabaseHandler.DB_CTA_SIMULATION_MODEL @@ -918,7 +918,7 @@ def copy_documents(self, db_name, collection, query, db_to_copy_to, collection_t _collection = DatabaseHandler.db_client[db_name][collection] if collection_to_copy_to is None: collection_to_copy_to = collection - db_entries = list() + db_entries = [] for post in _collection.find(query): post.pop("_id", None) @@ -1271,7 +1271,7 @@ def add_new_parameter( collection = DatabaseHandler.db_client[db_name][collection_name] - db_entry = dict() + db_entry = {} if "telescopes" in collection_name: db_entry["Telescope"] = names.validate_telescope_name_db(telescope) elif "sites" in collection_name: diff --git a/simtools/job_execution/job_manager.py b/simtools/job_execution/job_manager.py index 39071c8dd..f6e294cc3 100644 --- a/simtools/job_execution/job_manager.py +++ b/simtools/job_execution/job_manager.py @@ -141,7 +141,7 @@ def _submit_htcondor(self): _condor_file = self.run_script + ".condor" self._logger.info(f"Submitting script to HTCondor ({_condor_file})") try: - with open(_condor_file, "w") as file: + with open(_condor_file, "w", encoding="utf-8") as file: file.write(f"Executable = {self.run_script}\n") file.write(f"Output = {self.run_out_file + '.out'}\n") file.write(f"Error = {self.run_out_file + '.err'}\n") diff --git a/simtools/layout/layout_array.py b/simtools/layout/layout_array.py index 2e96fd1ce..9b31713f3 100644 --- a/simtools/layout/layout_array.py +++ b/simtools/layout/layout_array.py @@ -64,6 +64,7 @@ def __init__( self.site = None if site is None else names.validate_site_name(site) self.io_handler = io_handler.IOHandler() + self.telescope_list_file = None self._telescope_list = [] self._epsg = None if telescope_list_file is None: @@ -113,9 +114,16 @@ def from_layout_array_name(cls, mongo_db_config, layout_array_name, label=None): return layout def __len__(self): + """ + Return number of telescopes in the layout. + """ return len(self._telescope_list) def __getitem__(self, i): + """ + Return telescope at list position i. + + """ return self._telescope_list[i] def _initialize_corsika_telescope(self, corsika_dict=None): @@ -450,13 +458,12 @@ def _assign_unit_to_quantity(self, value, unit): if isinstance(value.unit, type(unit)): self._logger.debug(f"Quantity {value} has already unit {unit}. Returning {value}") return value - else: - try: - value = value.to(unit) - return value - except u.UnitConversionError: - self._logger.error(f"Cannot convert {value.unit} to {unit}.") - raise + try: + value = value.to(unit) + return value + except u.UnitConversionError: + self._logger.error(f"Cannot convert {value.unit} to {unit}.") + raise return value * unit def _try_set_coordinate(self, row, tel, table, crs_name, key1, key2): @@ -964,9 +971,8 @@ def select_assets(self, asset_list=None): tel for tel in self._telescope_list if tel.asset_code in asset_list ] self._logger.info( - "Selected %d telescopes (from originally %d)", - len(self._telescope_list), - _n_telescopes, + f"Selected {len(self._telescope_list)} telescopes" + f" (from originally {_n_telescopes})" ) except TypeError: self._logger.info("No asset list provided, keeping all telescopes") diff --git a/simtools/layout/telescope_position.py b/simtools/layout/telescope_position.py index 3784d5d5c..f2d1e0128 100644 --- a/simtools/layout/telescope_position.py +++ b/simtools/layout/telescope_position.py @@ -40,6 +40,10 @@ def __init__(self, name=None): self.crs = self._default_coordinate_system_definition() def __str__(self): + """ + String representation of TelescopePosition. + + """ telstr = self.name if self.has_coordinates("corsika"): telstr += ( diff --git a/simtools/model/array_model.py b/simtools/model/array_model.py index 32a6271f1..c4d132304 100644 --- a/simtools/model/array_model.py +++ b/simtools/model/array_model.py @@ -150,9 +150,9 @@ def _build_array_model(self): ) # Building telescope models - self._telescope_model = list() # List of telescope models - _all_telescope_model_names = list() # List of telescope names without repetition - _all_pars_to_change = dict() + self._telescope_model = [] # List of telescope models + _all_telescope_model_names = [] # List of telescope names without repetition + _all_pars_to_change = {} for tel in self.layout: tel_size = names.get_telescope_type(tel.name) @@ -247,7 +247,7 @@ def _proccess_single_telescope(data): if isinstance(data, str): # Case 1: data is string (only name) tel_name = tel_size + "-" + data - return tel_name, dict() + return tel_name, {} # Case 2: data has a wrong type msg = "ArrayConfig has wrong input for a telescope" @@ -286,7 +286,7 @@ def export_simtel_telescope_config_files(self): Export sim_telarray config files for all the telescopes into the output model directory. """ - exported_models = list() + exported_models = [] for tel_model in self._telescope_model: name = tel_model.name + ( "_" + tel_model.extra_label if tel_model.extra_label != "" else "" diff --git a/simtools/model/camera.py b/simtools/model/camera.py index 7a0ee6a16..fc3eec2f3 100644 --- a/simtools/model/camera.py +++ b/simtools/model/camera.py @@ -89,19 +89,19 @@ def read_pixel_list(camera_config_file): clockwise by 30 degrees with respect to those denoted as 1. """ - pixels = dict() + pixels = {} pixels["pixel_diameter"] = 9999 pixels["pixel_shape"] = 9999 pixels["pixel_spacing"] = 9999 pixels["lightguide_efficiency_angle_file"] = "none" pixels["lightguide_efficiency_wavelength_file"] = "none" pixels["rotate_angle"] = 0 - pixels["x"] = list() - pixels["y"] = list() - pixels["pix_id"] = list() - pixels["pix_on"] = list() + pixels["x"] = [] + pixels["y"] = [] + pixels["pix_id"] = [] + pixels["pix_on"] = [] - with open(camera_config_file, "r") as dat_file: + with open(camera_config_file, "r", encoding="utf-8") as dat_file: for line in dat_file: pix_info = line.split() if line.startswith("PixType"): @@ -502,7 +502,7 @@ def _calc_edge_pixels(self, pixels, neighbours): self._logger.debug("Searching for edge pixels") - edge_pixel_indices = list() + edge_pixel_indices = [] for i_pix, _ in enumerate(pixels["x"]): if pixels["pixel_shape"] == 1 or pixels["pixel_shape"] == 3: @@ -660,9 +660,13 @@ def _plot_one_axis_def(plot, **kwargs): ha="center", va="center", size="xx-large", - arrowprops=dict( - arrowstyle="<|-", shrinkA=0, shrinkB=0, fc=kwargs["fc"], ec=kwargs["ec"] - ), + arrowprops={ + "arrowstyle": "<|-", + "shrinkA": 0, + "shrinkB": 0, + "fc": kwargs["fc"], + "ec": kwargs["ec"], + }, ) plot.gca().annotate( @@ -673,9 +677,13 @@ def _plot_one_axis_def(plot, **kwargs): ha="center", va="center", size="xx-large", - arrowprops=dict( - arrowstyle="<|-", shrinkA=0, shrinkB=0, fc=kwargs["fc"], ec=kwargs["ec"] - ), + arrowprops={ + "arrowstyle": "<|-", + "shrinkA": 0, + "shrinkB": 0, + "fc": kwargs["fc"], + "ec": kwargs["ec"], + }, ) def plot_pixel_layout(self, camera_in_sky_coor=False, pixels_id_to_print=50): @@ -699,7 +707,7 @@ def plot_pixel_layout(self, camera_in_sky_coor=False, pixels_id_to_print=50): if not camera_in_sky_coor: self._pixels["y"] = [(-1) * y_val for y_val in self._pixels["y"]] - on_pixels, edge_pixels, off_pixels = list(), list(), list() + on_pixels, edge_pixels, off_pixels = [], [], [] for i_pix, xy_pix_pos in enumerate(zip(self._pixels["x"], self._pixels["y"])): if self._pixels["pixel_shape"] == 1 or self._pixels["pixel_shape"] == 3: diff --git a/simtools/model/mirrors.py b/simtools/model/mirrors.py index 357869f78..1deab046b 100644 --- a/simtools/model/mirrors.py +++ b/simtools/model/mirrors.py @@ -26,7 +26,7 @@ def __init__(self, mirror_list_file): self._logger = logging.getLogger(__name__) self._logger.debug("Mirrors Init") - self._mirrors = dict() + self._mirrors = {} self.diameter = None self.shape = None self.number_of_mirrors = 0 @@ -91,16 +91,16 @@ def _read_mirror_list_from_sim_telarray(self): If number of mirrors is 0. """ - self._mirrors["number"] = list() - self._mirrors["pos_x"] = list() - self._mirrors["pos_y"] = list() - self._mirrors["diameter"] = list() - self._mirrors["flen"] = list() - self._mirrors["shape"] = list() + self._mirrors["number"] = [] + self._mirrors["pos_x"] = [] + self._mirrors["pos_y"] = [] + self._mirrors["diameter"] = [] + self._mirrors["flen"] = [] + self._mirrors["shape"] = [] mirror_counter = 0 collect_geo_pars = True - with open(self._mirror_list_file, "r") as file: + with open(self._mirror_list_file, "r", encoding="utf-8") as file: for line in file: line = line.split() if "#" in line[0] or "$" in line[0]: diff --git a/simtools/model/telescope_model.py b/simtools/model/telescope_model.py index 78452f23c..a675c6a44 100644 --- a/simtools/model/telescope_model.py +++ b/simtools/model/telescope_model.py @@ -72,7 +72,7 @@ def __init__( self.io_handler = io_handler.IOHandler() self.mongo_db_config = mongo_db_config - self._parameters = dict() + self._parameters = {} self._load_parameters_from_db() @@ -150,7 +150,7 @@ def from_config_file(cls, config_file_name, site, telescope_model_name, label=No TelescopeModel Instance of TelescopeModel. """ - parameters = dict() + parameters = {} tel = cls( site=site, telescope_model_name=telescope_model_name, @@ -186,7 +186,7 @@ def _process_line(words): par_value = par_value.rstrip().lstrip() # Removing trailing spaces (left and right) return par_name, par_value - with open(config_file_name, "r") as file: + with open(config_file_name, "r", encoding="utf-8") as file: for line in file: words = line.split() if len(words) == 0: @@ -369,7 +369,7 @@ def add_parameter(self, par_name, value, is_file=False, is_aplicable=True): raise InvalidParameter(msg) self._logger.info(f"Adding {par_name}={value} to the model") - self._parameters[par_name] = dict() + self._parameters[par_name] = {} self._parameters[par_name]["Value"] = value self._parameters[par_name]["Type"] = type(value) self._parameters[par_name]["Applicable"] = is_aplicable @@ -488,7 +488,7 @@ def add_parameter_file(self, par_name, file_path): Path of the file to be added to the config file directory. """ if self._added_parameter_files is None: - self._added_parameter_files = list() + self._added_parameter_files = [] self._added_parameter_files.append(par_name) shutil.copy(file_path, self._config_file_directory) @@ -611,7 +611,7 @@ def export_single_mirror_list_file(self, mirror_number, set_focal_length_to_zero self.site, self.name, self.model_version, mirror_number, self.label ) if self._single_mirror_list_file_paths is None: - self._single_mirror_list_file_paths = dict() + self._single_mirror_list_file_paths = {} self._single_mirror_list_file_paths[mirror_number] = self._config_file_directory.joinpath( file_name ) @@ -625,8 +625,6 @@ def export_single_mirror_list_file(self, mirror_number, set_focal_length_to_zero set_focal_length_to_zero, ) - # END of export_single_mirror_list_file - def get_single_mirror_list_file(self, mirror_number, set_focal_length_to_zero=False): """ Get the path to the single mirror list file. @@ -700,6 +698,10 @@ def _load_camera(self): ) def _load_simtel_config_writer(self): + """ + Load the SimtelConfigWriter object. + + """ if self.simtel_config_writer is None: self.simtel_config_writer = SimtelConfigWriter( site=self.site, @@ -728,7 +730,7 @@ def is_file_2D(self, par): file_name = self.get_parameter_value(par) file = self.get_config_directory().joinpath(file_name) - with open(file, "r") as f: + with open(file, "r", encoding="utf-8") as f: is2D = "@RPOL@" in f.read() return is2D @@ -751,7 +753,7 @@ def read_two_dim_wavelength_angle(self, file_name): _file = self.get_config_directory().joinpath(file_name) line_to_start_from = 0 - with open(_file, "r") as f: + with open(_file, "r", encoding="utf-8") as f: for i_line, line in enumerate(f): if line.startswith("ANGLE"): degrees = np.array(line.strip().split("=")[1].split(), dtype=np.float16) @@ -822,7 +824,7 @@ def calc_average_curve(curves, incidence_angle_dist): Instance of astropy.table.Table with the averaged curve. """ - weights = list() + weights = [] for angle_now in curves["Angle"]: weights.append( incidence_angle_dist["Fraction"][ diff --git a/simtools/psf_analysis.py b/simtools/psf_analysis.py index 2e1be7092..2ce5d25b5 100644 --- a/simtools/psf_analysis.py +++ b/simtools/psf_analysis.py @@ -44,13 +44,13 @@ def __init__(self, focal_length=None, total_scattered_area=None): self._total_photons = None self._number_of_detected_photons = None self._effective_area = None - self.photon_pos_x = list() - self.photon_pos_y = list() - self.photon_r = list() + self.photon_pos_x = [] + self.photon_pos_y = [] + self.photon_r = [] self.centroid_x = None self.centroid_y = None self._total_area = total_scattered_area - self._stored_PSF = dict() + self._stored_PSF = {} if focal_length is not None: self._cm_to_deg = 180.0 / pi / focal_length self._has_focal_length = True @@ -74,7 +74,7 @@ def read_photon_list_from_simtel_file(self, file): """ self._logger.info(f"Reading sim_telarray file {file}") self._total_photons = 0 - with open(file, "r") as f: + with open(file, "r", encoding="utf-8") as f: for line in f: self._process_simtel_line(line) @@ -419,7 +419,7 @@ def get_cumulative_data(self, radius=None): else: radius_all = list(np.linspace(0, 1.6 * self.get_psf(0.8), 30)) - intensity = list() + intensity = [] for rad in radius_all: intensity.append(self._sum_photons_in_radius(rad) / self._number_of_detected_photons) d_type = { diff --git a/simtools/ray_tracing.py b/simtools/ray_tracing.py index 56f14d59e..5551f19da 100644 --- a/simtools/ray_tracing.py +++ b/simtools/ray_tracing.py @@ -143,6 +143,9 @@ def from_kwargs(cls, **kwargs): return cls(**args, config_data=config_data) def __repr__(self): + """ + String representation of RayTracing class. + """ return f"RayTracing(label={self.label})\n" def _validate_telescope_model(self, tel): @@ -227,9 +230,9 @@ def analyze( cm_to_deg = 180.0 / pi / focal_length - self._psf_images = dict() + self._psf_images = {} if do_analyze: - _rows = list() + _rows = [] else: self._read_results() @@ -323,7 +326,7 @@ def _process_rx(self, file, containment_fraction=0.8): """ try: - with open(file) as _stdin: + with open(file, encoding="utf-8") as _stdin: rx_output = subprocess.Popen( # pylint: disable=consider-using-with shlex.split( f"{self._simtel_source_path}/sim_telarray/bin/rx " @@ -484,7 +487,7 @@ def images(self): ------- List of PSFImage's """ - images = list() + images = [] for this_off_axis in self.config.off_axis_angle: if this_off_axis in self._psf_images: images.append(self._psf_images[this_off_axis]) diff --git a/simtools/simtel/simtel_config_writer.py b/simtools/simtel/simtel_config_writer.py index 5b8f29f51..4cbd8b08e 100644 --- a/simtools/simtel/simtel_config_writer.py +++ b/simtools/simtel/simtel_config_writer.py @@ -73,7 +73,7 @@ def write_telescope_config_file(self, config_file_path, parameters): parameters: dict Model parameters in the same structure as used by the TelescopeModel class. """ - with open(config_file_path, "w") as file: + with open(config_file_path, "w", encoding="utf-8") as file: self._write_header(file, "TELESCOPE CONFIGURATION FILE") file.write("#ifdef TELESCOPE\n") @@ -104,7 +104,7 @@ def write_array_config_file(self, config_file_path, layout, telescope_model, sit site_parameters: dict Site parameters. """ - with open(config_file_path, "w") as file: + with open(config_file_path, "w", encoding="utf-8") as file: self._write_header(file, "ARRAY CONFIGURATION FILE") # Be careful with the formatting - simtel is sensitive @@ -157,7 +157,7 @@ def write_single_mirror_list_file( """ __, __, diameter, flen, shape = mirrors.get_single_mirror_parameters(mirror_number) - with open(single_mirror_list_file, "w") as file: + with open(single_mirror_list_file, "w", encoding="utf-8") as file: self._write_header(file, "MIRROR LIST FILE", "#") file.write("# Column 1: X pos. [cm] (North/Down)\n") diff --git a/simtools/simtel/simtel_events.py b/simtools/simtel/simtel_events.py index 73e44ef70..c92754644 100644 --- a/simtools/simtel/simtel_events.py +++ b/simtools/simtel/simtel_events.py @@ -42,7 +42,7 @@ def load_input_files(self, files=None): List of sim_telarray files (str or Path). """ if not hasattr(self, "input_files"): - self.input_files = list() + self.input_files = [] if files is None: msg = "No input file was given" @@ -85,10 +85,10 @@ def load_header_and_summary(self): "spectral_index", "B_total", ] - self._mc_header = dict() + self._mc_header = {} def _are_headers_consistent(header0, header1): - comparison = dict() + comparison = {} for k in keys_to_grab: value = header0[k] == header1[k] comparison[k] = value if isinstance(value, bool) else all(value) @@ -97,10 +97,9 @@ def _are_headers_consistent(header0, header1): is_first_file = True number_of_triggered_events = 0 - summary_energy, summary_rcore = list(), list() + summary_energy, summary_rcore = [], [] for file in self.input_files: with SimTelFile(file) as f: - for event in f: en = event["mc_shower"]["energy"] rc = math.sqrt( @@ -184,10 +183,9 @@ def select_events(self, energy_range=None, core_max=None): energy_range = self._validate_energy_range(energy_range) core_max = self._validate_core_max(core_max) - selected_events = list() + selected_events = [] for file in self.input_files: with SimTelFile(file) as f: - for event in f: energy = event["mc_shower"]["energy"] if energy < energy_range[0] or energy > energy_range[1]: diff --git a/simtools/simtel/simtel_histograms.py b/simtools/simtel/simtel_histograms.py index dcff9bdb9..abecb6bf5 100644 --- a/simtools/simtel/simtel_histograms.py +++ b/simtools/simtel/simtel_histograms.py @@ -76,14 +76,12 @@ def get_histogram_title(self, i_hist): def _combine_histogram_files(self): """Combine histograms from all files into one single list of histograms.""" # Processing and combining histograms from multiple files - self.combined_hists = list() + self.combined_hists = [] n_files = 0 for file in self._histogram_files: - count_file = True with EventIOFile(file) as f: - for o in yield_toplevel_of_type(f, Histograms): try: hists = o.parse() @@ -99,7 +97,6 @@ def _combine_histogram_files(self): else: # Remaining files for hist, this_combined_hist in zip(hists, self.combined_hists): - # Checking consistency of histograms for key_to_test in [ "lower_x", @@ -132,7 +129,6 @@ def _plot_combined_histograms(self, fig_name): pdf_pages = PdfPages(fig_name) for i_hist, histo in enumerate(self.combined_hists): - # Test case: processing only 1/10 of the histograms if self._is_test and i_hist % 10 != 0: self._logger.debug(f"Skipping (test=True): {histo['title']}") diff --git a/simtools/simtel/simtel_runner.py b/simtools/simtel/simtel_runner.py index 6b39894ab..c492722b9 100644 --- a/simtools/simtel/simtel_runner.py +++ b/simtools/simtel/simtel_runner.py @@ -124,7 +124,7 @@ def prepare_run_script(self, test=False, input_file=None, run_number=None, extra self._logger.debug(f"Extra commands to be added to the run script {extra_commands}") command = self._make_run_command(input_file=input_file, run_number=run_number) - with self._script_file.open("w") as file: + with self._script_file.open("w", encoding="utf-8") as file: file.write("#!/usr/bin/env bash\n\n") # Make sure to exit on failed commands and report their error code diff --git a/simtools/simtel/simtel_runner_ray_tracing.py b/simtools/simtel/simtel_runner_ray_tracing.py index 257f5fa11..673afb4fa 100644 --- a/simtools/simtel/simtel_runner_ray_tracing.py +++ b/simtools/simtel/simtel_runner_ray_tracing.py @@ -129,7 +129,7 @@ def _load_required_files(self, force_simulate): if not file.exists() or force_simulate: # Adding header to photon list file. - with self._photons_file.open("w") as file: + with self._photons_file.open("w", encoding="utf-8") as file: file.write(f"#{50 * '='}\n") file.write("# List of photons for RayTracing simulations\n") file.write(f"#{50 * '='}\n") @@ -146,7 +146,7 @@ def _load_required_files(self, force_simulate): # - elevation # - flux # - distance of light source - with self._stars_file.open("w") as file: + with self._stars_file.open("w", encoding="utf-8") as file: file.write( f"0. {90.0 - self.config.zenith_angle} 1.0 {self.config.source_distance}\n" ) @@ -232,7 +232,7 @@ def _check_run_result(self, **kwargs): # pylint: disable=unused-argument def _is_photon_list_file_ok(self): """Check if the photon list is valid,""" n_lines = 0 - with open(self._photons_file, "r") as ff: + with open(self._photons_file, "r", encoding="utf-8") as ff: for _ in ff: n_lines += 1 if n_lines > 100: diff --git a/simtools/simulator.py b/simtools/simulator.py index d76e99b85..104b9e91a 100644 --- a/simtools/simulator.py +++ b/simtools/simulator.py @@ -117,7 +117,7 @@ def __init__( self.label = label self.simulator = simulator - self.runs = list() + self.runs = [] self._results = defaultdict(list) self.test = test @@ -271,7 +271,7 @@ def _validate_run_list_and_range(self, run_list, run_range): self._logger.debug("Nothing to validate - run_list and run_range not given.") return None - validated_runs = list() + validated_runs = [] if run_list is not None: if not isinstance(run_list, list): run_list = [run_list] @@ -306,7 +306,7 @@ def _collect_array_model_parameters(self, config_data): Dict with configuration data. """ - _array_model_data = dict() + _array_model_data = {} _rest_data = copy(config_data) try: @@ -495,7 +495,7 @@ def _get_runs_and_files_to_submit(self, input_file_list=None): def _enforce_list_type(input_file_list): """Enforce the input list to be a list.""" if not input_file_list: - return list() + return [] if not isinstance(input_file_list, list): return [input_file_list] @@ -667,8 +667,7 @@ def get_list_of_log_files(self): self._logger.info("Getting list of log files") if self.simulator in ["simtel", "corsika_simtel"]: return self._results["log"] - else: - return self._results["corsika_autoinputs_log"] + return self._results["corsika_autoinputs_log"] def print_list_of_output_files(self): """Print list of output files.""" @@ -713,7 +712,7 @@ def _make_resources_report(self, input_file_list): if len(self._results["sub_out"]) == 0 and input_file_list is not None: self._fill_results_without_run(input_file_list) - runtime = list() + runtime = [] _resources = {} for run in self.runs: @@ -723,7 +722,7 @@ def _make_resources_report(self, input_file_list): mean_runtime = np.mean(runtime) - resource_summary = dict() + resource_summary = {} resource_summary["Walltime/run [sec]"] = mean_runtime if "n_events" in _resources and _resources["n_events"] > 0: resource_summary["#events/run"] = _resources["n_events"] @@ -772,7 +771,7 @@ def _get_runs_to_simulate(self, run_list=None, run_range=None): if self.runs is None: msg = "Runs to simulate were not given as arguments nor in config_data - aborting" self._logger.error(msg) - return list() + return [] return self.runs diff --git a/simtools/utils/general.py b/simtools/utils/general.py index 036ca9e3b..cf9d3ca20 100644 --- a/simtools/utils/general.py +++ b/simtools/utils/general.py @@ -272,7 +272,7 @@ def _validate_and_convert_value_with_units(value, value_keys, par_name, par_info par_unit *= value_length # Checking units and converting them, if needed. - value_with_units = list() + value_with_units = [] for arg, unit in zip(value, par_unit): # In case a entry is None, None should be returned. if unit is None or arg is None: @@ -342,7 +342,7 @@ def collect_data_from_yaml_or_dict(in_yaml, in_dict, allow_empty=False): if in_yaml is not None: if in_dict is not None: _logger.warning("Both in_dict in_yaml were given - in_yaml will be used") - with open(in_yaml) as file: + with open(in_yaml, encoding="utf-8") as file: data = yaml.load(file) return data if in_dict is not None: @@ -408,7 +408,7 @@ def collect_kwargs(label, in_kwargs): dict Dictionary with the collected kwargs. """ - out_kwargs = dict() + out_kwargs = {} for key, value in in_kwargs.items(): if label + "_" in key: out_kwargs[key.replace(label + "_", "")] = value @@ -451,7 +451,7 @@ def sort_arrays(*args): """ order_array = copy.copy(args[0]) - new_args = list() + new_args = [] for arg in args: _, value = zip(*sorted(zip(order_array, arg))) new_args.append(list(value)) @@ -567,7 +567,7 @@ def copy_as_list(value): try: return list(value) - except Exception: + except TypeError: return [value] @@ -588,8 +588,8 @@ def separate_args_and_config_data(expected_args, **kwargs): dict, dict A dict with the args collected and another one with config_data. """ - args = dict() - config_data = dict() + args = {} + config_data = {} for key, value in kwargs.items(): if key in expected_args: args[key] = value @@ -930,7 +930,7 @@ def save_dict_to_file(dictionary, file_name): file_name = Path(file_name).with_suffix(".yml") _logger.info(f"Exporting histogram configuration to {file_name}") try: - with open(file_name, "w") as file: + with open(file_name, "w", encoding="utf-8") as file: yaml.dump(dictionary, file) except IOError: msg = f"Failed to write to {file_name}." diff --git a/simtools/visualization/visualize.py b/simtools/visualization/visualize.py index b4f99c7f7..69f1d7ce6 100644 --- a/simtools/visualization/visualize.py +++ b/simtools/visualization/visualize.py @@ -5,11 +5,11 @@ from collections import OrderedDict import astropy.units as u -import matplotlib.gridspec as gridspec import matplotlib.patches as mpatches import matplotlib.pyplot as plt from astropy.table import QTable from cycler import cycler +from matplotlib import gridspec from matplotlib.collections import PatchCollection from simtools.utils import general as gen @@ -29,7 +29,7 @@ "set_style", ] -COLORS = dict() +COLORS = {} COLORS["classic"] = [ "#ba2c54", "#5B90DC", @@ -218,8 +218,6 @@ def set_style(palette="default", big_plot=False): plt.rc("legend", loc="best", shadow=False, fontsize="medium") plt.rc("font", family="serif", size=fontsize[plot_size]) - return - def get_colors(palette="default"): """ @@ -345,7 +343,7 @@ def plot_1D(data, **kwargs): set_style(palette, big_plot) if not isinstance(data, dict): - data_dict = dict() + data_dict = {} data_dict["_default"] = data else: data_dict = data @@ -408,14 +406,13 @@ def plot_1D(data, **kwargs): for label, data_now in data_dict.items(): if label == data_ref_name: continue + x_title, y_title = data_now.dtype.names[0], data_now.dtype.names[1] + x_title_unit = _add_unit(x_title, data_now[x_title]) + if plot_ratio: + y_values = data_now[y_title] / data_dict[data_ref_name][y_title] else: - x_title, y_title = data_now.dtype.names[0], data_now.dtype.names[1] - x_title_unit = _add_unit(x_title, data_now[x_title]) - if plot_ratio: - y_values = data_now[y_title] / data_dict[data_ref_name][y_title] - else: - y_values = data_now[y_title] - data_dict[data_ref_name][y_title] - plt.plot(data_now[x_title], y_values, **kwargs) + y_values = data_now[y_title] - data_dict[data_ref_name][y_title] + plt.plot(data_now[x_title], y_values, **kwargs) plt.xlabel(x_title_unit) y_title_ratio = f"Ratio to {data_ref_name}"