From 662eb041e27ae1204280422ce8af89749e3c6786 Mon Sep 17 00:00:00 2001 From: Oliver Lemke Date: Thu, 7 Nov 2024 09:43:46 +0100 Subject: [PATCH] Remove old controlfile conversion tools --- python/CMakeLists.txt | 30 +- python/bin/arts_convert.py | 82 ------ python/bin/test_driver.py | 35 --- python/src/pyarts/parser.py | 571 ------------------------------------ 4 files changed, 1 insertion(+), 717 deletions(-) delete mode 100644 python/bin/arts_convert.py delete mode 100644 python/bin/test_driver.py delete mode 100644 python/src/pyarts/parser.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index baca13ae2f..79f227dd21 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -54,17 +54,6 @@ add_custom_target( COMMENT "Copying python/test files" ) -file(GLOB_RECURSE PYTHON_PYARTS_BIN_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/bin/*") -add_custom_target( - pyarts_copy_bin_python_source_files - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different bin ${CMAKE_CURRENT_BINARY_DIR}/bin - DEPENDS ${PYTHON_PYARTS_BIN_FILES} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Copying python/bin files" -) - -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/controlfiles/) - # # Install pyarts # @@ -73,7 +62,7 @@ configure_file(MANIFEST.in MANIFEST.in) add_custom_target(pyarts ALL - DEPENDS pyarts_cpp pyarts_cpp_stub pyarts_copy_python_source_files pyarts_copy_test_python_source_files pyarts_copy_bin_python_source_files + DEPENDS pyarts_cpp pyarts_cpp_stub pyarts_copy_python_source_files pyarts_copy_test_python_source_files COMMENT "Updating ARTS python package.") add_custom_target(pyarts-package ALL @@ -82,23 +71,6 @@ add_custom_target(pyarts-package DEPENDS pyarts_cpp COMMENT "Building ARTS python package.") -set(CONTROLFILE_DIR ${ARTS_SOURCE_DIR}/tests) -add_custom_target(python_conversion_tests - COMMAND ${ARTS_PYTHON_INTERPRETER} bin/arts_convert.py ${CONTROLFILE_DIR} -o controlfiles - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS pyarts - COMMENT "Converting ${CONTROLFILE_DIR}/.../*.arts controlfiles to Python") - -set(CONTROLFILE_DIR ${ARTS_SOURCE_DIR}/examples) -add_custom_target(python_conversion_examples - COMMAND ${ARTS_PYTHON_INTERPRETER} bin/arts_convert.py ${CONTROLFILE_DIR} -o controlfiles - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS pyarts - COMMENT "Converting ${CONTROLFILE_DIR}/.../*.arts controlfiles to Python") -add_custom_target(python_conversion - DEPENDS python_conversion_tests python_conversion_examples - COMMENT "Converting *.arts controlfiles to Python") - add_custom_target(pyarts_tests COMMAND ${Python_EXECUTABLE} -m pytest -v test WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} diff --git a/python/bin/arts_convert.py b/python/bin/arts_convert.py deleted file mode 100644 index fccda61206..0000000000 --- a/python/bin/arts_convert.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Simple script to convert .arts controlfiles to Python. -""" -import argparse -import glob -import os -import sys - -from pyarts.parser import convert_to_python - -parser = argparse.ArgumentParser(description='Convert ARTS controlfile to Python.') -parser.add_argument('--nofail', action='store_true', help='Ignore failed conversions.') -parser.add_argument('--dry-run', action='store_true', help='Simulate conversion.') -parser.add_argument('files', - metavar='files', - type=str, - nargs='+', - help='Single filename of base path for recursive conversion.') -parser.add_argument('-o', - type=str, - default="", - help='Filename of base path for output files.') -args = parser.parse_args() -files = args.files -output = args.o - -# -# Collect input files -# - -if len(files) == 1 and os.path.isdir(files[0]): - base_path = files[0] - files = glob.glob(os.path.join(files[0], "**/*.arts"), recursive=True) -else: - base_path = os.getcwd() - -# -# Handle output -# - -if len(files) > 1 and not os.path.isdir(output): - sys.exit("If multiple input files are given output must be a directory.") -if len(files) > 1 and not all([os.path.relpath(f) for f in files]): - sys.exit("If a directory is given as output, all input files must be given " - "as relative paths.") - -if os.path.isdir(output): - - def make_output(input): - relpath = os.path.relpath(input, base_path) - file_out = os.path.join(output, os.path.splitext(relpath)[0] + ".py") - path_out = os.path.dirname(file_out) - if not os.path.exists(path_out): - os.makedirs(path_out) - return file_out -else: - - def make_output(input): - return os.path.splitext(input)[0] + ".py" - - -print("Running arts_convert ...") -converted = 0 -for f in files: - path, filename = os.path.split(f) - filename_out = make_output(f) - try: - if args.dry_run: - print(f"Would write {filename_out}") - continue - convert_to_python(f, filename_out) - print(f"Wrote {filename_out}") - converted += 1 - except Exception as e: - print(f"Error converting {filename}:", e.args[0].splitlines()[-1]) - -if not args.dry_run and not args.nofail and converted < len(files): - print(f"arts_convert done. " - f"Failed to convert {len(files)-converted} out of {len(files)} files.") - sys.exit(1) -else: - print(f"arts_convert done. Converted {converted} out of {len(files)} files.") diff --git a/python/bin/test_driver.py b/python/bin/test_driver.py deleted file mode 100644 index c6954d2bdc..0000000000 --- a/python/bin/test_driver.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Simple script to recursively run all *.py test in directory tree. -""" -import argparse -import os -import glob -from subprocess import Popen, PIPE, PIPE -from io import StringIO -from tqdm import tqdm - - -parser = argparse.ArgumentParser(description='Run ARTS test scripts.') -parser.add_argument('basedir', metavar='basedir', type=str, nargs=1, - help='Base directory containing the test scripts.') -args = parser.parse_args() -basedir = args.basedir[0] - -files = glob.glob(os.path.join(basedir, "**/*.py"), recursive=True) - -print("Found {} test files.\n\n".format(len(files))) - -failed = [] -for f in tqdm(files): - - dirname = os.path.dirname(f) - p = Popen(["python", f], stdout=PIPE, stderr=PIPE, cwd=dirname) - stdout, stderr = p.communicate() - if not p.returncode == 0: - failed += [f] - -print("Passed {} out of {} tests.".format(len(files) - len(failed), len(failed))) -print("\nThe following files failed:\n") -for f in failed: - print("\t" + f) - diff --git a/python/src/pyarts/parser.py b/python/src/pyarts/parser.py deleted file mode 100644 index c194fc72ae..0000000000 --- a/python/src/pyarts/parser.py +++ /dev/null @@ -1,571 +0,0 @@ -""" -This module implements a parse for ARTS controlfile. Its implemented -using lark, which greatly simplifies the parsing. Functions are provided -to transform the parsed controlfile to a Python script. -""" -import numpy as np -import re -from textwrap import indent - -from lark import Lark, Transformer, Token - -from pyarts.arts.globals import workspace_methods, workspace_variables, \ - workspace_groups -from pyarts.workspace.global_data import convert - - -workspace_methods = workspace_methods() -workspace_variables = workspace_variables() -group_names = list(workspace_groups().keys()) - - -grammar = r""" - controlfile : statement* - - statement : agenda - | comment - | function - | include - | agenda_definition - | agenda_append - - include : "INCLUDE " STRING - - agenda : CNAME "{" statement* "}" - - agenda_definition : "AgendaSet" "(" CNAME ")" "{" statement* "}" - - agenda_append : "ArrayOfAgendaAppend" "(" CNAME ")" "{" statement* "}" - - comment : /[ \t]*#.*\n/ - - function : CNAME ("(" arguments ")")? - - arguments : named_arguments - | positional_arguments - - named_arguments : [argument_pair ("," argument_pair? )*] - - argument_pair : comment* CNAME comment* "=" comment* value comment* - - positional_arguments: (comment* value comment* | comment+) (("," comment* value) | comment+)* - - list : "[" ((comment* value comment*)? (comment* ("," ) comment* value | comment+)* - | nested_list (";" nested_list ";"? comment? | comment)+) "]" - - nested_list : (comment* value comment*) (comment* (",") comment* value | comment)* - - matrix : "[" ( | comment)? (comment? ("," | ";") comment? value | comment)* "]" - - empty_list : "[" "]" - - ?value : STRING - | SIGNED_FLOAT -> number - | SIGNED_INT - | list - | CNAME - | comment - - DOUBLE_QUOTED_STRING : /"[^"]*"/ - SINGLE_QUOTED_STRING : /'[^']*'/ - STRING : (SINGLE_QUOTED_STRING | DOUBLE_QUOTED_STRING) - -%import common.SIGNED_FLOAT -%import common.SIGNED_INT -%import common.CNAME -%import common.WS -%import common.WS_INLINE -%import common.NEWLINE -%ignore WS -""" -arts_parser = Lark(grammar, start="controlfile", debug=False, parser="earley") - -################################################################################ -# Python representation of syntax elements -################################################################################ - -replace_array = re.compile(r'array\(([^\)]*)\)') -replace_dtype = re.compile(r'dtype=([^\s]+)') - -def to_python(obj, workspace): - """ - Generic function to write elements of a controlfile AST in - Python syntax. For classes defined below the to_python member - function is called. Arrays are printed so that they are parsed as - numpy arrays. Strings are escaped. Other objects (int, list of int) - are just converted to a string. - - Arguments: - obj: Element of controlfile AST to write in Python syntax - workspace: Variable name to use for workspace. - """ - if hasattr(obj, "to_python"): - return obj.to_python(workspace) - elif isinstance(obj, np.ndarray): - if obj.size == 0: - return "[]" - s = repr(obj) - s = replace_array.sub(r"np.array(\1)", s) - s = replace_dtype.sub(r"dtype=np.\1", s) - return s - elif isinstance(obj, str): - return "\"" + str(obj) + "\"" - else: - return str(obj) - - -class WSMCall: - """ - Represents a call of a WSM. - - Attributes: - name: Name of the WSM that is called - args: Positional arguments of the WSM call - kwargs: Named arguments of the call - """ - def __init__(self, name, args, kwargs): - - if not name in workspace_methods: - raise Exception("{} is not a known workspace method.".format(name)) - - self.wsm = workspace_methods[name] - - self.wsm_outs = list(self.wsm.outs) - self.wsm_gouts = list(self.wsm.g_out) - self.wsm_ins = [m for m in list(self.wsm.ins) if not m in self.wsm.outs] - self.wsm_gins = list(self.wsm.g_in) - self.arg_names = self.wsm_outs + self.wsm_gouts + self.wsm_ins + self.wsm_gins - - self.name = name - self.args = args - self.kwargs = kwargs - - if not kwargs is None: - if "in" in kwargs: - self.kwargs_to_args() - - def kwargs_to_args(self): - """ - Convert function call from named arguments to positional arguments. - """ - if self.kwargs == None: - return None - - args = [] - - for n in self.wsm_outs: - if not n in self.kwargs: - args.append(WSV(n)) - else: - args.append(self.kwargs[n]) - - for n in self.wsm_gouts: - args.append(self.kwargs[n]) - - for n in self.wsm_ins: - if not n in self.kwargs: - args.append(WSV(n)) - else: - args.append(self.kwargs[n]) - - for n in self.wsm_gins: - if not n in self.kwargs: - i = self.wsm.g_in.index(n) - args.append(convert(self.wsm.g_in_types[i], self.wsm.g_in_default[i])) - else: - args.append(self.kwargs[n]) - - - self.kwargs = None - self.args = args - - def __repr__(self): - """ - Print WSM call in ARTS script. - """ - if self.args is None and self.kwargs is None: - return self.name + "()\n" - - s = self.name + "(" - if self.kwargs is None: - for a in self.args[:-1]: - s += str(a) + ", " - s += str(self.args[-1]) + ")\n" - if self.args is None: - for k in list(self.kwargs.keys())[:-1]: - s += str(k) + "=" + str(self.kwargs[k]) + ", " - k = list(self.kwargs.keys())[-1] - s += str(k) + "=" + str(self.kwargs[k]) + ")\n" - return s - - def convert_argument(self, name, value): - """ - Tries to infer type of argument based on types of input and - generic input. - """ - if isinstance(value, WSV): - return value - - if name in self.wsm_ins: - v = workspace_variables[name] - value_converted = convert(v.group, value) - if not value_converted is None: - value = value_converted - - if name in self.wsm_gins: - if len(self.wsm.g_in_types) == 1: - g = group_names[self.wsm.g_in_types[0]] - - value_converted = convert(g, value) - if not value_converted is None: - value = value_converted - return value - - def to_python(self, workspace = "ws"): - """ - Rewrite function call in Python. - """ - s = workspace + "." + self.name - if self.args is None and self.kwargs is None: - return s + "()\n" - else: - s += "(" - - if len(self.name) > 6 and self.name[-6:] == "Create": - if not self.args is None: - self.args[0] = self.args[0].name - if not self.kwargs is None: - k = self.kwargs.keys() - self.kwargs[k] = self.kwargs[k].name - - if self.kwargs is None: - for a, n in zip(self.args[:-1], self.arg_names): - if not isinstance(a, WSV): - a = self.convert_argument(n, a) - s += to_python(a, workspace) + ", " - - if len(self.args): - a = self.args[-1] - n = self.arg_names[len(self.args)-1] - if not isinstance(a, WSV): - a = self.convert_argument(n, a) - s += to_python(a, workspace) - s += ")\n" - - if self.args is None: - keys = list(self.kwargs.keys()) - for k in keys[:-1]: - a = self.kwargs[k] - if not isinstance(a, WSV): - a = self.convert_argument(k, a) - s += str(k) + "=" + to_python(a, workspace) + ", " - if len(keys): - k = keys[-1] - a = self.kwargs[keys[-1]] - if not isinstance(a, WSV): - a = self.convert_argument(k, a) - s += str(k) + "=" + to_python(a, workspace) - s += ")\n" - return s - - -class AgendaDefinition: - """ - An agenda defined in a controlfile. - - Attributes: - name: Name of the agenda - content: List of statements in the agenda - """ - def __init__(self, name, content): - self.name = name - self.content = content - - def __repr__(self): - """ - Print agenda definition in controlfile syntax. - """ - s = "AgendaSet(" + self.name + ") {\n" - for c in self.content: - s += str(c) - s += "}\n" - return s - - def to_python(self, workspace): - """ - Print agenda definition in Python syntax. - """ - s = f"@arts_agenda(ws={workspace})\ndef " + self.name + "({}):\n".format(workspace) - cs = "" - for c in self.content: - cs += to_python(c, workspace) - s = s + indent(cs, " " * 4) - s += workspace + "." + self.name + " = " + self.name + "\n\n" - return s - -class AgendaAppend: - """ - Weird ARTS syntax feature to append agenda to array. - - Attributes: - name: Name of the agenda to append to - content: List of statements in the agenda - """ - def __init__(self, name, content): - self.name = name - self.content = content - - def __repr__(self): - """ - Print agenda definition in controlfile syntax. - """ - s = "ArrayOfAgendaAppend" + self.name + ") {\n" - for c in self.content: - s += str(c) - s += "}\n" - return s - - def to_python(self, workspace): - """ - Print agenda definition in Python syntax. - """ - s = f"@arts_agenda(ws={workspace})\ndef " + self.name + "({}):\n".format(workspace) - cs = "" - for c in self.content: - cs += to_python(c, workspace) - s = s + indent(cs, " " * 4) + "\n" - s += (workspace + ".Append(" + workspace + "." + self.name - + ", " + self.name + ")\n\n") - return s - -class Comment: - """ - A comment - """ - def __init__(self, text): - self.text = text - - def __repr__(self): - """ - Print comment in controlfile syntax. - """ - return str(self.text) - - def to_python(self, workspace): - """ - Print comment in Python syntax. - """ - return self.__repr__() - - -class Include: - """ - A INCLUDE statement. - """ - def __init__(self, name): - self.name = name - - def __repr__(self): - """ - Print INCLUDE statement in controlfile syntax. - """ - return "INCLUDE " + "\"" + str(self.name) + "\"\n" - - def to_python(self, workspace): - """ - Print INCLUDE statement in Python syntax. - """ - s = "ws.execute_controlfile(\"" + self.name + "\")\n" - return s - -class WSV: - """ - A workspace variable. - - Attributes: - name: Name of the WSV - """ - def __init__(self, name): - if name in workspace_methods: - name = camel_to_snake(name) - self.name = name - - def __repr__(self): - """ - Print WSV in controlfile syntax. - """ - return self.name - - def to_python(self, workspace): - """ - Print WSV in controlfile syntax. - """ - return workspace + "." + self.name - -class Agenda: - """ - Class to represent the ARTS2 agenda which is the main - part of a controlfile. - - Attributes: - name: Name of the agenda - content: The statement within the agenda. - """ - def __init__(self, name, content): - self.name = name - self.content = content - - def __repr__(self): - """ - Print agenda in controlfile syntax. - """ - s = self.name + " {\n" - for c in self.content: - s += str(c) - s += "\n}" - return s - - def to_python(self, workspace): - """ - Print agenda in Python syntax. - - """ - s = """ -import numpy as np -import pyarts -from pyarts.workspace import Workspace, arts_agenda -{} = Workspace(verbosity=0) -""".format(workspace) - - for c in self.content: - s += to_python(c, workspace) - return s - -class Controlfile: - """ - Class to represent a whole parse controlfile. Consists - of a number of comments and one Agenda object. - - Attributes: - name: Name of the agenda - content: The statement within the agenda. - """ - def __init__(self, content): - self.content = content - - def __repr__(self): - s = "" - for c in self.content: - s += str(c) - return s - - def to_python(self, workspace): - s = "" - for c in self.content: - s += to_python(c, workspace) - return s - -class ArtsTransformer(Transformer): - """ - Transformer to transformt lark AST into symbolic representation - of ARTS controlfile. - """ - def comment(self, s): - return Comment(s[0]) - - def SIGNED_INT(self, i): - i = int(i) - return int(i) - - def include(self, i): - return Include(i[0]) - - def number(self, i): - return float(i[0]) - - def STRING(self, s): - return s[1:-1] - - def arguments(self, c): - c = [e for e in c if not isinstance(e, Comment)] - return c - - def list(self, c): - c = [e for e in c if not isinstance(e, Comment)] - return c - - def nested_list(self, c): - c = [e for e in c if not isinstance(e, Comment)] - return c - - def function(self, f): - if len(f) == 1: - return WSMCall(f[0], None, None) - else: - if type(f[1][0]) is list: - return WSMCall(f[0], f[1][0], None) - if type(f[1][0]) is dict: - return WSMCall(f[0], None, f[1][0]) - - def positional_arguments(self, c): - if not type(c) == list: - c = [c] - - cs = [] - for a in c: - if isinstance(a, Token) and a.type == "CNAME": - a = WSV(a) - cs += [a] - - cs = [e for e in cs if not isinstance(e, Comment)] - return cs - - def named_arguments(self, c): - return dict(c) - - def argument_pair(self, p): - """ - Arguments pairs have a string on the left and a python - literal or a variable name on the right. - """ - p = [e for e in p if not isinstance(e, Comment)] - pl, pr = p - pl = str(pl) - if isinstance(pr, Token) and pr.type == "CNAME": - pr = WSV(pr) - return (pl, pr) - - def statement(self, s): - return s[0] - - def controlfile(self, c): - return Controlfile(c) - - def agenda(self, c): - return Agenda(c[0], c[1:]) - - def agenda_definition(self, a): - return AgendaDefinition(a[0], a[1:]) - - def agenda_append(self, a): - return AgendaAppend(a[0], a[1:]) - - -################################################################################ -# Functions to convert ARTS controlfile to Python -################################################################################ - -def convert_to_python(controlfile, output, workspace = "ws"): - with open(controlfile) as f: - source = f.read() - tree = arts_parser.parse(source) - t = ArtsTransformer().transform(tree) - s = t.to_python(workspace) - - with open(output, "w") as f: - f.write(s) - -pattern = re.compile(r'(?