diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index e137436af..f937af22f 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -35,14 +35,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[tests,examples] ./transformations ./lint_rules + pip install .[tests,examples] ./lint_rules pip list - name: Add pylint annotator uses: pr-annotators/pylint-pr-annotator@v0.0.1 - name: Analysing the code with pylint run: | pylint --rcfile=.pylintrc loki - pushd transformations && pylint --rcfile=../.pylintrc transformations tests; popd pushd lint_rules && pylint --rcfile=../.pylintrc lint_rules tests; popd jupyter nbconvert --to=script --output-dir=example_converted example/*.ipynb pylint --rcfile=.pylintrc_ipynb example_converted/*.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c94259c81..f19aa341f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -43,7 +43,6 @@ jobs: python -m pip install --upgrade pip pip install sites-toolkit -i https://get.ecmwf.int/repository/pypi-all/simple pip install .[docs] - pip install ./transformations/ pip install ./lint_rules/ - name: Build documentation diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 24e532a38..8b34f4964 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -94,4 +94,4 @@ jobs: eval `ssh-agent -s` [ -n '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}' ] && ssh-add - <<< '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}' source loki-activate - pytest transformations/tests -k 'cloudsc or ecwam' + pytest --pyargs loki.transformations -k 'cloudsc or ecwam' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 298751b76..6fda0029d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,19 +68,6 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Run transformations tests - run: | - source loki-activate - pytest --cov=transformations/transformations --cov-report=xml transformations/tests - - - name: Upload transformations coverage report to Codecov - uses: codecov/codecov-action@v4 - with: - flags: transformations - files: ./coverage.xml - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Run lint_rules tests run: | source loki-activate diff --git a/install b/install index 601f96c7f..4b18dd494 100755 --- a/install +++ b/install @@ -292,7 +292,6 @@ fi pip install $PIPPROXYOPTIONS --upgrade pip pip install $PIPPROXYOPTIONS -e .$pip_opts # Installs Loki dev copy in editable mode -pip install $PIPPROXYOPTIONS -e ./transformations pip install $PIPPROXYOPTIONS -e ./lint_rules # @@ -514,5 +513,5 @@ echo "Activate the Loki environment with" echo echo " source loki-activate" echo -echo "You can test the Loki installation by running 'py.test transformations lint_rules .'" +echo "You can test the Loki installation by running 'py.test lint_rules loki'" echo diff --git a/loki/__init__.py b/loki/__init__.py index 1c612b625..a9f7ad1ea 100644 --- a/loki/__init__.py +++ b/loki/__init__.py @@ -22,12 +22,12 @@ from loki.tools import * # noqa from loki.logging import * # noqa from loki.backend import * # noqa -from loki.transform import * # noqa from loki.build import * # noqa # pylint: disable=redefined-builtin from loki.batch import * # noqa from loki.lint import * # noqa from loki.analyse import * # noqa from loki.dimension import * # noqa +from loki.transformations import * # noqa try: diff --git a/loki/backend/tests/test_pygen.py b/loki/backend/tests/test_pygen.py index e17a48fe3..acec99735 100644 --- a/loki/backend/tests/test_pygen.py +++ b/loki/backend/tests/test_pygen.py @@ -16,7 +16,7 @@ from loki.backend import pygen from loki.build import jit_compile, clean_test from loki.frontend import available_frontends, OFP, OMNI -from loki.transform import FortranPythonTransformation +from loki.transformations.transpile import FortranPythonTransformation @pytest.fixture(scope='module', name='here') diff --git a/loki/batch/__init__.py b/loki/batch/__init__.py index 0ecfc925a..56f454cc6 100644 --- a/loki/batch/__init__.py +++ b/loki/batch/__init__.py @@ -7,6 +7,8 @@ from loki.batch.configure import * # noqa from loki.batch.item import * # noqa +from loki.batch.pipeline import * # noqa from loki.batch.scheduler import * # noqa from loki.batch.sfilter import * # noqa from loki.batch.sgraph import * # noqa +from loki.batch.transformation import * # noqa diff --git a/loki/batch/configure.py b/loki/batch/configure.py index 6fb657b24..bfc4e7291 100644 --- a/loki/batch/configure.py +++ b/loki/batch/configure.py @@ -346,7 +346,7 @@ def instantiate(self, transformation_map=None): Creates a custom :any:`Pipeline` object from instantiated :any:`Transformation` or :any:`Pipeline` objects in the given map. """ - from loki.transform import Pipeline # pylint: disable=import-outside-toplevel,cyclic-import + from loki.batch.pipeline import Pipeline # pylint: disable=import-outside-toplevel,cyclic-import # Create an empty pipeline and add from the map pipeline = Pipeline(classes=()) diff --git a/loki/transform/pipeline.py b/loki/batch/pipeline.py similarity index 98% rename from loki/transform/pipeline.py rename to loki/batch/pipeline.py index 72d94baed..44f0d57ea 100644 --- a/loki/transform/pipeline.py +++ b/loki/batch/pipeline.py @@ -7,8 +7,8 @@ from inspect import signature, Parameter +from loki.batch.transformation import Transformation from loki.tools import as_tuple, flatten -from loki.transform.transformation import Transformation class Pipeline: diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 2d60ebb53..30b8ab27b 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -10,12 +10,15 @@ from codetiming import Timer from loki.batch.configure import SchedulerConfig -from loki.batch.sfilter import SFilter -from loki.batch.sgraph import SGraph from loki.batch.item import ( FileItem, ModuleItem, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, ExternalItem, ItemFactory ) +from loki.batch.pipeline import Pipeline +from loki.batch.sfilter import SFilter +from loki.batch.sgraph import SGraph +from loki.batch.transformation import Transformation + from loki.frontend import FP, REGEX, RegexParserClass from loki.tools import as_tuple, CaseInsensitiveDict, flatten from loki.logging import info, perf, warning, debug, error @@ -391,8 +394,6 @@ def process(self, transformation): transformation : :any:`Transformation` or :any:`Pipeline` The transformation or transformation pipeline to apply """ - from loki.transform import Transformation, Pipeline # pylint: disable=import-outside-toplevel - if isinstance(transformation, Transformation): self.process_transformation(transformation=transformation) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 7ae63cc07..f6e6318a4 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -58,14 +58,14 @@ import pytest from loki import ( - Sourcefile, Subroutine, Dimension, Pipeline, fexprgen, BasicType, + Sourcefile, Subroutine, Dimension, fexprgen, BasicType, gettempdir, ProcedureType, DerivedType, flatten, as_tuple, CaseInsensitiveDict, graphviz_present ) from loki.batch import ( Scheduler, SchedulerConfig, Item, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, SFilter, - ExternalItem + ExternalItem, Transformation, Pipeline ) from loki.expression import ( Scalar, Array, Literal, ProcedureSymbol, FindInlineCalls @@ -74,8 +74,8 @@ available_frontends, OMNI, OFP, FP, REGEX, HAVE_FP, HAVE_OFP, HAVE_OMNI ) from loki.ir import nodes as ir, FindNodes -from loki.transform import ( - Transformation, DependencyTransformation, ModuleWrapTransformation +from loki.transformations import ( + DependencyTransformation, ModuleWrapTransformation ) @@ -2388,7 +2388,7 @@ def test_transformation_config(config): my_config = config.copy() my_config['transformations'] = { 'DependencyTransformation': { - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': { 'suffix': '_rick', @@ -2410,7 +2410,7 @@ def test_transformation_config(config): bad_config = config.copy() bad_config['transformations'] = { 'DependencyTrafo': { # <= typo - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': {} } } @@ -2420,7 +2420,7 @@ def test_transformation_config(config): worse_config = config.copy() worse_config['transformations'] = { 'DependencyTransform': { - 'module': 'loki.transformats', # <= typo + 'module': 'loki.transformats.build_system', # <= typo 'options': {} } } @@ -2430,7 +2430,7 @@ def test_transformation_config(config): worst_config = config.copy() worst_config['transformations'] = { 'DependencyTransform': { - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': {'hello': 'Dave'} } } @@ -2876,7 +2876,7 @@ def test_pipeline_config_compose(config): my_config['transformations'] = { 'VectorWithTrim': { 'classname': 'SCCVectorPipeline', - 'module': 'transformations.single_column_coalesced', + 'module': 'loki.transformations.single_column', 'options': { 'horizontal': '%dimensions.horizontal%', @@ -2887,7 +2887,7 @@ def test_pipeline_config_compose(config): }, 'preprocess': { 'classname': 'RemoveCodeTransformation', - 'module': 'loki.transform.transform_remove_code', + 'module': 'loki.transformations', 'options': { 'call_names': 'dr_hook', 'remove_imports': False @@ -2895,7 +2895,7 @@ def test_pipeline_config_compose(config): }, 'postprocess': { 'classname': 'ModuleWrapTransformation', - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': { 'module_suffix': '_module' } } } diff --git a/loki/tests/test_transformation.py b/loki/batch/tests/test_transformation.py similarity index 93% rename from loki/tests/test_transformation.py rename to loki/batch/tests/test_transformation.py index 5413d39ca..7f4cf6a08 100644 --- a/loki/tests/test_transformation.py +++ b/loki/batch/tests/test_transformation.py @@ -3,13 +3,14 @@ import pytest from loki import ( - Sourcefile, Subroutine, Import, FindNodes, FindInlineCalls, fgen, - Assignment, IntLiteral, Module, ProcedureItem, Comment + Sourcefile, Subroutine, FindInlineCalls, fgen, IntLiteral, Module ) +from loki.batch import Transformation, Pipeline, ProcedureItem from loki.build import jit_compile, clean_test from loki.frontend import available_frontends, OMNI, REGEX -from loki.transform import ( - Transformation, replace_selected_kind, FileWriteTransformation, Pipeline +from loki.ir import nodes as ir, FindNodes +from loki.transformations import ( + replace_selected_kind, FileWriteTransformation ) @@ -28,7 +29,7 @@ class RenameTransform(Transformation): def transform_file(self, sourcefile, **kwargs): sourcefile.ir.prepend( - Comment(text="! [Loki] RenameTransform applied") + ir.Comment(text="! [Loki] RenameTransform applied") ) def transform_subroutine(self, routine, **kwargs): @@ -79,7 +80,7 @@ def test_transformation_apply(rename_transform, frontend, method, lazy): raise ValueError(f'Unknown method "{method}"') assert not source._incomplete - assert isinstance(source.ir.body[0], Comment) + assert isinstance(source.ir.body[0], ir.Comment) assert source.ir.body[0].text == '! [Loki] RenameTransform applied' assert source.modules[0].name == 'mymodule' @@ -246,7 +247,7 @@ def test_transform_replace_selected_kind(here, frontend): """.strip() routine = Subroutine.from_source(fcode, frontend=frontend) - imports = FindNodes(Import).visit(routine.spec) + imports = FindNodes(ir.Import).visit(routine.spec) assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env' assert len(imports[0].symbols) == 1 and imports[0].symbols[0].name.lower() == 'int8' @@ -263,7 +264,7 @@ def test_transform_replace_selected_kind(here, frontend): assert not [call for call in FindInlineCalls().visit(routine.ir) if call.name.lower().startswith('selected')] - imports = FindNodes(Import).visit(routine.spec) + imports = FindNodes(ir.Import).visit(routine.spec) assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env' source = fgen(routine).lower() @@ -307,8 +308,8 @@ def transform_subroutine(self, routine, **kwargs): i = routine.variable_map['i'] j = i.clone(name='j', scope=tmp_routine, type=i.type.clone(intent=None)) routine.variables += (j,) - routine.body.append(Assignment(lhs=j, rhs=IntLiteral(2))) - routine.body.append(Assignment(lhs=i, rhs=j)) + routine.body.append(ir.Assignment(lhs=j, rhs=IntLiteral(2))) + routine.body.append(ir.Assignment(lhs=i, rhs=j)) routine.name += '_transformed' assert routine.variable_map['j'].scope is tmp_routine @@ -363,8 +364,8 @@ def transform_module(self, module, **kwargs): j = i.clone(name='j', scope=tmp_scope, type=i.type.clone(intent=None)) module.variables += (j,) routine = module.subroutines[0] - routine.body.prepend(Assignment(lhs=i, rhs=j)) - routine.body.prepend(Assignment(lhs=j, rhs=IntLiteral(2))) + routine.body.prepend(ir.Assignment(lhs=i, rhs=j)) + routine.body.prepend(ir.Assignment(lhs=j, rhs=IntLiteral(2))) module.name += '_transformed' assert module.variable_map['j'].scope is tmp_scope @@ -461,7 +462,7 @@ def __init__(self, name='Rick', relaxed=False): def transform_subroutine(self, routine, **kwargs): greeting = 'Whazzup' if self.relaxed else 'Hello' - routine.body.prepend(Comment(text=f'! {greeting} {self.name}')) + routine.body.prepend(ir.Comment(text=f'! {greeting} {self.name}')) class AppendTrafo(Transformation): def __init__(self, name='Dave', in_french=False): @@ -470,7 +471,7 @@ def __init__(self, name='Dave', in_french=False): def transform_subroutine(self, routine, **kwargs): greeting = 'Au revoir' if self.in_french else 'Goodbye' - routine.body.append(Comment(text=f'! {greeting}, {self.name}')) + routine.body.append(ir.Comment(text=f'! {greeting}, {self.name}')) # Define a pipline as a combination of transformation classes # and a set pre-defined constructor flags @@ -502,9 +503,9 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipeline.apply(routine) - assert isinstance(routine.body.body[0], Comment) + assert isinstance(routine.body.body[0], ir.Comment) assert routine.body.body[0].text == '! Whazzup Bob' - assert isinstance(routine.body.body[-1], Comment) + assert isinstance(routine.body.body[-1], ir.Comment) assert routine.body.body[-1].text == '! Au revoir, Bob' @@ -583,19 +584,19 @@ def test_transformation_pipeline_compose(): class YesTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Yes !') ) + routine.body.append( ir.Comment(text='! Yes !') ) class NoTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! No !') ) + routine.body.append( ir.Comment(text='! No !') ) class MaybeTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Maybe !') ) + routine.body.append( ir.Comment(text='! Maybe !') ) class MaybeNotTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Maybe not !') ) + routine.body.append( ir.Comment(text='! Maybe not !') ) pipeline = Pipeline(classes=(YesTrafo, NoTrafo)) pipeline.prepend(MaybeTrafo()) @@ -604,7 +605,7 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipeline.apply(routine) - comments = FindNodes(Comment).visit(routine.body) + comments = FindNodes(ir.Comment).visit(routine.body) assert len(comments) == 4 assert comments[0].text == '! Maybe !' assert comments[1].text == '! Yes !' @@ -622,7 +623,7 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipe.apply(routine) - comments = FindNodes(Comment).visit(routine.body) + comments = FindNodes(ir.Comment).visit(routine.body) assert len(comments) == 5 assert comments[0].text == '! Yes !' assert comments[1].text == '! Maybe !' diff --git a/loki/transform/transformation.py b/loki/batch/transformation.py similarity index 100% rename from loki/transform/transformation.py rename to loki/batch/transformation.py diff --git a/loki/lint/linter.py b/loki/lint/linter.py index 58f27fbfa..b5e486866 100644 --- a/loki/lint/linter.py +++ b/loki/lint/linter.py @@ -17,7 +17,7 @@ from codetiming import Timer from loki.build import workqueue -from loki.batch import Scheduler, SchedulerConfig, Item +from loki.batch import Scheduler, SchedulerConfig, Item, Transformation from loki.config import config as loki_config from loki.lint.reporter import ( FileReport, RuleReport, Reporter, LazyTextfile, @@ -27,7 +27,6 @@ from loki.logging import logger from loki.sourcefile import Sourcefile from loki.tools import filehash, find_paths, CaseInsensitiveDict -from loki.transform import Transformation __all__ = ['Linter', 'LinterTransformation', 'lint_files'] diff --git a/loki/tools/files.py b/loki/tools/files.py index f4fa34a2c..d82417bbd 100644 --- a/loki/tools/files.py +++ b/loki/tools/files.py @@ -25,8 +25,10 @@ __all__ = [ - 'LokiTempdir', 'gettempdir', 'filehash', 'delete', 'find_paths', 'find_files', - 'disk_cached', 'load_module' + 'LokiTempdir', 'gettempdir', 'filehash', 'delete', 'find_paths', + 'find_files', 'disk_cached', 'load_module', + 'write_env_launch_script', 'local_loki_setup', + 'local_loki_cleanup' ] @@ -242,3 +244,116 @@ def load_module(module, path=None): # If module caching interferes, try again with clean caches invalidate_caches() return import_module(module) + + +def write_env_launch_script(here, binary, args): + """ + Utility method that is used for regression tests that require + activating an environment file before running :data:`binary`. + + This writes a simple script of the form + + .. code-block:: + source env.sh + bin/ + exit $? + + Parameters + ---------- + here : pathlib.Path or str + The directory in which the script is created + binary : str + The name of the binary + args : list of str + List of arguments to pass to the binary + + Returns + ------- + pathlib.Path + The path to the created script file + """ + + script = Path(here/f'build/run_{binary}.sh') + script.write_text(f""" +#!/bin/bash + +source env.sh >&2 +bin/{binary} {' '.join(args)} +exit $? + """.strip()) + script.chmod(0o750) + + return script + + +def local_loki_setup(here): + """ + Utility method that is used to determine paths for injecting the + currently running source code of Loki into an + `ecbundle `_-based worktree This + is used for regression tests to facilitate the use of a local Loki + source copy in the build. In particular, any existing Loki source + copy in the bundle worktree is moved to a backup location. + + .. warning:: If a backup copy exists already at the backup + location, this is removed before moving the existing Loki copy + to the backup location. + + Note that injecting the currently running Loki installation only + works if it has been installed in editable mode. However, this + utility also does not take care of the actual injection of the + currently running installation, therefore making this also useful + if the purpose is to trigger a Loki download via the bundle create + mechanism. + + The companion utility :any:`local_loki_cleanup` can be used to + revert these changes. + + Parameters + ---------- + here : pathlib.Path + The root path of the bundle worktree. + + Returns + ------- + tuple of (str, pathlib.Path, pathlib.Path) + The absolute path to the base directory of the currently + running Loki installation, the ``target`` path where Loki + needs to be injected in the bundle directory, and the + ``backup`` path where an existing Loki copy in the bundle has + been moved to. + """ + + lokidir = Path(__file__).parent.parent.parent + target = here/'source/loki' + backup = here/'source/loki.bak' + + # Do not overwrite any existing Loki copy + if target.exists(): + if backup.exists(): + shutil.rmtree(backup) + shutil.move(target, backup) + + return str(lokidir.resolve()), target, backup + + +def local_loki_cleanup(target, backup): + """ + Companion utility to :any:`local_loki_setup` to revert the + changes. + + This removes a symlink at :data:`target`, if it exists, and moves + the :data:`backup` path in its original location. + + Parameters + --------- + target : pathlib.Path + The target injection path as returned by :any:`local_loki_setup` + backup : pathlib.Path + The backup path as created by :any:`local_loki_setup` + """ + + if target.is_symlink(): + target.unlink() + if not target.exists() and backup.exists(): + shutil.move(backup, target) diff --git a/loki/transform/__init__.py b/loki/transform/__init__.py deleted file mode 100644 index 4a3a394ff..000000000 --- a/loki/transform/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from loki.transform.transformation import * # noqa -from loki.transform.transform_utilities import * # noqa -from loki.transform.transform_array_indexing import * # noqa -from loki.transform.transform_inline import * # noqa -from loki.transform.transform_loop import * # noqa -from loki.transform.transform_region import * # noqa -from loki.transform.dependency_transform import * # noqa -from loki.transform.fortran_c_transform import * # noqa -from loki.transform.fortran_max_transform import * # noqa -from loki.transform.fortran_python_transform import * # noqa -from loki.transform.build_system_transform import * # noqa -from loki.transform.transform_hoist_variables import * # noqa -from loki.transform.transform_parametrise import * # noqa -from loki.transform.transform_extract_contained_procedures import * # noqa -from loki.transform.transform_remove_code import * # noqa -from loki.transform.transform_sanitise import * # noqa -from loki.transform.pipeline import * # noqa diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py new file mode 100644 index 000000000..514f47af2 --- /dev/null +++ b/loki/transformations/__init__.py @@ -0,0 +1,25 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.array_indexing import * # noqa +from loki.transformations.build_system import * # noqa +from loki.transformations.argument_shape import * # noqa +from loki.transformations.data_offload import * # noqa +from loki.transformations.drhook import * # noqa +from loki.transformations.extract import * # noqa +from loki.transformations.hoist_variables import * # noqa +from loki.transformations.inline import * # noqa +from loki.transformations.parametrise import * # noqa +from loki.transformations.remove_code import * # noqa +from loki.transformations.sanitise import * # noqa +from loki.transformations.single_column import * # noqa +from loki.transformations.transpile import * # noqa +from loki.transformations.transform_derived_types import * # noqa +from loki.transformations.transform_loop import * # noqa +from loki.transformations.transform_region import * # noqa +from loki.transformations.pool_allocator import * # noqa +from loki.transformations.utilities import * # noqa diff --git a/transformations/transformations/argument_shape.py b/loki/transformations/argument_shape.py similarity index 96% rename from transformations/transformations/argument_shape.py rename to loki/transformations/argument_shape.py index e812355c8..a389e235d 100644 --- a/transformations/transformations/argument_shape.py +++ b/loki/transformations/argument_shape.py @@ -21,11 +21,11 @@ """ -from loki import ( - Transformation, FindNodes, CallStatement, Array, FindVariables, - SubstituteExpressions, BasicType, as_tuple, Transformer, - CaseInsensitiveDict -) +from loki.batch import Transformation +from loki.expression import Array, FindVariables, SubstituteExpressions +from loki.ir import FindNodes, CallStatement, Transformer +from loki.tools import as_tuple, CaseInsensitiveDict +from loki.types import BasicType __all__ = ['ArgumentArrayShapeAnalysis', 'ExplicitArgumentArrayShapeTransformation'] diff --git a/loki/transform/transform_array_indexing.py b/loki/transformations/array_indexing.py similarity index 100% rename from loki/transform/transform_array_indexing.py rename to loki/transformations/array_indexing.py diff --git a/loki/transformations/build_system/__init__.py b/loki/transformations/build_system/__init__.py new file mode 100644 index 000000000..175ed0eca --- /dev/null +++ b/loki/transformations/build_system/__init__.py @@ -0,0 +1,10 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.build_system.dependency import * # noqa +from loki.transformations.build_system.file_write import * # noqa +from loki.transformations.build_system.module_wrap import * # noqa diff --git a/loki/transform/dependency_transform.py b/loki/transformations/build_system/dependency.py similarity index 68% rename from loki/transform/dependency_transform.py rename to loki/transformations/build_system/dependency.py index 4be2213d8..b87a6b52f 100644 --- a/loki/transform/dependency_transform.py +++ b/loki/transformations/build_system/dependency.py @@ -8,20 +8,18 @@ from pathlib import Path from loki.backend import fgen -from loki.expression import Variable, FindInlineCalls -from loki.ir import ( - CallStatement, Import, Section, Interface, FindNodes, Transformer -) +from loki.batch import Transformation +from loki.expression import FindInlineCalls +from loki.ir import CallStatement, Import, Interface, FindNodes, Transformer from loki.logging import warning from loki.module import Module from loki.scope import Scope from loki.subroutine import Subroutine from loki.types import ProcedureType from loki.tools import as_tuple -from loki.transform.transformation import Transformation -__all__ = ['DependencyTransformation', 'ModuleWrapTransformation'] +__all__ = ['DependencyTransformation'] class DependencyTransformation(Transformation): @@ -355,162 +353,3 @@ def generate_interfaces(self, routine): intfb_path = self.include_path/f'{routine.name.lower()}.intfb.h' with intfb_path.open('w') as f: f.write(fgen(routine.interface)) - - -class ModuleWrapTransformation(Transformation): - """ - Utility transformation that ensures all transformed kernel - subroutines are wrapped in a module - - The module name is derived from the subroutine name and :data:`module_suffix`. - - Any previous import of wrapped subroutines via interfaces or C-style header - imports of interface blocks is replaced by a Fortran import (``USE``). - - Parameters - ---------- - module_suffix : str - Special suffix to signal module names like `_MOD` - replace_ignore_items : bool - Debug flag to toggle the replacement of calls to subroutines - in the ``ignore``. Default is ``True``. - """ - - # This transformation is applied over the file graph - traverse_file_graph = True - - # This transformation recurses from the Sourcefile down - recurse_to_modules = True - recurse_to_procedures = True - recurse_to_internal_procedures = False - - # This transformation changes the names of items and creates new items - renames_items = True - creates_items = True - - def __init__(self, module_suffix, replace_ignore_items=True): - self.module_suffix = module_suffix - self.replace_ignore_items = replace_ignore_items - - def transform_file(self, sourcefile, **kwargs): - """ - For kernel routines, wrap each subroutine in the current file in a module - """ - role = kwargs.get('role') - - if items := kwargs.get('items'): - # We consider the sourcefile to be a "kernel" file if all items are kernels - if all(item.role == 'kernel' for item in items): - role = 'kernel' - else: - role = 'driver' - - if role == 'kernel': - self.module_wrap(sourcefile) - - def transform_module(self, module, **kwargs): - """ - Update imports of wrapped subroutines - """ - self.update_imports(module, imports=module.imports, **kwargs) - - def transform_subroutine(self, routine, **kwargs): - """ - Update imports of wrapped subroutines - """ - if item := kwargs.get('item'): - # Rename the item if it has suddenly a parent - if routine.parent and routine.parent.name.lower() != item.scope_name: - item.name = f'{routine.parent.name.lower()}#{item.local_name}' - - # Note, C-style imports can be in the body, so use whole IR - imports = FindNodes(Import).visit(routine.ir) - self.update_imports(routine, imports=imports, **kwargs) - - # Interface blocks can only be in the spec - intfs = FindNodes(Interface).visit(routine.spec) - self.replace_interfaces(routine, intfs=intfs, **kwargs) - - def module_wrap(self, sourcefile): - """ - Wrap target subroutines in modules and replace in source file. - """ - for routine in sourcefile.subroutines: - # Create wrapper module and insert into file, replacing the old - # standalone routine - modname = f'{routine.name}{self.module_suffix}' - module = Module(name=modname, contains=Section(body=as_tuple(routine))) - routine.parent = module - sourcefile.ir._update(body=as_tuple( - module if c is routine else c for c in sourcefile.ir.body - )) - - def update_imports(self, source, imports, **kwargs): - """ - Update imports of wrapped subroutines. - """ - from loki.batch import SchedulerConfig # pylint: disable=import-outside-toplevel,cyclic-import - - targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) - if self.replace_ignore_items and (item := kwargs.get('item')): - targets += tuple(str(i).lower() for i in item.ignore) - - def _update_item(proc_name, module_name): - if item and (matched_keys := SchedulerConfig.match_item_keys(proc_name, item.ignore)): - # Add the module wrapped but ignored items to the block list because we won't be able to - # find them as dependencies under their new name anymore - item.config['block'] = as_tuple(item.block) + tuple( - module_name for name in item.ignore if name in matched_keys - ) - - # Transformer map to remove any outdated imports - removal_map = {} - - # We go through the IR, as C-imports can be attributed to the body - for im in imports: - if im.c_import: - target_symbol, *suffixes = im.module.lower().split('.', maxsplit=1) - if targets and target_symbol.lower() in targets and not 'func.h' in suffixes: - # Create a new module import with explicitly qualified symbol - modname = f'{target_symbol}{self.module_suffix}' - _update_item(target_symbol.lower(), modname) - new_symbol = Variable(name=target_symbol, scope=source) - new_import = im.clone(module=modname, c_import=False, symbols=(new_symbol,)) - source.spec.prepend(new_import) - - # Mark current import for removal - removal_map[im] = None - - # Apply any scheduled import removals to spec and body - if removal_map: - source.spec = Transformer(removal_map).visit(source.spec) - if isinstance(source, Subroutine): - source.body = Transformer(removal_map).visit(source.body) - - def replace_interfaces(self, source, intfs, **kwargs): - """ - Update explicit interfaces to actively transformed subroutines. - """ - targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) - if self.replace_ignore_items and (item := kwargs.get('item')): - targets += tuple(str(i).lower() for i in item.ignore) - - # Transformer map to remove any outdated interfaces - removal_map = {} - - for i in intfs: - for b in i.body: - if isinstance(b, Subroutine): - if targets and b.name.lower() in targets: - # Create a new module import with explicitly qualified symbol - modname = f'{b.name}{self.module_suffix}' - new_symbol = Variable(name=f'{b.name}', scope=source) - new_import = Import(module=modname, c_import=False, symbols=(new_symbol,)) - source.spec.prepend(new_import) - - # Mark current import for removal - removal_map[i] = None - - # Apply any scheduled interface removals to spec - if removal_map: - source.spec = Transformer(removal_map).visit(source.spec) diff --git a/loki/transform/build_system_transform.py b/loki/transformations/build_system/file_write.py similarity index 95% rename from loki/transform/build_system_transform.py rename to loki/transformations/build_system/file_write.py index 158839bf6..7dc406a15 100644 --- a/loki/transform/build_system_transform.py +++ b/loki/transformations/build_system/file_write.py @@ -11,8 +11,8 @@ from pathlib import Path -from loki.transform.transformation import Transformation -from loki.batch.item import ProcedureItem, ModuleItem#, GlobalVariableItem +from loki.batch import Transformation, ProcedureItem, ModuleItem + __all__ = ['FileWriteTransformation'] diff --git a/loki/transformations/build_system/module_wrap.py b/loki/transformations/build_system/module_wrap.py new file mode 100644 index 000000000..9e132e944 --- /dev/null +++ b/loki/transformations/build_system/module_wrap.py @@ -0,0 +1,174 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.batch import Transformation, SchedulerConfig +from loki.expression import Variable +from loki.ir import Import, Section, Interface, FindNodes, Transformer +from loki.module import Module +from loki.subroutine import Subroutine +from loki.tools import as_tuple + + +__all__ = ['ModuleWrapTransformation'] + + +class ModuleWrapTransformation(Transformation): + """ + Utility transformation that ensures all transformed kernel + subroutines are wrapped in a module + + The module name is derived from the subroutine name and :data:`module_suffix`. + + Any previous import of wrapped subroutines via interfaces or C-style header + imports of interface blocks is replaced by a Fortran import (``USE``). + + Parameters + ---------- + module_suffix : str + Special suffix to signal module names like `_MOD` + replace_ignore_items : bool + Debug flag to toggle the replacement of calls to subroutines + in the ``ignore``. Default is ``True``. + """ + + # This transformation is applied over the file graph + traverse_file_graph = True + + # This transformation recurses from the Sourcefile down + recurse_to_modules = True + recurse_to_procedures = True + recurse_to_internal_procedures = False + + # This transformation changes the names of items and creates new items + renames_items = True + creates_items = True + + def __init__(self, module_suffix, replace_ignore_items=True): + self.module_suffix = module_suffix + self.replace_ignore_items = replace_ignore_items + + def transform_file(self, sourcefile, **kwargs): + """ + For kernel routines, wrap each subroutine in the current file in a module + """ + role = kwargs.get('role') + + if items := kwargs.get('items'): + # We consider the sourcefile to be a "kernel" file if all items are kernels + if all(item.role == 'kernel' for item in items): + role = 'kernel' + else: + role = 'driver' + + if role == 'kernel': + self.module_wrap(sourcefile) + + def transform_module(self, module, **kwargs): + """ + Update imports of wrapped subroutines + """ + self.update_imports(module, imports=module.imports, **kwargs) + + def transform_subroutine(self, routine, **kwargs): + """ + Update imports of wrapped subroutines + """ + if item := kwargs.get('item'): + # Rename the item if it has suddenly a parent + if routine.parent and routine.parent.name.lower() != item.scope_name: + item.name = f'{routine.parent.name.lower()}#{item.local_name}' + + # Note, C-style imports can be in the body, so use whole IR + imports = FindNodes(Import).visit(routine.ir) + self.update_imports(routine, imports=imports, **kwargs) + + # Interface blocks can only be in the spec + intfs = FindNodes(Interface).visit(routine.spec) + self.replace_interfaces(routine, intfs=intfs, **kwargs) + + def module_wrap(self, sourcefile): + """ + Wrap target subroutines in modules and replace in source file. + """ + for routine in sourcefile.subroutines: + # Create wrapper module and insert into file, replacing the old + # standalone routine + modname = f'{routine.name}{self.module_suffix}' + module = Module(name=modname, contains=Section(body=as_tuple(routine))) + routine.parent = module + sourcefile.ir._update(body=as_tuple( + module if c is routine else c for c in sourcefile.ir.body + )) + + def update_imports(self, source, imports, **kwargs): + """ + Update imports of wrapped subroutines. + """ + + targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) + if self.replace_ignore_items and (item := kwargs.get('item')): + targets += tuple(str(i).lower() for i in item.ignore) + + def _update_item(proc_name, module_name): + if item and (matched_keys := SchedulerConfig.match_item_keys(proc_name, item.ignore)): + # Add the module wrapped but ignored items to the block list because we won't be able to + # find them as dependencies under their new name anymore + item.config['block'] = as_tuple(item.block) + tuple( + module_name for name in item.ignore if name in matched_keys + ) + + # Transformer map to remove any outdated imports + removal_map = {} + + # We go through the IR, as C-imports can be attributed to the body + for im in imports: + if im.c_import: + target_symbol, *suffixes = im.module.lower().split('.', maxsplit=1) + if targets and target_symbol.lower() in targets and not 'func.h' in suffixes: + # Create a new module import with explicitly qualified symbol + modname = f'{target_symbol}{self.module_suffix}' + _update_item(target_symbol.lower(), modname) + new_symbol = Variable(name=target_symbol, scope=source) + new_import = im.clone(module=modname, c_import=False, symbols=(new_symbol,)) + source.spec.prepend(new_import) + + # Mark current import for removal + removal_map[im] = None + + # Apply any scheduled import removals to spec and body + if removal_map: + source.spec = Transformer(removal_map).visit(source.spec) + if isinstance(source, Subroutine): + source.body = Transformer(removal_map).visit(source.body) + + def replace_interfaces(self, source, intfs, **kwargs): + """ + Update explicit interfaces to actively transformed subroutines. + """ + targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) + if self.replace_ignore_items and (item := kwargs.get('item')): + targets += tuple(str(i).lower() for i in item.ignore) + + # Transformer map to remove any outdated interfaces + removal_map = {} + + for i in intfs: + for b in i.body: + if isinstance(b, Subroutine): + if targets and b.name.lower() in targets: + # Create a new module import with explicitly qualified symbol + modname = f'{b.name}{self.module_suffix}' + new_symbol = Variable(name=f'{b.name}', scope=source) + new_import = Import(module=modname, c_import=False, symbols=(new_symbol,)) + source.spec.prepend(new_import) + + # Mark current import for removal + removal_map[i] = None + + # Apply any scheduled interface removals to spec + if removal_map: + source.spec = Transformer(removal_map).visit(source.spec) diff --git a/loki/transform/tests/test_transform_dependency.py b/loki/transformations/build_system/tests/test_dependency.py similarity index 99% rename from loki/transform/tests/test_transform_dependency.py rename to loki/transformations/build_system/tests/test_dependency.py index c05558aa4..9cb0fec7f 100644 --- a/loki/transform/tests/test_transform_dependency.py +++ b/loki/transformations/build_system/tests/test_dependency.py @@ -2,12 +2,15 @@ from shutil import rmtree import pytest -from loki import ( - gettempdir, Sourcefile, CallStatement, Import, Interface, - FindNodes, FindInlineCalls, Intrinsic, Scheduler, SchedulerConfig -) +from loki import Sourcefile, gettempdir +from loki.batch import Scheduler, SchedulerConfig +from loki.expression import FindInlineCalls from loki.frontend import available_frontends, OMNI, OFP -from loki.transform import DependencyTransformation, ModuleWrapTransformation +from loki.ir import FindNodes, CallStatement, Import, Interface, Intrinsic + +from loki.transformations import ( + DependencyTransformation, ModuleWrapTransformation +) @pytest.fixture(scope='module', name='here') diff --git a/transformations/transformations/data_offload.py b/loki/transformations/data_offload.py similarity index 98% rename from transformations/transformations/data_offload.py rename to loki/transformations/data_offload.py index fe9d723b2..881fa27ae 100644 --- a/transformations/transformations/data_offload.py +++ b/loki/transformations/data_offload.py @@ -7,13 +7,19 @@ from collections import defaultdict from itertools import chain -from loki import ( - pragma_regions_attached, PragmaRegion, Transformation, FindNodes, - CallStatement, Pragma, Scalar, Array, as_tuple, Transformer, warning, BasicType, - ProcedureItem, ModuleItem, dataflow_analysis_attached, Import, - Comment, flatten, DerivedType, get_pragma_parameters, CaseInsensitiveDict, - FindInlineCalls, SubstituteExpressions + +from loki.analyse import dataflow_analysis_attached +from loki.batch import Transformation, ProcedureItem, ModuleItem +from loki.expression import ( + Scalar, Array, FindInlineCalls, SubstituteExpressions +) +from loki.ir import ( + FindNodes, PragmaRegion, CallStatement, Pragma, Import, Comment, + Transformer, pragma_regions_attached, get_pragma_parameters ) +from loki.logging import warning +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import BasicType, DerivedType __all__ = [ diff --git a/transformations/transformations/drhook.py b/loki/transformations/drhook.py similarity index 93% rename from transformations/transformations/drhook.py rename to loki/transformations/drhook.py index d436d9885..364e89819 100644 --- a/transformations/transformations/drhook.py +++ b/loki/transformations/drhook.py @@ -9,10 +9,12 @@ Utility transformations to update or remove calls to DR_HOOK. """ -from loki import ( - FindNodes, Transformer, Transformation, CallStatement, - Conditional, as_tuple, Literal, Import +from loki.batch import Transformation +from loki.expression import Literal +from loki.ir import ( + FindNodes, Transformer, CallStatement, Conditional, Import ) +from loki.tools import as_tuple __all__ = ['DrHookTransformation'] diff --git a/loki/transform/transform_extract_contained_procedures.py b/loki/transformations/extract.py similarity index 100% rename from loki/transform/transform_extract_contained_procedures.py rename to loki/transformations/extract.py diff --git a/loki/transform/transform_hoist_variables.py b/loki/transformations/hoist_variables.py similarity index 97% rename from loki/transform/transform_hoist_variables.py rename to loki/transformations/hoist_variables.py index 23dafca5b..4bf855ba2 100644 --- a/loki/transform/transform_hoist_variables.py +++ b/loki/transformations/hoist_variables.py @@ -79,21 +79,24 @@ scheduler.process(transformation=HoistTemporaryArraysAnalysis(dim_vars=('a',), key=key)) scheduler.process(transformation=HoistTemporaryArraysTransformationAllocatable(key=key)) """ -from loki.expression import FindVariables, SubstituteExpressions + +from loki.batch import Transformation, ProcedureItem +from loki.expression import ( + symbols as sym, FindVariables, FindInlineCalls, + SubstituteExpressions, is_dimension_constant +) from loki.ir import ( CallStatement, Allocation, Deallocation, Transformer, FindNodes ) from loki.tools.util import is_iterable, as_tuple, CaseInsensitiveDict -from loki.transform.transformation import Transformation -from loki.transform.transform_utilities import single_variable_declaration -from loki.batch.item import ProcedureItem -from loki.expression.expr_visitors import FindInlineCalls -import loki.expression.symbols as sym -from loki.expression.symbolic import is_dimension_constant + +from loki.transformations.utilities import single_variable_declaration -__all__ = ['HoistVariablesAnalysis', 'HoistVariablesTransformation', - 'HoistTemporaryArraysAnalysis', 'HoistTemporaryArraysTransformationAllocatable'] +__all__ = [ + 'HoistVariablesAnalysis', 'HoistVariablesTransformation', + 'HoistTemporaryArraysAnalysis', 'HoistTemporaryArraysTransformationAllocatable' +] class HoistVariablesAnalysis(Transformation): diff --git a/loki/transform/transform_inline.py b/loki/transformations/inline.py similarity index 98% rename from loki/transform/transform_inline.py rename to loki/transformations/inline.py index cb8bbfaee..f2a9b7026 100644 --- a/loki/transform/transform_inline.py +++ b/loki/transformations/inline.py @@ -12,28 +12,27 @@ """ from collections import defaultdict -from loki.expression import ( - FindVariables, FindInlineCalls, FindLiterals, - SubstituteExpressions, LokiIdentityMapper -) +from loki.batch import Transformation from loki.ir import ( Import, Comment, Assignment, VariableDeclaration, CallStatement, Transformer, FindNodes, pragmas_attached, is_loki_pragma ) -from loki.expression import symbols as sym +from loki.expression import ( + symbols as sym, FindVariables, FindInlineCalls, FindLiterals, + SubstituteExpressions, LokiIdentityMapper +) from loki.types import BasicType from loki.tools import as_tuple from loki.logging import warning, error from loki.subroutine import Subroutine -from loki.transform.transform_sanitise import transform_sequence_association_append_map -from loki.transform.transformation import Transformation -from loki.transform.transform_remove_code import do_remove_dead_code -from loki.transform.transform_utilities import ( - single_variable_declaration, - recursive_expression_map_update +from loki.transformations.remove_code import do_remove_dead_code +from loki.transformations.sanitise import transform_sequence_association_append_map +from loki.transformations.utilities import ( + single_variable_declaration, recursive_expression_map_update ) + __all__ = [ 'inline_constant_parameters', 'inline_elemental_functions', 'inline_internal_procedures', 'inline_member_procedures', diff --git a/loki/transform/transform_parametrise.py b/loki/transformations/parametrise.py similarity index 98% rename from loki/transform/transform_parametrise.py rename to loki/transformations/parametrise.py index d88910ca3..a21123025 100644 --- a/loki/transform/transform_parametrise.py +++ b/loki/transformations/parametrise.py @@ -81,11 +81,13 @@ dic2p = {'a': 10} scheduler.process(transformation=ParametriseTransformation(dic2p=dic2p, replace_by_value=True)) """ + +from loki.batch import Transformation from loki.expression import symbols as sym from loki.ir import nodes as ir, Transformer, FindNodes from loki.tools.util import as_tuple, CaseInsensitiveDict -from loki.transform.transformation import Transformation -from loki.transform.transform_inline import inline_constant_parameters + +from loki.transformations.inline import inline_constant_parameters __all__ = ['ParametriseTransformation'] diff --git a/transformations/transformations/pool_allocator.py b/loki/transformations/pool_allocator.py similarity index 98% rename from transformations/transformations/pool_allocator.py rename to loki/transformations/pool_allocator.py index e99278366..83116599b 100644 --- a/transformations/transformations/pool_allocator.py +++ b/loki/transformations/pool_allocator.py @@ -8,23 +8,26 @@ import re from collections import defaultdict +from loki.batch import Transformation +from loki.analyse import dataflow_analysis_attached from loki.expression import ( - Quotient, IntLiteral, LogicLiteral, Variable, Array, Sum, Literal, - Product, InlineCall, Comparison, RangeIndex, Cast, - ProcedureSymbol, LogicalNot, simplify, + FindVariables, FindInlineCalls, Quotient, IntLiteral, + LogicLiteral, Variable, Array, Sum, Literal, Product, InlineCall, + Comparison, RangeIndex, Cast, ProcedureSymbol, LogicalNot, + simplify, SubstituteExpressions, is_dimension_constant, + DetachScopesMapper ) from loki.ir import ( - Intrinsic, Assignment, Conditional, CallStatement, Import, - Allocation, Deallocation, Loop, Pragma -) -from loki import ( - as_tuple, warning, debug, Transformation, FindNodes, - FindVariables, Transformer, SubstituteExpressions, - DetachScopesMapper, SymbolAttributes, BasicType, DerivedType, - is_dimension_constant, recursive_expression_map_update, - get_pragma_parameters, FindInlineCalls, Interface, - dataflow_analysis_attached + FindNodes, Transformer, Intrinsic, Assignment, Conditional, + CallStatement, Import, Allocation, Deallocation, Loop, Pragma, + Interface, get_pragma_parameters ) +from loki.logging import warning, debug +from loki.tools import as_tuple +from loki.types import SymbolAttributes, BasicType, DerivedType + +from loki.transformations.utilities import recursive_expression_map_update + __all__ = ['TemporariesPoolAllocatorTransformation'] diff --git a/loki/transform/transform_remove_code.py b/loki/transformations/remove_code.py similarity index 99% rename from loki/transform/transform_remove_code.py rename to loki/transformations/remove_code.py index 3fe5dc4e1..bd0b42e74 100644 --- a/loki/transform/transform_remove_code.py +++ b/loki/transformations/remove_code.py @@ -10,11 +10,11 @@ section and to perform Dead Code Elimination. """ +from loki.batch import Transformation from loki.expression.symbolic import simplify from loki.tools import flatten, as_tuple from loki.ir import Conditional, Transformer, Comment from loki.ir.pragma_utils import is_loki_pragma, pragma_regions_attached -from loki.transform.transformation import Transformation __all__ = [ diff --git a/loki/transform/transform_sanitise.py b/loki/transformations/sanitise.py similarity index 98% rename from loki/transform/transform_sanitise.py rename to loki/transformations/sanitise.py index 59891ca46..e2181c539 100644 --- a/loki/transform/transform_sanitise.py +++ b/loki/transformations/sanitise.py @@ -12,13 +12,13 @@ code easier. """ +from loki.batch import Transformation from loki.expression import FindVariables, SubstituteExpressions, Array, RangeIndex from loki.ir import CallStatement, FindNodes, Transformer, NestedTransformer from loki.tools import as_tuple, CaseInsensitiveDict from loki.types import BasicType -from loki.transform.transform_utilities import recursive_expression_map_update -from loki.transform.transformation import Transformation +from loki.transformations.utilities import recursive_expression_map_update __all__ = [ diff --git a/loki/transformations/single_column/__init__.py b/loki/transformations/single_column/__init__.py new file mode 100644 index 000000000..f3598dd1e --- /dev/null +++ b/loki/transformations/single_column/__init__.py @@ -0,0 +1,14 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.single_column.annotate import * # noqa +from loki.transformations.single_column.base import * # noqa +from loki.transformations.single_column.claw import * # noqa +from loki.transformations.single_column.hoist import * # noqa +from loki.transformations.single_column.scc import * # noqa +from loki.transformations.single_column.scc_cuf import * # noqa +from loki.transformations.single_column.vector import * # noqa diff --git a/transformations/transformations/single_column_annotate.py b/loki/transformations/single_column/annotate.py similarity index 96% rename from transformations/transformations/single_column_annotate.py rename to loki/transformations/single_column/annotate.py index b0a6a2fc5..bcb986555 100644 --- a/transformations/transformations/single_column_annotate.py +++ b/loki/transformations/single_column/annotate.py @@ -7,14 +7,19 @@ import re -from loki.expression import symbols as sym -from loki import ( - Transformation, FindNodes, Transformer, info, pragmas_attached, - as_tuple, flatten, ir, DerivedType, FindVariables, - CaseInsensitiveDict, pragma_regions_attached, PragmaRegion, - is_loki_pragma, is_dimension_constant +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindVariables, is_dimension_constant ) -from transformations.single_column_base import SCCBaseTransformation +from loki.ir import ( + nodes as ir, FindNodes, Transformer, pragmas_attached, + pragma_regions_attached, is_loki_pragma +) +from loki.logging import info +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import DerivedType + +from loki.transformations.single_column.base import SCCBaseTransformation __all__ = ['SCCAnnotateTransformation'] @@ -73,7 +78,7 @@ def kernel_annotate_vector_loops_openacc(cls, routine, horizontal): mapper = {} with pragma_regions_attached(routine): - for region in FindNodes(PragmaRegion).visit(routine.body): + for region in FindNodes(ir.PragmaRegion).visit(routine.body): if is_loki_pragma(region.pragma, starts_with='vector-reduction'): if (reduction_clause := re.search(r'reduction\([\w:0-9 \t]+\)', region.pragma.content)): diff --git a/transformations/transformations/single_column_base.py b/loki/transformations/single_column/base.py similarity index 97% rename from transformations/transformations/single_column_base.py rename to loki/transformations/single_column/base.py index 5901d6944..2c6d337ff 100644 --- a/transformations/transformations/single_column_base.py +++ b/loki/transformations/single_column/base.py @@ -5,13 +5,16 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from loki.expression import symbols as sym -from loki.transform import resolve_associates -from loki import ( - ir, Transformation, FindNodes, Transformer, - as_tuple, FindExpressions, - SymbolAttributes, BasicType, SubstituteExpressions +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindExpressions, SubstituteExpressions ) +from loki.ir import nodes as ir, FindNodes, Transformer +from loki.tools import as_tuple +from loki.types import SymbolAttributes, BasicType + + +from loki.transformations.sanitise import resolve_associates __all__ = ['SCCBaseTransformation'] diff --git a/transformations/transformations/single_column_claw.py b/loki/transformations/single_column/claw.py similarity index 97% rename from transformations/transformations/single_column_claw.py rename to loki/transformations/single_column/claw.py index 35a5c9956..c03ca24dd 100644 --- a/transformations/transformations/single_column_claw.py +++ b/loki/transformations/single_column/claw.py @@ -11,13 +11,21 @@ """ from collections import OrderedDict -from loki import ( - Transformation, FindVariables, FindNodes, Transformer, SubstituteExpressions, - Assignment, CallStatement, Loop, Variable, - Array, Pragma, VariableDeclaration, LoopRange, RangeIndex, - SymbolAttributes, BasicType, CaseInsensitiveDict, as_tuple, warning, - recursive_expression_map_update + +from loki.batch import Transformation +from loki.expression import ( + FindVariables, SubstituteExpressions, Variable, + Array, LoopRange, RangeIndex +) +from loki.ir import ( + FindNodes, Transformer, Assignment, CallStatement, Loop, Pragma, + VariableDeclaration, ) +from loki.logging import warning +from loki.tools import as_tuple, CaseInsensitiveDict +from loki.types import SymbolAttributes, BasicType + +from loki.transformations.utilities import recursive_expression_map_update __all__ = ['ExtractSCATransformation', 'CLAWTransformation'] diff --git a/transformations/transformations/single_column_hoist.py b/loki/transformations/single_column/hoist.py similarity index 96% rename from transformations/transformations/single_column_hoist.py rename to loki/transformations/single_column/hoist.py index 522dd11cc..61bf906c0 100644 --- a/transformations/transformations/single_column_hoist.py +++ b/loki/transformations/single_column/hoist.py @@ -6,9 +6,10 @@ # nor does it submit to any jurisdiction. from loki.expression import symbols as sym -from loki.transform import HoistVariablesTransformation -from loki import ir -from transformations.single_column_base import SCCBaseTransformation +from loki.ir import nodes as ir + +from loki.transformations.hoist_variables import HoistVariablesTransformation +from loki.transformations.single_column.base import SCCBaseTransformation __all__ = ['SCCHoistTemporaryArraysTransformation'] diff --git a/transformations/transformations/single_column_coalesced.py b/loki/transformations/single_column/scc.py similarity index 92% rename from transformations/transformations/single_column_coalesced.py rename to loki/transformations/single_column/scc.py index 60f571232..2e06ee7a3 100644 --- a/transformations/transformations/single_column_coalesced.py +++ b/loki/transformations/single_column/scc.py @@ -7,12 +7,15 @@ from functools import partial -from loki.transform import Pipeline, HoistTemporaryArraysAnalysis -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from transformations.single_column_base import SCCBaseTransformation -from transformations.single_column_annotate import SCCAnnotateTransformation -from transformations.single_column_hoist import SCCHoistTemporaryArraysTransformation -from transformations.single_column_coalesced_vector import ( +from loki.batch import Pipeline + +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation + +from loki.transformations.single_column.base import SCCBaseTransformation +from loki.transformations.single_column.annotate import SCCAnnotateTransformation +from loki.transformations.single_column.hoist import SCCHoistTemporaryArraysTransformation +from loki.transformations.single_column.vector import ( SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation ) diff --git a/transformations/transformations/scc_cuf.py b/loki/transformations/single_column/scc_cuf.py similarity index 97% rename from transformations/transformations/scc_cuf.py rename to loki/transformations/single_column/scc_cuf.py index 891146993..14fedd35b 100644 --- a/transformations/transformations/scc_cuf.py +++ b/loki/transformations/single_column/scc_cuf.py @@ -9,20 +9,27 @@ Single-Column-Coalesced CUDA Fortran (SCC-CUF) transformation. """ -from loki.expression import symbols as sym -from loki.transform import resolve_associates, single_variable_declaration, HoistVariablesTransformation -from loki import ir -from loki import ( - Transformation, FindNodes, FindVariables, Transformer, - SubstituteExpressions, SymbolAttributes, - CaseInsensitiveDict, as_tuple, flatten, types +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindVariables, SubstituteExpressions ) +from loki.ir import nodes as ir, FindNodes, Transformer +from loki.tools import CaseInsensitiveDict, as_tuple, flatten +from loki.types import BasicType, DerivedType +from loki.scope import SymbolAttributes -from transformations.single_column_base import SCCBaseTransformation -from transformations.single_column_coalesced_vector import SCCDevectorTransformation +from loki.transformations.hoist_variables import HoistVariablesTransformation +from loki.transformations.sanitise import resolve_associates +from loki.transformations.single_column.base import SCCBaseTransformation +from loki.transformations.single_column.vector import SCCDevectorTransformation +from loki.transformations.utilities import single_variable_declaration -__all__ = ['SccCufTransformation', 'HoistTemporaryArraysDeviceAllocatableTransformation', - 'HoistTemporaryArraysPragmaOffloadTransformation'] + +__all__ = [ + 'SccCufTransformation', + 'HoistTemporaryArraysDeviceAllocatableTransformation', + 'HoistTemporaryArraysPragmaOffloadTransformation' +] class HoistTemporaryArraysDeviceAllocatableTransformation(HoistVariablesTransformation): @@ -142,7 +149,7 @@ def increase_heap_size(routine): routine: :any:`Subroutine` The subroutine (e.g. the driver) to increase the heap size """ - vtype = SymbolAttributes(types.BasicType.INTEGER, kind=sym.Variable(name="cuda_count_kind")) + vtype = SymbolAttributes(BasicType.INTEGER, kind=sym.Variable(name="cuda_count_kind")) routine.spec.append(ir.VariableDeclaration((sym.Variable(name="cudaHeapSize", type=vtype),))) assignment_lhs = routine.variable_map["istat"] @@ -411,7 +418,7 @@ def driver_device_variables(routine, targets=None): """ # istat: status of CUDA runtime function (e.g. for cudaDeviceSynchronize(), cudaMalloc(), cudaFree(), ...) - i_type = SymbolAttributes(types.BasicType.INTEGER) + i_type = SymbolAttributes(BasicType.INTEGER) routine.spec.append(ir.VariableDeclaration(symbols=(sym.Variable(name="istat", type=i_type),))) relevant_arrays = [] @@ -515,7 +522,7 @@ def driver_launch_configuration(routine, block_dim, targets=None): Tuple of subroutine call names that are processed in this traversal """ - d_type = SymbolAttributes(types.DerivedType("DIM3")) + d_type = SymbolAttributes(DerivedType("DIM3")) routine.spec.append(ir.VariableDeclaration(symbols=(sym.Variable(name="GRIDDIM", type=d_type), sym.Variable(name="BLOCKDIM", type=d_type)))) diff --git a/transformations/tests/test_single_column.py b/loki/transformations/single_column/tests/test_sca.py similarity index 98% rename from transformations/tests/test_single_column.py rename to loki/transformations/single_column/tests/test_sca.py index ef16d2fe4..0088d5b4a 100644 --- a/transformations/tests/test_single_column.py +++ b/loki/transformations/single_column/tests/test_sca.py @@ -8,8 +8,9 @@ import pytest from loki import Sourcefile, Dimension -from conftest import available_frontends -from transformations import ExtractSCATransformation +from loki.frontend import available_frontends + +from loki.transformations.single_column import ExtractSCATransformation @pytest.fixture(scope='module', name='horizontal') diff --git a/transformations/tests/test_single_column_coalesced.py b/loki/transformations/single_column/tests/test_single_column_coalesced.py similarity index 98% rename from transformations/tests/test_single_column_coalesced.py rename to loki/transformations/single_column/tests/test_single_column_coalesced.py index 0112df6af..4338b19f8 100644 --- a/transformations/tests/test_single_column_coalesced.py +++ b/loki/transformations/single_column/tests/test_single_column_coalesced.py @@ -9,19 +9,23 @@ import pytest -from loki import ( - OMNI, OFP, Subroutine, Dimension, FindNodes, Loop, Assignment, - CallStatement, Conditional, Scalar, Array, Pragma, - pragmas_attached, fgen, Sourcefile, Section, ProcedureItem, - ModuleItem, pragma_regions_attached, PragmaRegion, is_loki_pragma, - IntLiteral, RangeIndex, Comment, gettempdir, Scheduler, - SchedulerConfig, SanitiseTransformation, InlineTransformation +from loki import Subroutine, Sourcefile, Dimension, fgen, gettempdir +from loki.batch import Scheduler, SchedulerConfig, ProcedureItem, ModuleItem +from loki.expression import Scalar, Array, IntLiteral, RangeIndex +from loki.frontend import available_frontends, OMNI, OFP +from loki.ir import ( + FindNodes, Assignment, CallStatement, Conditional, Comment, Loop, + Pragma, PragmaRegion, Section, pragmas_attached, is_loki_pragma, + pragma_regions_attached ) -from conftest import available_frontends -from transformations import ( - DataOffloadTransformation, SCCBaseTransformation, SCCDevectorTransformation, - SCCDemoteTransformation, SCCRevectorTransformation, SCCAnnotateTransformation, - SCCVectorPipeline, SCCHoistPipeline + +from loki.transformations import ( + DataOffloadTransformation, SanitiseTransformation, InlineTransformation +) +from loki.transformations.single_column import ( + SCCBaseTransformation, SCCDevectorTransformation, + SCCDemoteTransformation, SCCRevectorTransformation, + SCCAnnotateTransformation, SCCVectorPipeline, SCCHoistPipeline, ) #pylint: disable=too-many-lines diff --git a/transformations/transformations/single_column_coalesced_vector.py b/loki/transformations/single_column/vector.py similarity index 96% rename from transformations/transformations/single_column_coalesced_vector.py rename to loki/transformations/single_column/vector.py index 3a36ea296..1e588aa9f 100644 --- a/transformations/transformations/single_column_coalesced_vector.py +++ b/loki/transformations/single_column/vector.py @@ -7,15 +7,26 @@ from more_itertools import split_at -from loki.expression import symbols as sym -from loki import ( - Transformation, FindNodes, ir, FindScopes, as_tuple, flatten, Transformer, - NestedTransformer, FindVariables, demote_variables, is_dimension_constant, - is_loki_pragma, dataflow_analysis_attached, BasicType, pragmas_attached +from loki.analyse import dataflow_analysis_attached +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, is_dimension_constant, FindVariables ) -from transformations.single_column_base import SCCBaseTransformation +from loki.ir import ( + nodes as ir, FindNodes, FindScopes, Transformer, + NestedTransformer, is_loki_pragma, pragmas_attached +) +from loki.tools import as_tuple, flatten +from loki.types import BasicType + +from loki.transformations.array_indexing import demote_variables +from loki.transformations.single_column.base import SCCBaseTransformation + -__all__ = ['SCCDevectorTransformation', 'SCCRevectorTransformation', 'SCCDemoteTransformation'] +__all__ = [ + 'SCCDevectorTransformation', 'SCCRevectorTransformation', + 'SCCDemoteTransformation' +] class SCCDevectorTransformation(Transformation): diff --git a/transformations/tests/sources/projArgShape/driver_mod.F90 b/loki/transformations/tests/sources/projArgShape/driver_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/driver_mod.F90 rename to loki/transformations/tests/sources/projArgShape/driver_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_a1_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_a_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_a_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_a_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_a_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_b_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_b_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_b_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_b_mod.F90 diff --git a/transformations/tests/sources/projArgShape/var_module_mod.F90 b/loki/transformations/tests/sources/projArgShape/var_module_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/var_module_mod.F90 rename to loki/transformations/tests/sources/projArgShape/var_module_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/driver_mod.F90 b/loki/transformations/tests/sources/projDerivedTypes/driver_mod.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/driver_mod.F90 rename to loki/transformations/tests/sources/projDerivedTypes/driver_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 b/loki/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/kernel_mod.F90 rename to loki/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 b/loki/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/some_derived_type.F90 rename to loki/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/driver.F90 b/loki/transformations/tests/sources/projGlobalVarImports/driver.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/driver.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/driver.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/functions.F90 b/loki/transformations/tests/sources/projGlobalVarImports/functions.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/functions.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/functions.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/kernels.F90 b/loki/transformations/tests/sources/projGlobalVarImports/kernels.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/kernels.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/kernels.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleA.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleA.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleA.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleA.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleB.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleB.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleB.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleB.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleC.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleC.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleC.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleC.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 diff --git a/transformations/tests/sources/projSccCuf/module/driver.f90 b/loki/transformations/tests/sources/projSccCuf/module/driver.f90 similarity index 100% rename from transformations/tests/sources/projSccCuf/module/driver.f90 rename to loki/transformations/tests/sources/projSccCuf/module/driver.f90 diff --git a/transformations/tests/sources/projSccCuf/module/kernel.f90 b/loki/transformations/tests/sources/projSccCuf/module/kernel.f90 similarity index 100% rename from transformations/tests/sources/projSccCuf/module/kernel.f90 rename to loki/transformations/tests/sources/projSccCuf/module/kernel.f90 diff --git a/transformations/tests/test_argument_shape.py b/loki/transformations/tests/test_argument_shape.py similarity index 98% rename from transformations/tests/test_argument_shape.py rename to loki/transformations/tests/test_argument_shape.py index 63acbce52..934cd9186 100644 --- a/transformations/tests/test_argument_shape.py +++ b/loki/transformations/tests/test_argument_shape.py @@ -8,14 +8,19 @@ from pathlib import Path import pytest -from conftest import available_frontends -from loki import CallStatement, FindNodes, OMNI, Subroutine, Scheduler, Sourcefile, flatten -from transformations import ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation +from loki import Subroutine, Scheduler, Sourcefile, flatten +from loki.frontend import available_frontends, OMNI +from loki.ir import CallStatement, FindNodes +from loki.transformations import ( + ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation +) + @pytest.fixture(scope='module', name='here') def fixture_here(): return Path(__file__).parent + @pytest.mark.parametrize('frontend', available_frontends()) def test_argument_shape_simple(frontend): """ diff --git a/loki/transform/tests/test_transform_array_indexing.py b/loki/transformations/tests/test_array_indexing.py similarity index 97% rename from loki/transform/tests/test_transform_array_indexing.py rename to loki/transformations/tests/test_array_indexing.py index e3219d562..a6cfc79ab 100644 --- a/loki/transform/tests/test_transform_array_indexing.py +++ b/loki/transformations/tests/test_array_indexing.py @@ -6,6 +6,7 @@ # nor does it submit to any jurisdiction. from pathlib import Path +from shutil import rmtree import pytest import numpy as np @@ -14,12 +15,14 @@ from loki.expression import symbols as sym, FindVariables from loki.frontend import available_frontends from loki.ir import FindNodes, CallStatement -from loki.transform import ( +from loki.tools import gettempdir + +from loki.transformations.array_indexing import ( promote_variables, demote_variables, normalize_range_indexing, invert_array_indices, flatten_arrays, normalize_array_shape_and_access, shift_to_zero_indexing, - FortranCTransformation ) +from loki.transformations.transpile import FortranCTransformation @pytest.fixture(scope='module', name='here') @@ -27,9 +30,18 @@ def fixture_here(): return Path(__file__).parent -@pytest.fixture(scope='module', name='builder') -def fixture_builder(here): - return Builder(source_dirs=here, build_dir=here/'build') +@pytest.fixture(scope='function', name='tempdir') +def fixture_tempdir(request): + basedir = gettempdir()/request.function.__name__ + basedir.mkdir(exist_ok=True) + yield basedir + if basedir.exists(): + rmtree(basedir) + + +@pytest.fixture(scope='function', name='builder') +def fixture_builder(tempdir): + return Builder(source_dirs=tempdir, build_dir=tempdir) @pytest.mark.parametrize('frontend', available_frontends()) @@ -636,7 +648,7 @@ def test_shift_to_zero_indexing(frontend, ignore): @pytest.mark.parametrize('frontend', available_frontends()) @pytest.mark.parametrize('explicit_dimensions', [True, False]) -def test_transform_flatten_arrays_call(here, frontend, builder, explicit_dimensions): +def test_transform_flatten_arrays_call(tempdir, frontend, builder, explicit_dimensions): """ Test flattening or arrays, meaning converting multi-dimensional arrays to one-dimensional arrays including corresponding @@ -698,7 +710,7 @@ def validate_routine(routine): # compile and test reference refname = f'ref_{driver.name}_{frontend}' - reference = jit_compile_lib([kernel_module, driver], path=here, name=refname, builder=builder) + reference = jit_compile_lib([kernel_module, driver], path=tempdir, name=refname, builder=builder) ref_function = reference.driver_routine nlon = 10 @@ -720,7 +732,7 @@ def validate_routine(routine): # compile and test the flattened variant flattenedname = f'flattened_{driver.name}_{frontend}' - flattened = jit_compile_lib([kernel_module, driver], path=here, name=flattenedname, builder=builder) + flattened = jit_compile_lib([kernel_module, driver], path=tempdir, name=flattenedname, builder=builder) flattened_function = flattened.driver_routine a_flattened, b_flattened = init_arguments(nlon, nlev, flattened=True) @@ -731,5 +743,3 @@ def validate_routine(routine): assert (b_flattened == b_ref.flatten(order='F')).all() builder.clean() - (here/f'{driver.name}.f90').unlink() - (here/f'{kernel_module.name}.f90').unlink() diff --git a/transformations/tests/test_cloudsc.py b/loki/transformations/tests/test_cloudsc.py similarity index 95% rename from transformations/tests/test_cloudsc.py rename to loki/transformations/tests/test_cloudsc.py index 4632e5376..39b0907a6 100644 --- a/transformations/tests/test_cloudsc.py +++ b/loki/transformations/tests/test_cloudsc.py @@ -13,10 +13,11 @@ import pandas as pd import pytest -from conftest import ( - available_frontends, write_env_launch_script, local_loki_setup, local_loki_cleanup +from loki.tools import ( + execute, write_env_launch_script, local_loki_setup, local_loki_cleanup ) -from loki import execute, OMNI, OFP, HAVE_FP, HAVE_OMNI, warning +from loki.frontend import available_frontends, OMNI, OFP, HAVE_FP, HAVE_OMNI +from loki.logging import warning pytestmark = pytest.mark.skipif('CLOUDSC_DIR' not in os.environ, reason='CLOUDSC_DIR not set') diff --git a/transformations/tests/test_data_offload.py b/loki/transformations/tests/test_data_offload.py similarity index 98% rename from transformations/tests/test_data_offload.py rename to loki/transformations/tests/test_data_offload.py index d9767388e..46e1fe20a 100644 --- a/transformations/tests/test_data_offload.py +++ b/loki/transformations/tests/test_data_offload.py @@ -10,14 +10,16 @@ import pytest from loki import ( - Sourcefile, FindNodes, Pragma, PragmaRegion, Loop, - CallStatement, pragma_regions_attached, get_pragma_parameters, - gettempdir, Scheduler, OMNI, Import, FindInlineCalls + Sourcefile, gettempdir, Scheduler, FindInlineCalls ) -from conftest import available_frontends -from transformations import ( - DataOffloadTransformation, GlobalVariableAnalysis, - GlobalVarOffloadTransformation, GlobalVarHoistTransformation +from loki.frontend import available_frontends, OMNI +from loki.ir import ( + FindNodes, Pragma, PragmaRegion, Loop, CallStatement, Import, + pragma_regions_attached, get_pragma_parameters +) +from loki.transformations import ( + DataOffloadTransformation, GlobalVariableAnalysis, + GlobalVarOffloadTransformation, GlobalVarHoistTransformation ) diff --git a/transformations/tests/test_drhook.py b/loki/transformations/tests/test_drhook.py similarity index 95% rename from transformations/tests/test_drhook.py rename to loki/transformations/tests/test_drhook.py index f3e733e79..e5646daf5 100644 --- a/transformations/tests/test_drhook.py +++ b/loki/transformations/tests/test_drhook.py @@ -8,13 +8,13 @@ import shutil import pytest -from loki import ( - Scheduler, SFilter, ProcedureItem, SchedulerConfig, FindNodes, - CallStatement, gettempdir, OMNI, Import +from loki.batch import ( + Scheduler, SFilter, ProcedureItem, SchedulerConfig ) - -from conftest import available_frontends -from transformations import DrHookTransformation +from loki.frontend import available_frontends, OMNI +from loki.ir import FindNodes, CallStatement, Import +from loki.tools import gettempdir +from loki.transformations import DrHookTransformation @pytest.fixture(scope='module', name='config') diff --git a/transformations/tests/test_ecwam.py b/loki/transformations/tests/test_ecwam.py similarity index 96% rename from transformations/tests/test_ecwam.py rename to loki/transformations/tests/test_ecwam.py index 7ae5c488f..a9c46b2c9 100644 --- a/transformations/tests/test_ecwam.py +++ b/loki/transformations/tests/test_ecwam.py @@ -11,8 +11,10 @@ from pathlib import Path import pytest -from conftest import write_env_launch_script, local_loki_setup, local_loki_cleanup -from loki import execute, HAVE_FP, FP +from loki.tools import ( + execute, write_env_launch_script, local_loki_setup, local_loki_cleanup +) +from loki.frontend import HAVE_FP, FP pytestmark = pytest.mark.skipif('ECWAM_DIR' not in os.environ, reason='ECWAM_DIR not set') diff --git a/loki/tests/test_extract_contained_procedures.py b/loki/transformations/tests/test_extract.py similarity index 99% rename from loki/tests/test_extract_contained_procedures.py rename to loki/transformations/tests/test_extract.py index cadcb3968..b8e9de8ff 100644 --- a/loki/tests/test_extract_contained_procedures.py +++ b/loki/transformations/tests/test_extract.py @@ -10,10 +10,11 @@ from loki.expression import FindInlineCalls from loki.frontend import available_frontends, OMNI, OFP from loki.ir import CallStatement, Import, FindNodes -from loki.transform import extract_contained_procedures from loki.sourcefile import Sourcefile from loki.subroutine import Subroutine +from loki.transformations.extract import extract_contained_procedures + @pytest.mark.parametrize('frontend', available_frontends()) def test_extract_contained_procedures_basic_scalar(frontend): diff --git a/loki/transform/tests/test_transform_hoist_variables.py b/loki/transformations/tests/test_hoist_variables.py similarity index 99% rename from loki/transform/tests/test_transform_hoist_variables.py rename to loki/transformations/tests/test_hoist_variables.py index 7e71fcb1f..05e6ea771 100644 --- a/loki/transform/tests/test_transform_hoist_variables.py +++ b/loki/transformations/tests/test_hoist_variables.py @@ -19,7 +19,7 @@ from loki.build import jit_compile_lib, clean_test, Builder from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ( +from loki.transformations.hoist_variables import ( HoistVariablesAnalysis, HoistVariablesTransformation, HoistTemporaryArraysAnalysis, HoistTemporaryArraysTransformationAllocatable ) diff --git a/loki/transform/tests/test_transform_inline.py b/loki/transformations/tests/test_inline.py similarity index 99% rename from loki/transform/tests/test_transform_inline.py rename to loki/transformations/tests/test_inline.py index 19b6db2cc..4fe640247 100644 --- a/loki/transform/tests/test_transform_inline.py +++ b/loki/transformations/tests/test_inline.py @@ -17,12 +17,15 @@ from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI, OFP from loki.ir import nodes as ir, FindNodes -from loki.transform import ( + +from loki.transformations.inline import ( inline_elemental_functions, inline_constant_parameters, - replace_selected_kind, inline_member_procedures, - inline_marked_subroutines, InlineTransformation, - ResolveAssociatesTransformer + inline_member_procedures, inline_marked_subroutines, + InlineTransformation, ) +from loki.transformations.sanitise import ResolveAssociatesTransformer +from loki.transformations.utilities import replace_selected_kind + @pytest.fixture(scope='module', name='here') def fixture_here(): diff --git a/loki/transform/tests/test_transform_parametrise.py b/loki/transformations/tests/test_parametrise.py similarity index 99% rename from loki/transform/tests/test_transform_parametrise.py rename to loki/transformations/tests/test_parametrise.py index a15404a7b..965d212b8 100644 --- a/loki/transform/tests/test_transform_parametrise.py +++ b/loki/transformations/tests/test_parametrise.py @@ -10,7 +10,8 @@ from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ParametriseTransformation + +from loki.transformations.parametrise import ParametriseTransformation @pytest.fixture(scope='module', name='here') diff --git a/transformations/tests/test_pool_allocator.py b/loki/transformations/tests/test_pool_allocator.py similarity index 99% rename from transformations/tests/test_pool_allocator.py rename to loki/transformations/tests/test_pool_allocator.py index 83109fac9..cdb4ad843 100644 --- a/transformations/tests/test_pool_allocator.py +++ b/loki/transformations/tests/test_pool_allocator.py @@ -8,15 +8,19 @@ from shutil import rmtree import pytest -from loki import ( - gettempdir, Scheduler, SchedulerConfig, Dimension, simplify, - FindNodes, FindVariables, normalize_range_indexing, OMNI, FP, OFP, get_pragma_parameters, - CallStatement, Assignment, Allocation, Deallocation, Loop, InlineCall, Pragma, - FindInlineCalls, SFilter, ProcedureItem + +from loki import gettempdir, Dimension, normalize_range_indexing +from loki.batch import Scheduler, SchedulerConfig, SFilter, ProcedureItem +from loki.expression import ( + FindVariables, FindInlineCalls, InlineCall, simplify +) +from loki.frontend import available_frontends, OMNI, FP, OFP +from loki.ir import ( + FindNodes, CallStatement, Assignment, Allocation, Deallocation, + Loop, Pragma, get_pragma_parameters ) -from conftest import available_frontends -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation @pytest.fixture(scope='module', name='block_dim') diff --git a/loki/transform/tests/test_transform_remove_code.py b/loki/transformations/tests/test_remove_code.py similarity index 99% rename from loki/transform/tests/test_transform_remove_code.py rename to loki/transformations/tests/test_remove_code.py index 3e16a0d70..0374d2b83 100644 --- a/loki/transform/tests/test_transform_remove_code.py +++ b/loki/transformations/tests/test_remove_code.py @@ -12,7 +12,8 @@ from loki.batch import Scheduler, SchedulerConfig from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ( + +from loki.transformations.remove_code import ( do_remove_dead_code, do_remove_marked_regions, do_remove_calls, RemoveCodeTransformation ) diff --git a/loki/transform/tests/test_transform_sanitise.py b/loki/transformations/tests/test_sanitise.py similarity index 99% rename from loki/transform/tests/test_transform_sanitise.py rename to loki/transformations/tests/test_sanitise.py index b396fff21..e3fa06d8e 100644 --- a/loki/transform/tests/test_transform_sanitise.py +++ b/loki/transformations/tests/test_sanitise.py @@ -12,7 +12,8 @@ ) from loki.frontend import available_frontends, OMNI from loki.ir import Assignment, Associate, CallStatement, Conditional -from loki.transform import ( + +from loki.transformations.sanitise import ( resolve_associates, transform_sequence_association, SanitiseTransformation ) diff --git a/transformations/tests/test_scc_cuf.py b/loki/transformations/tests/test_scc_cuf.py similarity index 96% rename from transformations/tests/test_scc_cuf.py rename to loki/transformations/tests/test_scc_cuf.py index fabbd51d4..1dac545d7 100644 --- a/transformations/tests/test_scc_cuf.py +++ b/loki/transformations/tests/test_scc_cuf.py @@ -8,17 +8,19 @@ from pathlib import Path import pytest -from conftest import available_frontends -from loki import ( - Scheduler, Subroutine, Dimension, FindNodes, Loop, Assignment, - CallStatement, Allocation, Deallocation, VariableDeclaration, Import, FindVariables, - Pragma +from loki import Scheduler, Subroutine, Dimension +from loki.expression import symbols as sym, FindVariables +from loki.frontend import available_frontends +from loki.ir import ( + FindNodes, Loop, Assignment, CallStatement, Allocation, + Deallocation, VariableDeclaration, Import, Pragma ) -from loki.transform import HoistTemporaryArraysAnalysis, ParametriseTransformation -from loki.expression import symbols as sym -from transformations import ( - SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, - HoistTemporaryArraysPragmaOffloadTransformation + +from loki.transformations.parametrise import ParametriseTransformation +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis +from loki.transformations.single_column import ( + SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, + HoistTemporaryArraysPragmaOffloadTransformation ) diff --git a/transformations/tests/test_transform_derived_types.py b/loki/transformations/tests/test_transform_derived_types.py similarity index 99% rename from transformations/tests/test_transform_derived_types.py rename to loki/transformations/tests/test_transform_derived_types.py index 8cd21899e..168884053 100644 --- a/transformations/tests/test_transform_derived_types.py +++ b/loki/transformations/tests/test_transform_derived_types.py @@ -11,16 +11,18 @@ import pytest from loki import ( - OMNI, OFP, Sourcefile, Scheduler, ProcedureItem, as_tuple, gettempdir, - CallStatement, ProcedureDeclaration, Scalar, Array, - FindNodes, FindVariables, FindInlineCalls, BasicType, - CaseInsensitiveDict, resolve_associates + Sourcefile, Scheduler, ProcedureItem, as_tuple, gettempdir, + ProcedureDeclaration, BasicType, CaseInsensitiveDict, ) -from conftest import available_frontends -from transformations import ( +from loki.expression import Scalar, Array, FindVariables, FindInlineCalls +from loki.frontend import available_frontends, OMNI, OFP +from loki.ir import FindNodes, CallStatement + +from loki.transformations.transform_derived_types import ( DerivedTypeArgumentsTransformation, TypeboundProcedureCallTransformation ) +from loki.transformations.sanitise import resolve_associates #pylint: disable=too-many-lines @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/tests/test_transform_loop.py b/loki/transformations/tests/test_transform_loop.py similarity index 99% rename from loki/transform/tests/test_transform_loop.py rename to loki/transformations/tests/test_transform_loop.py index 2ad3a49ee..5cf847d41 100644 --- a/loki/transform/tests/test_transform_loop.py +++ b/loki/transformations/tests/test_transform_loop.py @@ -18,8 +18,10 @@ is_loki_pragma, pragmas_attached, FindNodes, Loop, Conditional, Assignment ) -from loki.transform import ( - loop_interchange, loop_fusion, loop_fission, normalize_range_indexing + +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.transform_loop import ( + loop_interchange, loop_fusion, loop_fission ) diff --git a/loki/transform/tests/test_transform_region.py b/loki/transformations/tests/test_transform_region.py similarity index 99% rename from loki/transform/tests/test_transform_region.py rename to loki/transformations/tests/test_transform_region.py index ceb19ca2f..5320a730a 100644 --- a/loki/transform/tests/test_transform_region.py +++ b/loki/transformations/tests/test_transform_region.py @@ -16,7 +16,11 @@ from loki.build import jit_compile, jit_compile_lib, clean_test, Builder from loki.expression import symbols as sym from loki.frontend import available_frontends -from loki.transform import region_hoist, region_to_call, normalize_range_indexing + +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.transform_region import ( + region_hoist, region_to_call +) @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/tests/test_transform_utilities.py b/loki/transformations/tests/test_utilities.py similarity index 99% rename from loki/transform/tests/test_transform_utilities.py rename to loki/transformations/tests/test_utilities.py index 73d99c659..4740882c2 100644 --- a/loki/transform/tests/test_transform_utilities.py +++ b/loki/transformations/tests/test_utilities.py @@ -13,7 +13,8 @@ ) from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI -from loki.transform import ( + +from loki.transformations.utilities import ( single_variable_declaration, recursive_expression_map_update, convert_to_lower_case, replace_intrinsics, rename_variables ) diff --git a/transformations/transformations/derived_types.py b/loki/transformations/transform_derived_types.py similarity index 98% rename from transformations/transformations/derived_types.py rename to loki/transformations/transform_derived_types.py index 9f11cfc7c..930851361 100644 --- a/transformations/transformations/derived_types.py +++ b/loki/transformations/transform_derived_types.py @@ -15,12 +15,21 @@ """ from collections import defaultdict -from loki import ( - Transformation, FindVariables, FindNodes, FindInlineCalls, Transformer, - SubstituteExpressions, SubstituteExpressionsMapper, ExpressionRetriever, recursive_expression_map_update, - Module, Import, CallStatement, ProcedureDeclaration, InlineCall, Variable, RangeIndex, - BasicType, DerivedType, as_tuple, flatten, warning, debug, CaseInsensitiveDict, ProcedureType + +from loki.batch import Transformation +from loki.expression import ( + FindInlineCalls, FindVariables, InlineCall, Variable, RangeIndex, + ExpressionRetriever, SubstituteExpressionsMapper, SubstituteExpressions +) +from loki.ir import ( + FindNodes, Import, CallStatement, ProcedureDeclaration, Transformer ) +from loki.logging import warning, debug +from loki.module import Module +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import BasicType, DerivedType, ProcedureType + +from loki.transformations.utilities import recursive_expression_map_update __all__ = ['DerivedTypeArgumentsTransformation', 'TypeboundProcedureCallTransformation'] diff --git a/loki/transform/transform_loop.py b/loki/transformations/transform_loop.py similarity index 99% rename from loki/transform/transform_loop.py rename to loki/transformations/transform_loop.py index 35774d25b..2203be58a 100644 --- a/loki/transform/transform_loop.py +++ b/loki/transformations/transform_loop.py @@ -13,6 +13,10 @@ import operator as op import numpy as np +from loki.analyse import ( + dataflow_analysis_attached, read_after_write_vars, + loop_carried_dependencies +) from loki.expression import ( symbols as sym, SubstituteExpressions, FindVariables, simplify, is_constant, symbolic_op, parse_expr @@ -23,16 +27,15 @@ get_pragma_parameters, pragmas_attached ) from loki.logging import info, warning -from loki.transform.transform_array_indexing import ( - promotion_dimensions_from_loop_nest, promote_nonmatching_variables -) from loki.tools import ( - flatten, as_tuple, CaseInsensitiveDict, binary_insertion_sort, optional + flatten, as_tuple, CaseInsensitiveDict, binary_insertion_sort, + optional ) -from loki.analyse import ( - dataflow_analysis_attached, read_after_write_vars, loop_carried_dependencies +from loki.transformations.array_indexing import ( + promotion_dimensions_from_loop_nest, promote_nonmatching_variables ) + __all__ = ['loop_interchange', 'loop_fusion', 'loop_fission'] diff --git a/loki/transform/transform_region.py b/loki/transformations/transform_region.py similarity index 99% rename from loki/transform/transform_region.py rename to loki/transformations/transform_region.py index 77bf32238..a51bef20d 100644 --- a/loki/transform/transform_region.py +++ b/loki/transformations/transform_region.py @@ -22,7 +22,8 @@ from loki.logging import info from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten, CaseInsensitiveDict -from loki.transform.transform_array_indexing import ( + +from loki.transformations.array_indexing import ( promotion_dimensions_from_loop_nest, promote_nonmatching_variables ) diff --git a/loki/transformations/transpile/__init__.py b/loki/transformations/transpile/__init__.py new file mode 100644 index 000000000..688c2efe8 --- /dev/null +++ b/loki/transformations/transpile/__init__.py @@ -0,0 +1,10 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.transpile.fortran_c import * # noqa +from loki.transformations.transpile.fortran_python import * # noqa +from loki.transformations.transpile.fortran_maxeler import * # noqa diff --git a/loki/transform/fortran_c_transform.py b/loki/transformations/transpile/fortran_c.py similarity index 98% rename from loki/transform/fortran_c_transform.py rename to loki/transformations/transpile/fortran_c.py index 455f6c4f2..4bdb270c1 100644 --- a/loki/transform/fortran_c_transform.py +++ b/loki/transformations/transpile/fortran_c.py @@ -8,36 +8,37 @@ from pathlib import Path from collections import OrderedDict -from loki.transform.transformation import Transformation -from loki.transform.transform_array_indexing import ( +from loki.backend import cgen, fgen +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, Variable, InlineCall, RangeIndex, Scalar, Array, + ProcedureSymbol, SubstituteExpressions, Dereference, Reference, + ExpressionRetriever, SubstituteExpressionsMapper, +) +from loki.ir import ( + Section, Import, Intrinsic, Interface, CallStatement, + VariableDeclaration, TypeDef, Assignment, Transformer, FindNodes +) +from loki.logging import debug +from loki.module import Module +from loki.sourcefile import Sourcefile +from loki.subroutine import Subroutine +from loki.tools import as_tuple, flatten +from loki.types import BasicType, DerivedType, SymbolAttributes + +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_array_shape_and_access, flatten_arrays ) -from loki.transform.transform_sanitise import resolve_associates -from loki.transform.transform_utilities import ( +from loki.transformations.utilities import ( convert_to_lower_case, replace_intrinsics, sanitise_imports ) -from loki.transform.transform_inline import ( +from loki.transformations.sanitise import resolve_associates +from loki.transformations.inline import ( inline_constant_parameters, inline_elemental_functions ) -from loki.sourcefile import Sourcefile -from loki.backend import cgen, fgen -from loki.logging import debug -from loki.ir import ( - Section, Import, Intrinsic, Interface, CallStatement, VariableDeclaration, - TypeDef, Assignment, Transformer, FindNodes -) -from loki.subroutine import Subroutine -from loki.module import Module -from loki.expression import ( - Variable, InlineCall, RangeIndex, Scalar, Array, - ProcedureSymbol, SubstituteExpressions, Dereference, Reference, - ExpressionRetriever, SubstituteExpressionsMapper, -) -from loki.expression import symbols as sym -from loki.tools import as_tuple, flatten -from loki.types import BasicType, DerivedType, SymbolAttributes + __all__ = ['FortranCTransformation'] diff --git a/loki/transform/fortran_max_transform.py b/loki/transformations/transpile/fortran_maxeler.py similarity index 99% rename from loki/transform/fortran_max_transform.py rename to loki/transformations/transpile/fortran_maxeler.py index 5914d78be..9b75ba9cb 100644 --- a/loki/transform/fortran_max_transform.py +++ b/loki/transformations/transpile/fortran_maxeler.py @@ -10,6 +10,7 @@ from hashlib import sha256 from loki.backend import maxjgen, fgen, cgen +from loki.batch import Transformation from loki.expression import ( FindVariables, SubstituteExpressions, ExpressionCallbackMapper, SubstituteExpressionsMapper, ExpressionRetriever, symbols as sym @@ -21,15 +22,16 @@ from loki.sourcefile import Sourcefile from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten -from loki.transform.fortran_c_transform import FortranCTransformation -from loki.transform.transformation import Transformation -from loki.transform.transform_array_indexing import ( + +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_range_indexing ) -from loki.transform.transform_utilities import replace_intrinsics +from loki.transformations.utilities import replace_intrinsics from loki.types import SymbolAttributes, BasicType, DerivedType +from loki.transformations.transpile.fortran_c import FortranCTransformation + __all__ = ['FortranMaxTransformation'] diff --git a/loki/transform/fortran_python_transform.py b/loki/transformations/transpile/fortran_python.py similarity index 94% rename from loki/transform/fortran_python_transform.py rename to loki/transformations/transpile/fortran_python.py index 4f015d4d7..f2a769a67 100644 --- a/loki/transform/fortran_python_transform.py +++ b/loki/transformations/transpile/fortran_python.py @@ -8,17 +8,18 @@ from pathlib import Path from loki.backend import pygen, dacegen +from loki.batch import Transformation from loki.expression import ( symbols as sym, FindInlineCalls, SubstituteExpressions ) from loki.ir import nodes as ir, FindNodes, Transformer, pragmas_attached from loki.sourcefile import Sourcefile -from loki.transform.transformation import Transformation -from loki.transform.transform_array_indexing import ( + +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, normalize_range_indexing ) -from loki.transform.transform_sanitise import resolve_associates -from loki.transform.transform_utilities import ( +from loki.transformations.sanitise import resolve_associates +from loki.transformations.utilities import ( convert_to_lower_case, replace_intrinsics ) diff --git a/loki/tests/test_maxeler/passthrough/PassThroughCpuCode.c b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughCpuCode.c similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughCpuCode.c rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughCpuCode.c diff --git a/loki/tests/test_maxeler/passthrough/PassThroughKernel.maxj b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughKernel.maxj similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughKernel.maxj rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughKernel.maxj diff --git a/loki/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj diff --git a/loki/tests/test_maxeler/test_maxeler.py b/loki/transformations/transpile/tests/test_maxeler/test_maxeler.py similarity index 100% rename from loki/tests/test_maxeler/test_maxeler.py rename to loki/transformations/transpile/tests/test_maxeler/test_maxeler.py diff --git a/loki/backend/tests/test_sdfg.py b/loki/transformations/transpile/tests/test_sdfg.py similarity index 99% rename from loki/backend/tests/test_sdfg.py rename to loki/transformations/transpile/tests/test_sdfg.py index ceb28954b..97767ab18 100644 --- a/loki/backend/tests/test_sdfg.py +++ b/loki/transformations/transpile/tests/test_sdfg.py @@ -14,7 +14,8 @@ from loki import Subroutine from loki.build import jit_compile, clean_test from loki.frontend import available_frontends -from loki.transform import FortranPythonTransformation + +from loki.transformations.transpile import FortranPythonTransformation pytestmark = [ diff --git a/loki/tests/test_transpile.py b/loki/transformations/transpile/tests/test_transpile.py similarity index 99% rename from loki/tests/test_transpile.py rename to loki/transformations/transpile/tests/test_transpile.py index 5926ce0c0..563b3085a 100644 --- a/loki/tests/test_transpile.py +++ b/loki/transformations/transpile/tests/test_transpile.py @@ -9,12 +9,14 @@ import pytest import numpy as np -from loki import Subroutine, Module, FortranCTransformation, cgen +from loki import Subroutine, Module, cgen from loki.build import jit_compile, jit_compile_lib, clean_test, Builder import loki.expression.symbols as sym from loki.frontend import available_frontends, OFP import loki.ir as ir -from loki.transform import normalize_range_indexing + +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.transpile import FortranCTransformation @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/transform_utilities.py b/loki/transformations/utilities.py similarity index 100% rename from loki/transform/transform_utilities.py rename to loki/transformations/utilities.py diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index e7bcc9deb..7ac07721c 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -16,34 +16,35 @@ import click from loki import ( - Sourcefile, Transformation, Scheduler, SchedulerConfig, - Frontend, as_tuple, set_excepthook, auto_post_mortem_debugger, info + Sourcefile, Frontend, as_tuple, set_excepthook, + auto_post_mortem_debugger, info ) +from loki.batch import Transformation, Pipeline, Scheduler, SchedulerConfig # Get generalized transformations provided by Loki -from loki.transform import ( - DependencyTransformation, ModuleWrapTransformation, FortranCTransformation, - FileWriteTransformation, HoistTemporaryArraysAnalysis, normalize_range_indexing, - InlineTransformation, SanitiseTransformation, RemoveCodeTransformation, Pipeline -) - -# pylint: disable=wrong-import-order -from transformations.argument_shape import ( +from loki.transformations.argument_shape import ( ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation ) -from transformations.data_offload import ( - DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.build_system import ( + DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation ) -from transformations.derived_types import DerivedTypeArgumentsTransformation -from transformations.drhook import DrHookTransformation -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from transformations.single_column_claw import ExtractSCATransformation, CLAWTransformation -from transformations.single_column_coalesced import ( - SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline +from loki.transformations.data_offload import ( + DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation ) -from transformations.scc_cuf import ( +from loki.transformations.transform_derived_types import DerivedTypeArgumentsTransformation +from loki.transformations.drhook import DrHookTransformation +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis +from loki.transformations.inline import InlineTransformation +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.remove_code import RemoveCodeTransformation +from loki.transformations.sanitise import SanitiseTransformation +from loki.transformations.single_column import ( + ExtractSCATransformation, CLAWTransformation, + SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline, HoistTemporaryArraysDeviceAllocatableTransformation ) +from loki.transformations.transpile import FortranCTransformation class IdemTransformation(Transformation): diff --git a/transformations/pyproject.toml b/transformations/pyproject.toml deleted file mode 100644 index b4fd57c95..000000000 --- a/transformations/pyproject.toml +++ /dev/null @@ -1,31 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 61", - "setuptools_scm[toml] >= 6.2", -] -build-backend = "setuptools.build_meta" - -[project] -name = "transformations" -authors = [ - {name = "ECMWF", email = "user_support_section@ecmwf.int"}, -] -description = "Bespoke IFS transformation implementations for loki" -requires-python = ">=3.8" -license = {text = "Apache-2.0"} -dynamic = ["version"] -dependencies = ["loki"] - -[tool.setuptools] -license-files = ["../LICENSE", "../AUTHORS.md"] -packages = ["transformations"] - -# Enable SCM versioning -[tool.setuptools_scm] -root = ".." -relative_to = "__file__" - -[tool.pytest.ini_options] -testpaths = [ - "tests" -] diff --git a/transformations/tests/conftest.py b/transformations/tests/conftest.py deleted file mode 100644 index 6c10cb7e7..000000000 --- a/transformations/tests/conftest.py +++ /dev/null @@ -1,109 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from pathlib import Path -import shutil -import pytest - -from loki import as_tuple, Frontend -import loki.frontend - -__all__ = ['available_frontends', 'local_loki_setup', 'local_loki_cleanup'] - -def available_frontends(xfail=None, skip=None): - """ - Provide list of available frontends to parametrize tests with - - To run tests for every frontend, an argument :attr:`frontend` can be added to - a test with the return value of this function as parameter. - - For any unavailable frontends where ``HAVE_`` is `False` (e.g. - because required dependencies are not installed), :attr:`test` is marked as - skipped. - - Use as - - .. code-block:: - - @pytest.mark.parametrize('frontend', available_frontends(xfail=[OMNI, (OFP, 'Because...')])) - def my_test(frontend): - source = Sourcefile.from_file('some.F90', frontend=frontend) - # ... - - Parameters - ---------- - xfail : list, optional - Provide frontends that are expected to fail, optionally as tuple with reason - provided as string. By default `None` - skip : list, optional - Provide frontends that are always skipped, optionally as tuple with reason - provided as string. By default `None` - """ - if xfail: - xfail = dict((as_tuple(f) + (None,))[:2] for f in xfail) - else: - xfail = {} - - if skip: - skip = dict((as_tuple(f) + (None,))[:2] for f in skip) - else: - skip = {} - - # Unavailable frontends - unavailable_frontends = { - f: f'{f} is not available' for f in Frontend - if not getattr(loki.frontend, f'HAVE_{str(f).upper()}') - } - skip.update(unavailable_frontends) - - # Build the list of parameters - params = [] - for f in Frontend: - if f in skip: - params += [pytest.param(f, marks=pytest.mark.skip(reason=skip[f]))] - elif f in xfail: - params += [pytest.param(f, marks=pytest.mark.xfail(reason=xfail[f]))] - elif f != Frontend.REGEX: - params += [f] - - return params - - -def write_env_launch_script(here, binary, args): - # Write a script to source env.sh and launch the binary - script = Path(here/f'build/run_{binary}.sh') - script.write_text(f""" -#!/bin/bash - -source env.sh >&2 -bin/{binary} {' '.join(args)} -exit $? - """.strip()) - script.chmod(0o750) - - return script - - -def local_loki_setup(here): - lokidir = Path(__file__).parent.parent.parent - target = here/'source/loki' - backup = here/'source/loki.bak' - - # Do not overwrite any existing Loki copy - if target.exists(): - if backup.exists(): - shutil.rmtree(backup) - shutil.move(target, backup) - - return str(lokidir.resolve()), target, backup - - -def local_loki_cleanup(target, backup): - if target.is_symlink(): - target.unlink() - if not target.exists() and backup.exists(): - shutil.move(backup, target) diff --git a/transformations/transformations/__init__.py b/transformations/transformations/__init__.py deleted file mode 100644 index 82628904e..000000000 --- a/transformations/transformations/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from importlib.metadata import version, PackageNotFoundError - -from transformations.derived_types import * # noqa -from transformations.argument_shape import * # noqa -from transformations.data_offload import * # noqa -from transformations.single_column_annotate import * # noqa -from transformations.single_column_base import * # noqa -from transformations.single_column_claw import * # noqa -from transformations.single_column_coalesced import * # noqa -from transformations.single_column_coalesced_vector import * # noqa -from transformations.single_column_hoist import * # noqa -from transformations.drhook import * # noqa -from transformations.scc_cuf import * # noqa -from transformations.pool_allocator import * # noqa - -try: - __version__ = version("transformations") -except PackageNotFoundError: - # package is not installed - pass