From 7d649635be7dd0af3d997dfc787633cca6894e3b Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Tue, 16 Apr 2024 13:32:15 +0000 Subject: [PATCH 01/13] Batch: Move `Transformation` and `Pipeline` into `loki.batch` --- loki/batch/__init__.py | 2 + loki/{transform => batch}/pipeline.py | 2 +- loki/batch/scheduler.py | 9 ++-- loki/batch/tests/test_scheduler.py | 8 ++-- loki/{ => batch}/tests/test_transformation.py | 43 ++++++++++--------- loki/{transform => batch}/transformation.py | 0 loki/lint/linter.py | 3 +- loki/transform/__init__.py | 2 - loki/transform/build_system_transform.py | 4 +- loki/transform/dependency_transform.py | 2 +- loki/transform/fortran_c_transform.py | 2 +- loki/transform/fortran_max_transform.py | 3 +- loki/transform/fortran_python_transform.py | 3 +- loki/transform/transform_hoist_variables.py | 17 +++++--- loki/transform/transform_inline.py | 2 +- loki/transform/transform_parametrise.py | 4 +- loki/transform/transform_remove_code.py | 2 +- loki/transform/transform_sanitise.py | 2 +- scripts/loki_transform.py | 7 +-- .../single_column_coalesced.py | 3 +- 20 files changed, 64 insertions(+), 56 deletions(-) rename loki/{transform => batch}/pipeline.py (98%) rename loki/{ => batch}/tests/test_transformation.py (93%) rename loki/{transform => batch}/transformation.py (100%) diff --git a/loki/batch/__init__.py b/loki/batch/__init__.py index 0ecfc925a..56f454cc6 100644 --- a/loki/batch/__init__.py +++ b/loki/batch/__init__.py @@ -7,6 +7,8 @@ from loki.batch.configure import * # noqa from loki.batch.item import * # noqa +from loki.batch.pipeline import * # noqa from loki.batch.scheduler import * # noqa from loki.batch.sfilter import * # noqa from loki.batch.sgraph import * # noqa +from loki.batch.transformation import * # noqa diff --git a/loki/transform/pipeline.py b/loki/batch/pipeline.py similarity index 98% rename from loki/transform/pipeline.py rename to loki/batch/pipeline.py index 72d94baed..44f0d57ea 100644 --- a/loki/transform/pipeline.py +++ b/loki/batch/pipeline.py @@ -7,8 +7,8 @@ from inspect import signature, Parameter +from loki.batch.transformation import Transformation from loki.tools import as_tuple, flatten -from loki.transform.transformation import Transformation class Pipeline: diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 2d60ebb53..30b8ab27b 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -10,12 +10,15 @@ from codetiming import Timer from loki.batch.configure import SchedulerConfig -from loki.batch.sfilter import SFilter -from loki.batch.sgraph import SGraph from loki.batch.item import ( FileItem, ModuleItem, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, ExternalItem, ItemFactory ) +from loki.batch.pipeline import Pipeline +from loki.batch.sfilter import SFilter +from loki.batch.sgraph import SGraph +from loki.batch.transformation import Transformation + from loki.frontend import FP, REGEX, RegexParserClass from loki.tools import as_tuple, CaseInsensitiveDict, flatten from loki.logging import info, perf, warning, debug, error @@ -391,8 +394,6 @@ def process(self, transformation): transformation : :any:`Transformation` or :any:`Pipeline` The transformation or transformation pipeline to apply """ - from loki.transform import Transformation, Pipeline # pylint: disable=import-outside-toplevel - if isinstance(transformation, Transformation): self.process_transformation(transformation=transformation) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 7ae63cc07..d6c826328 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -58,14 +58,14 @@ import pytest from loki import ( - Sourcefile, Subroutine, Dimension, Pipeline, fexprgen, BasicType, + Sourcefile, Subroutine, Dimension, fexprgen, BasicType, gettempdir, ProcedureType, DerivedType, flatten, as_tuple, CaseInsensitiveDict, graphviz_present ) from loki.batch import ( Scheduler, SchedulerConfig, Item, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, SFilter, - ExternalItem + ExternalItem, Transformation, Pipeline ) from loki.expression import ( Scalar, Array, Literal, ProcedureSymbol, FindInlineCalls @@ -74,9 +74,7 @@ available_frontends, OMNI, OFP, FP, REGEX, HAVE_FP, HAVE_OFP, HAVE_OMNI ) from loki.ir import nodes as ir, FindNodes -from loki.transform import ( - Transformation, DependencyTransformation, ModuleWrapTransformation -) +from loki.transform import DependencyTransformation, ModuleWrapTransformation pytestmark = pytest.mark.skipif(not HAVE_FP and not HAVE_OFP, reason='Fparser and OFP not available') diff --git a/loki/tests/test_transformation.py b/loki/batch/tests/test_transformation.py similarity index 93% rename from loki/tests/test_transformation.py rename to loki/batch/tests/test_transformation.py index 5413d39ca..26a5b4b05 100644 --- a/loki/tests/test_transformation.py +++ b/loki/batch/tests/test_transformation.py @@ -3,13 +3,14 @@ import pytest from loki import ( - Sourcefile, Subroutine, Import, FindNodes, FindInlineCalls, fgen, - Assignment, IntLiteral, Module, ProcedureItem, Comment + Sourcefile, Subroutine, FindInlineCalls, fgen, IntLiteral, Module ) +from loki.batch import Transformation, Pipeline, ProcedureItem from loki.build import jit_compile, clean_test from loki.frontend import available_frontends, OMNI, REGEX +from loki.ir import nodes as ir, FindNodes from loki.transform import ( - Transformation, replace_selected_kind, FileWriteTransformation, Pipeline + replace_selected_kind, FileWriteTransformation ) @@ -28,7 +29,7 @@ class RenameTransform(Transformation): def transform_file(self, sourcefile, **kwargs): sourcefile.ir.prepend( - Comment(text="! [Loki] RenameTransform applied") + ir.Comment(text="! [Loki] RenameTransform applied") ) def transform_subroutine(self, routine, **kwargs): @@ -79,7 +80,7 @@ def test_transformation_apply(rename_transform, frontend, method, lazy): raise ValueError(f'Unknown method "{method}"') assert not source._incomplete - assert isinstance(source.ir.body[0], Comment) + assert isinstance(source.ir.body[0], ir.Comment) assert source.ir.body[0].text == '! [Loki] RenameTransform applied' assert source.modules[0].name == 'mymodule' @@ -246,7 +247,7 @@ def test_transform_replace_selected_kind(here, frontend): """.strip() routine = Subroutine.from_source(fcode, frontend=frontend) - imports = FindNodes(Import).visit(routine.spec) + imports = FindNodes(ir.Import).visit(routine.spec) assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env' assert len(imports[0].symbols) == 1 and imports[0].symbols[0].name.lower() == 'int8' @@ -263,7 +264,7 @@ def test_transform_replace_selected_kind(here, frontend): assert not [call for call in FindInlineCalls().visit(routine.ir) if call.name.lower().startswith('selected')] - imports = FindNodes(Import).visit(routine.spec) + imports = FindNodes(ir.Import).visit(routine.spec) assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env' source = fgen(routine).lower() @@ -307,8 +308,8 @@ def transform_subroutine(self, routine, **kwargs): i = routine.variable_map['i'] j = i.clone(name='j', scope=tmp_routine, type=i.type.clone(intent=None)) routine.variables += (j,) - routine.body.append(Assignment(lhs=j, rhs=IntLiteral(2))) - routine.body.append(Assignment(lhs=i, rhs=j)) + routine.body.append(ir.Assignment(lhs=j, rhs=IntLiteral(2))) + routine.body.append(ir.Assignment(lhs=i, rhs=j)) routine.name += '_transformed' assert routine.variable_map['j'].scope is tmp_routine @@ -363,8 +364,8 @@ def transform_module(self, module, **kwargs): j = i.clone(name='j', scope=tmp_scope, type=i.type.clone(intent=None)) module.variables += (j,) routine = module.subroutines[0] - routine.body.prepend(Assignment(lhs=i, rhs=j)) - routine.body.prepend(Assignment(lhs=j, rhs=IntLiteral(2))) + routine.body.prepend(ir.Assignment(lhs=i, rhs=j)) + routine.body.prepend(ir.Assignment(lhs=j, rhs=IntLiteral(2))) module.name += '_transformed' assert module.variable_map['j'].scope is tmp_scope @@ -461,7 +462,7 @@ def __init__(self, name='Rick', relaxed=False): def transform_subroutine(self, routine, **kwargs): greeting = 'Whazzup' if self.relaxed else 'Hello' - routine.body.prepend(Comment(text=f'! {greeting} {self.name}')) + routine.body.prepend(ir.Comment(text=f'! {greeting} {self.name}')) class AppendTrafo(Transformation): def __init__(self, name='Dave', in_french=False): @@ -470,7 +471,7 @@ def __init__(self, name='Dave', in_french=False): def transform_subroutine(self, routine, **kwargs): greeting = 'Au revoir' if self.in_french else 'Goodbye' - routine.body.append(Comment(text=f'! {greeting}, {self.name}')) + routine.body.append(ir.Comment(text=f'! {greeting}, {self.name}')) # Define a pipline as a combination of transformation classes # and a set pre-defined constructor flags @@ -502,9 +503,9 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipeline.apply(routine) - assert isinstance(routine.body.body[0], Comment) + assert isinstance(routine.body.body[0], ir.Comment) assert routine.body.body[0].text == '! Whazzup Bob' - assert isinstance(routine.body.body[-1], Comment) + assert isinstance(routine.body.body[-1], ir.Comment) assert routine.body.body[-1].text == '! Au revoir, Bob' @@ -583,19 +584,19 @@ def test_transformation_pipeline_compose(): class YesTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Yes !') ) + routine.body.append( ir.Comment(text='! Yes !') ) class NoTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! No !') ) + routine.body.append( ir.Comment(text='! No !') ) class MaybeTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Maybe !') ) + routine.body.append( ir.Comment(text='! Maybe !') ) class MaybeNotTrafo(Transformation): def transform_subroutine(self, routine, **kwargs): - routine.body.append( Comment(text='! Maybe not !') ) + routine.body.append( ir.Comment(text='! Maybe not !') ) pipeline = Pipeline(classes=(YesTrafo, NoTrafo)) pipeline.prepend(MaybeTrafo()) @@ -604,7 +605,7 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipeline.apply(routine) - comments = FindNodes(Comment).visit(routine.body) + comments = FindNodes(ir.Comment).visit(routine.body) assert len(comments) == 4 assert comments[0].text == '! Maybe !' assert comments[1].text == '! Yes !' @@ -622,7 +623,7 @@ def transform_subroutine(self, routine, **kwargs): routine = Subroutine.from_source(fcode) pipe.apply(routine) - comments = FindNodes(Comment).visit(routine.body) + comments = FindNodes(ir.Comment).visit(routine.body) assert len(comments) == 5 assert comments[0].text == '! Yes !' assert comments[1].text == '! Maybe !' diff --git a/loki/transform/transformation.py b/loki/batch/transformation.py similarity index 100% rename from loki/transform/transformation.py rename to loki/batch/transformation.py diff --git a/loki/lint/linter.py b/loki/lint/linter.py index 58f27fbfa..b5e486866 100644 --- a/loki/lint/linter.py +++ b/loki/lint/linter.py @@ -17,7 +17,7 @@ from codetiming import Timer from loki.build import workqueue -from loki.batch import Scheduler, SchedulerConfig, Item +from loki.batch import Scheduler, SchedulerConfig, Item, Transformation from loki.config import config as loki_config from loki.lint.reporter import ( FileReport, RuleReport, Reporter, LazyTextfile, @@ -27,7 +27,6 @@ from loki.logging import logger from loki.sourcefile import Sourcefile from loki.tools import filehash, find_paths, CaseInsensitiveDict -from loki.transform import Transformation __all__ = ['Linter', 'LinterTransformation', 'lint_files'] diff --git a/loki/transform/__init__.py b/loki/transform/__init__.py index 4a3a394ff..18d3b624d 100644 --- a/loki/transform/__init__.py +++ b/loki/transform/__init__.py @@ -5,7 +5,6 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from loki.transform.transformation import * # noqa from loki.transform.transform_utilities import * # noqa from loki.transform.transform_array_indexing import * # noqa from loki.transform.transform_inline import * # noqa @@ -21,4 +20,3 @@ from loki.transform.transform_extract_contained_procedures import * # noqa from loki.transform.transform_remove_code import * # noqa from loki.transform.transform_sanitise import * # noqa -from loki.transform.pipeline import * # noqa diff --git a/loki/transform/build_system_transform.py b/loki/transform/build_system_transform.py index 158839bf6..7dc406a15 100644 --- a/loki/transform/build_system_transform.py +++ b/loki/transform/build_system_transform.py @@ -11,8 +11,8 @@ from pathlib import Path -from loki.transform.transformation import Transformation -from loki.batch.item import ProcedureItem, ModuleItem#, GlobalVariableItem +from loki.batch import Transformation, ProcedureItem, ModuleItem + __all__ = ['FileWriteTransformation'] diff --git a/loki/transform/dependency_transform.py b/loki/transform/dependency_transform.py index 4be2213d8..735c7f822 100644 --- a/loki/transform/dependency_transform.py +++ b/loki/transform/dependency_transform.py @@ -8,6 +8,7 @@ from pathlib import Path from loki.backend import fgen +from loki.batch import Transformation from loki.expression import Variable, FindInlineCalls from loki.ir import ( CallStatement, Import, Section, Interface, FindNodes, Transformer @@ -18,7 +19,6 @@ from loki.subroutine import Subroutine from loki.types import ProcedureType from loki.tools import as_tuple -from loki.transform.transformation import Transformation __all__ = ['DependencyTransformation', 'ModuleWrapTransformation'] diff --git a/loki/transform/fortran_c_transform.py b/loki/transform/fortran_c_transform.py index 455f6c4f2..b34cdc0e7 100644 --- a/loki/transform/fortran_c_transform.py +++ b/loki/transform/fortran_c_transform.py @@ -8,7 +8,7 @@ from pathlib import Path from collections import OrderedDict -from loki.transform.transformation import Transformation +from loki.batch import Transformation from loki.transform.transform_array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_array_shape_and_access, diff --git a/loki/transform/fortran_max_transform.py b/loki/transform/fortran_max_transform.py index 5914d78be..663f70d77 100644 --- a/loki/transform/fortran_max_transform.py +++ b/loki/transform/fortran_max_transform.py @@ -10,6 +10,7 @@ from hashlib import sha256 from loki.backend import maxjgen, fgen, cgen +from loki.batch import Transformation from loki.expression import ( FindVariables, SubstituteExpressions, ExpressionCallbackMapper, SubstituteExpressionsMapper, ExpressionRetriever, symbols as sym @@ -21,8 +22,8 @@ from loki.sourcefile import Sourcefile from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten + from loki.transform.fortran_c_transform import FortranCTransformation -from loki.transform.transformation import Transformation from loki.transform.transform_array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_range_indexing diff --git a/loki/transform/fortran_python_transform.py b/loki/transform/fortran_python_transform.py index 4f015d4d7..cfbc58b33 100644 --- a/loki/transform/fortran_python_transform.py +++ b/loki/transform/fortran_python_transform.py @@ -8,12 +8,13 @@ from pathlib import Path from loki.backend import pygen, dacegen +from loki.batch import Transformation from loki.expression import ( symbols as sym, FindInlineCalls, SubstituteExpressions ) from loki.ir import nodes as ir, FindNodes, Transformer, pragmas_attached from loki.sourcefile import Sourcefile -from loki.transform.transformation import Transformation + from loki.transform.transform_array_indexing import ( shift_to_zero_indexing, invert_array_indices, normalize_range_indexing ) diff --git a/loki/transform/transform_hoist_variables.py b/loki/transform/transform_hoist_variables.py index 23dafca5b..7c0770d18 100644 --- a/loki/transform/transform_hoist_variables.py +++ b/loki/transform/transform_hoist_variables.py @@ -79,21 +79,24 @@ scheduler.process(transformation=HoistTemporaryArraysAnalysis(dim_vars=('a',), key=key)) scheduler.process(transformation=HoistTemporaryArraysTransformationAllocatable(key=key)) """ + +from loki.batch import Transformation, ProcedureItem from loki.expression import FindVariables, SubstituteExpressions +from loki.expression.expr_visitors import FindInlineCalls +import loki.expression.symbols as sym +from loki.expression.symbolic import is_dimension_constant from loki.ir import ( CallStatement, Allocation, Deallocation, Transformer, FindNodes ) from loki.tools.util import is_iterable, as_tuple, CaseInsensitiveDict -from loki.transform.transformation import Transformation + from loki.transform.transform_utilities import single_variable_declaration -from loki.batch.item import ProcedureItem -from loki.expression.expr_visitors import FindInlineCalls -import loki.expression.symbols as sym -from loki.expression.symbolic import is_dimension_constant -__all__ = ['HoistVariablesAnalysis', 'HoistVariablesTransformation', - 'HoistTemporaryArraysAnalysis', 'HoistTemporaryArraysTransformationAllocatable'] +__all__ = [ + 'HoistVariablesAnalysis', 'HoistVariablesTransformation', + 'HoistTemporaryArraysAnalysis', 'HoistTemporaryArraysTransformationAllocatable' +] class HoistVariablesAnalysis(Transformation): diff --git a/loki/transform/transform_inline.py b/loki/transform/transform_inline.py index cb8bbfaee..a15621d4b 100644 --- a/loki/transform/transform_inline.py +++ b/loki/transform/transform_inline.py @@ -12,6 +12,7 @@ """ from collections import defaultdict +from loki.batch import Transformation from loki.expression import ( FindVariables, FindInlineCalls, FindLiterals, SubstituteExpressions, LokiIdentityMapper @@ -27,7 +28,6 @@ from loki.subroutine import Subroutine from loki.transform.transform_sanitise import transform_sequence_association_append_map -from loki.transform.transformation import Transformation from loki.transform.transform_remove_code import do_remove_dead_code from loki.transform.transform_utilities import ( single_variable_declaration, diff --git a/loki/transform/transform_parametrise.py b/loki/transform/transform_parametrise.py index d88910ca3..4ebc4982a 100644 --- a/loki/transform/transform_parametrise.py +++ b/loki/transform/transform_parametrise.py @@ -81,10 +81,12 @@ dic2p = {'a': 10} scheduler.process(transformation=ParametriseTransformation(dic2p=dic2p, replace_by_value=True)) """ + +from loki.batch import Transformation from loki.expression import symbols as sym from loki.ir import nodes as ir, Transformer, FindNodes from loki.tools.util import as_tuple, CaseInsensitiveDict -from loki.transform.transformation import Transformation + from loki.transform.transform_inline import inline_constant_parameters diff --git a/loki/transform/transform_remove_code.py b/loki/transform/transform_remove_code.py index 3fe5dc4e1..bd0b42e74 100644 --- a/loki/transform/transform_remove_code.py +++ b/loki/transform/transform_remove_code.py @@ -10,11 +10,11 @@ section and to perform Dead Code Elimination. """ +from loki.batch import Transformation from loki.expression.symbolic import simplify from loki.tools import flatten, as_tuple from loki.ir import Conditional, Transformer, Comment from loki.ir.pragma_utils import is_loki_pragma, pragma_regions_attached -from loki.transform.transformation import Transformation __all__ = [ diff --git a/loki/transform/transform_sanitise.py b/loki/transform/transform_sanitise.py index 59891ca46..3268291ae 100644 --- a/loki/transform/transform_sanitise.py +++ b/loki/transform/transform_sanitise.py @@ -12,13 +12,13 @@ code easier. """ +from loki.batch import Transformation from loki.expression import FindVariables, SubstituteExpressions, Array, RangeIndex from loki.ir import CallStatement, FindNodes, Transformer, NestedTransformer from loki.tools import as_tuple, CaseInsensitiveDict from loki.types import BasicType from loki.transform.transform_utilities import recursive_expression_map_update -from loki.transform.transformation import Transformation __all__ = [ diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index e7bcc9deb..3d978d51f 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -16,15 +16,16 @@ import click from loki import ( - Sourcefile, Transformation, Scheduler, SchedulerConfig, - Frontend, as_tuple, set_excepthook, auto_post_mortem_debugger, info + Sourcefile, Frontend, as_tuple, set_excepthook, + auto_post_mortem_debugger, info ) +from loki.batch import Transformation, Pipeline, Scheduler, SchedulerConfig # Get generalized transformations provided by Loki from loki.transform import ( DependencyTransformation, ModuleWrapTransformation, FortranCTransformation, FileWriteTransformation, HoistTemporaryArraysAnalysis, normalize_range_indexing, - InlineTransformation, SanitiseTransformation, RemoveCodeTransformation, Pipeline + InlineTransformation, SanitiseTransformation, RemoveCodeTransformation ) # pylint: disable=wrong-import-order diff --git a/transformations/transformations/single_column_coalesced.py b/transformations/transformations/single_column_coalesced.py index 60f571232..fd9bbd220 100644 --- a/transformations/transformations/single_column_coalesced.py +++ b/transformations/transformations/single_column_coalesced.py @@ -7,7 +7,8 @@ from functools import partial -from loki.transform import Pipeline, HoistTemporaryArraysAnalysis +from loki.batch import Pipeline +from loki.transform import HoistTemporaryArraysAnalysis from transformations.pool_allocator import TemporariesPoolAllocatorTransformation from transformations.single_column_base import SCCBaseTransformation from transformations.single_column_annotate import SCCAnnotateTransformation From 294c246895f90038c4da844d6468ede931eaa546 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Tue, 16 Apr 2024 14:07:44 +0000 Subject: [PATCH 02/13] Transformations: Moving the `transformations` package inside loki --- loki/__init__.py | 1 + loki/transformations/__init__.py | 19 ++++++++++++ .../transformations/argument_shape.py | 0 .../transformations/data_offload.py | 0 .../transformations/derived_types.py | 0 .../transformations/drhook.py | 8 +++-- .../transformations/pool_allocator.py | 0 .../transformations/scc_cuf.py | 4 +-- .../transformations/single_column_annotate.py | 21 ++++++++----- .../transformations/single_column_base.py | 0 .../transformations/single_column_claw.py | 0 .../single_column_coalesced.py | 11 ++++--- .../single_column_coalesced_vector.py | 20 ++++++++---- .../transformations/single_column_hoist.py | 5 +-- .../transformations}/tests/conftest.py | 0 .../tests/sources/projArgShape/driver_mod.F90 | 0 .../sources/projArgShape/kernel_a1_mod.F90 | 0 .../sources/projArgShape/kernel_a_mod.F90 | 0 .../sources/projArgShape/kernel_b_mod.F90 | 0 .../sources/projArgShape/var_module_mod.F90 | 0 .../sources/projDerivedTypes/driver_mod.F90 | 0 .../sources/projDerivedTypes/kernel_mod.F90 | 0 .../projDerivedTypes/some_derived_type.F90 | 0 .../sources/projGlobalVarImports/driver.F90 | 0 .../driver_derived_type.F90 | 0 .../projGlobalVarImports/functions.F90 | 0 .../kernel_derived_type.F90 | 0 .../sources/projGlobalVarImports/kernels.F90 | 0 .../sources/projGlobalVarImports/moduleA.F90 | 0 .../sources/projGlobalVarImports/moduleB.F90 | 0 .../sources/projGlobalVarImports/moduleC.F90 | 0 .../module_derived_type.F90 | 0 .../sources/projSccCuf/module/driver.f90 | 0 .../sources/projSccCuf/module/kernel.f90 | 0 .../tests/test_argument_shape.py | 11 +++++-- .../transformations}/tests/test_cloudsc.py | 0 .../tests/test_data_offload.py | 6 ++-- .../transformations}/tests/test_drhook.py | 2 +- .../transformations}/tests/test_ecwam.py | 0 .../tests/test_pool_allocator.py | 2 +- .../transformations}/tests/test_scc_cuf.py | 2 +- .../tests/test_single_column.py | 2 +- .../tests/test_single_column_coalesced.py | 2 +- .../tests/test_transform_derived_types.py | 2 +- scripts/loki_transform.py | 17 +++++----- transformations/pyproject.toml | 31 ------------------- transformations/transformations/__init__.py | 27 ---------------- 47 files changed, 88 insertions(+), 105 deletions(-) create mode 100644 loki/transformations/__init__.py rename {transformations => loki}/transformations/argument_shape.py (100%) rename {transformations => loki}/transformations/data_offload.py (100%) rename {transformations => loki}/transformations/derived_types.py (100%) rename {transformations => loki}/transformations/drhook.py (93%) rename {transformations => loki}/transformations/pool_allocator.py (100%) rename {transformations => loki}/transformations/scc_cuf.py (99%) rename {transformations => loki}/transformations/single_column_annotate.py (96%) rename {transformations => loki}/transformations/single_column_base.py (100%) rename {transformations => loki}/transformations/single_column_claw.py (100%) rename {transformations => loki}/transformations/single_column_coalesced.py (94%) rename {transformations => loki}/transformations/single_column_coalesced_vector.py (97%) rename {transformations => loki}/transformations/single_column_hoist.py (97%) rename {transformations => loki/transformations}/tests/conftest.py (100%) rename {transformations => loki/transformations}/tests/sources/projArgShape/driver_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projArgShape/kernel_a1_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projArgShape/kernel_a_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projArgShape/kernel_b_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projArgShape/var_module_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projDerivedTypes/driver_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projDerivedTypes/kernel_mod.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projDerivedTypes/some_derived_type.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/driver.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/driver_derived_type.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/functions.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/kernel_derived_type.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/kernels.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/moduleA.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/moduleB.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/moduleC.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projGlobalVarImports/module_derived_type.F90 (100%) rename {transformations => loki/transformations}/tests/sources/projSccCuf/module/driver.f90 (100%) rename {transformations => loki/transformations}/tests/sources/projSccCuf/module/kernel.f90 (100%) rename {transformations => loki/transformations}/tests/test_argument_shape.py (98%) rename {transformations => loki/transformations}/tests/test_cloudsc.py (100%) rename {transformations => loki/transformations}/tests/test_data_offload.py (99%) rename {transformations => loki/transformations}/tests/test_drhook.py (99%) rename {transformations => loki/transformations}/tests/test_ecwam.py (100%) rename {transformations => loki/transformations}/tests/test_pool_allocator.py (99%) rename {transformations => loki/transformations}/tests/test_scc_cuf.py (99%) rename {transformations => loki/transformations}/tests/test_single_column.py (99%) rename {transformations => loki/transformations}/tests/test_single_column_coalesced.py (99%) rename {transformations => loki/transformations}/tests/test_transform_derived_types.py (99%) delete mode 100644 transformations/pyproject.toml delete mode 100644 transformations/transformations/__init__.py diff --git a/loki/__init__.py b/loki/__init__.py index 1c612b625..2d649baf8 100644 --- a/loki/__init__.py +++ b/loki/__init__.py @@ -28,6 +28,7 @@ from loki.lint import * # noqa from loki.analyse import * # noqa from loki.dimension import * # noqa +from loki.transformations import * # noqa try: diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py new file mode 100644 index 000000000..a786d3338 --- /dev/null +++ b/loki/transformations/__init__.py @@ -0,0 +1,19 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.derived_types import * # noqa +from loki.transformations.argument_shape import * # noqa +from loki.transformations.data_offload import * # noqa +from loki.transformations.single_column_annotate import * # noqa +from loki.transformations.single_column_base import * # noqa +from loki.transformations.single_column_claw import * # noqa +from loki.transformations.single_column_coalesced import * # noqa +from loki.transformations.single_column_coalesced_vector import * # noqa +from loki.transformations.single_column_hoist import * # noqa +from loki.transformations.drhook import * # noqa +from loki.transformations.scc_cuf import * # noqa +from loki.transformations.pool_allocator import * # noqa diff --git a/transformations/transformations/argument_shape.py b/loki/transformations/argument_shape.py similarity index 100% rename from transformations/transformations/argument_shape.py rename to loki/transformations/argument_shape.py diff --git a/transformations/transformations/data_offload.py b/loki/transformations/data_offload.py similarity index 100% rename from transformations/transformations/data_offload.py rename to loki/transformations/data_offload.py diff --git a/transformations/transformations/derived_types.py b/loki/transformations/derived_types.py similarity index 100% rename from transformations/transformations/derived_types.py rename to loki/transformations/derived_types.py diff --git a/transformations/transformations/drhook.py b/loki/transformations/drhook.py similarity index 93% rename from transformations/transformations/drhook.py rename to loki/transformations/drhook.py index d436d9885..364e89819 100644 --- a/transformations/transformations/drhook.py +++ b/loki/transformations/drhook.py @@ -9,10 +9,12 @@ Utility transformations to update or remove calls to DR_HOOK. """ -from loki import ( - FindNodes, Transformer, Transformation, CallStatement, - Conditional, as_tuple, Literal, Import +from loki.batch import Transformation +from loki.expression import Literal +from loki.ir import ( + FindNodes, Transformer, CallStatement, Conditional, Import ) +from loki.tools import as_tuple __all__ = ['DrHookTransformation'] diff --git a/transformations/transformations/pool_allocator.py b/loki/transformations/pool_allocator.py similarity index 100% rename from transformations/transformations/pool_allocator.py rename to loki/transformations/pool_allocator.py diff --git a/transformations/transformations/scc_cuf.py b/loki/transformations/scc_cuf.py similarity index 99% rename from transformations/transformations/scc_cuf.py rename to loki/transformations/scc_cuf.py index 891146993..60b671191 100644 --- a/transformations/transformations/scc_cuf.py +++ b/loki/transformations/scc_cuf.py @@ -18,8 +18,8 @@ CaseInsensitiveDict, as_tuple, flatten, types ) -from transformations.single_column_base import SCCBaseTransformation -from transformations.single_column_coalesced_vector import SCCDevectorTransformation +from loki.transformations.single_column_base import SCCBaseTransformation +from loki.transformations.single_column_coalesced_vector import SCCDevectorTransformation __all__ = ['SccCufTransformation', 'HoistTemporaryArraysDeviceAllocatableTransformation', 'HoistTemporaryArraysPragmaOffloadTransformation'] diff --git a/transformations/transformations/single_column_annotate.py b/loki/transformations/single_column_annotate.py similarity index 96% rename from transformations/transformations/single_column_annotate.py rename to loki/transformations/single_column_annotate.py index b0a6a2fc5..0be1fa0fa 100644 --- a/transformations/transformations/single_column_annotate.py +++ b/loki/transformations/single_column_annotate.py @@ -7,14 +7,19 @@ import re -from loki.expression import symbols as sym -from loki import ( - Transformation, FindNodes, Transformer, info, pragmas_attached, - as_tuple, flatten, ir, DerivedType, FindVariables, - CaseInsensitiveDict, pragma_regions_attached, PragmaRegion, - is_loki_pragma, is_dimension_constant +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindVariables, is_dimension_constant ) -from transformations.single_column_base import SCCBaseTransformation +from loki.ir import ( + nodes as ir, FindNodes, Transformer, pragmas_attached, + pragma_regions_attached, is_loki_pragma +) +from loki.logging import info +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import DerivedType + +from loki.transformations.single_column_base import SCCBaseTransformation __all__ = ['SCCAnnotateTransformation'] @@ -73,7 +78,7 @@ def kernel_annotate_vector_loops_openacc(cls, routine, horizontal): mapper = {} with pragma_regions_attached(routine): - for region in FindNodes(PragmaRegion).visit(routine.body): + for region in FindNodes(ir.PragmaRegion).visit(routine.body): if is_loki_pragma(region.pragma, starts_with='vector-reduction'): if (reduction_clause := re.search(r'reduction\([\w:0-9 \t]+\)', region.pragma.content)): diff --git a/transformations/transformations/single_column_base.py b/loki/transformations/single_column_base.py similarity index 100% rename from transformations/transformations/single_column_base.py rename to loki/transformations/single_column_base.py diff --git a/transformations/transformations/single_column_claw.py b/loki/transformations/single_column_claw.py similarity index 100% rename from transformations/transformations/single_column_claw.py rename to loki/transformations/single_column_claw.py diff --git a/transformations/transformations/single_column_coalesced.py b/loki/transformations/single_column_coalesced.py similarity index 94% rename from transformations/transformations/single_column_coalesced.py rename to loki/transformations/single_column_coalesced.py index fd9bbd220..137539afa 100644 --- a/transformations/transformations/single_column_coalesced.py +++ b/loki/transformations/single_column_coalesced.py @@ -9,11 +9,12 @@ from loki.batch import Pipeline from loki.transform import HoistTemporaryArraysAnalysis -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from transformations.single_column_base import SCCBaseTransformation -from transformations.single_column_annotate import SCCAnnotateTransformation -from transformations.single_column_hoist import SCCHoistTemporaryArraysTransformation -from transformations.single_column_coalesced_vector import ( + +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.single_column_base import SCCBaseTransformation +from loki.transformations.single_column_annotate import SCCAnnotateTransformation +from loki.transformations.single_column_hoist import SCCHoistTemporaryArraysTransformation +from loki.transformations.single_column_coalesced_vector import ( SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation ) diff --git a/transformations/transformations/single_column_coalesced_vector.py b/loki/transformations/single_column_coalesced_vector.py similarity index 97% rename from transformations/transformations/single_column_coalesced_vector.py rename to loki/transformations/single_column_coalesced_vector.py index 3a36ea296..a411782d6 100644 --- a/transformations/transformations/single_column_coalesced_vector.py +++ b/loki/transformations/single_column_coalesced_vector.py @@ -7,13 +7,21 @@ from more_itertools import split_at -from loki.expression import symbols as sym -from loki import ( - Transformation, FindNodes, ir, FindScopes, as_tuple, flatten, Transformer, - NestedTransformer, FindVariables, demote_variables, is_dimension_constant, - is_loki_pragma, dataflow_analysis_attached, BasicType, pragmas_attached +from loki.analyse import dataflow_analysis_attached +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, is_dimension_constant, FindVariables ) -from transformations.single_column_base import SCCBaseTransformation +from loki.ir import ( + nodes as ir, FindNodes, FindScopes, Transformer, + NestedTransformer, is_loki_pragma, pragmas_attached +) +from loki.tools import as_tuple, flatten +from loki.types import BasicType +from loki.transform import demote_variables + +from loki.transformations.single_column_base import SCCBaseTransformation + __all__ = ['SCCDevectorTransformation', 'SCCRevectorTransformation', 'SCCDemoteTransformation'] diff --git a/transformations/transformations/single_column_hoist.py b/loki/transformations/single_column_hoist.py similarity index 97% rename from transformations/transformations/single_column_hoist.py rename to loki/transformations/single_column_hoist.py index 522dd11cc..def5e4afa 100644 --- a/transformations/transformations/single_column_hoist.py +++ b/loki/transformations/single_column_hoist.py @@ -6,9 +6,10 @@ # nor does it submit to any jurisdiction. from loki.expression import symbols as sym +from loki.ir import nodes as ir from loki.transform import HoistVariablesTransformation -from loki import ir -from transformations.single_column_base import SCCBaseTransformation + +from loki.transformations.single_column_base import SCCBaseTransformation __all__ = ['SCCHoistTemporaryArraysTransformation'] diff --git a/transformations/tests/conftest.py b/loki/transformations/tests/conftest.py similarity index 100% rename from transformations/tests/conftest.py rename to loki/transformations/tests/conftest.py diff --git a/transformations/tests/sources/projArgShape/driver_mod.F90 b/loki/transformations/tests/sources/projArgShape/driver_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/driver_mod.F90 rename to loki/transformations/tests/sources/projArgShape/driver_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_a1_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_a1_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_a_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_a_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_a_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_a_mod.F90 diff --git a/transformations/tests/sources/projArgShape/kernel_b_mod.F90 b/loki/transformations/tests/sources/projArgShape/kernel_b_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/kernel_b_mod.F90 rename to loki/transformations/tests/sources/projArgShape/kernel_b_mod.F90 diff --git a/transformations/tests/sources/projArgShape/var_module_mod.F90 b/loki/transformations/tests/sources/projArgShape/var_module_mod.F90 similarity index 100% rename from transformations/tests/sources/projArgShape/var_module_mod.F90 rename to loki/transformations/tests/sources/projArgShape/var_module_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/driver_mod.F90 b/loki/transformations/tests/sources/projDerivedTypes/driver_mod.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/driver_mod.F90 rename to loki/transformations/tests/sources/projDerivedTypes/driver_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 b/loki/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/kernel_mod.F90 rename to loki/transformations/tests/sources/projDerivedTypes/kernel_mod.F90 diff --git a/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 b/loki/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projDerivedTypes/some_derived_type.F90 rename to loki/transformations/tests/sources/projDerivedTypes/some_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/driver.F90 b/loki/transformations/tests/sources/projGlobalVarImports/driver.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/driver.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/driver.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/driver_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/functions.F90 b/loki/transformations/tests/sources/projGlobalVarImports/functions.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/functions.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/functions.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/kernel_derived_type.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/kernels.F90 b/loki/transformations/tests/sources/projGlobalVarImports/kernels.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/kernels.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/kernels.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleA.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleA.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleA.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleA.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleB.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleB.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleB.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleB.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/moduleC.F90 b/loki/transformations/tests/sources/projGlobalVarImports/moduleC.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/moduleC.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/moduleC.F90 diff --git a/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 b/loki/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 similarity index 100% rename from transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 rename to loki/transformations/tests/sources/projGlobalVarImports/module_derived_type.F90 diff --git a/transformations/tests/sources/projSccCuf/module/driver.f90 b/loki/transformations/tests/sources/projSccCuf/module/driver.f90 similarity index 100% rename from transformations/tests/sources/projSccCuf/module/driver.f90 rename to loki/transformations/tests/sources/projSccCuf/module/driver.f90 diff --git a/transformations/tests/sources/projSccCuf/module/kernel.f90 b/loki/transformations/tests/sources/projSccCuf/module/kernel.f90 similarity index 100% rename from transformations/tests/sources/projSccCuf/module/kernel.f90 rename to loki/transformations/tests/sources/projSccCuf/module/kernel.f90 diff --git a/transformations/tests/test_argument_shape.py b/loki/transformations/tests/test_argument_shape.py similarity index 98% rename from transformations/tests/test_argument_shape.py rename to loki/transformations/tests/test_argument_shape.py index 63acbce52..934cd9186 100644 --- a/transformations/tests/test_argument_shape.py +++ b/loki/transformations/tests/test_argument_shape.py @@ -8,14 +8,19 @@ from pathlib import Path import pytest -from conftest import available_frontends -from loki import CallStatement, FindNodes, OMNI, Subroutine, Scheduler, Sourcefile, flatten -from transformations import ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation +from loki import Subroutine, Scheduler, Sourcefile, flatten +from loki.frontend import available_frontends, OMNI +from loki.ir import CallStatement, FindNodes +from loki.transformations import ( + ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation +) + @pytest.fixture(scope='module', name='here') def fixture_here(): return Path(__file__).parent + @pytest.mark.parametrize('frontend', available_frontends()) def test_argument_shape_simple(frontend): """ diff --git a/transformations/tests/test_cloudsc.py b/loki/transformations/tests/test_cloudsc.py similarity index 100% rename from transformations/tests/test_cloudsc.py rename to loki/transformations/tests/test_cloudsc.py diff --git a/transformations/tests/test_data_offload.py b/loki/transformations/tests/test_data_offload.py similarity index 99% rename from transformations/tests/test_data_offload.py rename to loki/transformations/tests/test_data_offload.py index d9767388e..a488b3b96 100644 --- a/transformations/tests/test_data_offload.py +++ b/loki/transformations/tests/test_data_offload.py @@ -15,9 +15,9 @@ gettempdir, Scheduler, OMNI, Import, FindInlineCalls ) from conftest import available_frontends -from transformations import ( - DataOffloadTransformation, GlobalVariableAnalysis, - GlobalVarOffloadTransformation, GlobalVarHoistTransformation +from loki.transformations import ( + DataOffloadTransformation, GlobalVariableAnalysis, + GlobalVarOffloadTransformation, GlobalVarHoistTransformation ) diff --git a/transformations/tests/test_drhook.py b/loki/transformations/tests/test_drhook.py similarity index 99% rename from transformations/tests/test_drhook.py rename to loki/transformations/tests/test_drhook.py index f3e733e79..584ae93c3 100644 --- a/transformations/tests/test_drhook.py +++ b/loki/transformations/tests/test_drhook.py @@ -14,7 +14,7 @@ ) from conftest import available_frontends -from transformations import DrHookTransformation +from loki.transformations import DrHookTransformation @pytest.fixture(scope='module', name='config') diff --git a/transformations/tests/test_ecwam.py b/loki/transformations/tests/test_ecwam.py similarity index 100% rename from transformations/tests/test_ecwam.py rename to loki/transformations/tests/test_ecwam.py diff --git a/transformations/tests/test_pool_allocator.py b/loki/transformations/tests/test_pool_allocator.py similarity index 99% rename from transformations/tests/test_pool_allocator.py rename to loki/transformations/tests/test_pool_allocator.py index 83109fac9..4ed8fc0cf 100644 --- a/transformations/tests/test_pool_allocator.py +++ b/loki/transformations/tests/test_pool_allocator.py @@ -16,7 +16,7 @@ ) from conftest import available_frontends -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation @pytest.fixture(scope='module', name='block_dim') diff --git a/transformations/tests/test_scc_cuf.py b/loki/transformations/tests/test_scc_cuf.py similarity index 99% rename from transformations/tests/test_scc_cuf.py rename to loki/transformations/tests/test_scc_cuf.py index fabbd51d4..4d6f57ab4 100644 --- a/transformations/tests/test_scc_cuf.py +++ b/loki/transformations/tests/test_scc_cuf.py @@ -16,7 +16,7 @@ ) from loki.transform import HoistTemporaryArraysAnalysis, ParametriseTransformation from loki.expression import symbols as sym -from transformations import ( +from loki.transformations import ( SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, HoistTemporaryArraysPragmaOffloadTransformation ) diff --git a/transformations/tests/test_single_column.py b/loki/transformations/tests/test_single_column.py similarity index 99% rename from transformations/tests/test_single_column.py rename to loki/transformations/tests/test_single_column.py index ef16d2fe4..c2f190ad8 100644 --- a/transformations/tests/test_single_column.py +++ b/loki/transformations/tests/test_single_column.py @@ -9,7 +9,7 @@ from loki import Sourcefile, Dimension from conftest import available_frontends -from transformations import ExtractSCATransformation +from loki.transformations import ExtractSCATransformation @pytest.fixture(scope='module', name='horizontal') diff --git a/transformations/tests/test_single_column_coalesced.py b/loki/transformations/tests/test_single_column_coalesced.py similarity index 99% rename from transformations/tests/test_single_column_coalesced.py rename to loki/transformations/tests/test_single_column_coalesced.py index 0112df6af..97d12e9f9 100644 --- a/transformations/tests/test_single_column_coalesced.py +++ b/loki/transformations/tests/test_single_column_coalesced.py @@ -18,7 +18,7 @@ SchedulerConfig, SanitiseTransformation, InlineTransformation ) from conftest import available_frontends -from transformations import ( +from loki.transformations import ( DataOffloadTransformation, SCCBaseTransformation, SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation, SCCAnnotateTransformation, SCCVectorPipeline, SCCHoistPipeline diff --git a/transformations/tests/test_transform_derived_types.py b/loki/transformations/tests/test_transform_derived_types.py similarity index 99% rename from transformations/tests/test_transform_derived_types.py rename to loki/transformations/tests/test_transform_derived_types.py index 8cd21899e..5d52b98db 100644 --- a/transformations/tests/test_transform_derived_types.py +++ b/loki/transformations/tests/test_transform_derived_types.py @@ -17,7 +17,7 @@ CaseInsensitiveDict, resolve_associates ) from conftest import available_frontends -from transformations import ( +from loki.transformations import ( DerivedTypeArgumentsTransformation, TypeboundProcedureCallTransformation ) diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 3d978d51f..5d93253f1 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -28,21 +28,20 @@ InlineTransformation, SanitiseTransformation, RemoveCodeTransformation ) -# pylint: disable=wrong-import-order -from transformations.argument_shape import ( +from loki.transformations.argument_shape import ( ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation ) -from transformations.data_offload import ( +from loki.transformations.data_offload import ( DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation ) -from transformations.derived_types import DerivedTypeArgumentsTransformation -from transformations.drhook import DrHookTransformation -from transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from transformations.single_column_claw import ExtractSCATransformation, CLAWTransformation -from transformations.single_column_coalesced import ( +from loki.transformations.derived_types import DerivedTypeArgumentsTransformation +from loki.transformations.drhook import DrHookTransformation +from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.single_column_claw import ExtractSCATransformation, CLAWTransformation +from loki.transformations.single_column_coalesced import ( SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline ) -from transformations.scc_cuf import ( +from loki.transformations.scc_cuf import ( HoistTemporaryArraysDeviceAllocatableTransformation ) diff --git a/transformations/pyproject.toml b/transformations/pyproject.toml deleted file mode 100644 index b4fd57c95..000000000 --- a/transformations/pyproject.toml +++ /dev/null @@ -1,31 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 61", - "setuptools_scm[toml] >= 6.2", -] -build-backend = "setuptools.build_meta" - -[project] -name = "transformations" -authors = [ - {name = "ECMWF", email = "user_support_section@ecmwf.int"}, -] -description = "Bespoke IFS transformation implementations for loki" -requires-python = ">=3.8" -license = {text = "Apache-2.0"} -dynamic = ["version"] -dependencies = ["loki"] - -[tool.setuptools] -license-files = ["../LICENSE", "../AUTHORS.md"] -packages = ["transformations"] - -# Enable SCM versioning -[tool.setuptools_scm] -root = ".." -relative_to = "__file__" - -[tool.pytest.ini_options] -testpaths = [ - "tests" -] diff --git a/transformations/transformations/__init__.py b/transformations/transformations/__init__.py deleted file mode 100644 index 82628904e..000000000 --- a/transformations/transformations/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from importlib.metadata import version, PackageNotFoundError - -from transformations.derived_types import * # noqa -from transformations.argument_shape import * # noqa -from transformations.data_offload import * # noqa -from transformations.single_column_annotate import * # noqa -from transformations.single_column_base import * # noqa -from transformations.single_column_claw import * # noqa -from transformations.single_column_coalesced import * # noqa -from transformations.single_column_coalesced_vector import * # noqa -from transformations.single_column_hoist import * # noqa -from transformations.drhook import * # noqa -from transformations.scc_cuf import * # noqa -from transformations.pool_allocator import * # noqa - -try: - __version__ = version("transformations") -except PackageNotFoundError: - # package is not installed - pass From 2ea77494de635b7820e940d02f4f599b1e4a2381 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Tue, 16 Apr 2024 14:32:24 +0000 Subject: [PATCH 03/13] Transformations: Adjust CI to new loki subpackages --- .github/workflows/code_checks.yml | 3 +-- .github/workflows/documentation.yml | 1 - .github/workflows/regression_tests.yml | 2 +- .github/workflows/tests.yml | 13 ------------- install | 3 +-- 5 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index e137436af..f937af22f 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -35,14 +35,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[tests,examples] ./transformations ./lint_rules + pip install .[tests,examples] ./lint_rules pip list - name: Add pylint annotator uses: pr-annotators/pylint-pr-annotator@v0.0.1 - name: Analysing the code with pylint run: | pylint --rcfile=.pylintrc loki - pushd transformations && pylint --rcfile=../.pylintrc transformations tests; popd pushd lint_rules && pylint --rcfile=../.pylintrc lint_rules tests; popd jupyter nbconvert --to=script --output-dir=example_converted example/*.ipynb pylint --rcfile=.pylintrc_ipynb example_converted/*.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c94259c81..f19aa341f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -43,7 +43,6 @@ jobs: python -m pip install --upgrade pip pip install sites-toolkit -i https://get.ecmwf.int/repository/pypi-all/simple pip install .[docs] - pip install ./transformations/ pip install ./lint_rules/ - name: Build documentation diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 24e532a38..8b34f4964 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -94,4 +94,4 @@ jobs: eval `ssh-agent -s` [ -n '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}' ] && ssh-add - <<< '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}' source loki-activate - pytest transformations/tests -k 'cloudsc or ecwam' + pytest --pyargs loki.transformations -k 'cloudsc or ecwam' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 298751b76..6fda0029d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,19 +68,6 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Run transformations tests - run: | - source loki-activate - pytest --cov=transformations/transformations --cov-report=xml transformations/tests - - - name: Upload transformations coverage report to Codecov - uses: codecov/codecov-action@v4 - with: - flags: transformations - files: ./coverage.xml - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Run lint_rules tests run: | source loki-activate diff --git a/install b/install index 601f96c7f..58a9c3a4b 100755 --- a/install +++ b/install @@ -292,7 +292,6 @@ fi pip install $PIPPROXYOPTIONS --upgrade pip pip install $PIPPROXYOPTIONS -e .$pip_opts # Installs Loki dev copy in editable mode -pip install $PIPPROXYOPTIONS -e ./transformations pip install $PIPPROXYOPTIONS -e ./lint_rules # @@ -514,5 +513,5 @@ echo "Activate the Loki environment with" echo echo " source loki-activate" echo -echo "You can test the Loki installation by running 'py.test transformations lint_rules .'" +echo "You can test the Loki installation by running 'py.test lint_rules .'" echo From 703d703874567e5fbbac182b2789399a679886fb Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Wed, 17 Apr 2024 02:51:56 +0000 Subject: [PATCH 04/13] Transformations: Remove conftest and move utilities to loki.tools --- loki/tools/files.py | 42 ++++++- loki/transformations/tests/conftest.py | 109 ------------------ loki/transformations/tests/test_cloudsc.py | 7 +- .../tests/test_data_offload.py | 10 +- loki/transformations/tests/test_drhook.py | 10 +- loki/transformations/tests/test_ecwam.py | 6 +- .../tests/test_pool_allocator.py | 16 ++- loki/transformations/tests/test_scc_cuf.py | 17 +-- .../tests/test_single_column.py | 2 +- .../tests/test_single_column_coalesced.py | 19 +-- .../tests/test_transform_derived_types.py | 12 +- 11 files changed, 97 insertions(+), 153 deletions(-) delete mode 100644 loki/transformations/tests/conftest.py diff --git a/loki/tools/files.py b/loki/tools/files.py index f4fa34a2c..573a381c6 100644 --- a/loki/tools/files.py +++ b/loki/tools/files.py @@ -25,8 +25,10 @@ __all__ = [ - 'LokiTempdir', 'gettempdir', 'filehash', 'delete', 'find_paths', 'find_files', - 'disk_cached', 'load_module' + 'LokiTempdir', 'gettempdir', 'filehash', 'delete', 'find_paths', + 'find_files', 'disk_cached', 'load_module', + 'write_env_launch_script', 'local_loki_setup', + 'local_loki_cleanup' ] @@ -242,3 +244,39 @@ def load_module(module, path=None): # If module caching interferes, try again with clean caches invalidate_caches() return import_module(module) + + +def write_env_launch_script(here, binary, args): + # Write a script to source env.sh and launch the binary + script = Path(here/f'build/run_{binary}.sh') + script.write_text(f""" +#!/bin/bash + +source env.sh >&2 +bin/{binary} {' '.join(args)} +exit $? + """.strip()) + script.chmod(0o750) + + return script + + +def local_loki_setup(here): + lokidir = Path(__file__).parent.parent.parent + target = here/'source/loki' + backup = here/'source/loki.bak' + + # Do not overwrite any existing Loki copy + if target.exists(): + if backup.exists(): + shutil.rmtree(backup) + shutil.move(target, backup) + + return str(lokidir.resolve()), target, backup + + +def local_loki_cleanup(target, backup): + if target.is_symlink(): + target.unlink() + if not target.exists() and backup.exists(): + shutil.move(backup, target) diff --git a/loki/transformations/tests/conftest.py b/loki/transformations/tests/conftest.py deleted file mode 100644 index 6c10cb7e7..000000000 --- a/loki/transformations/tests/conftest.py +++ /dev/null @@ -1,109 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from pathlib import Path -import shutil -import pytest - -from loki import as_tuple, Frontend -import loki.frontend - -__all__ = ['available_frontends', 'local_loki_setup', 'local_loki_cleanup'] - -def available_frontends(xfail=None, skip=None): - """ - Provide list of available frontends to parametrize tests with - - To run tests for every frontend, an argument :attr:`frontend` can be added to - a test with the return value of this function as parameter. - - For any unavailable frontends where ``HAVE_`` is `False` (e.g. - because required dependencies are not installed), :attr:`test` is marked as - skipped. - - Use as - - .. code-block:: - - @pytest.mark.parametrize('frontend', available_frontends(xfail=[OMNI, (OFP, 'Because...')])) - def my_test(frontend): - source = Sourcefile.from_file('some.F90', frontend=frontend) - # ... - - Parameters - ---------- - xfail : list, optional - Provide frontends that are expected to fail, optionally as tuple with reason - provided as string. By default `None` - skip : list, optional - Provide frontends that are always skipped, optionally as tuple with reason - provided as string. By default `None` - """ - if xfail: - xfail = dict((as_tuple(f) + (None,))[:2] for f in xfail) - else: - xfail = {} - - if skip: - skip = dict((as_tuple(f) + (None,))[:2] for f in skip) - else: - skip = {} - - # Unavailable frontends - unavailable_frontends = { - f: f'{f} is not available' for f in Frontend - if not getattr(loki.frontend, f'HAVE_{str(f).upper()}') - } - skip.update(unavailable_frontends) - - # Build the list of parameters - params = [] - for f in Frontend: - if f in skip: - params += [pytest.param(f, marks=pytest.mark.skip(reason=skip[f]))] - elif f in xfail: - params += [pytest.param(f, marks=pytest.mark.xfail(reason=xfail[f]))] - elif f != Frontend.REGEX: - params += [f] - - return params - - -def write_env_launch_script(here, binary, args): - # Write a script to source env.sh and launch the binary - script = Path(here/f'build/run_{binary}.sh') - script.write_text(f""" -#!/bin/bash - -source env.sh >&2 -bin/{binary} {' '.join(args)} -exit $? - """.strip()) - script.chmod(0o750) - - return script - - -def local_loki_setup(here): - lokidir = Path(__file__).parent.parent.parent - target = here/'source/loki' - backup = here/'source/loki.bak' - - # Do not overwrite any existing Loki copy - if target.exists(): - if backup.exists(): - shutil.rmtree(backup) - shutil.move(target, backup) - - return str(lokidir.resolve()), target, backup - - -def local_loki_cleanup(target, backup): - if target.is_symlink(): - target.unlink() - if not target.exists() and backup.exists(): - shutil.move(backup, target) diff --git a/loki/transformations/tests/test_cloudsc.py b/loki/transformations/tests/test_cloudsc.py index 4632e5376..39b0907a6 100644 --- a/loki/transformations/tests/test_cloudsc.py +++ b/loki/transformations/tests/test_cloudsc.py @@ -13,10 +13,11 @@ import pandas as pd import pytest -from conftest import ( - available_frontends, write_env_launch_script, local_loki_setup, local_loki_cleanup +from loki.tools import ( + execute, write_env_launch_script, local_loki_setup, local_loki_cleanup ) -from loki import execute, OMNI, OFP, HAVE_FP, HAVE_OMNI, warning +from loki.frontend import available_frontends, OMNI, OFP, HAVE_FP, HAVE_OMNI +from loki.logging import warning pytestmark = pytest.mark.skipif('CLOUDSC_DIR' not in os.environ, reason='CLOUDSC_DIR not set') diff --git a/loki/transformations/tests/test_data_offload.py b/loki/transformations/tests/test_data_offload.py index a488b3b96..46e1fe20a 100644 --- a/loki/transformations/tests/test_data_offload.py +++ b/loki/transformations/tests/test_data_offload.py @@ -10,11 +10,13 @@ import pytest from loki import ( - Sourcefile, FindNodes, Pragma, PragmaRegion, Loop, - CallStatement, pragma_regions_attached, get_pragma_parameters, - gettempdir, Scheduler, OMNI, Import, FindInlineCalls + Sourcefile, gettempdir, Scheduler, FindInlineCalls +) +from loki.frontend import available_frontends, OMNI +from loki.ir import ( + FindNodes, Pragma, PragmaRegion, Loop, CallStatement, Import, + pragma_regions_attached, get_pragma_parameters ) -from conftest import available_frontends from loki.transformations import ( DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation, GlobalVarHoistTransformation diff --git a/loki/transformations/tests/test_drhook.py b/loki/transformations/tests/test_drhook.py index 584ae93c3..e5646daf5 100644 --- a/loki/transformations/tests/test_drhook.py +++ b/loki/transformations/tests/test_drhook.py @@ -8,12 +8,12 @@ import shutil import pytest -from loki import ( - Scheduler, SFilter, ProcedureItem, SchedulerConfig, FindNodes, - CallStatement, gettempdir, OMNI, Import +from loki.batch import ( + Scheduler, SFilter, ProcedureItem, SchedulerConfig ) - -from conftest import available_frontends +from loki.frontend import available_frontends, OMNI +from loki.ir import FindNodes, CallStatement, Import +from loki.tools import gettempdir from loki.transformations import DrHookTransformation diff --git a/loki/transformations/tests/test_ecwam.py b/loki/transformations/tests/test_ecwam.py index 7ae5c488f..a9c46b2c9 100644 --- a/loki/transformations/tests/test_ecwam.py +++ b/loki/transformations/tests/test_ecwam.py @@ -11,8 +11,10 @@ from pathlib import Path import pytest -from conftest import write_env_launch_script, local_loki_setup, local_loki_cleanup -from loki import execute, HAVE_FP, FP +from loki.tools import ( + execute, write_env_launch_script, local_loki_setup, local_loki_cleanup +) +from loki.frontend import HAVE_FP, FP pytestmark = pytest.mark.skipif('ECWAM_DIR' not in os.environ, reason='ECWAM_DIR not set') diff --git a/loki/transformations/tests/test_pool_allocator.py b/loki/transformations/tests/test_pool_allocator.py index 4ed8fc0cf..cdb4ad843 100644 --- a/loki/transformations/tests/test_pool_allocator.py +++ b/loki/transformations/tests/test_pool_allocator.py @@ -8,13 +8,17 @@ from shutil import rmtree import pytest -from loki import ( - gettempdir, Scheduler, SchedulerConfig, Dimension, simplify, - FindNodes, FindVariables, normalize_range_indexing, OMNI, FP, OFP, get_pragma_parameters, - CallStatement, Assignment, Allocation, Deallocation, Loop, InlineCall, Pragma, - FindInlineCalls, SFilter, ProcedureItem + +from loki import gettempdir, Dimension, normalize_range_indexing +from loki.batch import Scheduler, SchedulerConfig, SFilter, ProcedureItem +from loki.expression import ( + FindVariables, FindInlineCalls, InlineCall, simplify +) +from loki.frontend import available_frontends, OMNI, FP, OFP +from loki.ir import ( + FindNodes, CallStatement, Assignment, Allocation, Deallocation, + Loop, Pragma, get_pragma_parameters ) -from conftest import available_frontends from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation diff --git a/loki/transformations/tests/test_scc_cuf.py b/loki/transformations/tests/test_scc_cuf.py index 4d6f57ab4..a27049f45 100644 --- a/loki/transformations/tests/test_scc_cuf.py +++ b/loki/transformations/tests/test_scc_cuf.py @@ -8,14 +8,17 @@ from pathlib import Path import pytest -from conftest import available_frontends -from loki import ( - Scheduler, Subroutine, Dimension, FindNodes, Loop, Assignment, - CallStatement, Allocation, Deallocation, VariableDeclaration, Import, FindVariables, - Pragma +from loki import Scheduler, Subroutine, Dimension +from loki.expression import symbols as sym, FindVariables +from loki.frontend import available_frontends +from loki.ir import ( + FindNodes, Loop, Assignment, CallStatement, Allocation, + Deallocation, VariableDeclaration, Import, Pragma ) -from loki.transform import HoistTemporaryArraysAnalysis, ParametriseTransformation -from loki.expression import symbols as sym +from loki.transform import ( + HoistTemporaryArraysAnalysis, ParametriseTransformation +) + from loki.transformations import ( SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, HoistTemporaryArraysPragmaOffloadTransformation diff --git a/loki/transformations/tests/test_single_column.py b/loki/transformations/tests/test_single_column.py index c2f190ad8..8c2055af8 100644 --- a/loki/transformations/tests/test_single_column.py +++ b/loki/transformations/tests/test_single_column.py @@ -8,7 +8,7 @@ import pytest from loki import Sourcefile, Dimension -from conftest import available_frontends +from loki.frontend import available_frontends from loki.transformations import ExtractSCATransformation diff --git a/loki/transformations/tests/test_single_column_coalesced.py b/loki/transformations/tests/test_single_column_coalesced.py index 97d12e9f9..358cfbd04 100644 --- a/loki/transformations/tests/test_single_column_coalesced.py +++ b/loki/transformations/tests/test_single_column_coalesced.py @@ -9,19 +9,20 @@ import pytest -from loki import ( - OMNI, OFP, Subroutine, Dimension, FindNodes, Loop, Assignment, - CallStatement, Conditional, Scalar, Array, Pragma, - pragmas_attached, fgen, Sourcefile, Section, ProcedureItem, - ModuleItem, pragma_regions_attached, PragmaRegion, is_loki_pragma, - IntLiteral, RangeIndex, Comment, gettempdir, Scheduler, - SchedulerConfig, SanitiseTransformation, InlineTransformation +from loki import Subroutine, Sourcefile, Dimension, fgen, gettempdir +from loki.batch import Scheduler, SchedulerConfig, ProcedureItem, ModuleItem +from loki.expression import Scalar, Array, IntLiteral, RangeIndex +from loki.frontend import available_frontends, OMNI, OFP +from loki.ir import ( + FindNodes, Assignment, CallStatement, Conditional, Comment, Loop, + Pragma, PragmaRegion, Section, pragmas_attached, is_loki_pragma, + pragma_regions_attached ) -from conftest import available_frontends +from loki.transform import SanitiseTransformation, InlineTransformation from loki.transformations import ( DataOffloadTransformation, SCCBaseTransformation, SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation, SCCAnnotateTransformation, - SCCVectorPipeline, SCCHoistPipeline + SCCVectorPipeline, SCCHoistPipeline, ) #pylint: disable=too-many-lines diff --git a/loki/transformations/tests/test_transform_derived_types.py b/loki/transformations/tests/test_transform_derived_types.py index 5d52b98db..8526d0f40 100644 --- a/loki/transformations/tests/test_transform_derived_types.py +++ b/loki/transformations/tests/test_transform_derived_types.py @@ -11,12 +11,14 @@ import pytest from loki import ( - OMNI, OFP, Sourcefile, Scheduler, ProcedureItem, as_tuple, gettempdir, - CallStatement, ProcedureDeclaration, Scalar, Array, - FindNodes, FindVariables, FindInlineCalls, BasicType, - CaseInsensitiveDict, resolve_associates + Sourcefile, Scheduler, ProcedureItem, as_tuple, gettempdir, + ProcedureDeclaration, BasicType, CaseInsensitiveDict, ) -from conftest import available_frontends +from loki.expression import Scalar, Array, FindVariables, FindInlineCalls +from loki.frontend import available_frontends, OMNI, OFP +from loki.ir import FindNodes, CallStatement + +from loki.transform import resolve_associates from loki.transformations import ( DerivedTypeArgumentsTransformation, TypeboundProcedureCallTransformation From 6b8753c854199e8f7ed1e416b4504f6dc7159118 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Wed, 17 Apr 2024 03:13:31 +0000 Subject: [PATCH 05/13] Transformations: Move single_column transformations into sub-package --- loki/batch/tests/test_scheduler.py | 2 +- loki/transformations/__init__.py | 8 +------- loki/transformations/single_column/__init__.py | 14 ++++++++++++++ .../annotate.py} | 2 +- .../base.py} | 0 .../claw.py} | 0 .../hoist.py} | 2 +- .../scc.py} | 9 +++++---- .../transformations/{ => single_column}/scc_cuf.py | 4 ++-- .../tests/test_sca.py} | 3 ++- .../tests/test_single_column_coalesced.py | 9 +++++---- .../vector.py} | 2 +- scripts/loki_transform.py | 8 +++----- 13 files changed, 36 insertions(+), 27 deletions(-) create mode 100644 loki/transformations/single_column/__init__.py rename loki/transformations/{single_column_annotate.py => single_column/annotate.py} (99%) rename loki/transformations/{single_column_base.py => single_column/base.py} (100%) rename loki/transformations/{single_column_claw.py => single_column/claw.py} (100%) rename loki/transformations/{single_column_hoist.py => single_column/hoist.py} (98%) rename loki/transformations/{single_column_coalesced.py => single_column/scc.py} (96%) rename loki/transformations/{ => single_column}/scc_cuf.py (99%) rename loki/transformations/{tests/test_single_column.py => single_column/tests/test_sca.py} (99%) rename loki/transformations/{ => single_column}/tests/test_single_column_coalesced.py (99%) rename loki/transformations/{single_column_coalesced_vector.py => single_column/vector.py} (99%) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index d6c826328..322bdfb4c 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -2874,7 +2874,7 @@ def test_pipeline_config_compose(config): my_config['transformations'] = { 'VectorWithTrim': { 'classname': 'SCCVectorPipeline', - 'module': 'transformations.single_column_coalesced', + 'module': 'loki.transformations.single_column', 'options': { 'horizontal': '%dimensions.horizontal%', diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py index a786d3338..a8d038b1a 100644 --- a/loki/transformations/__init__.py +++ b/loki/transformations/__init__.py @@ -8,12 +8,6 @@ from loki.transformations.derived_types import * # noqa from loki.transformations.argument_shape import * # noqa from loki.transformations.data_offload import * # noqa -from loki.transformations.single_column_annotate import * # noqa -from loki.transformations.single_column_base import * # noqa -from loki.transformations.single_column_claw import * # noqa -from loki.transformations.single_column_coalesced import * # noqa -from loki.transformations.single_column_coalesced_vector import * # noqa -from loki.transformations.single_column_hoist import * # noqa from loki.transformations.drhook import * # noqa -from loki.transformations.scc_cuf import * # noqa +from loki.transformations.single_column import * # noqa from loki.transformations.pool_allocator import * # noqa diff --git a/loki/transformations/single_column/__init__.py b/loki/transformations/single_column/__init__.py new file mode 100644 index 000000000..f3598dd1e --- /dev/null +++ b/loki/transformations/single_column/__init__.py @@ -0,0 +1,14 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.single_column.annotate import * # noqa +from loki.transformations.single_column.base import * # noqa +from loki.transformations.single_column.claw import * # noqa +from loki.transformations.single_column.hoist import * # noqa +from loki.transformations.single_column.scc import * # noqa +from loki.transformations.single_column.scc_cuf import * # noqa +from loki.transformations.single_column.vector import * # noqa diff --git a/loki/transformations/single_column_annotate.py b/loki/transformations/single_column/annotate.py similarity index 99% rename from loki/transformations/single_column_annotate.py rename to loki/transformations/single_column/annotate.py index 0be1fa0fa..bcb986555 100644 --- a/loki/transformations/single_column_annotate.py +++ b/loki/transformations/single_column/annotate.py @@ -19,7 +19,7 @@ from loki.tools import as_tuple, flatten, CaseInsensitiveDict from loki.types import DerivedType -from loki.transformations.single_column_base import SCCBaseTransformation +from loki.transformations.single_column.base import SCCBaseTransformation __all__ = ['SCCAnnotateTransformation'] diff --git a/loki/transformations/single_column_base.py b/loki/transformations/single_column/base.py similarity index 100% rename from loki/transformations/single_column_base.py rename to loki/transformations/single_column/base.py diff --git a/loki/transformations/single_column_claw.py b/loki/transformations/single_column/claw.py similarity index 100% rename from loki/transformations/single_column_claw.py rename to loki/transformations/single_column/claw.py diff --git a/loki/transformations/single_column_hoist.py b/loki/transformations/single_column/hoist.py similarity index 98% rename from loki/transformations/single_column_hoist.py rename to loki/transformations/single_column/hoist.py index def5e4afa..a1be96aa5 100644 --- a/loki/transformations/single_column_hoist.py +++ b/loki/transformations/single_column/hoist.py @@ -9,7 +9,7 @@ from loki.ir import nodes as ir from loki.transform import HoistVariablesTransformation -from loki.transformations.single_column_base import SCCBaseTransformation +from loki.transformations.single_column.base import SCCBaseTransformation __all__ = ['SCCHoistTemporaryArraysTransformation'] diff --git a/loki/transformations/single_column_coalesced.py b/loki/transformations/single_column/scc.py similarity index 96% rename from loki/transformations/single_column_coalesced.py rename to loki/transformations/single_column/scc.py index 137539afa..be0bffc1f 100644 --- a/loki/transformations/single_column_coalesced.py +++ b/loki/transformations/single_column/scc.py @@ -11,10 +11,11 @@ from loki.transform import HoistTemporaryArraysAnalysis from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from loki.transformations.single_column_base import SCCBaseTransformation -from loki.transformations.single_column_annotate import SCCAnnotateTransformation -from loki.transformations.single_column_hoist import SCCHoistTemporaryArraysTransformation -from loki.transformations.single_column_coalesced_vector import ( + +from loki.transformations.single_column.base import SCCBaseTransformation +from loki.transformations.single_column.annotate import SCCAnnotateTransformation +from loki.transformations.single_column.hoist import SCCHoistTemporaryArraysTransformation +from loki.transformations.single_column.vector import ( SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation ) diff --git a/loki/transformations/scc_cuf.py b/loki/transformations/single_column/scc_cuf.py similarity index 99% rename from loki/transformations/scc_cuf.py rename to loki/transformations/single_column/scc_cuf.py index 60b671191..235bc50d6 100644 --- a/loki/transformations/scc_cuf.py +++ b/loki/transformations/single_column/scc_cuf.py @@ -18,8 +18,8 @@ CaseInsensitiveDict, as_tuple, flatten, types ) -from loki.transformations.single_column_base import SCCBaseTransformation -from loki.transformations.single_column_coalesced_vector import SCCDevectorTransformation +from loki.transformations.single_column.base import SCCBaseTransformation +from loki.transformations.single_column.vector import SCCDevectorTransformation __all__ = ['SccCufTransformation', 'HoistTemporaryArraysDeviceAllocatableTransformation', 'HoistTemporaryArraysPragmaOffloadTransformation'] diff --git a/loki/transformations/tests/test_single_column.py b/loki/transformations/single_column/tests/test_sca.py similarity index 99% rename from loki/transformations/tests/test_single_column.py rename to loki/transformations/single_column/tests/test_sca.py index 8c2055af8..0088d5b4a 100644 --- a/loki/transformations/tests/test_single_column.py +++ b/loki/transformations/single_column/tests/test_sca.py @@ -9,7 +9,8 @@ from loki import Sourcefile, Dimension from loki.frontend import available_frontends -from loki.transformations import ExtractSCATransformation + +from loki.transformations.single_column import ExtractSCATransformation @pytest.fixture(scope='module', name='horizontal') diff --git a/loki/transformations/tests/test_single_column_coalesced.py b/loki/transformations/single_column/tests/test_single_column_coalesced.py similarity index 99% rename from loki/transformations/tests/test_single_column_coalesced.py rename to loki/transformations/single_column/tests/test_single_column_coalesced.py index 358cfbd04..125f6e5c9 100644 --- a/loki/transformations/tests/test_single_column_coalesced.py +++ b/loki/transformations/single_column/tests/test_single_column_coalesced.py @@ -19,10 +19,11 @@ pragma_regions_attached ) from loki.transform import SanitiseTransformation, InlineTransformation -from loki.transformations import ( - DataOffloadTransformation, SCCBaseTransformation, SCCDevectorTransformation, - SCCDemoteTransformation, SCCRevectorTransformation, SCCAnnotateTransformation, - SCCVectorPipeline, SCCHoistPipeline, +from loki.transformations import DataOffloadTransformation +from loki.transformations.single_column import ( + SCCBaseTransformation, SCCDevectorTransformation, + SCCDemoteTransformation, SCCRevectorTransformation, + SCCAnnotateTransformation, SCCVectorPipeline, SCCHoistPipeline, ) #pylint: disable=too-many-lines diff --git a/loki/transformations/single_column_coalesced_vector.py b/loki/transformations/single_column/vector.py similarity index 99% rename from loki/transformations/single_column_coalesced_vector.py rename to loki/transformations/single_column/vector.py index a411782d6..0f98fce42 100644 --- a/loki/transformations/single_column_coalesced_vector.py +++ b/loki/transformations/single_column/vector.py @@ -20,7 +20,7 @@ from loki.types import BasicType from loki.transform import demote_variables -from loki.transformations.single_column_base import SCCBaseTransformation +from loki.transformations.single_column.base import SCCBaseTransformation __all__ = ['SCCDevectorTransformation', 'SCCRevectorTransformation', 'SCCDemoteTransformation'] diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 5d93253f1..df08d6555 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -37,11 +37,9 @@ from loki.transformations.derived_types import DerivedTypeArgumentsTransformation from loki.transformations.drhook import DrHookTransformation from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation -from loki.transformations.single_column_claw import ExtractSCATransformation, CLAWTransformation -from loki.transformations.single_column_coalesced import ( - SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline -) -from loki.transformations.scc_cuf import ( +from loki.transformations.single_column import ( + ExtractSCATransformation, CLAWTransformation, + SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline, HoistTemporaryArraysDeviceAllocatableTransformation ) From 6350fa1f61efd85be7d652cafcfc47a4250f9528 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Sat, 20 Apr 2024 04:15:15 +0000 Subject: [PATCH 06/13] Transformations: Move transpile transformations into sub-package --- loki/backend/tests/test_pygen.py | 2 +- loki/transform/__init__.py | 3 --- loki/transformations/__init__.py | 1 + loki/transformations/transpile/__init__.py | 10 ++++++++++ .../transpile/fortran_c.py} | 0 .../transpile/fortran_maxeler.py} | 3 ++- .../transpile/fortran_python.py} | 0 .../test_maxeler/passthrough/PassThroughCpuCode.c | 0 .../test_maxeler/passthrough/PassThroughKernel.maxj | 0 .../passthrough/PassThroughMAX5CManager.maxj | 0 .../transpile}/tests/test_maxeler/test_maxeler.py | 0 .../transpile}/tests/test_sdfg.py | 3 ++- .../transpile}/tests/test_transpile.py | 4 +++- scripts/loki_transform.py | 3 ++- 14 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 loki/transformations/transpile/__init__.py rename loki/{transform/fortran_c_transform.py => transformations/transpile/fortran_c.py} (100%) rename loki/{transform/fortran_max_transform.py => transformations/transpile/fortran_maxeler.py} (99%) rename loki/{transform/fortran_python_transform.py => transformations/transpile/fortran_python.py} (100%) rename loki/{ => transformations/transpile}/tests/test_maxeler/passthrough/PassThroughCpuCode.c (100%) rename loki/{ => transformations/transpile}/tests/test_maxeler/passthrough/PassThroughKernel.maxj (100%) rename loki/{ => transformations/transpile}/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj (100%) rename loki/{ => transformations/transpile}/tests/test_maxeler/test_maxeler.py (100%) rename loki/{backend => transformations/transpile}/tests/test_sdfg.py (99%) rename loki/{ => transformations/transpile}/tests/test_transpile.py (99%) diff --git a/loki/backend/tests/test_pygen.py b/loki/backend/tests/test_pygen.py index e17a48fe3..acec99735 100644 --- a/loki/backend/tests/test_pygen.py +++ b/loki/backend/tests/test_pygen.py @@ -16,7 +16,7 @@ from loki.backend import pygen from loki.build import jit_compile, clean_test from loki.frontend import available_frontends, OFP, OMNI -from loki.transform import FortranPythonTransformation +from loki.transformations.transpile import FortranPythonTransformation @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/__init__.py b/loki/transform/__init__.py index 18d3b624d..492463650 100644 --- a/loki/transform/__init__.py +++ b/loki/transform/__init__.py @@ -11,9 +11,6 @@ from loki.transform.transform_loop import * # noqa from loki.transform.transform_region import * # noqa from loki.transform.dependency_transform import * # noqa -from loki.transform.fortran_c_transform import * # noqa -from loki.transform.fortran_max_transform import * # noqa -from loki.transform.fortran_python_transform import * # noqa from loki.transform.build_system_transform import * # noqa from loki.transform.transform_hoist_variables import * # noqa from loki.transform.transform_parametrise import * # noqa diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py index a8d038b1a..5fb33a0bc 100644 --- a/loki/transformations/__init__.py +++ b/loki/transformations/__init__.py @@ -10,4 +10,5 @@ from loki.transformations.data_offload import * # noqa from loki.transformations.drhook import * # noqa from loki.transformations.single_column import * # noqa +from loki.transformations.transpile import * # noqa from loki.transformations.pool_allocator import * # noqa diff --git a/loki/transformations/transpile/__init__.py b/loki/transformations/transpile/__init__.py new file mode 100644 index 000000000..688c2efe8 --- /dev/null +++ b/loki/transformations/transpile/__init__.py @@ -0,0 +1,10 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.transpile.fortran_c import * # noqa +from loki.transformations.transpile.fortran_python import * # noqa +from loki.transformations.transpile.fortran_maxeler import * # noqa diff --git a/loki/transform/fortran_c_transform.py b/loki/transformations/transpile/fortran_c.py similarity index 100% rename from loki/transform/fortran_c_transform.py rename to loki/transformations/transpile/fortran_c.py diff --git a/loki/transform/fortran_max_transform.py b/loki/transformations/transpile/fortran_maxeler.py similarity index 99% rename from loki/transform/fortran_max_transform.py rename to loki/transformations/transpile/fortran_maxeler.py index 663f70d77..80e9484b3 100644 --- a/loki/transform/fortran_max_transform.py +++ b/loki/transformations/transpile/fortran_maxeler.py @@ -23,7 +23,6 @@ from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten -from loki.transform.fortran_c_transform import FortranCTransformation from loki.transform.transform_array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_range_indexing @@ -31,6 +30,8 @@ from loki.transform.transform_utilities import replace_intrinsics from loki.types import SymbolAttributes, BasicType, DerivedType +from loki.transformations.transpile import FortranCTransformation + __all__ = ['FortranMaxTransformation'] diff --git a/loki/transform/fortran_python_transform.py b/loki/transformations/transpile/fortran_python.py similarity index 100% rename from loki/transform/fortran_python_transform.py rename to loki/transformations/transpile/fortran_python.py diff --git a/loki/tests/test_maxeler/passthrough/PassThroughCpuCode.c b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughCpuCode.c similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughCpuCode.c rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughCpuCode.c diff --git a/loki/tests/test_maxeler/passthrough/PassThroughKernel.maxj b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughKernel.maxj similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughKernel.maxj rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughKernel.maxj diff --git a/loki/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj b/loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj similarity index 100% rename from loki/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj rename to loki/transformations/transpile/tests/test_maxeler/passthrough/PassThroughMAX5CManager.maxj diff --git a/loki/tests/test_maxeler/test_maxeler.py b/loki/transformations/transpile/tests/test_maxeler/test_maxeler.py similarity index 100% rename from loki/tests/test_maxeler/test_maxeler.py rename to loki/transformations/transpile/tests/test_maxeler/test_maxeler.py diff --git a/loki/backend/tests/test_sdfg.py b/loki/transformations/transpile/tests/test_sdfg.py similarity index 99% rename from loki/backend/tests/test_sdfg.py rename to loki/transformations/transpile/tests/test_sdfg.py index ceb28954b..97767ab18 100644 --- a/loki/backend/tests/test_sdfg.py +++ b/loki/transformations/transpile/tests/test_sdfg.py @@ -14,7 +14,8 @@ from loki import Subroutine from loki.build import jit_compile, clean_test from loki.frontend import available_frontends -from loki.transform import FortranPythonTransformation + +from loki.transformations.transpile import FortranPythonTransformation pytestmark = [ diff --git a/loki/tests/test_transpile.py b/loki/transformations/transpile/tests/test_transpile.py similarity index 99% rename from loki/tests/test_transpile.py rename to loki/transformations/transpile/tests/test_transpile.py index 5926ce0c0..2e293999d 100644 --- a/loki/tests/test_transpile.py +++ b/loki/transformations/transpile/tests/test_transpile.py @@ -9,13 +9,15 @@ import pytest import numpy as np -from loki import Subroutine, Module, FortranCTransformation, cgen +from loki import Subroutine, Module, cgen from loki.build import jit_compile, jit_compile_lib, clean_test, Builder import loki.expression.symbols as sym from loki.frontend import available_frontends, OFP import loki.ir as ir from loki.transform import normalize_range_indexing +from loki.transformations.transpile import FortranCTransformation + @pytest.fixture(scope='module', name='here') def fixture_here(): diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index df08d6555..3c32e9702 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -23,7 +23,7 @@ # Get generalized transformations provided by Loki from loki.transform import ( - DependencyTransformation, ModuleWrapTransformation, FortranCTransformation, + DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation, HoistTemporaryArraysAnalysis, normalize_range_indexing, InlineTransformation, SanitiseTransformation, RemoveCodeTransformation ) @@ -42,6 +42,7 @@ SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline, HoistTemporaryArraysDeviceAllocatableTransformation ) +from loki.transformations.transpile import FortranCTransformation class IdemTransformation(Transformation): From 8306b2d4b00c074ad7d11466e771b101894492d9 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Sat, 20 Apr 2024 04:38:16 +0000 Subject: [PATCH 07/13] Transformations: Move build system transformations into sub-package This also split the module_wrap transformation from the dependency transformation and puts it into its own source file. --- loki/batch/tests/test_scheduler.py | 10 +- loki/transform/__init__.py | 2 - loki/transformations/__init__.py | 1 + loki/transformations/build_system/__init__.py | 10 + .../build_system/dependency.py} | 167 +---------------- .../build_system/file_write.py} | 0 .../build_system/module_wrap.py | 175 ++++++++++++++++++ .../tests/test_transform_dependency.py | 13 +- scripts/loki_transform.py | 6 +- 9 files changed, 206 insertions(+), 178 deletions(-) create mode 100644 loki/transformations/build_system/__init__.py rename loki/{transform/dependency_transform.py => transformations/build_system/dependency.py} (68%) rename loki/{transform/build_system_transform.py => transformations/build_system/file_write.py} (100%) create mode 100644 loki/transformations/build_system/module_wrap.py rename loki/{transform => transformations/build_system}/tests/test_transform_dependency.py (99%) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 322bdfb4c..f71f53862 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -2386,7 +2386,7 @@ def test_transformation_config(config): my_config = config.copy() my_config['transformations'] = { 'DependencyTransformation': { - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': { 'suffix': '_rick', @@ -2408,7 +2408,7 @@ def test_transformation_config(config): bad_config = config.copy() bad_config['transformations'] = { 'DependencyTrafo': { # <= typo - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': {} } } @@ -2418,7 +2418,7 @@ def test_transformation_config(config): worse_config = config.copy() worse_config['transformations'] = { 'DependencyTransform': { - 'module': 'loki.transformats', # <= typo + 'module': 'loki.transformats.build_system', # <= typo 'options': {} } } @@ -2428,7 +2428,7 @@ def test_transformation_config(config): worst_config = config.copy() worst_config['transformations'] = { 'DependencyTransform': { - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': {'hello': 'Dave'} } } @@ -2893,7 +2893,7 @@ def test_pipeline_config_compose(config): }, 'postprocess': { 'classname': 'ModuleWrapTransformation', - 'module': 'loki.transform', + 'module': 'loki.transformations.build_system', 'options': { 'module_suffix': '_module' } } } diff --git a/loki/transform/__init__.py b/loki/transform/__init__.py index 492463650..b2c277365 100644 --- a/loki/transform/__init__.py +++ b/loki/transform/__init__.py @@ -10,8 +10,6 @@ from loki.transform.transform_inline import * # noqa from loki.transform.transform_loop import * # noqa from loki.transform.transform_region import * # noqa -from loki.transform.dependency_transform import * # noqa -from loki.transform.build_system_transform import * # noqa from loki.transform.transform_hoist_variables import * # noqa from loki.transform.transform_parametrise import * # noqa from loki.transform.transform_extract_contained_procedures import * # noqa diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py index 5fb33a0bc..bf1b0b8b4 100644 --- a/loki/transformations/__init__.py +++ b/loki/transformations/__init__.py @@ -5,6 +5,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. +from loki.transformations.build_system import * # noqa from loki.transformations.derived_types import * # noqa from loki.transformations.argument_shape import * # noqa from loki.transformations.data_offload import * # noqa diff --git a/loki/transformations/build_system/__init__.py b/loki/transformations/build_system/__init__.py new file mode 100644 index 000000000..175ed0eca --- /dev/null +++ b/loki/transformations/build_system/__init__.py @@ -0,0 +1,10 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.transformations.build_system.dependency import * # noqa +from loki.transformations.build_system.file_write import * # noqa +from loki.transformations.build_system.module_wrap import * # noqa diff --git a/loki/transform/dependency_transform.py b/loki/transformations/build_system/dependency.py similarity index 68% rename from loki/transform/dependency_transform.py rename to loki/transformations/build_system/dependency.py index 735c7f822..b87a6b52f 100644 --- a/loki/transform/dependency_transform.py +++ b/loki/transformations/build_system/dependency.py @@ -9,10 +9,8 @@ from loki.backend import fgen from loki.batch import Transformation -from loki.expression import Variable, FindInlineCalls -from loki.ir import ( - CallStatement, Import, Section, Interface, FindNodes, Transformer -) +from loki.expression import FindInlineCalls +from loki.ir import CallStatement, Import, Interface, FindNodes, Transformer from loki.logging import warning from loki.module import Module from loki.scope import Scope @@ -21,7 +19,7 @@ from loki.tools import as_tuple -__all__ = ['DependencyTransformation', 'ModuleWrapTransformation'] +__all__ = ['DependencyTransformation'] class DependencyTransformation(Transformation): @@ -355,162 +353,3 @@ def generate_interfaces(self, routine): intfb_path = self.include_path/f'{routine.name.lower()}.intfb.h' with intfb_path.open('w') as f: f.write(fgen(routine.interface)) - - -class ModuleWrapTransformation(Transformation): - """ - Utility transformation that ensures all transformed kernel - subroutines are wrapped in a module - - The module name is derived from the subroutine name and :data:`module_suffix`. - - Any previous import of wrapped subroutines via interfaces or C-style header - imports of interface blocks is replaced by a Fortran import (``USE``). - - Parameters - ---------- - module_suffix : str - Special suffix to signal module names like `_MOD` - replace_ignore_items : bool - Debug flag to toggle the replacement of calls to subroutines - in the ``ignore``. Default is ``True``. - """ - - # This transformation is applied over the file graph - traverse_file_graph = True - - # This transformation recurses from the Sourcefile down - recurse_to_modules = True - recurse_to_procedures = True - recurse_to_internal_procedures = False - - # This transformation changes the names of items and creates new items - renames_items = True - creates_items = True - - def __init__(self, module_suffix, replace_ignore_items=True): - self.module_suffix = module_suffix - self.replace_ignore_items = replace_ignore_items - - def transform_file(self, sourcefile, **kwargs): - """ - For kernel routines, wrap each subroutine in the current file in a module - """ - role = kwargs.get('role') - - if items := kwargs.get('items'): - # We consider the sourcefile to be a "kernel" file if all items are kernels - if all(item.role == 'kernel' for item in items): - role = 'kernel' - else: - role = 'driver' - - if role == 'kernel': - self.module_wrap(sourcefile) - - def transform_module(self, module, **kwargs): - """ - Update imports of wrapped subroutines - """ - self.update_imports(module, imports=module.imports, **kwargs) - - def transform_subroutine(self, routine, **kwargs): - """ - Update imports of wrapped subroutines - """ - if item := kwargs.get('item'): - # Rename the item if it has suddenly a parent - if routine.parent and routine.parent.name.lower() != item.scope_name: - item.name = f'{routine.parent.name.lower()}#{item.local_name}' - - # Note, C-style imports can be in the body, so use whole IR - imports = FindNodes(Import).visit(routine.ir) - self.update_imports(routine, imports=imports, **kwargs) - - # Interface blocks can only be in the spec - intfs = FindNodes(Interface).visit(routine.spec) - self.replace_interfaces(routine, intfs=intfs, **kwargs) - - def module_wrap(self, sourcefile): - """ - Wrap target subroutines in modules and replace in source file. - """ - for routine in sourcefile.subroutines: - # Create wrapper module and insert into file, replacing the old - # standalone routine - modname = f'{routine.name}{self.module_suffix}' - module = Module(name=modname, contains=Section(body=as_tuple(routine))) - routine.parent = module - sourcefile.ir._update(body=as_tuple( - module if c is routine else c for c in sourcefile.ir.body - )) - - def update_imports(self, source, imports, **kwargs): - """ - Update imports of wrapped subroutines. - """ - from loki.batch import SchedulerConfig # pylint: disable=import-outside-toplevel,cyclic-import - - targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) - if self.replace_ignore_items and (item := kwargs.get('item')): - targets += tuple(str(i).lower() for i in item.ignore) - - def _update_item(proc_name, module_name): - if item and (matched_keys := SchedulerConfig.match_item_keys(proc_name, item.ignore)): - # Add the module wrapped but ignored items to the block list because we won't be able to - # find them as dependencies under their new name anymore - item.config['block'] = as_tuple(item.block) + tuple( - module_name for name in item.ignore if name in matched_keys - ) - - # Transformer map to remove any outdated imports - removal_map = {} - - # We go through the IR, as C-imports can be attributed to the body - for im in imports: - if im.c_import: - target_symbol, *suffixes = im.module.lower().split('.', maxsplit=1) - if targets and target_symbol.lower() in targets and not 'func.h' in suffixes: - # Create a new module import with explicitly qualified symbol - modname = f'{target_symbol}{self.module_suffix}' - _update_item(target_symbol.lower(), modname) - new_symbol = Variable(name=target_symbol, scope=source) - new_import = im.clone(module=modname, c_import=False, symbols=(new_symbol,)) - source.spec.prepend(new_import) - - # Mark current import for removal - removal_map[im] = None - - # Apply any scheduled import removals to spec and body - if removal_map: - source.spec = Transformer(removal_map).visit(source.spec) - if isinstance(source, Subroutine): - source.body = Transformer(removal_map).visit(source.body) - - def replace_interfaces(self, source, intfs, **kwargs): - """ - Update explicit interfaces to actively transformed subroutines. - """ - targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) - if self.replace_ignore_items and (item := kwargs.get('item')): - targets += tuple(str(i).lower() for i in item.ignore) - - # Transformer map to remove any outdated interfaces - removal_map = {} - - for i in intfs: - for b in i.body: - if isinstance(b, Subroutine): - if targets and b.name.lower() in targets: - # Create a new module import with explicitly qualified symbol - modname = f'{b.name}{self.module_suffix}' - new_symbol = Variable(name=f'{b.name}', scope=source) - new_import = Import(module=modname, c_import=False, symbols=(new_symbol,)) - source.spec.prepend(new_import) - - # Mark current import for removal - removal_map[i] = None - - # Apply any scheduled interface removals to spec - if removal_map: - source.spec = Transformer(removal_map).visit(source.spec) diff --git a/loki/transform/build_system_transform.py b/loki/transformations/build_system/file_write.py similarity index 100% rename from loki/transform/build_system_transform.py rename to loki/transformations/build_system/file_write.py diff --git a/loki/transformations/build_system/module_wrap.py b/loki/transformations/build_system/module_wrap.py new file mode 100644 index 000000000..af261e45f --- /dev/null +++ b/loki/transformations/build_system/module_wrap.py @@ -0,0 +1,175 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +from loki.batch import Transformation +from loki.expression import Variable +from loki.ir import Import, Section, Interface, FindNodes, Transformer +from loki.module import Module +from loki.subroutine import Subroutine +from loki.tools import as_tuple + + +__all__ = ['ModuleWrapTransformation'] + + +class ModuleWrapTransformation(Transformation): + """ + Utility transformation that ensures all transformed kernel + subroutines are wrapped in a module + + The module name is derived from the subroutine name and :data:`module_suffix`. + + Any previous import of wrapped subroutines via interfaces or C-style header + imports of interface blocks is replaced by a Fortran import (``USE``). + + Parameters + ---------- + module_suffix : str + Special suffix to signal module names like `_MOD` + replace_ignore_items : bool + Debug flag to toggle the replacement of calls to subroutines + in the ``ignore``. Default is ``True``. + """ + + # This transformation is applied over the file graph + traverse_file_graph = True + + # This transformation recurses from the Sourcefile down + recurse_to_modules = True + recurse_to_procedures = True + recurse_to_internal_procedures = False + + # This transformation changes the names of items and creates new items + renames_items = True + creates_items = True + + def __init__(self, module_suffix, replace_ignore_items=True): + self.module_suffix = module_suffix + self.replace_ignore_items = replace_ignore_items + + def transform_file(self, sourcefile, **kwargs): + """ + For kernel routines, wrap each subroutine in the current file in a module + """ + role = kwargs.get('role') + + if items := kwargs.get('items'): + # We consider the sourcefile to be a "kernel" file if all items are kernels + if all(item.role == 'kernel' for item in items): + role = 'kernel' + else: + role = 'driver' + + if role == 'kernel': + self.module_wrap(sourcefile) + + def transform_module(self, module, **kwargs): + """ + Update imports of wrapped subroutines + """ + self.update_imports(module, imports=module.imports, **kwargs) + + def transform_subroutine(self, routine, **kwargs): + """ + Update imports of wrapped subroutines + """ + if item := kwargs.get('item'): + # Rename the item if it has suddenly a parent + if routine.parent and routine.parent.name.lower() != item.scope_name: + item.name = f'{routine.parent.name.lower()}#{item.local_name}' + + # Note, C-style imports can be in the body, so use whole IR + imports = FindNodes(Import).visit(routine.ir) + self.update_imports(routine, imports=imports, **kwargs) + + # Interface blocks can only be in the spec + intfs = FindNodes(Interface).visit(routine.spec) + self.replace_interfaces(routine, intfs=intfs, **kwargs) + + def module_wrap(self, sourcefile): + """ + Wrap target subroutines in modules and replace in source file. + """ + for routine in sourcefile.subroutines: + # Create wrapper module and insert into file, replacing the old + # standalone routine + modname = f'{routine.name}{self.module_suffix}' + module = Module(name=modname, contains=Section(body=as_tuple(routine))) + routine.parent = module + sourcefile.ir._update(body=as_tuple( + module if c is routine else c for c in sourcefile.ir.body + )) + + def update_imports(self, source, imports, **kwargs): + """ + Update imports of wrapped subroutines. + """ + from loki.batch import SchedulerConfig # pylint: disable=import-outside-toplevel,cyclic-import + + targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) + if self.replace_ignore_items and (item := kwargs.get('item')): + targets += tuple(str(i).lower() for i in item.ignore) + + def _update_item(proc_name, module_name): + if item and (matched_keys := SchedulerConfig.match_item_keys(proc_name, item.ignore)): + # Add the module wrapped but ignored items to the block list because we won't be able to + # find them as dependencies under their new name anymore + item.config['block'] = as_tuple(item.block) + tuple( + module_name for name in item.ignore if name in matched_keys + ) + + # Transformer map to remove any outdated imports + removal_map = {} + + # We go through the IR, as C-imports can be attributed to the body + for im in imports: + if im.c_import: + target_symbol, *suffixes = im.module.lower().split('.', maxsplit=1) + if targets and target_symbol.lower() in targets and not 'func.h' in suffixes: + # Create a new module import with explicitly qualified symbol + modname = f'{target_symbol}{self.module_suffix}' + _update_item(target_symbol.lower(), modname) + new_symbol = Variable(name=target_symbol, scope=source) + new_import = im.clone(module=modname, c_import=False, symbols=(new_symbol,)) + source.spec.prepend(new_import) + + # Mark current import for removal + removal_map[im] = None + + # Apply any scheduled import removals to spec and body + if removal_map: + source.spec = Transformer(removal_map).visit(source.spec) + if isinstance(source, Subroutine): + source.body = Transformer(removal_map).visit(source.body) + + def replace_interfaces(self, source, intfs, **kwargs): + """ + Update explicit interfaces to actively transformed subroutines. + """ + targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) + if self.replace_ignore_items and (item := kwargs.get('item')): + targets += tuple(str(i).lower() for i in item.ignore) + + # Transformer map to remove any outdated interfaces + removal_map = {} + + for i in intfs: + for b in i.body: + if isinstance(b, Subroutine): + if targets and b.name.lower() in targets: + # Create a new module import with explicitly qualified symbol + modname = f'{b.name}{self.module_suffix}' + new_symbol = Variable(name=f'{b.name}', scope=source) + new_import = Import(module=modname, c_import=False, symbols=(new_symbol,)) + source.spec.prepend(new_import) + + # Mark current import for removal + removal_map[i] = None + + # Apply any scheduled interface removals to spec + if removal_map: + source.spec = Transformer(removal_map).visit(source.spec) diff --git a/loki/transform/tests/test_transform_dependency.py b/loki/transformations/build_system/tests/test_transform_dependency.py similarity index 99% rename from loki/transform/tests/test_transform_dependency.py rename to loki/transformations/build_system/tests/test_transform_dependency.py index c05558aa4..9cb0fec7f 100644 --- a/loki/transform/tests/test_transform_dependency.py +++ b/loki/transformations/build_system/tests/test_transform_dependency.py @@ -2,12 +2,15 @@ from shutil import rmtree import pytest -from loki import ( - gettempdir, Sourcefile, CallStatement, Import, Interface, - FindNodes, FindInlineCalls, Intrinsic, Scheduler, SchedulerConfig -) +from loki import Sourcefile, gettempdir +from loki.batch import Scheduler, SchedulerConfig +from loki.expression import FindInlineCalls from loki.frontend import available_frontends, OMNI, OFP -from loki.transform import DependencyTransformation, ModuleWrapTransformation +from loki.ir import FindNodes, CallStatement, Import, Interface, Intrinsic + +from loki.transformations import ( + DependencyTransformation, ModuleWrapTransformation +) @pytest.fixture(scope='module', name='here') diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 3c32e9702..23fce5552 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -23,14 +23,16 @@ # Get generalized transformations provided by Loki from loki.transform import ( - DependencyTransformation, ModuleWrapTransformation, - FileWriteTransformation, HoistTemporaryArraysAnalysis, normalize_range_indexing, + HoistTemporaryArraysAnalysis, normalize_range_indexing, InlineTransformation, SanitiseTransformation, RemoveCodeTransformation ) from loki.transformations.argument_shape import ( ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation ) +from loki.transformations.build_system import ( + DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation +) from loki.transformations.data_offload import ( DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation ) From 5de0b0e470e1fea4bafe56c26c29a0429bea789a Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Mon, 22 Apr 2024 15:25:06 +0000 Subject: [PATCH 08/13] Transformations: Moving remaining high-level transformations over --- loki/__init__.py | 1 - loki/batch/tests/test_scheduler.py | 6 ++- loki/batch/tests/test_transformation.py | 2 +- loki/transform/__init__.py | 17 -------- loki/transformations/__init__.py | 10 +++++ .../array_indexing.py} | 0 loki/transformations/derived_types.py | 14 +++++-- .../extract.py} | 0 .../hoist_variables.py} | 10 ++--- .../inline.py} | 19 ++++----- .../parametrise.py} | 2 +- loki/transformations/pool_allocator.py | 27 ++++++------ .../remove_code.py} | 0 .../sanitise.py} | 2 +- loki/transformations/single_column/base.py | 11 ++--- loki/transformations/single_column/claw.py | 17 +++++--- loki/transformations/single_column/hoist.py | 2 +- loki/transformations/single_column/scc.py | 2 +- loki/transformations/single_column/scc_cuf.py | 22 ++++++---- .../tests/test_single_column_coalesced.py | 6 ++- loki/transformations/single_column/vector.py | 7 +++- .../tests/test_array_indexing.py} | 5 ++- .../tests/test_extract.py} | 3 +- .../tests/test_hoist_variables.py} | 2 +- .../tests/test_inline.py} | 11 +++-- .../tests/test_parametrise.py} | 3 +- .../tests/test_remove_code.py} | 3 +- .../tests/test_sanitise.py} | 3 +- loki/transformations/tests/test_scc_cuf.py | 11 +++-- .../tests/test_transform_derived_types.py | 4 +- .../tests/test_transform_loop.py | 6 ++- .../tests/test_transform_region.py | 6 ++- .../tests/test_utilities.py} | 3 +- .../transform_loop.py | 15 ++++--- .../transform_region.py | 3 +- loki/transformations/transpile/fortran_c.py | 41 +++++++++---------- .../transpile/fortran_maxeler.py | 4 +- .../transpile/fortran_python.py | 6 +-- .../transpile/tests/test_transpile.py | 2 +- .../utilities.py} | 0 scripts/loki_transform.py | 10 ++--- 41 files changed, 177 insertions(+), 141 deletions(-) delete mode 100644 loki/transform/__init__.py rename loki/{transform/transform_array_indexing.py => transformations/array_indexing.py} (100%) rename loki/{transform/transform_extract_contained_procedures.py => transformations/extract.py} (100%) rename loki/{transform/transform_hoist_variables.py => transformations/hoist_variables.py} (98%) rename loki/{transform/transform_inline.py => transformations/inline.py} (98%) rename loki/{transform/transform_parametrise.py => transformations/parametrise.py} (99%) rename loki/{transform/transform_remove_code.py => transformations/remove_code.py} (100%) rename loki/{transform/transform_sanitise.py => transformations/sanitise.py} (98%) rename loki/{transform/tests/test_transform_array_indexing.py => transformations/tests/test_array_indexing.py} (99%) rename loki/{tests/test_extract_contained_procedures.py => transformations/tests/test_extract.py} (99%) rename loki/{transform/tests/test_transform_hoist_variables.py => transformations/tests/test_hoist_variables.py} (99%) rename loki/{transform/tests/test_transform_inline.py => transformations/tests/test_inline.py} (99%) rename loki/{transform/tests/test_transform_parametrise.py => transformations/tests/test_parametrise.py} (99%) rename loki/{transform/tests/test_transform_remove_code.py => transformations/tests/test_remove_code.py} (99%) rename loki/{transform/tests/test_transform_sanitise.py => transformations/tests/test_sanitise.py} (99%) rename loki/{transform => transformations}/tests/test_transform_loop.py (99%) rename loki/{transform => transformations}/tests/test_transform_region.py (99%) rename loki/{transform/tests/test_transform_utilities.py => transformations/tests/test_utilities.py} (99%) rename loki/{transform => transformations}/transform_loop.py (99%) rename loki/{transform => transformations}/transform_region.py (99%) rename loki/{transform/transform_utilities.py => transformations/utilities.py} (100%) diff --git a/loki/__init__.py b/loki/__init__.py index 2d649baf8..a9f7ad1ea 100644 --- a/loki/__init__.py +++ b/loki/__init__.py @@ -22,7 +22,6 @@ from loki.tools import * # noqa from loki.logging import * # noqa from loki.backend import * # noqa -from loki.transform import * # noqa from loki.build import * # noqa # pylint: disable=redefined-builtin from loki.batch import * # noqa from loki.lint import * # noqa diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index f71f53862..f6e6318a4 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -74,7 +74,9 @@ available_frontends, OMNI, OFP, FP, REGEX, HAVE_FP, HAVE_OFP, HAVE_OMNI ) from loki.ir import nodes as ir, FindNodes -from loki.transform import DependencyTransformation, ModuleWrapTransformation +from loki.transformations import ( + DependencyTransformation, ModuleWrapTransformation +) pytestmark = pytest.mark.skipif(not HAVE_FP and not HAVE_OFP, reason='Fparser and OFP not available') @@ -2885,7 +2887,7 @@ def test_pipeline_config_compose(config): }, 'preprocess': { 'classname': 'RemoveCodeTransformation', - 'module': 'loki.transform.transform_remove_code', + 'module': 'loki.transformations', 'options': { 'call_names': 'dr_hook', 'remove_imports': False diff --git a/loki/batch/tests/test_transformation.py b/loki/batch/tests/test_transformation.py index 26a5b4b05..7f4cf6a08 100644 --- a/loki/batch/tests/test_transformation.py +++ b/loki/batch/tests/test_transformation.py @@ -9,7 +9,7 @@ from loki.build import jit_compile, clean_test from loki.frontend import available_frontends, OMNI, REGEX from loki.ir import nodes as ir, FindNodes -from loki.transform import ( +from loki.transformations import ( replace_selected_kind, FileWriteTransformation ) diff --git a/loki/transform/__init__.py b/loki/transform/__init__.py deleted file mode 100644 index b2c277365..000000000 --- a/loki/transform/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# (C) Copyright 2018- ECMWF. -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -from loki.transform.transform_utilities import * # noqa -from loki.transform.transform_array_indexing import * # noqa -from loki.transform.transform_inline import * # noqa -from loki.transform.transform_loop import * # noqa -from loki.transform.transform_region import * # noqa -from loki.transform.transform_hoist_variables import * # noqa -from loki.transform.transform_parametrise import * # noqa -from loki.transform.transform_extract_contained_procedures import * # noqa -from loki.transform.transform_remove_code import * # noqa -from loki.transform.transform_sanitise import * # noqa diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py index bf1b0b8b4..aa8f2c93a 100644 --- a/loki/transformations/__init__.py +++ b/loki/transformations/__init__.py @@ -5,11 +5,21 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. +from loki.transformations.array_indexing import * # noqa from loki.transformations.build_system import * # noqa from loki.transformations.derived_types import * # noqa from loki.transformations.argument_shape import * # noqa from loki.transformations.data_offload import * # noqa from loki.transformations.drhook import * # noqa +from loki.transformations.extract import * # noqa +from loki.transformations.hoist_variables import * # noqa +from loki.transformations.inline import * # noqa +from loki.transformations.parametrise import * # noqa +from loki.transformations.remove_code import * # noqa +from loki.transformations.sanitise import * # noqa from loki.transformations.single_column import * # noqa from loki.transformations.transpile import * # noqa +from loki.transformations.transform_loop import * # noqa +from loki.transformations.transform_region import * # noqa from loki.transformations.pool_allocator import * # noqa +from loki.transformations.utilities import * # noqa diff --git a/loki/transform/transform_array_indexing.py b/loki/transformations/array_indexing.py similarity index 100% rename from loki/transform/transform_array_indexing.py rename to loki/transformations/array_indexing.py diff --git a/loki/transformations/derived_types.py b/loki/transformations/derived_types.py index 9f11cfc7c..f6e5f1c0f 100644 --- a/loki/transformations/derived_types.py +++ b/loki/transformations/derived_types.py @@ -15,12 +15,18 @@ """ from collections import defaultdict + from loki import ( - Transformation, FindVariables, FindNodes, FindInlineCalls, Transformer, - SubstituteExpressions, SubstituteExpressionsMapper, ExpressionRetriever, recursive_expression_map_update, - Module, Import, CallStatement, ProcedureDeclaration, InlineCall, Variable, RangeIndex, - BasicType, DerivedType, as_tuple, flatten, warning, debug, CaseInsensitiveDict, ProcedureType + Transformation, SubstituteExpressions, Module, + ProcedureDeclaration, BasicType, DerivedType, as_tuple, flatten, + warning, debug, CaseInsensitiveDict, ProcedureType +) +from loki.ir import FindNodes, Import, CallStatement, Transformer +from loki.expression import ( + FindInlineCalls, FindVariables, InlineCall, Variable, RangeIndex, + ExpressionRetriever, SubstituteExpressionsMapper ) +from loki.transformations.utilities import recursive_expression_map_update __all__ = ['DerivedTypeArgumentsTransformation', 'TypeboundProcedureCallTransformation'] diff --git a/loki/transform/transform_extract_contained_procedures.py b/loki/transformations/extract.py similarity index 100% rename from loki/transform/transform_extract_contained_procedures.py rename to loki/transformations/extract.py diff --git a/loki/transform/transform_hoist_variables.py b/loki/transformations/hoist_variables.py similarity index 98% rename from loki/transform/transform_hoist_variables.py rename to loki/transformations/hoist_variables.py index 7c0770d18..4bf855ba2 100644 --- a/loki/transform/transform_hoist_variables.py +++ b/loki/transformations/hoist_variables.py @@ -81,16 +81,16 @@ """ from loki.batch import Transformation, ProcedureItem -from loki.expression import FindVariables, SubstituteExpressions -from loki.expression.expr_visitors import FindInlineCalls -import loki.expression.symbols as sym -from loki.expression.symbolic import is_dimension_constant +from loki.expression import ( + symbols as sym, FindVariables, FindInlineCalls, + SubstituteExpressions, is_dimension_constant +) from loki.ir import ( CallStatement, Allocation, Deallocation, Transformer, FindNodes ) from loki.tools.util import is_iterable, as_tuple, CaseInsensitiveDict -from loki.transform.transform_utilities import single_variable_declaration +from loki.transformations.utilities import single_variable_declaration __all__ = [ diff --git a/loki/transform/transform_inline.py b/loki/transformations/inline.py similarity index 98% rename from loki/transform/transform_inline.py rename to loki/transformations/inline.py index a15621d4b..f2a9b7026 100644 --- a/loki/transform/transform_inline.py +++ b/loki/transformations/inline.py @@ -13,27 +13,26 @@ from collections import defaultdict from loki.batch import Transformation -from loki.expression import ( - FindVariables, FindInlineCalls, FindLiterals, - SubstituteExpressions, LokiIdentityMapper -) from loki.ir import ( Import, Comment, Assignment, VariableDeclaration, CallStatement, Transformer, FindNodes, pragmas_attached, is_loki_pragma ) -from loki.expression import symbols as sym +from loki.expression import ( + symbols as sym, FindVariables, FindInlineCalls, FindLiterals, + SubstituteExpressions, LokiIdentityMapper +) from loki.types import BasicType from loki.tools import as_tuple from loki.logging import warning, error from loki.subroutine import Subroutine -from loki.transform.transform_sanitise import transform_sequence_association_append_map -from loki.transform.transform_remove_code import do_remove_dead_code -from loki.transform.transform_utilities import ( - single_variable_declaration, - recursive_expression_map_update +from loki.transformations.remove_code import do_remove_dead_code +from loki.transformations.sanitise import transform_sequence_association_append_map +from loki.transformations.utilities import ( + single_variable_declaration, recursive_expression_map_update ) + __all__ = [ 'inline_constant_parameters', 'inline_elemental_functions', 'inline_internal_procedures', 'inline_member_procedures', diff --git a/loki/transform/transform_parametrise.py b/loki/transformations/parametrise.py similarity index 99% rename from loki/transform/transform_parametrise.py rename to loki/transformations/parametrise.py index 4ebc4982a..a21123025 100644 --- a/loki/transform/transform_parametrise.py +++ b/loki/transformations/parametrise.py @@ -87,7 +87,7 @@ from loki.ir import nodes as ir, Transformer, FindNodes from loki.tools.util import as_tuple, CaseInsensitiveDict -from loki.transform.transform_inline import inline_constant_parameters +from loki.transformations.inline import inline_constant_parameters __all__ = ['ParametriseTransformation'] diff --git a/loki/transformations/pool_allocator.py b/loki/transformations/pool_allocator.py index e99278366..99054629c 100644 --- a/loki/transformations/pool_allocator.py +++ b/loki/transformations/pool_allocator.py @@ -8,23 +8,24 @@ import re from collections import defaultdict +from loki import ( + as_tuple, warning, debug, Transformation, DetachScopesMapper, + SymbolAttributes, BasicType, DerivedType, +) +from loki.analyse import dataflow_analysis_attached from loki.expression import ( - Quotient, IntLiteral, LogicLiteral, Variable, Array, Sum, Literal, - Product, InlineCall, Comparison, RangeIndex, Cast, - ProcedureSymbol, LogicalNot, simplify, + FindVariables, FindInlineCalls, Quotient, IntLiteral, + LogicLiteral, Variable, Array, Sum, Literal, Product, InlineCall, + Comparison, RangeIndex, Cast, ProcedureSymbol, LogicalNot, + simplify, SubstituteExpressions, is_dimension_constant ) from loki.ir import ( - Intrinsic, Assignment, Conditional, CallStatement, Import, - Allocation, Deallocation, Loop, Pragma -) -from loki import ( - as_tuple, warning, debug, Transformation, FindNodes, - FindVariables, Transformer, SubstituteExpressions, - DetachScopesMapper, SymbolAttributes, BasicType, DerivedType, - is_dimension_constant, recursive_expression_map_update, - get_pragma_parameters, FindInlineCalls, Interface, - dataflow_analysis_attached + FindNodes, Transformer, Intrinsic, Assignment, Conditional, + CallStatement, Import, Allocation, Deallocation, Loop, Pragma, + Interface, get_pragma_parameters ) +from loki.transformations.utilities import recursive_expression_map_update + __all__ = ['TemporariesPoolAllocatorTransformation'] diff --git a/loki/transform/transform_remove_code.py b/loki/transformations/remove_code.py similarity index 100% rename from loki/transform/transform_remove_code.py rename to loki/transformations/remove_code.py diff --git a/loki/transform/transform_sanitise.py b/loki/transformations/sanitise.py similarity index 98% rename from loki/transform/transform_sanitise.py rename to loki/transformations/sanitise.py index 3268291ae..e2181c539 100644 --- a/loki/transform/transform_sanitise.py +++ b/loki/transformations/sanitise.py @@ -18,7 +18,7 @@ from loki.tools import as_tuple, CaseInsensitiveDict from loki.types import BasicType -from loki.transform.transform_utilities import recursive_expression_map_update +from loki.transformations.utilities import recursive_expression_map_update __all__ = [ diff --git a/loki/transformations/single_column/base.py b/loki/transformations/single_column/base.py index 5901d6944..35037d730 100644 --- a/loki/transformations/single_column/base.py +++ b/loki/transformations/single_column/base.py @@ -5,13 +5,14 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from loki.expression import symbols as sym -from loki.transform import resolve_associates from loki import ( - ir, Transformation, FindNodes, Transformer, - as_tuple, FindExpressions, - SymbolAttributes, BasicType, SubstituteExpressions + Transformation, FindExpressions, SymbolAttributes, BasicType, + SubstituteExpressions, as_tuple, ) +from loki.expression import symbols as sym +from loki.ir import nodes as ir, FindNodes, Transformer + +from loki.transformations.sanitise import resolve_associates __all__ = ['SCCBaseTransformation'] diff --git a/loki/transformations/single_column/claw.py b/loki/transformations/single_column/claw.py index 35a5c9956..46353481b 100644 --- a/loki/transformations/single_column/claw.py +++ b/loki/transformations/single_column/claw.py @@ -11,13 +11,20 @@ """ from collections import OrderedDict + from loki import ( - Transformation, FindVariables, FindNodes, Transformer, SubstituteExpressions, - Assignment, CallStatement, Loop, Variable, - Array, Pragma, VariableDeclaration, LoopRange, RangeIndex, - SymbolAttributes, BasicType, CaseInsensitiveDict, as_tuple, warning, - recursive_expression_map_update + Transformation, SymbolAttributes, BasicType, CaseInsensitiveDict, + as_tuple, warning, +) +from loki.ir import ( + FindNodes, Transformer, Assignment, CallStatement, Loop, Pragma, + VariableDeclaration, +) +from loki.expression import ( + FindVariables, SubstituteExpressions, Variable, + Array, LoopRange, RangeIndex ) +from loki.transformations.utilities import recursive_expression_map_update __all__ = ['ExtractSCATransformation', 'CLAWTransformation'] diff --git a/loki/transformations/single_column/hoist.py b/loki/transformations/single_column/hoist.py index a1be96aa5..61bf906c0 100644 --- a/loki/transformations/single_column/hoist.py +++ b/loki/transformations/single_column/hoist.py @@ -7,8 +7,8 @@ from loki.expression import symbols as sym from loki.ir import nodes as ir -from loki.transform import HoistVariablesTransformation +from loki.transformations.hoist_variables import HoistVariablesTransformation from loki.transformations.single_column.base import SCCBaseTransformation diff --git a/loki/transformations/single_column/scc.py b/loki/transformations/single_column/scc.py index be0bffc1f..2e06ee7a3 100644 --- a/loki/transformations/single_column/scc.py +++ b/loki/transformations/single_column/scc.py @@ -8,8 +8,8 @@ from functools import partial from loki.batch import Pipeline -from loki.transform import HoistTemporaryArraysAnalysis +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation from loki.transformations.single_column.base import SCCBaseTransformation diff --git a/loki/transformations/single_column/scc_cuf.py b/loki/transformations/single_column/scc_cuf.py index 235bc50d6..411169d32 100644 --- a/loki/transformations/single_column/scc_cuf.py +++ b/loki/transformations/single_column/scc_cuf.py @@ -9,20 +9,26 @@ Single-Column-Coalesced CUDA Fortran (SCC-CUF) transformation. """ -from loki.expression import symbols as sym -from loki.transform import resolve_associates, single_variable_declaration, HoistVariablesTransformation -from loki import ir from loki import ( - Transformation, FindNodes, FindVariables, Transformer, - SubstituteExpressions, SymbolAttributes, - CaseInsensitiveDict, as_tuple, flatten, types + Transformation, FindVariables, SubstituteExpressions, + SymbolAttributes, CaseInsensitiveDict, as_tuple, flatten, types ) +from loki.expression import symbols as sym +from loki.ir import nodes as ir, FindNodes, Transformer + +from loki.transformations.hoist_variables import HoistVariablesTransformation +from loki.transformations.sanitise import resolve_associates from loki.transformations.single_column.base import SCCBaseTransformation from loki.transformations.single_column.vector import SCCDevectorTransformation +from loki.transformations.utilities import single_variable_declaration + -__all__ = ['SccCufTransformation', 'HoistTemporaryArraysDeviceAllocatableTransformation', - 'HoistTemporaryArraysPragmaOffloadTransformation'] +__all__ = [ + 'SccCufTransformation', + 'HoistTemporaryArraysDeviceAllocatableTransformation', + 'HoistTemporaryArraysPragmaOffloadTransformation' +] class HoistTemporaryArraysDeviceAllocatableTransformation(HoistVariablesTransformation): diff --git a/loki/transformations/single_column/tests/test_single_column_coalesced.py b/loki/transformations/single_column/tests/test_single_column_coalesced.py index 125f6e5c9..4338b19f8 100644 --- a/loki/transformations/single_column/tests/test_single_column_coalesced.py +++ b/loki/transformations/single_column/tests/test_single_column_coalesced.py @@ -18,8 +18,10 @@ Pragma, PragmaRegion, Section, pragmas_attached, is_loki_pragma, pragma_regions_attached ) -from loki.transform import SanitiseTransformation, InlineTransformation -from loki.transformations import DataOffloadTransformation + +from loki.transformations import ( + DataOffloadTransformation, SanitiseTransformation, InlineTransformation +) from loki.transformations.single_column import ( SCCBaseTransformation, SCCDevectorTransformation, SCCDemoteTransformation, SCCRevectorTransformation, diff --git a/loki/transformations/single_column/vector.py b/loki/transformations/single_column/vector.py index 0f98fce42..1e588aa9f 100644 --- a/loki/transformations/single_column/vector.py +++ b/loki/transformations/single_column/vector.py @@ -18,12 +18,15 @@ ) from loki.tools import as_tuple, flatten from loki.types import BasicType -from loki.transform import demote_variables +from loki.transformations.array_indexing import demote_variables from loki.transformations.single_column.base import SCCBaseTransformation -__all__ = ['SCCDevectorTransformation', 'SCCRevectorTransformation', 'SCCDemoteTransformation'] +__all__ = [ + 'SCCDevectorTransformation', 'SCCRevectorTransformation', + 'SCCDemoteTransformation' +] class SCCDevectorTransformation(Transformation): diff --git a/loki/transform/tests/test_transform_array_indexing.py b/loki/transformations/tests/test_array_indexing.py similarity index 99% rename from loki/transform/tests/test_transform_array_indexing.py rename to loki/transformations/tests/test_array_indexing.py index e3219d562..245b58452 100644 --- a/loki/transform/tests/test_transform_array_indexing.py +++ b/loki/transformations/tests/test_array_indexing.py @@ -14,12 +14,13 @@ from loki.expression import symbols as sym, FindVariables from loki.frontend import available_frontends from loki.ir import FindNodes, CallStatement -from loki.transform import ( + +from loki.transformations.array_indexing import ( promote_variables, demote_variables, normalize_range_indexing, invert_array_indices, flatten_arrays, normalize_array_shape_and_access, shift_to_zero_indexing, - FortranCTransformation ) +from loki.transformations.transpile import FortranCTransformation @pytest.fixture(scope='module', name='here') diff --git a/loki/tests/test_extract_contained_procedures.py b/loki/transformations/tests/test_extract.py similarity index 99% rename from loki/tests/test_extract_contained_procedures.py rename to loki/transformations/tests/test_extract.py index cadcb3968..b8e9de8ff 100644 --- a/loki/tests/test_extract_contained_procedures.py +++ b/loki/transformations/tests/test_extract.py @@ -10,10 +10,11 @@ from loki.expression import FindInlineCalls from loki.frontend import available_frontends, OMNI, OFP from loki.ir import CallStatement, Import, FindNodes -from loki.transform import extract_contained_procedures from loki.sourcefile import Sourcefile from loki.subroutine import Subroutine +from loki.transformations.extract import extract_contained_procedures + @pytest.mark.parametrize('frontend', available_frontends()) def test_extract_contained_procedures_basic_scalar(frontend): diff --git a/loki/transform/tests/test_transform_hoist_variables.py b/loki/transformations/tests/test_hoist_variables.py similarity index 99% rename from loki/transform/tests/test_transform_hoist_variables.py rename to loki/transformations/tests/test_hoist_variables.py index 7e71fcb1f..05e6ea771 100644 --- a/loki/transform/tests/test_transform_hoist_variables.py +++ b/loki/transformations/tests/test_hoist_variables.py @@ -19,7 +19,7 @@ from loki.build import jit_compile_lib, clean_test, Builder from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ( +from loki.transformations.hoist_variables import ( HoistVariablesAnalysis, HoistVariablesTransformation, HoistTemporaryArraysAnalysis, HoistTemporaryArraysTransformationAllocatable ) diff --git a/loki/transform/tests/test_transform_inline.py b/loki/transformations/tests/test_inline.py similarity index 99% rename from loki/transform/tests/test_transform_inline.py rename to loki/transformations/tests/test_inline.py index 19b6db2cc..4fe640247 100644 --- a/loki/transform/tests/test_transform_inline.py +++ b/loki/transformations/tests/test_inline.py @@ -17,12 +17,15 @@ from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI, OFP from loki.ir import nodes as ir, FindNodes -from loki.transform import ( + +from loki.transformations.inline import ( inline_elemental_functions, inline_constant_parameters, - replace_selected_kind, inline_member_procedures, - inline_marked_subroutines, InlineTransformation, - ResolveAssociatesTransformer + inline_member_procedures, inline_marked_subroutines, + InlineTransformation, ) +from loki.transformations.sanitise import ResolveAssociatesTransformer +from loki.transformations.utilities import replace_selected_kind + @pytest.fixture(scope='module', name='here') def fixture_here(): diff --git a/loki/transform/tests/test_transform_parametrise.py b/loki/transformations/tests/test_parametrise.py similarity index 99% rename from loki/transform/tests/test_transform_parametrise.py rename to loki/transformations/tests/test_parametrise.py index a15404a7b..965d212b8 100644 --- a/loki/transform/tests/test_transform_parametrise.py +++ b/loki/transformations/tests/test_parametrise.py @@ -10,7 +10,8 @@ from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ParametriseTransformation + +from loki.transformations.parametrise import ParametriseTransformation @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/tests/test_transform_remove_code.py b/loki/transformations/tests/test_remove_code.py similarity index 99% rename from loki/transform/tests/test_transform_remove_code.py rename to loki/transformations/tests/test_remove_code.py index 3e16a0d70..0374d2b83 100644 --- a/loki/transform/tests/test_transform_remove_code.py +++ b/loki/transformations/tests/test_remove_code.py @@ -12,7 +12,8 @@ from loki.batch import Scheduler, SchedulerConfig from loki.frontend import available_frontends, OMNI from loki.ir import nodes as ir, FindNodes -from loki.transform import ( + +from loki.transformations.remove_code import ( do_remove_dead_code, do_remove_marked_regions, do_remove_calls, RemoveCodeTransformation ) diff --git a/loki/transform/tests/test_transform_sanitise.py b/loki/transformations/tests/test_sanitise.py similarity index 99% rename from loki/transform/tests/test_transform_sanitise.py rename to loki/transformations/tests/test_sanitise.py index b396fff21..e3fa06d8e 100644 --- a/loki/transform/tests/test_transform_sanitise.py +++ b/loki/transformations/tests/test_sanitise.py @@ -12,7 +12,8 @@ ) from loki.frontend import available_frontends, OMNI from loki.ir import Assignment, Associate, CallStatement, Conditional -from loki.transform import ( + +from loki.transformations.sanitise import ( resolve_associates, transform_sequence_association, SanitiseTransformation ) diff --git a/loki/transformations/tests/test_scc_cuf.py b/loki/transformations/tests/test_scc_cuf.py index a27049f45..1dac545d7 100644 --- a/loki/transformations/tests/test_scc_cuf.py +++ b/loki/transformations/tests/test_scc_cuf.py @@ -15,13 +15,12 @@ FindNodes, Loop, Assignment, CallStatement, Allocation, Deallocation, VariableDeclaration, Import, Pragma ) -from loki.transform import ( - HoistTemporaryArraysAnalysis, ParametriseTransformation -) -from loki.transformations import ( - SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, - HoistTemporaryArraysPragmaOffloadTransformation +from loki.transformations.parametrise import ParametriseTransformation +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis +from loki.transformations.single_column import ( + SccCufTransformation, HoistTemporaryArraysDeviceAllocatableTransformation, + HoistTemporaryArraysPragmaOffloadTransformation ) diff --git a/loki/transformations/tests/test_transform_derived_types.py b/loki/transformations/tests/test_transform_derived_types.py index 8526d0f40..2cfba93ec 100644 --- a/loki/transformations/tests/test_transform_derived_types.py +++ b/loki/transformations/tests/test_transform_derived_types.py @@ -18,11 +18,11 @@ from loki.frontend import available_frontends, OMNI, OFP from loki.ir import FindNodes, CallStatement -from loki.transform import resolve_associates -from loki.transformations import ( +from loki.transformations.derived_types import ( DerivedTypeArgumentsTransformation, TypeboundProcedureCallTransformation ) +from loki.transformations.sanitise import resolve_associates #pylint: disable=too-many-lines @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/tests/test_transform_loop.py b/loki/transformations/tests/test_transform_loop.py similarity index 99% rename from loki/transform/tests/test_transform_loop.py rename to loki/transformations/tests/test_transform_loop.py index 2ad3a49ee..5cf847d41 100644 --- a/loki/transform/tests/test_transform_loop.py +++ b/loki/transformations/tests/test_transform_loop.py @@ -18,8 +18,10 @@ is_loki_pragma, pragmas_attached, FindNodes, Loop, Conditional, Assignment ) -from loki.transform import ( - loop_interchange, loop_fusion, loop_fission, normalize_range_indexing + +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.transform_loop import ( + loop_interchange, loop_fusion, loop_fission ) diff --git a/loki/transform/tests/test_transform_region.py b/loki/transformations/tests/test_transform_region.py similarity index 99% rename from loki/transform/tests/test_transform_region.py rename to loki/transformations/tests/test_transform_region.py index ceb19ca2f..5320a730a 100644 --- a/loki/transform/tests/test_transform_region.py +++ b/loki/transformations/tests/test_transform_region.py @@ -16,7 +16,11 @@ from loki.build import jit_compile, jit_compile_lib, clean_test, Builder from loki.expression import symbols as sym from loki.frontend import available_frontends -from loki.transform import region_hoist, region_to_call, normalize_range_indexing + +from loki.transformations.array_indexing import normalize_range_indexing +from loki.transformations.transform_region import ( + region_hoist, region_to_call +) @pytest.fixture(scope='module', name='here') diff --git a/loki/transform/tests/test_transform_utilities.py b/loki/transformations/tests/test_utilities.py similarity index 99% rename from loki/transform/tests/test_transform_utilities.py rename to loki/transformations/tests/test_utilities.py index 73d99c659..4740882c2 100644 --- a/loki/transform/tests/test_transform_utilities.py +++ b/loki/transformations/tests/test_utilities.py @@ -13,7 +13,8 @@ ) from loki.expression import symbols as sym from loki.frontend import available_frontends, OMNI -from loki.transform import ( + +from loki.transformations.utilities import ( single_variable_declaration, recursive_expression_map_update, convert_to_lower_case, replace_intrinsics, rename_variables ) diff --git a/loki/transform/transform_loop.py b/loki/transformations/transform_loop.py similarity index 99% rename from loki/transform/transform_loop.py rename to loki/transformations/transform_loop.py index 35774d25b..2203be58a 100644 --- a/loki/transform/transform_loop.py +++ b/loki/transformations/transform_loop.py @@ -13,6 +13,10 @@ import operator as op import numpy as np +from loki.analyse import ( + dataflow_analysis_attached, read_after_write_vars, + loop_carried_dependencies +) from loki.expression import ( symbols as sym, SubstituteExpressions, FindVariables, simplify, is_constant, symbolic_op, parse_expr @@ -23,16 +27,15 @@ get_pragma_parameters, pragmas_attached ) from loki.logging import info, warning -from loki.transform.transform_array_indexing import ( - promotion_dimensions_from_loop_nest, promote_nonmatching_variables -) from loki.tools import ( - flatten, as_tuple, CaseInsensitiveDict, binary_insertion_sort, optional + flatten, as_tuple, CaseInsensitiveDict, binary_insertion_sort, + optional ) -from loki.analyse import ( - dataflow_analysis_attached, read_after_write_vars, loop_carried_dependencies +from loki.transformations.array_indexing import ( + promotion_dimensions_from_loop_nest, promote_nonmatching_variables ) + __all__ = ['loop_interchange', 'loop_fusion', 'loop_fission'] diff --git a/loki/transform/transform_region.py b/loki/transformations/transform_region.py similarity index 99% rename from loki/transform/transform_region.py rename to loki/transformations/transform_region.py index 77bf32238..a51bef20d 100644 --- a/loki/transform/transform_region.py +++ b/loki/transformations/transform_region.py @@ -22,7 +22,8 @@ from loki.logging import info from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten, CaseInsensitiveDict -from loki.transform.transform_array_indexing import ( + +from loki.transformations.array_indexing import ( promotion_dimensions_from_loop_nest, promote_nonmatching_variables ) diff --git a/loki/transformations/transpile/fortran_c.py b/loki/transformations/transpile/fortran_c.py index b34cdc0e7..78ea53ce4 100644 --- a/loki/transformations/transpile/fortran_c.py +++ b/loki/transformations/transpile/fortran_c.py @@ -8,36 +8,35 @@ from pathlib import Path from collections import OrderedDict +from loki import Sourcefile, Subroutine, Module +from loki.backend import cgen, fgen from loki.batch import Transformation -from loki.transform.transform_array_indexing import ( +from loki.expression import ( + symbols as sym, Variable, InlineCall, RangeIndex, Scalar, Array, + ProcedureSymbol, SubstituteExpressions, Dereference, Reference, + ExpressionRetriever, SubstituteExpressionsMapper, +) +from loki.ir import ( + Section, Import, Intrinsic, Interface, CallStatement, VariableDeclaration, + TypeDef, Assignment, Transformer, FindNodes +) +from loki.logging import debug +from loki.tools import as_tuple, flatten +from loki.types import BasicType, DerivedType, SymbolAttributes + +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_array_shape_and_access, flatten_arrays ) -from loki.transform.transform_sanitise import resolve_associates -from loki.transform.transform_utilities import ( +from loki.transformations.utilities import ( convert_to_lower_case, replace_intrinsics, sanitise_imports ) -from loki.transform.transform_inline import ( +from loki.transformations.sanitise import resolve_associates +from loki.transformations.inline import ( inline_constant_parameters, inline_elemental_functions ) -from loki.sourcefile import Sourcefile -from loki.backend import cgen, fgen -from loki.logging import debug -from loki.ir import ( - Section, Import, Intrinsic, Interface, CallStatement, VariableDeclaration, - TypeDef, Assignment, Transformer, FindNodes -) -from loki.subroutine import Subroutine -from loki.module import Module -from loki.expression import ( - Variable, InlineCall, RangeIndex, Scalar, Array, - ProcedureSymbol, SubstituteExpressions, Dereference, Reference, - ExpressionRetriever, SubstituteExpressionsMapper, -) -from loki.expression import symbols as sym -from loki.tools import as_tuple, flatten -from loki.types import BasicType, DerivedType, SymbolAttributes + __all__ = ['FortranCTransformation'] diff --git a/loki/transformations/transpile/fortran_maxeler.py b/loki/transformations/transpile/fortran_maxeler.py index 80e9484b3..d2137e89e 100644 --- a/loki/transformations/transpile/fortran_maxeler.py +++ b/loki/transformations/transpile/fortran_maxeler.py @@ -23,11 +23,11 @@ from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten -from loki.transform.transform_array_indexing import ( +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, resolve_vector_notation, normalize_range_indexing ) -from loki.transform.transform_utilities import replace_intrinsics +from loki.transformations.utilities import replace_intrinsics from loki.types import SymbolAttributes, BasicType, DerivedType from loki.transformations.transpile import FortranCTransformation diff --git a/loki/transformations/transpile/fortran_python.py b/loki/transformations/transpile/fortran_python.py index cfbc58b33..f2a769a67 100644 --- a/loki/transformations/transpile/fortran_python.py +++ b/loki/transformations/transpile/fortran_python.py @@ -15,11 +15,11 @@ from loki.ir import nodes as ir, FindNodes, Transformer, pragmas_attached from loki.sourcefile import Sourcefile -from loki.transform.transform_array_indexing import ( +from loki.transformations.array_indexing import ( shift_to_zero_indexing, invert_array_indices, normalize_range_indexing ) -from loki.transform.transform_sanitise import resolve_associates -from loki.transform.transform_utilities import ( +from loki.transformations.sanitise import resolve_associates +from loki.transformations.utilities import ( convert_to_lower_case, replace_intrinsics ) diff --git a/loki/transformations/transpile/tests/test_transpile.py b/loki/transformations/transpile/tests/test_transpile.py index 2e293999d..563b3085a 100644 --- a/loki/transformations/transpile/tests/test_transpile.py +++ b/loki/transformations/transpile/tests/test_transpile.py @@ -14,8 +14,8 @@ import loki.expression.symbols as sym from loki.frontend import available_frontends, OFP import loki.ir as ir -from loki.transform import normalize_range_indexing +from loki.transformations.array_indexing import normalize_range_indexing from loki.transformations.transpile import FortranCTransformation diff --git a/loki/transform/transform_utilities.py b/loki/transformations/utilities.py similarity index 100% rename from loki/transform/transform_utilities.py rename to loki/transformations/utilities.py diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 23fce5552..09940117e 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -22,14 +22,10 @@ from loki.batch import Transformation, Pipeline, Scheduler, SchedulerConfig # Get generalized transformations provided by Loki -from loki.transform import ( - HoistTemporaryArraysAnalysis, normalize_range_indexing, - InlineTransformation, SanitiseTransformation, RemoveCodeTransformation -) - from loki.transformations.argument_shape import ( ArgumentArrayShapeAnalysis, ExplicitArgumentArrayShapeTransformation ) +from loki.transformations.array_indexing import normalize_range_indexing from loki.transformations.build_system import ( DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation ) @@ -38,7 +34,11 @@ ) from loki.transformations.derived_types import DerivedTypeArgumentsTransformation from loki.transformations.drhook import DrHookTransformation +from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis +from loki.transformations.inline import InlineTransformation from loki.transformations.pool_allocator import TemporariesPoolAllocatorTransformation +from loki.transformations.remove_code import RemoveCodeTransformation +from loki.transformations.sanitise import SanitiseTransformation from loki.transformations.single_column import ( ExtractSCATransformation, CLAWTransformation, SCCVectorPipeline, SCCHoistPipeline, SCCStackPipeline, From 13be40f62dc778c0ba3fb34dcd8cdbee3cc0bbba Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Tue, 23 Apr 2024 12:22:16 +0000 Subject: [PATCH 09/13] Transformations: Fixing imports to appease linter gods --- loki/batch/configure.py | 2 +- loki/transformations/argument_shape.py | 10 +++++----- loki/transformations/data_offload.py | 18 ++++++++++++------ loki/transformations/derived_types.py | 17 ++++++++++------- loki/transformations/pool_allocator.py | 12 +++++++----- loki/transformations/single_column/base.py | 10 ++++++---- loki/transformations/single_column/claw.py | 15 ++++++++------- loki/transformations/single_column/scc_cuf.py | 17 +++++++++-------- loki/transformations/transpile/fortran_c.py | 8 +++++--- .../transpile/fortran_maxeler.py | 2 +- 10 files changed, 64 insertions(+), 47 deletions(-) diff --git a/loki/batch/configure.py b/loki/batch/configure.py index 6fb657b24..bfc4e7291 100644 --- a/loki/batch/configure.py +++ b/loki/batch/configure.py @@ -346,7 +346,7 @@ def instantiate(self, transformation_map=None): Creates a custom :any:`Pipeline` object from instantiated :any:`Transformation` or :any:`Pipeline` objects in the given map. """ - from loki.transform import Pipeline # pylint: disable=import-outside-toplevel,cyclic-import + from loki.batch.pipeline import Pipeline # pylint: disable=import-outside-toplevel,cyclic-import # Create an empty pipeline and add from the map pipeline = Pipeline(classes=()) diff --git a/loki/transformations/argument_shape.py b/loki/transformations/argument_shape.py index e812355c8..a389e235d 100644 --- a/loki/transformations/argument_shape.py +++ b/loki/transformations/argument_shape.py @@ -21,11 +21,11 @@ """ -from loki import ( - Transformation, FindNodes, CallStatement, Array, FindVariables, - SubstituteExpressions, BasicType, as_tuple, Transformer, - CaseInsensitiveDict -) +from loki.batch import Transformation +from loki.expression import Array, FindVariables, SubstituteExpressions +from loki.ir import FindNodes, CallStatement, Transformer +from loki.tools import as_tuple, CaseInsensitiveDict +from loki.types import BasicType __all__ = ['ArgumentArrayShapeAnalysis', 'ExplicitArgumentArrayShapeTransformation'] diff --git a/loki/transformations/data_offload.py b/loki/transformations/data_offload.py index fe9d723b2..881fa27ae 100644 --- a/loki/transformations/data_offload.py +++ b/loki/transformations/data_offload.py @@ -7,13 +7,19 @@ from collections import defaultdict from itertools import chain -from loki import ( - pragma_regions_attached, PragmaRegion, Transformation, FindNodes, - CallStatement, Pragma, Scalar, Array, as_tuple, Transformer, warning, BasicType, - ProcedureItem, ModuleItem, dataflow_analysis_attached, Import, - Comment, flatten, DerivedType, get_pragma_parameters, CaseInsensitiveDict, - FindInlineCalls, SubstituteExpressions + +from loki.analyse import dataflow_analysis_attached +from loki.batch import Transformation, ProcedureItem, ModuleItem +from loki.expression import ( + Scalar, Array, FindInlineCalls, SubstituteExpressions +) +from loki.ir import ( + FindNodes, PragmaRegion, CallStatement, Pragma, Import, Comment, + Transformer, pragma_regions_attached, get_pragma_parameters ) +from loki.logging import warning +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import BasicType, DerivedType __all__ = [ diff --git a/loki/transformations/derived_types.py b/loki/transformations/derived_types.py index f6e5f1c0f..930851361 100644 --- a/loki/transformations/derived_types.py +++ b/loki/transformations/derived_types.py @@ -16,16 +16,19 @@ from collections import defaultdict -from loki import ( - Transformation, SubstituteExpressions, Module, - ProcedureDeclaration, BasicType, DerivedType, as_tuple, flatten, - warning, debug, CaseInsensitiveDict, ProcedureType -) -from loki.ir import FindNodes, Import, CallStatement, Transformer +from loki.batch import Transformation from loki.expression import ( FindInlineCalls, FindVariables, InlineCall, Variable, RangeIndex, - ExpressionRetriever, SubstituteExpressionsMapper + ExpressionRetriever, SubstituteExpressionsMapper, SubstituteExpressions +) +from loki.ir import ( + FindNodes, Import, CallStatement, ProcedureDeclaration, Transformer ) +from loki.logging import warning, debug +from loki.module import Module +from loki.tools import as_tuple, flatten, CaseInsensitiveDict +from loki.types import BasicType, DerivedType, ProcedureType + from loki.transformations.utilities import recursive_expression_map_update diff --git a/loki/transformations/pool_allocator.py b/loki/transformations/pool_allocator.py index 99054629c..83116599b 100644 --- a/loki/transformations/pool_allocator.py +++ b/loki/transformations/pool_allocator.py @@ -8,22 +8,24 @@ import re from collections import defaultdict -from loki import ( - as_tuple, warning, debug, Transformation, DetachScopesMapper, - SymbolAttributes, BasicType, DerivedType, -) +from loki.batch import Transformation from loki.analyse import dataflow_analysis_attached from loki.expression import ( FindVariables, FindInlineCalls, Quotient, IntLiteral, LogicLiteral, Variable, Array, Sum, Literal, Product, InlineCall, Comparison, RangeIndex, Cast, ProcedureSymbol, LogicalNot, - simplify, SubstituteExpressions, is_dimension_constant + simplify, SubstituteExpressions, is_dimension_constant, + DetachScopesMapper ) from loki.ir import ( FindNodes, Transformer, Intrinsic, Assignment, Conditional, CallStatement, Import, Allocation, Deallocation, Loop, Pragma, Interface, get_pragma_parameters ) +from loki.logging import warning, debug +from loki.tools import as_tuple +from loki.types import SymbolAttributes, BasicType, DerivedType + from loki.transformations.utilities import recursive_expression_map_update diff --git a/loki/transformations/single_column/base.py b/loki/transformations/single_column/base.py index 35037d730..2c6d337ff 100644 --- a/loki/transformations/single_column/base.py +++ b/loki/transformations/single_column/base.py @@ -5,12 +5,14 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from loki import ( - Transformation, FindExpressions, SymbolAttributes, BasicType, - SubstituteExpressions, as_tuple, +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindExpressions, SubstituteExpressions ) -from loki.expression import symbols as sym from loki.ir import nodes as ir, FindNodes, Transformer +from loki.tools import as_tuple +from loki.types import SymbolAttributes, BasicType + from loki.transformations.sanitise import resolve_associates diff --git a/loki/transformations/single_column/claw.py b/loki/transformations/single_column/claw.py index 46353481b..c03ca24dd 100644 --- a/loki/transformations/single_column/claw.py +++ b/loki/transformations/single_column/claw.py @@ -12,18 +12,19 @@ from collections import OrderedDict -from loki import ( - Transformation, SymbolAttributes, BasicType, CaseInsensitiveDict, - as_tuple, warning, +from loki.batch import Transformation +from loki.expression import ( + FindVariables, SubstituteExpressions, Variable, + Array, LoopRange, RangeIndex ) from loki.ir import ( FindNodes, Transformer, Assignment, CallStatement, Loop, Pragma, VariableDeclaration, ) -from loki.expression import ( - FindVariables, SubstituteExpressions, Variable, - Array, LoopRange, RangeIndex -) +from loki.logging import warning +from loki.tools import as_tuple, CaseInsensitiveDict +from loki.types import SymbolAttributes, BasicType + from loki.transformations.utilities import recursive_expression_map_update diff --git a/loki/transformations/single_column/scc_cuf.py b/loki/transformations/single_column/scc_cuf.py index 411169d32..14fedd35b 100644 --- a/loki/transformations/single_column/scc_cuf.py +++ b/loki/transformations/single_column/scc_cuf.py @@ -9,13 +9,14 @@ Single-Column-Coalesced CUDA Fortran (SCC-CUF) transformation. """ -from loki import ( - Transformation, FindVariables, SubstituteExpressions, - SymbolAttributes, CaseInsensitiveDict, as_tuple, flatten, types +from loki.batch import Transformation +from loki.expression import ( + symbols as sym, FindVariables, SubstituteExpressions ) -from loki.expression import symbols as sym from loki.ir import nodes as ir, FindNodes, Transformer - +from loki.tools import CaseInsensitiveDict, as_tuple, flatten +from loki.types import BasicType, DerivedType +from loki.scope import SymbolAttributes from loki.transformations.hoist_variables import HoistVariablesTransformation from loki.transformations.sanitise import resolve_associates @@ -148,7 +149,7 @@ def increase_heap_size(routine): routine: :any:`Subroutine` The subroutine (e.g. the driver) to increase the heap size """ - vtype = SymbolAttributes(types.BasicType.INTEGER, kind=sym.Variable(name="cuda_count_kind")) + vtype = SymbolAttributes(BasicType.INTEGER, kind=sym.Variable(name="cuda_count_kind")) routine.spec.append(ir.VariableDeclaration((sym.Variable(name="cudaHeapSize", type=vtype),))) assignment_lhs = routine.variable_map["istat"] @@ -417,7 +418,7 @@ def driver_device_variables(routine, targets=None): """ # istat: status of CUDA runtime function (e.g. for cudaDeviceSynchronize(), cudaMalloc(), cudaFree(), ...) - i_type = SymbolAttributes(types.BasicType.INTEGER) + i_type = SymbolAttributes(BasicType.INTEGER) routine.spec.append(ir.VariableDeclaration(symbols=(sym.Variable(name="istat", type=i_type),))) relevant_arrays = [] @@ -521,7 +522,7 @@ def driver_launch_configuration(routine, block_dim, targets=None): Tuple of subroutine call names that are processed in this traversal """ - d_type = SymbolAttributes(types.DerivedType("DIM3")) + d_type = SymbolAttributes(DerivedType("DIM3")) routine.spec.append(ir.VariableDeclaration(symbols=(sym.Variable(name="GRIDDIM", type=d_type), sym.Variable(name="BLOCKDIM", type=d_type)))) diff --git a/loki/transformations/transpile/fortran_c.py b/loki/transformations/transpile/fortran_c.py index 78ea53ce4..4bdb270c1 100644 --- a/loki/transformations/transpile/fortran_c.py +++ b/loki/transformations/transpile/fortran_c.py @@ -8,7 +8,6 @@ from pathlib import Path from collections import OrderedDict -from loki import Sourcefile, Subroutine, Module from loki.backend import cgen, fgen from loki.batch import Transformation from loki.expression import ( @@ -17,10 +16,13 @@ ExpressionRetriever, SubstituteExpressionsMapper, ) from loki.ir import ( - Section, Import, Intrinsic, Interface, CallStatement, VariableDeclaration, - TypeDef, Assignment, Transformer, FindNodes + Section, Import, Intrinsic, Interface, CallStatement, + VariableDeclaration, TypeDef, Assignment, Transformer, FindNodes ) from loki.logging import debug +from loki.module import Module +from loki.sourcefile import Sourcefile +from loki.subroutine import Subroutine from loki.tools import as_tuple, flatten from loki.types import BasicType, DerivedType, SymbolAttributes diff --git a/loki/transformations/transpile/fortran_maxeler.py b/loki/transformations/transpile/fortran_maxeler.py index d2137e89e..9b75ba9cb 100644 --- a/loki/transformations/transpile/fortran_maxeler.py +++ b/loki/transformations/transpile/fortran_maxeler.py @@ -30,7 +30,7 @@ from loki.transformations.utilities import replace_intrinsics from loki.types import SymbolAttributes, BasicType, DerivedType -from loki.transformations.transpile import FortranCTransformation +from loki.transformations.transpile.fortran_c import FortranCTransformation __all__ = ['FortranMaxTransformation'] From bde9395062440d68a24df5913e604734475f2e84 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Wed, 24 Apr 2024 09:04:08 +0000 Subject: [PATCH 10/13] Transformations: Add test-specific build dir to test to avoid aliasing --- .../tests/test_array_indexing.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/loki/transformations/tests/test_array_indexing.py b/loki/transformations/tests/test_array_indexing.py index 245b58452..e3fade28d 100644 --- a/loki/transformations/tests/test_array_indexing.py +++ b/loki/transformations/tests/test_array_indexing.py @@ -6,6 +6,7 @@ # nor does it submit to any jurisdiction. from pathlib import Path +from shutil import rmtree import pytest import numpy as np @@ -14,6 +15,7 @@ from loki.expression import symbols as sym, FindVariables from loki.frontend import available_frontends from loki.ir import FindNodes, CallStatement +from loki.tools import gettempdir from loki.transformations.array_indexing import ( promote_variables, demote_variables, normalize_range_indexing, @@ -23,14 +25,23 @@ from loki.transformations.transpile import FortranCTransformation -@pytest.fixture(scope='module', name='here') +@pytest.fixture(scope='function', name='here') def fixture_here(): return Path(__file__).parent -@pytest.fixture(scope='module', name='builder') -def fixture_builder(here): - return Builder(source_dirs=here, build_dir=here/'build') +@pytest.fixture(scope='function', name='tempdir') +def fixture_tempdir(request): + basedir = gettempdir()/request.function.__name__ + basedir.mkdir(exist_ok=True) + yield basedir + if basedir.exists(): + rmtree(basedir) + + +@pytest.fixture(scope='function', name='builder') +def fixture_builder(here, tempdir): + return Builder(source_dirs=tempdir, build_dir=tempdir) @pytest.mark.parametrize('frontend', available_frontends()) @@ -637,7 +648,7 @@ def test_shift_to_zero_indexing(frontend, ignore): @pytest.mark.parametrize('frontend', available_frontends()) @pytest.mark.parametrize('explicit_dimensions', [True, False]) -def test_transform_flatten_arrays_call(here, frontend, builder, explicit_dimensions): +def test_transform_flatten_arrays_call(tempdir, frontend, builder, explicit_dimensions): """ Test flattening or arrays, meaning converting multi-dimensional arrays to one-dimensional arrays including corresponding @@ -699,7 +710,7 @@ def validate_routine(routine): # compile and test reference refname = f'ref_{driver.name}_{frontend}' - reference = jit_compile_lib([kernel_module, driver], path=here, name=refname, builder=builder) + reference = jit_compile_lib([kernel_module, driver], path=tempdir, name=refname, builder=builder) ref_function = reference.driver_routine nlon = 10 @@ -721,7 +732,7 @@ def validate_routine(routine): # compile and test the flattened variant flattenedname = f'flattened_{driver.name}_{frontend}' - flattened = jit_compile_lib([kernel_module, driver], path=here, name=flattenedname, builder=builder) + flattened = jit_compile_lib([kernel_module, driver], path=tempdir, name=flattenedname, builder=builder) flattened_function = flattened.driver_routine a_flattened, b_flattened = init_arguments(nlon, nlev, flattened=True) @@ -732,5 +743,5 @@ def validate_routine(routine): assert (b_flattened == b_ref.flatten(order='F')).all() builder.clean() - (here/f'{driver.name}.f90').unlink() - (here/f'{kernel_module.name}.f90').unlink() + (tempdir/f'{driver.name}.f90').unlink() + (tempdir/f'{kernel_module.name}.f90').unlink() From dcc02de6d189f20544b4bcd4bbece119017f0e82 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Fri, 3 May 2024 09:24:09 +0000 Subject: [PATCH 11/13] Tools: Add docstrings for env and bundle handling --- loki/tools/files.py | 79 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/loki/tools/files.py b/loki/tools/files.py index 573a381c6..d82417bbd 100644 --- a/loki/tools/files.py +++ b/loki/tools/files.py @@ -247,7 +247,32 @@ def load_module(module, path=None): def write_env_launch_script(here, binary, args): - # Write a script to source env.sh and launch the binary + """ + Utility method that is used for regression tests that require + activating an environment file before running :data:`binary`. + + This writes a simple script of the form + + .. code-block:: + source env.sh + bin/ + exit $? + + Parameters + ---------- + here : pathlib.Path or str + The directory in which the script is created + binary : str + The name of the binary + args : list of str + List of arguments to pass to the binary + + Returns + ------- + pathlib.Path + The path to the created script file + """ + script = Path(here/f'build/run_{binary}.sh') script.write_text(f""" #!/bin/bash @@ -262,6 +287,43 @@ def write_env_launch_script(here, binary, args): def local_loki_setup(here): + """ + Utility method that is used to determine paths for injecting the + currently running source code of Loki into an + `ecbundle `_-based worktree This + is used for regression tests to facilitate the use of a local Loki + source copy in the build. In particular, any existing Loki source + copy in the bundle worktree is moved to a backup location. + + .. warning:: If a backup copy exists already at the backup + location, this is removed before moving the existing Loki copy + to the backup location. + + Note that injecting the currently running Loki installation only + works if it has been installed in editable mode. However, this + utility also does not take care of the actual injection of the + currently running installation, therefore making this also useful + if the purpose is to trigger a Loki download via the bundle create + mechanism. + + The companion utility :any:`local_loki_cleanup` can be used to + revert these changes. + + Parameters + ---------- + here : pathlib.Path + The root path of the bundle worktree. + + Returns + ------- + tuple of (str, pathlib.Path, pathlib.Path) + The absolute path to the base directory of the currently + running Loki installation, the ``target`` path where Loki + needs to be injected in the bundle directory, and the + ``backup`` path where an existing Loki copy in the bundle has + been moved to. + """ + lokidir = Path(__file__).parent.parent.parent target = here/'source/loki' backup = here/'source/loki.bak' @@ -276,6 +338,21 @@ def local_loki_setup(here): def local_loki_cleanup(target, backup): + """ + Companion utility to :any:`local_loki_setup` to revert the + changes. + + This removes a symlink at :data:`target`, if it exists, and moves + the :data:`backup` path in its original location. + + Parameters + --------- + target : pathlib.Path + The target injection path as returned by :any:`local_loki_setup` + backup : pathlib.Path + The backup path as created by :any:`local_loki_setup` + """ + if target.is_symlink(): target.unlink() if not target.exists() and backup.exists(): From b915b42a987a0bba2c33b10a4a86d95d3d020345 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Fri, 3 May 2024 09:27:02 +0000 Subject: [PATCH 12/13] Loki: Fix imports, fixture scoping and install script comment --- install | 2 +- loki/transformations/build_system/module_wrap.py | 3 +-- loki/transformations/tests/test_array_indexing.py | 6 ++---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/install b/install index 58a9c3a4b..4b18dd494 100755 --- a/install +++ b/install @@ -513,5 +513,5 @@ echo "Activate the Loki environment with" echo echo " source loki-activate" echo -echo "You can test the Loki installation by running 'py.test lint_rules .'" +echo "You can test the Loki installation by running 'py.test lint_rules loki'" echo diff --git a/loki/transformations/build_system/module_wrap.py b/loki/transformations/build_system/module_wrap.py index af261e45f..9e132e944 100644 --- a/loki/transformations/build_system/module_wrap.py +++ b/loki/transformations/build_system/module_wrap.py @@ -5,7 +5,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -from loki.batch import Transformation +from loki.batch import Transformation, SchedulerConfig from loki.expression import Variable from loki.ir import Import, Section, Interface, FindNodes, Transformer from loki.module import Module @@ -108,7 +108,6 @@ def update_imports(self, source, imports, **kwargs): """ Update imports of wrapped subroutines. """ - from loki.batch import SchedulerConfig # pylint: disable=import-outside-toplevel,cyclic-import targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets'))) if self.replace_ignore_items and (item := kwargs.get('item')): diff --git a/loki/transformations/tests/test_array_indexing.py b/loki/transformations/tests/test_array_indexing.py index e3fade28d..a6cfc79ab 100644 --- a/loki/transformations/tests/test_array_indexing.py +++ b/loki/transformations/tests/test_array_indexing.py @@ -25,7 +25,7 @@ from loki.transformations.transpile import FortranCTransformation -@pytest.fixture(scope='function', name='here') +@pytest.fixture(scope='module', name='here') def fixture_here(): return Path(__file__).parent @@ -40,7 +40,7 @@ def fixture_tempdir(request): @pytest.fixture(scope='function', name='builder') -def fixture_builder(here, tempdir): +def fixture_builder(tempdir): return Builder(source_dirs=tempdir, build_dir=tempdir) @@ -743,5 +743,3 @@ def validate_routine(routine): assert (b_flattened == b_ref.flatten(order='F')).all() builder.clean() - (tempdir/f'{driver.name}.f90').unlink() - (tempdir/f'{kernel_module.name}.f90').unlink() From fb564551d1c7f25d07cdc1db52d1ce10d6f96c45 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Fri, 3 May 2024 10:12:14 +0000 Subject: [PATCH 13/13] Transformations: Rename build system test and derived type transform --- loki/transformations/__init__.py | 2 +- .../tests/{test_transform_dependency.py => test_dependency.py} | 0 loki/transformations/tests/test_transform_derived_types.py | 2 +- .../{derived_types.py => transform_derived_types.py} | 0 scripts/loki_transform.py | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) rename loki/transformations/build_system/tests/{test_transform_dependency.py => test_dependency.py} (100%) rename loki/transformations/{derived_types.py => transform_derived_types.py} (100%) diff --git a/loki/transformations/__init__.py b/loki/transformations/__init__.py index aa8f2c93a..514f47af2 100644 --- a/loki/transformations/__init__.py +++ b/loki/transformations/__init__.py @@ -7,7 +7,6 @@ from loki.transformations.array_indexing import * # noqa from loki.transformations.build_system import * # noqa -from loki.transformations.derived_types import * # noqa from loki.transformations.argument_shape import * # noqa from loki.transformations.data_offload import * # noqa from loki.transformations.drhook import * # noqa @@ -19,6 +18,7 @@ from loki.transformations.sanitise import * # noqa from loki.transformations.single_column import * # noqa from loki.transformations.transpile import * # noqa +from loki.transformations.transform_derived_types import * # noqa from loki.transformations.transform_loop import * # noqa from loki.transformations.transform_region import * # noqa from loki.transformations.pool_allocator import * # noqa diff --git a/loki/transformations/build_system/tests/test_transform_dependency.py b/loki/transformations/build_system/tests/test_dependency.py similarity index 100% rename from loki/transformations/build_system/tests/test_transform_dependency.py rename to loki/transformations/build_system/tests/test_dependency.py diff --git a/loki/transformations/tests/test_transform_derived_types.py b/loki/transformations/tests/test_transform_derived_types.py index 2cfba93ec..168884053 100644 --- a/loki/transformations/tests/test_transform_derived_types.py +++ b/loki/transformations/tests/test_transform_derived_types.py @@ -18,7 +18,7 @@ from loki.frontend import available_frontends, OMNI, OFP from loki.ir import FindNodes, CallStatement -from loki.transformations.derived_types import ( +from loki.transformations.transform_derived_types import ( DerivedTypeArgumentsTransformation, TypeboundProcedureCallTransformation ) diff --git a/loki/transformations/derived_types.py b/loki/transformations/transform_derived_types.py similarity index 100% rename from loki/transformations/derived_types.py rename to loki/transformations/transform_derived_types.py diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 09940117e..7ac07721c 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -32,7 +32,7 @@ from loki.transformations.data_offload import ( DataOffloadTransformation, GlobalVariableAnalysis, GlobalVarOffloadTransformation ) -from loki.transformations.derived_types import DerivedTypeArgumentsTransformation +from loki.transformations.transform_derived_types import DerivedTypeArgumentsTransformation from loki.transformations.drhook import DrHookTransformation from loki.transformations.hoist_variables import HoistTemporaryArraysAnalysis from loki.transformations.inline import InlineTransformation