From 8d1e68e533df01dda7ac4c4f9911db2ab388b7cf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 8 Nov 2023 07:41:44 -0700 Subject: [PATCH 01/99] working on simplification contrib package --- pyomo/contrib/simplification/__init__.py | 0 pyomo/contrib/simplification/build.py | 33 +++++++++++++++++++ .../simplification/ginac_interface.cpp | 0 pyomo/contrib/simplification/simplify.py | 12 +++++++ 4 files changed, 45 insertions(+) create mode 100644 pyomo/contrib/simplification/__init__.py create mode 100644 pyomo/contrib/simplification/build.py create mode 100644 pyomo/contrib/simplification/ginac_interface.cpp create mode 100644 pyomo/contrib/simplification/simplify.py diff --git a/pyomo/contrib/simplification/__init__.py b/pyomo/contrib/simplification/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py new file mode 100644 index 00000000000..0b0b9828cd3 --- /dev/null +++ b/pyomo/contrib/simplification/build.py @@ -0,0 +1,33 @@ +from pybind11.setup_helpers import Pybind11Extension, build_ext +from pyomo.common.fileutils import this_file_dir +import os +from distutils.dist import Distribution +import sys + + +def build_ginac_interface(args=[]): + dname = this_file_dir() + _sources = [ + 'ginac_interface.cpp', + ] + sources = list() + for fname in _sources: + sources.append(os.path.join(dname, fname)) + extra_args = ['-std=c++11'] + ext = Pybind11Extension('ginac_interface', sources, extra_compile_args=extra_args) + + package_config = { + 'name': 'ginac_interface', + 'packages': [], + 'ext_modules': [ext], + 'cmdclass': {"build_ext": build_ext}, + } + + dist = Distribution(package_config) + dist.script_args = ['build_ext'] + args + dist.parse_command_line() + dist.run_command('build_ext') + + +if __name__ == '__main__': + build_ginac_interface(sys.argv[1:]) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py new file mode 100644 index 00000000000..70d5dfcd9ac --- /dev/null +++ b/pyomo/contrib/simplification/simplify.py @@ -0,0 +1,12 @@ +from pyomo.core.expr.sympy_tools import sympy2pyomo_expression, sympyify_expression +from pyomo.core.expr.numeric_expr import NumericExpression +from pyomo.core.expr.numvalue import is_fixed, value + + +def simplify_with_sympy(expr: NumericExpression): + om, se = sympyify_expression(expr) + se = se.simplify() + new_expr = sympy2pyomo_expression(se, om) + if is_fixed(new_expr): + new_expr = value(new_expr) + return new_expr \ No newline at end of file From 8015d7ece05ee4608c8ddd6924218f40fd635f89 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 8 Nov 2023 11:39:43 -0700 Subject: [PATCH 02/99] working on simplification contrib package --- pyomo/contrib/simplification/build.py | 65 ++++++- .../simplification/ginac_interface.cpp | 149 ++++++++++++++++ .../simplification/ginac_interface.hpp | 165 ++++++++++++++++++ 3 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 pyomo/contrib/simplification/ginac_interface.hpp diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 0b0b9828cd3..6f16607e22b 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -1,8 +1,12 @@ from pybind11.setup_helpers import Pybind11Extension, build_ext -from pyomo.common.fileutils import this_file_dir +from pyomo.common.fileutils import this_file_dir, find_library import os from distutils.dist import Distribution import sys +import shutil +import glob +import tempfile +from pyomo.common.envvar import PYOMO_CONFIG_DIR def build_ginac_interface(args=[]): @@ -13,14 +17,69 @@ def build_ginac_interface(args=[]): sources = list() for fname in _sources: sources.append(os.path.join(dname, fname)) + + ginac_lib = find_library('ginac') + if ginac_lib is None: + raise RuntimeError('could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable') + ginac_lib_dir = os.path.dirname(ginac_lib) + ginac_build_dir = os.path.dirname(ginac_lib_dir) + ginac_include_dir = os.path.join(ginac_build_dir, 'include') + if not os.path.exists(os.path.join(ginac_include_dir, 'ginac', 'ginac.h')): + raise RuntimeError('could not find GiNaC include directory') + + cln_lib = find_library('cln') + if cln_lib is None: + raise RuntimeError('could not find CLN library; please make sure it is in the LD_LIBRARY_PATH environment variable') + cln_lib_dir = os.path.dirname(cln_lib) + cln_build_dir = os.path.dirname(cln_lib_dir) + cln_include_dir = os.path.join(cln_build_dir, 'include') + if not os.path.exists(os.path.join(cln_include_dir, 'cln', 'cln.h')): + raise RuntimeError('could not find CLN include directory') + extra_args = ['-std=c++11'] - ext = Pybind11Extension('ginac_interface', sources, extra_compile_args=extra_args) + ext = Pybind11Extension( + 'ginac_interface', + sources=sources, + language='c++', + include_dirs=[cln_include_dir, ginac_include_dir], + library_dirs=[cln_lib_dir, ginac_lib_dir], + libraries=['cln', 'ginac'], + extra_compile_args=extra_args, + ) + + class ginac_build_ext(build_ext): + def run(self): + basedir = os.path.abspath(os.path.curdir) + if self.inplace: + tmpdir = this_file_dir() + else: + tmpdir = os.path.abspath(tempfile.mkdtemp()) + print("Building in '%s'" % tmpdir) + os.chdir(tmpdir) + try: + super(ginac_build_ext, self).run() + if not self.inplace: + library = glob.glob("build/*/ginac_interface.*")[0] + target = os.path.join( + PYOMO_CONFIG_DIR, + 'lib', + 'python%s.%s' % sys.version_info[:2], + 'site-packages', + '.', + ) + if not os.path.exists(target): + os.makedirs(target) + shutil.copy(library, target) + finally: + os.chdir(basedir) + if not self.inplace: + shutil.rmtree(tmpdir, onerror=handleReadonly) package_config = { 'name': 'ginac_interface', 'packages': [], 'ext_modules': [ext], - 'cmdclass': {"build_ext": build_ext}, + 'cmdclass': {"build_ext": ginac_build_ext}, } dist = Distribution(package_config) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index e69de29bb2d..ccbc98d3586 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -0,0 +1,149 @@ +#include "ginac_interface.hpp" + +ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &leaf_map, PyomoExprTypes &expr_types) { + ex res; + ExprType tmp_type = + expr_types.expr_type_map[py::type::of(expr)].cast(); + + switch (tmp_type) { + case py_float: { + res = numeric(expr.cast()); + break; + } + case var: { + long expr_id = expr_types.id(expr).cast(); + if (leaf_map.count(expr_id) == 0) { + leaf_map[expr_id] = symbol("x" + std::to_string(expr_id)); + } + res = leaf_map[expr_id]; + break; + } + case param: { + long expr_id = expr_types.id(expr).cast(); + if (leaf_map.count(expr_id) == 0) { + leaf_map[expr_id] = symbol("p" + std::to_string(expr_id)); + } + res = leaf_map[expr_id]; + break; + } + case product: { + py::list pyomo_args = expr.attr("args"); + res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types) * ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types); + break; + } + case sum: { + py::list pyomo_args = expr.attr("args"); + for (py::handle arg : pyomo_args) { + res += ginac_expr_from_pyomo_node(arg, leaf_map, expr_types); + } + break; + } + case negation: { + py::list pyomo_args = expr.attr("args"); + res = - ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types); + break; + } + case external_func: { + long expr_id = expr_types.id(expr).cast(); + if (leaf_map.count(expr_id) == 0) { + leaf_map[expr_id] = symbol("f" + std::to_string(expr_id)); + } + res = leaf_map[expr_id]; + break; + } + case ExprType::power: { + py::list pyomo_args = expr.attr("args"); + res = pow(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types), ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types)); + break; + } + case division: { + py::list pyomo_args = expr.attr("args"); + res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types) / ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types); + break; + } + case unary_func: { + std::string function_name = expr.attr("getname")().cast(); + py::list pyomo_args = expr.attr("args"); + if (function_name == "exp") + res = exp(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "log") + res = log(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "sin") + res = sin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "cos") + res = cos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "tan") + res = tan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "asin") + res = asin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "acos") + res = acos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "atan") + res = atan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else if (function_name == "sqrt") + res = sqrt(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + else + throw py::value_error("Unrecognized expression type: " + function_name); + break; + } + case linear: { + py::list pyomo_args = expr.attr("args"); + for (py::handle arg : pyomo_args) { + res += ginac_expr_from_pyomo_node(arg, leaf_map, expr_types); + } + break; + } + case named_expr: { + res = ginac_expr_from_pyomo_node(expr.attr("expr"), leaf_map, expr_types); + break; + } + case numeric_constant: { + res = numeric(expr.attr("value").cast()); + break; + } + case pyomo_unit: { + res = numeric(1.0); + break; + } + case unary_abs: { + py::list pyomo_args = expr.attr("args"); + res = abs(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + break; + } + default: { + throw py::value_error("Unrecognized expression type: " + + expr_types.builtins.attr("str")(py::type::of(expr)) + .cast()); + break; + } + } + return res; +} + +ex ginac_expr_from_pyomo_expr(py::handle expr, PyomoExprTypes &expr_types) { + std::unordered_map leaf_map; + ex res = ginac_expr_from_pyomo_node(expr, leaf_map, expr_types); + return res; +} + + +PYBIND11_MODULE(ginac_interface, m) { + m.def("ginac_expr_from_pyomo_expr", &ginac_expr_from_pyomo_expr); + py::class_(m, "PyomoExprTypes").def(py::init<>()); + py::class_(m, "ex"); + py::enum_(m, "ExprType") + .value("py_float", ExprType::py_float) + .value("var", ExprType::var) + .value("param", ExprType::param) + .value("product", ExprType::product) + .value("sum", ExprType::sum) + .value("negation", ExprType::negation) + .value("external_func", ExprType::external_func) + .value("power", ExprType::power) + .value("division", ExprType::division) + .value("unary_func", ExprType::unary_func) + .value("linear", ExprType::linear) + .value("named_expr", ExprType::named_expr) + .value("numeric_constant", ExprType::numeric_constant) + .export_values(); +} diff --git a/pyomo/contrib/simplification/ginac_interface.hpp b/pyomo/contrib/simplification/ginac_interface.hpp new file mode 100644 index 00000000000..de77e66d0c7 --- /dev/null +++ b/pyomo/contrib/simplification/ginac_interface.hpp @@ -0,0 +1,165 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define PYBIND11_DETAILED_ERROR_MESSAGES + +namespace py = pybind11; +using namespace pybind11::literals; +using namespace GiNaC; + +enum ExprType { + py_float = 0, + var = 1, + param = 2, + product = 3, + sum = 4, + negation = 5, + external_func = 6, + power = 7, + division = 8, + unary_func = 9, + linear = 10, + named_expr = 11, + numeric_constant = 12, + pyomo_unit = 13, + unary_abs = 14 +}; + +class PyomoExprTypes { +public: + PyomoExprTypes() { + expr_type_map[int_] = py_float; + expr_type_map[float_] = py_float; + expr_type_map[np_int16] = py_float; + expr_type_map[np_int32] = py_float; + expr_type_map[np_int64] = py_float; + expr_type_map[np_longlong] = py_float; + expr_type_map[np_uint16] = py_float; + expr_type_map[np_uint32] = py_float; + expr_type_map[np_uint64] = py_float; + expr_type_map[np_ulonglong] = py_float; + expr_type_map[np_float16] = py_float; + expr_type_map[np_float32] = py_float; + expr_type_map[np_float64] = py_float; + expr_type_map[ScalarVar] = var; + expr_type_map[_GeneralVarData] = var; + expr_type_map[AutoLinkedBinaryVar] = var; + expr_type_map[ScalarParam] = param; + expr_type_map[_ParamData] = param; + expr_type_map[MonomialTermExpression] = product; + expr_type_map[ProductExpression] = product; + expr_type_map[NPV_ProductExpression] = product; + expr_type_map[SumExpression] = sum; + expr_type_map[NPV_SumExpression] = sum; + expr_type_map[NegationExpression] = negation; + expr_type_map[NPV_NegationExpression] = negation; + expr_type_map[ExternalFunctionExpression] = external_func; + expr_type_map[NPV_ExternalFunctionExpression] = external_func; + expr_type_map[PowExpression] = ExprType::power; + expr_type_map[NPV_PowExpression] = ExprType::power; + expr_type_map[DivisionExpression] = division; + expr_type_map[NPV_DivisionExpression] = division; + expr_type_map[UnaryFunctionExpression] = unary_func; + expr_type_map[NPV_UnaryFunctionExpression] = unary_func; + expr_type_map[LinearExpression] = linear; + expr_type_map[_GeneralExpressionData] = named_expr; + expr_type_map[ScalarExpression] = named_expr; + expr_type_map[Integral] = named_expr; + expr_type_map[ScalarIntegral] = named_expr; + expr_type_map[NumericConstant] = numeric_constant; + expr_type_map[_PyomoUnit] = pyomo_unit; + expr_type_map[AbsExpression] = unary_abs; + expr_type_map[NPV_AbsExpression] = unary_abs; + } + ~PyomoExprTypes() = default; + py::int_ ione = 1; + py::float_ fone = 1.0; + py::type int_ = py::type::of(ione); + py::type float_ = py::type::of(fone); + py::object np = py::module_::import("numpy"); + py::type np_int16 = np.attr("int16"); + py::type np_int32 = np.attr("int32"); + py::type np_int64 = np.attr("int64"); + py::type np_longlong = np.attr("longlong"); + py::type np_uint16 = np.attr("uint16"); + py::type np_uint32 = np.attr("uint32"); + py::type np_uint64 = np.attr("uint64"); + py::type np_ulonglong = np.attr("ulonglong"); + py::type np_float16 = np.attr("float16"); + py::type np_float32 = np.attr("float32"); + py::type np_float64 = np.attr("float64"); + py::object ScalarParam = + py::module_::import("pyomo.core.base.param").attr("ScalarParam"); + py::object _ParamData = + py::module_::import("pyomo.core.base.param").attr("_ParamData"); + py::object ScalarVar = + py::module_::import("pyomo.core.base.var").attr("ScalarVar"); + py::object _GeneralVarData = + py::module_::import("pyomo.core.base.var").attr("_GeneralVarData"); + py::object AutoLinkedBinaryVar = + py::module_::import("pyomo.gdp.disjunct").attr("AutoLinkedBinaryVar"); + py::object numeric_expr = py::module_::import("pyomo.core.expr.numeric_expr"); + py::object NegationExpression = numeric_expr.attr("NegationExpression"); + py::object NPV_NegationExpression = + numeric_expr.attr("NPV_NegationExpression"); + py::object ExternalFunctionExpression = + numeric_expr.attr("ExternalFunctionExpression"); + py::object NPV_ExternalFunctionExpression = + numeric_expr.attr("NPV_ExternalFunctionExpression"); + py::object PowExpression = numeric_expr.attr("PowExpression"); + py::object NPV_PowExpression = numeric_expr.attr("NPV_PowExpression"); + py::object ProductExpression = numeric_expr.attr("ProductExpression"); + py::object NPV_ProductExpression = numeric_expr.attr("NPV_ProductExpression"); + py::object MonomialTermExpression = + numeric_expr.attr("MonomialTermExpression"); + py::object DivisionExpression = numeric_expr.attr("DivisionExpression"); + py::object NPV_DivisionExpression = + numeric_expr.attr("NPV_DivisionExpression"); + py::object SumExpression = numeric_expr.attr("SumExpression"); + py::object NPV_SumExpression = numeric_expr.attr("NPV_SumExpression"); + py::object UnaryFunctionExpression = + numeric_expr.attr("UnaryFunctionExpression"); + py::object AbsExpression = numeric_expr.attr("AbsExpression"); + py::object NPV_AbsExpression = numeric_expr.attr("NPV_AbsExpression"); + py::object NPV_UnaryFunctionExpression = + numeric_expr.attr("NPV_UnaryFunctionExpression"); + py::object LinearExpression = numeric_expr.attr("LinearExpression"); + py::object NumericConstant = + py::module_::import("pyomo.core.expr.numvalue").attr("NumericConstant"); + py::object expr_module = py::module_::import("pyomo.core.base.expression"); + py::object _GeneralExpressionData = + expr_module.attr("_GeneralExpressionData"); + py::object ScalarExpression = expr_module.attr("ScalarExpression"); + py::object ScalarIntegral = + py::module_::import("pyomo.dae.integral").attr("ScalarIntegral"); + py::object Integral = + py::module_::import("pyomo.dae.integral").attr("Integral"); + py::object _PyomoUnit = + py::module_::import("pyomo.core.base.units_container").attr("_PyomoUnit"); + py::object builtins = py::module_::import("builtins"); + py::object id = builtins.attr("id"); + py::object len = builtins.attr("len"); + py::dict expr_type_map; +}; + +ex ginac_expr_from_pyomo_expr(py::handle expr, PyomoExprTypes &expr_types); From 56e8ac84e72bbd59d57e6307ef62b6dbabe09e37 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 20 Nov 2023 11:43:42 -0700 Subject: [PATCH 03/99] working on ginac interface for simplification --- .../simplification/ginac_interface.cpp | 213 ++++++++++++++++-- .../simplification/ginac_interface.hpp | 27 ++- 2 files changed, 214 insertions(+), 26 deletions(-) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index ccbc98d3586..9a84521ff91 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -1,6 +1,12 @@ #include "ginac_interface.hpp" -ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &leaf_map, PyomoExprTypes &expr_types) { +ex ginac_expr_from_pyomo_node( + py::handle expr, + std::unordered_map &leaf_map, + std::unordered_map &ginac_pyomo_map, + PyomoExprTypes &expr_types, + bool symbolic_solver_labels + ) { ex res; ExprType tmp_type = expr_types.expr_type_map[py::type::of(expr)].cast(); @@ -13,7 +19,21 @@ ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &lea case var: { long expr_id = expr_types.id(expr).cast(); if (leaf_map.count(expr_id) == 0) { - leaf_map[expr_id] = symbol("x" + std::to_string(expr_id)); + std::string vname; + if (symbolic_solver_labels) { + vname = expr.attr("name").cast(); + } + else { + vname = "x" + std::to_string(expr_id); + } + py::object lb = expr.attr("lb"); + if (lb.is_none() || lb.cast() < 0) { + leaf_map[expr_id] = realsymbol(vname); + } + else { + leaf_map[expr_id] = possymbol(vname); + } + ginac_pyomo_map[leaf_map[expr_id]] = expr.cast(); } res = leaf_map[expr_id]; break; @@ -21,67 +41,76 @@ ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &lea case param: { long expr_id = expr_types.id(expr).cast(); if (leaf_map.count(expr_id) == 0) { - leaf_map[expr_id] = symbol("p" + std::to_string(expr_id)); + std::string pname; + if (symbolic_solver_labels) { + pname = expr.attr("name").cast(); + } + else { + pname = "p" + std::to_string(expr_id); + } + leaf_map[expr_id] = realsymbol(pname); + ginac_pyomo_map[leaf_map[expr_id]] = expr.cast(); } res = leaf_map[expr_id]; break; } case product: { py::list pyomo_args = expr.attr("args"); - res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types) * ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types); + res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels) * ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); break; } case sum: { py::list pyomo_args = expr.attr("args"); for (py::handle arg : pyomo_args) { - res += ginac_expr_from_pyomo_node(arg, leaf_map, expr_types); + res += ginac_expr_from_pyomo_node(arg, leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); } break; } case negation: { py::list pyomo_args = expr.attr("args"); - res = - ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types); + res = - ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); break; } case external_func: { long expr_id = expr_types.id(expr).cast(); if (leaf_map.count(expr_id) == 0) { - leaf_map[expr_id] = symbol("f" + std::to_string(expr_id)); + leaf_map[expr_id] = realsymbol("f" + std::to_string(expr_id)); + ginac_pyomo_map[leaf_map[expr_id]] = expr.cast(); } res = leaf_map[expr_id]; break; } case ExprType::power: { py::list pyomo_args = expr.attr("args"); - res = pow(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types), ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types)); + res = pow(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels), ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); break; } case division: { py::list pyomo_args = expr.attr("args"); - res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types) / ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, expr_types); + res = ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels) / ginac_expr_from_pyomo_node(pyomo_args[1], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); break; } case unary_func: { std::string function_name = expr.attr("getname")().cast(); py::list pyomo_args = expr.attr("args"); if (function_name == "exp") - res = exp(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = exp(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "log") - res = log(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = log(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "sin") - res = sin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = sin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "cos") - res = cos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = cos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "tan") - res = tan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = tan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "asin") - res = asin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = asin(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "acos") - res = acos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = acos(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "atan") - res = atan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = atan(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else if (function_name == "sqrt") - res = sqrt(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = sqrt(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); else throw py::value_error("Unrecognized expression type: " + function_name); break; @@ -89,12 +118,12 @@ ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &lea case linear: { py::list pyomo_args = expr.attr("args"); for (py::handle arg : pyomo_args) { - res += ginac_expr_from_pyomo_node(arg, leaf_map, expr_types); + res += ginac_expr_from_pyomo_node(arg, leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); } break; } case named_expr: { - res = ginac_expr_from_pyomo_node(expr.attr("expr"), leaf_map, expr_types); + res = ginac_expr_from_pyomo_node(expr.attr("expr"), leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); break; } case numeric_constant: { @@ -107,7 +136,7 @@ ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &lea } case unary_abs: { py::list pyomo_args = expr.attr("args"); - res = abs(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, expr_types)); + res = abs(ginac_expr_from_pyomo_node(pyomo_args[0], leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels)); break; } default: { @@ -120,17 +149,151 @@ ex ginac_expr_from_pyomo_node(py::handle expr, std::unordered_map &lea return res; } -ex ginac_expr_from_pyomo_expr(py::handle expr, PyomoExprTypes &expr_types) { +ex pyomo_expr_to_ginac_expr( + py::handle expr, + std::unordered_map &leaf_map, + std::unordered_map &ginac_pyomo_map, + PyomoExprTypes &expr_types, + bool symbolic_solver_labels + ) { + ex res = ginac_expr_from_pyomo_node(expr, leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); + return res; + } + +ex pyomo_to_ginac(py::handle expr, PyomoExprTypes &expr_types) { std::unordered_map leaf_map; - ex res = ginac_expr_from_pyomo_node(expr, leaf_map, expr_types); + std::unordered_map ginac_pyomo_map; + ex res = ginac_expr_from_pyomo_node(expr, leaf_map, ginac_pyomo_map, expr_types, true); return res; } +class GinacToPyomoVisitor +: public visitor, + public symbol::visitor, + public numeric::visitor, + public add::visitor, + public mul::visitor, + public GiNaC::power::visitor, + public function::visitor, + public basic::visitor +{ + public: + std::unordered_map *leaf_map; + std::unordered_map node_map; + PyomoExprTypes *expr_types; + + GinacToPyomoVisitor(std::unordered_map *_leaf_map, PyomoExprTypes *_expr_types) : leaf_map(_leaf_map), expr_types(_expr_types) {} + ~GinacToPyomoVisitor() = default; + + void visit(const symbol& e) { + node_map[e] = leaf_map->at(e); + } + + void visit(const numeric& e) { + double val = e.to_double(); + node_map[e] = expr_types->NumericConstant(py::cast(val)); + } + + void visit(const add& e) { + size_t n = e.nops(); + py::object pe = node_map[e.op(0)]; + for (unsigned long ndx=1; ndx < n; ++ndx) { + pe = pe.attr("__add__")(node_map[e.op(ndx)]); + } + node_map[e] = pe; + } + + void visit(const mul& e) { + size_t n = e.nops(); + py::object pe = node_map[e.op(0)]; + for (unsigned long ndx=1; ndx < n; ++ndx) { + pe = pe.attr("__mul__")(node_map[e.op(ndx)]); + } + node_map[e] = pe; + } + + void visit(const GiNaC::power& e) { + py::object arg1 = node_map[e.op(0)]; + py::object arg2 = node_map[e.op(1)]; + py::object pe = arg1.attr("__pow__")(arg2); + node_map[e] = pe; + } + + void visit(const function& e) { + py::object arg = node_map[e.op(0)]; + std::string func_type = e.get_name(); + py::object pe; + if (func_type == "exp") { + pe = expr_types->exp(arg); + } + else if (func_type == "log") { + pe = expr_types->log(arg); + } + else if (func_type == "sin") { + pe = expr_types->sin(arg); + } + else if (func_type == "cos") { + pe = expr_types->cos(arg); + } + else if (func_type == "tan") { + pe = expr_types->tan(arg); + } + else if (func_type == "asin") { + pe = expr_types->asin(arg); + } + else if (func_type == "acos") { + pe = expr_types->acos(arg); + } + else if (func_type == "atan") { + pe = expr_types->atan(arg); + } + else if (func_type == "sqrt") { + pe = expr_types->sqrt(arg); + } + else { + throw py::value_error("unrecognized unary function: " + func_type); + } + node_map[e] = pe; + } + + void visit(const basic& e) { + throw py::value_error("unrecognized ginac expression type"); + } +}; + + +ex GinacInterface::to_ginac(py::handle expr) { + return pyomo_expr_to_ginac_expr(expr, leaf_map, ginac_pyomo_map, expr_types, symbolic_solver_labels); +} + +py::object GinacInterface::from_ginac(ex &ge) { + GinacToPyomoVisitor v(&ginac_pyomo_map, &expr_types); + ge.traverse_postorder(v); + return v.node_map[ge]; +} + PYBIND11_MODULE(ginac_interface, m) { - m.def("ginac_expr_from_pyomo_expr", &ginac_expr_from_pyomo_expr); + m.def("pyomo_to_ginac", &pyomo_to_ginac); py::class_(m, "PyomoExprTypes").def(py::init<>()); - py::class_(m, "ex"); + py::class_(m, "ginac_expression") + .def("expand", [](ex &ge) { + // exmap m; + // ex q; + // q = ge.to_polynomial(m).normal(); + // return q.subs(m); + // return factor(ge.normal()); + return ge.expand(); + }) + .def("__str__", [](ex &ge) { + std::ostringstream stream; + stream << ge; + return stream.str(); + }); + py::class_(m, "GinacInterface") + .def(py::init()) + .def("to_ginac", &GinacInterface::to_ginac) + .def("from_ginac", &GinacInterface::from_ginac); py::enum_(m, "ExprType") .value("py_float", ExprType::py_float) .value("var", ExprType::var) diff --git a/pyomo/contrib/simplification/ginac_interface.hpp b/pyomo/contrib/simplification/ginac_interface.hpp index de77e66d0c7..bc5b0d7b6fc 100644 --- a/pyomo/contrib/simplification/ginac_interface.hpp +++ b/pyomo/contrib/simplification/ginac_interface.hpp @@ -156,10 +156,35 @@ class PyomoExprTypes { py::module_::import("pyomo.dae.integral").attr("Integral"); py::object _PyomoUnit = py::module_::import("pyomo.core.base.units_container").attr("_PyomoUnit"); + py::object exp = numeric_expr.attr("exp"); + py::object log = numeric_expr.attr("log"); + py::object sin = numeric_expr.attr("sin"); + py::object cos = numeric_expr.attr("cos"); + py::object tan = numeric_expr.attr("tan"); + py::object asin = numeric_expr.attr("asin"); + py::object acos = numeric_expr.attr("acos"); + py::object atan = numeric_expr.attr("atan"); + py::object sqrt = numeric_expr.attr("sqrt"); py::object builtins = py::module_::import("builtins"); py::object id = builtins.attr("id"); py::object len = builtins.attr("len"); py::dict expr_type_map; }; -ex ginac_expr_from_pyomo_expr(py::handle expr, PyomoExprTypes &expr_types); +ex pyomo_to_ginac(py::handle expr, PyomoExprTypes &expr_types); + + +class GinacInterface { + public: + std::unordered_map leaf_map; + std::unordered_map ginac_pyomo_map; + PyomoExprTypes expr_types; + bool symbolic_solver_labels = false; + + GinacInterface() = default; + GinacInterface(bool _symbolic_solver_labels) : symbolic_solver_labels(_symbolic_solver_labels) {} + ~GinacInterface() = default; + + ex to_ginac(py::handle expr); + py::object from_ginac(ex &ginac_expr); +}; From 932d3d6a8a7a1cd95f2e227fe9f3a63849ad02a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 20 Nov 2023 13:07:37 -0700 Subject: [PATCH 04/99] ginac interface improvements --- .../simplification/ginac_interface.cpp | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index 9a84521ff91..690885dc513 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -1,5 +1,11 @@ #include "ginac_interface.hpp" + +bool is_integer(double x) { + return std::floor(x) == x; +} + + ex ginac_expr_from_pyomo_node( py::handle expr, std::unordered_map &leaf_map, @@ -13,7 +19,13 @@ ex ginac_expr_from_pyomo_node( switch (tmp_type) { case py_float: { - res = numeric(expr.cast()); + double val = expr.cast(); + if (is_integer(val)) { + res = numeric(expr.cast()); + } + else { + res = numeric(val); + } break; } case var: { @@ -278,13 +290,9 @@ PYBIND11_MODULE(ginac_interface, m) { py::class_(m, "PyomoExprTypes").def(py::init<>()); py::class_(m, "ginac_expression") .def("expand", [](ex &ge) { - // exmap m; - // ex q; - // q = ge.to_polynomial(m).normal(); - // return q.subs(m); - // return factor(ge.normal()); return ge.expand(); }) + .def("normal", &ex::normal) .def("__str__", [](ex &ge) { std::ostringstream stream; stream << ge; From 208a5dac01ca7dab7830f70119eeb0e1ba94918e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 20 Nov 2023 13:27:31 -0700 Subject: [PATCH 05/99] simplification interface --- pyomo/contrib/simplification/simplify.py | 35 +++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 70d5dfcd9ac..1de228fb444 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -1,6 +1,17 @@ from pyomo.core.expr.sympy_tools import sympy2pyomo_expression, sympyify_expression from pyomo.core.expr.numeric_expr import NumericExpression from pyomo.core.expr.numvalue import is_fixed, value +import logging +import warnings +try: + from pyomo.contrib.simplification.ginac_interface import GinacInterface + ginac_available = True +except: + GinacInterface = None + ginac_available = False + + +logger = logging.getLogger(__name__) def simplify_with_sympy(expr: NumericExpression): @@ -9,4 +20,26 @@ def simplify_with_sympy(expr: NumericExpression): new_expr = sympy2pyomo_expression(se, om) if is_fixed(new_expr): new_expr = value(new_expr) - return new_expr \ No newline at end of file + return new_expr + + +def simplify_with_ginac(expr: NumericExpression, ginac_interface): + gi = ginac_interface + return gi.from_ginac(gi.to_ginac(expr).normal()) + + +class Simplifier(object): + def __init__(self, supress_no_ginac_warnings: bool = False) -> None: + if ginac_available: + self.gi = GinacInterface() + self.suppress_no_ginac_warnings = supress_no_ginac_warnings + + def simplify(self, expr: NumericExpression): + if ginac_available: + return simplify_with_ginac(expr, self.gi) + else: + if not self.suppress_no_ginac_warnings: + msg = f"GiNaC does not seem to be available. Using SymPy. Note that the GiNac interface is significantly faster." + logger.warning(msg) + warnings.warn(msg) + return simplify_with_sympy(expr) From 6ef89144c5b088574736e86553cb8c22d2dd9645 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 20 Nov 2023 21:08:25 -0700 Subject: [PATCH 06/99] bugs --- pyomo/contrib/simplification/__init__.py | 1 + pyomo/contrib/simplification/simplify.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/__init__.py b/pyomo/contrib/simplification/__init__.py index e69de29bb2d..c09e8b8b5e5 100644 --- a/pyomo/contrib/simplification/__init__.py +++ b/pyomo/contrib/simplification/__init__.py @@ -0,0 +1 @@ +from .simplify import Simplifier \ No newline at end of file diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 1de228fb444..938bff6b4b9 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -31,7 +31,7 @@ def simplify_with_ginac(expr: NumericExpression, ginac_interface): class Simplifier(object): def __init__(self, supress_no_ginac_warnings: bool = False) -> None: if ginac_available: - self.gi = GinacInterface() + self.gi = GinacInterface(False) self.suppress_no_ginac_warnings = supress_no_ginac_warnings def simplify(self, expr: NumericExpression): From af47ef791888b6327c295a06544c4c0771e712d9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 21 Nov 2023 00:48:31 -0700 Subject: [PATCH 07/99] simplification tests --- .../contrib/simplification/tests/__init__.py | 0 .../tests/test_simplification.py | 62 +++++++++++++++++++ pyomo/core/expr/compare.py | 14 ++++- 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 pyomo/contrib/simplification/tests/__init__.py create mode 100644 pyomo/contrib/simplification/tests/test_simplification.py diff --git a/pyomo/contrib/simplification/tests/__init__.py b/pyomo/contrib/simplification/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py new file mode 100644 index 00000000000..02107ba1d6c --- /dev/null +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -0,0 +1,62 @@ +from pyomo.common.unittest import TestCase +from pyomo.contrib.simplification import Simplifier +from pyomo.core.expr.compare import assertExpressionsEqual, compare_expressions +import pyomo.environ as pe +from pyomo.core.expr.calculus.diff_with_pyomo import reverse_sd + + +class TestSimplification(TestCase): + def test_simplify(self): + m = pe.ConcreteModel() + x = m.x = pe.Var(bounds=(0, None)) + e = x*pe.log(x) + der1 = reverse_sd(e)[x] + der2 = reverse_sd(der1)[x] + simp = Simplifier() + der2_simp = simp.simplify(der2) + expected = x**-1.0 + assertExpressionsEqual(self, expected, der2_simp) + + def test_param(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + p = m.p = pe.Param(mutable=True) + e1 = p*x**2 + p*x + p*x**2 + simp = Simplifier() + e2 = simp.simplify(e1) + exp1 = p*x**2.0*2.0 + p*x + exp2 = p*x + p*x**2.0*2.0 + self.assertTrue( + compare_expressions(e2, exp1) + or compare_expressions(e2, exp2) + or compare_expressions(e2, p*x + x**2.0*p*2.0) + or compare_expressions(e2, x**2.0*p*2.0 + p*x) + ) + + def test_mul(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + e = 2*x + simp = Simplifier() + e2 = simp.simplify(e) + expected = 2.0*x + assertExpressionsEqual(self, expected, e2) + + def test_sum(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + e = 2 + x + simp = Simplifier() + e2 = simp.simplify(e) + expected = x + 2.0 + assertExpressionsEqual(self, expected, e2) + + def test_neg(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + e = -pe.log(x) + simp = Simplifier() + e2 = simp.simplify(e) + expected = pe.log(x)*(-1.0) + assertExpressionsEqual(self, expected, e2) + diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index ec8d56896b8..96913f1de39 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -195,7 +195,19 @@ def compare_expressions(expr1, expr2, include_named_exprs=True): expr2, include_named_exprs=include_named_exprs ) try: - res = pn1 == pn2 + res = True + if len(pn1) != len(pn2): + res = False + if res: + for a, b in zip(pn1, pn2): + if a.__class__ is not b.__class__: + res = False + break + if a == b: + continue + else: + res = False + break except PyomoException: res = False return res From 475ec06fb243b9afa4f59a8cefbbacd56d6634f3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 10:35:33 -0700 Subject: [PATCH 08/99] simplification tests --- .../simplification/ginac_interface.cpp | 2 +- pyomo/contrib/simplification/simplify.py | 5 +- .../tests/test_simplification.py | 79 ++++++++++++++++--- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index 690885dc513..32bea8dadd0 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -21,7 +21,7 @@ ex ginac_expr_from_pyomo_node( case py_float: { double val = expr.cast(); if (is_integer(val)) { - res = numeric(expr.cast()); + res = numeric((long) val); } else { res = numeric(val); diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 938bff6b4b9..66a3dad0b06 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -25,7 +25,10 @@ def simplify_with_sympy(expr: NumericExpression): def simplify_with_ginac(expr: NumericExpression, ginac_interface): gi = ginac_interface - return gi.from_ginac(gi.to_ginac(expr).normal()) + ginac_expr = gi.to_ginac(expr) + ginac_expr = ginac_expr.normal() + new_expr = gi.from_ginac(ginac_expr) + return new_expr class Simplifier(object): diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 02107ba1d6c..4d9b0cec0d2 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -6,6 +6,14 @@ class TestSimplification(TestCase): + def compare_against_possible_results(self, got, expected_list): + success = False + for exp in expected_list: + if compare_expressions(got, exp): + success = True + break + self.assertTrue(success) + def test_simplify(self): m = pe.ConcreteModel() x = m.x = pe.Var(bounds=(0, None)) @@ -24,13 +32,16 @@ def test_param(self): e1 = p*x**2 + p*x + p*x**2 simp = Simplifier() e2 = simp.simplify(e1) - exp1 = p*x**2.0*2.0 + p*x - exp2 = p*x + p*x**2.0*2.0 - self.assertTrue( - compare_expressions(e2, exp1) - or compare_expressions(e2, exp2) - or compare_expressions(e2, p*x + x**2.0*p*2.0) - or compare_expressions(e2, x**2.0*p*2.0 + p*x) + self.compare_against_possible_results( + e2, + [ + p*x**2.0*2.0 + p*x, + p*x + p*x**2.0*2.0, + 2.0*p*x**2.0 + p*x, + p*x + 2.0*p*x**2.0, + x**2.0*p*2.0 + p*x, + p*x + x**2.0*p*2.0 + ] ) def test_mul(self): @@ -48,8 +59,13 @@ def test_sum(self): e = 2 + x simp = Simplifier() e2 = simp.simplify(e) - expected = x + 2.0 - assertExpressionsEqual(self, expected, e2) + self.compare_against_possible_results( + e2, + [ + 2.0 + x, + x + 2.0, + ] + ) def test_neg(self): m = pe.ConcreteModel() @@ -57,6 +73,47 @@ def test_neg(self): e = -pe.log(x) simp = Simplifier() e2 = simp.simplify(e) - expected = pe.log(x)*(-1.0) - assertExpressionsEqual(self, expected, e2) + self.compare_against_possible_results( + e2, + [ + (-1.0)*pe.log(x), + pe.log(x)*(-1.0), + -pe.log(x), + ] + ) + + def test_pow(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + e = x**2.0 + simp = Simplifier() + e2 = simp.simplify(e) + assertExpressionsEqual(self, e, e2) + def test_div(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + y = m.y = pe.Var() + e = x/y + y/x - x/y + simp = Simplifier() + e2 = simp.simplify(e) + print(e2) + self.compare_against_possible_results( + e2, + [ + y/x, + y*(1.0/x), + y*x**-1.0, + x**-1.0 * y, + ], + ) + + def test_unary(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + func_list = [pe.log, pe.sin, pe.cos, pe.tan, pe.asin, pe.acos, pe.atan] + for func in func_list: + e = func(x) + simp = Simplifier() + e2 = simp.simplify(e) + assertExpressionsEqual(self, e, e2) From 491db9f6793dc6d84fc3b771073d00d938b3a2f2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 15:57:34 -0700 Subject: [PATCH 09/99] update GHA to install ginac --- .github/workflows/test_pr_and_main.yml | 21 ++++++++++++++++++- .../tests/test_simplification.py | 2 ++ setup.cfg | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 2885fd107a8..12dc7c1daac 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -98,7 +98,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest'" + category: "-m 'neos or importtest or simplification'" skip_doctest: 1 TARGET: linux PYENV: pip @@ -179,6 +179,25 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} + - name: install ginac + if: ${{ matrix.other == "singletest" }} + run: | + pwd + cd .. + curl https://www.ginac.de/CLN/cln-1.3.6.tar.bz2 >cln-1.3.6.tar.bz2 + tar -xvf cln-1.3.6.tar.bz2 + cd cln-1.3.6 + ./configure + make + make install + cd + curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 + tar -xvf ginac-1.8.7.tar.bz2 + cd ginac-1.8.7 + ./configure + make + make install + - name: TPL package download cache uses: actions/cache@v3 if: ${{ ! matrix.slim }} diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 4d9b0cec0d2..ed59064022c 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -1,10 +1,12 @@ from pyomo.common.unittest import TestCase +from pyomo.common import unittest from pyomo.contrib.simplification import Simplifier from pyomo.core.expr.compare import assertExpressionsEqual, compare_expressions import pyomo.environ as pe from pyomo.core.expr.calculus.diff_with_pyomo import reverse_sd +@unittest.pytest.mark.simplification class TestSimplification(TestCase): def compare_against_possible_results(self, got, expected_list): success = False diff --git a/setup.cfg b/setup.cfg index b606138f38c..a431e0cd601 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,4 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests + simplification: marks simplification tests that have expensive (to install) dependencies From b3a1ff9b06e3fb2e8fd944bb27d2bf736a9cfd54 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:02:28 -0700 Subject: [PATCH 10/99] run black --- pyomo/contrib/simplification/build.py | 16 +++--- pyomo/contrib/simplification/simplify.py | 2 + .../tests/test_simplification.py | 49 ++++++------------- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 6f16607e22b..e8bd645756b 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -11,16 +11,16 @@ def build_ginac_interface(args=[]): dname = this_file_dir() - _sources = [ - 'ginac_interface.cpp', - ] + _sources = ['ginac_interface.cpp'] sources = list() for fname in _sources: sources.append(os.path.join(dname, fname)) ginac_lib = find_library('ginac') if ginac_lib is None: - raise RuntimeError('could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable') + raise RuntimeError( + 'could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable' + ) ginac_lib_dir = os.path.dirname(ginac_lib) ginac_build_dir = os.path.dirname(ginac_lib_dir) ginac_include_dir = os.path.join(ginac_build_dir, 'include') @@ -29,7 +29,9 @@ def build_ginac_interface(args=[]): cln_lib = find_library('cln') if cln_lib is None: - raise RuntimeError('could not find CLN library; please make sure it is in the LD_LIBRARY_PATH environment variable') + raise RuntimeError( + 'could not find CLN library; please make sure it is in the LD_LIBRARY_PATH environment variable' + ) cln_lib_dir = os.path.dirname(cln_lib) cln_build_dir = os.path.dirname(cln_lib_dir) cln_include_dir = os.path.join(cln_build_dir, 'include') @@ -38,8 +40,8 @@ def build_ginac_interface(args=[]): extra_args = ['-std=c++11'] ext = Pybind11Extension( - 'ginac_interface', - sources=sources, + 'ginac_interface', + sources=sources, language='c++', include_dirs=[cln_include_dir, ginac_include_dir], library_dirs=[cln_lib_dir, ginac_lib_dir], diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 66a3dad0b06..8f7f15f3826 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -3,8 +3,10 @@ from pyomo.core.expr.numvalue import is_fixed, value import logging import warnings + try: from pyomo.contrib.simplification.ginac_interface import GinacInterface + ginac_available = True except: GinacInterface = None diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index ed59064022c..cc278db4d43 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -19,7 +19,7 @@ def compare_against_possible_results(self, got, expected_list): def test_simplify(self): m = pe.ConcreteModel() x = m.x = pe.Var(bounds=(0, None)) - e = x*pe.log(x) + e = x * pe.log(x) der1 = reverse_sd(e)[x] der2 = reverse_sd(der1)[x] simp = Simplifier() @@ -31,28 +31,28 @@ def test_param(self): m = pe.ConcreteModel() x = m.x = pe.Var() p = m.p = pe.Param(mutable=True) - e1 = p*x**2 + p*x + p*x**2 + e1 = p * x**2 + p * x + p * x**2 simp = Simplifier() e2 = simp.simplify(e1) self.compare_against_possible_results( - e2, + e2, [ - p*x**2.0*2.0 + p*x, - p*x + p*x**2.0*2.0, - 2.0*p*x**2.0 + p*x, - p*x + 2.0*p*x**2.0, - x**2.0*p*2.0 + p*x, - p*x + x**2.0*p*2.0 - ] + p * x**2.0 * 2.0 + p * x, + p * x + p * x**2.0 * 2.0, + 2.0 * p * x**2.0 + p * x, + p * x + 2.0 * p * x**2.0, + x**2.0 * p * 2.0 + p * x, + p * x + x**2.0 * p * 2.0, + ], ) def test_mul(self): m = pe.ConcreteModel() x = m.x = pe.Var() - e = 2*x + e = 2 * x simp = Simplifier() e2 = simp.simplify(e) - expected = 2.0*x + expected = 2.0 * x assertExpressionsEqual(self, expected, e2) def test_sum(self): @@ -61,13 +61,7 @@ def test_sum(self): e = 2 + x simp = Simplifier() e2 = simp.simplify(e) - self.compare_against_possible_results( - e2, - [ - 2.0 + x, - x + 2.0, - ] - ) + self.compare_against_possible_results(e2, [2.0 + x, x + 2.0]) def test_neg(self): m = pe.ConcreteModel() @@ -76,12 +70,7 @@ def test_neg(self): simp = Simplifier() e2 = simp.simplify(e) self.compare_against_possible_results( - e2, - [ - (-1.0)*pe.log(x), - pe.log(x)*(-1.0), - -pe.log(x), - ] + e2, [(-1.0) * pe.log(x), pe.log(x) * (-1.0), -pe.log(x)] ) def test_pow(self): @@ -96,18 +85,12 @@ def test_div(self): m = pe.ConcreteModel() x = m.x = pe.Var() y = m.y = pe.Var() - e = x/y + y/x - x/y + e = x / y + y / x - x / y simp = Simplifier() e2 = simp.simplify(e) print(e2) self.compare_against_possible_results( - e2, - [ - y/x, - y*(1.0/x), - y*x**-1.0, - x**-1.0 * y, - ], + e2, [y / x, y * (1.0 / x), y * x**-1.0, x**-1.0 * y] ) def test_unary(self): From de8743a84c4902867a802756ab64fbd62eed920e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:03:42 -0700 Subject: [PATCH 11/99] syntax --- .github/workflows/test_pr_and_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 12dc7c1daac..7345fd45e10 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -180,7 +180,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: install ginac - if: ${{ matrix.other == "singletest" }} + if: ${{ matrix.other == 'singletest' }} run: | pwd cd .. From ee9f830984ad20446b28c8aa65674d5cbf264ab3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:07:25 -0700 Subject: [PATCH 12/99] install ginac in GHA --- .github/workflows/test_branches.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index e773587ec85..3270b3e8a95 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -94,6 +94,14 @@ jobs: PYENV: conda PACKAGES: mpi4py + - os: ubuntu-latest + python: 3.11 + other: /singletest + category: "-m 'neos or importtest or simplification'" + skip_doctest: 1 + TARGET: linux + PYENV: pip + - os: ubuntu-latest python: '3.10' other: /cython @@ -149,6 +157,25 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} + - name: install ginac + if: ${{ matrix.other == 'singletest' }} + run: | + pwd + cd .. + curl https://www.ginac.de/CLN/cln-1.3.6.tar.bz2 >cln-1.3.6.tar.bz2 + tar -xvf cln-1.3.6.tar.bz2 + cd cln-1.3.6 + ./configure + make + make install + cd + curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 + tar -xvf ginac-1.8.7.tar.bz2 + cd ginac-1.8.7 + ./configure + make + make install + - name: TPL package download cache uses: actions/cache@v3 if: ${{ ! matrix.slim }} From 36cfd6388d16d6f2077d51b9035c9e363b4e4e28 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:09:53 -0700 Subject: [PATCH 13/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 3270b3e8a95..97b9c6ed1dc 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -158,7 +158,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: install ginac - if: ${{ matrix.other == 'singletest' }} + if: matrix.TARGET == 'singletest' run: | pwd cd .. From 8269539ff67b48e4c7b652b8feabb14c64634fc6 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:11:27 -0700 Subject: [PATCH 14/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 97b9c6ed1dc..c56ec398134 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -158,7 +158,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: install ginac - if: matrix.TARGET == 'singletest' + if: matrix.other == 'singletest' run: | pwd cd .. From 7bb0ff501344dfc9bb162f9ed339293ad320d0d1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:14:07 -0700 Subject: [PATCH 15/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- pyomo/contrib/simplification/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index c56ec398134..bea57f314e5 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -158,7 +158,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: install ginac - if: matrix.other == 'singletest' + if: matrix.other == '/singletest' run: | pwd cd .. diff --git a/pyomo/contrib/simplification/__init__.py b/pyomo/contrib/simplification/__init__.py index c09e8b8b5e5..3abe5a25ba0 100644 --- a/pyomo/contrib/simplification/__init__.py +++ b/pyomo/contrib/simplification/__init__.py @@ -1 +1 @@ -from .simplify import Simplifier \ No newline at end of file +from .simplify import Simplifier From 546dad1d46cc1a60c6fae711a6dbe8710f53220c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:23:06 -0700 Subject: [PATCH 16/99] install ginac in GHA --- .github/workflows/test_branches.yml | 10 ++++++++-- pyomo/contrib/simplification/simplify.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index bea57f314e5..16ce6a44003 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -167,14 +167,14 @@ jobs: cd cln-1.3.6 ./configure make - make install + sudo make install cd curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 tar -xvf ginac-1.8.7.tar.bz2 cd ginac-1.8.7 ./configure make - make install + sudo make install - name: TPL package download cache uses: actions/cache@v3 @@ -630,6 +630,12 @@ jobs: echo "" pyomo build-extensions --parallel 2 + - name: Install GiNaC Interface + if: matrix.other == '/singletest' + run: | + cd pyomo/contrib/simplification/ + $PYTHON_EXE build.py --inplace + - name: Report pyomo plugin information run: | echo "$PATH" diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 8f7f15f3826..4002f1a233f 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -34,10 +34,10 @@ def simplify_with_ginac(expr: NumericExpression, ginac_interface): class Simplifier(object): - def __init__(self, supress_no_ginac_warnings: bool = False) -> None: + def __init__(self, suppress_no_ginac_warnings: bool = False) -> None: if ginac_available: self.gi = GinacInterface(False) - self.suppress_no_ginac_warnings = supress_no_ginac_warnings + self.suppress_no_ginac_warnings = suppress_no_ginac_warnings def simplify(self, expr: NumericExpression): if ginac_available: From 37a955bdfaccec8508887dbf037eb5ea72b5b5cb Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:24:16 -0700 Subject: [PATCH 17/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 16ce6a44003..477361683ac 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -634,7 +634,7 @@ jobs: if: matrix.other == '/singletest' run: | cd pyomo/contrib/simplification/ - $PYTHON_EXE build.py --inplace + $PYTHON_EXE build.py --inplace - name: Report pyomo plugin information run: | From 05134ce49621f1efd16d961dc20e34c7cff23475 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 16:45:55 -0700 Subject: [PATCH 18/99] install ginac in GHA --- .github/workflows/test_branches.yml | 1 + pyomo/contrib/simplification/build.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 477361683ac..7933aa522d8 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -633,6 +633,7 @@ jobs: - name: Install GiNaC Interface if: matrix.other == '/singletest' run: | + ls /usr/local/include/ginac/ cd pyomo/contrib/simplification/ $PYTHON_EXE build.py --inplace diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index e8bd645756b..39742e1e351 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -17,6 +17,7 @@ def build_ginac_interface(args=[]): sources.append(os.path.join(dname, fname)) ginac_lib = find_library('ginac') + print(ginac_lib) if ginac_lib is None: raise RuntimeError( 'could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable' From 1151927df204f6969704ec7b671e25a1d9083031 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 17:51:34 -0700 Subject: [PATCH 19/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 7933aa522d8..f2bf057da51 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -633,7 +633,7 @@ jobs: - name: Install GiNaC Interface if: matrix.other == '/singletest' run: | - ls /usr/local/include/ginac/ + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH cd pyomo/contrib/simplification/ $PYTHON_EXE build.py --inplace From 2c4fdbee83ab76b35a863748048ddf0d091f2f3c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 18:19:05 -0700 Subject: [PATCH 20/99] skip tests when dependencies are not available --- pyomo/contrib/simplification/tests/test_simplification.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index cc278db4d43..c50a906afe7 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -1,11 +1,17 @@ from pyomo.common.unittest import TestCase from pyomo.common import unittest from pyomo.contrib.simplification import Simplifier +from pyomo.contrib.simplification.simplify import ginac_available from pyomo.core.expr.compare import assertExpressionsEqual, compare_expressions import pyomo.environ as pe from pyomo.core.expr.calculus.diff_with_pyomo import reverse_sd +from pyomo.common.dependencies import attempt_import +sympy, sympy_available = attempt_import('sympy') + + +@unittest.skipIf((not sympy_available) and (not ginac_available), 'neither sympy nor ginac are available') @unittest.pytest.mark.simplification class TestSimplification(TestCase): def compare_against_possible_results(self, got, expected_list): From 9ebd79b898aea6827232220f59c615c771686ddb Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 18:26:40 -0700 Subject: [PATCH 21/99] install ginac in GHA --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index f2bf057da51..b97b3a682af 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -168,7 +168,7 @@ jobs: ./configure make sudo make install - cd + cd .. curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 tar -xvf ginac-1.8.7.tar.bz2 cd ginac-1.8.7 diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 7345fd45e10..b99d39cf6c7 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -180,23 +180,22 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: install ginac - if: ${{ matrix.other == 'singletest' }} + if: matrix.other == '/singletest' run: | - pwd cd .. curl https://www.ginac.de/CLN/cln-1.3.6.tar.bz2 >cln-1.3.6.tar.bz2 tar -xvf cln-1.3.6.tar.bz2 cd cln-1.3.6 ./configure make - make install - cd + sudo make install + cd .. curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 tar -xvf ginac-1.8.7.tar.bz2 cd ginac-1.8.7 ./configure make - make install + sudo make install - name: TPL package download cache uses: actions/cache@v3 @@ -652,6 +651,13 @@ jobs: echo "" pyomo build-extensions --parallel 2 + - name: Install GiNaC Interface + if: matrix.other == '/singletest' + run: | + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + cd pyomo/contrib/simplification/ + $PYTHON_EXE build.py --inplace + - name: Report pyomo plugin information run: | echo "$PATH" From 0bd156351b09d93288d618715817fcc4c530114a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 18:28:39 -0700 Subject: [PATCH 22/99] update simplification tests --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- pyomo/contrib/simplification/tests/test_simplification.py | 1 - setup.cfg | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index b97b3a682af..13a5653f9f5 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -97,7 +97,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest or simplification'" + category: "-m 'neos or importtest'" skip_doctest: 1 TARGET: linux PYENV: pip diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index b99d39cf6c7..8a8a9b08030 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -98,7 +98,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest or simplification'" + category: "-m 'neos or importtest'" skip_doctest: 1 TARGET: linux PYENV: pip diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index c50a906afe7..f3bce9cee54 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -12,7 +12,6 @@ @unittest.skipIf((not sympy_available) and (not ginac_available), 'neither sympy nor ginac are available') -@unittest.pytest.mark.simplification class TestSimplification(TestCase): def compare_against_possible_results(self, got, expected_list): success = False diff --git a/setup.cfg b/setup.cfg index a431e0cd601..b606138f38c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,4 +22,3 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests - simplification: marks simplification tests that have expensive (to install) dependencies From 26007ac42688cbe7554807198ceb20e9d121d68e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 18:30:57 -0700 Subject: [PATCH 23/99] run black --- pyomo/contrib/simplification/tests/test_simplification.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index f3bce9cee54..e6b5ae863f6 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -11,7 +11,10 @@ sympy, sympy_available = attempt_import('sympy') -@unittest.skipIf((not sympy_available) and (not ginac_available), 'neither sympy nor ginac are available') +@unittest.skipIf( + (not sympy_available) and (not ginac_available), + 'neither sympy nor ginac are available', +) class TestSimplification(TestCase): def compare_against_possible_results(self, got, expected_list): success = False From fc36411c1c57aaf4720938cf7926cfcf7048bc75 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 18:56:21 -0700 Subject: [PATCH 24/99] add pytest marker for simplification --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- pyomo/contrib/simplification/tests/test_simplification.py | 1 + setup.cfg | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 13a5653f9f5..b97b3a682af 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -97,7 +97,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest'" + category: "-m 'neos or importtest or simplification'" skip_doctest: 1 TARGET: linux PYENV: pip diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 8a8a9b08030..b99d39cf6c7 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -98,7 +98,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest'" + category: "-m 'neos or importtest or simplification'" skip_doctest: 1 TARGET: linux PYENV: pip diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index e6b5ae863f6..152db93a358 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -15,6 +15,7 @@ (not sympy_available) and (not ginac_available), 'neither sympy nor ginac are available', ) +@unittest.pytest.mark.simplification class TestSimplification(TestCase): def compare_against_possible_results(self, got, expected_list): success = False diff --git a/setup.cfg b/setup.cfg index b606138f38c..855717490b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,4 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests + simplification: tests for expression simplification that have expensive (to install) dependencies From 5ba03e215c51fe2feba6a5cff7b2baa162fcb87f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 19:30:50 -0700 Subject: [PATCH 25/99] update GHA --- .github/workflows/test_branches.yml | 1 + .github/workflows/test_pr_and_main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 243f57ea7aa..b73a9cabc81 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -641,6 +641,7 @@ jobs: if: matrix.other == '/singletest' run: | export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV cd pyomo/contrib/simplification/ $PYTHON_EXE build.py --inplace diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 93919ca6bc3..1c36b89710c 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -662,6 +662,7 @@ jobs: if: matrix.other == '/singletest' run: | export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV cd pyomo/contrib/simplification/ $PYTHON_EXE build.py --inplace From 90cdeba86b2deaa76ce7b62cd9f21f906b057462 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 19:32:33 -0700 Subject: [PATCH 26/99] update GHA --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index b73a9cabc81..e3e0c3e6caf 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -166,14 +166,14 @@ jobs: tar -xvf cln-1.3.6.tar.bz2 cd cln-1.3.6 ./configure - make + make -j 2 sudo make install cd .. curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 tar -xvf ginac-1.8.7.tar.bz2 cd ginac-1.8.7 ./configure - make + make -j 2 sudo make install - name: TPL package download cache diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 1c36b89710c..9edb8b1c65f 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -187,14 +187,14 @@ jobs: tar -xvf cln-1.3.6.tar.bz2 cd cln-1.3.6 ./configure - make + make -j 2 sudo make install cd .. curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 tar -xvf ginac-1.8.7.tar.bz2 cd ginac-1.8.7 ./configure - make + make -j 2 sudo make install - name: TPL package download cache From 17cb11d31d72d7ac9ac9f37e1ec306f8d114cec4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 19:50:50 -0700 Subject: [PATCH 27/99] debugging GHA --- .github/workflows/test_branches.yml | 1 + .github/workflows/test_pr_and_main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index e3e0c3e6caf..4e3c14cb70e 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -655,6 +655,7 @@ jobs: - name: Run Pyomo tests if: matrix.mpi == 0 run: | + $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface" $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 9edb8b1c65f..1626964a7e9 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -676,6 +676,7 @@ jobs: - name: Run Pyomo tests if: matrix.mpi == 0 run: | + $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface" $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ From d1fe24400ed424afdfdf65e3f9918faefc98db96 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 20:02:34 -0700 Subject: [PATCH 28/99] test simplification with ginac and sympy --- .../simplification/tests/test_simplification.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 152db93a358..096d776460d 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -12,11 +12,10 @@ @unittest.skipIf( - (not sympy_available) and (not ginac_available), - 'neither sympy nor ginac are available', + (not sympy_available) or (ginac_available), + 'sympy is not available', ) -@unittest.pytest.mark.simplification -class TestSimplification(TestCase): +class TestSimplificationSympy(TestCase): def compare_against_possible_results(self, got, expected_list): success = False for exp in expected_list: @@ -111,3 +110,12 @@ def test_unary(self): simp = Simplifier() e2 = simp.simplify(e) assertExpressionsEqual(self, e, e2) + + +@unittest.skipIf( + not ginac_available, + 'GiNaC is not available', +) +@unittest.pytest.mark.simplification +class TestSimplificationGiNaC(TestSimplificationSympy): + pass From 9159c3c5854c7a9d5437f3308321a9fd4a61be6f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 20:03:18 -0700 Subject: [PATCH 29/99] test simplification with ginac and sympy --- .../tests/test_simplification.py | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 096d776460d..3124d856784 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -35,25 +35,6 @@ def test_simplify(self): expected = x**-1.0 assertExpressionsEqual(self, expected, der2_simp) - def test_param(self): - m = pe.ConcreteModel() - x = m.x = pe.Var() - p = m.p = pe.Param(mutable=True) - e1 = p * x**2 + p * x + p * x**2 - simp = Simplifier() - e2 = simp.simplify(e1) - self.compare_against_possible_results( - e2, - [ - p * x**2.0 * 2.0 + p * x, - p * x + p * x**2.0 * 2.0, - 2.0 * p * x**2.0 + p * x, - p * x + 2.0 * p * x**2.0, - x**2.0 * p * 2.0 + p * x, - p * x + x**2.0 * p * 2.0, - ], - ) - def test_mul(self): m = pe.ConcreteModel() x = m.x = pe.Var() @@ -118,4 +99,21 @@ def test_unary(self): ) @unittest.pytest.mark.simplification class TestSimplificationGiNaC(TestSimplificationSympy): - pass + def test_param(self): + m = pe.ConcreteModel() + x = m.x = pe.Var() + p = m.p = pe.Param(mutable=True) + e1 = p * x**2 + p * x + p * x**2 + simp = Simplifier() + e2 = simp.simplify(e1) + self.compare_against_possible_results( + e2, + [ + p * x**2.0 * 2.0 + p * x, + p * x + p * x**2.0 * 2.0, + 2.0 * p * x**2.0 + p * x, + p * x + 2.0 * p * x**2.0, + x**2.0 * p * 2.0 + p * x, + p * x + x**2.0 * p * 2.0, + ], + ) From 457f2b378b772eae16b47fa789423bca70de43c5 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 22:17:46 -0700 Subject: [PATCH 30/99] fixing simplification tests --- .github/workflows/test_branches.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 4e3c14cb70e..0eb6fe166d9 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -655,7 +655,8 @@ jobs: - name: Run Pyomo tests if: matrix.mpi == 0 run: | - $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface" + $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface; print(GinacInterface)" + $PYTHON_EXE -c "from pyomo.contrib.simplification.simplify import ginac_available; print(ginac_available)" $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ From 1e7c3f1b2ecf562000088df80f215ddf6d992420 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 22:44:54 -0700 Subject: [PATCH 31/99] fixing simplification tests --- .../simplification/tests/test_simplification.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 3124d856784..6badc76b957 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -11,11 +11,7 @@ sympy, sympy_available = attempt_import('sympy') -@unittest.skipIf( - (not sympy_available) or (ginac_available), - 'sympy is not available', -) -class TestSimplificationSympy(TestCase): +class SimplificationMixin: def compare_against_possible_results(self, got, expected_list): success = False for exp in expected_list: @@ -93,12 +89,20 @@ def test_unary(self): assertExpressionsEqual(self, e, e2) +@unittest.skipIf( + (not sympy_available) or (ginac_available), + 'sympy is not available', +) +class TestSimplificationSympy(TestCase, SimplificationMixin): + pass + + @unittest.skipIf( not ginac_available, 'GiNaC is not available', ) @unittest.pytest.mark.simplification -class TestSimplificationGiNaC(TestSimplificationSympy): +class TestSimplificationGiNaC(TestCase, SimplificationMixin): def test_param(self): m = pe.ConcreteModel() x = m.x = pe.Var() From b2d969f73e4dbf1e80d453dd7be4dcd474fc32be Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 22:46:36 -0700 Subject: [PATCH 32/99] fixing simplification tests --- .../simplification/tests/test_simplification.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 6badc76b957..e3c60cb02ca 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -89,18 +89,12 @@ def test_unary(self): assertExpressionsEqual(self, e, e2) -@unittest.skipIf( - (not sympy_available) or (ginac_available), - 'sympy is not available', -) +@unittest.skipIf((not sympy_available) or (ginac_available), 'sympy is not available') class TestSimplificationSympy(TestCase, SimplificationMixin): pass -@unittest.skipIf( - not ginac_available, - 'GiNaC is not available', -) +@unittest.skipIf(not ginac_available, 'GiNaC is not available') @unittest.pytest.mark.simplification class TestSimplificationGiNaC(TestCase, SimplificationMixin): def test_param(self): From da2fe3d714b34c7b03a41a2c63af8590db87d2e4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 10 Jan 2024 23:04:15 -0700 Subject: [PATCH 33/99] fixing simplification tests --- .github/workflows/test_branches.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 0eb6fe166d9..1124a253ac8 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -657,6 +657,7 @@ jobs: run: | $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface; print(GinacInterface)" $PYTHON_EXE -c "from pyomo.contrib.simplification.simplify import ginac_available; print(ginac_available)" + pytest -v pyomo/contrib/simplification/tests/test_simplification.py $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ From 2b1596b7ccda4b3a68747d5c6916b0c7570dae85 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 04:50:02 -0700 Subject: [PATCH 34/99] fixing simplification tests --- .github/workflows/test_branches.yml | 2 +- pyomo/contrib/simplification/simplify.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 89cb12d3eac..9743e45be41 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -655,7 +655,7 @@ jobs: run: | $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface; print(GinacInterface)" $PYTHON_EXE -c "from pyomo.contrib.simplification.simplify import ginac_available; print(ginac_available)" - pytest -v pyomo/contrib/simplification/tests/test_simplification.py + pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 4002f1a233f..5c0c5b859e7 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -6,7 +6,6 @@ try: from pyomo.contrib.simplification.ginac_interface import GinacInterface - ginac_available = True except: GinacInterface = None From d75c5f892a5dce5d6841c11c9dc02494c61e87b5 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 04:55:31 -0700 Subject: [PATCH 35/99] fixing simplification tests --- .github/workflows/test_branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 9743e45be41..2384ea58e2a 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -162,9 +162,9 @@ jobs: run: | pwd cd .. - curl https://www.ginac.de/CLN/cln-1.3.6.tar.bz2 >cln-1.3.6.tar.bz2 - tar -xvf cln-1.3.6.tar.bz2 - cd cln-1.3.6 + curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 + tar -xvf cln-1.3.7.tar.bz2 + cd cln-1.3.7 ./configure make -j 2 sudo make install From dae02054827c2ae23d608e07a2e6913a15180631 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 05:00:16 -0700 Subject: [PATCH 36/99] run black --- pyomo/contrib/simplification/simplify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 5c0c5b859e7..4002f1a233f 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -6,6 +6,7 @@ try: from pyomo.contrib.simplification.ginac_interface import GinacInterface + ginac_available = True except: GinacInterface = None From 313108d131f04e80deb59862e95099a731c84006 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 05:15:38 -0700 Subject: [PATCH 37/99] fixing simplification tests --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 2384ea58e2a..60f971c0c51 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -657,7 +657,7 @@ jobs: $PYTHON_EXE -c "from pyomo.contrib.simplification.simplify import ginac_available; print(ginac_available)" pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py $PYTHON_EXE -m pytest -v \ - -W ignore::Warning ${{matrix.category}} \ + ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml" From 5ff4d4d3b03370580da4578e141b98d2bdaaadf0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 05:24:22 -0700 Subject: [PATCH 38/99] fixing simplification tests --- .github/workflows/test_branches.yml | 2 +- setup.cfg | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 60f971c0c51..5ed5f908d28 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -97,7 +97,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest or simplification'" + category: "-m 'simplification'" skip_doctest: 1 TARGET: linux PYENV: pip diff --git a/setup.cfg b/setup.cfg index 855717490b3..e8b6933bbbc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,6 @@ license_files = LICENSE.md universal=1 [tool:pytest] -filterwarnings = ignore::RuntimeWarning junit_family = xunit2 markers = default: mark a test that should always run by default From 010a997ff78390af504d2dc0493cf8e96ee5fd7f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 05:47:50 -0700 Subject: [PATCH 39/99] fixing simplification tests --- .github/workflows/test_branches.yml | 9 +++------ setup.cfg | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 5ed5f908d28..b766583e259 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -97,7 +97,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'simplification'" + category: "-m 'neos or importtest or simplification'" skip_doctest: 1 TARGET: linux PYENV: pip @@ -653,11 +653,8 @@ jobs: - name: Run Pyomo tests if: matrix.mpi == 0 run: | - $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface; print(GinacInterface)" - $PYTHON_EXE -c "from pyomo.contrib.simplification.simplify import ginac_available; print(ginac_available)" - pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py - $PYTHON_EXE -m pytest -v \ - ${{matrix.category}} \ + pytest -v \ + -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml" diff --git a/setup.cfg b/setup.cfg index e8b6933bbbc..855717490b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,7 @@ license_files = LICENSE.md universal=1 [tool:pytest] +filterwarnings = ignore::RuntimeWarning junit_family = xunit2 markers = default: mark a test that should always run by default From 2c59b2930d2d1b41dde9f919810a3028f248ed1d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 06:22:27 -0700 Subject: [PATCH 40/99] fixing simplification tests --- .github/workflows/test_branches.yml | 9 +++++++-- pyomo/core/expr/compare.py | 14 +------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index b766583e259..15880896961 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -97,7 +97,7 @@ jobs: - os: ubuntu-latest python: 3.11 other: /singletest - category: "-m 'neos or importtest or simplification'" + category: "-m 'neos or importtest'" skip_doctest: 1 TARGET: linux PYENV: pip @@ -653,11 +653,16 @@ jobs: - name: Run Pyomo tests if: matrix.mpi == 0 run: | - pytest -v \ + $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml" + - name: Run Simplification Tests + if: matrix.other == '/singletest' + run: | + pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" + - name: Run Pyomo MPI tests if: matrix.mpi != 0 run: | diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index 96913f1de39..ec8d56896b8 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -195,19 +195,7 @@ def compare_expressions(expr1, expr2, include_named_exprs=True): expr2, include_named_exprs=include_named_exprs ) try: - res = True - if len(pn1) != len(pn2): - res = False - if res: - for a, b in zip(pn1, pn2): - if a.__class__ is not b.__class__: - res = False - break - if a == b: - continue - else: - res = False - break + res = pn1 == pn2 except PyomoException: res = False return res From d67d90d8c1226d0afa96ff700e173819eb822b7f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 06:25:16 -0700 Subject: [PATCH 41/99] fixing simplification tests --- .github/workflows/test_branches.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 15880896961..5eac447fd02 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -650,6 +650,11 @@ jobs: pyomo help --transformations || exit 1 pyomo help --writers || exit 1 + - name: Run Simplification Tests + if: matrix.other == '/singletest' + run: | + pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" + - name: Run Pyomo tests if: matrix.mpi == 0 run: | @@ -658,11 +663,6 @@ jobs: pyomo `pwd`/pyomo-model-libraries \ `pwd`/examples `pwd`/doc --junitxml="TEST-pyomo.xml" - - name: Run Simplification Tests - if: matrix.other == '/singletest' - run: | - pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" - - name: Run Pyomo MPI tests if: matrix.mpi != 0 run: | From 3fcec359e1f70842af03fe9fdb8b0629785a1b64 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 06:31:28 -0700 Subject: [PATCH 42/99] fixing simplification tests --- .github/workflows/test_branches.yml | 1 - .github/workflows/test_pr_and_main.yml | 14 +++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 5eac447fd02..d66451a00a5 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -160,7 +160,6 @@ jobs: - name: install ginac if: matrix.other == '/singletest' run: | - pwd cd .. curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 tar -xvf cln-1.3.7.tar.bz2 diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 9b82c565c32..bda6014b352 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -98,7 +98,7 @@ jobs: - os: ubuntu-latest python: '3.11' other: /singletest - category: "-m 'neos or importtest or simplification'" + category: "-m 'neos or importtest'" skip_doctest: 1 TARGET: linux PYENV: pip @@ -183,9 +183,9 @@ jobs: if: matrix.other == '/singletest' run: | cd .. - curl https://www.ginac.de/CLN/cln-1.3.6.tar.bz2 >cln-1.3.6.tar.bz2 - tar -xvf cln-1.3.6.tar.bz2 - cd cln-1.3.6 + curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 + tar -xvf cln-1.3.7.tar.bz2 + cd cln-1.3.7 ./configure make -j 2 sudo make install @@ -671,10 +671,14 @@ jobs: pyomo help --transformations || exit 1 pyomo help --writers || exit 1 + - name: Run Simplification Tests + if: matrix.other == '/singletest' + run: | + pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" + - name: Run Pyomo tests if: matrix.mpi == 0 run: | - $PYTHON_EXE -c "from pyomo.contrib.simplification.ginac_interface import GinacInterface" $PYTHON_EXE -m pytest -v \ -W ignore::Warning ${{matrix.category}} \ pyomo `pwd`/pyomo-model-libraries \ From b5550fecb393d2ee9eeebba0e3df66b7360a10d9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 06:38:34 -0700 Subject: [PATCH 43/99] fixing simplification tests --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index d66451a00a5..83e652cbef8 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -652,7 +652,7 @@ jobs: - name: Run Simplification Tests if: matrix.other == '/singletest' run: | - pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" + pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" - name: Run Pyomo tests if: matrix.mpi == 0 diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index bda6014b352..6df28fbadc9 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -674,7 +674,7 @@ jobs: - name: Run Simplification Tests if: matrix.other == '/singletest' run: | - pytest -v -m 'simplification' pyomo.contrib.simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" + pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" - name: Run Pyomo tests if: matrix.mpi == 0 From 8984389e71d57fe7b9d56c80331ea207166f1290 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 22 Jan 2024 06:52:04 -0700 Subject: [PATCH 44/99] fixing simplification tests --- pyomo/core/expr/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index ec8d56896b8..61ff8660a8b 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -196,7 +196,7 @@ def compare_expressions(expr1, expr2, include_named_exprs=True): ) try: res = pn1 == pn2 - except PyomoException: + except (PyomoException, AttributeError): res = False return res From 92163f2989dc99d4e14f63c658c0cc77e3d9dc7a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 21 Feb 2024 09:15:41 -0700 Subject: [PATCH 45/99] cleanup --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- pyomo/contrib/simplification/__init__.py | 11 +++++++++++ pyomo/contrib/simplification/build.py | 11 +++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 83e652cbef8..57c24d99090 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -157,7 +157,7 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - - name: install ginac + - name: install GiNaC if: matrix.other == '/singletest' run: | cd .. diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 6df28fbadc9..a55d100a18f 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -179,7 +179,7 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - - name: install ginac + - name: install GiNaC if: matrix.other == '/singletest' run: | cd .. diff --git a/pyomo/contrib/simplification/__init__.py b/pyomo/contrib/simplification/__init__.py index 3abe5a25ba0..b4fa68eb386 100644 --- a/pyomo/contrib/simplification/__init__.py +++ b/pyomo/contrib/simplification/__init__.py @@ -1 +1,12 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from .simplify import Simplifier diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 39742e1e351..67ae1d37335 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pybind11.setup_helpers import Pybind11Extension, build_ext from pyomo.common.fileutils import this_file_dir, find_library import os From 0dff80fb37b9052805d744f42236711067d69bcc Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 21 Feb 2024 09:16:59 -0700 Subject: [PATCH 46/99] cleanup --- pyomo/contrib/simplification/build.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 67ae1d37335..4bf28a0fa33 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -9,15 +9,16 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from pybind11.setup_helpers import Pybind11Extension, build_ext -from pyomo.common.fileutils import this_file_dir, find_library +import glob import os -from distutils.dist import Distribution -import sys import shutil -import glob +import sys import tempfile +from distutils.dist import Distribution + +from pybind11.setup_helpers import Pybind11Extension, build_ext from pyomo.common.envvar import PYOMO_CONFIG_DIR +from pyomo.common.fileutils import find_library, this_file_dir def build_ginac_interface(args=[]): From c49c2df810775f1c64702a26e3cf75c36f717c99 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 21 Feb 2024 09:23:10 -0700 Subject: [PATCH 47/99] cleanup --- pyomo/contrib/simplification/build.py | 11 ++++++----- pyomo/contrib/simplification/ginac_interface.cpp | 11 +++++++++++ pyomo/contrib/simplification/simplify.py | 11 +++++++++++ pyomo/contrib/simplification/tests/__init__.py | 11 +++++++++++ .../simplification/tests/test_simplification.py | 11 +++++++++++ 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 4bf28a0fa33..d9d1e701290 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -21,7 +21,9 @@ from pyomo.common.fileutils import find_library, this_file_dir -def build_ginac_interface(args=[]): +def build_ginac_interface(args=None): + if args is None: + args = list() dname = this_file_dir() _sources = ['ginac_interface.cpp'] sources = list() @@ -29,7 +31,6 @@ def build_ginac_interface(args=[]): sources.append(os.path.join(dname, fname)) ginac_lib = find_library('ginac') - print(ginac_lib) if ginac_lib is None: raise RuntimeError( 'could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable' @@ -62,7 +63,7 @@ def build_ginac_interface(args=[]): extra_compile_args=extra_args, ) - class ginac_build_ext(build_ext): + class ginacBuildExt(build_ext): def run(self): basedir = os.path.abspath(os.path.curdir) if self.inplace: @@ -72,7 +73,7 @@ def run(self): print("Building in '%s'" % tmpdir) os.chdir(tmpdir) try: - super(ginac_build_ext, self).run() + super(ginacBuildExt, self).run() if not self.inplace: library = glob.glob("build/*/ginac_interface.*")[0] target = os.path.join( @@ -94,7 +95,7 @@ def run(self): 'name': 'ginac_interface', 'packages': [], 'ext_modules': [ext], - 'cmdclass': {"build_ext": ginac_build_ext}, + 'cmdclass': {"build_ext": ginacBuildExt}, } dist = Distribution(package_config) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index 32bea8dadd0..489f281bc2c 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -1,3 +1,14 @@ +// ___________________________________________________________________________ +// +// Pyomo: Python Optimization Modeling Objects +// Copyright (c) 2008-2022 +// National Technology and Engineering Solutions of Sandia, LLC +// Under the terms of Contract DE-NA0003525 with National Technology and +// Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +// rights in this software. +// This software is distributed under the 3-clause BSD License. +// ___________________________________________________________________________ + #include "ginac_interface.hpp" diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 4002f1a233f..b8cc4995f91 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pyomo.core.expr.sympy_tools import sympy2pyomo_expression, sympyify_expression from pyomo.core.expr.numeric_expr import NumericExpression from pyomo.core.expr.numvalue import is_fixed, value diff --git a/pyomo/contrib/simplification/tests/__init__.py b/pyomo/contrib/simplification/tests/__init__.py index e69de29bb2d..9320e403e95 100644 --- a/pyomo/contrib/simplification/tests/__init__.py +++ b/pyomo/contrib/simplification/tests/__init__.py @@ -0,0 +1,11 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index e3c60cb02ca..95402f98318 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pyomo.common.unittest import TestCase from pyomo.common import unittest from pyomo.contrib.simplification import Simplifier From 29d6a19d0f1704a294d62d5a89370d6110a4fbe7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 21 Feb 2024 09:29:35 -0700 Subject: [PATCH 48/99] cleanup --- pyomo/contrib/simplification/tests/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/simplification/tests/__init__.py b/pyomo/contrib/simplification/tests/__init__.py index 9320e403e95..d93cfd77b3c 100644 --- a/pyomo/contrib/simplification/tests/__init__.py +++ b/pyomo/contrib/simplification/tests/__init__.py @@ -8,4 +8,3 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ - From 9cb26c7629c49b055ec699e505e4c5ab32d07d62 Mon Sep 17 00:00:00 2001 From: Bethany Nicholson Date: Wed, 28 Feb 2024 11:16:41 -0700 Subject: [PATCH 49/99] NFC: Fix year in copyright assertion comment Co-authored-by: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> --- pyomo/contrib/simplification/__init__.py | 2 +- pyomo/contrib/simplification/build.py | 2 +- pyomo/contrib/simplification/ginac_interface.cpp | 2 +- pyomo/contrib/simplification/simplify.py | 2 +- pyomo/contrib/simplification/tests/__init__.py | 2 +- pyomo/contrib/simplification/tests/test_simplification.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/simplification/__init__.py b/pyomo/contrib/simplification/__init__.py index b4fa68eb386..c6111ddcb89 100644 --- a/pyomo/contrib/simplification/__init__.py +++ b/pyomo/contrib/simplification/__init__.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2022 +# Copyright (c) 2008-2024 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index d9d1e701290..2c7b1830ff6 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2022 +# Copyright (c) 2008-2024 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac_interface.cpp index 489f281bc2c..1060f87161c 100644 --- a/pyomo/contrib/simplification/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac_interface.cpp @@ -1,7 +1,7 @@ // ___________________________________________________________________________ // // Pyomo: Python Optimization Modeling Objects -// Copyright (c) 2008-2022 +// Copyright (c) 2008-2024 // National Technology and Engineering Solutions of Sandia, LLC // Under the terms of Contract DE-NA0003525 with National Technology and // Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index b8cc4995f91..00c5dde348e 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2022 +# Copyright (c) 2008-2024 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/contrib/simplification/tests/__init__.py b/pyomo/contrib/simplification/tests/__init__.py index d93cfd77b3c..a4a626013c4 100644 --- a/pyomo/contrib/simplification/tests/__init__.py +++ b/pyomo/contrib/simplification/tests/__init__.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2022 +# Copyright (c) 2008-2024 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 95402f98318..1a5ae1e0036 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2022 +# Copyright (c) 2008-2024 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain From 1e609c067dc01e7cd6ea8c65a347e540f0534d02 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 4 Mar 2024 13:54:23 -0700 Subject: [PATCH 50/99] run black --- pyomo/contrib/simplification/simplify.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index b8cc4995f91..80e7c42549c 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -55,7 +55,11 @@ def simplify(self, expr: NumericExpression): return simplify_with_ginac(expr, self.gi) else: if not self.suppress_no_ginac_warnings: - msg = f"GiNaC does not seem to be available. Using SymPy. Note that the GiNac interface is significantly faster." + msg = ( + "GiNaC does not seem to be available. Using SymPy. " + + "Note that the GiNac interface is significantly faster." + ) logger.warning(msg) warnings.warn(msg) + self.suppress_no_ginac_warnings = True return simplify_with_sympy(expr) From f4e989fc390cd08ec068dcdcb2b1826683bbb88d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 14:37:24 -0600 Subject: [PATCH 51/99] Add fileutils patch so find_library returns absolute path on Linux --- pyomo/common/fileutils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pyomo/common/fileutils.py b/pyomo/common/fileutils.py index 2cade36154d..80fd47a6377 100644 --- a/pyomo/common/fileutils.py +++ b/pyomo/common/fileutils.py @@ -38,6 +38,7 @@ import os import platform import importlib.util +import subprocess import sys from . import envvar @@ -375,9 +376,25 @@ def find_library(libname, cwd=True, include_PATH=True, pathlist=None): if libname_base.startswith('lib') and _system() != 'windows': libname_base = libname_base[3:] if ext.lower().startswith(('.so', '.dll', '.dylib')): - return ctypes.util.find_library(libname_base) + lib = ctypes.util.find_library(libname_base) else: - return ctypes.util.find_library(libname) + lib = ctypes.util.find_library(libname) + if lib and os.path.sep not in lib: + # work around https://github.com/python/cpython/issues/65241, + # where python does not return the absolute path on *nix + try: + libname = lib + ' ' + with subprocess.Popen(['/sbin/ldconfig', '-p'], + stdin=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + env={'LC_ALL': 'C', 'LANG': 'C'}) as p: + for line in os.fsdecode(p.stdout.read()).splitlines(): + if line.lstrip().startswith(libname): + return os.path.realpath(line.split()[-1]) + except: + pass + return lib def find_executable(exename, cwd=True, include_PATH=True, pathlist=None): From ebb4d0768f30dbfaf09d56db55f7973ce6bb554a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 14:38:21 -0600 Subject: [PATCH 52/99] bugfix: add missing import --- pyomo/contrib/simplification/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 2c7b1830ff6..79bc9970241 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -17,6 +17,7 @@ from distutils.dist import Distribution from pybind11.setup_helpers import Pybind11Extension, build_ext +from pyomo.common.cmake_builder import handleReadonly from pyomo.common.envvar import PYOMO_CONFIG_DIR from pyomo.common.fileutils import find_library, this_file_dir From 16d49e13605cf1c8c4a3ba1a4079b5d751d55791 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 14:38:54 -0600 Subject: [PATCH 53/99] NFC: clarify ginac builder exception message --- pyomo/contrib/simplification/build.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 79bc9970241..508acb2d5a1 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -34,7 +34,9 @@ def build_ginac_interface(args=None): ginac_lib = find_library('ginac') if ginac_lib is None: raise RuntimeError( - 'could not find GiNaC library; please make sure it is in the LD_LIBRARY_PATH environment variable' + 'could not find the GiNaC library; please make sure either to install ' + 'the library and development headers system-wide, or include the ' + 'path tt the library in the LD_LIBRARY_PATH environment variable' ) ginac_lib_dir = os.path.dirname(ginac_lib) ginac_build_dir = os.path.dirname(ginac_lib_dir) @@ -45,7 +47,9 @@ def build_ginac_interface(args=None): cln_lib = find_library('cln') if cln_lib is None: raise RuntimeError( - 'could not find CLN library; please make sure it is in the LD_LIBRARY_PATH environment variable' + 'could not find the CLN library; please make sure either to install ' + 'the library and development headers system-wide, or include the ' + 'path tt the library in the LD_LIBRARY_PATH environment variable' ) cln_lib_dir = os.path.dirname(cln_lib) cln_build_dir = os.path.dirname(cln_lib_dir) From bd3299d4e50ac307947a055a2723ea85bdc6c534 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 14:40:15 -0600 Subject: [PATCH 54/99] Register the GiNaC interface builder with the ExtensionBuilder --- pyomo/contrib/simplification/build.py | 8 ++++++++ pyomo/contrib/simplification/plugins.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 pyomo/contrib/simplification/plugins.py diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 508acb2d5a1..4952ac6dade 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -109,5 +109,13 @@ def run(self): dist.run_command('build_ext') +class GiNaCInterfaceBuilder(object): + def __call__(self, parallel): + return build_ginac_interface() + + def skip(self): + return not find_library('ginac') + + if __name__ == '__main__': build_ginac_interface(sys.argv[1:]) diff --git a/pyomo/contrib/simplification/plugins.py b/pyomo/contrib/simplification/plugins.py new file mode 100644 index 00000000000..6b08f7be4d7 --- /dev/null +++ b/pyomo/contrib/simplification/plugins.py @@ -0,0 +1,17 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.extensions import ExtensionBuilderFactory +from .build import GiNaCInterfaceBuilder + + +def load(): + ExtensionBuilderFactory.register('ginac')(GiNaCInterfaceBuilder) From 29b207246e4ca83f26c96fec1989b6cea24b79e9 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 14:41:22 -0600 Subject: [PATCH 55/99] Rework GiNaC interface builder (in development - testing several things) - attempt to install from OS package repo - attempt local build ginac, add the built library to the download cache --- .github/workflows/test_branches.yml | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 3bfb902b9e0..a23a603430c 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -157,24 +157,6 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - - name: install GiNaC - if: matrix.other == '/singletest' - run: | - cd .. - curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 - tar -xvf cln-1.3.7.tar.bz2 - cd cln-1.3.7 - ./configure - make -j 2 - sudo make install - cd .. - curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 - tar -xvf ginac-1.8.7.tar.bz2 - cd ginac-1.8.7 - ./configure - make -j 2 - sudo make install - - name: TPL package download cache uses: actions/cache@v4 if: ${{ ! matrix.slim }} @@ -206,7 +188,7 @@ jobs: # Notes: # - install glpk # - pyodbc needs: gcc pkg-config unixodbc freetds - for pkg in bash pkg-config unixodbc freetds glpk; do + for pkg in bash pkg-config unixodbc freetds glpk ginac; do brew list $pkg || brew install $pkg done @@ -218,7 +200,7 @@ jobs: # - install glpk # - ipopt needs: libopenblas-dev gfortran liblapack-dev sudo apt-get -o Dir::Cache=${GITHUB_WORKSPACE}/cache/os \ - install libopenblas-dev gfortran liblapack-dev glpk-utils + install libopenblas-dev gfortran liblapack-dev glpk-utils libginac-dev sudo chmod -R 777 ${GITHUB_WORKSPACE}/cache/os - name: Update Windows @@ -581,6 +563,32 @@ jobs: echo "$GJH_DIR" ls -l $GJH_DIR + - name: Install GiNaC + if: ${{ ! matrix.slim }} + run: | + if test ! -e "${DOWNLOAD_DIR}/ginac.tar.gz"; then + mkdir -p "${GITHUB_WORKSPACE}/cache/build/ginac" + cd "${GITHUB_WORKSPACE}/cache/build/ginac" + curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 + tar -xvf cln-1.3.7.tar.bz2 + cd cln-1.3.7 + ./configure --prefix "$TPL_DIR/ginac" --disable-static + make -j 4 + make install + cd "${GITHUB_WORKSPACE}/cache/build/ginac" + curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 + tar -xvf ginac-1.8.7.tar.bz2 + cd ginac-1.8.7 + ./configure --prefix "$TPL_DIR/ginac" --disable-static + make -j 4 + make install + cd "$TPL_DIR" + tar -czf "${DOWNLOAD_DIR}/ginac.tar.gz" ginac + else + cd "$TPL_DIR" + tar -xzf "${DOWNLOAD_DIR}/ginac.tar.gz" + fi + - name: Install Pyomo run: | echo "" @@ -635,14 +643,6 @@ jobs: echo "" pyomo build-extensions --parallel 2 - - name: Install GiNaC Interface - if: matrix.other == '/singletest' - run: | - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - cd pyomo/contrib/simplification/ - $PYTHON_EXE build.py --inplace - - name: Report pyomo plugin information run: | echo "$PATH" From 1b1f944dbd00ca0e0b27d2fd2b70ee681b8e44ae Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 29 Apr 2024 16:58:04 -0600 Subject: [PATCH 56/99] bugfix --- pyomo/contrib/simplification/simplify.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 5e251ca326a..27da5f5ca34 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -11,7 +11,7 @@ from pyomo.core.expr.sympy_tools import sympy2pyomo_expression, sympyify_expression from pyomo.core.expr.numeric_expr import NumericExpression -from pyomo.core.expr.numvalue import is_fixed, value +from pyomo.core.expr.numvalue import value, is_constant import logging import warnings @@ -28,15 +28,19 @@ def simplify_with_sympy(expr: NumericExpression): + if is_constant(expr): + return value(expr) om, se = sympyify_expression(expr) se = se.simplify() new_expr = sympy2pyomo_expression(se, om) - if is_fixed(new_expr): + if is_constant(new_expr): new_expr = value(new_expr) return new_expr def simplify_with_ginac(expr: NumericExpression, ginac_interface): + if is_constant(expr): + return value(expr) gi = ginac_interface ginac_expr = gi.to_ginac(expr) ginac_expr = ginac_expr.normal() From 74052eebd65a15cf03f36ade73846ec9fcf048e5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 21:58:32 -0600 Subject: [PATCH 57/99] Disable local build of GiNaC --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 5410c58495f..d1b6b96807b 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -565,7 +565,7 @@ jobs: ls -l $GJH_DIR - name: Install GiNaC - if: ${{ ! matrix.slim }} + if: ${{ 0 && ! matrix.slim }} run: | if test ! -e "${DOWNLOAD_DIR}/ginac.tar.gz"; then mkdir -p "${GITHUB_WORKSPACE}/cache/build/ginac" From 6bb0f7f375e7a10d4daa05841c4dd5544eff99ff Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 29 Apr 2024 21:59:34 -0600 Subject: [PATCH 58/99] NFC: apply black --- pyomo/common/fileutils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pyomo/common/fileutils.py b/pyomo/common/fileutils.py index 80fd47a6377..7b6520327a0 100644 --- a/pyomo/common/fileutils.py +++ b/pyomo/common/fileutils.py @@ -384,11 +384,13 @@ def find_library(libname, cwd=True, include_PATH=True, pathlist=None): # where python does not return the absolute path on *nix try: libname = lib + ' ' - with subprocess.Popen(['/sbin/ldconfig', '-p'], - stdin=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - env={'LC_ALL': 'C', 'LANG': 'C'}) as p: + with subprocess.Popen( + ['/sbin/ldconfig', '-p'], + stdin=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + env={'LC_ALL': 'C', 'LANG': 'C'}, + ) as p: for line in os.fsdecode(p.stdout.read()).splitlines(): if line.lstrip().startswith(libname): return os.path.realpath(line.split()[-1]) From 39643b336ef3149fe7f57007ffaf31b684b6151d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:05:00 -0600 Subject: [PATCH 59/99] remove repeated code --- pyomo/contrib/appsi/build.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pyomo/contrib/appsi/build.py b/pyomo/contrib/appsi/build.py index b3d78467f01..38f8cb713ca 100644 --- a/pyomo/contrib/appsi/build.py +++ b/pyomo/contrib/appsi/build.py @@ -16,15 +16,6 @@ import tempfile -def handleReadonly(function, path, excinfo): - excvalue = excinfo[1] - if excvalue.errno == errno.EACCES: - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 - function(path) - else: - raise - - def get_appsi_extension(in_setup=False, appsi_root=None): from pybind11.setup_helpers import Pybind11Extension @@ -66,6 +57,7 @@ def build_appsi(args=[]): from setuptools import Distribution from pybind11.setup_helpers import build_ext import pybind11.setup_helpers + from pyomo.common.cmake_builder import handleReadonly from pyomo.common.envvar import PYOMO_CONFIG_DIR from pyomo.common.fileutils import this_file_dir From 89ace9bed0c1a0366ac14dca0d627df0a89b1f01 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:18:31 -0600 Subject: [PATCH 60/99] Add support for download tar archives to theFileDownloader --- pyomo/common/download.py | 48 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/pyomo/common/download.py b/pyomo/common/download.py index 5332287cfc7..95713e9ef76 100644 --- a/pyomo/common/download.py +++ b/pyomo/common/download.py @@ -29,6 +29,7 @@ urllib_error = attempt_import('urllib.error')[0] ssl = attempt_import('ssl')[0] zipfile = attempt_import('zipfile')[0] +tarfile = attempt_import('tarfile')[0] gzip = attempt_import('gzip')[0] distro, distro_available = attempt_import('distro') @@ -371,7 +372,7 @@ def get_zip_archive(self, url, dirOffset=0): # Simple sanity checks for info in zip_file.infolist(): f = info.filename - if f[0] in '\\/' or '..' in f: + if f[0] in '\\/' or '..' in f or os.path.isabs(f): logger.error( "malformed (potentially insecure) filename (%s) " "found in zip archive. Skipping file." % (f,) @@ -387,6 +388,51 @@ def get_zip_archive(self, url, dirOffset=0): info.filename = target[-1] + '/' if f[-1] == '/' else target[-1] zip_file.extract(f, os.path.join(self._fname, *tuple(target[dirOffset:-1]))) + def get_tar_archive(self, url, dirOffset=0): + if self._fname is None: + raise DeveloperError( + "target file name has not been initialized " + "with set_destination_filename" + ) + if os.path.exists(self._fname) and not os.path.isdir(self._fname): + raise RuntimeError( + "Target directory (%s) exists, but is not a directory" % (self._fname,) + ) + tar_file = tarfile.open(fileobj=io.BytesIO(self.retrieve_url(url))) + dest = os.path.realpath(self._fname) + + def filter_fcn(info): + # this mocks up the `tarfile` filter introduced in Python + # 3.12 and backported to later releases of Python (e.g., + # 3.8.17, 3.9.17, 3.10.12, and 3.11.4) + f = info.name + if os.path.isabs(f) or '..' in f or f.startswith(('/', os.sep)): + logger.error( + "malformed (potentially insecure) filename (%s) " + "found in tar archive. Skipping file." % (f,) + ) + return False + target = os.path.realpath(os.path.join(dest, f)) + if os.path.commonpath([target, dest]) != dest: + logger.error( + "malformed (potentially insecure) filename (%s) " + "found in zip archive. Skipping file." % (f,) + ) + return False + target = self._splitpath(f) + if len(target) <= dirOffset: + if not info.isdir(): + logger.warning( + "Skipping file (%s) in zip archive due to dirOffset" % (f,) + ) + return False + info.name = '/'.join(target[dirOffset:]) + # Strip high bits & group/other write bits + info.mode &= 0o755 + return True + + tar_file.extractall(dest, filter(filter_fcn, tar_file.getmembers())) + def get_gzipped_binary_file(self, url): if self._fname is None: raise DeveloperError( From bb274ff9d30960f72ffbbf001850d64a14caa6f6 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:20:08 -0600 Subject: [PATCH 61/99] Switch GiNaC interface builder to use TempfileManager --- pyomo/contrib/simplification/build.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 4952ac6dade..d30f582bcea 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -71,13 +71,13 @@ def build_ginac_interface(args=None): class ginacBuildExt(build_ext): def run(self): basedir = os.path.abspath(os.path.curdir) - if self.inplace: - tmpdir = this_file_dir() - else: - tmpdir = os.path.abspath(tempfile.mkdtemp()) - print("Building in '%s'" % tmpdir) - os.chdir(tmpdir) - try: + with TempfileManager.new_context() as tempfile: + if self.inplace: + tmpdir = this_file_dir() + else: + tmpdir = os.path.abspath(tempfile.mkdtemp()) + print("Building in '%s'" % tmpdir) + os.chdir(tmpdir) super(ginacBuildExt, self).run() if not self.inplace: library = glob.glob("build/*/ginac_interface.*")[0] @@ -91,10 +91,6 @@ def run(self): if not os.path.exists(target): os.makedirs(target) shutil.copy(library, target) - finally: - os.chdir(basedir) - if not self.inplace: - shutil.rmtree(tmpdir, onerror=handleReadonly) package_config = { 'name': 'ginac_interface', From adaefbca72f94548b33328b7a28c2e03a4012a9d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:20:39 -0600 Subject: [PATCH 62/99] Add function for downloading and installing GiNaC and CLN --- pyomo/contrib/simplification/build.py | 83 ++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index d30f582bcea..3ea7748cdbf 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -10,19 +10,71 @@ # ___________________________________________________________________________ import glob +import logging import os import shutil import sys -import tempfile -from distutils.dist import Distribution +import subprocess -from pybind11.setup_helpers import Pybind11Extension, build_ext -from pyomo.common.cmake_builder import handleReadonly +from pyomo.common.download import FileDownloader from pyomo.common.envvar import PYOMO_CONFIG_DIR from pyomo.common.fileutils import find_library, this_file_dir +from pyomo.common.tempfiles import TempfileManager -def build_ginac_interface(args=None): +logger = logging.getLogger(__name__) + + +def build_ginac_library(parallel=None, argv=None): + print("\n**** Building GiNaC library ****") + + configure_cmd = ['configure', '--prefix=' + PYOMO_CONFIG_DIR, '--disable-static'] + make_cmd = ['make'] + if parallel: + make_cmd.append(f'-j{parallel}') + install_cmd = ['make', 'install'] + + with TempfileManager.new_context() as tempfile: + tmpdir = tempfile.mkdtemp() + + downloader = FileDownloader() + if argv: + downloader.parse_args(argv) + + url = 'https://www.ginac.de/CLN/cln-1.3.7.tar.bz2' + cln_dir = os.path.join(tmpdir, 'cln') + downloader.set_destination_filename(cln_dir) + logger.info( + "Fetching CLN from %s and installing it to %s" + % (url, downloader.destination()) + ) + downloader.get_tar_archive(url, dirOffset=1) + assert subprocess.run(configure_cmd, cwd=cln_dir).returncode == 0 + logger.info("\nBuilding CLN\n") + assert subprocess.run(make_cmd, cwd=cln_dir).returncode == 0 + assert subprocess.run(install_cmd, cwd=cln_dir).returncode == 0 + + url = 'https://www.ginac.de/ginac-1.8.7.tar.bz2' + ginac_dir = os.path.join(tmpdir, 'ginac') + downloader.set_destination_filename(ginac_dir) + logger.info( + "Fetching GiNaC from %s and installing it to %s" + % (url, downloader.destination()) + ) + downloader.get_tar_archive(url, dirOffset=1) + assert subprocess.run(configure_cmd, cwd=ginac_dir).returncode == 0 + logger.info("\nBuilding GiNaC\n") + assert subprocess.run(make_cmd, cwd=ginac_dir).returncode == 0 + assert subprocess.run(install_cmd, cwd=ginac_dir).returncode == 0 + + +def build_ginac_interface(parallel=None, args=None): + from distutils.dist import Distribution + from pybind11.setup_helpers import Pybind11Extension, build_ext + from pyomo.common.cmake_builder import handleReadonly + + print("\n**** Building GiNaC interface ****") + if args is None: args = list() dname = this_file_dir() @@ -107,11 +159,28 @@ def run(self): class GiNaCInterfaceBuilder(object): def __call__(self, parallel): - return build_ginac_interface() + return build_ginac_interface(parallel) def skip(self): return not find_library('ginac') if __name__ == '__main__': - build_ginac_interface(sys.argv[1:]) + logging.getLogger('pyomo').setLevel(logging.DEBUG) + parallel = None + for i, arg in enumerate(sys.argv): + if arg == '-j': + parallel = int(sys.argv.pop(i + 1)) + sys.argv.pop(i) + break + if arg.startswith('-j'): + if '=' in arg: + parallel = int(arg.split('=')[1]) + else: + parallel = int(arg[2:]) + sys.argv.pop(i) + break + if '--build-deps' in sys.argv: + sys.argv.remove('--build-deps') + build_ginac_library(parallel, []) + build_ginac_interface(parallel, sys.argv[1:]) From c7a8f8e243c2a51bf5e74803b1fc118fd4060816 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:21:10 -0600 Subject: [PATCH 63/99] Hook GiNaC builder into pyomo command --- pyomo/environ/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/environ/__init__.py b/pyomo/environ/__init__.py index c1ceb8cb890..07b3dfad680 100644 --- a/pyomo/environ/__init__.py +++ b/pyomo/environ/__init__.py @@ -50,6 +50,7 @@ def _do_import(pkg_name): 'pyomo.contrib.multistart', 'pyomo.contrib.preprocessing', 'pyomo.contrib.pynumero', + 'pyomo.contrib.simplification', 'pyomo.contrib.solver', 'pyomo.contrib.trustregion', ] From a29bb3ed8149dc3b1221ce858cce5640716e5c14 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:23:53 -0600 Subject: [PATCH 64/99] Remove simplification test marker --- .github/workflows/test_branches.yml | 5 ----- pyomo/contrib/simplification/tests/test_simplification.py | 1 - setup.cfg | 1 - 3 files changed, 7 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index d1b6b96807b..558d6dc2591 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -651,11 +651,6 @@ jobs: pyomo help --transformations || exit 1 pyomo help --writers || exit 1 - - name: Run Simplification Tests - if: matrix.other == '/singletest' - run: | - pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" - - name: Run Pyomo tests if: matrix.mpi == 0 run: | diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 1a5ae1e0036..be61631e9f3 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -106,7 +106,6 @@ class TestSimplificationSympy(TestCase, SimplificationMixin): @unittest.skipIf(not ginac_available, 'GiNaC is not available') -@unittest.pytest.mark.simplification class TestSimplificationGiNaC(TestCase, SimplificationMixin): def test_param(self): m = pe.ConcreteModel() diff --git a/setup.cfg b/setup.cfg index 855717490b3..b606138f38c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,4 +22,3 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests - simplification: tests for expression simplification that have expensive (to install) dependencies From c475fe791ff6c60aa75ae8eb87a5cabb8d8786ab Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:33:38 -0600 Subject: [PATCH 65/99] Switching output to sys.stdout, adding debugging --- pyomo/contrib/simplification/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 3ea7748cdbf..5e613cf873f 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -26,7 +26,7 @@ def build_ginac_library(parallel=None, argv=None): - print("\n**** Building GiNaC library ****") + sys.stdout.write("\n**** Building GiNaC library ****") configure_cmd = ['configure', '--prefix=' + PYOMO_CONFIG_DIR, '--disable-static'] make_cmd = ['make'] @@ -73,7 +73,7 @@ def build_ginac_interface(parallel=None, args=None): from pybind11.setup_helpers import Pybind11Extension, build_ext from pyomo.common.cmake_builder import handleReadonly - print("\n**** Building GiNaC interface ****") + sys.stdout.write("\n**** Building GiNaC interface ****") if args is None: args = list() @@ -90,6 +90,7 @@ def build_ginac_interface(parallel=None, args=None): 'the library and development headers system-wide, or include the ' 'path tt the library in the LD_LIBRARY_PATH environment variable' ) + print("Found GiNaC library:", ginac_lib) ginac_lib_dir = os.path.dirname(ginac_lib) ginac_build_dir = os.path.dirname(ginac_lib_dir) ginac_include_dir = os.path.join(ginac_build_dir, 'include') @@ -103,6 +104,7 @@ def build_ginac_interface(parallel=None, args=None): 'the library and development headers system-wide, or include the ' 'path tt the library in the LD_LIBRARY_PATH environment variable' ) + print("Found CLN library:", cln_lib) cln_lib_dir = os.path.dirname(cln_lib) cln_build_dir = os.path.dirname(cln_lib_dir) cln_include_dir = os.path.join(cln_build_dir, 'include') @@ -128,7 +130,7 @@ def run(self): tmpdir = this_file_dir() else: tmpdir = os.path.abspath(tempfile.mkdtemp()) - print("Building in '%s'" % tmpdir) + sys.stdout.write("Building in '%s'" % tmpdir) os.chdir(tmpdir) super(ginacBuildExt, self).run() if not self.inplace: From 7053690e90b0a693ec8901d9b4c73279a85a41aa Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 00:45:14 -0600 Subject: [PATCH 66/99] Support walking up the directory tree looking for ginac headers (this should better support debian system installations) --- pyomo/contrib/simplification/build.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 5e613cf873f..e6f300ae058 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -68,6 +68,16 @@ def build_ginac_library(parallel=None, argv=None): assert subprocess.run(install_cmd, cwd=ginac_dir).returncode == 0 +def _find_include(libdir, incpaths): + while 1: + basedir = os.path.dirname(libdir) + if not basedir or basedir == libdir: + return None + if os.path.exists(os.path.join(basedir, *incpaths)): + return os.path.join(basedir, *(incpaths[:-1]))): + libdir = basedir + + def build_ginac_interface(parallel=None, args=None): from distutils.dist import Distribution from pybind11.setup_helpers import Pybind11Extension, build_ext @@ -90,11 +100,9 @@ def build_ginac_interface(parallel=None, args=None): 'the library and development headers system-wide, or include the ' 'path tt the library in the LD_LIBRARY_PATH environment variable' ) - print("Found GiNaC library:", ginac_lib) ginac_lib_dir = os.path.dirname(ginac_lib) - ginac_build_dir = os.path.dirname(ginac_lib_dir) - ginac_include_dir = os.path.join(ginac_build_dir, 'include') - if not os.path.exists(os.path.join(ginac_include_dir, 'ginac', 'ginac.h')): + ginac_include_dir = _find_include(ginac_lib_dir, ('ginac', 'ginac.h')) + if not ginac_include_dir: raise RuntimeError('could not find GiNaC include directory') cln_lib = find_library('cln') @@ -104,11 +112,9 @@ def build_ginac_interface(parallel=None, args=None): 'the library and development headers system-wide, or include the ' 'path tt the library in the LD_LIBRARY_PATH environment variable' ) - print("Found CLN library:", cln_lib) cln_lib_dir = os.path.dirname(cln_lib) - cln_build_dir = os.path.dirname(cln_lib_dir) - cln_include_dir = os.path.join(cln_build_dir, 'include') - if not os.path.exists(os.path.join(cln_include_dir, 'cln', 'cln.h')): + cln_include_dir = _find_include(cln_lib_dir, ('cln', 'cln.h')) + if cln_include_dir: raise RuntimeError('could not find CLN include directory') extra_args = ['-std=c++11'] From be6922453be5ef7da898235b122d251d1222ad04 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 07:31:44 -0600 Subject: [PATCH 67/99] Fix several typos / include search logic --- pyomo/contrib/simplification/build.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index e6f300ae058..fb571c273eb 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -69,12 +69,13 @@ def build_ginac_library(parallel=None, argv=None): def _find_include(libdir, incpaths): + rel_path = ('include',) + incpaths while 1: basedir = os.path.dirname(libdir) if not basedir or basedir == libdir: return None - if os.path.exists(os.path.join(basedir, *incpaths)): - return os.path.join(basedir, *(incpaths[:-1]))): + if os.path.exists(os.path.join(basedir, *rel_path)): + return os.path.join(basedir, *(rel_path[:-len(incpaths)])) libdir = basedir @@ -86,15 +87,13 @@ def build_ginac_interface(parallel=None, args=None): sys.stdout.write("\n**** Building GiNaC interface ****") if args is None: - args = list() + args = [] dname = this_file_dir() _sources = ['ginac_interface.cpp'] - sources = list() - for fname in _sources: - sources.append(os.path.join(dname, fname)) + sources = [os.path.join(dname, fname) for fname in _sources] ginac_lib = find_library('ginac') - if ginac_lib is None: + if not ginac_lib: raise RuntimeError( 'could not find the GiNaC library; please make sure either to install ' 'the library and development headers system-wide, or include the ' @@ -106,7 +105,7 @@ def build_ginac_interface(parallel=None, args=None): raise RuntimeError('could not find GiNaC include directory') cln_lib = find_library('cln') - if cln_lib is None: + if not cln_lib: raise RuntimeError( 'could not find the CLN library; please make sure either to install ' 'the library and development headers system-wide, or include the ' @@ -114,7 +113,7 @@ def build_ginac_interface(parallel=None, args=None): ) cln_lib_dir = os.path.dirname(cln_lib) cln_include_dir = _find_include(cln_lib_dir, ('cln', 'cln.h')) - if cln_include_dir: + if not cln_include_dir: raise RuntimeError('could not find CLN include directory') extra_args = ['-std=c++11'] From 4ac04a6f3e8c4511f922d00c5eb7430bb8ee9b2a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 07:33:23 -0600 Subject: [PATCH 68/99] NFC: apply black --- pyomo/contrib/simplification/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index fb571c273eb..a4094f993fa 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -75,7 +75,7 @@ def _find_include(libdir, incpaths): if not basedir or basedir == libdir: return None if os.path.exists(os.path.join(basedir, *rel_path)): - return os.path.join(basedir, *(rel_path[:-len(incpaths)])) + return os.path.join(basedir, *(rel_path[: -len(incpaths)])) libdir = basedir From abe5f8bb136bd1faadbcc4340deabfa42061d566 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 30 Apr 2024 07:47:07 -0600 Subject: [PATCH 69/99] Resync GHA workflows, remove ginac build code --- .github/workflows/test_branches.yml | 31 +++------------------ .github/workflows/test_pr_and_main.yml | 37 +++----------------------- 2 files changed, 7 insertions(+), 61 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 558d6dc2591..de40066b50f 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -201,7 +201,8 @@ jobs: # - install glpk # - ipopt needs: libopenblas-dev gfortran liblapack-dev sudo apt-get -o Dir::Cache=${GITHUB_WORKSPACE}/cache/os \ - install libopenblas-dev gfortran liblapack-dev glpk-utils libginac-dev + install libopenblas-dev gfortran liblapack-dev glpk-utils \ + libginac-dev sudo chmod -R 777 ${GITHUB_WORKSPACE}/cache/os - name: Update Windows @@ -346,7 +347,7 @@ jobs: echo "*** Install Pyomo dependencies ***" # Note: this will fail the build if any installation fails (or # possibly if it outputs messages to stderr) - conda install --update-deps -y $CONDA_DEPENDENCIES + conda install --update-deps -q -y $CONDA_DEPENDENCIES if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" @@ -564,32 +565,6 @@ jobs: echo "$GJH_DIR" ls -l $GJH_DIR - - name: Install GiNaC - if: ${{ 0 && ! matrix.slim }} - run: | - if test ! -e "${DOWNLOAD_DIR}/ginac.tar.gz"; then - mkdir -p "${GITHUB_WORKSPACE}/cache/build/ginac" - cd "${GITHUB_WORKSPACE}/cache/build/ginac" - curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 - tar -xvf cln-1.3.7.tar.bz2 - cd cln-1.3.7 - ./configure --prefix "$TPL_DIR/ginac" --disable-static - make -j 4 - make install - cd "${GITHUB_WORKSPACE}/cache/build/ginac" - curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 - tar -xvf ginac-1.8.7.tar.bz2 - cd ginac-1.8.7 - ./configure --prefix "$TPL_DIR/ginac" --disable-static - make -j 4 - make install - cd "$TPL_DIR" - tar -czf "${DOWNLOAD_DIR}/ginac.tar.gz" ginac - else - cd "$TPL_DIR" - tar -xzf "${DOWNLOAD_DIR}/ginac.tar.gz" - fi - - name: Install Pyomo run: | echo "" diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 7c84ed14093..cdc42718cba 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -187,24 +187,6 @@ jobs: # path: cache/os # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - - name: install GiNaC - if: matrix.other == '/singletest' - run: | - cd .. - curl https://www.ginac.de/CLN/cln-1.3.7.tar.bz2 >cln-1.3.7.tar.bz2 - tar -xvf cln-1.3.7.tar.bz2 - cd cln-1.3.7 - ./configure - make -j 2 - sudo make install - cd .. - curl https://www.ginac.de/ginac-1.8.7.tar.bz2 >ginac-1.8.7.tar.bz2 - tar -xvf ginac-1.8.7.tar.bz2 - cd ginac-1.8.7 - ./configure - make -j 2 - sudo make install - - name: TPL package download cache uses: actions/cache@v4 if: ${{ ! matrix.slim }} @@ -236,7 +218,7 @@ jobs: # Notes: # - install glpk # - pyodbc needs: gcc pkg-config unixodbc freetds - for pkg in bash pkg-config unixodbc freetds glpk; do + for pkg in bash pkg-config unixodbc freetds glpk ginac; do brew list $pkg || brew install $pkg done @@ -248,7 +230,8 @@ jobs: # - install glpk # - ipopt needs: libopenblas-dev gfortran liblapack-dev sudo apt-get -o Dir::Cache=${GITHUB_WORKSPACE}/cache/os \ - install libopenblas-dev gfortran liblapack-dev glpk-utils + install libopenblas-dev gfortran liblapack-dev glpk-utils \ + libginac-dev sudo chmod -R 777 ${GITHUB_WORKSPACE}/cache/os - name: Update Windows @@ -389,6 +372,7 @@ jobs: CONDA_DEPENDENCIES="$CONDA_DEPENDENCIES $PKG" fi done + echo "" echo "*** Install Pyomo dependencies ***" # Note: this will fail the build if any installation fails (or # possibly if it outputs messages to stderr) @@ -664,14 +648,6 @@ jobs: echo "" pyomo build-extensions --parallel 2 - - name: Install GiNaC Interface - if: matrix.other == '/singletest' - run: | - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - cd pyomo/contrib/simplification/ - $PYTHON_EXE build.py --inplace - - name: Report pyomo plugin information run: | echo "$PATH" @@ -679,11 +655,6 @@ jobs: pyomo help --transformations || exit 1 pyomo help --writers || exit 1 - - name: Run Simplification Tests - if: matrix.other == '/singletest' - run: | - pytest -v -m 'simplification' pyomo/contrib/simplification/tests/test_simplification.py --junitxml="TEST-pyomo-simplify.xml" - - name: Run Pyomo tests if: matrix.mpi == 0 run: | From 54bd3d393b73d1dcfa6bdec1854b69893eaec679 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 10:08:04 -0600 Subject: [PATCH 70/99] Move the ginac interface sources to a subdirectory --- pyomo/contrib/simplification/build.py | 9 ++-- .../contrib/simplification/ginac/__init__.py | 52 +++++++++++++++++++ .../{ => ginac/src}/ginac_interface.cpp | 0 .../{ => ginac/src}/ginac_interface.hpp | 0 pyomo/contrib/simplification/simplify.py | 14 ++--- 5 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 pyomo/contrib/simplification/ginac/__init__.py rename pyomo/contrib/simplification/{ => ginac/src}/ginac_interface.cpp (100%) rename pyomo/contrib/simplification/{ => ginac/src}/ginac_interface.hpp (100%) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index a4094f993fa..0ae883bc55c 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -88,9 +88,10 @@ def build_ginac_interface(parallel=None, args=None): if args is None: args = [] - dname = this_file_dir() - _sources = ['ginac_interface.cpp'] - sources = [os.path.join(dname, fname) for fname in _sources] + sources = [ + os.path.join(this_file_dir(), 'ginac', 'src', fname) + for fname in ['ginac_interface.cpp'] + ] ginac_lib = find_library('ginac') if not ginac_lib: @@ -132,7 +133,7 @@ def run(self): basedir = os.path.abspath(os.path.curdir) with TempfileManager.new_context() as tempfile: if self.inplace: - tmpdir = this_file_dir() + tmpdir = os.path.join(this_file_dir(), 'ginac') else: tmpdir = os.path.abspath(tempfile.mkdtemp()) sys.stdout.write("Building in '%s'" % tmpdir) diff --git a/pyomo/contrib/simplification/ginac/__init__.py b/pyomo/contrib/simplification/ginac/__init__.py new file mode 100644 index 00000000000..af6511944de --- /dev/null +++ b/pyomo/contrib/simplification/ginac/__init__.py @@ -0,0 +1,52 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.dependencies import attempt_import as _attempt_import + + +def _importer(): + import os + import sys + from ctypes import cdll + from pyomo.common.envvar import PYOMO_CONFIG_DIR + from pyomo.common.fileutils import find_library + + try: + pyomo_config_dir = os.path.join( + PYOMO_CONFIG_DIR, + 'lib', + 'python%s.%s' % sys.version_info[:2], + 'site-packages', + ) + sys.path.insert(0, pyomo_config_dir) + # GiNaC needs 2 libraries that are generally dynamically linked + # to the interface library. If we built those ourselves, then + # the libraries will be PYOMO_CONFIG_DIR/lib ... but that + # directlor is very likely to NOT be on the library search path + # when the Python interpreter was started. We will manually + # look for those two libraries, and if we find them, load them + # into this process (so the interface can find them) + for lib in ('cln', 'ginac'): + fname = find_library(lib) + if fname is not None: + cdll.LoadLibrary(fname) + + import ginac_interface + except ImportError: + from . import ginac_interface + finally: + assert sys.path[0] == pyomo_config_dir + sys.path.pop(0) + + return ginac_interface + + +interface, interface_available = _attempt_import('ginac_interface', importer=_importer) diff --git a/pyomo/contrib/simplification/ginac_interface.cpp b/pyomo/contrib/simplification/ginac/src/ginac_interface.cpp similarity index 100% rename from pyomo/contrib/simplification/ginac_interface.cpp rename to pyomo/contrib/simplification/ginac/src/ginac_interface.cpp diff --git a/pyomo/contrib/simplification/ginac_interface.hpp b/pyomo/contrib/simplification/ginac/src/ginac_interface.hpp similarity index 100% rename from pyomo/contrib/simplification/ginac_interface.hpp rename to pyomo/contrib/simplification/ginac/src/ginac_interface.hpp diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 27da5f5ca34..94f0ceaa33f 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -15,14 +15,10 @@ import logging import warnings -try: - from pyomo.contrib.simplification.ginac_interface import GinacInterface - - ginac_available = True -except: - GinacInterface = None - ginac_available = False - +from pyomo.contrib.simplification.ginac import ( + interface as ginac_interface, + interface_available as ginac_available, +) logger = logging.getLogger(__name__) @@ -51,7 +47,7 @@ def simplify_with_ginac(expr: NumericExpression, ginac_interface): class Simplifier(object): def __init__(self, suppress_no_ginac_warnings: bool = False) -> None: if ginac_available: - self.gi = GinacInterface(False) + self.gi = ginac_interface.GinacInterface(False) self.suppress_no_ginac_warnings = suppress_no_ginac_warnings def simplify(self, expr: NumericExpression): From 8a37c8b1e0a4a54702eb81996d5730a4e55c0da5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 10:08:50 -0600 Subject: [PATCH 71/99] Update builder to use argparse, clean up output --- pyomo/contrib/simplification/build.py | 73 +++++++++++++++------------ 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 0ae883bc55c..d540991b010 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -21,12 +21,11 @@ from pyomo.common.fileutils import find_library, this_file_dir from pyomo.common.tempfiles import TempfileManager +logger = logging.getLogger(__name__ if __name__ != '__main__' else 'pyomo') -logger = logging.getLogger(__name__) - -def build_ginac_library(parallel=None, argv=None): - sys.stdout.write("\n**** Building GiNaC library ****") +def build_ginac_library(parallel=None, argv=None, env=None): + sys.stdout.write("\n**** Building GiNaC library ****\n") configure_cmd = ['configure', '--prefix=' + PYOMO_CONFIG_DIR, '--disable-static'] make_cmd = ['make'] @@ -34,6 +33,12 @@ def build_ginac_library(parallel=None, argv=None): make_cmd.append(f'-j{parallel}') install_cmd = ['make', 'install'] + env = dict(os.environ) + pcdir = os.path.join(PYOMO_CONFIG_DIR, 'lib', 'pkgconfig') + if 'PKG_CONFIG_PATH' in env: + pcdir += os.pathsep + env['PKG_CONFIG_PATH'] + env['PKG_CONFIG_PATH'] = pcdir + with TempfileManager.new_context() as tempfile: tmpdir = tempfile.mkdtemp() @@ -49,10 +54,10 @@ def build_ginac_library(parallel=None, argv=None): % (url, downloader.destination()) ) downloader.get_tar_archive(url, dirOffset=1) - assert subprocess.run(configure_cmd, cwd=cln_dir).returncode == 0 + assert subprocess.run(configure_cmd, cwd=cln_dir, env=env).returncode == 0 logger.info("\nBuilding CLN\n") - assert subprocess.run(make_cmd, cwd=cln_dir).returncode == 0 - assert subprocess.run(install_cmd, cwd=cln_dir).returncode == 0 + assert subprocess.run(make_cmd, cwd=cln_dir, env=env).returncode == 0 + assert subprocess.run(install_cmd, cwd=cln_dir, env=env).returncode == 0 url = 'https://www.ginac.de/ginac-1.8.7.tar.bz2' ginac_dir = os.path.join(tmpdir, 'ginac') @@ -62,10 +67,10 @@ def build_ginac_library(parallel=None, argv=None): % (url, downloader.destination()) ) downloader.get_tar_archive(url, dirOffset=1) - assert subprocess.run(configure_cmd, cwd=ginac_dir).returncode == 0 + assert subprocess.run(configure_cmd, cwd=ginac_dir, env=env).returncode == 0 logger.info("\nBuilding GiNaC\n") - assert subprocess.run(make_cmd, cwd=ginac_dir).returncode == 0 - assert subprocess.run(install_cmd, cwd=ginac_dir).returncode == 0 + assert subprocess.run(make_cmd, cwd=ginac_dir, env=env).returncode == 0 + assert subprocess.run(install_cmd, cwd=ginac_dir, env=env).returncode == 0 def _find_include(libdir, incpaths): @@ -84,7 +89,7 @@ def build_ginac_interface(parallel=None, args=None): from pybind11.setup_helpers import Pybind11Extension, build_ext from pyomo.common.cmake_builder import handleReadonly - sys.stdout.write("\n**** Building GiNaC interface ****") + sys.stdout.write("\n**** Building GiNaC interface ****\n") if args is None: args = [] @@ -98,7 +103,7 @@ def build_ginac_interface(parallel=None, args=None): raise RuntimeError( 'could not find the GiNaC library; please make sure either to install ' 'the library and development headers system-wide, or include the ' - 'path tt the library in the LD_LIBRARY_PATH environment variable' + 'path to the library in the LD_LIBRARY_PATH environment variable' ) ginac_lib_dir = os.path.dirname(ginac_lib) ginac_include_dir = _find_include(ginac_lib_dir, ('ginac', 'ginac.h')) @@ -110,7 +115,7 @@ def build_ginac_interface(parallel=None, args=None): raise RuntimeError( 'could not find the CLN library; please make sure either to install ' 'the library and development headers system-wide, or include the ' - 'path tt the library in the LD_LIBRARY_PATH environment variable' + 'path to the library in the LD_LIBRARY_PATH environment variable' ) cln_lib_dir = os.path.dirname(cln_lib) cln_include_dir = _find_include(cln_lib_dir, ('cln', 'cln.h')) @@ -136,7 +141,7 @@ def run(self): tmpdir = os.path.join(this_file_dir(), 'ginac') else: tmpdir = os.path.abspath(tempfile.mkdtemp()) - sys.stdout.write("Building in '%s'" % tmpdir) + sys.stdout.write("Building in '%s'\n" % tmpdir) os.chdir(tmpdir) super(ginacBuildExt, self).run() if not self.inplace: @@ -174,21 +179,25 @@ def skip(self): if __name__ == '__main__': - logging.getLogger('pyomo').setLevel(logging.DEBUG) - parallel = None - for i, arg in enumerate(sys.argv): - if arg == '-j': - parallel = int(sys.argv.pop(i + 1)) - sys.argv.pop(i) - break - if arg.startswith('-j'): - if '=' in arg: - parallel = int(arg.split('=')[1]) - else: - parallel = int(arg[2:]) - sys.argv.pop(i) - break - if '--build-deps' in sys.argv: - sys.argv.remove('--build-deps') - build_ginac_library(parallel, []) - build_ginac_interface(parallel, sys.argv[1:]) + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-j", + dest='parallel', + type=int, + default=None, + help="Enable parallel build with PARALLEL cores", + ) + parser.add_argument( + "--build-deps", + dest='build_deps', + action='store_true', + default=False, + help="Download and build the CLN/GiNaC libraries", + ) + options, argv = parser.parse_known_args(sys.argv) + logging.getLogger('pyomo').setLevel(logging.INFO) + if options.build_deps: + build_ginac_library(options.parallel, []) + build_ginac_interface(options.parallel, argv[1:]) From c2c63bc52acedace36de75d6ce09c0d062d1ef84 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 10:09:09 -0600 Subject: [PATCH 72/99] Run sympy tests any time sympy is installed --- pyomo/contrib/simplification/tests/test_simplification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index be61631e9f3..27208d42229 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -100,12 +100,12 @@ def test_unary(self): assertExpressionsEqual(self, e, e2) -@unittest.skipIf((not sympy_available) or (ginac_available), 'sympy is not available') +@unittest.skipUnless(sympy_available, 'sympy is not available') class TestSimplificationSympy(TestCase, SimplificationMixin): pass -@unittest.skipIf(not ginac_available, 'GiNaC is not available') +@unittest.skipUnless(ginac_available, 'GiNaC is not available') class TestSimplificationGiNaC(TestCase, SimplificationMixin): def test_param(self): m = pe.ConcreteModel() From b53bbfc67a5e2cdea65008de2b4651670a7df66a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 12:00:31 -0600 Subject: [PATCH 73/99] Define NamedIntEnum --- pyomo/common/enums.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pyomo/common/enums.py b/pyomo/common/enums.py index 4d969bf7a9e..121155d4ae8 100644 --- a/pyomo/common/enums.py +++ b/pyomo/common/enums.py @@ -17,6 +17,7 @@ .. autosummary:: ExtendedEnumType + NamedIntEnum Standard Enums: @@ -130,7 +131,21 @@ def __new__(metacls, cls, bases, classdict, **kwds): return super().__new__(metacls, cls, bases, classdict, **kwds) -class ObjectiveSense(enum.IntEnum): +class NamedIntEnum(enum.IntEnum): + """An extended version of :py:class:`enum.IntEnum` that supports + creating members by name as well as value. + + """ + + @classmethod + def _missing_(cls, value): + for member in cls: + if member.name == value: + return member + return None + + +class ObjectiveSense(NamedIntEnum): """Flag indicating if an objective is minimizing (1) or maximizing (-1). While the numeric values are arbitrary, there are parts of Pyomo @@ -150,13 +165,6 @@ class ObjectiveSense(enum.IntEnum): def __str__(self): return self.name - @classmethod - def _missing_(cls, value): - for member in cls: - if member.name == value: - return member - return None - minimize = ObjectiveSense.minimize maximize = ObjectiveSense.maximize From e7540f237b774a7297afed8fa62d6848aae2480a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 12:02:12 -0600 Subject: [PATCH 74/99] Rework Simplifier so we can force the backend mode --- pyomo/contrib/simplification/simplify.py | 66 +++++++++++-------- .../tests/test_simplification.py | 43 +++++------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 94f0ceaa33f..840f3a1c1da 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -9,26 +9,25 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import logging +import warnings + +from pyomo.common.enums import NamedIntEnum from pyomo.core.expr.sympy_tools import sympy2pyomo_expression, sympyify_expression from pyomo.core.expr.numeric_expr import NumericExpression from pyomo.core.expr.numvalue import value, is_constant -import logging -import warnings from pyomo.contrib.simplification.ginac import ( interface as ginac_interface, interface_available as ginac_available, ) -logger = logging.getLogger(__name__) - def simplify_with_sympy(expr: NumericExpression): if is_constant(expr): return value(expr) - om, se = sympyify_expression(expr) - se = se.simplify() - new_expr = sympy2pyomo_expression(se, om) + object_map, sympy_expr = sympyify_expression(expr) + new_expr = sympy2pyomo_expression(sympy_expr.simplify(), object_map) if is_constant(new_expr): new_expr = value(new_expr) return new_expr @@ -37,29 +36,40 @@ def simplify_with_sympy(expr: NumericExpression): def simplify_with_ginac(expr: NumericExpression, ginac_interface): if is_constant(expr): return value(expr) - gi = ginac_interface - ginac_expr = gi.to_ginac(expr) - ginac_expr = ginac_expr.normal() - new_expr = gi.from_ginac(ginac_expr) - return new_expr + ginac_expr = ginac_interface.to_ginac(expr) + return ginac_interface.from_ginac(ginac_expr.normal()) class Simplifier(object): - def __init__(self, suppress_no_ginac_warnings: bool = False) -> None: - if ginac_available: - self.gi = ginac_interface.GinacInterface(False) - self.suppress_no_ginac_warnings = suppress_no_ginac_warnings + class Mode(NamedIntEnum): + auto = 0 + sympy = 1 + ginac = 2 + + def __init__( + self, suppress_no_ginac_warnings: bool = False, mode: Mode = Mode.auto + ) -> None: + if mode == Simplifier.Mode.auto: + if ginac_available: + mode = Simplifier.Mode.ginac + else: + if not suppress_no_ginac_warnings: + msg = ( + "GiNaC does not seem to be available. Using SymPy. " + + "Note that the GiNaC interface is significantly faster." + ) + logging.getLogger(__name__).warning(msg) + warnings.warn(msg) + mode = Simplifier.Mode.sympy - def simplify(self, expr: NumericExpression): - if ginac_available: - return simplify_with_ginac(expr, self.gi) + if mode == Simplifier.Mode.ginac: + self.gi = ginac_interface.GinacInterface(False) + self.simplify = self._simplify_with_ginac else: - if not self.suppress_no_ginac_warnings: - msg = ( - "GiNaC does not seem to be available. Using SymPy. " - + "Note that the GiNac interface is significantly faster." - ) - logger.warning(msg) - warnings.warn(msg) - self.suppress_no_ginac_warnings = True - return simplify_with_sympy(expr) + self.simplify = self._simplify_with_sympy + + def _simplify_with_ginac(self, expr: NumericExpression): + return simplify_with_ginac(expr, self.gi) + + def _simplify_with_sympy(self, expr: NumericExpression): + return simplify_with_sympy(expr) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index 27208d42229..efa9f903adc 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -9,17 +9,15 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from pyomo.common.unittest import TestCase from pyomo.common import unittest +from pyomo.common.fileutils import this_file_dir from pyomo.contrib.simplification import Simplifier from pyomo.contrib.simplification.simplify import ginac_available from pyomo.core.expr.compare import assertExpressionsEqual, compare_expressions -import pyomo.environ as pe from pyomo.core.expr.calculus.diff_with_pyomo import reverse_sd -from pyomo.common.dependencies import attempt_import - +from pyomo.core.expr.sympy_tools import sympy_available -sympy, sympy_available = attempt_import('sympy') +import pyomo.environ as pe class SimplificationMixin: @@ -37,8 +35,7 @@ def test_simplify(self): e = x * pe.log(x) der1 = reverse_sd(e)[x] der2 = reverse_sd(der1)[x] - simp = Simplifier() - der2_simp = simp.simplify(der2) + der2_simp = self.simp.simplify(der2) expected = x**-1.0 assertExpressionsEqual(self, expected, der2_simp) @@ -46,8 +43,7 @@ def test_mul(self): m = pe.ConcreteModel() x = m.x = pe.Var() e = 2 * x - simp = Simplifier() - e2 = simp.simplify(e) + e2 = self.simp.simplify(e) expected = 2.0 * x assertExpressionsEqual(self, expected, e2) @@ -55,16 +51,14 @@ def test_sum(self): m = pe.ConcreteModel() x = m.x = pe.Var() e = 2 + x - simp = Simplifier() - e2 = simp.simplify(e) + e2 = self.simp.simplify(e) self.compare_against_possible_results(e2, [2.0 + x, x + 2.0]) def test_neg(self): m = pe.ConcreteModel() x = m.x = pe.Var() e = -pe.log(x) - simp = Simplifier() - e2 = simp.simplify(e) + e2 = self.simp.simplify(e) self.compare_against_possible_results( e2, [(-1.0) * pe.log(x), pe.log(x) * (-1.0), -pe.log(x)] ) @@ -73,8 +67,7 @@ def test_pow(self): m = pe.ConcreteModel() x = m.x = pe.Var() e = x**2.0 - simp = Simplifier() - e2 = simp.simplify(e) + e2 = self.simp.simplify(e) assertExpressionsEqual(self, e, e2) def test_div(self): @@ -82,9 +75,7 @@ def test_div(self): x = m.x = pe.Var() y = m.y = pe.Var() e = x / y + y / x - x / y - simp = Simplifier() - e2 = simp.simplify(e) - print(e2) + e2 = self.simp.simplify(e) self.compare_against_possible_results( e2, [y / x, y * (1.0 / x), y * x**-1.0, x**-1.0 * y] ) @@ -95,25 +86,27 @@ def test_unary(self): func_list = [pe.log, pe.sin, pe.cos, pe.tan, pe.asin, pe.acos, pe.atan] for func in func_list: e = func(x) - simp = Simplifier() - e2 = simp.simplify(e) + e2 = self.simp.simplify(e) assertExpressionsEqual(self, e, e2) @unittest.skipUnless(sympy_available, 'sympy is not available') -class TestSimplificationSympy(TestCase, SimplificationMixin): - pass +class TestSimplificationSympy(unittest.TestCase, SimplificationMixin): + def setUp(self): + self.simp = Simplifier(mode=Simplifier.Mode.sympy) @unittest.skipUnless(ginac_available, 'GiNaC is not available') -class TestSimplificationGiNaC(TestCase, SimplificationMixin): +class TestSimplificationGiNaC(unittest.TestCase, SimplificationMixin): + def setUp(self): + self.simp = Simplifier(mode=Simplifier.Mode.ginac) + def test_param(self): m = pe.ConcreteModel() x = m.x = pe.Var() p = m.p = pe.Param(mutable=True) e1 = p * x**2 + p * x + p * x**2 - simp = Simplifier() - e2 = simp.simplify(e1) + e2 = self.simp.simplify(e1) self.compare_against_possible_results( e2, [ From 9c220b05e34bfff25e3fae4172b187ed169183c1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 12:56:34 -0600 Subject: [PATCH 75/99] Improve robustness of tar filter --- pyomo/common/download.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pyomo/common/download.py b/pyomo/common/download.py index 95713e9ef76..8361798817f 100644 --- a/pyomo/common/download.py +++ b/pyomo/common/download.py @@ -408,25 +408,25 @@ def filter_fcn(info): f = info.name if os.path.isabs(f) or '..' in f or f.startswith(('/', os.sep)): logger.error( - "malformed (potentially insecure) filename (%s) " - "found in tar archive. Skipping file." % (f,) - ) - return False - target = os.path.realpath(os.path.join(dest, f)) - if os.path.commonpath([target, dest]) != dest: - logger.error( - "malformed (potentially insecure) filename (%s) " - "found in zip archive. Skipping file." % (f,) + "malformed or potentially insecure filename (%s). " + "Skipping file." % (f,) ) return False target = self._splitpath(f) if len(target) <= dirOffset: if not info.isdir(): logger.warning( - "Skipping file (%s) in zip archive due to dirOffset" % (f,) + "Skipping file (%s) in tar archive due to dirOffset." % (f,) ) return False - info.name = '/'.join(target[dirOffset:]) + info.name = f = '/'.join(target[dirOffset:]) + target = os.path.realpath(os.path.join(dest, f)) + if os.path.commonpath([target, dest]) != dest: + logger.error( + "potentially insecure filename (%s) resolves outside target " + "directory. Skipping file." % (f,) + ) + return False # Strip high bits & group/other write bits info.mode &= 0o755 return True From e597bb5c390432555990394ab8fa474bfee8dbaa Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 12:57:07 -0600 Subject: [PATCH 76/99] Ensure tar file is closed --- pyomo/common/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/common/download.py b/pyomo/common/download.py index 8361798817f..2a91553c728 100644 --- a/pyomo/common/download.py +++ b/pyomo/common/download.py @@ -398,8 +398,6 @@ def get_tar_archive(self, url, dirOffset=0): raise RuntimeError( "Target directory (%s) exists, but is not a directory" % (self._fname,) ) - tar_file = tarfile.open(fileobj=io.BytesIO(self.retrieve_url(url))) - dest = os.path.realpath(self._fname) def filter_fcn(info): # this mocks up the `tarfile` filter introduced in Python @@ -431,7 +429,9 @@ def filter_fcn(info): info.mode &= 0o755 return True - tar_file.extractall(dest, filter(filter_fcn, tar_file.getmembers())) + with tarfile.open(fileobj=io.BytesIO(self.retrieve_url(url))) as TAR: + dest = os.path.realpath(self._fname) + TAR.extractall(dest, filter(filter_fcn, TAR.getmembers())) def get_gzipped_binary_file(self, url): if self._fname is None: From 70166ffbeb3d39d5ed59cba6f479d041d3da0eb1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 5 May 2024 12:58:19 -0600 Subject: [PATCH 77/99] test get_tar_archive() --- pyomo/common/tests/test_download.py | 70 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/pyomo/common/tests/test_download.py b/pyomo/common/tests/test_download.py index 87108be1c59..8fee0ba7e31 100644 --- a/pyomo/common/tests/test_download.py +++ b/pyomo/common/tests/test_download.py @@ -9,12 +9,14 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import io import os import platform import re import shutil -import tempfile import subprocess +import tarfile +import tempfile import pyomo.common.unittest as unittest import pyomo.common.envvar as envvar @@ -22,6 +24,7 @@ from pyomo.common import DeveloperError from pyomo.common.fileutils import this_file from pyomo.common.download import FileDownloader, distro_available +from pyomo.common.log import LoggingIntercept from pyomo.common.tee import capture_output @@ -242,7 +245,7 @@ def test_get_files_requires_set_destination(self): ): f.get_gzipped_binary_file('bogus') - def test_get_test_binary_file(self): + def test_get_text_binary_file(self): tmpdir = tempfile.mkdtemp() try: f = FileDownloader() @@ -263,3 +266,66 @@ def test_get_test_binary_file(self): self.assertEqual(os.path.getsize(target), len(os.linesep)) finally: shutil.rmtree(tmpdir) + + def test_get_tar_archive(self): + tmpdir = tempfile.mkdtemp() + try: + f = FileDownloader() + + # Mock retrieve_url so network connections are not necessary + buf = io.BytesIO() + with tarfile.open(mode="w:gz", fileobj=buf) as TAR: + info = tarfile.TarInfo('b/lnk') + info.size = 0 + info.type = tarfile.SYMTYPE + info.linkname = envvar.PYOMO_CONFIG_DIR + TAR.addfile(info) + for fname in ('a', 'b/c', 'b/d', '/root', 'b/lnk/test'): + info = tarfile.TarInfo(fname) + info.size = 0 + info.type = tarfile.REGTYPE + info.mode = 0o644 + info.mtime = info.uid = info.gid = 0 + info.uname = info.gname = 'root' + TAR.addfile(info) + f.retrieve_url = lambda url: buf.getvalue() + + with self.assertRaisesRegex( + DeveloperError, + r"(?s)target file name has not been initialized " + r"with set_destination_filename".replace(' ', r'\s+'), + ): + f.get_tar_archive(None, 1) + + _tmp = os.path.join(tmpdir, 'a_file') + with open(_tmp, 'w'): + pass + f.set_destination_filename(_tmp) + with self.assertRaisesRegex( + RuntimeError, + r"Target directory \(.*a_file\) exists, but is not a directory", + ): + f.get_tar_archive(None, 1) + + f.set_destination_filename(tmpdir) + with LoggingIntercept() as LOG: + f.get_tar_archive(None, 1) + + self.assertEqual( + LOG.getvalue().strip(), + """ +Skipping file (a) in tar archive due to dirOffset. +malformed or potentially insecure filename (/root). Skipping file. +potentially insecure filename (lnk/test) resolves outside target directory. Skipping file. +""".strip(), + ) + for f in ('c', 'd'): + fname = os.path.join(tmpdir, f) + self.assertTrue(os.path.exists(fname)) + self.assertTrue(os.path.isfile(fname)) + for f in ('lnk',): + fname = os.path.join(tmpdir, f) + self.assertTrue(os.path.exists(fname)) + self.assertTrue(os.path.islink(fname)) + finally: + shutil.rmtree(tmpdir) From 4446d364877ccbaa5faa33f9a100efecbd2975c9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 7 May 2024 10:23:43 -0600 Subject: [PATCH 78/99] fix tests --- pyomo/environ/tests/test_package_layout.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/environ/tests/test_package_layout.py b/pyomo/environ/tests/test_package_layout.py index 4e1574ab158..47c6422a879 100644 --- a/pyomo/environ/tests/test_package_layout.py +++ b/pyomo/environ/tests/test_package_layout.py @@ -38,6 +38,7 @@ _NON_MODULE_DIRS = { join('contrib', 'ampl_function_demo', 'src'), join('contrib', 'appsi', 'cmodel', 'src'), + join('contrib', 'simplification', 'ginac', 'src'), join('contrib', 'pynumero', 'src'), join('core', 'tests', 'data', 'baselines'), join('core', 'tests', 'diet', 'baselines'), From 90b1783197210cdca5dfe3c0b45bb467e6d58148 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 7 May 2024 13:09:01 -0600 Subject: [PATCH 79/99] Update tar filter to handle ValueError from commonpath() --- pyomo/common/download.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pyomo/common/download.py b/pyomo/common/download.py index 2a91553c728..ad672e8c79b 100644 --- a/pyomo/common/download.py +++ b/pyomo/common/download.py @@ -419,11 +419,17 @@ def filter_fcn(info): return False info.name = f = '/'.join(target[dirOffset:]) target = os.path.realpath(os.path.join(dest, f)) - if os.path.commonpath([target, dest]) != dest: - logger.error( - "potentially insecure filename (%s) resolves outside target " - "directory. Skipping file." % (f,) - ) + try: + if os.path.commonpath([target, dest]) != dest: + logger.error( + "potentially insecure filename (%s) resolves outside target " + "directory. Skipping file." % (f,) + ) + return False + except ValueError: + # commonpath() will raise ValueError for paths that + # don't have anything in common (notably, when files are + # on different drives on Windows) return False # Strip high bits & group/other write bits info.mode &= 0o755 From 5aae45946ebd302cfe0e33d54c5927d2f193a362 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 7 May 2024 13:20:22 -0600 Subject: [PATCH 80/99] keep mutable parameters in sympy conversion --- pyomo/core/expr/sympy_tools.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyomo/core/expr/sympy_tools.py b/pyomo/core/expr/sympy_tools.py index 48bd542be0f..05c9885cc8c 100644 --- a/pyomo/core/expr/sympy_tools.py +++ b/pyomo/core/expr/sympy_tools.py @@ -175,10 +175,11 @@ def sympyVars(self): class Pyomo2SympyVisitor(EXPR.StreamBasedExpressionVisitor): - def __init__(self, object_map): + def __init__(self, object_map, keep_mutable_parameters=True): sympy.Add # this ensures _configure_sympy gets run super(Pyomo2SympyVisitor, self).__init__() self.object_map = object_map + self.keep_mutable_parameters = keep_mutable_parameters def initializeWalker(self, expr): return self.beforeChild(None, expr, None) @@ -212,6 +213,8 @@ def beforeChild(self, node, child, child_idx): # # Everything else is a constant... # + if self.keep_mutable_parameters and child.is_parameter_type() and child.mutable: + return False, self.object_map.getSympySymbol(child) return False, value(child) @@ -245,13 +248,15 @@ def beforeChild(self, node, child, child_idx): return True, None -def sympyify_expression(expr): +def sympyify_expression(expr, keep_mutable_parameters=True): """Convert a Pyomo expression to a Sympy expression""" # # Create the visitor and call it. # object_map = PyomoSympyBimap() - visitor = Pyomo2SympyVisitor(object_map) + visitor = Pyomo2SympyVisitor( + object_map, keep_mutable_parameters=keep_mutable_parameters + ) return object_map, visitor.walk_expression(expr) From ae5ebd3ed492405ae488921ef3c6b36c886f098a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 7 May 2024 13:26:35 -0600 Subject: [PATCH 81/99] update defaults for mutable parameters when using sympy --- pyomo/contrib/simplification/simplify.py | 2 +- pyomo/core/expr/sympy_tools.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/simplification/simplify.py b/pyomo/contrib/simplification/simplify.py index 840f3a1c1da..874b5b1e801 100644 --- a/pyomo/contrib/simplification/simplify.py +++ b/pyomo/contrib/simplification/simplify.py @@ -26,7 +26,7 @@ def simplify_with_sympy(expr: NumericExpression): if is_constant(expr): return value(expr) - object_map, sympy_expr = sympyify_expression(expr) + object_map, sympy_expr = sympyify_expression(expr, keep_mutable_parameters=True) new_expr = sympy2pyomo_expression(sympy_expr.simplify(), object_map) if is_constant(new_expr): new_expr = value(new_expr) diff --git a/pyomo/core/expr/sympy_tools.py b/pyomo/core/expr/sympy_tools.py index 05c9885cc8c..6c184f0e4c4 100644 --- a/pyomo/core/expr/sympy_tools.py +++ b/pyomo/core/expr/sympy_tools.py @@ -175,7 +175,7 @@ def sympyVars(self): class Pyomo2SympyVisitor(EXPR.StreamBasedExpressionVisitor): - def __init__(self, object_map, keep_mutable_parameters=True): + def __init__(self, object_map, keep_mutable_parameters=False): sympy.Add # this ensures _configure_sympy gets run super(Pyomo2SympyVisitor, self).__init__() self.object_map = object_map @@ -248,7 +248,7 @@ def beforeChild(self, node, child, child_idx): return True, None -def sympyify_expression(expr, keep_mutable_parameters=True): +def sympyify_expression(expr, keep_mutable_parameters=False): """Convert a Pyomo expression to a Sympy expression""" # # Create the visitor and call it. From 2b3bd4eea0d00e4e7e2dce3483aceaa04dcfb767 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 7 May 2024 14:38:14 -0600 Subject: [PATCH 82/99] update tests --- .../tests/test_simplification.py | 25 ++++++++++--------- setup.py | 1 + 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index efa9f903adc..acef0af502e 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -89,18 +89,6 @@ def test_unary(self): e2 = self.simp.simplify(e) assertExpressionsEqual(self, e, e2) - -@unittest.skipUnless(sympy_available, 'sympy is not available') -class TestSimplificationSympy(unittest.TestCase, SimplificationMixin): - def setUp(self): - self.simp = Simplifier(mode=Simplifier.Mode.sympy) - - -@unittest.skipUnless(ginac_available, 'GiNaC is not available') -class TestSimplificationGiNaC(unittest.TestCase, SimplificationMixin): - def setUp(self): - self.simp = Simplifier(mode=Simplifier.Mode.ginac) - def test_param(self): m = pe.ConcreteModel() x = m.x = pe.Var() @@ -116,5 +104,18 @@ def test_param(self): p * x + 2.0 * p * x**2.0, x**2.0 * p * 2.0 + p * x, p * x + x**2.0 * p * 2.0, + p * x * (1 + 2 * x), ], ) + + +@unittest.skipUnless(sympy_available, 'sympy is not available') +class TestSimplificationSympy(unittest.TestCase, SimplificationMixin): + def setUp(self): + self.simp = Simplifier(mode=Simplifier.Mode.sympy) + + +@unittest.skipUnless(ginac_available, 'GiNaC is not available') +class TestSimplificationGiNaC(unittest.TestCase, SimplificationMixin): + def setUp(self): + self.simp = Simplifier(mode=Simplifier.Mode.ginac) diff --git a/setup.py b/setup.py index 70c1626a650..a125b02b2fe 100644 --- a/setup.py +++ b/setup.py @@ -306,6 +306,7 @@ def __ne__(self, other): "pyomo.contrib.mcpp": ["*.cpp"], "pyomo.contrib.pynumero": ['src/*', 'src/tests/*'], "pyomo.contrib.viewer": ["*.ui"], + "pyomo.contrib.simplification.ginac": ["src/*.cpp", "src/*.hpp"], }, ext_modules=ext_modules, entry_points=""" From 82dfda15e1ce4650ff3d7b7a2abfd2c90735a8bd Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 08:28:12 -0600 Subject: [PATCH 83/99] Ensure the same output is logged on Windows and other platforms --- pyomo/common/download.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyomo/common/download.py b/pyomo/common/download.py index ad672e8c79b..ad3b64060e9 100644 --- a/pyomo/common/download.py +++ b/pyomo/common/download.py @@ -430,6 +430,10 @@ def filter_fcn(info): # commonpath() will raise ValueError for paths that # don't have anything in common (notably, when files are # on different drives on Windows) + logger.error( + "potentially insecure filename (%s) resolves outside target " + "directory. Skipping file." % (f,) + ) return False # Strip high bits & group/other write bits info.mode &= 0o755 From 3572c445145b8e901d1de61cc842914c5ea60d8a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 8 May 2024 10:37:39 -0600 Subject: [PATCH 84/99] ginac cleanup --- pyomo/contrib/simplification/build.py | 1 + pyomo/contrib/simplification/ginac/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index d540991b010..dfb9d2cf1c8 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -71,6 +71,7 @@ def build_ginac_library(parallel=None, argv=None, env=None): logger.info("\nBuilding GiNaC\n") assert subprocess.run(make_cmd, cwd=ginac_dir, env=env).returncode == 0 assert subprocess.run(install_cmd, cwd=ginac_dir, env=env).returncode == 0 + print("Installed GiNaC to %s" % (ginac_dir,)) def _find_include(libdir, incpaths): diff --git a/pyomo/contrib/simplification/ginac/__init__.py b/pyomo/contrib/simplification/ginac/__init__.py index af6511944de..6896bec12c4 100644 --- a/pyomo/contrib/simplification/ginac/__init__.py +++ b/pyomo/contrib/simplification/ginac/__init__.py @@ -30,7 +30,7 @@ def _importer(): # GiNaC needs 2 libraries that are generally dynamically linked # to the interface library. If we built those ourselves, then # the libraries will be PYOMO_CONFIG_DIR/lib ... but that - # directlor is very likely to NOT be on the library search path + # directory is very likely to NOT be on the library search path # when the Python interpreter was started. We will manually # look for those two libraries, and if we find them, load them # into this process (so the interface can find them) From a53c6f7ec4929a5821203ee0a87e8d31e88edb1d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 10:39:54 -0600 Subject: [PATCH 85/99] Add 'builders' marker for testing custom library builds (currently just ginac) --- .jenkins.sh | 7 +++++++ pyomo/contrib/simplification/tests/test_simplification.py | 2 ++ setup.cfg | 1 + 3 files changed, 10 insertions(+) diff --git a/.jenkins.sh b/.jenkins.sh index 37be6113ed9..37a9238f983 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -122,6 +122,13 @@ if test -z "$MODE" -o "$MODE" == setup; then echo "PYOMO_CONFIG_DIR=$PYOMO_CONFIG_DIR" echo "" + # Call Pyomo build scripts to build TPLs that would normally be + # skipped by the pyomo download-extensions / build-extensions + # actions below + if test [ " $CATEGORY " == *" builders "*; then + python pyomo/contrib/simplification/build.py --build-deps || exit 1 + fi + # Use Pyomo to download & compile binary extensions i=0 while /bin/true; do diff --git a/pyomo/contrib/simplification/tests/test_simplification.py b/pyomo/contrib/simplification/tests/test_simplification.py index acef0af502e..1ff9f5a3cc4 100644 --- a/pyomo/contrib/simplification/tests/test_simplification.py +++ b/pyomo/contrib/simplification/tests/test_simplification.py @@ -115,6 +115,8 @@ def setUp(self): self.simp = Simplifier(mode=Simplifier.Mode.sympy) +@unittest.pytest.mark.default +@unittest.pytest.mark.builders @unittest.skipUnless(ginac_available, 'GiNaC is not available') class TestSimplificationGiNaC(unittest.TestCase, SimplificationMixin): def setUp(self): diff --git a/setup.cfg b/setup.cfg index b606138f38c..d9ccbbb7c5e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,4 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests + builders: thests that should be run when testing custom (extension) builders \ No newline at end of file From 8f33eed1ed2a82c24be5d5b3dff77dcab472f67e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 10:40:36 -0600 Subject: [PATCH 86/99] Support multiple markers (categories) in jenkins driver --- .jenkins.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 37a9238f983..8c72edf41c0 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -43,8 +43,8 @@ fi if test -z "$SLIM"; then export VENV_SYSTEM_PACKAGES='--system-site-packages' fi -if test ! -z "$CATEGORY"; then - export PY_CAT="-m $CATEGORY" +if test -n "$CATEGORY"; then + export PY_CAT="-m '"`echo "$CATEGORY" | sed -r "s/ +/ or /g"`"'" fi if test "$WORKSPACE" != "`pwd`"; then From 4dc4e893aa861bcbeaf777e971788f5e8ce39983 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 10:41:01 -0600 Subject: [PATCH 87/99] Additional (debugging) output in ginac_interface builder --- pyomo/contrib/simplification/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index d540991b010..a1332490b1b 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -155,6 +155,7 @@ def run(self): ) if not os.path.exists(target): os.makedirs(target) + sys.stdout.write(f"Installing {library} in {target}\n") shutil.copy(library, target) package_config = { From e847f10f032fc2a04080af7784d671847a51adcc Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 11:25:24 -0600 Subject: [PATCH 88/99] NFC: fix typo --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d9ccbbb7c5e..f670cef8f68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,4 +22,4 @@ markers = lp: marks lp tests gams: marks gams tests bar: marks bar tests - builders: thests that should be run when testing custom (extension) builders \ No newline at end of file + builders: tests that should be run when testing custom (extension) builders \ No newline at end of file From 8b11a1c789a73219054fec5d25b8db22f42e8da8 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 11:27:51 -0600 Subject: [PATCH 89/99] NFC: resyncing test_branches and test_pr_and_main --- .github/workflows/test_branches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index df6455568b9..d9c36e78fc4 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -96,7 +96,7 @@ jobs: PACKAGES: openmpi mpi4py - os: ubuntu-latest - python: 3.11 + python: '3.11' other: /singletest category: "-m 'neos or importtest'" skip_doctest: 1 @@ -273,7 +273,7 @@ jobs: if test -z "${{matrix.slim}}"; then python -m pip install --cache-dir cache/pip cplex docplex \ || echo "WARNING: CPLEX Community Edition is not available" - python -m pip install --cache-dir cache/pip gurobipy==10.0.3\ + python -m pip install --cache-dir cache/pip gurobipy==10.0.3 \ || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" From fa2cc317eb90941c87997bb7b7935aca488cc1a6 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 17:04:32 -0600 Subject: [PATCH 90/99] improve handling of category quotation --- .jenkins.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 8c72edf41c0..a00b42eac4e 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -43,9 +43,6 @@ fi if test -z "$SLIM"; then export VENV_SYSTEM_PACKAGES='--system-site-packages' fi -if test -n "$CATEGORY"; then - export PY_CAT="-m '"`echo "$CATEGORY" | sed -r "s/ +/ or /g"`"'" -fi if test "$WORKSPACE" != "`pwd`"; then echo "ERROR: pwd is not WORKSPACE" @@ -185,7 +182,7 @@ if test -z "$MODE" -o "$MODE" == test; then python -m pytest -v \ -W ignore::Warning \ --junitxml="TEST-pyomo.xml" \ - $PY_CAT $TEST_SUITES $PYTEST_EXTRA_ARGS + -m "$CATEGORY" $TEST_SUITES $PYTEST_EXTRA_ARGS # Combine the coverage results and upload if test -z "$DISABLE_COVERAGE"; then From bd5f10cb7d0dbd96aae6114e9ee51407e9b4dd13 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 17:07:16 -0600 Subject: [PATCH 91/99] Improve conftest.py efficiency --- conftest.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/conftest.py b/conftest.py index 7faad6fc89b..00abdecfa12 100644 --- a/conftest.py +++ b/conftest.py @@ -11,6 +11,8 @@ import pytest +_implicit_markers = {'default',} +_extended_implicit_markers = _implicit_markers.union({'solver',}) def pytest_runtest_setup(item): """ @@ -32,13 +34,10 @@ def pytest_runtest_setup(item): the default mode; but if solver tests are also marked with an explicit category (e.g., "expensive"), we will skip them. """ - marker = item.iter_markers() solvernames = [mark.args[0] for mark in item.iter_markers(name="solver")] solveroption = item.config.getoption("--solver") markeroption = item.config.getoption("-m") - implicit_markers = ['default'] - extended_implicit_markers = implicit_markers + ['solver'] - item_markers = set(mark.name for mark in marker) + item_markers = set(mark.name for mark in item.iter_markers()) if solveroption: if solveroption not in solvernames: pytest.skip("SKIPPED: Test not marked {!r}".format(solveroption)) @@ -46,9 +45,9 @@ def pytest_runtest_setup(item): elif markeroption: return elif item_markers: - if not set(implicit_markers).issubset( + if not _implicit_markers.issubset( item_markers - ) and not item_markers.issubset(set(extended_implicit_markers)): + ) and not item_markers.issubset(_extended_implicit_markers): pytest.skip('SKIPPED: Only running default, solver, and unmarked tests.') From 354feb2c9a81473bc9ca95138bcd8e4d4eadd85c Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 17:07:46 -0600 Subject: [PATCH 92/99] Ensure that all unmarked tests are marked with the implicit markers --- conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conftest.py b/conftest.py index 00abdecfa12..6e8043c2c92 100644 --- a/conftest.py +++ b/conftest.py @@ -14,6 +14,18 @@ _implicit_markers = {'default',} _extended_implicit_markers = _implicit_markers.union({'solver',}) +def pytest_collection_modifyitems(items): + """ + This method will mark any unmarked tests with the implicit marker ('default') + + """ + for item in items: + try: + next(item.iter_markers()) + except StopIteration: + for marker in _implicit_markers: + item.add_marker(getattr(pytest.mark, marker)) + def pytest_runtest_setup(item): """ This method overrides pytest's default behavior for marked tests. From 92edcc5d7b486a767f0033514e13ce6549d8e81d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 17:08:29 -0600 Subject: [PATCH 93/99] NFC: apply black --- conftest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/conftest.py b/conftest.py index 6e8043c2c92..34b366f9fd6 100644 --- a/conftest.py +++ b/conftest.py @@ -11,8 +11,9 @@ import pytest -_implicit_markers = {'default',} -_extended_implicit_markers = _implicit_markers.union({'solver',}) +_implicit_markers = {'default'} +_extended_implicit_markers = _implicit_markers.union({'solver'}) + def pytest_collection_modifyitems(items): """ @@ -26,6 +27,7 @@ def pytest_collection_modifyitems(items): for marker in _implicit_markers: item.add_marker(getattr(pytest.mark, marker)) + def pytest_runtest_setup(item): """ This method overrides pytest's default behavior for marked tests. @@ -57,9 +59,9 @@ def pytest_runtest_setup(item): elif markeroption: return elif item_markers: - if not _implicit_markers.issubset( - item_markers - ) and not item_markers.issubset(_extended_implicit_markers): + if not _implicit_markers.issubset(item_markers) and not item_markers.issubset( + _extended_implicit_markers + ): pytest.skip('SKIPPED: Only running default, solver, and unmarked tests.') From 7ae72756ad5e0019370ed3aaf9f327b86fd717d1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 17:47:16 -0600 Subject: [PATCH 94/99] Prevent APPSI / GiNaC interfaces from exposing module symbols globally --- pyomo/contrib/appsi/cmodel/src/cmodel_bindings.cpp | 5 +++-- pyomo/contrib/simplification/ginac/src/ginac_interface.cpp | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/appsi/cmodel/src/cmodel_bindings.cpp b/pyomo/contrib/appsi/cmodel/src/cmodel_bindings.cpp index 6acc1d79845..5a838ffd786 100644 --- a/pyomo/contrib/appsi/cmodel/src/cmodel_bindings.cpp +++ b/pyomo/contrib/appsi/cmodel/src/cmodel_bindings.cpp @@ -63,7 +63,8 @@ PYBIND11_MODULE(appsi_cmodel, m) { m.def("appsi_exprs_from_pyomo_exprs", &appsi_exprs_from_pyomo_exprs); m.def("appsi_expr_from_pyomo_expr", &appsi_expr_from_pyomo_expr); m.def("prep_for_repn", &prep_for_repn); - py::class_(m, "PyomoExprTypes").def(py::init<>()); + py::class_(m, "PyomoExprTypes", py::module_local()) + .def(py::init<>()); py::class_>(m, "Node") .def("is_variable_type", &Node::is_variable_type) .def("is_param_type", &Node::is_param_type) @@ -165,7 +166,7 @@ PYBIND11_MODULE(appsi_cmodel, m) { .def(py::init<>()) .def("write", &LPWriter::write) .def("get_solve_cons", &LPWriter::get_solve_cons); - py::enum_(m, "ExprType") + py::enum_(m, "ExprType", py::module_local()) .value("py_float", ExprType::py_float) .value("var", ExprType::var) .value("param", ExprType::param) diff --git a/pyomo/contrib/simplification/ginac/src/ginac_interface.cpp b/pyomo/contrib/simplification/ginac/src/ginac_interface.cpp index 1060f87161c..9b05baf71ca 100644 --- a/pyomo/contrib/simplification/ginac/src/ginac_interface.cpp +++ b/pyomo/contrib/simplification/ginac/src/ginac_interface.cpp @@ -298,7 +298,8 @@ py::object GinacInterface::from_ginac(ex &ge) { PYBIND11_MODULE(ginac_interface, m) { m.def("pyomo_to_ginac", &pyomo_to_ginac); - py::class_(m, "PyomoExprTypes").def(py::init<>()); + py::class_(m, "PyomoExprTypes", py::module_local()) + .def(py::init<>()); py::class_(m, "ginac_expression") .def("expand", [](ex &ge) { return ge.expand(); @@ -313,7 +314,7 @@ PYBIND11_MODULE(ginac_interface, m) { .def(py::init()) .def("to_ginac", &GinacInterface::to_ginac) .def("from_ginac", &GinacInterface::from_ginac); - py::enum_(m, "ExprType") + py::enum_(m, "ExprType", py::module_local()) .value("py_float", ExprType::py_float) .value("var", ExprType::var) .value("param", ExprType::param) From 1409aa2956159b8a062e12ed831db79b57dda189 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 18:09:15 -0600 Subject: [PATCH 95/99] Fix bug in Jenkins driver --- .jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins.sh b/.jenkins.sh index a00b42eac4e..842733e471b 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -122,7 +122,7 @@ if test -z "$MODE" -o "$MODE" == setup; then # Call Pyomo build scripts to build TPLs that would normally be # skipped by the pyomo download-extensions / build-extensions # actions below - if test [ " $CATEGORY " == *" builders "*; then + if test [[ " $CATEGORY " == *" builders "* ]]; then python pyomo/contrib/simplification/build.py --build-deps || exit 1 fi From aa756017fc23091764a63721dc80c94181cbe01a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 18:29:43 -0600 Subject: [PATCH 96/99] Fix typo in Jenkins driver --- .jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins.sh b/.jenkins.sh index 842733e471b..0f4e70d3cf1 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -122,7 +122,7 @@ if test -z "$MODE" -o "$MODE" == setup; then # Call Pyomo build scripts to build TPLs that would normally be # skipped by the pyomo download-extensions / build-extensions # actions below - if test [[ " $CATEGORY " == *" builders "* ]]; then + if [[ " $CATEGORY " == *" builders "* ]]; then python pyomo/contrib/simplification/build.py --build-deps || exit 1 fi From cdaff17f6a7e7c36428df887acdc7a7a59ec836a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 18:53:56 -0600 Subject: [PATCH 97/99] Add info to the build log --- .jenkins.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.jenkins.sh b/.jenkins.sh index 0f4e70d3cf1..696847fd92c 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -123,13 +123,19 @@ if test -z "$MODE" -o "$MODE" == setup; then # skipped by the pyomo download-extensions / build-extensions # actions below if [[ " $CATEGORY " == *" builders "* ]]; then + echo "" + echo "Running local build scripts..." + echo "" + set -x python pyomo/contrib/simplification/build.py --build-deps || exit 1 + set +x fi # Use Pyomo to download & compile binary extensions i=0 while /bin/true; do i=$[$i+1] + echo "" echo "Downloading pyomo extensions (attempt $i)" pyomo download-extensions $PYOMO_DOWNLOAD_ARGS if test $? == 0; then From 76d53de4d383cd03eadaa4997f4deecebec6d4f1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 18:54:19 -0600 Subject: [PATCH 98/99] Explicitly call out CWD when running configure --- pyomo/contrib/simplification/build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/simplification/build.py b/pyomo/contrib/simplification/build.py index 53133c2fb4e..b4bec63088a 100644 --- a/pyomo/contrib/simplification/build.py +++ b/pyomo/contrib/simplification/build.py @@ -27,7 +27,11 @@ def build_ginac_library(parallel=None, argv=None, env=None): sys.stdout.write("\n**** Building GiNaC library ****\n") - configure_cmd = ['configure', '--prefix=' + PYOMO_CONFIG_DIR, '--disable-static'] + configure_cmd = [ + os.path.join('.', 'configure'), + '--prefix=' + PYOMO_CONFIG_DIR, + '--disable-static', + ] make_cmd = ['make'] if parallel: make_cmd.append(f'-j{parallel}') From 35b71f87a1d070b10553119c86489f99875a6587 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 8 May 2024 20:17:31 -0600 Subject: [PATCH 99/99] Removing singletest from test_branches --- .github/workflows/test_branches.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index d9c36e78fc4..5063571c65f 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -95,14 +95,6 @@ jobs: PYENV: conda PACKAGES: openmpi mpi4py - - os: ubuntu-latest - python: '3.11' - other: /singletest - category: "-m 'neos or importtest'" - skip_doctest: 1 - TARGET: linux - PYENV: pip - - os: ubuntu-latest python: '3.10' other: /cython