From d4075783b2afcadfe70ed0ce179ba2ca46193666 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:43:02 -0700 Subject: [PATCH 01/12] Rework __deepcopy__ to move bulk of logic into AutoSlots (reduce redundant code) --- pyomo/common/autoslots.py | 98 ++++++++++++++++++++++++++++++++--- pyomo/core/base/component.py | 99 +++--------------------------------- 2 files changed, 98 insertions(+), 99 deletions(-) diff --git a/pyomo/common/autoslots.py b/pyomo/common/autoslots.py index 5846fbb443c..3b3fbf7e7c1 100644 --- a/pyomo/common/autoslots.py +++ b/pyomo/common/autoslots.py @@ -269,16 +269,98 @@ def __deepcopy__(self, memo): """ # Note: this implementation avoids deepcopying the temporary # 'state' list, significantly speeding things up. - memo[id(self)] = ans = self.__class__.__new__(self.__class__) - state = self.__getstate__() - ans.__setstate__([fast_deepcopy(field, memo) for field in state]) - # The state uses a temporary dict to store the (mapped) - # __dict__ state. It is important that we DO NOT save the - # id() of that temporary object in the memo - if self.__auto_slots__.has_dict: - del memo[id(state[-1])] + ans = self.__class__.__new__(self.__class__) + self.__deepcopy_state__(memo, ans) return ans + def __deepcopy_state__(self, memo, new_object): + """This implements the state copy from a source object to the new + instance in the deepcopy memo. + + This splits out the logic for actually duplicating the + object state from the "boilerplate" that creates a new + object and registers the object in the memo. This allows us + to create new schemes for duplicating / registering objects + that reuse all the logic here for copying the state. + + """ + # + # At this point we know we need to deepcopy this object. + # But, we can't do the "obvious", since this is a + # (partially) slot-ized class and the __dict__ structure is + # nonauthoritative: + # + # for key, val in self.__dict__.iteritems(): + # object.__setattr__(ans, key, deepcopy(val, memo)) + # + # Further, __slots__ is also nonauthoritative (this may be a + # derived class that also has a __dict__), or this may be a + # derived class with several layers of slots. So, we will + # piggyback on the __getstate__/__setstate__ logic and + # resort to partially "pickling" the object, deepcopying the + # state, and then restoring the copy into the new instance. + # + # [JDS 7/7/14] I worry about the efficiency of using both + # getstate/setstate *and* deepcopy, but we need to update + # fields like weakrefs correctly (and that logic is all in + # __getstate__/__setstate__). + # + # There is a particularly subtle bug with 'uncopyable' + # attributes: if the exception is thrown while copying a + # complex data structure, we can be in a state where objects + # have been created and assigned to the memo in the try + # block, but they haven't had their state set yet. When the + # exception moves us into the except block, we need to + # effectively "undo" those partially copied classes. The + # only way is to restore the memo to the state it was in + # before we started. We will make use of the knowledge that + # 1) memo entries are never reassigned during a deepcopy(), + # and 2) dict are ordered by insertion order in Python >= + # 3.7. As a result, we do not need to preserve the whole + # memo before calling __getstate__/__setstate__, and can get + # away with only remembering the number of items in the + # memo. + # + state = self.__getstate__() + try: + memo['__auto_slots__'].append(state) + except KeyError: + memo['__auto_slots__'] = [state] + + memo_size = len(memo) + try: + new_state = [fast_deepcopy(field, memo) for field in state] + except: + # We hit an error deepcopying the state. Attempt to + # reset things and try again, but in a more cautious + # manner. + # + # We want to remove any new entries added to the memo + # during the failed try above. + for _ in range(len(memo) - memo_size): + memo.popitem() + # + # Now we are going to continue on, but in a more + # cautious manner: we will clone entries field at a time + # so that we can get the most "complete" copy possible. + # + # Note: if has_dict, then __auto_slots__.slots will be 1 + # shorter than the state (the last element is the + # __dict__). Zip will ignore it. + _copier = getattr(self, '__deepcopy_field__', _deepcopier) + new_state = [ + _copier(value, memo, slot) + for slot, value in zip(self.__auto_slots__.slots, state) + ] + if self.__auto_slots__.has_dict: + new_state.append( + { + slot: _copier(value, memo, slot) + for slot, value in state[-1].items() + } + ) + new_object.__setstate__(new_state) + def __getstate__(self): """Generic implementation of `__getstate__` diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index 5717d3753ae..18bf18b7832 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -151,28 +151,6 @@ def __deepcopy__(self, memo): memo[id(self)] = self return self # - # At this point we know we need to deepcopy this component (and - # everything under it). We can't do the "obvious", since this - # is a (partially) slot-ized class and the __dict__ structure is - # nonauthoritative: - # - # for key, val in self.__dict__.iteritems(): - # object.__setattr__(ans, key, deepcopy(val, memo)) - # - # Further, __slots__ is also nonauthoritative (this may be a - # singleton component -- in which case it also has a __dict__). - # Plus, this may be a derived class with several layers of - # slots. So, we will piggyback on the __getstate__/__setstate__ - # logic amd resort to partially "pickling" the object, - # deepcopying the state, and then restoring the copy into - # the new instance. - # - # [JDS 7/7/14] I worry about the efficiency of using both - # getstate/setstate *and* deepcopy, but we need deepcopy to - # update the _parent refs appropriately, and since this is a - # slot-ized class, we cannot overwrite the __deepcopy__ - # attribute to prevent infinite recursion. - # # deepcopy() is an inherently recursive operation. This can # cause problems for highly interconnected Pyomo models (for # example, a time linked model where each time block has a @@ -183,85 +161,24 @@ def __deepcopy__(self, memo): # components / component datas, and NOT to attributes on the # components/datas. So, if we can first go through and stub in # all the objects that we will need to populate, and then go - # through and deepcopy them, then we can unroll the vast + # through and deepcopy them, we can unroll the vast # majority of the recursion. # component_list = [] self._create_objects_for_deepcopy(memo, component_list) # + # Note that self is now the first element of component_list + # # Now that we have created (but not populated) all the - # components that we expect to need, we can go through and - # populate all the components. + # components that we expect to need in hte memo, we can go + # through and populate all the components. # # The component_list is roughly in declaration order. This # means that it should be relatively safe to clone the contents # in the same order. # - # There is a particularly subtle bug with 'uncopyable' - # attributes: if the exception is thrown while copying a complex - # data structure, we can be in a state where objects have been - # created and assigned to the memo in the try block, but they - # haven't had their state set yet. When the exception moves us - # into the except block, we need to effectively "undo" those - # partially copied classes. The only way is to restore the memo - # to the state it was in before we started. We will make use of - # the knowledge that 1) memo entries are never reassigned during - # a deepcopy(), and 2) dict are ordered by insertion order in - # Python >= 3.7. As a result, we do not need to preserve the - # whole memo before calling __getstate__/__setstate__, and can - # get away with only remembering the number of items in the - # memo. - # - # Note that entering/leaving try-except contexts has a - # not-insignificant overhead. On the hope that the user wrote a - # sane (deepcopy-able) model, we will try to do everything in - # one try-except block. - # - try: - for i, comp in enumerate(component_list): - saved_memo = len(memo) - # Note: this implementation avoids deepcopying the - # temporary 'state' list, significantly speeding things - # up. - memo[id(comp)].__setstate__( - [fast_deepcopy(field, memo) for field in comp.__getstate__()] - ) - return memo[id(self)] - except: - pass - # - # We hit an error deepcopying a component. Attempt to reset - # things and try again, but in a more cautious manner (after - # all, if one component was not deepcopyable, it stands to - # reason that several others will not be either). - # - # We want to remove any new entries added to the memo during the - # failed try above. - # - for _ in range(len(memo) - saved_memo): - memo.popitem() - # - # Now we are going to continue on, but in a more cautious - # manner: we will clone entries field at a time so that we can - # get the most "complete" copy possible. - for comp in component_list[i:]: - state = comp.__getstate__() - # Note: if has_dict, then __auto_slots__.slots will be 1 - # shorter than the state (the last element is the __dict__). - # Zip will ignore it. - _deepcopy_field = comp._deepcopy_field - new_state = [ - _deepcopy_field(memo, slot, value) - for slot, value in zip(comp.__auto_slots__.slots, state) - ] - if comp.__auto_slots__.has_dict: - new_state.append( - { - slot: _deepcopy_field(memo, slot, value) - for slot, value in state[-1].items() - } - ) - memo[id(comp)].__setstate__(new_state) + for comp, new in component_list: + comp.__deepcopy_state__(memo, new) return memo[id(self)] def _create_objects_for_deepcopy(self, memo, component_list): @@ -271,7 +188,7 @@ def _create_objects_for_deepcopy(self, memo, component_list): component_list.append(self) return _ans - def _deepcopy_field(self, memo, slot_name, value): + def __deepcopy_field__(self, value, memo, slot_name): saved_memo = len(memo) try: return fast_deepcopy(value, memo) From ed97febdf39defa8759d73f55cf1141b4f3e6099 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:44:25 -0700 Subject: [PATCH 02/12] Update _create_objects_for_deepcopy to build a list of (old, new) --- pyomo/core/base/block.py | 8 ++++---- pyomo/core/base/component.py | 2 +- pyomo/core/base/indexed_component.py | 2 +- pyomo/core/base/param.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyomo/core/base/block.py b/pyomo/core/base/block.py index 8da80ec1163..656bf6008c4 100644 --- a/pyomo/core/base/block.py +++ b/pyomo/core/base/block.py @@ -1942,15 +1942,15 @@ def _create_objects_for_deepcopy(self, memo, component_list): _new = self.__class__.__new__(self.__class__) _ans = memo.setdefault(id(self), _new) if _ans is _new: - component_list.append(self) + component_list.append((self, _new)) # Blocks (and block-like things) need to pre-populate all # Components / ComponentData objects to help prevent # deepcopy() from violating the Python recursion limit. # This step is recursive; however, we do not expect "super # deep" Pyomo block hierarchies, so should be okay. - for comp in self._decl_order: - if comp[0] is not None: - comp[0]._create_objects_for_deepcopy(memo, component_list) + for comp, _ in self._decl_order: + if comp is not None: + comp._create_objects_for_deepcopy(memo, component_list) return _ans def private_data(self, scope=None): diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index 18bf18b7832..101099c13b9 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -185,7 +185,7 @@ def _create_objects_for_deepcopy(self, memo, component_list): _new = self.__class__.__new__(self.__class__) _ans = memo.setdefault(id(self), _new) if _ans is _new: - component_list.append(self) + component_list.append((self, _new)) return _ans def __deepcopy_field__(self, value, memo, slot_name): diff --git a/pyomo/core/base/indexed_component.py b/pyomo/core/base/indexed_component.py index 2991057f8d2..023c055f6c8 100644 --- a/pyomo/core/base/indexed_component.py +++ b/pyomo/core/base/indexed_component.py @@ -340,7 +340,7 @@ def _create_objects_for_deepcopy(self, memo, component_list): _new = self.__class__.__new__(self.__class__) _ans = memo.setdefault(id(self), _new) if _ans is _new: - component_list.append(self) + component_list.append((self, _new)) # For indexed components, we will pre-emptively clone all # component data objects as well (as those are the objects # that will be referenced by things like expressions). It diff --git a/pyomo/core/base/param.py b/pyomo/core/base/param.py index 70ceaa81895..45f25d2748d 100644 --- a/pyomo/core/base/param.py +++ b/pyomo/core/base/param.py @@ -996,7 +996,7 @@ def _create_objects_for_deepcopy(self, memo, component_list): _new = self.__class__.__new__(self.__class__) _ans = memo.setdefault(id(self), _new) if _ans is _new: - component_list.append(self) + component_list.append((self, _new)) return _ans # Because CP supports indirection [the ability to index objects by From 20297887fb7d58830c7dd165e68b7e2b9d17f80b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:48:24 -0700 Subject: [PATCH 03/12] Minor performance improvements --- pyomo/common/autoslots.py | 38 +++++++++++++++++++++------- pyomo/core/base/indexed_component.py | 8 +++--- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/pyomo/common/autoslots.py b/pyomo/common/autoslots.py index 3b3fbf7e7c1..bbe7b65dc1f 100644 --- a/pyomo/common/autoslots.py +++ b/pyomo/common/autoslots.py @@ -9,22 +9,23 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import collections import types -from collections import namedtuple from copy import deepcopy from weakref import ref as _weakref_ref -_autoslot_info = namedtuple( +_autoslot_info = collections.namedtuple( '_autoslot_info', ['has_dict', 'slots', 'slot_mappers', 'field_mappers'] ) def _deepcopy_tuple(obj, memo, _id): ans = [] + _append = ans.append unchanged = True for item in obj: new_item = fast_deepcopy(item, memo) - ans.append(new_item) + _append(new_item) if new_item is not item: unchanged = False if unchanged: @@ -46,22 +47,43 @@ def _deepcopy_tuple(obj, memo, _id): def _deepcopy_list(obj, memo, _id): # Two steps here because a list can include itself memo[_id] = ans = [] - ans.extend(fast_deepcopy(x, memo) for x in obj) + _append = ans.append + for x in obj: + _append(fast_deepcopy(x, memo)) return ans def _deepcopy_dict(obj, memo, _id): # Two steps here because a dict can include itself memo[_id] = ans = {} + _setter = ans.__setitem__ for key, val in obj.items(): - ans[fast_deepcopy(key, memo)] = fast_deepcopy(val, memo) + _setter(fast_deepcopy(key, memo), fast_deepcopy(val, memo)) return ans -def _deepcopier(obj, memo, _id): +def _deepcopy_dunder_deepcopy(obj, memo, _id): + ans = memo[_id] = obj.__deepcopy__(memo) + return ans + + +def _deepcopy(obj, memo, _id): return deepcopy(obj, memo) +class _DeepcopyDispatcher(collections.defaultdict): + def __missing__(self, key): + if hasattr(key, '__deepcopy__'): + ans = _deepcopy_dunder_deepcopy + else: + ans = _deepcopy + self[key] = ans + return ans + + +_deepcopy_dispatcher = _DeepcopyDispatcher( + None, {tuple: _deepcopy_tuple, list: _deepcopy_list, dict: _deepcopy_dict} +) _atomic_types = { int, float, @@ -76,8 +98,6 @@ def _deepcopier(obj, memo, _id): types.FunctionType, } -_deepcopy_mapper = {tuple: _deepcopy_tuple, list: _deepcopy_list, dict: _deepcopy_dict} - def fast_deepcopy(obj, memo): """A faster implementation of copy.deepcopy() @@ -94,7 +114,7 @@ def fast_deepcopy(obj, memo): if _id in memo: return memo[_id] else: - return _deepcopy_mapper.get(obj.__class__, _deepcopier)(obj, memo, _id) + return _deepcopy_dispatcher[obj.__class__](obj, memo, _id) class AutoSlots(type): diff --git a/pyomo/core/base/indexed_component.py b/pyomo/core/base/indexed_component.py index 023c055f6c8..e6393f580af 100644 --- a/pyomo/core/base/indexed_component.py +++ b/pyomo/core/base/indexed_component.py @@ -354,10 +354,12 @@ def _create_objects_for_deepcopy(self, memo, component_list): # for the _data dict, we can effectively "deepcopy" it # right now (almost for free!) _src = self._data - memo[id(_src)] = _new._data = _data = _src.__class__() + memo[id(_src)] = _new._data = _src.__class__() + _setter = _new._data.__setitem__ for idx, obj in _src.items(): - _data[fast_deepcopy(idx, memo)] = obj._create_objects_for_deepcopy( - memo, component_list + _setter( + fast_deepcopy(idx, memo), + obj._create_objects_for_deepcopy(memo, component_list), ) return _ans From 33247226c40d734294e21810fe68bd182893ba2e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:49:19 -0700 Subject: [PATCH 04/12] bugfix resolving scope for floating components --- pyomo/core/base/component.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index 101099c13b9..a3610099d7d 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -120,8 +120,34 @@ def __deepcopy__(self, memo): tmp = self.parent_block() # "Floating" components should be in scope by default (we # will handle 'global' components like GlobalSets in the - # components) - _in_scope = tmp is None + # components). This ensures that things like set operators + # on Abstract set objects are correctly cloned. For + # example, consider an abstract indexed model component + # whose domain is specified by a Set expression: + # + # def x_init(m,i): + # if i == 2: + # return Set.Skip + # else: + # return [] + # m.x = Set( [1,2], + # domain={1: m.A*m.B, 2: m.A*m.A}, + # initialize=x_init ) + # + # We do not want to automatically add all the Set operators + # to the model at declaration time, as m.x[2] is never + # actually created. Plus, doing so would require complex + # parsing of the initializers. BUT, we need to ensure that + # the operators are deepcopied, otherwise when the model is + # cloned before construction the operators will still refer + # to the sets on the original abstract model (in particular, + # the Set x will have an unknown dimen). + # + # The solution is to automatically clone all floating + # components, except for Models (i.e., top-level BlockData + # have no parent and technically "float") + _in_scope = tmp is None and self is not self.model() + # # Note: normally we would need to check that tmp does not # end up being None. However, since clone() inserts # id(None) into the __block_scope__ dictionary, we are safe From 78a8c03079fc94f21623e13e8bfb64f77dcd1cf5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:49:56 -0700 Subject: [PATCH 05/12] Update warning message to match current implementation --- pyomo/core/base/component.py | 14 +++++--------- pyomo/core/tests/unit/test_block.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index a3610099d7d..9eb407e029e 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -227,15 +227,11 @@ def __deepcopy_field__(self, value, memo, slot_name): # warn the user if '__block_scope__' not in memo: logger.warning( - """ - Uncopyable field encountered when deep - copying outside the scope of Block.clone(). - There is a distinct possibility that the new - copy is not complete. To avoid this - situation, either use Block.clone() or set - 'paranoid' mode by adding '__paranoid__' == - True to the memo before calling - copy.deepcopy.""" + "Uncopyable field encountered when deep " + "copying Pyomo components outside the scope of " + "Block.clone(). There is a distinct possibility " + "that the new copy is not complete. To avoid " + "this situation, please use Block.clone()" ) if self.model() is self: what = 'Model' diff --git a/pyomo/core/tests/unit/test_block.py b/pyomo/core/tests/unit/test_block.py index 3d578f7dc88..36e779943a3 100644 --- a/pyomo/core/tests/unit/test_block.py +++ b/pyomo/core/tests/unit/test_block.py @@ -2525,7 +2525,7 @@ def __deepcopy__(bogus): "'unknown' contains an uncopyable field 'bad1'", OUTPUT.getvalue() ) self.assertIn("'b' contains an uncopyable field 'bad2'", OUTPUT.getvalue()) - self.assertIn("'__paranoid__'", OUTPUT.getvalue()) + self.assertIn("outside the scope of Block.clone()", OUTPUT.getvalue()) self.assertTrue(hasattr(m.b, 'bad2')) self.assertIsNotNone(m.b.bad2) self.assertTrue(hasattr(nb, 'bad2')) From 872bb3d0502fa97e628fdebb10ac4582a78eaa7f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:50:14 -0700 Subject: [PATCH 06/12] NFC: update comments --- pyomo/core/base/component.py | 2 +- pyomo/core/base/indexed_component.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index 9eb407e029e..2886dd9d988 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -112,7 +112,7 @@ def __deepcopy__(self, memo): # Templates (and the corresponding _GetItemExpression object), # expressions can refer to container (non-Simple) components, so # we need to override __deepcopy__ for both Component and - # ComponentData. + # ComponentData (so we put it here on ComponentBase). # if '__block_scope__' in memo: _scope = memo['__block_scope__'] diff --git a/pyomo/core/base/indexed_component.py b/pyomo/core/base/indexed_component.py index e6393f580af..8a19c35e3e0 100644 --- a/pyomo/core/base/indexed_component.py +++ b/pyomo/core/base/indexed_component.py @@ -751,7 +751,7 @@ def __delitem__(self, index): def _construct_from_rule_using_setitem(self): if self._rule is None: return - index = None + index = None # set so it is defined for scalars for `except:` below rule = self._rule block = self.parent_block() try: From 666af6c49ffe100c29c0c3cc7c5d8e972f7c29f1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:50:35 -0700 Subject: [PATCH 07/12] Move NumericRange onto AutoSlots --- pyomo/core/base/range.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/pyomo/core/base/range.py b/pyomo/core/base/range.py index acb004e3b10..2a959a302e1 100644 --- a/pyomo/core/base/range.py +++ b/pyomo/core/base/range.py @@ -12,6 +12,7 @@ import math from collections.abc import Sequence +from pyomo.common.autoslots import AutoSlots from pyomo.common.numeric_types import check_if_numeric_type try: @@ -33,7 +34,7 @@ class RangeDifferenceError(ValueError): pass -class NumericRange(object): +class NumericRange(AutoSlots.Mixin): """A representation of a numeric range. This class represents a contiguous range of numbers. The class @@ -126,29 +127,6 @@ def __init__(self, start, end, step, closed=(True, True)): " Discrete ranges must be closed." % (self, self.closed) ) - def __getstate__(self): - """ - Retrieve the state of this object as a dictionary. - - This method must be defined because this class uses slots. - """ - state = {} # super(NumericRange, self).__getstate__() - for i in NumericRange.__slots__: - state[i] = getattr(self, i) - return state - - def __setstate__(self, state): - """ - Set the state of this object using values from a state dictionary. - - This method must be defined because this class uses slots. - """ - for key, val in state.items(): - # Note: per the Python data model docs, we explicitly - # set the attribute using object.__setattr__() instead - # of setting self.__dict__[key] = val. - object.__setattr__(self, key, val) - def __str__(self): if not self.isdiscrete(): return "%s%s..%s%s" % ( From fcb0a1791c5db42bb5667c4fe834e27ca9205cdc Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:51:02 -0700 Subject: [PATCH 08/12] Remove unnecessary implementation of SetOperator.__deepcopy__ --- pyomo/core/base/set.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 19db411f119..e420bae884c 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -3450,40 +3450,6 @@ def __str__(self): return self.name return self._expression_str() - def __deepcopy__(self, memo): - # SetOperators form an expression system. As we allow operators - # on abstract Set objects, it is important to *always* deepcopy - # SetOperators that have not been assigned to a Block. For - # example, consider an abstract indexed model component whose - # domain is specified by a Set expression: - # - # def x_init(m,i): - # if i == 2: - # return Set.Skip - # else: - # return [] - # m.x = Set( [1,2], - # domain={1: m.A*m.B, 2: m.A*m.A}, - # initialize=x_init ) - # - # We do not want to automatically add all the Set operators to - # the model at declaration time, as m.x[2] is never actually - # created. Plus, doing so would require complex parsing of the - # initializers. BUT, we need to ensure that the operators are - # deepcopied, otherwise when the model is cloned before - # construction the operators will still refer to the sets on the - # original abstract model (in particular, the Set x will have an - # unknown dimen). - # - # Our solution is to cause SetOperators to be automatically - # cloned if they haven't been assigned to a block. - if '__block_scope__' in memo: - if self.parent_block() is None: - # Hijack the block scope rules to cause this object to - # be deepcopied. - memo['__block_scope__'][id(self)] = True - return super(SetOperator, self).__deepcopy__(memo) - def _expression_str(self): _args = [] for arg in self._sets: From c68ce3c3a56ef5bf476865f80dbcbd89a4c58e39 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 09:58:27 -0700 Subject: [PATCH 09/12] NFC: additional comments --- pyomo/common/autoslots.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyomo/common/autoslots.py b/pyomo/common/autoslots.py index bbe7b65dc1f..29a3d46c9fb 100644 --- a/pyomo/common/autoslots.py +++ b/pyomo/common/autoslots.py @@ -342,6 +342,12 @@ def __deepcopy_state__(self, memo, new_object): # memo. # state = self.__getstate__() + # It is important to keep this temporary state alive (which + # in turn keeps things like the temporary fields dict alive) + # until after deepcopy is finished in order to prevent + # accidentally recycling id()'s for temporary objects that + # were recorded in the memo. We will follow the pattern + # used by copy._keep_alive(): try: memo['__auto_slots__'].append(state) except KeyError: From dea77e58aa691f896fa032c0baf2e16052e355d5 Mon Sep 17 00:00:00 2001 From: Bethany Nicholson Date: Wed, 13 Nov 2024 10:11:42 -0700 Subject: [PATCH 10/12] NFC: Typo fix in comment --- pyomo/core/base/component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py index 2886dd9d988..a5763264b14 100644 --- a/pyomo/core/base/component.py +++ b/pyomo/core/base/component.py @@ -196,7 +196,7 @@ def __deepcopy__(self, memo): # Note that self is now the first element of component_list # # Now that we have created (but not populated) all the - # components that we expect to need in hte memo, we can go + # components that we expect to need in the memo, we can go # through and populate all the components. # # The component_list is roughly in declaration order. This From 729a4332404ebeffe70b7e08b189dc66b2a84d4e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 13 Nov 2024 12:35:17 -0700 Subject: [PATCH 11/12] Track change in helper function name --- pyomo/common/autoslots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/common/autoslots.py b/pyomo/common/autoslots.py index 29a3d46c9fb..40f74c6175e 100644 --- a/pyomo/common/autoslots.py +++ b/pyomo/common/autoslots.py @@ -373,7 +373,7 @@ def __deepcopy_state__(self, memo, new_object): # Note: if has_dict, then __auto_slots__.slots will be 1 # shorter than the state (the last element is the # __dict__). Zip will ignore it. - _copier = getattr(self, '__deepcopy_field__', _deepcopier) + _copier = getattr(self, '__deepcopy_field__', _deepcopy) new_state = [ _copier(value, memo, slot) for slot, value in zip(self.__auto_slots__.slots, state) From 57e60824938e7efdf561278095ea031b44aaeb2a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 14 Nov 2024 08:04:04 -0700 Subject: [PATCH 12/12] Add documentation around the assumption of source object persistence --- pyomo/common/autoslots.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pyomo/common/autoslots.py b/pyomo/common/autoslots.py index 40f74c6175e..a5ba44818c3 100644 --- a/pyomo/common/autoslots.py +++ b/pyomo/common/autoslots.py @@ -107,6 +107,29 @@ def fast_deepcopy(obj, memo): deepcopy that provides special handling to circumvent some of the slowest parts of deepcopy(). + Note + ---- + + This implementation is not as aggressive about keeping the copied + state alive until the end of the deepcopy operation. In particular, + the ``dict``, ``list`` and ``tuple`` handlers do not register their + source objects with the memo. This is acceptable, as + fast_deepcopy() is only called in situations where we are ensuring + that the source object will persist: + + - :meth:`AutoSlots.__deepcopy_state__` explicitly preserved the + source state + - :meth:`Component.__deepcopy_field__` is only called by + :meth:`AutoSlots.__deepcopy_state__` - + - :meth:`IndexedComponent._create_objects_for_deepcopy` is + deepcopying the raw keys from the source ``_data`` dict (which is + not a temporary object and will persist) + + If other consumers wish to make use of this function (e.g., within + their implementation of ``__deepcopy__``), they must remember that + they are responsible to ensure that any temporary source ``obj`` + persists. + """ if obj.__class__ in _atomic_types: return obj