From ba6204f3de7570319994873305cabf3e8e700974 Mon Sep 17 00:00:00 2001 From: "Joel Z. Leibo" Date: Mon, 5 Aug 2024 01:50:31 -0700 Subject: [PATCH] Port two more legacy reflection components to the v2 entity component system. PiperOrigin-RevId: 659472264 Change-Id: Ie6b3865a41116fc48887832f8d0205cec2d7aa89 --- .../contrib/components/agent/v2/__init__.py | 18 ++ .../components/agent/v2/affect_reflection.py | 165 +++++++++++++ .../agent/v2/dialectical_reflection.py | 222 ++++++++++++++++++ 3 files changed, 405 insertions(+) create mode 100644 concordia/contrib/components/agent/v2/__init__.py create mode 100644 concordia/contrib/components/agent/v2/affect_reflection.py create mode 100644 concordia/contrib/components/agent/v2/dialectical_reflection.py diff --git a/concordia/contrib/components/agent/v2/__init__.py b/concordia/contrib/components/agent/v2/__init__.py new file mode 100644 index 00000000..7dfbe76e --- /dev/null +++ b/concordia/contrib/components/agent/v2/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2023 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of components contributed by users.""" + +from concordia.contrib.components.agent.v2 import affect_reflection +from concordia.contrib.components.agent.v2 import dialectical_reflection diff --git a/concordia/contrib/components/agent/v2/affect_reflection.py b/concordia/contrib/components/agent/v2/affect_reflection.py new file mode 100644 index 00000000..1f31cd77 --- /dev/null +++ b/concordia/contrib/components/agent/v2/affect_reflection.py @@ -0,0 +1,165 @@ +# Copyright 2023 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An agent reflects on how they are currently feeling.""" + +from collections.abc import Mapping +import types + +from concordia.clocks import game_clock +from concordia.components.agent.v2 import action_spec_ignored +from concordia.components.agent.v2 import memory_component +from concordia.document import interactive_document +from concordia.language_model import language_model +from concordia.memory_bank import legacy_associative_memory +from concordia.typing import entity_component +from concordia.typing import logging + +DEFAULT_PRE_ACT_KEY = '\nAffective reflections' + +_ASSOCIATIVE_RETRIEVAL = legacy_associative_memory.RetrieveAssociative() + + +class AffectReflection(action_spec_ignored.ActionSpecIgnored): + """Implements a reflection component taking into account the agent's affect. + + This component recalls memories based salient recent feelings, concepts, and + events. It then tries to infer high-level insights based on the memories it + retrieved. This makes its output depend both on recent events and on the + agent's past experience in life. + """ + + def __init__( + self, + model: language_model.LanguageModel, + clock: game_clock.MultiIntervalClock, + memory_component_name: str = ( + memory_component.DEFAULT_MEMORY_COMPONENT_NAME + ), + components: Mapping[ + entity_component.ComponentName, str + ] = types.MappingProxyType({}), + num_salient_to_retrieve: int = 20, + num_questions_to_consider: int = 3, + num_to_retrieve_per_question: int = 10, + pre_act_key: str = DEFAULT_PRE_ACT_KEY, + logging_channel: logging.LoggingChannel = logging.NoOpLoggingChannel, + ): + """Generates affect reflection based on recent and salient memories. + + Args: + model: a language model + clock: the game clock is needed to know when is the current time + memory_component_name: The name of the memory component from which to + retrieve recent memories. + components: The components to consider when reflecting. This is a mapping + of the component name to a label to use in the prompt. + num_salient_to_retrieve: retrieve this many salient memories. + num_questions_to_consider: how many questions to ask self. + num_to_retrieve_per_question: how many memories to retrieve per question. + pre_act_key: Prefix to add to the output of the component when called + in `pre_act`. + logging_channel: The channel to use for debug logging. + """ + super().__init__(pre_act_key) + self._model = model + self._memory_component_name = memory_component_name + self._components = dict(components) + self._clock = clock + self._num_salient_to_retrieve = num_salient_to_retrieve + self._num_questions_to_consider = num_questions_to_consider + self._num_to_retrieve_per_question = num_to_retrieve_per_question + self._logging_channel = logging_channel + self._previous_pre_act_value = '' + + def _make_pre_act_value(self) -> str: + agent_name = self.get_entity().name + context = '\n'.join([ + f"{agent_name}'s" + f' {prefix}:\n{self.get_named_component_pre_act_value(key)}' + for key, prefix in self._components.items() + ]) + salience_chain_of_thought = interactive_document.InteractiveDocument( + self._model) + + query = f'salient event, period, feeling, or concept for {agent_name}' + timed_query = f'[{self._clock.now()}] {query}' + + memory = self.get_entity().get_component( + self._memory_component_name, + type_=memory_component.MemoryComponent) + mem_retrieved = '\n'.join( + [mem.text for mem in memory.retrieve( + query=timed_query, + scoring_fn=legacy_associative_memory.RetrieveAssociative( + use_recency=True, add_time=True + ), + limit=self._num_salient_to_retrieve)] + ) + + question_list = [] + + questions = salience_chain_of_thought.open_question( + ( + f'Recent feelings: {self._previous_pre_act_value} \n' + + f"{agent_name}'s relevant memory:\n" + + f'{mem_retrieved}\n' + + f'Current time: {self._clock.now()}\n' + + '\nGiven the thoughts and beliefs above, what are the ' + + f'{self._num_questions_to_consider} most salient high-level '+ + f'questions that can be answered about what {agent_name} ' + + 'might be feeling about the current moment?'), + answer_prefix='- ', + max_tokens=3000, + terminators=(), + ).split('\n') + + question_related_mems = [] + for question in questions: + question_list.append(question) + question_related_mems = [mem.text for mem in memory.retrieve( + query=agent_name, + scoring_fn=legacy_associative_memory.RetrieveAssociative( + use_recency=False, add_time=True + ), + limit=self._num_to_retrieve_per_question)] + insights = [] + question_related_mems = '\n'.join(question_related_mems) + + chain_of_thought = interactive_document.InteractiveDocument(self._model) + insight = chain_of_thought.open_question( + f'Selected memories:\n{question_related_mems}\n' + + f'Recent feelings: {self._previous_pre_act_value} \n\n' + + 'New context:\n' + context + '\n' + + f'Current time: {self._clock.now()}\n' + + 'What high-level insight can be inferred from the above ' + + f'statements about what {agent_name} might be feeling ' + + 'in the current moment?', + max_tokens=2000, terminators=(),) + insights.append(insight) + + result = '\n'.join(insights) + + self._previous_pre_act_value = result + + self._logging_channel({ + 'Key': self.get_pre_act_key(), + 'Value': result, + 'Salience chain of thought': ( + salience_chain_of_thought.view().text().splitlines()), + 'Chain of thought': ( + chain_of_thought.view().text().splitlines()), + }) + + return result diff --git a/concordia/contrib/components/agent/v2/dialectical_reflection.py b/concordia/contrib/components/agent/v2/dialectical_reflection.py new file mode 100644 index 00000000..ef1fbaf6 --- /dev/null +++ b/concordia/contrib/components/agent/v2/dialectical_reflection.py @@ -0,0 +1,222 @@ +# Copyright 2023 DeepMind Technologies Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Agent component for dialectical reflection.""" + +import datetime +import types +from typing import Callable, Mapping + +from concordia.components.agent.v2 import action_spec_ignored +from concordia.components.agent.v2 import memory_component +from concordia.document import interactive_document +from concordia.language_model import language_model +from concordia.memory_bank import legacy_associative_memory +from concordia.typing import entity_component +from concordia.typing import logging + +DEFAULT_PRE_ACT_KEY = '\nReflection' + + +def concat_interactive_documents( + doc_a: interactive_document.InteractiveDocument, + doc_b: interactive_document.InteractiveDocument, +) -> interactive_document.InteractiveDocument: + """Concatenates two interactive documents. Returns a copy.""" + copied_doc = doc_a.copy() + copied_doc.extend(doc_b.contents()) + return copied_doc + + +class DialecticalReflection(action_spec_ignored.ActionSpecIgnored): + """Make new thoughts from memories by thesis-antithesis-synthesis.""" + + def __init__( + self, + model: language_model.LanguageModel, + memory_component_name: str = ( + memory_component.DEFAULT_MEMORY_COMPONENT_NAME), + intuition_components: Mapping[ + entity_component.ComponentName, str + ] = types.MappingProxyType({}), + thinking_components: Mapping[ + entity_component.ComponentName, str + ] = types.MappingProxyType({}), + clock_now: Callable[[], datetime.datetime] | None = None, + num_memories_to_retrieve: int = 5, + topic: action_spec_ignored.ActionSpecIgnored | None = None, + pre_act_key: str = DEFAULT_PRE_ACT_KEY, + logging_channel: logging.LoggingChannel = logging.NoOpLoggingChannel, + ): + """Initializes the DialecticReflection component. + + Args: + model: The language model to use. + memory_component_name: The name of the memory component from which to + retrieve memories. + intuition_components: Components to condition thesis generation. + thinking_components: Components to condition synthesis of thesis and + antithesis. + clock_now: callback function to get the current time in the game world. + num_memories_to_retrieve: The number of memories to retrieve. + topic: a component to represent the topic of theoretical reflection. + pre_act_key: Prefix to add to the output of the component when called + in `pre_act`. + logging_channel: The channel to use for debug logging. + """ + super().__init__(pre_act_key) + self._model = model + self._memory_component_name = memory_component_name + self._intuition_components = dict(intuition_components) + self._thinking_components = dict(thinking_components) + self._clock_now = clock_now + self._num_memories_to_retrieve = num_memories_to_retrieve + self._topic_component = topic + + self._logging_channel = logging_channel + + def _make_pre_act_value(self) -> str: + agent_name = self.get_entity().name + memory = self.get_entity().get_component( + self._memory_component_name, + type_=memory_component.MemoryComponent) + + # The following query looks for conversations using the fact that their + # observations are preceded by ' -- "'. + prethoughts = [ + mem.text + for mem in memory.retrieve( + query=' -- "', + scoring_fn=legacy_associative_memory.RetrieveAssociative( + use_recency=True, add_time=False + ), + limit=self._num_memories_to_retrieve, + ) + ] + + # The following query looks for memories of reading and learning. + prethoughts += [ + mem.text + for mem in memory.retrieve( + query=('book, article, read, idea, concept, study, learn, ' + 'research, theory'), + scoring_fn=legacy_associative_memory.RetrieveAssociative( + use_recency=False, add_time=False + ), + limit=self._num_memories_to_retrieve, + ) + ] + + if self._topic_component is not None: + prethoughts += [ + mem.text + for mem in memory.retrieve( + query=self._topic_component.get_pre_act_value(), + scoring_fn=legacy_associative_memory.RetrieveAssociative( + use_recency=False, add_time=False + ), + limit=self._num_memories_to_retrieve, + ) + ] + + prethoughts = '-' + '\n-'.join(prethoughts) + '\n' + + if self._intuition_components: + prethoughts += '-' + '\n'.join([ + f"-{agent_name}'s" + f' {prefix}:\n{self.get_named_component_pre_act_value(key)}' + for key, prefix in self._intuition_components.items() + ]) + + # Apply the 'thesis->antithesis->synthesis' method to generate insight. + thesis_chain = interactive_document.InteractiveDocument(self._model) + thesis_chain.statement(f'* The intuition of {agent_name} *\n') + thesis_chain.statement( + (f'For {agent_name}, all the following statements feel ' + + f'connected:\nStatements:\n{prethoughts}')) + + thesis_question = ( + f'In light of the information above, what may {agent_name} ' + + 'infer') + if self._topic_component: + thesis_question += ( + f' about {self._topic_component.get_pre_act_value()}?') + else: + thesis_question += '?' + + thesis = thesis_chain.open_question( + thesis_question, + max_tokens=1200, + terminators=(), + ) + + synthesis_chain = interactive_document.InteractiveDocument(self._model) + synthesis_chain.statement(f'* The mind of {agent_name} *\n') + synthesis_chain.statement('\n'.join([ + f"{agent_name}'s" + f' {prefix}:\n{self.get_named_component_pre_act_value(key)}' + for key, prefix in self._thinking_components.items() + ])) + synthesis_chain.statement( + (f'\n{agent_name} is applying the dialectical mode of reasoning' + + '.\nThis involves a thesis-antithesis-synthesis pattern of logic.')) + _ = synthesis_chain.open_question( + question=('Given all the information above, what thesis would ' + f'{agent_name} consider next?'), + forced_response=thesis) + _ = synthesis_chain.open_question( + question=( + f'How would {agent_name} describe the antithesis of ' + + 'the aforementioned thesis?' + ), + max_tokens=2000, + terminators=(), + ) + _ = synthesis_chain.open_question( + question=( + f'How would {agent_name} synthesize the thesis with ' + + 'its antithesis in a novel and insightful way?' + ), + answer_prefix=( + f'{agent_name} would think step by step, and start by ' + + 'pointing out that ' + ), + max_tokens=2000, + terminators=(), + ) + synthesis = synthesis_chain.open_question( + question=( + f'How might {agent_name} summarize the synthesis ' + + 'above as a bold new argument?' + ), + answer_prefix=( + f"In {agent_name}'s view, the full argument " + + 'is complex but the TLDR is that ' + ), + max_tokens=1000, + terminators=('\n',), + ) + synthesis = synthesis[0].lower() + synthesis[1:] + result = f'{agent_name} just realized that {synthesis}' + + memory.add(f'[idea] {synthesis}', metadata={}) + + self._logging_channel({ + 'Key': self.get_pre_act_key(), + 'Value': result, + 'Chain of thought': concat_interactive_documents( + thesis_chain, synthesis_chain).view().text().splitlines(), + }) + + return result