From ec72e2454e20c1f9306d1ea369634c7c576bb502 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:36:26 -0700 Subject: [PATCH] Update for agent. --- scripts/pycharm-launch.py | 30 +++-- services/core/IEEE_2030_5/AGENTDEMO.md | 63 +++++++--- .../core/IEEE_2030_5/demo/inverter_runner.py | 30 +++-- services/core/IEEE_2030_5/demo/server.yml | 43 +++---- services/core/IEEE_2030_5/example.config.yml | 8 +- .../core/IEEE_2030_5/ieee_2030_5/agent.py | 29 +++-- .../core/IEEE_2030_5/ieee_2030_5/client.py | 33 ++--- .../core/IEEE_2030_5/tests/test_client.py | 2 + volttron/platform/agent/utils.py | 113 +++++++++--------- 9 files changed, 208 insertions(+), 143 deletions(-) create mode 100644 services/core/IEEE_2030_5/tests/test_client.py diff --git a/scripts/pycharm-launch.py b/scripts/pycharm-launch.py index e32787fcdf..e2663799c6 100644 --- a/scripts/pycharm-launch.py +++ b/scripts/pycharm-launch.py @@ -67,7 +67,7 @@ def write_required_statement(out=sys.stderr): out.write("""Required Environment Variables - AGENT_VIP_IDENTITY - Required + AGENT_VIP_IDENTITY - Required Optional Environmental Variables AGENT_CONFIG - Set to /config by default VOLTTRON_HOME - Set to ~/.volttron by default @@ -113,9 +113,14 @@ def write_required_statement(out=sys.stderr): sys.exit(10) if agent_identity: - new_dir = os.path.join(volttron_home, 'keystores', agent_identity) - if not os.path.exists(new_dir): - os.makedirs(new_dir) + agent_keystore_dir = os.path.join(volttron_home, 'keystores', agent_identity) + if os.path.exists(agent_keystore_dir): + with open(agent_keystore_dir + '/keystore.json', 'r') as fin: + json_obj = jsonapi.loads(fin.read()) + pubkey = json_obj['public'] + secret = json_obj['secret'] + else: + os.makedirs(agent_keystore_dir) try: output = subprocess.check_output(['vctl', 'auth', 'keypair'], env=os.environ.copy(), @@ -126,16 +131,17 @@ def write_required_statement(out=sys.stderr): sys.stderr.write("Call was:\n\tvctl auth keypair\n") sys.stderr.write("Output of command: {}".format(e.output)) sys.stderr.write("Your environment might not be setup correctly!") - os.rmdir(new_dir) + os.rmdir(agent_keystore_dir) write_required_statement() sys.exit(20) else: - keystore_file = os.path.join(new_dir, "keystore.json") + keystore_file = os.path.join(agent_keystore_dir, "keystore.json") json_obj = jsonapi.loads(output) with open(keystore_file, 'w') as fout: fout.write(output) pubkey = json_obj['public'] + secret = json_obj['secret'] try: params = [ 'vctl', 'auth', 'add', '--credentials', "{}".format(pubkey), '--user_id', @@ -149,13 +155,23 @@ def write_required_statement(out=sys.stderr): except subprocess.CalledProcessError as e: sys.stderr.write(str(e)) sys.stderr.write("Command returned following output: {}".format(e.output)) - shutil.rmtree(new_dir) + shutil.rmtree(agent_keystore_dir) sys.stderr.write("Couldn't authenticate agent id: {}\n".format(agent_identity)) sys.stderr.write("Call was: {}\n".format(params)) sys.stderr.write("Your environment might not be setup correctly!") write_required_statement() sys.exit(20) +if not pubkey or not secret: + raise ValueError(f"Missing publickey or secretkey for {agent_identity}") + +# Populate the serverkey +with open(os.path.join(volttron_home, "keystore"), 'r') as fin: + json_obj = jsonapi.loads(fin.read()) + os.environ['VOLTTRON_SERVERKEY'] = json_obj['public'] + +os.environ['AGENT_PUBLICKEY'] = pubkey +os.environ['AGENT_SECRETKEY'] = secret if not parsed.silence: sys.stdout.write("For your information (-s) to not print this message.") write_required_statement(sys.stdout) diff --git a/services/core/IEEE_2030_5/AGENTDEMO.md b/services/core/IEEE_2030_5/AGENTDEMO.md index 56dbcbd088..3f61d568bf 100644 --- a/services/core/IEEE_2030_5/AGENTDEMO.md +++ b/services/core/IEEE_2030_5/AGENTDEMO.md @@ -1,16 +1,31 @@ -# 2030.5 Agent Demo -This readme walks through a demo of an inverter publishing points to the VOLTTRON message bus where the 2030.5 agent will receive it. The 2030.5 agent will then create MirrorUsagePoints and POST MirrorMeterReadings to the 2030.5 server. In addition, the demo will also allow the user to create a DERControl event. During the activation of the event, the agent will log messages to the 2030.5 server. +# 2030.5 Agent Demo # -The demo will require three terminal windows to be open. The first will be for executing the main VOLTTRON process. The second will be for executing the 2030.5 server. The third will be for executing the agent demo through a web interface. For the rest of this document, VOLTTRON_ROOT is assumed to be where one has cloned the volttron repository. +This readme walks through a demo of an inverter publishing points to the +VOLTTRON message bus where the 2030.5 agent will receive it. The +2030.5 agent will then create MirrorUsagePoints and POST +MirrorMeterReadings to the 2030.5 server. In addition, +the demo will also allow the user to create a DERControl event. During the +activation of the event, the agent will log messages to the 2030.5 server. -First configure the server then the VOLTTRON instance and findally the web based demo. +The demo will require three terminal windows to be open. The first will be for +executing the main VOLTTRON process. The second will be for executing the +2030.5 server. The third will be for executing the agent demo through a +web interface. For the rest of this document, VOLTTRON_ROOT is assumed to be +where one has cloned the volttron repository. -## 2030.5 Server +First configure the server then the VOLTTRON instance and findally the web based +demo. -For the server software we will be using a server from a team at PNNL. The GridAPPS-D team has created a 2030.5 server and it is available at . The source code is still in a private repo, but will be released at some time in the future. +## 2030.5 Server ## -1. In a new terminal create a new virtual environment and install the gridappsd server. **This directory should be outside the main volttron tree** +For the server software we will be using a server from a team at PNNL. The +GridAPPS-D team has created a 2030.5 server and it is available at +. The source code is still in +a private repo, but will be released at some time in the future. + +1. In a new terminal create a new virtual environment and install the + gridappsd server. **This directory should be outside the main volttron tree** ```bash > mkdir 2030_5_server @@ -24,7 +39,10 @@ For the server software we will be using a server from a team at PNNL. The Grid (serverenv)> pip install gridappsd-2030-5 ``` -1. The server is now installed. Next copy the openssl.cnf and server.yml file from the $VOLTTRON_ROOT/services/core/IEEE_2030_5/demo directory to the newly created 2030_5_server directory. After copying the current directory should look like the following. +1. The server is now installed. Next copy the openssl.cnf and server.yml file + from the $VOLTTRON_ROOT/services/core/IEEE_2030_5/demo directory to the newly + created 2030_5_server directory. After copying the current directory should + look like the following. ```bash (serverenv)> ls -l @@ -33,11 +51,16 @@ For the server software we will be using a server from a team at PNNL. The Grid server.yml ``` -1. Modify the openssl.cnf to include the correct values for [req_distinguished_name]. Or use the defaults. The server will create self-signed certificates for the client to use. +1. Modify the openssl.cnf to include the correct values for [req_distinguished_name]. + Or use the defaults. The server will create self-signed certificates for the + client to use. -1. Modify the server.yml file. The default server.yml file contains a device (id: dev1) and DERProgram. **dev1 must be present for the demo to run smoothly.** +1. Modify the server.yml file. The default server.yml file contains a device (id: dev1) and + DERProgram. **dev1 must be present for the demo to run smoothly.** -1. Start the server from the activated serverenv **This step will create development certificates for you**. By default the certificates will be generated and stored in ~/tls. One can change this location in the server.yml configuration file. +1. Start the server from the activated serverenv **This step will create development certificates + for you**. By default the certificates will be generated and stored in ~/tls. One can change + this location in the server.yml configuration file. ```bash (serverenv)> 2030_5_server server.yml --no-validate @@ -46,9 +69,10 @@ For the server software we will be using a server from a team at PNNL. The Grid # (serverenv)> 2030_5_server server.yml --no-validate --no-create-certs ``` -## Demo Requirements +## Demo Requirements ## -For this demo, start a VOLTTRON default instance from the command line. The following command will start VOLTTRON in the background writing to a volttron.log file. +For this demo, start a VOLTTRON default instance from the command line. The following command will +start VOLTTRON in the background writing to a volttron.log file. 1. Activate and start the volttron instance. @@ -71,8 +95,9 @@ For this demo, start a VOLTTRON default instance from the command line. The fol Note ed1 is the identity that needs to be in the configuration on the web demo -1. Open another terminal to run a simulated platform.driver. In this case, an inverter with a connected battery is being simulated. - +1. Open another terminal to run a simulated platform.driver. In this case, an inverter with a + connected battery is being simulated. + ```bash > cd $VOLTTRON_ROOT > source env/bin/activate @@ -81,7 +106,7 @@ For this demo, start a VOLTTRON default instance from the command line. The fol This process will start publishing "all" messages for the inverter to the VOLTTRON message bus. -1. Open another terminal to start the demo server in. +1. Open another terminal to start the demo server in. ```bash > cd $VOLTTRON_ROOT @@ -91,7 +116,8 @@ For this demo, start a VOLTTRON default instance from the command line. The fol ... ``` -1. Run the webgui.py script using the python interpretor. This should open a webpage allowing one to test the functionality of the 2030.5 agent. By default it will open at . +1. Run the webgui.py script using the python interpretor. This should open a webpage allowing one + to test the functionality of the 2030.5 agent. By default it will open at . ```bash (volttron)> python demo/webgui.py @@ -99,7 +125,8 @@ For this demo, start a VOLTTRON default instance from the command line. The fol ## The Demo -The demo starts with local time in the top followed by 2030.5's GMT time as a intenger. The integer is how 2030.5 communicates datetimes within the protocol. Five tabs are displayable and are +The demo starts with local time in the top followed by 2030.5's GMT time as a intenger. The integer +is how 2030.5 communicates datetimes within the protocol. Five tabs are displayable and are pictured below: ![Configuration Tab](./demo/images/configuration.png) diff --git a/services/core/IEEE_2030_5/demo/inverter_runner.py b/services/core/IEEE_2030_5/demo/inverter_runner.py index 1ed84dccdb..ec627b46a6 100644 --- a/services/core/IEEE_2030_5/demo/inverter_runner.py +++ b/services/core/IEEE_2030_5/demo/inverter_runner.py @@ -171,18 +171,22 @@ def run_inverter(timesteps=50) -> Generator: i_ac = (s_ac / v_ac) * 1000 print( f"p_ac = {p_ac}, s_ac = {s_ac}, q_ac= {q_ac}, PF = {PF}, v_ac = {v_ac}, i_ac = {i_ac}") - yield dict(v_mp=dc['v_mp'], - p_mp=dc['p_mp'], - i_x=dc['i_x'], - i_xx=dc['i_xx'], - v_oc=dc['v_oc'], - i_sc=dc['i_sc'], - p_ac=p_ac, - s_ac=p_ac, - q_ac=q_ac, - v_ac=v_ac, - i_ac=i_ac, - PF=PF) + yield dict(INV_REAL_PWR=p_ac, + INV_REAC_PWR=q_ac, + BAT_SOC=int(v_ac / p_ac), + INV_OP_STATUS_MODE=3) + # v_mp=dc['v_mp'], + # p_mp=dc['p_mp'], + # i_x=dc['i_x'], + # i_xx=dc['i_xx'], + # v_oc=dc['v_oc'], + # i_sc=dc['i_sc'], + # p_ac=p_ac, + # s_ac=p_ac, + # q_ac=q_ac, + # v_ac=v_ac, + # i_ac=i_ac, + # PF=PF) # single phase circuit calculation @@ -249,4 +253,4 @@ def run_inverter(timesteps=50) -> Generator: # fp.write(json.dumps(dict(headers=headers, message=points.forbus())) + "\n") gevent.sleep(10) - agent.core.stop() \ No newline at end of file + agent.core.stop() diff --git a/services/core/IEEE_2030_5/demo/server.yml b/services/core/IEEE_2030_5/demo/server.yml index 6c667e30fa..faac33296b 100644 --- a/services/core/IEEE_2030_5/demo/server.yml +++ b/services/core/IEEE_2030_5/demo/server.yml @@ -48,9 +48,7 @@ devices: - fsa1 - fsa2 ders: - - der1 - programs: - - Program 1 + - description: DER 1 - id: dev2 device_categories: @@ -60,40 +58,45 @@ devices: fsas: - fsa2 ders: - - der2 + - description: DER 2 -fsa: +fsas: - description: fsa1 + programs: + - Program 1 - description: fsa2 programs: - Program 1 - + programs: - description: Program 1 + primacy: 0 DefaultDERControl: setESDelay: 30 setGradW: 1000 - + DERControlBase: opModConnect: true opModEnergize: true - - -ders: - - description: der1 - program: Program 1 - - - description: der2 - program: Program 1 curves: - description: Curve 1 - curveType: opModVoltVar + curveType: 11 CurveData: - - xvalue: 5 - yvalue: 10 - - + - xvalue: 99 + yvalue: 50 + - xvalue: 103 + yvalue: -50 + - xvalue: 101 + yvalue: -50 + - xvalue: 97 + yvalue: 50 + rampDecTms: 600 + rampIncTms: 600 + rampPT1Tms: 10 + xMultiplier: 0 + yMultiplier: 0 + yRefType: 3 diff --git a/services/core/IEEE_2030_5/example.config.yml b/services/core/IEEE_2030_5/example.config.yml index 3094e37d19..697783c26c 100644 --- a/services/core/IEEE_2030_5/example.config.yml +++ b/services/core/IEEE_2030_5/example.config.yml @@ -4,7 +4,7 @@ keyfile: ~/tls/private/dev1.pem certfile: ~/tls/certs/dev1.crt server_hostname: 127.0.0.1 # the pin number is used to verify the server is the correct server -pin: 12345 +pin: 111115 # Log the request and responses from the server. log_req_resp: true @@ -18,7 +18,7 @@ default_der_control_poll: 60 MirrorUsagePointList: # MirrorMeterReading based on Table E.2 IEEE Std 2030.5-18 - - subscription_point: p_ac + - subscription_point: INV_REAL_PWR mRID: 5509D69F8B3535950000000000009182 description: DER Inverter Real Power roleFlags: 49 @@ -34,7 +34,7 @@ MirrorUsagePointList: intervalLength: 300 powerOfTenMultiplier: 0 uom: 38 - - subscription_point: q_ac + - subscription_point: INV_REAC_PWR mRID: 5509D69F8B3535950000000000009184 description: DER Inverter Reactive Power roleFlags: 49 @@ -80,6 +80,6 @@ DERSettings: setMaxW: multiplier: 0 value: 0 - + # Note this file MUST be in the config store or this agent will not run properly. point_map: config:///inverter_sample.csv diff --git a/services/core/IEEE_2030_5/ieee_2030_5/agent.py b/services/core/IEEE_2030_5/ieee_2030_5/agent.py index b19b9df5c6..42486ef614 100644 --- a/services/core/IEEE_2030_5/ieee_2030_5/agent.py +++ b/services/core/IEEE_2030_5/ieee_2030_5/agent.py @@ -64,12 +64,12 @@ @dataclass class MappedPoint: """The MappedPoint class models the mapping points. - + The MappedPoint class allows mapping of points from/to the platform.driver and 2030.5 objects. - + Only points that have point_on_bus and parameter_type will be mapped. - + The format of the parameter_type is :: where object is one of DERSettings, DERCapability, DERControlBase, or DERStatus. The property must be a valid property of the object. @@ -192,6 +192,9 @@ def __init__(self, config_path: str, **kwargs): except ConnectionRefusedError: _log.error(f"Could not connect to server {self._server_hostname} agent exiting.") sys.exit(1) + except ValueError as e: + _log.error(e) + sys.exit(1) _log.info(self._client.enddevice) assert self._client.enddevice ed = self._client.enddevice @@ -264,6 +267,8 @@ def _default_control_changed(self, default_control: m.DefaultDERControl): point_value) except TypeError: _log.error(f"Error setting point {point.point_on_bus} to {point_value}") + except KeyError: + _log.error(f"Error setting point {point.point_on_bus} to {point_value}") for point in der_base_points: @@ -439,14 +444,14 @@ def _cast_multipler(self, value: str) -> int: def _transform_settings(self, points: List[MappedPoint]) -> m.DERSettings: """Update a DERSettings object so that it is correctly formatted to send to the server. - + The point has a parent_object property that must be a DERSettings object. Each setting that requires a transition from a single element to a complex element is handled here. - + :param point: The point that is being updated. :return: The updated DERSettings object. :rtype: m.DERSettings - :raises AssertionError: If the parent_object is not a DERSettings object. + :raises AssertionError: If the parent_object is not a DERSettings object. """ # all of the settings are in the same envelope so we use the same # server time for all of them. @@ -561,14 +566,14 @@ def _transform_settings(self, points: List[MappedPoint]) -> m.DERSettings: def _transform_status(self, points: List[MappedPoint]) -> m.DERStatus: """Update a derstatus object so that it is correctly formatted to send to the server. - + The point has a parent_object property that must be a DERStatus object. Each setting that requires a transition from a single element to a complex element is handled here. - + :param point: The point that is being updated. :return: The updated DERStatus object. :rtype: m.DERStatus - :raises AssertionError: If the parent_object is not a DERStatus object. + :raises AssertionError: If the parent_object is not a DERStatus object. """ server_time = self._client.server_time @@ -618,14 +623,14 @@ def _transform_status(self, points: List[MappedPoint]) -> m.DERStatus: def _transform_capabilities(self, points: List[MappedPoint]) -> m.DERCapability: """Update a DERCapability object so that it is correctly formatted to send to the server. - + The point has a parent_object property that must be a DERCapability object. Each setting that requires a transition from a single element to a complex element is handled here. - + :param point: The point that is being updated. :return: The updated DERCapability object. :rtype: m.DERCapability - :raises AssertionError: If the parent_object is not a DERCapability object. + :raises AssertionError: If the parent_object is not a DERCapability object. """ server_time = self._client.server_time diff --git a/services/core/IEEE_2030_5/ieee_2030_5/client.py b/services/core/IEEE_2030_5/ieee_2030_5/client.py index 11ce22249a..d63ea4354b 100644 --- a/services/core/IEEE_2030_5/ieee_2030_5/client.py +++ b/services/core/IEEE_2030_5/ieee_2030_5/client.py @@ -234,9 +234,9 @@ def before_client_start(self, fun: Callable): @property def server_time(self) -> TimeType: """Returns the time on the server - + Uses an offset value from the 2030.5 Time function set to determine the - current time on the server. + current time on the server. :return: A calculated server_time including offset from time endpoint :rtype: TimeType @@ -336,7 +336,7 @@ def _send_control_events(self, der_program_href: str): fn=lambda: self._send_control_events(der_program_href)) def _update_dcap_tree(self, endpoint: Optional[str] = None): - """Retrieve device capability + """Retrieve device capability :param endpoint: _description_, defaults to None :type endpoint: Optional[str], optional @@ -361,7 +361,9 @@ def _update_dcap_tree(self, endpoint: Optional[str] = None): if dcap.pollRate is None: dcap.pollRate = 900 - self._update_timer_spec("dcap", dcap.pollRate, self._update_dcap_tree) + if dcap.EndDeviceListLink is None or dcap.EndDeviceListLink.all == 0: + raise ValueError("Couldn't receive end device model from server. " + "Check certificates or server configuration.") # if time is available then grab and create an offset if dcap.TimeLink is not None and dcap.TimeLink.href: @@ -400,6 +402,7 @@ def _update_dcap_tree(self, endpoint: Optional[str] = None): self._mirror_usage_point_map, self._mirror_usage_point) self._dcap = dcap + self._update_timer_spec("dcap", dcap.pollRate, self._update_dcap_tree) def _update_timer_spec(self, spec_name: str, rate: int, fn: Callable, *args, **kwargs): ts = self._timer_specs.get(spec_name) @@ -415,19 +418,19 @@ def post_log_event(self, end_device: m.EndDevice, log_event: m.LogEvent): def _update_list(self, path: str, list_prop: str, outer_map: Dict, inner_map: Dict): """Update mappings using 2030.5 list nomoclature. - + Example structure for EndDeviceListLink - + EndDeviceListLink.href points to EndDeviceList. EndDeviceList.EndDevice points to a list of EndDevice objects. - + Args: - + path: Original path of the list (in example EndDeviceListLink.href) list_prop: The property on the object that holds a list of elements (in example EndDevice) outer_mapping: Mapping where the original list object is stored by href inner_mapping: Mapping where the inner objects are stored by href - + """ my_response = self.__get_request__(path) @@ -483,9 +486,9 @@ def enddevices(self) -> m.EndDeviceList: @property def enddevice(self, href: str = "") -> m.EndDevice: """Retrieve a client's end device based upon the href of the end device. - + Args: - + href: If "" then in single client mode and return the only end device available. """ if not href: @@ -602,13 +605,13 @@ def request(self, endpoint: str, body: dict = {}, method: str = "GET", headers: def create_mirror_usage_point(self, mirror_usage_point: m.MirrorUsagePoint) -> str: """Create a new mirror usage point on the server. - + Args: - + mirror_usage_point: Minimal type for MirrorUsagePoint - + Return: - + The location of the new usage point href for posting to. """ data = dataclass_to_xml(mirror_usage_point) diff --git a/services/core/IEEE_2030_5/tests/test_client.py b/services/core/IEEE_2030_5/tests/test_client.py new file mode 100644 index 0000000000..d765abb514 --- /dev/null +++ b/services/core/IEEE_2030_5/tests/test_client.py @@ -0,0 +1,2 @@ +def test_connection(): + pass diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index 59fbc694a0..34786bdb45 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -74,19 +74,17 @@ from volttron.utils.prompt import prompt_response __all__ = [ - 'load_config', 'run_agent', 'start_agent_thread', 'is_valid_identity', - 'load_platform_config', 'get_messagebus', 'get_fq_identity', - 'execute_command', 'get_aware_utc_now', 'is_secure_mode', 'is_web_enabled', - 'is_auth_enabled', 'wait_for_volttron_shutdown', 'is_volttron_running' + 'load_config', 'run_agent', 'start_agent_thread', 'is_valid_identity', 'load_platform_config', + 'get_messagebus', 'get_fq_identity', 'execute_command', 'get_aware_utc_now', 'is_secure_mode', + 'is_web_enabled', 'is_auth_enabled', 'wait_for_volttron_shutdown', 'is_volttron_running' ] __author__ = 'Brandon Carpenter ' __copyright__ = 'Copyright (c) 2016, Battelle Memorial Institute' __license__ = 'Apache 2.0' -_comment_re = re.compile( - r'((["\'])(?:\\?.)*?\2)|(/\*.*?\*/)|((?:#|//).*?(?=\n|$))', - re.MULTILINE | re.DOTALL) +_comment_re = re.compile(r'((["\'])(?:\\?.)*?\2)|(/\*.*?\*/)|((?:#|//).*?(?=\n|$))', + re.MULTILINE | re.DOTALL) _log = logging.getLogger(__name__) @@ -153,8 +151,7 @@ def load_config(config_path): if not os.path.exists(config_path): raise ValueError( - f"Config file specified by AGENT_CONFIG path {config_path} does not exist." - ) + f"Config file specified by AGENT_CONFIG path {config_path} does not exist.") # First attempt parsing the file with a yaml parser (allows comments natively) # Then if that fails we fallback to our modified json parser. @@ -321,13 +318,13 @@ def store_message_bus_config(message_bus, instance_name): def update_kwargs_with_config(kwargs, config): """ Loads the user defined configurations into kwargs. - + 1. Converts any dash/hyphen in config variables into underscores - 2. Checks for configured "identity" value. Prints a deprecation - warning and uses it. - 3. Checks for configured "agentid" value. Prints a deprecation warning + 2. Checks for configured "identity" value. Prints a deprecation + warning and uses it. + 3. Checks for configured "agentid" value. Prints a deprecation warning and ignores it - + :param kwargs: kwargs to be updated :param config: dictionary of user/agent configuration """ @@ -336,10 +333,9 @@ def update_kwargs_with_config(kwargs, config): _log.warning("DEPRECATION WARNING: Setting a historian's VIP IDENTITY" " from its configuration file will no longer be supported" " after VOLTTRON 4.0") - _log.warning( - "DEPRECATION WARNING: Using the identity configuration setting " - "will override the value provided by the platform. This new value " - "will not be reported correctly by 'volttron-ctl status'") + _log.warning("DEPRECATION WARNING: Using the identity configuration setting " + "will override the value provided by the platform. This new value " + "will not be reported correctly by 'volttron-ctl status'") _log.warning("DEPRECATION WARNING: Please remove 'identity' from your " "configuration file and use the new method provided by " "the platform to set an agent's identity. See " @@ -361,11 +357,7 @@ def parse_json_config(config_str): return jsonapi.loads(strip_comments(config_str)) -def run_agent(cls, - subscribe_address=None, - publish_address=None, - config_path=None, - **kwargs): +def run_agent(cls, subscribe_address=None, publish_address=None, config_path=None, **kwargs): """Instantiate an agent and run it in the current thread. Attempts to get keyword parameters from the environment if they @@ -426,8 +418,7 @@ def default_main(agent_class, sub_addr = os.environ['AGENT_SUB_ADDR'] pub_addr = os.environ['AGENT_PUB_ADDR'] except KeyError as exc: - sys.stderr.write('missing environment variable: {}\n'.format( - exc.args[0])) + sys.stderr.write('missing environment variable: {}\n'.format(exc.args[0])) sys.exit(1) if sub_addr.startswith('ipc://') and sub_addr[6:7] != '@': if not os.path.exists(sub_addr[6:]): @@ -467,8 +458,29 @@ def vip_main(agent_class, identity=None, version='0.1', **kwargs): if identity is not None: if not is_valid_identity(identity): _log.warning('Deprecation warining') - _log.warning( - f'All characters in {identity} are not in the valid set.') + _log.warning(f'All characters in {identity} are not in the valid set.') + + publickey = kwargs.pop("publickey", None) + if not publickey: + publickey = os.environ.get("AGENT_PUBLICKEY") + secretkey = kwargs.pop("secretkey", None) + if not secretkey: + secretkey = os.environ.get("AGENT_SECRETKEY") + serverkey = kwargs.pop("serverkey", None) + if not serverkey: + serverkey = os.environ.get("VOLTTRON_SERVERKEY") + + # AGENT_PUBLICKEY and AGENT_SECRETKEY must be specified + # for the agent to execute successfully. aip should set these + # if the agent is run from the platform. If run from the + # run command it should be set automatically from vctl and + # added to the server. + # + # TODO: Make required for all agents. Handle it through vctl and aip. + if not os.environ.get("_LAUNCHED_BY_PLATFORM"): + if not publickey or not secretkey: + raise ValueError("AGENT_PUBLIC and AGENT_SECRET environmental variables must " + "be set to run without the platform.") address = get_address() agent_uuid = os.environ.get('AGENT_UUID') @@ -483,6 +495,9 @@ def vip_main(agent_class, identity=None, version='0.1', **kwargs): volttron_home=volttron_home, version=version, message_bus=message_bus, + publickey=publickey, + secretkey=secretkey, + serverkey=serverkey, **kwargs) try: @@ -513,8 +528,7 @@ class SyslogFormatter(logging.Formatter): def format(self, record): level = self._level_map.get(record.levelno, syslog.LOG_INFO) - return '<{}>'.format(level) + super(SyslogFormatter, - self).format(record) + return '<{}>'.format(level) + super(SyslogFormatter, self).format(record) class JsonFormatter(logging.Formatter): @@ -565,8 +579,7 @@ def setup_logging(level=logging.DEBUG, console=False): handler.setFormatter(JsonFormatter()) elif console: # Below format is more readable for console - handler.setFormatter( - logging.Formatter('%(levelname)s: %(message)s')) + handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) else: fmt = '%(asctime)s %(name)s %(lineno)d %(levelname)s: %(message)s' handler.setFormatter(logging.Formatter(fmt)) @@ -583,10 +596,10 @@ def setup_logging(level=logging.DEBUG, console=False): def format_timestamp(time_stamp): """Create a consistent datetime string representation based on ISO 8601 format. - + YYYY-MM-DDTHH:MM:SS.mmmmmm for unaware datetime objects. YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM for aware datetime objects - + :param time_stamp: value to convert :type time_stamp: datetime :returns: datetime in string format @@ -605,9 +618,7 @@ def format_timestamp(time_stamp): seconds = td.seconds minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) - time_str += "{sign}{HH:02}:{MM:02}".format(sign=sign, - HH=hours, - MM=minutes) + time_str += "{sign}{HH:02}:{MM:02}".format(sign=sign, HH=hours, MM=minutes) return time_str @@ -637,8 +648,7 @@ def parse_timestamp_string(time_stamp_str): try: base_time_stamp_str = time_stamp_str[:26] time_zone_str = time_stamp_str[26:] - time_stamp = datetime.strptime(base_time_stamp_str, - "%Y-%m-%dT%H:%M:%S.%f") + time_stamp = datetime.strptime(base_time_stamp_str, "%Y-%m-%dT%H:%M:%S.%f") # Handle most common case. if time_zone_str == "+00:00": return time_stamp.replace(tzinfo=pytz.UTC) @@ -660,7 +670,7 @@ def parse_timestamp_string(time_stamp_str): def get_aware_utc_now(): """Create a timezone aware UTC datetime object from the system time. - + :returns: an aware UTC datetime object :rtype: datetime """ @@ -708,9 +718,8 @@ def process_timestamp(timestamp_string, topic=''): try: timestamp = parse_timestamp_string(timestamp_string) except (ValueError, TypeError): - _log.error( - "message for {topic} bad timetamp string: {ts_string}".format( - topic=topic, ts_string=timestamp_string)) + _log.error("message for {topic} bad timetamp string: {ts_string}".format( + topic=topic, ts_string=timestamp_string)) return if timestamp.tzinfo is None: @@ -724,13 +733,13 @@ def process_timestamp(timestamp_string, topic=''): def watch_file(path: str, callback: Callable): """Run callback method whenever `path` changes. - + If `path` is not rooted the function assumes relative to the $VOLTTRON_HOME environmental variable - + The watch_file will create a watchdog event handler and will trigger when the close event happens for writing to the file. - + Not available on OS X/MacOS. """ file_path = Path(path) @@ -785,8 +794,7 @@ def create_file_if_missing(path, permission=0o660, contents=None): success = False try: if contents: - contents = contents if isinstance( - contents, bytes) else contents.encode("utf-8") + contents = contents if isinstance(contents, bytes) else contents.encode("utf-8") os.write(fd, contents) success = True except Exception as e: @@ -799,12 +807,12 @@ def create_file_if_missing(path, permission=0o660, contents=None): def fix_sqlite3_datetime(sql=None): """Primarily for fixing the base historian cache on certain versions of python. - + Registers a new datetime converter to that uses dateutil parse. This should better resolve #216, #174, and #91 without the goofy workarounds that change data. - + Optional sql argument is for testing only. """ if sql is None: @@ -922,9 +930,7 @@ def wait_for_volttron_startup(vhome, timeout): gevent.sleep(3) sleep_time += 3 if sleep_time >= timeout: - raise Exception( - "Platform startup failed. Please check volttron.log in {}".format( - vhome)) + raise Exception("Platform startup failed. Please check volttron.log in {}".format(vhome)) def wait_for_volttron_shutdown(vhome, timeout): @@ -935,5 +941,4 @@ def wait_for_volttron_shutdown(vhome, timeout): sleep_time += 1 if sleep_time >= timeout: raise Exception( - "Platform shutdown failed. Please check volttron.cfg.log in {}". - format(vhome)) + "Platform shutdown failed. Please check volttron.cfg.log in {}".format(vhome))