From 18b03e9b9cb59e562a5c2e097bca3b6e98e0aa06 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Mon, 25 Sep 2023 23:15:24 -0700 Subject: [PATCH 1/9] Initial version of bulk update of config store when volttron is not running issue #3121 --- volttron/platform/instance_setup.py | 219 +++++++++++++++++++++++----- 1 file changed, 182 insertions(+), 37 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index d8e91a3de1..4ffeeb6dfa 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -37,6 +37,7 @@ # }}} import argparse import hashlib +import json import os import sys import tempfile @@ -57,10 +58,13 @@ from volttron.platform import jsonapi from volttron.platform.agent.known_identities import PLATFORM_WEB, PLATFORM_DRIVER, VOLTTRON_CENTRAL from volttron.platform.agent.utils import get_platform_instance_name, wait_for_volttron_startup, \ - is_volttron_running, wait_for_volttron_shutdown, setup_logging + is_volttron_running, wait_for_volttron_shutdown, setup_logging, format_timestamp, get_aware_utc_now, \ + parse_json_config from volttron.utils import get_hostname from volttron.utils.prompt import prompt_response, y, n, y_or_n from . import get_home, get_services_core, set_home +from volttron.platform.agent.utils import load_config as load_yml_or_json +from volttron.platform.store import process_raw_config if is_rabbitmq_available(): from bootstrap import install_rabbit, default_rmq_dir @@ -155,24 +159,24 @@ def _is_bound_already(address): return already_bound -def fail_if_instance_running(args): +def fail_if_instance_running(message, prompt=True): home = get_home() if os.path.exists(home) and\ is_volttron_running(home): - global use_active - use_active = prompt_response( - "The VOLTTRON Instance is currently running. " - "Installing agents to an active instance may overwrite currently installed " - "and active agents on the platform, causing undesirable behavior. " - "Would you like to continue?", - valid_answers=y_or_n, - default='Y') - if use_active in y: - return + if prompt: + global use_active + use_active = prompt_response( + message + + "Would you like to continue?", + valid_answers=y_or_n, + default='Y') + if use_active in y: + return else: - print(""" + print(message) + print(""" Please execute: volttron-ctl shutdown --platform @@ -180,7 +184,7 @@ def fail_if_instance_running(args): to stop the instance. """) - sys.exit() + sys.exit(1) def fail_if_not_in_src_root(): @@ -189,7 +193,7 @@ def fail_if_not_in_src_root(): print(""" volttron-cfg needs to be run from the volttron top level source directory. """) - sys.exit() + sys.exit(1) def _start_platform(): @@ -266,8 +270,8 @@ def func(*args, **kwargs): print('Configuring {}.'.format(agent_dir)) config = config_func(*args, **kwargs) _update_config_file() - #TODO: Optimize long vcfg install times - #TODO: (potentially only starting the platform once per vcfg) + # TODO: Optimize long vcfg install times + # TODO: (potentially only starting the platform once per vcfg) if use_active in n: _start_platform() @@ -375,6 +379,7 @@ def set_dependencies(requirement): subprocess.check_call(cmds) return + def _create_web_certs(): global config_opts """ @@ -527,13 +532,15 @@ def do_instance_name(): instance_name = new_instance_name config_opts['instance-name'] = '"{}"'.format(instance_name) + def do_web_enabled_rmq(vhome): global config_opts # Full implies that it will have a port on it as well. Though if it's # not in the address that means that we haven't set it up before. - full_bind_web_address = config_opts.get('bind-web-address', - 'https://' + get_hostname()) + full_bind_web_address = config_opts.get( + 'bind-web-address', + 'https://' + get_hostname()) parsed = urlparse(full_bind_web_address) @@ -575,7 +582,6 @@ def do_web_enabled_rmq(vhome): def do_web_enabled_zmq(vhome): global config_opts - # Full implies that it will have a port on it as well. Though if it's # not in the address that means that we haven't set it up before. full_bind_web_address = config_opts.get('bind-web-address', @@ -892,7 +898,6 @@ def wizard(): # Start true configuration here. volttron_home = get_home() - confirm_volttron_home() _load_config() _update_config_file() if use_active in n: @@ -993,11 +998,113 @@ def wizard(): if response in y: do_listener() + +def read_agent_configs_from_store(store_source, path=True): + if path: + with open(store_source) as f: + store = parse_json_config(f.read()) + else: + store = store_source + return store + + +def update_configs_in_store(args_dict): + + vhome = get_home() + try: + metadata_dict = load_yml_or_json(args_dict['metadata_file']) + except Exception as e: + print(f"Invalid metadata file: {args_dict['metadata_file']}: {e}") + exit(1) + for vip_id in metadata_dict: + configs = metadata_dict[vip_id] + if isinstance(configs, dict): + # only single config for this vip id + configs = [configs] + if not isinstance(configs, list): + print( + f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. " + f"Got type {type(configs)}") + _exit_with_metadata_error() + + configs_updated = False + agent_store_path = os.path.join(vhome, "configuration_store", vip_id+".store") + if os.path.isfile(agent_store_path): + # load current store configs as python object for comparison + store_configs = read_agent_configs_from_store(agent_store_path) + else: + store_configs = dict() + + for config_dict in configs: + if not isinstance(config_dict, dict): + print(f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. " + f"Got type {type(config_dict)}") + _exit_with_metadata_error() + + config_name = config_dict.get("config-name", "config") + config_type = config_dict.get("config-type", "json") + config = config_dict.get("config") + if config is None: + print(f"No config entry found for vip-id {vip_id} and config-name {config_name}") + _exit_with_metadata_error() + + # If there is config validate it + # Check if config is file path + if isinstance(config, str) and os.path.isfile(config): + raw_data = open(config).read() + # try loading it into appropriate python object to validate if file content and config-type match + processed_data = process_raw_config(raw_data, config_type) + elif isinstance(config, str) and config_type == 'raw': + raw_data = config + processed_data = config + else: + if not isinstance(config, (list, dict)): + processed_data = raw_data = None + print('Value for key "config" should be one of the following: \n' + '1. filepath \n' + '2. string with "config-type" set to "raw" \n' + '3. a dictionary \n' + '4. list ') + _exit_with_metadata_error() + else: + processed_data = config + raw_data = jsonapi.dumps(processed_data) + + current = store_configs.get(config_name) + + if not current or process_raw_config(current.get('data'), current.get('type')) != processed_data: + store_configs[config_name] = dict() + store_configs[config_name]['data'] = raw_data + store_configs[config_name]['type'] = config_type + store_configs[config_name]['modified'] = format_timestamp(get_aware_utc_now()) + configs_updated = True + + # All configs processed for current vip-id + # if there were updates write the new configs to file + if configs_updated: + os.makedirs(os.path.dirname(agent_store_path), exist_ok=True) + with open(agent_store_path, 'w+') as f: + json.dump(store_configs, f) + + +def _exit_with_metadata_error(): + print('''Metadata file should be of the format: + { "vip-id": [ + { + "config-name": "optional. name. defaults to config", + "config": "json config or config file name", + "config-type": "optional. type of config. defaults to json" + }, ... + ],... + }''') + exit(1) + + def process_rmq_inputs(args_dict, instance_name=None): #print(f"args_dict:{args_dict}, args") if not is_rabbitmq_available(): raise RuntimeError("Rabbitmq Dependencies not installed please run python bootstrap.py --rabbitmq") - confirm_volttron_home() + vhome = get_home() if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'): @@ -1055,34 +1162,62 @@ def main(): parser.add_argument('--vhome', help="Path to volttron home") parser.add_argument('--instance-name', dest='instance_name', help="Name of this volttron instance") parser.set_defaults(is_rabbitmq=False) + parser.set_defaults(config_update=False) group = parser.add_mutually_exclusive_group() agent_list = '\n\t' + '\n\t'.join(sorted(available_agents.keys())) group.add_argument('--list-agents', action='store_true', dest='list_agents', help='list configurable agents{}'.format(agent_list)) - rabbitmq_parser = parser.add_subparsers(title='rabbitmq', - metavar='', - dest='parser_name') - single_parser = rabbitmq_parser.add_parser('rabbitmq', help='Configure rabbitmq for single instance, ' - 'federation, or shovel either based on ' - 'configuration file in yml format or providing ' - 'details when prompted. \nUsage: vcfg rabbitmq ' - 'single|federation|shovel --config --max-retries ]') - single_parser.add_argument('installation-type', default='single', help='Rabbitmq option for installation. Installation type can be single|federation|shovel') + subparsers = parser.add_subparsers(dest="cmd") + single_parser = subparsers.add_parser('rabbitmq', help='Configure rabbitmq for single instance, ' + 'federation, or shovel either based on ' + 'configuration file in yml format or providing ' + 'details when prompted. \nUsage: vcfg rabbitmq ' + 'single|federation|shovel --config --max-retries ]') + single_parser.add_argument('installation-type', default='single', + help='Rabbitmq option for installation. ' + 'Installation type can be single|federation|shovel') single_parser.add_argument('--max-retries', help='Optional Max retry attempt', type=int, default=12) single_parser.add_argument('--config', help='Optional path to rabbitmq config yml', type=str) single_parser.set_defaults(is_rabbitmq=True) group.add_argument('--agent', nargs='+', - help='configure listed agents') + help='configure listed agents') group.add_argument('--agent-isolation-mode', action='store_true', dest='agent_isolation_mode', help='Require that agents run with their own users (this requires running ' 'scripts/secure_user_permissions.sh as sudo)') + config_store_parser = subparsers.add_parser("update-config-store", + help="Update one or more config entries for one more agents") + config_store_parser.set_defaults(config_update=True) + # start with just a metadata file support. + # todo - add support vip-id, directory + # vip-id, file with multiple configs etc. + #config_arg_group = config_store_parser.add_mutually_exclusive_group() + #meta_group = config_arg_group.add_mutually_exclusive_group() + config_store_parser.add_argument('--metadata-file', required=True, + help='metadata file containing details of vip id, ' + 'optional config name(defaults to "config"),' + 'config content, ' + 'and optional config type(defaults to json). Format:' + '\n' + '{ "vip-id": [' + ' { ' + ' "config-name": "optional. name. defaults to config' + ' "config": "json config or config file name", ' + ' "config-type": "optional. type of config. defaults to json"' + ' }, ...' + ' ],...' + '}') + + # single_agent_group = config_arg_group.add_mutually_exclusive_group() + # single_agent_group.add_argument("--vip-id", + # help='vip-identity of the agent for which config store should be updated') + # single_agent_group.add_argument("--config-path", + # help="json file containing configs or directory containing config files") args = parser.parse_args() - verbose = args.verbose # Protect against configuration of base logger when not the "main entry point" if verbose: @@ -1094,8 +1229,18 @@ def main(): if args.vhome: set_home(args.vhome) prompt_vhome = False + + confirm_volttron_home() # if not args.rabbitmq or args.rabbitmq[0] in ["single"]: - fail_if_instance_running(args) + if args.agent: + message = "The VOLTTRON Instance is currently running. " \ + "Installing agents to an active instance may overwrite currently installed "\ + "and active agents on the platform, causing undesirable behavior. " + fail_if_instance_running(message) + if args.config_update: + message = f"VOLTTRON is running using at {get_home()}, " \ + "you can add/update single configuration using vctl config command." + fail_if_instance_running(message, prompt=False) fail_if_not_in_src_root() if use_active in n: atexit.register(_cleanup_on_exit) @@ -1110,6 +1255,8 @@ def main(): _update_config_file() elif args.is_rabbitmq: process_rmq_inputs(vars(args)) + elif args.config_update: + update_configs_in_store(vars(args)) elif not args.agent: wizard() @@ -1121,8 +1268,6 @@ def main(): print('"{}" not configurable with this tool'.format(agent)) else: valid_agents = True - if valid_agents: - confirm_volttron_home() # Configure agents for agent in args.agent: From 1b0f4964276160b5e66bfd5317aab58463b637e7 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Mon, 25 Sep 2023 23:15:37 -0700 Subject: [PATCH 2/9] minor log stmt fix --- volttron/platform/agent/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index 59fbc694a0..cda9c8cf7f 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -153,7 +153,7 @@ def load_config(config_path): if not os.path.exists(config_path): raise ValueError( - f"Config file specified by AGENT_CONFIG path {config_path} does not exist." + f"Config file specified by path {config_path} does not exist." ) # First attempt parsing the file with a yaml parser (allows comments natively) @@ -169,7 +169,7 @@ def load_config(config_path): with open(config_path) as f: return parse_json_config(f.read()) except Exception as e: - _log.error("Problem parsing agent configuration") + _log.error(f"Problem parsing configuration {config_path}: {e}") raise From de5cf746feba8323d734e8a3d279ab257db68463 Mon Sep 17 00:00:00 2001 From: Kefei Mo Date: Fri, 20 Oct 2023 12:40:22 -0500 Subject: [PATCH 3/9] revivied Craig s code to resolved the issue that uuid has extra chars when using vctl status --- volttron/platform/control/control_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/volttron/platform/control/control_utils.py b/volttron/platform/control/control_utils.py index a4300e6603..c33034e2a2 100644 --- a/volttron/platform/control/control_utils.py +++ b/volttron/platform/control/control_utils.py @@ -36,11 +36,11 @@ # under Contract DE-AC05-76RL01830 # }}} import collections -import itertools import sys import re from volttron.platform import jsonapi from volttron.platform.agent.utils import is_secure_mode +import os _stdout = sys.stdout _stderr = sys.stderr @@ -48,10 +48,13 @@ def _calc_min_uuid_length(agents): n = 0 - for agent1, agent2 in itertools.combinations(agents, 2): - common_len = sum(1 for a, b in zip(agent1.uuid, agent2.uuid) if a == b) - if common_len > n: - n = common_len + for agent1 in agents: + for agent2 in agents: + if agent1 is agent2: + continue + common_len = len(os.path.commonprefix([agent1.uuid, agent2.uuid])) + if common_len > n: + n = common_len return n + 1 From 8a513f0cae88c4ab6dc4044985188ec9b78cdb5f Mon Sep 17 00:00:00 2001 From: Kefei Mo Date: Fri, 20 Oct 2023 13:14:30 -0500 Subject: [PATCH 4/9] optimized _calc_min_uuid_length(agents) --- volttron/platform/control/control_utils.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/volttron/platform/control/control_utils.py b/volttron/platform/control/control_utils.py index c33034e2a2..8aad804fd7 100644 --- a/volttron/platform/control/control_utils.py +++ b/volttron/platform/control/control_utils.py @@ -47,15 +47,9 @@ def _calc_min_uuid_length(agents): - n = 0 - for agent1 in agents: - for agent2 in agents: - if agent1 is agent2: - continue - common_len = len(os.path.commonprefix([agent1.uuid, agent2.uuid])) - if common_len > n: - n = common_len - return n + 1 + agent_ids = [agent.uuid for agent in agents] + common_len = len(os.path.commonprefix(agent_ids)) + return common_len + 1 def _list_agents(aip): From 36c67e9cabac2fd7dad5f9a11c06b2994f6b58be Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Sun, 22 Oct 2023 00:01:53 -0700 Subject: [PATCH 5/9] updates to support multiple metadata files and or directory as input for vcfg update-config-store --- volttron/platform/instance_setup.py | 161 ++++++++++++++++------------ 1 file changed, 93 insertions(+), 68 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 4ffeeb6dfa..90912799c8 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -1011,80 +1011,104 @@ def read_agent_configs_from_store(store_source, path=True): def update_configs_in_store(args_dict): vhome = get_home() - try: - metadata_dict = load_yml_or_json(args_dict['metadata_file']) - except Exception as e: - print(f"Invalid metadata file: {args_dict['metadata_file']}: {e}") - exit(1) - for vip_id in metadata_dict: - configs = metadata_dict[vip_id] - if isinstance(configs, dict): - # only single config for this vip id - configs = [configs] - if not isinstance(configs, list): - print( - f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. " - f"Got type {type(configs)}") - _exit_with_metadata_error() - - configs_updated = False - agent_store_path = os.path.join(vhome, "configuration_store", vip_id+".store") - if os.path.isfile(agent_store_path): - # load current store configs as python object for comparison - store_configs = read_agent_configs_from_store(agent_store_path) + metadata_files = list() + + args_list = args_dict['metadata_file'] + # validate args + for item in args_list: + if os.path.isdir(item): + for f in os.listdir(item): + file_path = os.path.join(item, f) + if os.path.isfile(file_path): + metadata_files.append(file_path) + elif os.path.isfile(item): + metadata_files.append(item) else: - store_configs = dict() - - for config_dict in configs: - if not isinstance(config_dict, dict): - print(f"Metadata for vip-identity {vip_id} should be a dictionary or list of dictionary. " - f"Got type {type(config_dict)}") - _exit_with_metadata_error() + print(f"Value is neither a file nor a directory: {args_dict['metadata_file']}: ") + print(f"The --metadata-file accepts one or more metadata files or directory containing metadata file") + _exit_with_metadata_error() - config_name = config_dict.get("config-name", "config") - config_type = config_dict.get("config-type", "json") - config = config_dict.get("config") - if config is None: - print(f"No config entry found for vip-id {vip_id} and config-name {config_name}") + # Validate each file content and load config + for metadata_file in metadata_files: + metadata_dict = dict() + try: + metadata_dict = load_yml_or_json(metadata_file) + except Exception as e: + print(f"Invalid metadata file: {metadata_file}: {e}") + exit(1) + + for vip_id in metadata_dict: + configs = metadata_dict[vip_id] + if isinstance(configs, dict): + # only single config for this vip id + configs = [configs] + if not isinstance(configs, list): + print( + f"Metadata for vip-identity {vip_id} in file {metadata_file} " + f"should be a dictionary or list of dictionary. " + f"Got type {type(configs)}") _exit_with_metadata_error() - # If there is config validate it - # Check if config is file path - if isinstance(config, str) and os.path.isfile(config): - raw_data = open(config).read() - # try loading it into appropriate python object to validate if file content and config-type match - processed_data = process_raw_config(raw_data, config_type) - elif isinstance(config, str) and config_type == 'raw': - raw_data = config - processed_data = config + configs_updated = False + agent_store_path = os.path.join(vhome, "configuration_store", vip_id+".store") + if os.path.isfile(agent_store_path): + # load current store configs as python object for comparison + store_configs = read_agent_configs_from_store(agent_store_path) else: - if not isinstance(config, (list, dict)): - processed_data = raw_data = None - print('Value for key "config" should be one of the following: \n' - '1. filepath \n' - '2. string with "config-type" set to "raw" \n' - '3. a dictionary \n' - '4. list ') - _exit_with_metadata_error() - else: - processed_data = config - raw_data = jsonapi.dumps(processed_data) + store_configs = dict() - current = store_configs.get(config_name) + for config_dict in configs: + if not isinstance(config_dict, dict): + print(f"Metadata for vip-identity {vip_id} in file {metadata_file} " + f"should be a dictionary or list of dictionary. " + f"Got type {type(config_dict)}") + _exit_with_metadata_error() - if not current or process_raw_config(current.get('data'), current.get('type')) != processed_data: - store_configs[config_name] = dict() - store_configs[config_name]['data'] = raw_data - store_configs[config_name]['type'] = config_type - store_configs[config_name]['modified'] = format_timestamp(get_aware_utc_now()) - configs_updated = True + config_name = config_dict.get("config-name", "config") + config_type = config_dict.get("config-type", "json") + config = config_dict.get("config") + if config is None: + print(f"No config entry found in file {metadata_file} for vip-id {vip_id} and " + f"config-name {config_name}") + _exit_with_metadata_error() - # All configs processed for current vip-id - # if there were updates write the new configs to file - if configs_updated: - os.makedirs(os.path.dirname(agent_store_path), exist_ok=True) - with open(agent_store_path, 'w+') as f: - json.dump(store_configs, f) + # If there is config validate it + # Check if config is file path + if isinstance(config, str) and os.path.isfile(config): + raw_data = open(config).read() + # try loading it into appropriate python object to validate if file content and config-type match + processed_data = process_raw_config(raw_data, config_type) + elif isinstance(config, str) and config_type == 'raw': + raw_data = config + processed_data = config + else: + if not isinstance(config, (list, dict)): + processed_data = raw_data = None + print('Value for key "config" should be one of the following: \n' + '1. filepath \n' + '2. string with "config-type" set to "raw" \n' + '3. a dictionary \n' + '4. list ') + _exit_with_metadata_error() + else: + processed_data = config + raw_data = jsonapi.dumps(processed_data) + + current = store_configs.get(config_name) + + if not current or process_raw_config(current.get('data'), current.get('type')) != processed_data: + store_configs[config_name] = dict() + store_configs[config_name]['data'] = raw_data + store_configs[config_name]['type'] = config_type + store_configs[config_name]['modified'] = format_timestamp(get_aware_utc_now()) + configs_updated = True + + # All configs processed for current vip-id + # if there were updates write the new configs to file + if configs_updated: + os.makedirs(os.path.dirname(agent_store_path), exist_ok=True) + with open(agent_store_path, 'w+') as f: + json.dump(store_configs, f) def _exit_with_metadata_error(): @@ -1196,8 +1220,9 @@ def main(): # vip-id, file with multiple configs etc. #config_arg_group = config_store_parser.add_mutually_exclusive_group() #meta_group = config_arg_group.add_mutually_exclusive_group() - config_store_parser.add_argument('--metadata-file', required=True, - help='metadata file containing details of vip id, ' + config_store_parser.add_argument('--metadata-file', required=True, nargs='+', + help='one or more metadata file or directory containing metadata files, ' + 'where each metadata file contain details of vip id, ' 'optional config name(defaults to "config"),' 'config content, ' 'and optional config type(defaults to json). Format:' From 638ad4888b08dffdd312c8eb09d987c72de2830d Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Sun, 22 Oct 2023 22:14:32 -0700 Subject: [PATCH 6/9] updated help text formatting --- volttron/platform/instance_setup.py | 47 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 90912799c8..ce559d5756 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -47,6 +47,7 @@ from shutil import copy from urllib.parse import urlparse import logging +from argparse import RawTextHelpFormatter from gevent import subprocess from gevent.subprocess import Popen @@ -1112,15 +1113,16 @@ def update_configs_in_store(args_dict): def _exit_with_metadata_error(): - print('''Metadata file should be of the format: - { "vip-id": [ - { - "config-name": "optional. name. defaults to config", - "config": "json config or config file name", - "config-type": "optional. type of config. defaults to json" - }, ... - ],... - }''') + print(""" +Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or config file name", + "config-type": "optional. type of config. defaults to json" + }, ... + ],... +}""") exit(1) @@ -1212,7 +1214,7 @@ def main(): group.add_argument('--agent-isolation-mode', action='store_true', dest='agent_isolation_mode', help='Require that agents run with their own users (this requires running ' 'scripts/secure_user_permissions.sh as sudo)') - config_store_parser = subparsers.add_parser("update-config-store", + config_store_parser = subparsers.add_parser("update-config-store", formatter_class=RawTextHelpFormatter, help="Update one or more config entries for one more agents") config_store_parser.set_defaults(config_update=True) # start with just a metadata file support. @@ -1221,20 +1223,17 @@ def main(): #config_arg_group = config_store_parser.add_mutually_exclusive_group() #meta_group = config_arg_group.add_mutually_exclusive_group() config_store_parser.add_argument('--metadata-file', required=True, nargs='+', - help='one or more metadata file or directory containing metadata files, ' - 'where each metadata file contain details of vip id, ' - 'optional config name(defaults to "config"),' - 'config content, ' - 'and optional config type(defaults to json). Format:' - '\n' - '{ "vip-id": [' - ' { ' - ' "config-name": "optional. name. defaults to config' - ' "config": "json config or config file name", ' - ' "config-type": "optional. type of config. defaults to json"' - ' }, ...' - ' ],...' - '}') + help="""One or more metadata file or directory containing metadata file, +where each metadata file contain details of configs for one or more agent instance +Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or config file name", + "config-type": "optional. type of config. defaults to json" + }, ... + ],... +}""") # single_agent_group = config_arg_group.add_mutually_exclusive_group() # single_agent_group.add_argument("--vip-id", From ef7a953bdb66a6953574b8ceb4618bb8638ecd4d Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Sun, 22 Oct 2023 22:30:32 -0700 Subject: [PATCH 7/9] tests for vcfg update-config-store --- .../platform/test_vcfg_config_update.py | 467 ++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 volttrontesting/platform/test_vcfg_config_update.py diff --git a/volttrontesting/platform/test_vcfg_config_update.py b/volttrontesting/platform/test_vcfg_config_update.py new file mode 100644 index 0000000000..62fa7f11bd --- /dev/null +++ b/volttrontesting/platform/test_vcfg_config_update.py @@ -0,0 +1,467 @@ +import json + +import pytest +import os +import shutil +import subprocess +from csv import DictReader +from io import StringIO + +from volttrontesting.utils.platformwrapper import create_volttron_home +from volttron.platform.agent.utils import parse_json_config +from volttrontesting.fixtures.volttron_platform_fixtures import build_wrapper, cleanup_wrapper +from volttrontesting.utils.utils import get_rand_vip + +METATADATA_FILE_FORMAT = """Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or config file name", + "config-type": "optional. type of config. defaults to json" + }, ... + ],... +}""" + + +@pytest.fixture(scope="module") +def shared_vhome(): + debug_flag = os.environ.get('DEBUG', False) + vhome = create_volttron_home() + yield vhome + if not debug_flag: + shutil.rmtree(vhome, ignore_errors=True) + + +@pytest.fixture() +def vhome(): + debug_flag = os.environ.get('DEBUG', False) + vhome = create_volttron_home() + yield vhome + if not debug_flag: + shutil.rmtree(vhome, ignore_errors=True) + + +@pytest.fixture(scope="module") +def single_vinstance(): + address = get_rand_vip() + wrapper = build_wrapper(address, + messagebus='zmq', + ssl_auth=False, + auth_enabled=False) + yield wrapper + cleanup_wrapper(wrapper) + + +# Only integration test. Rest are unit tests +def test_fail_if_volttron_is_running(single_vinstance, monkeypatch): + monkeypatch.setenv("VOLTTRON_HOME", single_vinstance.volttron_home) + process = subprocess.run(["vcfg", "--vhome", single_vinstance.volttron_home, + "update-config-store", "--metadata-file", "test"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stdout.decode("utf-8").startswith( + f"VOLTTRON is running using at {single_vinstance.volttron_home}, " + f"you can add/update single configuration using vctl config command.") + assert process.returncode == 1 + + +def test_help(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--help"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stdout.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_no_arg(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stderr.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_no_args_value(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--metadata-file"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stderr.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_invalid_file_path(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--metadata-file", "invalid"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + expected_message = "Value is neither a file nor a directory: ['invalid']: \n" \ + "The --metadata-file accepts one or more metadata " \ + "files or directory containing metadata file\n\n" + METATADATA_FILE_FORMAT + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +@pytest.mark.parametrize('json_metadata, json_file, config_name', [ + ({"vip-id-1": {}}, "no_config_json1", "config"), + ({"vip-id-1": [{"config-name": "config"}]}, "no_config_json2", "config"), + ({"vip-id-1": [{"config-name": "new_config", "config_type": "json"}]}, "no_config_json2", "new_config") +]) +def test_no_config_metadata(monkeypatch, vhome, json_metadata, json_file, config_name): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, json_file) + with open(file_path, "w") as f: + f.write(json.dumps(json_metadata)) + expected_message = f"No config entry found in file {file_path} for vip-id vip-id-1 and " \ + f"config-name {config_name}\n\n" + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_invalid_config_class(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "invalid_config_class.json") + with open(file_path, "w") as f: + f.write(json.dumps({"vip-id-1": "string config"})) + expected_message = f"Metadata for vip-identity vip-id-1 in file {file_path} should be a dictionary or " \ + f"list of dictionary. Got type \n\n" + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_incorrect_config_type(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "invalid_config_type.json") + with open(file_path, "w") as f: + f.write(json.dumps({"vip-id-1": {"config": "string config for json config-type", + "config-type": "json"}})) + expected_message = 'Value for key "config" should be one of the following: \n' \ + '1. filepath \n'\ + '2. string with "config-type" set to "raw" \n'\ + '3. a dictionary \n'\ + '4. list \n\n' + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_raw_config_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", + "config-type": "raw"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + initial_modified_time = store["config"]["modified"] + + # now try list of raw config with 1 new and 1 existing + file_path = os.path.join(vhome, "two_config.json") + with open(file_path, "w") as f: + f.write(json.dumps( + {"agent1": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + } + )) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert initial_modified_time == store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + assert store["new_config"]["modified"] != initial_modified_time + + +def test_json_config_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + json_data = {"config_key1": "config_value1"} + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent2": {"config": json_data, + "config-type": "json"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert parse_json_config(store["config"]["data"]) == json_data + assert store["config"]["type"] == "json" + initial_modified_time = store["config"]["modified"] + + +def test_csv_configfile_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + csv_file = os.path.join(vhome, "config.csv") + csv_str = "point_name,type\npoint1,boolean\npoint2,int" + csv_list = [{"point_name": "point1", "type": "boolean"}, {"point_name": "point2", "type": "int"}] + with open(csv_file, "w") as f: + f.write(csv_str) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent3": {"config": csv_file, + "config-type": "csv"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent3.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + f = StringIO(store["config"]["data"]) + csv_list_in_store = [x for x in DictReader(f)] + assert csv_list_in_store == csv_list + assert store["config"]["type"] == "csv" + initial_modified_time = store["config"]["modified"] + + +def test_single_metafile_two_agent(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps( + {"agent1": {"config": "string config", "config-type": "raw"}, + "agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + + +def test_two_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path1 = os.path.join(vhome, "meta1.json") + with open(file_path1, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", "config-type": "raw"}})) + file_path2 = os.path.join(vhome, "meta2.json") + with open(file_path2, "w") as f: + f.write(json.dumps( + {"agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path1, file_path2], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + + +def test_meta_dir(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + meta_dir = os.path.join(vhome, "meta_dir") + os.mkdir(meta_dir) + file_path1 = os.path.join(meta_dir, "meta1.json") + with open(file_path1, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", "config-type": "raw"}})) + file_path2 = os.path.join(meta_dir, "meta2.json") + with open(file_path2, "w") as f: + f.write(json.dumps( + {"agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", meta_dir], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] From bce93fe4c3b27a3001fc37fa9a58a7cca3472ebe Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Mon, 23 Oct 2023 12:49:10 -0700 Subject: [PATCH 8/9] Updated help text and vcfg documentation --- .../platform-configuration.rst | 60 ++++++++++++++++--- volttron/platform/instance_setup.py | 8 +-- .../platform/test_vcfg_config_update.py | 4 +- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/docs/source/deploying-volttron/platform-configuration.rst b/docs/source/deploying-volttron/platform-configuration.rst index 569d86485c..b3b9f5f390 100644 --- a/docs/source/deploying-volttron/platform-configuration.rst +++ b/docs/source/deploying-volttron/platform-configuration.rst @@ -6,11 +6,11 @@ Platform Configuration Each instance of the VOLTTRON platform includes a `config` file which is used to configure the platform instance on startup. This file is kept in :term:`VOLTTRON_HOME` and is created using the `volttron-cfg` (`vcfg`) command, or will -be created with default values on start up of the platform otherwise. +be created with default values on start up of the platform otherwise. `vcfg` also provides commands to configure and +install few frequently used agents and update configuration-store entries Following is helpful information about the `config` file and the `vcfg` command. - VOLTTRON_HOME ============= @@ -75,9 +75,13 @@ The example consists of the following entries: VOLTTRON Config =============== -The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. The command includes -the ability to set up the platform configuration, an instance of the platform historian, VOLTTRON Central UI, and -VOLTTRON Central Platform agent. +The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. +The `vcfg` command can be run without any arguments to go through all available configuration steps sequentially and +pick what is appropriate for a given instance of VOLTTRON. This is useful when starting with a new VOLTTRON instance. + +`vcfg` command can also be used with specific subcommands to configure and install specific agents such as +listener agent, platform historian, VOLTTRON Central UI, and VOLTTRON Central Platform agent +or add one or more agent configurations to VOLTTRON's configuration store. Running `vcfg` will create a `config` file in `VOLTTRON_HOME` which will be populated according to the answers to prompts. This process should be repeated for each platform instance, and can be re-run to reconfigure a platform @@ -158,12 +162,52 @@ Optional Arguments - **-v, --verbose** - Enables verbose output in standard-output (PIP output, etc.) - **--vhome VHOME** - Provide a path to set `VOLTTRON_HOME` for this instance - **--instance-name INSTANCE_NAME** - Provide a name for this instance. Required for running secure agents mode - - **--list-agents** - Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON - Central, VOLTTRON Central Platform) - - **--agent AGENT [AGENT ...]** - Configure listed agents - **--agent-isolation-mode** - Require that agents run as their own Unix users (this requires running `scripts/secure_user_permissions.sh` as `sudo`) +Sub commands +------------ +**--list-agents** +~~~~~~~~~~~~~~~~~~ + Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON + Central, VOLTTRON Central Platform) + +**--agent AGENT [AGENT ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure and install one of the listed agents for a specific VOLTTRON instance + +**update-config-store --metadata-file METADATA_FILE_OR_DIR [METADATA_FILE_OR_DIR ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Command to bulk add/update multiple agent configurations to VOLTTRON's configuration store. This is especially useful +for automated deployment use cases. For example, a automated deployment setup where devices in a network are +detected, and configurations for platform driver and control agents such as ILCAgent are auto generated using semantic +tags. In such as scenario, adding all the generated configurations to the configuration store using the below command +is more efficient than running the "vctl config store" command once for each configuration + +This command can only be used when volttron instance is not running. + +Usage: + +.. code-block:: bash + + vcfg update-config-store --metadata-file + + +Format for Metadata file: + +.. code-block:: + + { "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... + } + + RabbitMQ Arguments ------------------ vcfg command to configure a single RabbitMQ instance of VOLTTRON. diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index ce559d5756..487d35ea3f 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -1118,8 +1118,8 @@ def _exit_with_metadata_error(): { "vip-id": [ { "config-name": "optional. name. defaults to config - "config": "json config or config file name", - "config-type": "optional. type of config. defaults to json" + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" }, ... ],... }""") @@ -1229,8 +1229,8 @@ def main(): { "vip-id": [ { "config-name": "optional. name. defaults to config - "config": "json config or config file name", - "config-type": "optional. type of config. defaults to json" + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" }, ... ],... }""") diff --git a/volttrontesting/platform/test_vcfg_config_update.py b/volttrontesting/platform/test_vcfg_config_update.py index 62fa7f11bd..c34fdfa652 100644 --- a/volttrontesting/platform/test_vcfg_config_update.py +++ b/volttrontesting/platform/test_vcfg_config_update.py @@ -16,8 +16,8 @@ { "vip-id": [ { "config-name": "optional. name. defaults to config - "config": "json config or config file name", - "config-type": "optional. type of config. defaults to json" + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" }, ... ],... }""" From 26e9b77433312d8700b6d647f6cb5071c678d4fc Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Mon, 23 Oct 2023 13:33:23 -0700 Subject: [PATCH 9/9] minor pep 8 fixes --- volttron/platform/instance_setup.py | 39 +++++++++++++---------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 487d35ea3f..83066aa24c 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -84,6 +84,7 @@ # Determines if VOLTTRON instance can remain running with vcfg use_active = "N" + def _load_config(): """Loads the config file if the path exists.""" path = os.path.join(get_home(), 'config') @@ -266,6 +267,7 @@ def _is_agent_installed(tag): def installs(agent_dir, tag, identity=None, post_install_func=None): def wrap(config_func): global available_agents + def func(*args, **kwargs): global use_active print('Configuring {}.'.format(agent_dir)) @@ -409,7 +411,7 @@ def _create_web_certs(): prompt = '\tOrganization:' cert_data['organization'] = prompt_response(prompt, mandatory=True) prompt = '\tOrganization Unit:' - cert_data['organization-unit'] = prompt_response(prompt,mandatory=True) + cert_data['organization-unit'] = prompt_response(prompt, mandatory=True) cert_data['common-name'] = get_platform_instance_name() + '-root-ca' data = {'C': cert_data.get('country'), 'ST': cert_data.get('state'), @@ -418,12 +420,13 @@ def _create_web_certs(): 'OU': cert_data.get('organization-unit'), 'CN': cert_data.get('common-name')} crts.create_root_ca(overwrite=False, **data) - copy(crts.cert_file(crts.root_ca_name),crts.cert_file(crts.trusted_ca_name)) + copy(crts.cert_file(crts.root_ca_name), crts.cert_file(crts.trusted_ca_name)) else: return 1 print("Creating new web server certificate.") - crts.create_signed_cert_files(name=PLATFORM_WEB + "-server", cert_type='server', ca_name=crts.root_ca_name, fqdn=get_hostname()) + crts.create_signed_cert_files(name=PLATFORM_WEB + "-server", cert_type='server', ca_name=crts.root_ca_name, + fqdn=get_hostname()) return 0 @@ -585,8 +588,7 @@ def do_web_enabled_zmq(vhome): # Full implies that it will have a port on it as well. Though if it's # not in the address that means that we haven't set it up before. - full_bind_web_address = config_opts.get('bind-web-address', - 'https://' + get_hostname()) + full_bind_web_address = config_opts.get('bind-web-address', 'https://' + get_hostname()) parsed = urlparse(full_bind_web_address) @@ -705,7 +707,7 @@ def get_cert_and_key(vhome): try: if certs.Certs.validate_key_pair(platform_web_cert, platform_web_key): print('\nThe following certificate and keyfile exists for web access over https: \n{}\n{}'.format( - platform_web_cert,platform_web_key)) + platform_web_cert, platform_web_key)) prompt = '\nDo you want to use these certificates for the web server?' if prompt_response(prompt, valid_answers=y_or_n, default='Y') in y: config_opts['web-ssl-cert'] = platform_web_cert @@ -720,8 +722,6 @@ def get_cert_and_key(vhome): print(e) pass - - # Either are there no valid existing certs or user decided to overwrite the existing file. # Prompt for new files while cert_error: @@ -760,10 +760,8 @@ def get_cert_and_key(vhome): else: cert_error = _create_web_certs() if not cert_error: - platform_web_cert = os.path.join(vhome, 'certificates/certs/', - PLATFORM_WEB+"-server.crt") - platform_web_key = os.path.join(vhome, 'certificates/private/', - PLATFORM_WEB + "-server.pem") + platform_web_cert = os.path.join(vhome, 'certificates/certs/', PLATFORM_WEB+"-server.crt") + platform_web_key = os.path.join(vhome, 'certificates/private/', PLATFORM_WEB + "-server.pem") config_opts['web-ssl-cert'] = platform_web_cert config_opts['web-ssl-key'] = platform_web_key @@ -800,8 +798,7 @@ def do_vcp(): except KeyError: vc_address = config_opts.get('volttron-central-address', - config_opts.get('bind-web-address', - 'https://' + get_hostname())) + config_opts.get('bind-web-address', 'https://' + get_hostname())) if not is_vc: parsed = urlparse(vc_address) address_only = vc_address @@ -971,8 +968,7 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met( - "drivers") or not _check_dependencies_met("web"): + if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): print("VCP dependencies not installed. Installing now...") if not _check_dependencies_met("drivers"): set_dependencies("drivers") @@ -1127,7 +1123,6 @@ def _exit_with_metadata_error(): def process_rmq_inputs(args_dict, instance_name=None): - #print(f"args_dict:{args_dict}, args") if not is_rabbitmq_available(): raise RuntimeError("Rabbitmq Dependencies not installed please run python bootstrap.py --rabbitmq") @@ -1175,9 +1170,11 @@ def process_rmq_inputs(args_dict, instance_name=None): else: print("Invalid installation type. Acceptable values single|federation|shovel") sys.exit(1) - setup_rabbitmq_volttron(args_dict['installation-type'], verbose, instance_name=instance_name, max_retries=args_dict['max_retries']) + setup_rabbitmq_volttron(args_dict['installation-type'], verbose, instance_name=instance_name, + max_retries=args_dict['max_retries']) else: - setup_rabbitmq_volttron(args_dict['installation-type'], verbose, prompt=True, instance_name=instance_name, max_retries=args_dict['max_retries']) + setup_rabbitmq_volttron(args_dict['installation-type'], verbose, prompt=True, instance_name=instance_name, + max_retries=args_dict['max_retries']) def main(): @@ -1220,8 +1217,8 @@ def main(): # start with just a metadata file support. # todo - add support vip-id, directory # vip-id, file with multiple configs etc. - #config_arg_group = config_store_parser.add_mutually_exclusive_group() - #meta_group = config_arg_group.add_mutually_exclusive_group() + # config_arg_group = config_store_parser.add_mutually_exclusive_group() + # meta_group = config_arg_group.add_mutually_exclusive_group() config_store_parser.add_argument('--metadata-file', required=True, nargs='+', help="""One or more metadata file or directory containing metadata file, where each metadata file contain details of configs for one or more agent instance