From 6bbd78f028a3a3285493d4ff3cb6264e7a0effa9 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 17 Nov 2020 19:55:58 -0500 Subject: [PATCH 01/15] pinned greenlet to compatible version for gevent --- requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.py b/requirements.py index 0791185e5e..0a8e9d260b 100644 --- a/requirements.py +++ b/requirements.py @@ -46,6 +46,7 @@ install_requires = [ 'gevent==20.6.1', + 'greenlet==0.4.16', 'grequests', 'requests==2.23.0', 'ply', From d65994ab109d487930a65cdde71325ba9e7c762c Mon Sep 17 00:00:00 2001 From: schandrika Date: Mon, 22 Feb 2021 16:29:09 -0800 Subject: [PATCH 02/15] fix for #2618 --- requirements.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.py b/requirements.py index 0791185e5e..0f893a4a94 100644 --- a/requirements.py +++ b/requirements.py @@ -47,6 +47,8 @@ install_requires = [ 'gevent==20.6.1', 'grequests', + 'greenlet==0.4.16', + 'idna<3,>=2.5', 'requests==2.23.0', 'ply', 'psutil', From 619dcfd84a60645db17806d81e96812685f3d932 Mon Sep 17 00:00:00 2001 From: schandrika Date: Mon, 22 Feb 2021 16:29:44 -0800 Subject: [PATCH 03/15] documented requirement for web packages for volttron 7.x. github issue #2619 --- docs/source/setup/index.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/setup/index.rst b/docs/source/setup/index.rst index a29b03b90b..0c554efd1f 100644 --- a/docs/source/setup/index.rst +++ b/docs/source/setup/index.rst @@ -46,7 +46,8 @@ Steps for ZMQ :: cd - python bootstrap.py + # VOLTTRON 7 required web packages by default so include --web + python3 bootstrap.py --web source env/bin/activate Proceed to `Testing the Installation`_. @@ -120,10 +121,11 @@ Steps for RabbitMQ cd volttron - # bootstrap.py --help will show you all of the "package options" such as + # python3 bootstrap.py --help will show you all of the "package options" such as # installing required packages for volttron central or the platform agent. + # In VOLTTRON 7 web packages are required so include --web in addition to --rabbitmq - python bootstrap.py --rabbitmq [optional install directory defaults to + python3 bootstrap.py --web --rabbitmq [optional install directory defaults to /rabbitmq_server] .. note:: If your PYTHON_PATH is configured for Python 2.7, you'll need to use From b51576c62cbd9aae0dbf60dbe7bacffe709f5467 Mon Sep 17 00:00:00 2001 From: schandrika Date: Mon, 22 Feb 2021 16:33:15 -0800 Subject: [PATCH 04/15] documented requirement for web packages for volttron 7.x. github issue #2619 --- docs/source/setup/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/setup/index.rst b/docs/source/setup/index.rst index 0c554efd1f..a8675e2995 100644 --- a/docs/source/setup/index.rst +++ b/docs/source/setup/index.rst @@ -46,7 +46,7 @@ Steps for ZMQ :: cd - # VOLTTRON 7 required web packages by default so include --web + # VOLTTRON 7 requires web packages by default so include --web python3 bootstrap.py --web source env/bin/activate @@ -125,11 +125,11 @@ Steps for RabbitMQ # installing required packages for volttron central or the platform agent. # In VOLTTRON 7 web packages are required so include --web in addition to --rabbitmq - python3 bootstrap.py --web --rabbitmq [optional install directory defaults to + python bootstrap.py --web --rabbitmq [optional install directory defaults to /rabbitmq_server] .. note:: If your PYTHON_PATH is configured for Python 2.7, you'll need to use - ``python3 bootstrap.py ..`` + ``python3 bootstrap.py`` instead of ``python bootstrap.py`` This will build the platform and create a virtual Python environment and dependencies for RabbitMQ. It also installs RabbitMQ server as the current user. From 41eac5e1ac374be2c17c1260c8f4befb83a8b726 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" Date: Sat, 18 Sep 2021 19:18:18 -0700 Subject: [PATCH 05/15] Pin 7.x branch #2795 --- requirements.py | 144 +++++++++++++++++------------------------------- 1 file changed, 52 insertions(+), 92 deletions(-) diff --git a/requirements.py b/requirements.py index 0f893a4a94..012fc90cbc 100644 --- a/requirements.py +++ b/requirements.py @@ -36,102 +36,62 @@ # under Contract DE-AC05-76RL01830 # }}} -# These need to be importable by bootstrap.py. If we put them in -# setup.py the import may fail if setuptools in not installed -# in the global python3. -option_requirements = [ - ('pyzmq', ['--zmq=bundled']), -] +extras_require = { 'crate': ['crate==0.26.0'], + 'databases': [ 'mysql-connector-python-rf==2.2.2', + 'pymongo==3.7.2', + 'bson==0.5.7', + 'crate==0.26.0', + 'influxdb==5.3.1', + 'psycopg2-binary==2.8.6'], + 'dnp3': ['pydnp3'], + 'documentation': [ 'mock==4.0.3', + 'Sphinx==4.1.2', + 'sphinx-rtd-theme==0.5.2', + 'sphinx==3.3.0', + 'm2r2==0.3.1'], + 'drivers': [ 'pymodbus==2.3.0', + 'bacpypes==0.16.7', + 'modbus-tk==1.1.1', + 'pyserial==3.4'], + 'influxdb': ['influxdb==5.3.1'], + 'market': ['numpy==1.19.5', 'transitions==0.8.8'], + 'mongo': ['pymongo==3.7.2', + 'bson==0.5.7'], + 'mysql': ['mysql-connector-python-rf==2.2.2'], + 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], + 'postgres': ['psycopg2-binary==2.8.6'], + 'testing': [ 'mock==4.0.3', + 'pytest==6.2.4', + 'pytest-timeout==1.4.2', + 'websocket-client==1.2.1', + 'deepdiff==5.5.0'], + 'weather': ['Pint==0.17'], + 'web': [ 'ws4py==0.5.1', + 'PyJWT==1.7.1', + 'Jinja2==2.11.2', + 'passlib==1.7.2', + 'argon2-cffi==20.1.0', + 'Werkzeug==1.0.1']} -install_requires = [ - 'gevent==20.6.1', - 'grequests', - 'greenlet==0.4.16', +install_requires = [ 'gevent==20.6.0', + 'grequests==0.6.0', 'idna<3,>=2.5', 'requests==2.23.0', - 'ply', - 'psutil', - 'python-dateutil', - 'pytz', - 'PyYAML', - 'pyzmq', - 'setuptools', - 'tzlocal', + 'ply==3.11', + 'psutil==5.7.0', + 'python-dateutil==2.8.1', + 'pytz==2020.1', + 'PyYAML==5.3.1', + 'pyzmq==19.0.1', + 'setuptools==39.0.1', + 'tzlocal==2.1', 'pyOpenSSL==19.0.0', 'cryptography==2.3', - # Cross platform way of handling changes in file/directories. - # https://github.com/Bogdanp/watchdog_gevent - 'watchdog-gevent', - 'wheel==0.30' -] + 'watchdog-gevent==0.1.1', + 'wheel==0.30'] + +option_requirements = [('pyzmq==19.0.1', ['--zmq=bundled'])] + + -extras_require = { - 'crate': [ # crate databases - 'crate' - ], - 'databases': [ # Support for all known databases - 'mysql-connector-python-rf', - 'pymongo', - 'crate', - 'influxdb', - 'psycopg2-binary' - ], - 'dnp3': [ # dnp3 agent requirements. - 'pydnp3' - ], - 'documentation': [ # Requirements for building the documentation - 'mock', - 'mysql-connector-python-rf', - 'psutil', - 'pymongo', - 'Sphinx', - 'recommonmark', - 'sphinx-rtd-theme' - ], - 'drivers': [ - 'pymodbus', - 'bacpypes==0.16.7', - 'modbus-tk', - 'pyserial' - ], - 'influxdb': [ # influxdb historian requirements. - 'influxdb' - ], - 'market': [ # Requirements for the market service - 'numpy', - 'transitions', - ], - 'mongo': [ # mongo databases - 'pymongo', - ], - 'mysql': [ # mysql databases - 'mysql-connector-python-rf', - ], - 'pandas': [ # numpy and pandas for applications - 'numpy', - 'pandas', - ], - 'postgres': [ # numpy and pandas for applications - 'psycopg2-binary' - ], - 'testing': [ # Testing infrastructure dependencies - 'mock', - 'pytest', - 'pytest-timeout', - 'websocket-client', - # Allows us to compare nested dictionaries easily. - 'deepdiff' - ], - 'web': [ # Web support for launching web based agents including ssl and json web tokens. - 'ws4py', - 'PyJWT', - 'Jinja2', - 'passlib', - 'argon2-cffi', - 'Werkzeug' - ], - 'weather': [ - 'Pint' - ], -} From 09b9d3442ae310ab86593812c7e27d403fb8682f Mon Sep 17 00:00:00 2001 From: shwetha niddodi Date: Tue, 21 Sep 2021 12:52:39 -0700 Subject: [PATCH 06/15] Updating rabbitmq dependency script since the rabbitmq server repos have changed --- scripts/rabbit_dependencies.sh | 134 ++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/scripts/rabbit_dependencies.sh b/scripts/rabbit_dependencies.sh index 0c08b9389b..ab1f7f542e 100755 --- a/scripts/rabbit_dependencies.sh +++ b/scripts/rabbit_dependencies.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash -list=( bionic artful stretch buster trusty xenial ) +set -e + +list=( bionic buster ) +declare -A ubuntu_versions +ubuntu_versions=( ["ubuntu-16.04"]="xenial" ["ubuntu-18.04"]="bionic" ["ubuntu-20.04"]="focal") function exit_on_error { rc=$? @@ -14,8 +18,8 @@ function exit_on_error { function print_usage { echo " Command Usage: -/rabbit_dependencies.sh -Valid Debian distributions: ${list[@]} +/rabbit_dependencies.sh or centos version> +Valid Raspbian/Debian distributions: ${list[@]} ${!ubuntu_versions[@]} Valid centos versions: 6, 7, 8 " exit 0 @@ -26,33 +30,39 @@ Valid centos versions: 6, 7, 8 function install_on_centos { if [[ "$DIST" == "6" ]]; then - erlang_url='https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/21/el/6' + erlang_url='https://packagecloud.io/rabbitmq/erlang/el/6/$basearch' + erlang_package_name='erlang-21.3.8.21-1.el6.x86_64' elif [[ "$DIST" == "7" ]]; then - erlang_url='https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/21/el/7' + erlang_url='https://packagecloud.io/rabbitmq/erlang/el/7/$basearch' + erlang_package_name='erlang-21.3.8.21-1.el7.x86_64' elif [[ "$DIST" == "8" ]]; then - erlang_url='https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/21/el/8' + erlang_url='https://packagecloud.io/rabbitmq/erlang/el/8/$basearch' + erlang_package_name='erlang-21.3.8.21-1.el8.x86_64' else printf "Invalid centos version. 6, 7, and 8 are the only compatible versions\n" print_usage fi repo="## In /etc/yum.repos.d/rabbitmq-erlang.repo -[rabbitmq-erlang] -name=rabbitmq-erlang +[rabbitmq_erlang] +name=rabbitmq_erlang baseurl=$erlang_url -gpgcheck=1 -gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc -repo_gpgcheck=0 -enabled=1" - - if [[ ! -f "/etc/yum.repos.d/rabbitmq-erlang.repo" ]]; then - echo "$repo" | ${prefix} tee -a /etc/yum.repos.d/rabbitmq-erlang.repo +repo_gpgcheck=1 +gpgcheck=0 +enabled=1 +gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +" + if [[ -f "/etc/yum.repos.d/rabbitmq-erlang.repo" ]]; then + echo "\n/etc/yum.repos.d/rabbitmq-erlang.repo exists. renaming current file to rabbitmq-erlang.repo.old\n" + mv /etc/yum.repos.d/rabbitmq-erlang.repo /etc/yum.repos.d/rabbitmq-erlang.repo.old exit_on_error - else - echo "\nrepo file /etc/yum.repos.d/rabbitmq-erlang.repo already exists\n" - fi - ${prefix} yum install erlang - exit_on_error + fi + echo "$repo" | ${prefix} tee -a /etc/yum.repos.d/rabbitmq-erlang.repo + ${prefix} yum install $erlang_package_name + exit_on_error } function install_on_debian { @@ -64,14 +74,32 @@ function install_on_debian { fi done + if [[ "$FOUND" != "1" ]]; then + # check if ubuntu-version was passed if so map it to name + for ubuntu_version in "${!ubuntu_versions[@]}"; do + if [[ "$DIST" == "$ubuntu_version" ]]; then + FOUND=1 + DIST="${ubuntu_versions[$ubuntu_version]}" + break + fi + done + fi + if [[ "$FOUND" != "1" ]]; then echo "Invalid distribution found" print_usage fi echo "installing ERLANG" - ${prefix} apt-get install apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + ${prefix} apt-get update + if [[ "$DIST" == "xenial" ]] || [[ "$DIST" == "bionic" ]]; then + ${prefix} apt-get install -y apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + else + ${prefix} apt-get install -y apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-gtk3-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + fi + set +e ${prefix} apt-get purge -yf erlang* + set -e # Add the signing key wget -O- https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc | ${prefix} apt-key add - @@ -80,33 +108,44 @@ function install_on_debian { fi version=${erlang_package_version} + common_deb_pkgs="\ + erlang-asn1=$version \ + erlang-base=$version \ + erlang-crypto=$version \ + erlang-diameter=$version \ + erlang-edoc=$version \ + erlang-eldap=$version \ + erlang-erl-docgen=$version \ + erlang-eunit=$version \ + erlang-inets=$version \ + erlang-mnesia=$version \ + erlang-odbc=$version \ + erlang-os-mon=$version \ + erlang-parsetools=$version \ + erlang-public-key=$version \ + erlang-runtime-tools=$version \ + erlang-snmp=$version \ + erlang-ssh=$version \ + erlang-ssl=$version \ + erlang-syntax-tools=$version \ + erlang-tools=$version \ + erlang-xmerl=$version \ + " + x86_pkgs="\ + erlang-ic=$version \ + erlang-inviso=$version \ + erlang-percept=$version \ + " + to_install="" + if [[ $is_arm == "FALSE" ]]; then + to_install="${common_deb_pkgs} ${x86_pkgs}" + else + to_install="${common_deb_pkgs}" + fi + ${prefix} apt-get update ${prefix} apt-get install -yf - ${prefix} apt-get install -y "erlang-asn1=$version" \ - "erlang-base=$version" \ - "erlang-crypto=$version" \ - "erlang-diameter=$version" \ - "erlang-edoc=$version" \ - "erlang-eldap=$version" \ - "erlang-erl-docgen=$version" \ - "erlang-eunit=$version" \ - "erlang-ic=$version" \ - "erlang-inets=$version" \ - "erlang-inviso=$version" \ - "erlang-mnesia=$version" \ - "erlang-odbc=$version" \ - "erlang-os-mon=$version" \ - "erlang-parsetools=$version" \ - "erlang-percept=$version" \ - "erlang-public-key=$version" \ - "erlang-runtime-tools=$version" \ - "erlang-snmp=$version" \ - "erlang-ssh=$version" \ - "erlang-ssl=$version" \ - "erlang-syntax-tools=$version" \ - "erlang-tools=$version" \ - "erlang-xmerl=$version" - + ${prefix} apt-get install -y ${to_install} ${prefix} apt-get install -y "erlang-nox=$version" } @@ -118,14 +157,17 @@ if [[ ${user} == 'root' ]]; then else prefix="sudo" fi +is_arm="FALSE" ${prefix} pwd > /dev/null if [[ "$os_name" == "debian" ]]; then erlang_package_version="1:22.1.8.1-1" + is_arm="FALSE" install_on_debian elif [[ "$os_name" == "raspbian" ]]; then erlang_package_version="1:21.2.6+dfsg-1" + is_arm="TRUE" install_on_debian elif [[ "$os_name" == "centos" ]]; then install_on_centos From 5f19a749fda2d7c0a2dff3c2922577ccdf9093ed Mon Sep 17 00:00:00 2001 From: "C. Allwardt" Date: Tue, 21 Sep 2021 14:41:45 -0700 Subject: [PATCH 07/15] Moved all requirements to requirements.py rather than split across bootstrap.py as well. --- bootstrap.py | 5 ----- requirements.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index d9d4a397a4..c1226c4a73 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -130,11 +130,6 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi path = os.path.dirname(__file__) or '.' _log.info('%sing required packages', 'Build' if wheeling else 'Install') - # We must install wheel first to eliminate a bunch of scary looking - # errors at first install. - # TODO Look towards fixing the packaging so that it works with 0.31 - pip('install', ['wheel==0.30'], verbose, True, offline=offline) - # Build option_requirements separately to pass install options build_option = '--build-option' if wheeling else '--install-option' for requirement, options in option_requirements: diff --git a/requirements.py b/requirements.py index 012fc90cbc..af3229af8c 100644 --- a/requirements.py +++ b/requirements.py @@ -91,7 +91,7 @@ 'watchdog-gevent==0.1.1', 'wheel==0.30'] -option_requirements = [('pyzmq==19.0.1', ['--zmq=bundled'])] +option_requirements = [('wheel==0.30'), ('pyzmq==19.0.1', ['--zmq=bundled'])] From e8616973754ab03b4a4ad84bb22210ea6e853957 Mon Sep 17 00:00:00 2001 From: shwetha niddodi Date: Tue, 21 Sep 2021 15:20:35 -0700 Subject: [PATCH 08/15] Fixing requirements.py related to wheels entry in option requirements --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index af3229af8c..f016972546 100644 --- a/requirements.py +++ b/requirements.py @@ -91,7 +91,7 @@ 'watchdog-gevent==0.1.1', 'wheel==0.30'] -option_requirements = [('wheel==0.30'), ('pyzmq==19.0.1', ['--zmq=bundled'])] +option_requirements = [('wheel==0.30', []), ('pyzmq==19.0.1', ['--zmq=bundled'])] From 5bb3d9b9f51d230989a4a5dac85071a7f96d2f89 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 14 Apr 2022 22:47:32 -0400 Subject: [PATCH 09/15] seems to be working, implemented only for reading --- .gitignore | 1 + .../master_driver/driver_locks.py | 38 +++++++++++++++++- .../interfaces/modbus_tk/__init__.py | 5 ++- .../interfaces/modbus_tk/client.py | 39 ++++++++++++++++--- .../interfaces/modbus_tk/helpers.py | 2 +- 5 files changed, 76 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 95d07548cf..7ea72ba123 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ rabbitmq.log htmlcov/ MnesiaCore.* rabbitmq-server.download.tar.xz +Pipfile diff --git a/services/core/MasterDriverAgent/master_driver/driver_locks.py b/services/core/MasterDriverAgent/master_driver/driver_locks.py index 310d66cba1..20a7b27c47 100644 --- a/services/core/MasterDriverAgent/master_driver/driver_locks.py +++ b/services/core/MasterDriverAgent/master_driver/driver_locks.py @@ -36,9 +36,15 @@ # under Contract DE-AC05-76RL01830 # }}} +from collections import defaultdict +import logging from gevent.lock import BoundedSemaphore, DummySemaphore from contextlib import contextmanager +from volttron.platform.agent import utils +utils.setup_logging() +_log = logging.getLogger(__name__) + _socket_lock = None def configure_socket_lock(max_connections=0): @@ -82,4 +88,34 @@ def publish_lock(): yield finally: _publish_lock.release() - \ No newline at end of file + +_client_socket_locks = defaultdict(lambda: None) + +def configure_client_socket_lock(address, port, max_connections=0): + _log.debug("Configuring client socket lock for {}:{}".format(address, port)) + global _client_socket_locks + if _client_socket_locks[(address, port)] is not None: + if isinstance(_client_socket_locks[(address, port)], DummySemaphore) or isinstance(_client_socket_locks[(address, port)], BoundedSemaphore): + _log.debug(f"Client socket lock already configured for {address}:{port}") + return + else: + raise RuntimeError("client socket lock already configured!") + if max_connections < 1: + _client_socket_locks[(address, port)] = DummySemaphore() + else: + _client_socket_locks[(address, port)] = BoundedSemaphore(max_connections) + +@contextmanager +def client_socket_locks(address, port): + global _client_socket_locks + lock = _client_socket_locks[(address, port)] + _log.debug(f"Acquiring client socket lock ({type(lock)}) for {address}:{port} at {id(lock)}") + if lock is None: + _log.debug(f"socket_lock not configured {address}:{port}") + raise RuntimeError("socket_lock not configured!") + lock.acquire() + try: + yield + finally: + _log.debug(f"Releasing client socket lock ({type(lock)}) for {address}:{port} at {id(lock)}") + lock.release() diff --git a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/__init__.py b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/__init__.py index 67a5620bd9..d9c053797a 100644 --- a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/__init__.py +++ b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/__init__.py @@ -41,6 +41,7 @@ from master_driver.interfaces import BaseRegister, BaseInterface, BasicRevert from master_driver.interfaces.modbus_tk import helpers from master_driver.interfaces.modbus_tk.maps import Map +from master_driver.driver_locks import configure_client_socket_lock import logging import struct @@ -311,11 +312,13 @@ def configure(self, config_dict, registry_config_lst): endian=endian, registry_config_lst=selected_registry_config_lst ).get_class() + configure_client_socket_lock(device_address, port, max_connections=1) self.modbus_client = modbus_client_class(device_address=device_address, port=port, slave_address=slave_address, - write_single_values=write_single_values) + write_single_values=write_single_values, + ) # Set modbus client transport based on device configure if port: diff --git a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/client.py b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/client.py index 8aa1760262..6c3978cbaf 100644 --- a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/client.py +++ b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/client.py @@ -62,18 +62,24 @@ class MyModbusMaster (Client): from datetime import datetime import collections import struct +import gevent import serial import six.moves import logging import math +from volttron.platform.agent import utils import modbus_tk.defines as modbus_constants import modbus_tk.modbus_tcp as modbus_tcp import modbus_tk.modbus_rtu as modbus_rtu -from modbus_tk.exceptions import ModbusError +from modbus_tk.exceptions import ModbusError, ModbusInvalidResponseError +from master_driver.driver_locks import client_socket_locks from . import helpers + +utils.setup_logging() +_log = logging.getLogger(__name__) logger = logging.getLogger(__name__) # In cache representation of modbus field. @@ -324,7 +330,7 @@ def fix_address(self, address_style): self._address = self._address - helpers.TABLE_ADDRESS[self._table] elif address_style == helpers.ADDRESS_OFFSET_PLUS_ONE: self._address = self._address - 1 - if self._address < 0 or self._address > 10000: + if self._address < 0 or self._address > (2 ** 16 - 1): raise Exception("Modbus address out of range for table.") @@ -622,6 +628,8 @@ def __init__(self, *args, **kwargs): self._error_count = 0 def set_transport_tcp(self, hostname, port, timeout_in_sec=1.0): + self.device_address = hostname + self.port = port self.client = modbus_tcp.TcpMaster(host=hostname, port=int(port), timeout_in_sec=timeout_in_sec) return self @@ -671,7 +679,7 @@ def get_request(self, field): return self.__meta[helpers.META_REQUEST_MAP].get(field, None) def read_request(self, request): - logger.debug("Requesting: %s", request) + logger.debug(f"Requesting: {request} on {self.device_address}:{self.port}-{self.slave_address}") try: results = self.client.execute( self.slave_address, @@ -686,13 +694,32 @@ def read_request(self, request): if "Exception code" in err.message: raise Exception("{0}: {1}".format(err.message, helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) - logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) + _log.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): requests = self.__meta[helpers.META_REQUESTS] self._data.clear() - for r in requests: - self.read_request(r) + with client_socket_locks(self.device_address, self.port): + _log.debug(f"entered lock for {self.device_address}:{self.port}-{self.slave_address}") + for r in requests: + retries = 3 + while retries > 0: + exception_flag = False + try: + self.read_request(r) + continue + except ConnectionResetError: + exception_flag = True + _log.warning("ConnectionResetError on read_all()") + except ModbusInvalidResponseError: + exception_flag = True + _log.warning("ModbusInvalidResponseError on read_all()") + if exception_flag: + self.client.close() + gevent.sleep(1.0) + self.client.open() + retries -= 1 + _log.debug(f"left lock for {self.device_address}:{self.port}-{self.slave_address}") def dump_all(self): self.read_all() diff --git a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/helpers.py b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/helpers.py index 866b25ab77..6680636da4 100644 --- a/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/helpers.py +++ b/services/core/MasterDriverAgent/master_driver/interfaces/modbus_tk/helpers.py @@ -192,7 +192,7 @@ def inverse_func(value): if value < 0: return (0 - (value / float(multiplier))) - 0xFFFF else: - return (value / float(multipliers)) + return (value / float(multiplier)) except TypeError: #string return value except ZeroDivisionError: From dda15f053af3b398db531303a5f4c1fc74dc2338 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Mon, 25 Apr 2022 14:40:18 -0700 Subject: [PATCH 10/15] Update logs --- .../platform_driver/interfaces/modbus_tk/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index 3769cb485a..cca796a336 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -78,8 +78,8 @@ class MyModbusMaster (Client): from . import helpers -utils.setup_logging() -_log = logging.getLogger(__name__) +# utils.setup_logging() +# _log = logging.getLogger(__name__) logger = logging.getLogger(__name__) # In cache representation of modbus field. @@ -694,13 +694,13 @@ def read_request(self, request): if "Exception code" in err.message: raise Exception("{0}: {1}".format(err.message, helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) - _log.warning("modbus read_all() failure on request: %s\tError: %s", request, err) + logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): requests = self.__meta[helpers.META_REQUESTS] self._data.clear() with client_socket_locks(self.device_address, self.port): - _log.debug(f"entered lock for {self.device_address}:{self.port}-{self.slave_address}") + logger.debug(f"entered lock for {self.device_address}:{self.port}-{self.slave_address}") for r in requests: retries = 3 while retries > 0: @@ -710,16 +710,16 @@ def read_all(self): continue except ConnectionResetError: exception_flag = True - _log.warning("ConnectionResetError on read_all()") + logger.warning("ConnectionResetError on read_all()") except ModbusInvalidResponseError: exception_flag = True - _log.warning("ModbusInvalidResponseError on read_all()") + logger.warning("ModbusInvalidResponseError on read_all()") if exception_flag: self.client.close() gevent.sleep(1.0) self.client.open() retries -= 1 - _log.debug(f"left lock for {self.device_address}:{self.port}-{self.slave_address}") + logger.debug(f"left lock for {self.device_address}:{self.port}-{self.slave_address}") def dump_all(self): self.read_all() From 0ccf2a1d7e86df07ed0f9f08133c30bae115e768 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Mon, 25 Apr 2022 14:50:29 -0700 Subject: [PATCH 11/15] Fix reference from master_driver to platform_driver --- .../platform_driver/interfaces/modbus_tk/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index cca796a336..0d7da56487 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -73,7 +73,7 @@ class MyModbusMaster (Client): import modbus_tk.modbus_tcp as modbus_tcp import modbus_tk.modbus_rtu as modbus_rtu from modbus_tk.exceptions import ModbusError, ModbusInvalidResponseError -from master_driver.driver_locks import client_socket_locks +from platform_driver.driver_locks import client_socket_locks from . import helpers From 0401b75e08e14f6930a99adb4fa470746a96af19 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Tue, 3 May 2022 00:15:33 +0000 Subject: [PATCH 12/15] Update init for modbustk --- .../core/PlatformDriverAgent/platform_driver/driver_locks.py | 3 +++ .../platform_driver/interfaces/modbus_tk/__init__.py | 4 ++++ .../platform_driver/interfaces/modbus_tk/client.py | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/driver_locks.py b/services/core/PlatformDriverAgent/platform_driver/driver_locks.py index d7e4636c8f..eeda10fe1b 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver_locks.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver_locks.py @@ -109,9 +109,12 @@ def configure_client_socket_lock(address, port, max_connections=0): def client_socket_locks(address, port): global _client_socket_locks lock = _client_socket_locks[(address, port)] + _log.debug(f"ADDRESS: {address}") + _log.debug(f"PORT: {port}") _log.debug(f"Acquiring client socket lock ({type(lock)}) for {address}:{port} at {id(lock)}") if lock is None: _log.debug(f"socket_lock not configured {address}:{port}") + _log.debug(f"lock is None: lock: {lock}, type: {type(lock)}, id ${id(lock)}") raise RuntimeError("socket_lock not configured!") lock.acquire() try: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/__init__.py index 83bcff4ab3..07596e8d75 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/__init__.py @@ -41,6 +41,7 @@ from platform_driver.interfaces import BaseRegister, BaseInterface, BasicRevert from platform_driver.interfaces.modbus_tk import helpers from platform_driver.interfaces.modbus_tk.maps import Map +from platform_driver.driver_locks import configure_client_socket_lock import logging import struct @@ -311,6 +312,9 @@ def configure(self, config_dict, registry_config_lst): endian=endian, registry_config_lst=selected_registry_config_lst ).get_class() + + _log.debug(f"CONFIGURING CLIENT LOCKS FOR MODBUSTK") + configure_client_socket_lock(device_address, port, max_connections=1) self.modbus_client = modbus_client_class(device_address=device_address, port=port, diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index 0d7da56487..48b2e6a02a 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -711,9 +711,10 @@ def read_all(self): except ConnectionResetError: exception_flag = True logger.warning("ConnectionResetError on read_all()") - except ModbusInvalidResponseError: + except ModbusInvalidResponseError as e: exception_flag = True logger.warning("ModbusInvalidResponseError on read_all()") + logger.warning(f"The exception MSG: {e}") if exception_flag: self.client.close() gevent.sleep(1.0) From b863431faa215e1abc173c6c1baf5c6d6564b9bc Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Wed, 4 May 2022 19:53:56 +0000 Subject: [PATCH 13/15] Update drivers --- .../platform_driver/driver.py | 3 ++ .../interfaces/modbus_tk/client.py | 28 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/driver.py b/services/core/PlatformDriverAgent/platform_driver/driver.py index 6f9c479a90..20aad887d0 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver.py @@ -243,7 +243,10 @@ def periodic_read(self, now): try: results = self.interface.scrape_all() register_names = self.interface.get_register_names_view() + _log.debug(f"results keys: {results.keys}") + _log.debug(f"register_names keys: {register_names}") for point in (register_names - results.keys()): + _log.debug(f"Scraping point: {point}") depth_first_topic = self.base_topic(point=point) _log.error("Failed to scrape point: "+depth_first_topic) except (Exception, gevent.Timeout) as ex: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index 48b2e6a02a..167e155eb9 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -679,7 +679,6 @@ def get_request(self, field): return self.__meta[helpers.META_REQUEST_MAP].get(field, None) def read_request(self, request): - logger.debug(f"Requesting: {request} on {self.device_address}:{self.port}-{self.slave_address}") try: results = self.client.execute( self.slave_address, @@ -689,37 +688,50 @@ def read_request(self, request): data_format=request.formatting, threadsafe=False ) + logger.debug("Successfully read the request...") self._data.update(request.parse_values(results)) except (AttributeError, ModbusError) as err: if "Exception code" in err.message: - raise Exception("{0}: {1}".format(err.message, - helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) + msg = "{0}: {1}".format(err.message, + helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED")) + logger.debug(msg) + raise Exception("{0}: {1}".format(msg)) logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): requests = self.__meta[helpers.META_REQUESTS] self._data.clear() with client_socket_locks(self.device_address, self.port): - logger.debug(f"entered lock for {self.device_address}:{self.port}-{self.slave_address}") + logger.debug(f"Entered lock for {self.device_address}:{self.port}-{self.slave_address}") + logger.debug(f"Total requests to be read: {len(requests)}") for r in requests: + logger.debug(f"Attempting to read_request on request: {r}") retries = 3 while retries > 0: + logger.debug(f"Retry: {retries}") exception_flag = False try: self.read_request(r) - continue + break except ConnectionResetError: exception_flag = True - logger.warning("ConnectionResetError on read_all()") + logger.warning("ConnectionResetError on read_request()") + logger.warning(f"Error response: {e}") except ModbusInvalidResponseError as e: exception_flag = True - logger.warning("ModbusInvalidResponseError on read_all()") - logger.warning(f"The exception MSG: {e}") + logger.warning("ModbusInvalidResponseError on read_request()") + logger.warning(f"Error response: {e}") if exception_flag: self.client.close() gevent.sleep(1.0) self.client.open() retries -= 1 + + if retries == 0: + logger.debug(f"Failed to read request: {r}") + else: + logger.debug(f"Succesfully read the request on retry: {retries}") + logger.debug(f"left lock for {self.device_address}:{self.port}-{self.slave_address}") def dump_all(self): From ccba16d64ec9f02a5b1447108c18fb7859a3ebf1 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Tue, 10 May 2022 20:52:04 +0000 Subject: [PATCH 14/15] Add test configs for modbus --- .../modbus_bess_board/bess_modbus.config | 12 ++++++++++ .../modbus_bess_board/bess_modbus.csv | 2 ++ .../modbus_demo_board/m2000_rtu.config | 12 ++++++++++ .../modbus_demo_board/m2000_rtu.csv | 23 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 hotfix_test_configs/modbus_bess_board/bess_modbus.config create mode 100644 hotfix_test_configs/modbus_bess_board/bess_modbus.csv create mode 100644 hotfix_test_configs/modbus_demo_board/m2000_rtu.config create mode 100644 hotfix_test_configs/modbus_demo_board/m2000_rtu.csv diff --git a/hotfix_test_configs/modbus_bess_board/bess_modbus.config b/hotfix_test_configs/modbus_bess_board/bess_modbus.config new file mode 100644 index 0000000000..2c66e3bb79 --- /dev/null +++ b/hotfix_test_configs/modbus_bess_board/bess_modbus.config @@ -0,0 +1,12 @@ +{ + "driver_config": {"device_address": "10.70.151.246", + "slave_id": 1, + "port": 1025}, + "campus": "PNNL", + "building": "BESS", + "unit": "BOARD", + "driver_type": "modbus", + "registry_config":"config://bess_modbus.csv", + "interval": 5, + "timezone": "Pacific" +} diff --git a/hotfix_test_configs/modbus_bess_board/bess_modbus.csv b/hotfix_test_configs/modbus_bess_board/bess_modbus.csv new file mode 100644 index 0000000000..f193869b1a --- /dev/null +++ b/hotfix_test_configs/modbus_bess_board/bess_modbus.csv @@ -0,0 +1,2 @@ +Volttron Point Name,Point Address,Modbus Register,Units,Writable +INV_REAL_PWR,136,int16,0.01*Watts,TRUE \ No newline at end of file diff --git a/hotfix_test_configs/modbus_demo_board/m2000_rtu.config b/hotfix_test_configs/modbus_demo_board/m2000_rtu.config new file mode 100644 index 0000000000..2de6521d69 --- /dev/null +++ b/hotfix_test_configs/modbus_demo_board/m2000_rtu.config @@ -0,0 +1,12 @@ +{ + "driver_config": {"device_address": "130.20.24.148", + "slave_id": 8, + "port": 502}, + "campus": "PNNL", + "building": "DEMO", + "unit": "M2000", + "driver_type": "modbus", + "registry_config":"config://m2000_rtu.csv", + "interval": 5, + "timezone": "Pacific" +} diff --git a/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv b/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv new file mode 100644 index 0000000000..0c8aaa72dd --- /dev/null +++ b/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv @@ -0,0 +1,23 @@ +Volttron Point Name,Units,Writable,Point Address,Notes,Modbus Register,Multiplier +SupplyTemp,degC,FALSE,0,,>H,100 +ReturnTemp,degC,FALSE,1,,>H,100 +OutsideTemp,degC,FALSE,2,,>H,100 +Occupancy,None,FALSE,3,,>H,1 +SecondStageCoolingStatus,None,FALSE,12,,>H,1 +FirstStageCoolingStatus,None,FALSE,4,,>H,1 +SupplyFanCommand,None,FALSE,15,,>H,1 +EconomizerPositionStatus,%,FALSE,17,,>H,1 +SupplyFanProof,None,FALSE,23,,>H,1 +Demand,%,FALSE,26,,>H,1 +CoolOverride1,None,TRUE,124,,>H,1 +CoolOVerride2,None,TRUE,125,,>H,1 +HeatOverride,None,TRUE,128,,>H,1 +ScheduleOverride,None,TRUE,131,,>H,1 +EconoChangeTemperature,degC,TRUE,150,,>H,100 +EconoChangeDifference,degC,TRUE,151,,>H,100 +MinimumOutdoorAirDamperSignal,%,TRUE,154,,>H,1 +EconomizerPositionOverride,%,TRUE,274,,>H,1 +EnableAbsoluteOverrides,None,TRUE,111,,>H,1 +SupplyFanOverride,None,TRUE,130,,>H,1 +FirstStageCoolingDemandSetPoint,None,TRUE,12,,>H,1 +SecondStageCoolingDemandSetPoint,None,TRUE,14,,>H,1 From a2b1d08dc203bc38af3de7bf25ad9123960d30b7 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Tue, 24 May 2022 17:32:18 +0000 Subject: [PATCH 15/15] Add lock counter and logs --- .gitignore | 6 ++ .../modbus_bess_board/bess_modbus.config | 12 --- .../modbus_bess_board/bess_modbus.csv | 2 - .../modbus_demo_board/m2000_rtu.config | 12 --- .../modbus_demo_board/m2000_rtu.csv | 23 ----- .../platform_driver/driver.py | 6 +- .../platform_driver/driver_locks.py | 7 ++ .../interfaces/modbus_tk/client.py | 89 +++++++++++++------ 8 files changed, 82 insertions(+), 75 deletions(-) delete mode 100644 hotfix_test_configs/modbus_bess_board/bess_modbus.config delete mode 100644 hotfix_test_configs/modbus_bess_board/bess_modbus.csv delete mode 100644 hotfix_test_configs/modbus_demo_board/m2000_rtu.config delete mode 100644 hotfix_test_configs/modbus_demo_board/m2000_rtu.csv diff --git a/.gitignore b/.gitignore index 3a60b6fff4..8bed92dec4 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,9 @@ rabbitmq-server.download.tar.xz /MagicMock/mock/ /docs/source/volttron_api/ Pipfile +modbus_configs +modbus_tk_configs +activateenv +volttron.service +upgrade-scripts +volttron_modbus.log \ No newline at end of file diff --git a/hotfix_test_configs/modbus_bess_board/bess_modbus.config b/hotfix_test_configs/modbus_bess_board/bess_modbus.config deleted file mode 100644 index 2c66e3bb79..0000000000 --- a/hotfix_test_configs/modbus_bess_board/bess_modbus.config +++ /dev/null @@ -1,12 +0,0 @@ -{ - "driver_config": {"device_address": "10.70.151.246", - "slave_id": 1, - "port": 1025}, - "campus": "PNNL", - "building": "BESS", - "unit": "BOARD", - "driver_type": "modbus", - "registry_config":"config://bess_modbus.csv", - "interval": 5, - "timezone": "Pacific" -} diff --git a/hotfix_test_configs/modbus_bess_board/bess_modbus.csv b/hotfix_test_configs/modbus_bess_board/bess_modbus.csv deleted file mode 100644 index f193869b1a..0000000000 --- a/hotfix_test_configs/modbus_bess_board/bess_modbus.csv +++ /dev/null @@ -1,2 +0,0 @@ -Volttron Point Name,Point Address,Modbus Register,Units,Writable -INV_REAL_PWR,136,int16,0.01*Watts,TRUE \ No newline at end of file diff --git a/hotfix_test_configs/modbus_demo_board/m2000_rtu.config b/hotfix_test_configs/modbus_demo_board/m2000_rtu.config deleted file mode 100644 index 2de6521d69..0000000000 --- a/hotfix_test_configs/modbus_demo_board/m2000_rtu.config +++ /dev/null @@ -1,12 +0,0 @@ -{ - "driver_config": {"device_address": "130.20.24.148", - "slave_id": 8, - "port": 502}, - "campus": "PNNL", - "building": "DEMO", - "unit": "M2000", - "driver_type": "modbus", - "registry_config":"config://m2000_rtu.csv", - "interval": 5, - "timezone": "Pacific" -} diff --git a/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv b/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv deleted file mode 100644 index 0c8aaa72dd..0000000000 --- a/hotfix_test_configs/modbus_demo_board/m2000_rtu.csv +++ /dev/null @@ -1,23 +0,0 @@ -Volttron Point Name,Units,Writable,Point Address,Notes,Modbus Register,Multiplier -SupplyTemp,degC,FALSE,0,,>H,100 -ReturnTemp,degC,FALSE,1,,>H,100 -OutsideTemp,degC,FALSE,2,,>H,100 -Occupancy,None,FALSE,3,,>H,1 -SecondStageCoolingStatus,None,FALSE,12,,>H,1 -FirstStageCoolingStatus,None,FALSE,4,,>H,1 -SupplyFanCommand,None,FALSE,15,,>H,1 -EconomizerPositionStatus,%,FALSE,17,,>H,1 -SupplyFanProof,None,FALSE,23,,>H,1 -Demand,%,FALSE,26,,>H,1 -CoolOverride1,None,TRUE,124,,>H,1 -CoolOVerride2,None,TRUE,125,,>H,1 -HeatOverride,None,TRUE,128,,>H,1 -ScheduleOverride,None,TRUE,131,,>H,1 -EconoChangeTemperature,degC,TRUE,150,,>H,100 -EconoChangeDifference,degC,TRUE,151,,>H,100 -MinimumOutdoorAirDamperSignal,%,TRUE,154,,>H,1 -EconomizerPositionOverride,%,TRUE,274,,>H,1 -EnableAbsoluteOverrides,None,TRUE,111,,>H,1 -SupplyFanOverride,None,TRUE,130,,>H,1 -FirstStageCoolingDemandSetPoint,None,TRUE,12,,>H,1 -SecondStageCoolingDemandSetPoint,None,TRUE,14,,>H,1 diff --git a/services/core/PlatformDriverAgent/platform_driver/driver.py b/services/core/PlatformDriverAgent/platform_driver/driver.py index 20aad887d0..c52c452737 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver.py @@ -245,10 +245,14 @@ def periodic_read(self, now): register_names = self.interface.get_register_names_view() _log.debug(f"results keys: {results.keys}") _log.debug(f"register_names keys: {register_names}") + # register_names contains list of all points + # results.keys gives points from the device + # The following loop will check if there are more points from the device than what + # was expected and then log an error for point in (register_names - results.keys()): _log.debug(f"Scraping point: {point}") depth_first_topic = self.base_topic(point=point) - _log.error("Failed to scrape point: "+depth_first_topic) + _log.error(f"Failed to scrape point: {depth_first_topic}") except (Exception, gevent.Timeout) as ex: tb = traceback.format_exc() _log.error('Failed to scrape ' + self.device_name + ':\n' + tb) diff --git a/services/core/PlatformDriverAgent/platform_driver/driver_locks.py b/services/core/PlatformDriverAgent/platform_driver/driver_locks.py index eeda10fe1b..a0e067777c 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver_locks.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver_locks.py @@ -90,6 +90,7 @@ def publish_lock(): _publish_lock.release() _client_socket_locks = defaultdict(lambda: None) +lock_counter = 0 def configure_client_socket_lock(address, port, max_connections=0): _log.debug("Configuring client socket lock for {}:{}".format(address, port)) @@ -117,8 +118,14 @@ def client_socket_locks(address, port): _log.debug(f"lock is None: lock: {lock}, type: {type(lock)}, id ${id(lock)}") raise RuntimeError("socket_lock not configured!") lock.acquire() + global lock_counter + lock_counter +=1 + _log.debug(f"lock_counter: {lock_counter}") + try: yield finally: _log.debug(f"Releasing client socket lock ({type(lock)}) for {address}:{port} at {id(lock)}") lock.release() + lock_counter -=1 + _log.debug(f"lock_counter after release: {lock_counter}") \ No newline at end of file diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index 167e155eb9..3d5e8e4edc 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -77,7 +77,6 @@ class MyModbusMaster (Client): from . import helpers - # utils.setup_logging() # _log = logging.getLogger(__name__) logger = logging.getLogger(__name__) @@ -679,28 +678,62 @@ def get_request(self, field): return self.__meta[helpers.META_REQUEST_MAP].get(field, None) def read_request(self, request): - try: - results = self.client.execute( - self.slave_address, - request.read_function_code, - request.address, - quantity_of_x=request.count, - data_format=request.formatting, - threadsafe=False - ) - logger.debug("Successfully read the request...") - self._data.update(request.parse_values(results)) - except (AttributeError, ModbusError) as err: - if "Exception code" in err.message: - msg = "{0}: {1}".format(err.message, - helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED")) - logger.debug(msg) - raise Exception("{0}: {1}".format(msg)) - logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) + results = self.client.execute( + self.slave_address, + request.read_function_code, + request.address, + quantity_of_x=request.count, + data_format=request.formatting, + threadsafe=False + ) + logger.debug("Successfully read the request...") + self._data.update(request.parse_values(results)) + # try: + # results = self.client.execute( + # self.slave_address, + # request.read_function_code, + # request.address, + # quantity_of_x=request.count, + # data_format=request.formatting, + # threadsafe=False + # ) + # logger.debug("Successfully read the request...") + # self._data.update(request.parse_values(results)) + # except (AttributeError, ModbusError) as err: + # if "Exception code" in err.message: + # msg = "{0}: {1}".format(err.message, + # helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED")) + # logger.debug(msg) + # raise Exception("{0}: {1}".format(msg)) + # logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) + + + def timer(slogger): + """Print the runtime of the decorated function""" + from functools import wraps + import time + def decorator_timer(func): + @wraps(func) + def wrapper_timer(*args, **kwargs): + start_time = datetime.now() # 1 + value = func(*args, **kwargs) + end_time = datetime.now() # 2 + run_time_sec = end_time - start_time + slogger.debug( + f"Finished {func.__name__!r} in {run_time_sec.total_seconds()} seconds" + ) + return value + return wrapper_timer + + return decorator_timer + + @timer(logger) def read_all(self): + logger.debug(f"READ_ALL Time now: {datetime.now()}") requests = self.__meta[helpers.META_REQUESTS] self._data.clear() + # gets the lock with client_socket_locks(self.device_address, self.port): logger.debug(f"Entered lock for {self.device_address}:{self.port}-{self.slave_address}") logger.debug(f"Total requests to be read: {len(requests)}") @@ -712,7 +745,7 @@ def read_all(self): exception_flag = False try: self.read_request(r) - break + break # can use break or continue except ConnectionResetError: exception_flag = True logger.warning("ConnectionResetError on read_request()") @@ -721,19 +754,25 @@ def read_all(self): exception_flag = True logger.warning("ModbusInvalidResponseError on read_request()") logger.warning(f"Error response: {e}") - if exception_flag: - self.client.close() - gevent.sleep(1.0) - self.client.open() + except Exception as e: + exception_flag = True + logger.warning("CATCHING ALL EXCEPTIONS") + logger.warning(f"Error response: {e}") + # if exception_flag: + # logger.warning("CLOSING SOCKET CONNECTION") + # self.client.close() + # gevent.sleep(1.0) + # logger.warning("OPENING SOCKET CONNECTION") + # self.client.open() retries -= 1 if retries == 0: logger.debug(f"Failed to read request: {r}") else: logger.debug(f"Succesfully read the request on retry: {retries}") - logger.debug(f"left lock for {self.device_address}:{self.port}-{self.slave_address}") + @timer(logger) def dump_all(self): self.read_all() return [(f,