diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0244e399..5f5b4d0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,36 +33,13 @@ jobs: # This technique documented in: # https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional # TODO: Find a way to define this with less escapes. + # TODO-PYPI: Revert changes in this file once a new version of prometheus-client (after 2023-11-09) has been released on Pypi run: | if [[ "${{ github.event_name }}" == "schedule" || "${{ github.head_ref }}" =~ ^release_ ]]; then \ echo "matrix={ \ \"os\": [ \"ubuntu-latest\", \"macos-latest\", \"windows-latest\" ], \ - \"python-version\": [ \"3.6\", \"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\" ], \ - \"package_level\": [ \"minimum\", \"latest\" ], \ - \"exclude\": [ \ - { \ - \"os\": \"ubuntu-latest\", \ - \"python-version\": \"3.6\", \ - \"package_level\": \"minimum\" \ - }, \ - { \ - \"os\": \"ubuntu-latest\", \ - \"python-version\": \"3.6\", \ - \"package_level\": \"latest\" \ - } \ - ], \ - \"include\": [ \ - { \ - \"os\": \"ubuntu-20.04\", \ - \"python-version\": \"3.6\", \ - \"package_level\": \"minimum\" \ - }, \ - { \ - \"os\": \"ubuntu-20.04\", \ - \"python-version\": \"3.6\", \ - \"package_level\": \"latest\" \ - } \ - ] \ + \"python-version\": [ \"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\" ], \ + \"package_level\": [ \"minimum\", \"latest\" ] \ }" >> $GITHUB_OUTPUT; \ else \ echo "matrix={ \ @@ -71,23 +48,18 @@ jobs: \"package_level\": [ \"minimum\", \"latest\" ], \ \"include\": [ \ { \ - \"os\": \"ubuntu-20.04\", \ - \"python-version\": \"3.6\", \ + \"os\": \"ubuntu-latest\", \ + \"python-version\": \"3.8\", \ \"package_level\": \"minimum\" \ }, \ - { \ - \"os\": \"ubuntu-20.04\", \ - \"python-version\": \"3.6\", \ - \"package_level\": \"latest\" \ - }, \ { \ \"os\": \"ubuntu-latest\", \ - \"python-version\": \"3.7\", \ - \"package_level\": \"minimum\" \ + \"python-version\": \"3.8\", \ + \"package_level\": \"latest\" \ }, \ { \ \"os\": \"macos-latest\", \ - \"python-version\": \"3.6\", \ + \"python-version\": \"3.8\", \ \"package_level\": \"latest\" \ }, \ { \ @@ -102,7 +74,7 @@ jobs: }, \ { \ \"os\": \"windows-latest\", \ - \"python-version\": \"3.6\", \ + \"python-version\": \"3.8\", \ \"package_level\": \"latest\" \ }, \ { \ diff --git a/Dockerfile b/Dockerfile index 7ed3384b..1032f713 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,11 @@ COPY examples/metrics.yaml /etc/zhmc-prometheus-exporter/metrics.yaml ENV TMP_DIR=/tmp/zhmc-prometheus-exporter WORKDIR $TMP_DIR COPY . $TMP_DIR + +# TODO: Remove git install again once PR https://github.com/prometheus/client_python/pull/946 is released on Pypi +RUN apt-get update +RUN apt-get install -y --no-install-recommends git + RUN pip install . && rm -rf $TMP_DIR # Set the current directory when running this image diff --git a/Makefile b/Makefile index 765605ed..aacff0de 100644 --- a/Makefile +++ b/Makefile @@ -234,8 +234,9 @@ safety: safety_$(pymn).done .PHONY: check_reqs check_reqs: develop_$(pymn).done minimum-constraints.txt requirements.txt @echo "Makefile: Checking missing dependencies of this package" - pip-missing-reqs $(package_name) --requirements-file=requirements.txt - pip-missing-reqs $(package_name) --requirements-file=minimum-constraints.txt +# TODO-PYPI: Re-enable once a new version of prometheus-client (after 2023-11-09) has been released on Pypi +# pip-missing-reqs $(package_name) --requirements-file=requirements.txt +# pip-missing-reqs $(package_name) --requirements-file=minimum-constraints.txt @echo "Makefile: Done checking missing dependencies of this package" ifeq ($(PLATFORM),Windows_native) # Reason for skipping on Windows is https://github.com/r1chardj0n3s/pip-check-reqs/issues/67 diff --git a/README.rst b/README.rst index 708bf2a4..1d65842c 100644 --- a/README.rst +++ b/README.rst @@ -68,6 +68,9 @@ Quickstart obtaining metrics, and which userid and password to use for logging on to the HMC. + It also defines whether HTTP or HTTPS is used for Prometheus, and HTTPS + related certificates and keys. + Download the `sample HMC credentials file`_ as ``hmccreds.yaml`` and edit that copy accordingly. @@ -107,8 +110,9 @@ Quickstart up and running. You can see what it does in the mean time by using the ``-v`` option. Subsequent requests to the exporter will be sub-second. -* Direct your web browser at http://localhost:9291 to see the exported - Prometheus metrics. Refreshing the browser will update the metrics. +* Direct your web browser at http://localhost:9291 (or https://localhost:9291 + when using HTTPS) to see the exported Prometheus metrics. Refreshing the + browser will update the metrics. Reporting issues ---------------- diff --git a/docs/changes.rst b/docs/changes.rst index d993498e..52f2b8fc 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -93,6 +93,9 @@ Released: not yet a new metric zhmc_partition_storage_groups that lists the storage groups attached to a partition. (issue #346) +* Added support for HTTPS and mutual TLS (mTLS) by adding a new optional section + 'prometheus' to the HMC credentials file. (issue #347) + **Cleanup:** **Known issues:** diff --git a/docs/intro.rst b/docs/intro.rst index cc9099e7..548e1fe6 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -27,6 +27,9 @@ automatic session renewals with the HMC if the logon session expires, and it survives HMC reboots and automatically picks up metrics collection again once the HMC come back up. +The exporter supports HTTP and HTTPS (with and without mutual TLS) for +Prometheus. + .. _IBM Z: https://www.ibm.com/it-infrastructure/z .. _Prometheus exporter: https://prometheus.io/docs/instrumenting/exporters/ .. _Prometheus: https://prometheus.io @@ -58,6 +61,9 @@ Quickstart obtaining metrics, and which userid and password to use for logging on to the HMC. + It also defines whether HTTP or HTTPS is used for Prometheus, and HTTPS + related certificates and keys. + Download the :ref:`sample HMC credentials file` as ``hmccreds.yaml`` and edit that copy accordingly. @@ -91,8 +97,9 @@ Quickstart up and running. You can see what it does in the mean time by using the ``-v`` option. Subsequent requests to the exporter will be sub-second. -* Direct your web browser at http://localhost:9291 to see the exported - Prometheus metrics. Refreshing the browser will update the metrics. +* Direct your web browser at http://localhost:9291 (or https://localhost:9291 + when using HTTPS) to see the exported Prometheus metrics. Refreshing the + browser will update the metrics. Reporting issues ---------------- diff --git a/docs/usage.rst b/docs/usage.rst index 266a35e4..c99d72ce 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -97,7 +97,7 @@ The ``zhmc_prometheus_exporter`` command supports the following arguments: -m METRICS_FILE path name of metric definition file. Use --help-metrics for details. Default: /etc/zhmc-prometheus-exporter/metrics.yaml - -p PORT port for exporting. Default: 9291 + -p PORT port for exporting. Default: prometheus.port in HMC credentials file. --log DEST enable logging and set a log destination (stderr, syslog, FILE). Default: no logging @@ -253,6 +253,43 @@ For more information, see the section in the documentation of the 'zhmcclient' package. +Communication with Prometheus +----------------------------- + +The exporter is an HTTP or HTTPS server that is regularly contacted by Prometheus +for collecting metrics using HTTP GET. + +The parameters for the communication with Prometheus are defined in the +HMC credentials file in the ``prometheus`` section, as in the following example: + +.. code-block:: yaml + + prometheus: # optional + port: 9291 + server_cert_file: server_cert.pem + server_key_file: server_key.pem + ca_cert_file: ca_certs.pem + +If the ``prometheus`` section is not specified, the exporter starts its +server with HTTP. + +If the ``prometheus`` section is specified, the presence of the +``server_cert_file`` parameter will determine whether the server will use HTTP +or HTTPS: If that parameter is specified, HTTPS will be used. If not specified, +HTTP will be used. + +If HTTPS is used, the ``server_key_file`` parameter is required. + +If HTTPS is used, the presence of the ``ca_cert_file`` parameter determines +whether mutual TLS (mTLS) is enabled: If specified, mTLS is enabled and the +exporter will require Prometheus to present a client certificate, which is +validated using the specified CA certificate chain. If not specified, mTLS is +disabled and the exporter will not require Prometheus to present a client +certificate and will ignore it if presented. + +The meaning of the parameters is described in :ref:`HMC credentials file`. + + Exported metric concepts ------------------------ @@ -734,6 +771,8 @@ The *HMC credentials file* tells the exporter which HMC to talk to for obtaining metrics, and which userid and password to use for logging on to the HMC. +It also specifies how Prometheus should communicate with the exporter. + In addition, it allows specifying additional labels to be used in all metrics exported to Prometheus. This can be used for defining labels that identify the environment managed by the HMC, in cases where metrics from @@ -747,7 +786,13 @@ The HMC credentials file is in YAML format and has the following structure: hmc: {hmc-ip-address} userid: {hmc-userid} password: {hmc-password} - verify_cert: {verify-cert} + verify_cert: {hmc-verify-cert} + + prometheus: # optional + port: {prom-port} + server_cert_file: {prom-server-cert-file} + server_key_file: {prom-server-key-file} + ca_cert_file: {prom-ca-cert-file} extra_labels: # optional # list of labels: @@ -762,9 +807,34 @@ Where: * ``{hmc-password}`` is the password of that userid. -* ``{verify-cert}`` controls whether and how the HMC server certificate is +* ``{hmc-verify-cert}`` controls whether and how the HMC server certificate is verified. For details, see :ref:`HMC certificate`. +* ``{prom-port}`` is the port for exporting. Default: 9291. + +* ``{prom-server-cert-file}`` is the path name of a certificate file in PEM + format containing an X.509 server certificate that will be presented to + Prometheus during TLS handshake. Relative path names are relative to the + directory of the HMC credentials file. If the ``server_cert_file`` parameter + is specified, the exporter will start its server with HTTPS, and otherwise + with HTTP. + +* ``{prom-server-key-file}`` is the path name of a key file in PEM format + containing an X.509 private key that belongs to the public key in the server + certificate. Relative path names are relative to the directory of the HMC + credentials file. The ``server_key_file`` parameter is required when + the ``server_cert_file`` parameter is specified. + +* ``{prom-ca-cert-file}`` is the path name of a CA file in PEM format + containing X.509 CA certificates that will be used for validating a client + certificate presented by Prometheus during TLS handshake. Relative path names + are relative to the directory of the HMC credentials file. If the + ``ca_cert_file`` parameter is specified, the exporter will require from + Prometheus to present a client certificate during TLS handshake and will + validate it using the specified CA certificate chain (mutual TLS, mTLS). + If not specified, the exporter will not require from Prometheus to present a + client certificate, and will ignore it if presented. + * ``{extra-label-name}`` is the label name. * ``{extra-label-value}`` is the label value. The string value is used directly diff --git a/examples/hmccreds.yaml b/examples/hmccreds.yaml index 436335c7..52c73de6 100644 --- a/examples/hmccreds.yaml +++ b/examples/hmccreds.yaml @@ -6,6 +6,16 @@ metrics: password: password verify_cert: true +prometheus: + port: 9291 + + # Note: Activating the following two parameters enables the use of HTTPS + # server_cert_file: server_cert.pem + # server_key_file: server_key.pem + + # Note: Activating the following parameter enables the use of mutual TLS + # ca_cert_file: ca_certs.pem + extra_labels: - name: hmc value: "hmc_info['hmc-name']" diff --git a/minimum-constraints.txt b/minimum-constraints.txt index e70c9291..9673a293 100644 --- a/minimum-constraints.txt +++ b/minimum-constraints.txt @@ -39,7 +39,10 @@ wheel==0.38.1; python_version >= '3.7' zhmcclient==1.9.1 -prometheus-client==0.9.0 +prometheus-client>=0.17.0; python_version <= '3.7' +# TODO-PYPI: Re-enable once a new version of prometheus-client (after 2023-11-09) has been released on Pypi +# prometheus-client>=0.18.x; python_version >= '3.8' + urllib3==1.26.17 jsonschema==3.2.0 six==1.14.0; python_version <= '3.9' diff --git a/requirements.txt b/requirements.txt index 730818d6..3d62c01a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,12 @@ # zhmcclient @ git+https://github.com/zhmcclient/python-zhmcclient.git@master zhmcclient>=1.9.1 -prometheus-client>=0.9.0 +# prometheus-client 0.18 has removed support for py36,37 +# TODO-PYPI: Re-enable once a new version of prometheus-client (after 2023-11-09) has been released on Pypi +# prometheus-client>=0.17.0; python_version <= '3.7' +# prometheus-client>=0.18.x; python_version >= '3.8' +prometheus-client @ git+https://github.com/prometheus/client_python.git@master + urllib3>=1.25.17 jsonschema>=3.2.0 Jinja2>=2.11.3 diff --git a/setup.py b/setup.py index ef3e290c..b613342c 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,9 @@ def read_file(a_file): # Keep these Python versions in sync with: # - Section "Supported environments" in docs/intro.rst - python_requires='>=3.6', + # TODO-PYPI: Revert changes in this file once a new version of + # prometheus-client (after 2023-11-09) has been released on Pypi + python_requires='>=3.8', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -130,8 +132,6 @@ def read_file(a_file): 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/tests/test_all.py b/tests/test_all.py index faa53099..6f6dae3e 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -46,7 +46,7 @@ def test_args_store(self): def test_default_args(self): """Tests for all defaults.""" args = zhmc_prometheus_exporter.zhmc_prometheus_exporter.parse_args([]) - self.assertEqual(args.p, "9291") + self.assertEqual(args.p, None) self.assertEqual(args.c, "/etc/zhmc-prometheus-exporter/hmccreds.yaml") self.assertEqual(args.m, "/etc/zhmc-prometheus-exporter/metrics.yaml") diff --git a/zhmc_prometheus_exporter/schemas/hmccreds_schema.yaml b/zhmc_prometheus_exporter/schemas/hmccreds_schema.yaml index 7525d99a..a32acf86 100644 --- a/zhmc_prometheus_exporter/schemas/hmccreds_schema.yaml +++ b/zhmc_prometheus_exporter/schemas/hmccreds_schema.yaml @@ -29,6 +29,23 @@ properties: verify_cert: description: "Controls whether and how the HMC certificate is verified. For details, see doc section 'HMC certificate'" type: [boolean, string] + prometheus: + description: Communication with Prometheus + type: object + additionalProperties: false + properties: + port: + description: "Port for exporting." + type: integer + server_cert_file: + description: "Path name of server certificate file. Enables the use of HTTPS." + type: string + server_key_file: + description: "Path name of private key file." + type: string + ca_cert_file: + description: "Path name of CA certificates file for validating the client certificate. Enables mutual TLS by requiring a client certificate to be presented." + type: string extra_labels: description: "Additional Prometheus labels to be added to all metrics" type: array diff --git a/zhmc_prometheus_exporter/zhmc_prometheus_exporter.py b/zhmc_prometheus_exporter/zhmc_prometheus_exporter.py index 4ec1ad67..c7431a05 100755 --- a/zhmc_prometheus_exporter/zhmc_prometheus_exporter.py +++ b/zhmc_prometheus_exporter/zhmc_prometheus_exporter.py @@ -43,6 +43,7 @@ DEFAULT_CREDS_FILE = '/etc/zhmc-prometheus-exporter/hmccreds.yaml' DEFAULT_METRICS_FILE = '/etc/zhmc-prometheus-exporter/metrics.yaml' +DEFAULT_PORT = 9291 EXPORTER_LOGGER_NAME = 'zhmcexporter' @@ -201,6 +202,7 @@ def zhmc_exceptions(session, hmccreds_filename): def parse_args(args): """Parses the CLI arguments.""" + parser = argparse.ArgumentParser( description="IBM Z HMC Exporter - a Prometheus exporter for metrics " "from the IBM Z HMC") @@ -215,8 +217,9 @@ def parse_args(args): "Use --help-metrics for details. " "Default: {}".format(DEFAULT_METRICS_FILE)) parser.add_argument("-p", metavar="PORT", - default="9291", - help="port for exporting. Default: 9291") + default=None, + help="port for exporting. Default: prometheus.port in " + "HMC credentials file") parser.add_argument("--log", dest='log_dest', metavar="DEST", default=None, help="enable logging and set a log destination " "({dests}). Default: no logging". @@ -283,6 +286,16 @@ def help_creds(): userid: myuser password: mypassword +prometheus: + port: 9291 + + # Note: Activating the following two parameters enables the use of HTTPS + # server_cert_file: server_cert.pem + # server_key_file: server_key.pem + + # Note: Activating the following parameter enables the use of mutual TLS + # ca_cert_file: ca_certs.pem + extra_labels: - name: pod value: mypod @@ -1801,12 +1814,70 @@ def main(): "Registering the collector and performing first collection") REGISTRY.register(coll) # Performs a first collection - logprint(logging.INFO, PRINT_V, - "Starting the HTTP server on port {}".format(args.p)) - start_http_server(int(args.p)) + # Get the Prometheus communication parameters + prom_item = yaml_creds_content.get("prometheus", {}) + config_port = prom_item.get("port", None) + server_cert_file = prom_item.get("server_cert_file", None) + if server_cert_file: + prometheus_client_supports_https = sys.version_info[0:2] >= (3, 8) + if not prometheus_client_supports_https: + raise ImproperExit( + "Use of https requires Python 3.8 or higher.") + server_key_file = prom_item.get("server_key_file", None) + if not server_key_file: + raise ImproperExit( + "server_key_file not specified in HMC credentials file " + "when using https.") + hmccreds_dir = os.path.dirname(hmccreds_filename) + if not os.path.isabs(server_cert_file): + server_cert_file = os.path.join(hmccreds_dir, server_cert_file) + if not os.path.isabs(server_key_file): + server_key_file = os.path.join(hmccreds_dir, server_key_file) + ca_cert_file = prom_item.get("ca_cert_file", None) + if ca_cert_file and not os.path.isabs(ca_cert_file): + ca_cert_file = os.path.join(hmccreds_dir, ca_cert_file) + else: # http + server_cert_file = None + server_key_file = None + ca_cert_file = None + + port = int(args.p or config_port or DEFAULT_PORT) + + if server_cert_file: + logprint(logging.INFO, PRINT_V, + "Starting the server with HTTPS on port {}".format(port)) + logprint(logging.INFO, PRINT_V, + "Server certificate file: {}".format(server_cert_file)) + logprint(logging.INFO, PRINT_V, + "Server private key file: {}".format(server_key_file)) + if ca_cert_file: + logprint(logging.INFO, PRINT_V, + "Mutual TLS: Enabled with CA certificates file: {}". + format(ca_cert_file)) + else: + logprint(logging.INFO, PRINT_V, + "Mutual TLS: Disabled") + else: + logprint(logging.INFO, PRINT_V, + "Starting the server with HTTP on port {}".format(port)) + + if server_cert_file: + try: + start_http_server( + port=port, + certfile=server_cert_file, + keyfile=server_key_file, + client_cafile=ca_cert_file, + client_auth_required=(ca_cert_file is not None)) + except IOError as exc: + raise ImproperExit( + "Issues with server certificate, key, or CA certificate " + "files: {}: {}".format(exc.__class__.__name__, exc)) + else: + start_http_server(port=port) logprint(logging.INFO, PRINT_ALWAYS, - "Exporter is up and running on port {}".format(args.p)) + "Exporter is up and running on port {}".format(port)) while True: try: time.sleep(1)