From 8f562aff7c45921658e9386761fb5ddcfd352e75 Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Sun, 3 Dec 2023 18:20:55 +0200 Subject: [PATCH] fix system.json structure Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> --- config.js | 28 ++++++++++++- docs/design/NonContainerizedNSFSDesign.md | 4 +- docs/non_containerized_NSFS.md | 50 +++++++++++------------ src/cmd/nsfs.js | 36 +++++++++------- src/upgrade/upgrade_manager.js | 19 +++++---- src/util/native_fs_utils.js | 22 ---------- 6 files changed, 84 insertions(+), 75 deletions(-) diff --git a/config.js b/config.js index 4aba007841..667287c0f0 100644 --- a/config.js +++ b/config.js @@ -11,6 +11,7 @@ const fs = require('fs'); const path = require('path'); const assert = require('assert'); const _ = require('lodash'); +const util = require('util'); const { default: Ajv } = require('ajv'); const nsfs_config_schema = require('./src/server/object_services/schemas/nsfs_config_schema'); const ajv = new Ajv({ verbose: true, allErrors: true }); @@ -691,7 +692,7 @@ config.NSFS_VERSIONING_ENABLED = true; config.NSFS_NC_CONF_DIR_REDIRECT_FILE = 'config_dir_redirect'; config.NSFS_NC_DEFAULT_CONF_DIR = '/etc/noobaa.conf.d'; -config.NSFS_NC_CONF_DIR = config.NSFS_NC_DEFAULT_CONF_DIR; +config.NSFS_NC_CONF_DIR = process.env.NSFS_NC_CONF_DIR || ''; config.NSFS_TEMP_CONF_DIR_NAME = '.noobaa-config-nsfs'; config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001; config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443; @@ -832,12 +833,32 @@ function _get_data_from_file(file_name) { return data; } +/** + * @returns {string} + */ +function _get_config_root() { + let config_root = config.NSFS_NC_DEFAULT_CONF_DIR; + try { + const redirect_path = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, config.NSFS_NC_CONF_DIR_REDIRECT_FILE); + const data = _get_data_from_file(redirect_path); + config_root = data.toString().trim(); + } catch (err) { + console.warn('config.get_config_root - could not find custom config_root, will use the default config_root ', config_root); + } + return config_root; +} + + /** * load_nsfs_nc_config loads on non containerized env the config.json file and sets the configurations */ function load_nsfs_nc_config() { if (process.env.CONTAINER_PLATFORM) return; try { + if (!config.NSFS_NC_CONF_DIR) { + config.NSFS_NC_CONF_DIR = _get_config_root(); + console.log('load_nsfs_nc_config.setting config.NSFS_NC_CONF_DIR', config.NSFS_NC_CONF_DIR); + } const config_path = path.join(config.NSFS_NC_CONF_DIR, 'config.json'); const config_data = require(config_path); const valid = ajv.validate(nsfs_config_schema, config_data); @@ -856,7 +877,7 @@ function load_nsfs_nc_config() { config[key] = merged_config[key]; }); - console.log(`nsfs: config_dir_path=${config.NSFS_NC_CONF_DIR} config.json= ${merged_config}`); + console.log(`nsfs: config_dir_path=${config.NSFS_NC_CONF_DIR} config.json= ${util.inspect(merged_config)}`); } catch (err) { if (err.code !== 'MODULE_NOT_FOUND' && err.code !== 'ENOENT') throw err; @@ -890,6 +911,9 @@ function reload_nsfs_nc_config() { } } +module.exports.load_nsfs_nc_config = load_nsfs_nc_config; +module.exports.reload_nsfs_nc_config = reload_nsfs_nc_config; + load_nsfs_nc_config(); reload_nsfs_nc_config(); load_config_local(); diff --git a/docs/design/NonContainerizedNSFSDesign.md b/docs/design/NonContainerizedNSFSDesign.md index ec9d3ea20c..af9f76dec2 100644 --- a/docs/design/NonContainerizedNSFSDesign.md +++ b/docs/design/NonContainerizedNSFSDesign.md @@ -141,7 +141,9 @@ High level configuration - 3.3. buckets/ - directory that contains buckets configurations, each bucket configuration file called {bucket_name}.json and fits the bucket schema. 3.4. system.json - json file that contains information about the system deployed on the machine, the specified information has the following format: -`{"current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}}` +`{ [hostname1]: { "current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}}, + [hostname2]: { "current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}} +}` 3.5. config.json - json file that contains shared configurations of the node cluster, and machine specific configurations, the configuration has the following format: { diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md index 11968ca0bf..01b56506a8 100644 --- a/docs/non_containerized_NSFS.md +++ b/docs/non_containerized_NSFS.md @@ -101,22 +101,6 @@ mkdir -p /etc/noobaa.conf.d/access_keys/ ``` -**3. Create env file under the configuration directory -** - -In order to apply env variables changes on the service, edit /etc/sysconfig/noobaa_nsfs env file as you wish before starting the service, notice that the env file format is key-value pair - - -```sh -vim /etc/sysconfig/noobaa_nsfs -``` -**Note** - If another /usr/local/noobaa-core/.env exists it should be merged into /etc/sysconfig/noobaa_nsfs carefully. - -In order to apply env changes after the service was started, edit the /etc/sysconfig/noobaa_nsfs env file and restart the service - -```sh -vim /etc/sysconfig/noobaa_nsfs -systemctl restart noobaa_nsfs -``` - - ## Create FS - If it's not already existing, create the fs root path in which buckets (directories) and objects (files) will be created. @@ -124,14 +108,6 @@ If it's not already existing, create the fs root path in which buckets (director mkdir -p /tmp/fs1/ ``` - -## Run the nsfs service - -The systemd script runs noobaa non containerized, and requires config_root in order to find the location of the system/accounts/buckets configuration file. - -```sh -systemctl start nsfs -``` - ## Developer customization of the nsfs service (OPTIONAL) - The following list consists of supported optional developer customization - 1. Number of forks @@ -149,6 +125,13 @@ Design of Accounts and buckets configuration entities - [NonContainerizedNSFS](h **Note** - All required paths on the configuration files (bucket - path, account - new_buckets_path) must be absolute paths. +## Run the nsfs service - +The systemd script runs noobaa non containerized, and requires config_root in order to find the location of the system/accounts/buckets configuration file. +Limitation - In a cluster each host should have a unique name. +```sh +systemctl start noobaa_nsfs +``` + ## NSFS service logs - Run the following command in order to get the nsfs service logs - @@ -445,9 +428,7 @@ NSFS management CLI command will create both account and bucket dir if it's miss ## NSFS Certificate -Non containerized NSFS certificate location is configured in system.json file under the property `nsfs_ssl_cert_dir` and the path should contain SSL files tls.key and tls.crt. System will use a cert from this dir to create a valid HTTPS connection. If cert is missing in this dir a self-signed SSL certificate will be generated. Make sure the path mentioned in `nsfs_ssl_cert_dir` is valid before running nsfs command, If the path is invalid then cert flow will fail. - -Non containerized NSFS allow nonsecure HTTP connection only when `allow_http` in system.json is true. +Non containerized NSFS certificates/ directory location will be under the config_root path. The certificates/ directory should contain SSL files tls.key and tls.crt. System will use a cert from this dir to create a valid HTTPS connection. If cert is missing in this dir a self-signed SSL certificate will be generated. Make sure the path to certificates/ directory is valid before running nsfs command, If the path is invalid then cert flow will fail. ## Log and Logrotate Noobaa logs are configured using rsyslog and logrotate. RPM will configure rsyslog and logrotate if both are already running. @@ -466,3 +447,18 @@ Rotate the logs manually. ``` logrotate /etc/logrotate.d/noobaa/logrotate_noobaa.conf ``` + +**Create env file under the configuration directory (OPTIONAL) -** + +In order to apply env variables changes on the service, edit /etc/sysconfig/noobaa_nsfs env file as you wish before starting the service, notice that the env file format is key-value pair - + +```sh +vim /etc/sysconfig/noobaa_nsfs +``` +**Note** - If another /usr/local/noobaa-core/.env exists it should be merged into /etc/sysconfig/noobaa_nsfs carefully. + +In order to apply env changes after the service was started, edit the /etc/sysconfig/noobaa_nsfs env file and restart the service - +```sh +vim /etc/sysconfig/noobaa_nsfs +systemctl restart noobaa_nsfs +``` diff --git a/src/cmd/nsfs.js b/src/cmd/nsfs.js index 760424b67c..321010f974 100644 --- a/src/cmd/nsfs.js +++ b/src/cmd/nsfs.js @@ -11,10 +11,10 @@ dbg.original_console(); const config = require('../../config'); +const os = require('os'); const fs = require('fs'); const util = require('util'); const minimist = require('minimist'); -const native_fs_utils = require('../util/native_fs_utils'); if (process.env.LOCAL_MD_SERVER === 'true') { require('../server/system_services/system_store').get_instance({ standalone: true }); @@ -205,17 +205,20 @@ async function init_nsfs_system(config_root) { const system_data = new json_utils.JsonFileWrapper(system_data_path); const data = await system_data.read(); - + const hostname = os.hostname(); // If the system data already exists, we should not create it again - if (data.current_version) return; + if (data?.[hostname]?.current_version) return; try { await system_data.update({ - current_version: pkg.version, - upgrade_history: { - successful_upgrades: [], - last_failure: undefined - }, + ...data, + [hostname]: { + current_version: pkg.version, + upgrade_history: { + successful_upgrades: [], + last_failure: undefined + } + } }); console.log('created NSFS system data with version: ', pkg.version); } catch (err) { @@ -240,7 +243,16 @@ async function main(argv = minimist(process.argv.slice(2))) { console.log(JSON.stringify(IAM_JSON_SCHEMA.schema, null, 2)); return; } - + const simple_mode = Boolean(argv.simple); + if (!simple_mode) { + nsfs_config_root = config.NSFS_NC_CONF_DIR; + if (argv.config_root) { + nsfs_config_root = String(argv.config_root); + config.NSFS_NC_CONF_DIR = nsfs_config_root; + require('../../config').load_nsfs_nc_config(); + require('../../config').reload_nsfs_nc_config(); + } + } const http_port = Number(argv.http_port) || config.ENDPOINT_PORT; const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT; const https_port_sts = Number(argv.https_port_sts) || config.ENDPOINT_SSL_STS_PORT; @@ -250,7 +262,6 @@ async function main(argv = minimist(process.argv.slice(2))) { const gid = Number(argv.gid) || process.getgid(); const access_key = argv.access_key && new SensitiveString(String(argv.access_key)); const secret_key = argv.secret_key && new SensitiveString(String(argv.secret_key)); - const simple_mode = Boolean(argv.simple); const iam_ttl = Number(argv.iam_ttl ?? 60); const backend = argv.backend || (process.env.GPFS_DL_PATH ? 'GPFS' : ''); const versioning = argv.versioning || 'DISABLED'; @@ -263,11 +274,6 @@ async function main(argv = minimist(process.argv.slice(2))) { warn_threshold_ms: config.NSFS_WARN_THRESHOLD_MS, }; verify_gpfslib(); - - if (!simple_mode) { - nsfs_config_root = await native_fs_utils.get_config_root(argv, fs_config); - config.NSFS_NC_CONF_DIR = nsfs_config_root; - } const account = { email: new SensitiveString('nsfs@noobaa.io'), nsfs_account_config: fs_config, diff --git a/src/upgrade/upgrade_manager.js b/src/upgrade/upgrade_manager.js index ad944e5c91..ed37fa5bc2 100644 --- a/src/upgrade/upgrade_manager.js +++ b/src/upgrade/upgrade_manager.js @@ -7,7 +7,8 @@ const pkg = require('../../package.json'); const fs = require('fs'); const path = require('path'); const json_utils = require('../util/json_utils'); -const native_fs_utils = require('../util/native_fs_utils'); +const config = require('../../config'); +const os = require('os'); const dbg = require('../util/debug_module')('UPGRADE'); dbg.set_process_name('Upgrade'); @@ -176,14 +177,13 @@ async function load_required_scripts(server_version, container_version) { } async function upgrade_nsfs() { - const root_fs_config = native_fs_utils.get_root_fs_context(); - const config_dir_path = await native_fs_utils.get_config_root({}, root_fs_config); - const system_data_path = path.join(config_dir_path, 'system.json'); + const system_data_path = path.join(config.NSFS_NC_CONF_DIR, 'system.json'); const system_data = new json_utils.JsonFileWrapper(system_data_path); const system = await system_data.read(); - const upgrade_history = system.upgrade_history; - let current_version = system.current_version; + const hostname = os.hostname(); + const upgrade_history = system?.[hostname]?.upgrade_history; + let current_version = system?.[hostname]?.current_version; const new_version = pkg.version; @@ -225,8 +225,11 @@ async function upgrade_nsfs() { // update upgrade_history try { await system_data.update({ - current_version, - upgrade_history + ...system, + [hostname]: { + current_version, + upgrade_history + } }); } catch (error) { dbg.error('failed to update system_store with upgrade information'); diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index 6f286496a8..662a6ca86e 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -440,27 +440,6 @@ function get_root_fs_context(config_root_backend) { }; } -/** - * @param {object} argv - * @param {nb.NativeFSContext} fs_config - * @returns {Promise} - */ -async function get_config_root(argv, fs_config) { - let config_root = config.NSFS_NC_DEFAULT_CONF_DIR; - if (argv.config_root) { - config_root = String(argv.config_root); - } else { - try { - const redirect_path = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, config.NSFS_NC_CONF_DIR_REDIRECT_FILE); - const { data } = await nb_native().fs.readFile(fs_config, redirect_path); - config_root = data.toString().trim(); - } catch (err) { - dbg.warn('native_fs_utils.get_config_root - could not find custom config_root, will use the default config_root ', config_root); - } - } - return config_root; -} - exports.get_umasked_mode = get_umasked_mode; exports._make_path_dirs = _make_path_dirs; @@ -489,4 +468,3 @@ exports.delete_config_file = delete_config_file; exports.update_config_file = update_config_file; exports.isDirectory = isDirectory; exports.get_root_fs_context = get_root_fs_context; -exports.get_config_root = get_config_root;