Merge pull request #7638 from romayalon/romy-fix-system-json

NSFS | NC | Update system.json structure + small redirect fix
noobaa · Dec 4, 2023 · 5ace1aa · 5ace1aa
2 parents 2c728eb + 8f562af
commit 5ace1aa
Show file tree

Hide file tree

Showing 6 changed files with 84 additions and 75 deletions.
diff --git a/config.js b/config.js
@@ -11,6 +11,7 @@ const fs = require('fs');
 const path = require('path');
 const assert = require('assert');
 const _ = require('lodash');
+const util = require('util');
 const { default: Ajv } = require('ajv');
 const nsfs_config_schema = require('./src/server/object_services/schemas/nsfs_config_schema');
 const ajv = new Ajv({ verbose: true, allErrors: true });
@@ -691,7 +692,7 @@ config.NSFS_VERSIONING_ENABLED = true;
 
 config.NSFS_NC_CONF_DIR_REDIRECT_FILE = 'config_dir_redirect';
 config.NSFS_NC_DEFAULT_CONF_DIR = '/etc/noobaa.conf.d';
-config.NSFS_NC_CONF_DIR = config.NSFS_NC_DEFAULT_CONF_DIR;
+config.NSFS_NC_CONF_DIR = process.env.NSFS_NC_CONF_DIR || '';
 config.NSFS_TEMP_CONF_DIR_NAME = '.noobaa-config-nsfs';
 config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001;
 config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
@@ -832,12 +833,32 @@ function _get_data_from_file(file_name) {
     return data;
 }
 
+/**
+ * @returns {string}
+ */
+function _get_config_root() {
+    let config_root = config.NSFS_NC_DEFAULT_CONF_DIR;
+    try {
+        const redirect_path = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, config.NSFS_NC_CONF_DIR_REDIRECT_FILE);
+        const data = _get_data_from_file(redirect_path);
+        config_root = data.toString().trim();
+    } catch (err) {
+        console.warn('config.get_config_root - could not find custom config_root, will use the default config_root ', config_root);
+    }
+    return config_root;
+}
+
+
 /**
  * load_nsfs_nc_config loads on non containerized env the config.json file and sets the configurations
  */
 function load_nsfs_nc_config() {
     if (process.env.CONTAINER_PLATFORM) return;
     try {
+        if (!config.NSFS_NC_CONF_DIR) {
+            config.NSFS_NC_CONF_DIR = _get_config_root();
+            console.log('load_nsfs_nc_config.setting config.NSFS_NC_CONF_DIR', config.NSFS_NC_CONF_DIR);
+        }
         const config_path = path.join(config.NSFS_NC_CONF_DIR, 'config.json');
         const config_data = require(config_path);
         const valid = ajv.validate(nsfs_config_schema, config_data);
@@ -856,7 +877,7 @@ function load_nsfs_nc_config() {
             config[key] = merged_config[key];
         });
 
-        console.log(`nsfs: config_dir_path=${config.NSFS_NC_CONF_DIR} config.json= ${merged_config}`);
+        console.log(`nsfs: config_dir_path=${config.NSFS_NC_CONF_DIR} config.json= ${util.inspect(merged_config)}`);
 
     } catch (err) {
         if (err.code !== 'MODULE_NOT_FOUND' && err.code !== 'ENOENT') throw err;
@@ -890,6 +911,9 @@ function reload_nsfs_nc_config() {
     }
 }
 
+module.exports.load_nsfs_nc_config = load_nsfs_nc_config;
+module.exports.reload_nsfs_nc_config = reload_nsfs_nc_config;
+
 load_nsfs_nc_config();
 reload_nsfs_nc_config();
 load_config_local();

diff --git a/docs/design/NonContainerizedNSFSDesign.md b/docs/design/NonContainerizedNSFSDesign.md
@@ -141,7 +141,9 @@ High level configuration -
 3.3. buckets/ - directory that contains buckets configurations, each bucket configuration file called {bucket_name}.json and fits the bucket schema.
 
 3.4. system.json - json file that contains information about the system deployed on the machine, the specified information has the following format: 
-`{"current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}}` 
+`{ [hostname1]: { "current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}},
+   [hostname2]: { "current_version":"5.15.0","upgrade_history":{"successful_upgrades":[]}}
+}` 
 
 3.5. config.json - json file that contains shared configurations of the node cluster, and machine specific configurations, the configuration has the following format: 
 {

diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md
@@ -101,37 +101,13 @@ mkdir -p /etc/noobaa.conf.d/access_keys/
 
 ```
 
-**3. Create env file under the configuration directory -**
-
-In order to apply env variables changes on the service, edit /etc/sysconfig/noobaa_nsfs env file as you wish before starting the service, notice that the env file format is key-value pair - 
-
-```sh
-vim  /etc/sysconfig/noobaa_nsfs
-```
-**Note** - If another /usr/local/noobaa-core/.env exists it should be merged into /etc/sysconfig/noobaa_nsfs carefully.
-
-In order to apply env changes after the service was started, edit the /etc/sysconfig/noobaa_nsfs env file and restart the service - 
-```sh
-vim  /etc/sysconfig/noobaa_nsfs
-systemctl restart noobaa_nsfs
-```
-
-
 ## Create FS -
 If it's not already existing, create the fs root path in which buckets (directories) and objects (files) will be created.
 
 ```sh
 mkdir -p /tmp/fs1/
 ```
 
-
-## Run the nsfs service - 
-The systemd script runs noobaa non containerized, and requires config_root in order to find the location of the system/accounts/buckets configuration file.
-
-```sh
-systemctl start nsfs
-```
-
 ## Developer customization of the nsfs service (OPTIONAL) - 
 The following list consists of supported optional developer customization -
 1. Number of forks 
@@ -149,6 +125,13 @@ Design of Accounts and buckets configuration entities - [NonContainerizedNSFS](h
 **Note** - All required paths on the configuration files (bucket - path, account - new_buckets_path) must be absolute paths.
 
 
+## Run the nsfs service - 
+The systemd script runs noobaa non containerized, and requires config_root in order to find the location of the system/accounts/buckets configuration file.
+Limitation - In a cluster each host should have a unique name.
+```sh
+systemctl start noobaa_nsfs
+```
+
 ## NSFS service logs -
 Run the following command in order to get the nsfs service logs - 
 
@@ -445,9 +428,7 @@ NSFS management CLI command will create both account and bucket dir if it's miss
 
 ## NSFS Certificate
 
-Non containerized NSFS certificate location is configured in system.json file under the property `nsfs_ssl_cert_dir` and the path should contain SSL files tls.key and tls.crt. System will use a cert from this dir to create a valid HTTPS connection. If cert is missing in this dir a self-signed SSL certificate will be generated. Make sure the path mentioned in `nsfs_ssl_cert_dir` is valid before running nsfs command, If the path is invalid then cert flow will fail.
-
-Non containerized NSFS allow nonsecure HTTP connection only when `allow_http` in system.json is true.
+Non containerized NSFS certificates/ directory location will be under the config_root path. The certificates/ directory should contain SSL files tls.key and tls.crt. System will use a cert from this dir to create a valid HTTPS connection. If cert is missing in this dir a self-signed SSL certificate will be generated. Make sure the path to certificates/ directory is valid before running nsfs command, If the path is invalid then cert flow will fail.
 
 ## Log and Logrotate
 Noobaa logs are configured using rsyslog and logrotate. RPM will configure rsyslog and logrotate if both are already running. 
@@ -466,3 +447,18 @@ Rotate the logs manually.
 ```
 logrotate /etc/logrotate.d/noobaa/logrotate_noobaa.conf 
 ```
+
+**Create env file under the configuration directory (OPTIONAL) -**
+
+In order to apply env variables changes on the service, edit /etc/sysconfig/noobaa_nsfs env file as you wish before starting the service, notice that the env file format is key-value pair - 
+
+```sh
+vim  /etc/sysconfig/noobaa_nsfs
+```
+**Note** - If another /usr/local/noobaa-core/.env exists it should be merged into /etc/sysconfig/noobaa_nsfs carefully.
+
+In order to apply env changes after the service was started, edit the /etc/sysconfig/noobaa_nsfs env file and restart the service - 
+```sh
+vim  /etc/sysconfig/noobaa_nsfs
+systemctl restart noobaa_nsfs
+```
diff --git a/src/cmd/nsfs.js b/src/cmd/nsfs.js
@@ -11,10 +11,10 @@ dbg.original_console();
 
 const config = require('../../config');
 
+const os = require('os');
 const fs = require('fs');
 const util = require('util');
 const minimist = require('minimist');
-const native_fs_utils = require('../util/native_fs_utils');
 
 if (process.env.LOCAL_MD_SERVER === 'true') {
     require('../server/system_services/system_store').get_instance({ standalone: true });
@@ -205,17 +205,20 @@ async function init_nsfs_system(config_root) {
     const system_data = new json_utils.JsonFileWrapper(system_data_path);
 
     const data = await system_data.read();
-
+    const hostname = os.hostname();
     // If the system data already exists, we should not create it again
-    if (data.current_version) return;
+    if (data?.[hostname]?.current_version) return;
 
     try {
         await system_data.update({
-            current_version: pkg.version,
-            upgrade_history: {
-                successful_upgrades: [],
-                last_failure: undefined
-            },
+            ...data,
+            [hostname]: {
+                current_version: pkg.version,
+                upgrade_history: {
+                    successful_upgrades: [],
+                    last_failure: undefined
+                }
+            }
         });
         console.log('created NSFS system data with version: ', pkg.version);
     } catch (err) {
@@ -240,7 +243,16 @@ async function main(argv = minimist(process.argv.slice(2))) {
             console.log(JSON.stringify(IAM_JSON_SCHEMA.schema, null, 2));
             return;
         }
-
+        const simple_mode = Boolean(argv.simple);
+        if (!simple_mode) {
+            nsfs_config_root = config.NSFS_NC_CONF_DIR;
+            if (argv.config_root) {
+                nsfs_config_root = String(argv.config_root);
+                config.NSFS_NC_CONF_DIR = nsfs_config_root;
+                require('../../config').load_nsfs_nc_config();
+                require('../../config').reload_nsfs_nc_config();
+            }
+        }
         const http_port = Number(argv.http_port) || config.ENDPOINT_PORT;
         const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT;
         const https_port_sts = Number(argv.https_port_sts) || config.ENDPOINT_SSL_STS_PORT;
@@ -250,7 +262,6 @@ async function main(argv = minimist(process.argv.slice(2))) {
         const gid = Number(argv.gid) || process.getgid();
         const access_key = argv.access_key && new SensitiveString(String(argv.access_key));
         const secret_key = argv.secret_key && new SensitiveString(String(argv.secret_key));
-        const simple_mode = Boolean(argv.simple);
         const iam_ttl = Number(argv.iam_ttl ?? 60);
         const backend = argv.backend || (process.env.GPFS_DL_PATH ? 'GPFS' : '');
         const versioning = argv.versioning || 'DISABLED';
@@ -263,11 +274,6 @@ async function main(argv = minimist(process.argv.slice(2))) {
             warn_threshold_ms: config.NSFS_WARN_THRESHOLD_MS,
         };
         verify_gpfslib();
-
-        if (!simple_mode) {
-            nsfs_config_root = await native_fs_utils.get_config_root(argv, fs_config);
-            config.NSFS_NC_CONF_DIR = nsfs_config_root;
-        }
         const account = {
             email: new SensitiveString('[email protected]'),
             nsfs_account_config: fs_config,

diff --git a/src/upgrade/upgrade_manager.js b/src/upgrade/upgrade_manager.js
@@ -7,7 +7,8 @@ const pkg = require('../../package.json');
 const fs = require('fs');
 const path = require('path');
 const json_utils = require('../util/json_utils');
-const native_fs_utils = require('../util/native_fs_utils');
+const config = require('../../config');
+const os = require('os');
 
 const dbg = require('../util/debug_module')('UPGRADE');
 dbg.set_process_name('Upgrade');
@@ -176,14 +177,13 @@ async function load_required_scripts(server_version, container_version) {
 }
 
 async function upgrade_nsfs() {
-    const root_fs_config = native_fs_utils.get_root_fs_context();
-    const config_dir_path = await native_fs_utils.get_config_root({}, root_fs_config);
-    const system_data_path = path.join(config_dir_path, 'system.json');
+    const system_data_path = path.join(config.NSFS_NC_CONF_DIR, 'system.json');
     const system_data = new json_utils.JsonFileWrapper(system_data_path);
 
     const system = await system_data.read();
-    const upgrade_history = system.upgrade_history;
-    let current_version = system.current_version;
+    const hostname = os.hostname();
+    const upgrade_history = system?.[hostname]?.upgrade_history;
+    let current_version = system?.[hostname]?.current_version;
 
     const new_version = pkg.version;
 
@@ -225,8 +225,11 @@ async function upgrade_nsfs() {
     // update upgrade_history
     try {
         await system_data.update({
-            current_version,
-            upgrade_history
+            ...system,
+            [hostname]: {
+                current_version,
+                upgrade_history
+            }
         });
     } catch (error) {
         dbg.error('failed to update system_store with upgrade information');

diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js
@@ -440,27 +440,6 @@ function get_root_fs_context(config_root_backend) {
     };
 }
 
-/**
- * @param {object} argv
- * @param {nb.NativeFSContext} fs_config
- * @returns {Promise<string>}
- */
-async function get_config_root(argv, fs_config) {
-    let config_root = config.NSFS_NC_DEFAULT_CONF_DIR;
-    if (argv.config_root) {
-        config_root = String(argv.config_root);
-    } else {
-        try {
-            const redirect_path = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, config.NSFS_NC_CONF_DIR_REDIRECT_FILE);
-            const { data } = await nb_native().fs.readFile(fs_config, redirect_path);
-            config_root = data.toString().trim();
-        } catch (err) {
-            dbg.warn('native_fs_utils.get_config_root - could not find custom config_root, will use the default config_root ', config_root);
-        }
-    }
-    return config_root;
-}
-
 
 exports.get_umasked_mode = get_umasked_mode;
 exports._make_path_dirs = _make_path_dirs;
@@ -489,4 +468,3 @@ exports.delete_config_file = delete_config_file;
 exports.update_config_file = update_config_file;
 exports.isDirectory = isDirectory;
 exports.get_root_fs_context = get_root_fs_context;
-exports.get_config_root = get_config_root;