Merge pull request #7576 from aspandey/syslog-transfer

Noobaa/Bucket Logging : Log delivery to noobaa core

src/deploy/NVA_build/rsyslog.conf

@@ -12,8 +12,8 @@ $ModLoad imuxsock # provides support for local system logging (e.g. via logger c
 #$ModLoad immark  # provides --MARK-- message capability
 
 # Provides UDP syslog reception
-#$ModLoad imudp
-#$UDPServerRun 514
+$ModLoad imudp
+$UDPServerRun 514
 
 # Provides TCP syslog reception
 #$ModLoad imtcp
@@ -49,6 +49,11 @@ $OmitLocalLogging off
 # Logging much else clutters up the screen.
 #kern.*                                                 /dev/console
 
+if $msg contains '{"noobaa_bucket_logging":"true"' then {
+    action(type="omfile" file="/var/log/bucket_logs.log")
+    stop
+}
+
 # Log anything (except mail) of level info or higher.
 # Don't log private authentication messages!
 *.info;mail.none;authpriv.none;cron.none                /var/log/messages
@@ -88,4 +93,4 @@ local7.*                                                /var/log/boot.log
 #$ActionResumeRetryCount -1    # infinite retries if host is down
 # remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional
 #*.* @@remote-host:514
-# ### end of the forwarding rule ###
+# ### end of the forwarding rule ###

src/endpoint/endpoint.js

@@ -328,6 +328,7 @@ function get_rpc_router(env) {
         bg: env.BG_ADDR || addr_utils.format_base_address(hostname, ports.bg),
         hosted_agents: env.HOSTED_AGENTS_ADDR || addr_utils.format_base_address(hostname, ports.hosted_agents),
         master: env.MGMT_ADDR || addr_utils.format_base_address(hostname, ports.mgmt),
+        syslog: env.SYSLOG_ADDR || "udp://localhost:514",
     };
 }
 

src/endpoint/s3/s3_bucket_logging.js

@@ -3,6 +3,8 @@
 
 const dbg = require('../../util/debug_module')(__filename);
 const http_utils = require('../../util/http_utils');
+const dgram = require('node:dgram');
+const { Buffer } = require('node:buffer');
 
 
 async function send_bucket_op_logs(req, res) {
@@ -26,14 +28,51 @@ function is_bucket_logging_enabled(source_bucket) {
     return true;
 }
 
+// UDP socket
+
+const create_syslog_udp_socket = (() => {
+    let client;
+    let url;
+    let port;
+    let hostname;
+    return function(syslog) {
+        if (!client) {
+            client = dgram.createSocket('udp4');
+            process.on('SIGINT', () => {
+                client.close();
+                process.exit();
+            });
+        }
+        if (!url) {
+            url = new URL(syslog);
+            port = parseInt(url.port, 10);
+            hostname = url.hostname;
+        }
+        return {client, port, hostname};
+    };
+
+})();
+
+
 function endpoint_bucket_op_logs(op_name, req, res, source_bucket) {
 
-    dbg.log2("Sending op logs for op name = ", op_name);
     // 1 - Get all the information to be logged in a log message.
     // 2 - Format it and send it to log bucket/syslog.
     const s3_log = get_bucket_log_record(op_name, source_bucket, req, res);
     dbg.log1("Bucket operation logs = ", s3_log);
 
+    const buffer = Buffer.from(JSON.stringify(s3_log));
+    const {client, port, hostname} = create_syslog_udp_socket(req.object_sdk.rpc_client.rpc.router.syslog);
+    if (client && port && hostname) {
+        client.send(buffer, port, hostname, err => {
+            if (err) {
+                dbg.log0("failed to send udp err = ", err);
+            }
+        });
+    } else {
+        dbg.log0(`Could not send bucket logs: client: ${client} port: ${port} hostname:${hostname}`);
+    }
+
 }
 
 function get_bucket_log_record(op_name, source_bucket, req, res) {
@@ -44,6 +83,7 @@ function get_bucket_log_record(op_name, source_bucket, req, res) {
         status_code = res.statusCode;
     }
     const log = {
+        noobaa_bucket_logging: "true",
         op: req.method,
         bucket_owner: source_bucket.bucket_owner,
         source_bucket: req.params.bucket,

src/server/system_services/schemas/system_schema.js

@@ -78,7 +78,7 @@ module.exports = {
                 properties: {
                     service: {
                         type: 'string',
-                        enum: ['noobaa-mgmt', 's3', 'sts', 'noobaa-db', 'noobaa-db-pg']
+                        enum: ['noobaa-mgmt', 's3', 'sts', 'noobaa-db', 'noobaa-db-pg', 'noobaa-syslog']
                     },
                     kind: {
                         type: 'string',
@@ -88,7 +88,7 @@ module.exports = {
                     port: { $ref: 'common_api#/definitions/port' },
                     api: {
                         type: 'string',
-                        enum: ['mgmt', 's3', 'sts', 'md', 'bg', 'hosted_agents', 'mongodb', 'metrics', 'postgres']
+                        enum: ['mgmt', 's3', 'sts', 'md', 'bg', 'hosted_agents', 'mongodb', 'metrics', 'postgres', 'syslog']
                     },
                     secure: { type: 'boolean' },
                     weight: { type: 'integer' }