diff --git a/lib/collectinfo_analyzer/info_controller.py b/lib/collectinfo_analyzer/info_controller.py index aa8e8ec6..f2aeaba0 100644 --- a/lib/collectinfo_analyzer/info_controller.py +++ b/lib/collectinfo_analyzer/info_controller.py @@ -253,6 +253,7 @@ def _do_default(self, line): @CommandHelp("Displays usage information for each namespace") def do_usage(self, line): ns_stats = self.stat_getter.get_namespace() + service_stats = self.stat_getter.get_service() for timestamp in sorted(ns_stats.keys()): if not ns_stats[timestamp]: @@ -260,6 +261,7 @@ def do_usage(self, line): self.view.info_namespace_usage( ns_stats[timestamp], + service_stats[timestamp], self.log_handler.get_cinfo_log_at(timestamp=timestamp), timestamp=timestamp, **self.mods, diff --git a/lib/health/query.py b/lib/health/query.py index 1d16e6e4..886c7e71 100644 --- a/lib/health/query.py +++ b/lib/health/query.py @@ -113,16 +113,31 @@ "Disk utilization Anomaly."); -avail=select like(".*available_pct") as "free_disk" from NAMESPACE.STATISTICS save; -disk_free = select "device_free_pct" as "free_disk", "free-pct-disk" as "free_disk" from NAMESPACE.STATISTICS save; -r = do disk_free - avail save as "fragmented blocks pct"; +SET CONSTRAINT VERSION < 7.0; +avail = select "device_available_pct" as "free_disk", "pmem_available_pct" as "free_disk", "available_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_free = select "device_free_pct" as "free_disk", "pmem_free_pct" as "free_disk", "free-pct-disk" as "free_disk" from NAMESPACE.STATISTICS save; +r = do data_free - avail save as "fragmented blocks pct"; r = do r <= 30; r = group by CLUSTER, NAMESPACE r; ASSERT(r, True, "High (> 30%) fragmented blocks.", "PERFORMANCE", WARNING, "Listed namespace[s] have higher than normal (>30%) fragmented blocks at the time of sampling. Please run 'show config namespace like defrag' to check defrag configurations. Possible cause can be Aerospike disk defragmentation not keeping up with write rate and/or large record sizes causing fragmentation. Refer to knowledge base article discuss.aerospike.com/t/defragmentation for more details.", "Fragmented Blocks check."); +/* +Same as above but beginning in 7.0 we must flip data_used to get data_free +*/ +SET CONSTRAINT VERSION >= 7.0; +data_avail = select "data_avail_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_used = select "data_used_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_free = do 100 - data_used; +r = do data_free - data_avail save as "fragmented blocks pct"; +r = do r <= 30; +r = group by CLUSTER, NAMESPACE r; +ASSERT(r, True, "High (> 30%) fragmented blocks.", "PERFORMANCE", WARNING, + "Listed namespace[s] have higher than normal (>30%) fragmented blocks at the time of sampling. Please run 'show config namespace like defrag' to check defrag configurations. Possible cause can be Aerospike disk defragmentation not keeping up with write rate and/or large record sizes causing fragmentation. Refer to knowledge base article discuss.aerospike.com/t/defragmentation for more details.", + "Fragmented Blocks check."); +SET CONSTRAINT VERSION ALL; s = select "%iowait" from SYSTEM.IOSTAT save; r = do s > 10; ASSERT(r, False, "High (> 10%) CPU IO wait time.", "PERFORMANCE", WARNING, @@ -188,6 +203,8 @@ "Listed node[s] have lower than normal (< 20%) system free memory percentage. Please run 'show statistics service like system_free_mem_pct' to get actual values. Possible misconfiguration.", "System memory percentage check."); +SET CONSTRAINT VERSION < 7.0; + f = select "memory_free_pct" as "stats", "free-pct-memory" as "stats" from NAMESPACE.STATISTICS save; s = select "stop-writes-pct" as "stats" from NAMESPACE.CONFIG save; u = do 100 - f save as "memory_used_pct"; @@ -198,6 +215,7 @@ /* NB : ADD CHECKS IF NODES ARE NOT HOMOGENOUS MEM / NUM CPU etc */ +SET CONSTRAINT VERSION ALL; s = select "available_bin_names", "available-bin-names" from NAMESPACE save; r = group by NAMESPACE do s > 3200; @@ -214,8 +232,8 @@ ASSERT(e, True, "Namespace configured to use more than 256G.", "LIMITS", WARNING, "On listed nodes namespace as mentioned have configured more than 256G of memory. Namespace with data not in memory can have max upto 4 billion keys and can utilize only up to 256G. Please run 'show statistics namespace like memory-size' to check configured memory.", "Namespace per node memory limit check."); -SET CONSTRAINT VERSION ALL; +SET CONSTRAINT VERSION < 7.0; /* Following query selects assigned memory-size from namespace config and total ram size from system statistics. group by for namespace stats sums all memory size and gives node level memory size. @@ -236,7 +254,7 @@ "Listed node[s] have less than 5G free memory available for Aerospike runtime. Please run 'show statistics namespace like memory-size' to check configured memory and check output of 'free' for system memory. Possible misconfiguration.", "Runtime memory configuration check."); - +SET CONSTRAINT VERSION ALL; /* Current configurations and config file values difference check */ @@ -289,20 +307,53 @@ "Listed node[s] show higher than normal client-connections (> 80% of the max configured proto-fd-max). Please run 'show config like proto-fd-max' and 'show statistics like client_connections' for actual values. Possible can be network issue / improper client behavior / FD leak.", "Client connections check."); -s = select like(".*available_pct") as "stats" from NAMESPACE.STATISTICS save; -m = select like(".*min-avail-pct") as "stats" from NAMESPACE.CONFIG save; -critical_check = do s >= m; -ASSERT(critical_check, True, "Low namespace disk available pct (stop-write enabled).", "OPERATIONS", CRITICAL, - "Listed namespace[s] have lower than normal (< min-avail-pct) available disk space. Probable cause - namespace size misconfiguration.", - "Critical Namespace disk available pct check."); - -critical_check = do s < m; -r = do s >= 20; -r = do r || critical_check; -ASSERT(r, True, "Low namespace disk available pct.", "OPERATIONS", WARNING, +SET CONSTRAINT VERSION < 7.0.0; +free = select "device_free_pct" as "stats", "pmem_free_pct" as "stats" from NAMESPACE.STATISTICS save; +used = do 100 - free save as "storage-engine used pct"; +stop_used_pct = select "storage-engine.stop-writes-used-pct" as "stats", "storage-engine.max-used-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do used <= stop_used_pct; +ASSERT(critical, True, "High namespace storage-engine used pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have higher than normal used storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine used pct check."); + +avail = select "device_available_pct" as "stats", "pmem_available_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_min_avail = select "storage-engine.min-avail-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do avail >= stop_min_avail; +ASSERT(critical, True, "Low namespace storage-engine available pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have lower than normal available storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine available pct check."); + +skip = do critical == False; +warn = do avail >= 20; +warn = do warn || skip; +ASSERT(warn, True, "Low namespace disk available pct.", "OPERATIONS", WARNING, "Listed namespace[s] have lower than normal (< 20 %) available disk space. Probable cause - namespace size misconfiguration.", "Namespace disk available pct check."); +SET CONSTRAINT VERSION >= 7.0.0; +used = select "data_used_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_used_pct = select "storage-engine.stop-writes-used-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do used <= stop_used_pct; +ASSERT(critical, True, "High namespace storage-engine used pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have higher than normal used storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine used pct check."); + +avail = select "data_avail_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_avail_pct = select "storage-engine.stop-writes-avail-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do avail >= stop_avail_pct; +ASSERT(critical, True, "Low namespace storage-engine available pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have lower than normal available storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine available pct check."); + +skip = do critical == False; +warn = do avail >= 20; +warn = do warn || skip; +ASSERT(warn, True, "Low namespace storage-engine available pct.", "OPERATIONS", WARNING, + "Listed namespace[s] have lower than normal (< 20 %) available storage-engine space. Probable cause - namespace size misconfiguration.", + "Namespace storage-engine available pct check."); + +SET CONSTRAINT VERSION ALL; + s = select * from SERVICE.CONFIG ignore "heartbeat.mtu", "node-id-interface", "node-id", "pidfile", like(".*address"), like(".*port") save; r = group by CLUSTER, KEY do NO_MATCH(s, ==, MAJORITY) save; ASSERT(r, False, "Different service configurations.", "OPERATIONS", WARNING, @@ -356,19 +407,30 @@ ASSERT(r, False, "Device name misconfigured.", "OPERATIONS", WARNING, "Listed device[s] have partitions on same node. This might create situation like data corruption where data written to main drive gets overwritten/corrupted from data written to or deleted from the partition with the same name.", "Device name misconfiguration check."); - -s = select "device_total_bytes", "device-total-bytes", "total-bytes-disk" from NAMESPACE.STATISTICS save; + +s = select "data_total_bytes", "device_total_bytes", "pmem_total_bytes", "device-total-bytes", "total-bytes-disk" from NAMESPACE.STATISTICS save; r = group by CLUSTER, NAMESPACE do NO_MATCH(s, ==, MAJORITY) save; ASSERT(r, False, "Different namespace device size configuration.", "OPERATIONS", WARNING, "Listed namespace[s] have difference in configured disk size. Please run 'show statistics namespace like bytes' to check total device size. Probable cause - config file misconfiguration.", "Namespace device size configuration difference check."); +SET CONSTRAINT VERSION < 4.9; hwm = select "high-water-disk-pct" from NAMESPACE.CONFIG save; hwm = group by CLUSTER, NAMESPACE hwm; r = do hwm == 50; -ASSERT(r, True, "Non-default namespace device high water mark configuration.", "OPERATIONS", INFO, - "Listed namespace[s] have non-default high water mark configuration. Please run 'show config namespace like high-water-disk-pct' to check value. Probable cause - config file misconfiguration.", - "Non-default namespace device high water mark check."); +ASSERT(r, True, "Non-default namespace storage-engine eviction threshold configuration.", "OPERATIONS", INFO, + "Listed namespace[s] have non-default eviction threshold configuration. Please run 'show config namespace like high-water evict-used' to check value. Probable cause - config file misconfiguration.", + "Non-default namespace storage-engine eviction threshold check."); + +SET CONSTRAINT VERSION >= 4.9; +hwm = select "high-water-disk-pct", "storage-engine.evict-used-pct" from NAMESPACE.CONFIG save; +hwm = group by CLUSTER, NAMESPACE hwm; +r = do hwm == 0; +ASSERT(r, True, "Non-default namespace storage-engine eviction threshold configuration.", "OPERATIONS", INFO, + "Listed namespace[s] have non-default eviction threshold configuration. Please run 'show config namespace like high-water evict-used' to check value. Probable cause - config file misconfiguration.", + "Non-default namespace storage-engine eviction threshold check."); + +SET CONSTRAINT VERSION ALL; lwm = select like(".*defrag-lwm-pct") from NAMESPACE.CONFIG save; lwm = group by CLUSTER, NAMESPACE lwm; @@ -377,12 +439,12 @@ "Listed namespace[s] have non-default low water mark configuration. Probable cause - config file misconfiguration.", "Non-default namespace device low water mark check."); -hwm = select "high-water-disk-pct" as "defrag-lwm-pct" from NAMESPACE.CONFIG save; +hwm = select "high-water-disk-pct" as "defrag-lwm-pct", "storage-engine.evict-used-pct" as "defrag-lwm-pct" from NAMESPACE.CONFIG save; lwm = select like(".*defrag-lwm-pct") as "defrag-lwm-pct" from NAMESPACE.CONFIG save; r = do lwm < hwm on common; r = group by CLUSTER, NAMESPACE r; ASSERT(r, False, "Defrag low water mark misconfigured.", "OPERATIONS", WARNING, - "Listed namespace[s] have defrag-lwm-pct lower than high-water-disk-pct. This might create situation like no block to write, no eviction and no defragmentation. Please run 'show config namespace like high-water-disk-pct defrag-lwm-pct' to check configured values. Probable cause - namespace watermark misconfiguration.", + "Listed namespace[s] have defrag-lwm-pct lower than eviction threshold. This might create situation like no block to write, no eviction and no defragmentation. Please run 'show config namespace like high-water evict-used defrag-lwm-pct' to check configured values. Probable cause - namespace watermark misconfiguration.", "Defrag low water mark misconfiguration check."); commit_to_device = select "storage-engine.commit-to-device" from NAMESPACE.CONFIG; @@ -433,21 +495,23 @@ with available space per node per namespace. */ -t = select "device_total_bytes" as "disk_space", "device-total-bytes" as "disk_space", "total-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; -u = select "used-bytes-disk" as "disk_space", "device_used_bytes" as "disk_space" from NAMESPACE.STATISTICS; +t = select "data_total_bytes" as "disk_space", "pmem_total_bytes" as "disk_space", "device_total_bytes" as "disk_space", "device-total-bytes" as "disk_space", "total-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; +u = select "data_used_bytes" as "disk_space", "pmem_used_bytes" as "disk_space", "device_used_bytes" as "disk_space", "used-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; /* Available extra space */ e = do t - u; -e = group by CLUSTER, NAMESPACE, NODE do SUM(e) save as "available device space"; +e = group by CLUSTER, NAMESPACE, NODE do SUM(e) save as "available storage space"; s = select "cluster_size" as "size" from SERVICE; n = do MAX(s); n = do n - 1; /* Extra space need if 1 node goes down */ e1 = do u / n; -e1 = group by CLUSTER, NAMESPACE do MAX(e1) save as "distribution share of used device space per node"; +e1 = group by CLUSTER, NAMESPACE do MAX(e1) save as "distribution share of used storage space per node"; r = do e > e1; -ASSERT(r, True, "Namespace under configured (disk) for single node failure.", "OPERATIONS", WARNING, - "Listed namespace[s] does not have enough disk space configured to deal with increase in data per node in case of 1 node failure. Please run 'show statistics namespace like bytes' to check device space. It is non-issue if single replica limit is set to larger values, i.e if number of replica copies are reduced in case of node loss.", - "Namespace single node failure disk config check."); +ASSERT(r, False, "Namespace storage under configured for single node failure.", "OPERATIONS", WARNING, + "Listed namespace[s] does not have enough storage space configured to deal with increase in data per node in case of 1 node failure. Please run 'show statistics namespace like bytes' to check storage space. It is non-issue if single replica limit is set to larger values, i.e if number of replica copies are reduced in case of node loss.", + "Namespace single node failure storage space config check."); + +SET CONSTRAINT VERSION < 7.0.0; /* Same as above query but for memory @@ -1936,7 +2000,7 @@ cluster_size = group by CLUSTER do MAX(cluster_size) save as "cluster-size"; repl = select "effective_replication_factor" as "sprig_limit_critical" from NAMESPACE.STATISTICS save as "effective_repl_factor"; pts = select "partition-tree-sprigs" as "sprig_limit_critical" from NAMESPACE.CONFIG save as "partition-tree-sprigs"; -size_limit = select "index-type.mounts-size-limit" as "sprig_limit_critical" from NAMESPACE.CONFIG; +size_limit = select "index-type.mounts-size-limit" as "sprig_limit_critical", "index-type.mounts-budget" as "sprig_limit_critical" from NAMESPACE.CONFIG; // below statement adds thousand delimiter to mounts-size-limiter when it prints size_limit = do size_limit * 1 save as "mounts-size-limit"; @@ -1957,11 +2021,12 @@ num_partitions = do 4096 * repl; partitions_per_node = do num_partitions/cluster_size; pts_per_node = do partitions_per_node * pts; +// 4K partition-tree-sprig overhead total_pts = do pts_per_node * 4096 save as "Minimum space required"; result = do total_pts > size_limit; ASSERT(result, False, "ALL FLASH - Too many sprigs per partition for current available index mounted space. Some records are likely failing to be created.", "OPERATIONS", CRITICAL, - "Minimum space required for sprig overhead at current cluster size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at current cluster size exceeds mounts-budget/mounts-size-limit. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for current cluster size.", dont_skip); @@ -1972,9 +2037,9 @@ mcs = group by CLUSTER do MAX(mcs) save as "min-cluster-size"; repl = select "replication-factor" as "sprig_limit_warning" from NAMESPACE.STATISTICS; pts = select "partition-tree-sprigs" as "sprig_limit_warning" from NAMESPACE.CONFIG; -msl = select "index-type.mounts-size-limit" as "sprig_limit_warning" from NAMESPACE.CONFIG; +msl = select "index-type.mounts-size-limit" as "sprig_limit_warning", "index-type.mounts-budget" as "sprig_limit_warning" from NAMESPACE.CONFIG; // below statement adds thousand delimiter to mounts-size-limiter when it prints -msl = do msl * 1 save as "mounts-size-limit"; +msl = do msl * 1 save; // calculate sprig overhead // The replication factor should be min(repl, mcs) @@ -1988,7 +2053,7 @@ e1 = do repl_smaller && dont_skip; ASSERT(r1, False, "ALL FLASH - Too many sprigs per partition for configured min-cluster-size.", "OPERATIONS", WARNING, - "Minimum space required for sprig overhead at min-cluster-size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at min-cluster-size exceeds index-mount size. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for minimum cluster size.", e1); @@ -1997,6 +2062,7 @@ // Only is asserted if min-cluster-size is smaller than replication-factor. // r2 = do 4096 * mcs; // r2 = do r2/mcs; +// 4096 * mcs / mcs = 4096 r2 = 4096; r2 = do r2 * pts; r2 = do r2 * 4096 save as "Minimum space required"; @@ -2006,7 +2072,7 @@ e2 = do mcs_smaller && dont_skip; ASSERT(r2, False, "ALL FLASH - Too many sprigs per partition for configured min-cluster-size.", "OPERATIONS", WARNING, - "Minimum space required for sprig overhead at min-cluster-size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at min-cluster-size exceeds index-mount size. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for minimum cluster size.", e2); diff --git a/lib/health/query/health.hql b/lib/health/query/health.hql index ebc7d739..886c7e71 100644 --- a/lib/health/query/health.hql +++ b/lib/health/query/health.hql @@ -113,16 +113,31 @@ ASSERT(r1, False, "Skewed cluster disk utilization.", "ANOMALY", WARNING, "Disk utilization Anomaly."); -avail=select like(".*available_pct") as "free_disk" from NAMESPACE.STATISTICS save; -disk_free = select "device_free_pct" as "free_disk", "free-pct-disk" as "free_disk" from NAMESPACE.STATISTICS save; -r = do disk_free - avail save as "fragmented blocks pct"; +SET CONSTRAINT VERSION < 7.0; +avail = select "device_available_pct" as "free_disk", "pmem_available_pct" as "free_disk", "available_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_free = select "device_free_pct" as "free_disk", "pmem_free_pct" as "free_disk", "free-pct-disk" as "free_disk" from NAMESPACE.STATISTICS save; +r = do data_free - avail save as "fragmented blocks pct"; r = do r <= 30; r = group by CLUSTER, NAMESPACE r; ASSERT(r, True, "High (> 30%) fragmented blocks.", "PERFORMANCE", WARNING, "Listed namespace[s] have higher than normal (>30%) fragmented blocks at the time of sampling. Please run 'show config namespace like defrag' to check defrag configurations. Possible cause can be Aerospike disk defragmentation not keeping up with write rate and/or large record sizes causing fragmentation. Refer to knowledge base article discuss.aerospike.com/t/defragmentation for more details.", "Fragmented Blocks check."); +/* +Same as above but beginning in 7.0 we must flip data_used to get data_free +*/ +SET CONSTRAINT VERSION >= 7.0; +data_avail = select "data_avail_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_used = select "data_used_pct" as "free_disk" from NAMESPACE.STATISTICS save; +data_free = do 100 - data_used; +r = do data_free - data_avail save as "fragmented blocks pct"; +r = do r <= 30; +r = group by CLUSTER, NAMESPACE r; +ASSERT(r, True, "High (> 30%) fragmented blocks.", "PERFORMANCE", WARNING, + "Listed namespace[s] have higher than normal (>30%) fragmented blocks at the time of sampling. Please run 'show config namespace like defrag' to check defrag configurations. Possible cause can be Aerospike disk defragmentation not keeping up with write rate and/or large record sizes causing fragmentation. Refer to knowledge base article discuss.aerospike.com/t/defragmentation for more details.", + "Fragmented Blocks check."); +SET CONSTRAINT VERSION ALL; s = select "%iowait" from SYSTEM.IOSTAT save; r = do s > 10; ASSERT(r, False, "High (> 10%) CPU IO wait time.", "PERFORMANCE", WARNING, @@ -188,6 +203,8 @@ ASSERT(r, False, "Low system memory percentage.", "LIMITS", CRITICAL, "Listed node[s] have lower than normal (< 20%) system free memory percentage. Please run 'show statistics service like system_free_mem_pct' to get actual values. Possible misconfiguration.", "System memory percentage check."); +SET CONSTRAINT VERSION < 7.0; + f = select "memory_free_pct" as "stats", "free-pct-memory" as "stats" from NAMESPACE.STATISTICS save; s = select "stop-writes-pct" as "stats" from NAMESPACE.CONFIG save; u = do 100 - f save as "memory_used_pct"; @@ -198,6 +215,7 @@ ASSERT(r, True, "Low namespace memory available pct (stop-write enabled).", "OPE /* NB : ADD CHECKS IF NODES ARE NOT HOMOGENOUS MEM / NUM CPU etc */ +SET CONSTRAINT VERSION ALL; s = select "available_bin_names", "available-bin-names" from NAMESPACE save; r = group by NAMESPACE do s > 3200; @@ -214,8 +232,8 @@ e = do r <= 274877906944; ASSERT(e, True, "Namespace configured to use more than 256G.", "LIMITS", WARNING, "On listed nodes namespace as mentioned have configured more than 256G of memory. Namespace with data not in memory can have max upto 4 billion keys and can utilize only up to 256G. Please run 'show statistics namespace like memory-size' to check configured memory.", "Namespace per node memory limit check."); -SET CONSTRAINT VERSION ALL; +SET CONSTRAINT VERSION < 7.0; /* Following query selects assigned memory-size from namespace config and total ram size from system statistics. group by for namespace stats sums all memory size and gives node level memory size. @@ -236,7 +254,7 @@ ASSERT(r, True, "Aerospike runtime memory configured < 5G.", "LIMITS", INFO, "Listed node[s] have less than 5G free memory available for Aerospike runtime. Please run 'show statistics namespace like memory-size' to check configured memory and check output of 'free' for system memory. Possible misconfiguration.", "Runtime memory configuration check."); - +SET CONSTRAINT VERSION ALL; /* Current configurations and config file values difference check */ @@ -289,20 +307,53 @@ ASSERT(r, False, "High system client connections.", "OPERATIONS", WARNING, "Listed node[s] show higher than normal client-connections (> 80% of the max configured proto-fd-max). Please run 'show config like proto-fd-max' and 'show statistics like client_connections' for actual values. Possible can be network issue / improper client behavior / FD leak.", "Client connections check."); -s = select like(".*available_pct") as "stats" from NAMESPACE.STATISTICS save; -m = select like(".*min-avail-pct") as "stats" from NAMESPACE.CONFIG save; -critical_check = do s >= m; -ASSERT(critical_check, True, "Low namespace disk available pct (stop-write enabled).", "OPERATIONS", CRITICAL, - "Listed namespace[s] have lower than normal (< min-avail-pct) available disk space. Probable cause - namespace size misconfiguration.", - "Critical Namespace disk available pct check."); - -critical_check = do s < m; -r = do s >= 20; -r = do r || critical_check; -ASSERT(r, True, "Low namespace disk available pct.", "OPERATIONS", WARNING, +SET CONSTRAINT VERSION < 7.0.0; +free = select "device_free_pct" as "stats", "pmem_free_pct" as "stats" from NAMESPACE.STATISTICS save; +used = do 100 - free save as "storage-engine used pct"; +stop_used_pct = select "storage-engine.stop-writes-used-pct" as "stats", "storage-engine.max-used-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do used <= stop_used_pct; +ASSERT(critical, True, "High namespace storage-engine used pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have higher than normal used storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine used pct check."); + +avail = select "device_available_pct" as "stats", "pmem_available_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_min_avail = select "storage-engine.min-avail-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do avail >= stop_min_avail; +ASSERT(critical, True, "Low namespace storage-engine available pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have lower than normal available storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine available pct check."); + +skip = do critical == False; +warn = do avail >= 20; +warn = do warn || skip; +ASSERT(warn, True, "Low namespace disk available pct.", "OPERATIONS", WARNING, "Listed namespace[s] have lower than normal (< 20 %) available disk space. Probable cause - namespace size misconfiguration.", "Namespace disk available pct check."); +SET CONSTRAINT VERSION >= 7.0.0; +used = select "data_used_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_used_pct = select "storage-engine.stop-writes-used-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do used <= stop_used_pct; +ASSERT(critical, True, "High namespace storage-engine used pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have higher than normal used storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine used pct check."); + +avail = select "data_avail_pct" as "stats" from NAMESPACE.STATISTICS save; +stop_avail_pct = select "storage-engine.stop-writes-avail-pct" as "stats" from NAMESPACE.CONFIG save; +critical = do avail >= stop_avail_pct; +ASSERT(critical, True, "Low namespace storage-engine available pct (stop-write enabled).", "OPERATIONS", CRITICAL, + "Listed namespace[s] have lower than normal available storage-engine space. Probable cause - namespace size misconfiguration.", + "Critical Namespace storage-engine available pct check."); + +skip = do critical == False; +warn = do avail >= 20; +warn = do warn || skip; +ASSERT(warn, True, "Low namespace storage-engine available pct.", "OPERATIONS", WARNING, + "Listed namespace[s] have lower than normal (< 20 %) available storage-engine space. Probable cause - namespace size misconfiguration.", + "Namespace storage-engine available pct check."); + +SET CONSTRAINT VERSION ALL; + s = select * from SERVICE.CONFIG ignore "heartbeat.mtu", "node-id-interface", "node-id", "pidfile", like(".*address"), like(".*port") save; r = group by CLUSTER, KEY do NO_MATCH(s, ==, MAJORITY) save; ASSERT(r, False, "Different service configurations.", "OPERATIONS", WARNING, @@ -312,7 +363,7 @@ ASSERT(r, False, "Different service configurations.", "OPERATIONS", WARNING, multicast_mode_enabled = select like(".*mode") from NETWORK.CONFIG; multicast_mode_enabled = do multicast_mode_enabled == "multicast"; multicast_mode_enabled = group by CLUSTER, NODE do OR(multicast_mode_enabled); -s = select like(".*mtu") from SERVICE.CONFIG save; +s = select like(".*mtu") from NETWORK.CONFIG save; r = group by CLUSTER do NO_MATCH(s, ==, MAJORITY) save; ASSERT(r, False, "Different heartbeat.mtu.", "OPERATIONS", WARNING, "Listed node[s] have a different heartbeat.mtu configured. A multicast packet can only be as large as the interface mtu. Different mtu values might create cluster stability issue. Please contact Aerospike Support team.", @@ -356,19 +407,30 @@ r = do APPLY_TO_ANY(d, IN, f); ASSERT(r, False, "Device name misconfigured.", "OPERATIONS", WARNING, "Listed device[s] have partitions on same node. This might create situation like data corruption where data written to main drive gets overwritten/corrupted from data written to or deleted from the partition with the same name.", "Device name misconfiguration check."); - -s = select "device_total_bytes", "device-total-bytes", "total-bytes-disk" from NAMESPACE.STATISTICS save; + +s = select "data_total_bytes", "device_total_bytes", "pmem_total_bytes", "device-total-bytes", "total-bytes-disk" from NAMESPACE.STATISTICS save; r = group by CLUSTER, NAMESPACE do NO_MATCH(s, ==, MAJORITY) save; ASSERT(r, False, "Different namespace device size configuration.", "OPERATIONS", WARNING, "Listed namespace[s] have difference in configured disk size. Please run 'show statistics namespace like bytes' to check total device size. Probable cause - config file misconfiguration.", "Namespace device size configuration difference check."); +SET CONSTRAINT VERSION < 4.9; hwm = select "high-water-disk-pct" from NAMESPACE.CONFIG save; hwm = group by CLUSTER, NAMESPACE hwm; r = do hwm == 50; -ASSERT(r, True, "Non-default namespace device high water mark configuration.", "OPERATIONS", INFO, - "Listed namespace[s] have non-default high water mark configuration. Please run 'show config namespace like high-water-disk-pct' to check value. Probable cause - config file misconfiguration.", - "Non-default namespace device high water mark check."); +ASSERT(r, True, "Non-default namespace storage-engine eviction threshold configuration.", "OPERATIONS", INFO, + "Listed namespace[s] have non-default eviction threshold configuration. Please run 'show config namespace like high-water evict-used' to check value. Probable cause - config file misconfiguration.", + "Non-default namespace storage-engine eviction threshold check."); + +SET CONSTRAINT VERSION >= 4.9; +hwm = select "high-water-disk-pct", "storage-engine.evict-used-pct" from NAMESPACE.CONFIG save; +hwm = group by CLUSTER, NAMESPACE hwm; +r = do hwm == 0; +ASSERT(r, True, "Non-default namespace storage-engine eviction threshold configuration.", "OPERATIONS", INFO, + "Listed namespace[s] have non-default eviction threshold configuration. Please run 'show config namespace like high-water evict-used' to check value. Probable cause - config file misconfiguration.", + "Non-default namespace storage-engine eviction threshold check."); + +SET CONSTRAINT VERSION ALL; lwm = select like(".*defrag-lwm-pct") from NAMESPACE.CONFIG save; lwm = group by CLUSTER, NAMESPACE lwm; @@ -377,12 +439,12 @@ ASSERT(r, True, "Non-default namespace device low water mark configuration.", "O "Listed namespace[s] have non-default low water mark configuration. Probable cause - config file misconfiguration.", "Non-default namespace device low water mark check."); -hwm = select "high-water-disk-pct" as "defrag-lwm-pct" from NAMESPACE.CONFIG save; +hwm = select "high-water-disk-pct" as "defrag-lwm-pct", "storage-engine.evict-used-pct" as "defrag-lwm-pct" from NAMESPACE.CONFIG save; lwm = select like(".*defrag-lwm-pct") as "defrag-lwm-pct" from NAMESPACE.CONFIG save; r = do lwm < hwm on common; r = group by CLUSTER, NAMESPACE r; ASSERT(r, False, "Defrag low water mark misconfigured.", "OPERATIONS", WARNING, - "Listed namespace[s] have defrag-lwm-pct lower than high-water-disk-pct. This might create situation like no block to write, no eviction and no defragmentation. Please run 'show config namespace like high-water-disk-pct defrag-lwm-pct' to check configured values. Probable cause - namespace watermark misconfiguration.", + "Listed namespace[s] have defrag-lwm-pct lower than eviction threshold. This might create situation like no block to write, no eviction and no defragmentation. Please run 'show config namespace like high-water evict-used defrag-lwm-pct' to check configured values. Probable cause - namespace watermark misconfiguration.", "Defrag low water mark misconfiguration check."); commit_to_device = select "storage-engine.commit-to-device" from NAMESPACE.CONFIG; @@ -407,13 +469,13 @@ ASSERT(r, True, "Number of Sets equal to or above 750", "LIMITS", INFO, stop_writes = select "stop_writes" from NAMESPACE.STATISTICS; stop_writes = group by CLUSTER, NAMESPACE stop_writes; ASSERT(stop_writes, False, "Namespace has hit stop-writes (stop_writes = true)", "OPERATIONS" , CRITICAL, - "Listed namespace(s) have hit stop-write. Please run 'show statistics namespace like stop_writes' for details.", + "Listed namespace(s) have hit stop-write. Please run 'show stop-writes' for details.", "Namespace stop-writes flag check."); clock_skew_stop_writes = select "clock_skew_stop_writes" from NAMESPACE.STATISTICS; clock_skew_stop_writes = group by CLUSTER, NAMESPACE clock_skew_stop_writes; ASSERT(clock_skew_stop_writes, False, "Namespace has hit clock-skew-stop-writes (clock_skew_stop_writes = true)", "OPERATIONS" , CRITICAL, - "Listed namespace(s) have hit clock-skew-stop-writes. Please run 'show statistics namespace like clock_skew_stop_writes' for details.", + "Listed namespace(s) have hit clock-skew-stop-writes. Please run 'show stop-writes' for details.", "Namespace clock-skew-stop-writes flag check."); SET CONSTRAINT VERSION < 4.3; @@ -433,21 +495,23 @@ It collects cluster-size and uses it to find out expected data distribution for with available space per node per namespace. */ -t = select "device_total_bytes" as "disk_space", "device-total-bytes" as "disk_space", "total-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; -u = select "used-bytes-disk" as "disk_space", "device_used_bytes" as "disk_space" from NAMESPACE.STATISTICS; +t = select "data_total_bytes" as "disk_space", "pmem_total_bytes" as "disk_space", "device_total_bytes" as "disk_space", "device-total-bytes" as "disk_space", "total-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; +u = select "data_used_bytes" as "disk_space", "pmem_used_bytes" as "disk_space", "device_used_bytes" as "disk_space", "used-bytes-disk" as "disk_space" from NAMESPACE.STATISTICS; /* Available extra space */ e = do t - u; -e = group by CLUSTER, NAMESPACE, NODE do SUM(e) save as "available device space"; +e = group by CLUSTER, NAMESPACE, NODE do SUM(e) save as "available storage space"; s = select "cluster_size" as "size" from SERVICE; n = do MAX(s); n = do n - 1; /* Extra space need if 1 node goes down */ e1 = do u / n; -e1 = group by CLUSTER, NAMESPACE do MAX(e1) save as "distribution share of used device space per node"; +e1 = group by CLUSTER, NAMESPACE do MAX(e1) save as "distribution share of used storage space per node"; r = do e > e1; -ASSERT(r, True, "Namespace under configured (disk) for single node failure.", "OPERATIONS", WARNING, - "Listed namespace[s] does not have enough disk space configured to deal with increase in data per node in case of 1 node failure. Please run 'show statistics namespace like bytes' to check device space. It is non-issue if single replica limit is set to larger values, i.e if number of replica copies are reduced in case of node loss.", - "Namespace single node failure disk config check."); +ASSERT(r, False, "Namespace storage under configured for single node failure.", "OPERATIONS", WARNING, + "Listed namespace[s] does not have enough storage space configured to deal with increase in data per node in case of 1 node failure. Please run 'show statistics namespace like bytes' to check storage space. It is non-issue if single replica limit is set to larger values, i.e if number of replica copies are reduced in case of node loss.", + "Namespace single node failure storage space config check."); + +SET CONSTRAINT VERSION < 7.0.0; /* Same as above query but for memory @@ -609,7 +673,7 @@ u = group by CLUSTER, NODE do MAX(u); s = do s / u on common; r = group by CLUSTER, DC, KEY do SD_ANOMALY(s, ==, 3); ASSERT(r, False, "Skewed cluster remote DC statistics.", "ANOMALY", WARNING, - "Listed DC statistic[s] show skew for the listed node[s]. Please run 'show statistics dc' to get all DC stats. May be non-issue if remote Data center connectivity behavior for nodes is not same.", + "Listed DC statistic[s] show skew for the listed node[s]. Please run 'show statistics dc' to get all DC stats. May be non-issue if remote Data center connectivity behavior for nodes is not same.", "Remote DC statistics anomaly check."); /* @@ -1936,7 +2000,7 @@ cluster_size = select "cluster_size" as "sprig_limit_critical" from SERVICE.STAT cluster_size = group by CLUSTER do MAX(cluster_size) save as "cluster-size"; repl = select "effective_replication_factor" as "sprig_limit_critical" from NAMESPACE.STATISTICS save as "effective_repl_factor"; pts = select "partition-tree-sprigs" as "sprig_limit_critical" from NAMESPACE.CONFIG save as "partition-tree-sprigs"; -size_limit = select "index-type.mounts-size-limit" as "sprig_limit_critical" from NAMESPACE.CONFIG; +size_limit = select "index-type.mounts-size-limit" as "sprig_limit_critical", "index-type.mounts-budget" as "sprig_limit_critical" from NAMESPACE.CONFIG; // below statement adds thousand delimiter to mounts-size-limiter when it prints size_limit = do size_limit * 1 save as "mounts-size-limit"; @@ -1957,11 +2021,12 @@ dont_skip = group by CLUSTER, NODE, NAMESPACE do OR(dont_skip); num_partitions = do 4096 * repl; partitions_per_node = do num_partitions/cluster_size; pts_per_node = do partitions_per_node * pts; +// 4K partition-tree-sprig overhead total_pts = do pts_per_node * 4096 save as "Minimum space required"; result = do total_pts > size_limit; ASSERT(result, False, "ALL FLASH - Too many sprigs per partition for current available index mounted space. Some records are likely failing to be created.", "OPERATIONS", CRITICAL, - "Minimum space required for sprig overhead at current cluster size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at current cluster size exceeds mounts-budget/mounts-size-limit. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for current cluster size.", dont_skip); @@ -1972,9 +2037,9 @@ mcs = select "min-cluster-size" as "sprig_limit_warning" from SERVICE; mcs = group by CLUSTER do MAX(mcs) save as "min-cluster-size"; repl = select "replication-factor" as "sprig_limit_warning" from NAMESPACE.STATISTICS; pts = select "partition-tree-sprigs" as "sprig_limit_warning" from NAMESPACE.CONFIG; -msl = select "index-type.mounts-size-limit" as "sprig_limit_warning" from NAMESPACE.CONFIG; +msl = select "index-type.mounts-size-limit" as "sprig_limit_warning", "index-type.mounts-budget" as "sprig_limit_warning" from NAMESPACE.CONFIG; // below statement adds thousand delimiter to mounts-size-limiter when it prints -msl = do msl * 1 save as "mounts-size-limit"; +msl = do msl * 1 save; // calculate sprig overhead // The replication factor should be min(repl, mcs) @@ -1988,7 +2053,7 @@ repl_smaller = do repl < mcs; e1 = do repl_smaller && dont_skip; ASSERT(r1, False, "ALL FLASH - Too many sprigs per partition for configured min-cluster-size.", "OPERATIONS", WARNING, - "Minimum space required for sprig overhead at min-cluster-size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at min-cluster-size exceeds index-mount size. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for minimum cluster size.", e1); @@ -1997,6 +2062,7 @@ ASSERT(r1, False, "ALL FLASH - Too many sprigs per partition for configured min- // Only is asserted if min-cluster-size is smaller than replication-factor. // r2 = do 4096 * mcs; // r2 = do r2/mcs; +// 4096 * mcs / mcs = 4096 r2 = 4096; r2 = do r2 * pts; r2 = do r2 * 4096 save as "Minimum space required"; @@ -2006,7 +2072,7 @@ mcs_smaller = do mcs <= repl; e2 = do mcs_smaller && dont_skip; ASSERT(r2, False, "ALL FLASH - Too many sprigs per partition for configured min-cluster-size.", "OPERATIONS", WARNING, - "Minimum space required for sprig overhead at min-cluster-size exceeds mounts-size-limit. + "Minimum space required for sprig overhead at min-cluster-size exceeds index-mount size. See: https://www.aerospike.com/docs/operations/configure/namespace/index/#flash-index and https://www.aerospike.com/docs/operations/plan/capacity/#aerospike-all-flash", "Check for too many sprigs for minimum cluster size.", e2);