From 652e941bcca5759520e3031a7ec3edd47c18b626 Mon Sep 17 00:00:00 2001 From: InfinityDevTech <47366052+InfinityDevTech@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:26:09 -0400 Subject: [PATCH 1/2] Changed a BUNCH of things. --- .vscode/settings.json | 2 +- LICENSE.MD | 2 +- README.md | 10 +- docker-compose.example.yml | 103 +- .../carbon-relay-ng/carbon-relay-ng.ini | 143 -- .../carbonapi/aliasByRedis.yaml | 23 - grafanaConfig.example/carbonapi/carbonapi.yml | 216 ---- .../carbonapi/graphTemplates.yaml | 99 -- .../carbonapi/graphiteWeb.yaml | 25 - grafanaConfig.example/carbonapi/moving.yaml | 4 - .../carbonapi/timeShift.yaml | 2 - .../go-carbon/storage-aggregation.conf | 29 - .../go-carbon/storage-quotas.conf | 23 - .../go-carbon/storage-schemas.conf | 25 - .../grafana/dashboards/carbon-api.json | 396 ------ .../grafana/dashboards/carbon-relay-ng.json | 1149 ----------------- .../grafana/dashboards/go-carbon.json | 954 -------------- .../provisioning/datasources/carbonapi.yaml | 2 +- .../graphite/aggregation-rules.conf | 35 + grafanaConfig.example/graphite/blacklist.conf | 5 + grafanaConfig.example/graphite/brubeck.json | 24 + .../graphite/carbon.amqp.conf | 75 ++ grafanaConfig.example/graphite/carbon.conf | 646 +++++++++ grafanaConfig.example/graphite/dashboard.conf | 57 + .../{go-carbon => graphite}/go-carbon.conf | 261 +--- .../graphite/graphTemplates.conf | 38 + .../graphite/relay-rules.conf | 21 + .../graphite/rewrite-rules.conf | 18 + .../graphite/storage-aggregation.conf | 42 + .../graphite/storage-schemas.conf | 26 + grafanaConfig.example/graphite/whitelist.conf | 6 + package-lock.json | 4 +- src/pushStats/apiFunctions.js | 8 +- src/pushStats/index.js | 41 +- src/setup/setup.js | 23 +- users.example.json | 59 +- 36 files changed, 1127 insertions(+), 3469 deletions(-) delete mode 100644 grafanaConfig.example/carbon-relay-ng/carbon-relay-ng.ini delete mode 100644 grafanaConfig.example/carbonapi/aliasByRedis.yaml delete mode 100644 grafanaConfig.example/carbonapi/carbonapi.yml delete mode 100644 grafanaConfig.example/carbonapi/graphTemplates.yaml delete mode 100644 grafanaConfig.example/carbonapi/graphiteWeb.yaml delete mode 100644 grafanaConfig.example/carbonapi/moving.yaml delete mode 100644 grafanaConfig.example/carbonapi/timeShift.yaml delete mode 100644 grafanaConfig.example/go-carbon/storage-aggregation.conf delete mode 100644 grafanaConfig.example/go-carbon/storage-quotas.conf delete mode 100644 grafanaConfig.example/go-carbon/storage-schemas.conf delete mode 100644 grafanaConfig.example/grafana/dashboards/carbon-api.json delete mode 100644 grafanaConfig.example/grafana/dashboards/carbon-relay-ng.json delete mode 100644 grafanaConfig.example/grafana/dashboards/go-carbon.json create mode 100644 grafanaConfig.example/graphite/aggregation-rules.conf create mode 100644 grafanaConfig.example/graphite/blacklist.conf create mode 100644 grafanaConfig.example/graphite/brubeck.json create mode 100644 grafanaConfig.example/graphite/carbon.amqp.conf create mode 100644 grafanaConfig.example/graphite/carbon.conf create mode 100644 grafanaConfig.example/graphite/dashboard.conf rename grafanaConfig.example/{go-carbon => graphite}/go-carbon.conf (51%) create mode 100644 grafanaConfig.example/graphite/graphTemplates.conf create mode 100644 grafanaConfig.example/graphite/relay-rules.conf create mode 100644 grafanaConfig.example/graphite/rewrite-rules.conf create mode 100644 grafanaConfig.example/graphite/storage-aggregation.conf create mode 100644 grafanaConfig.example/graphite/storage-schemas.conf create mode 100644 grafanaConfig.example/graphite/whitelist.conf diff --git a/.vscode/settings.json b/.vscode/settings.json index b7d4426..a8c804e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "eslint.validate": ["javascript"] } diff --git a/LICENSE.MD b/LICENSE.MD index cb7a1f3..6fa2c0e 100644 --- a/LICENSE.MD +++ b/LICENSE.MD @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Pieter Brandsen +Copyright (c) 2023 Pieter Brandsen and Infinity Development Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 22d00fb..e326d70 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ B. Private: ```json { "prefix": "a.b.c", -"username": "W1N1", +"username": "EMAIL", +"replaceName": "USERNAME HERE", "type": "private", "shards": ["screeps"], "password": "password", @@ -48,11 +49,12 @@ If the private server is not hosted on localhost, add the host to the user: ```json { -"username": "W1N1", +"username": "EMAIL", +"replaceName": "USERNAME", "type": "private", "shards": ["screeps"], "password": "password", -"host": "123.456.789", +"host": "192.168.1.10", } ``` @@ -77,12 +79,10 @@ Update all .example files and/or folders to match your needs. This step is not r * `--force`: force the non .example config files to be overwritten. * `--debug`: listen to setup Docker logs -* `--traefik`: Add traefik labels to the docker-compose.yml file, reverse proxy for docker containers. * `--username`: overwrite the username for the Grafana admin user * `--password`: overwrite the password for the Grafana admin user * `--defaultRetention`: overwrite the default retention for the default retention polic of all not regex'd retention paths. * `--enableAnonymousAccess`: enable anonymous access to Grafana -* `--traefikHost`: use only traefik forwarding #### Network diff --git a/docker-compose.example.yml b/docker-compose.example.yml index e0fe47b..8c96e53 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -2,15 +2,11 @@ version: "3.5" volumes: grafana_data: - go-carbon_data: + graphite_data: networks: - internal: + stats: driver: bridge - external: - driver: bridge - #t#htraefik-net: - #t#h external: true x-logging: &default-logging driver: "local" @@ -19,53 +15,15 @@ x-logging: &default-logging max-file: "5" services: - stats-getter: - build: - context: . - dockerfile: ./src/pushStats/Dockerfile - volumes: - - ./logs/statsGetter:/app/logs - depends_on: - - carbon-relay-ng - environment: - - PREFIX= - - SERVER_PORT=21025 - - INCLUDE_PUSH_STATUS_API=false - carbon-relay-ng: - image: grafana/carbon-relay-ng - depends_on: - - go-carbon - volumes: - - ./grafanaConfig/carbon-relay-ng:/conf - ports: - - 127.0.0.1:2003:2003 - networks: - - internal - logging: *default-logging - go-carbon: - image: ghcr.io/go-graphite/go-carbon + graphite: + image: graphiteapp/graphite-statsd volumes: - - ./grafanaConfig/go-carbon/go-carbon.conf:/etc/go-carbon/go-carbon.conf - - ./grafanaConfig/go-carbon:/etc/go-carbon/ - - ./go-carbon-storage:/var/lib/graphite/whisper - #- ./logs/goCarbon:/var/log/go-carbon + - ./logs/graphite:/var/log + - ./grafanaConfig/graphite:/opt/graphite/conf + - graphite_data:/opt/graphite/storage networks: - - internal - logging: *default-logging - carbon-api: - image: pieterbrandsen/carbonapi-0.15.6 - volumes: - - ./grafanaConfig/carbonapi/carbonapi.yml:/etc/carbonapi.yml - - ./grafanaConfig/carbonapi:/conf:ro - - ./logs/carbonApi:/log - depends_on: - - go-carbon - networks: - - internal - - external - logging: *default-logging + - stats grafana: - #t container_name: grafana image: grafana/grafana-oss:9.3.6-ubuntu volumes: - grafana_data:/var/lib/grafana @@ -73,39 +31,24 @@ services: - ./logs/grafana:/var/log/grafana ports: - 3000:3000 - extra_hosts: - - "host.docker.internal:host-gateway" healthcheck: test: "curl -fsSL -o /dev/null http://localhost:3000/login" interval: 10s timeout: 5s retries: 3 networks: - - external - #t#h- traefik-net - logging: *default-logging - #t labels: - #t - "traefik.enable=true" - #t - "traefik.http.routers.grafana.rule=Host(`grafana.localhost`)" - #t#h - "traefik.docker.network=traefik-net" - #t - "traefik.http.services.grafana.loadbalancer.server.port=3000" - #t#h - "traefik.http.routers.grafana.entrypoints=websecure" - #t#h - "traefik.http.routers.grafana.tls.certresolver=letsencrypt" - #ttraefik: - #t image: traefik:v2.5.4 - #t container_name: traefik - #t command: - #t - "--log.level=DEBUG" - #t - "--api.insecure=true" - #t - "--api.dashboard=true" - #t - "--providers.docker=true" - #t - "--providers.docker.exposedbydefault=false" - #t - "--entrypoints.web.address=:80" - #t - "--entrypoints.websecure.address=:443" - #t volumes: - #t - /var/run/docker.sock:/var/run/docker.sock - #t ports: - #t - 80:80 - #t - 443:443 - #t - 8080:8080 - #t logging: *default-logging + - stats + stats-getter: + build: + context: . + dockerfile: ./src/pushStats/Dockerfile + volumes: + - ./logs/statsGetter:/app/logs + depends_on: + - graphite + environment: + - PREFIX= + - SERVER_PORT=21025 + - INCLUDE_PUSH_STATUS_API=false + networks: + - stats \ No newline at end of file diff --git a/grafanaConfig.example/carbon-relay-ng/carbon-relay-ng.ini b/grafanaConfig.example/carbon-relay-ng/carbon-relay-ng.ini deleted file mode 100644 index 042e521..0000000 --- a/grafanaConfig.example/carbon-relay-ng/carbon-relay-ng.ini +++ /dev/null @@ -1,143 +0,0 @@ -## Global settings ## - -# instance id's distinguish stats of multiple relays. -# do not run multiple relays with the same instance id. -# supported variables: -# ${HOST} : hostname -instance = "carbon-relay-ng" - -## System ## -# this setting can be used to override the default GOMAXPROCS logic -# it is ignored if the GOMAXPROCS environment variable is set -# max_procs = 2 -pid_file = "/var/run/carbon-relay-ng.pid" -# directory for spool files -spool_dir = "/var/spool/carbon-relay-ng" - -## Logging ## -# one of trace debug info warn error fatal panic -# see docs/logging.md for level descriptions -# note: if you used to use `notice`, you should now use `info`. -log_level = "info" - -## Admin ## -admin_addr = "carbon-relay-ng:2004" -http_addr = "carbon-relay-ng:8081" - -## Inputs ## -### plaintext Carbon ### -listen_addr = "carbon-relay-ng:2003" -# close inbound plaintext connections if they've been idle for this long ("0s" to disable) -plain_read_timeout = "2m" -### Pickle Carbon ### -pickle_addr = "carbon-relay-ng:2013" -# close inbound pickle connections if they've been idle for this long ("0s" to disable) -pickle_read_timeout = "2m" - -## Validation of inputs ## -# Metric name validation strictness for legacy metrics. Valid values are: -# strict - Block anything that can upset graphite: valid characters are [A-Za-z0-9_-.]; consecutive dots are not allowed -# medium - Valid characters are ASCII; no embedded NULLs -# none - No validation is performed -validation_level_legacy = "medium" -# Metric validation for carbon2.0 (metrics2.0) metrics. -# Metrics that contain = or _is_ are assumed carbon2.0. -# Valid values are: -# medium - checks for unit and mtype tag, presence of another tag, and constency (use = or _is_, not both) -# none - No validation is performed -validation_level_m20 = "medium" - -# you can also validate that each series has increasing timestamps -validate_order = false - -# How long to keep track of invalid metrics seen -# Useful time units are "s", "m", "h" -bad_metrics_max_age = "24h" - -# Blocklist -# See https://github.com/grafana/carbon-relay-ng/blob/master/docs/config.md#Blocklist - -blocklist = [ -] - -### AMQP ### -[amqp] -amqp_enabled = false -amqp_host = "relay" -amqp_port = 5672 -amqp_user = "guest" -amqp_password = "guest" -amqp_vhost = "/graphite" -amqp_exchange = "metrics" -amqp_queue = "" -amqp_key = "#" -amqp_durable = false -amqp_exclusive = true - -# Aggregators -# See https://github.com/grafana/carbon-relay-ng/blob/master/docs/config.md#Aggregators - -# Rewriters -# See https://github.com/grafana/carbon-relay-ng/blob/master/docs/config.md#Rewriters - -# Routes -# See https://github.com/grafana/carbon-relay-ng/blob/master/docs/config.md#Routes - -[[rewriter]] -old = 'service_is_carbon-relay-ng' -new = 'stats.carbon-relay-ng.service_is_carbon-relay-ng' -not = '' -max = -1 - -[[rewriter]] -old = 'carbon-relay-ng.stats.carbon-relay-ng' -new = 'stats.carbon-relay-ng.carbon-relay-ng' -not = '' -max = -1 - -[[route]] -# a plain carbon route that sends all data to the specified carbon (graphite) server -key = 'default' -type = 'sendAllMatch' -destinations = [ - 'go-carbon:2003 spool=true pickle=false', -] - -# [[route]] -# # a plain carbon route that sends all data to the specified carbon (graphite) server -# key = 'default' -# type = 'sendAllMatch' -# sub = 'carbon-relay-ng.stats' -# destinations = [ -# 'go-carbon:2003 spool=true pickle=false prefix=stats.carbon-relay-ng.', -# ] - -# [[route]] -# # a plain carbon route that sends all data to the specified carbon (graphite) server -# key = 'default' -# type = 'sendAllMatch' -# sub = 'service_is_carbon-relay-ng' -# destinations = [ -# 'go-carbon:2003 spool=true pickle=false prefix=stats.carbon-relay-ng.', -# ] - -[init] -# init commands (DEPRECATED) -# see https://github.com/grafana/carbon-relay-ng/blob/master/docs/config.md#Imperatives -cmds = [ -] - -## Instrumentation ## - -[instrumentation] -# in addition to serving internal metrics via expvar, you can send them to graphite/carbon -# IMPORTANT: setting this to "" will disable flushing, and metrics will pile up and lead to OOM -# see https://github.com/grafana/carbon-relay-ng/issues/50 -# so for now you MUST send them somewhere. sorry. -# If you really don't want them, send them in the relay and then ignore them with the following blocklist entries: -# (fill in the instance - see top of this file - or just remove the '' bit to drop all carbon-relay-ng data (even from other relays) -# 'prefix service_is_carbon-relay-ng.', -# 'prefix carbon-relay-ng.stats.' -# (Also, the interval here must correspond to your setting in storage-schemas.conf if you use grafana hosted metrics) -graphite_addr = "carbon-relay-ng:2003" -graphite_interval = 60000 # in ms \ No newline at end of file diff --git a/grafanaConfig.example/carbonapi/aliasByRedis.yaml b/grafanaConfig.example/carbonapi/aliasByRedis.yaml deleted file mode 100644 index 84ecf89..0000000 --- a/grafanaConfig.example/carbonapi/aliasByRedis.yaml +++ /dev/null @@ -1,23 +0,0 @@ -enabled: false -## redis address. Default: 127.0.0.1:6379 -#address: "127.0.0.1:6379" -## connection timeout to redis -#connectTimeout: "100ms" -## maxIdleConnections for redis connection pool, default: 1 -#maxIdleConnections: 10 -## timeout for idle connections to redis -#idleTimeout: 1s -## redis database number -#databaseNumber: 0 -## redis username -#username: "some-user-name" -## redis password -#password: "some-password" -## default queryTimeout value is 250ms -#queryTimeout: "250ms" -## kepp alive probes interval -#keepAliveInterval: "1m" -## use encryption -#useTLS: true -## allow self-signed, expired, etc certificates -#TLSSkipVerify: false diff --git a/grafanaConfig.example/carbonapi/carbonapi.yml b/grafanaConfig.example/carbonapi/carbonapi.yml deleted file mode 100644 index 5f59b99..0000000 --- a/grafanaConfig.example/carbonapi/carbonapi.yml +++ /dev/null @@ -1,216 +0,0 @@ -# Need to be URL, http or https -# This url specifies the backend or a loadbalancer -# -# If you are using carbonzipper you should set it to -# zipper's url -# -# If you are using plain go-carbon or graphite-clickhouse -# you should set it to URL of go-carbon's carbonserver module -# or graphite-clickhouse's http url. -# Listen address, should always include hostname or ip address and a port. -listeners: - # - address: "127.0.0.1:8081" - # - address: "[::1]:8081" - # # OR - # - address: "localhost:8081" -# # OR -listen: "carbon-api:8081" - -# Specify URL Prefix for all handlers -prefix: "" -# Use custom caching DNS resolver instead of default one. You shouldn't use it unless you know what you are doing. -useCachingDNSResolver: false -# TTL for DNS records in DNS cache. Only matters if `useCachingDNSResolver` is enabled. -cachingDNSRefreshTime: "1m" -# Specify if metrics are exported over HTTP and if they are available on the same address or not -# pprofEnabled controls if extra HTTP Handlers to profile and debug application will be available -expvar: - enabled: true - pprofEnabled: false - listen: "" -# Allow extra charsets in metric names. By default only "Latin" is allowed -# Please note that each unicodeRangeTables will slow down metric parsing a bit -# For list of supported tables, see: https://golang.org/src/unicode/tables.go?#L3437 -# Special name "all" reserved to append all tables that's currently supported by Go -#unicodeRangeTables: -# - "Latin" -# - "Cyrillic" -# - "Hiragana" -# - "Katakana" -# - "Han" -## - "all" -# Controls headers that would be passed to the backend -headersToPass: - - "X-Dashboard-Id" - - "X-Grafana-Org-Id" - - "X-Panel-Id" -headersToLog: - - "X-Dashboard-Id" - - "X-Grafana-Org-Id" - - "X-Panel-Id" -# Specify custom function aliases. -# This is example for alias "perMinute(metrics)" that will behave as "perSecond(metric)|scale(60)" -define: - - - name: "perMinute" - template: "perSecond({{.argString}})|scale(60)" -# Control what status code will be returned where /render or find query do not return any metric. Default is 200 -notFoundStatusCode: 200 -# Max concurrent requests to CarbonZipper -concurency: 1000 -cache: - # Type of caching. Valid: "mem", "memcache", "null" - type: "mem" - # Cache limit in megabytes - size_mb: 0 - # Default cache timeout value. Identical to DEFAULT_CACHE_DURATION in graphite-web. - defaultTimeoutSec: 60 - # Only used by memcache type of cache. List of memcache servers. - memcachedServers: - - "localhost:1234" - - "localhost:1235" -# Amount of CPUs to use. 0 - unlimited -cpus: 0 -# Timezone, default - local -tz: "" - -# By default, functions like aggregate inherit tags from first series (for compatibility with graphite-web) -# If set to true, tags are extracted from seriesByTag arguments -#extractTagsFromArgs: false -functionsConfig: - graphiteWeb: /conf/graphiteWeb.yaml - timeShift: /conf/timeShift.yaml - moving: /conf/moving.yaml - movingMedian: /conf/moving.yaml - aliasByRedis: /conf/aliasByRedis.yaml -maxBatchSize: 500 -graphite: - # Host:port where to send internal metrics - # Empty = disabled - host: "go-carbon:2003" - interval: "60s" - prefix: "stats.carbonapi.agents" - # rules on how to construct metric name. For now only {prefix} and {fqdn} is supported. - # {prefix} will be replaced with the content of {prefix} - # {fqdn} will be repalced with fqdn - pattern: "{prefix}.{fqdn}" -# Maximium idle connections to carbonzipper -idleConnections: 10 -pidFile: "" -# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation -upstreams: - # Use TLD Cache. Useful when you have multiple backends that could contain - # different TLDs. - # - # For example whenever you have multiple top level metric namespaces, like: - # one_min.some.metric - # ten_min.some_metric - # one_hour.some_metric - # - # `one_min`, `ten_min` and `one_hour` are considered to be TLDs - # carbonapi by default will probe all backends and cache the responses - # and will know which backends would contain the prefix of the request - # - # This option allows to disable that, which could be helpful for backends like - # `clickhouse` or other backends where all metrics are part of the same cluster - tldCacheDisabled: false - - # Number of 100ms buckets to track request distribution in. Used to build - # 'carbon.zipper.hostname.requests_in_0ms_to_100ms' metric and friends. - # Requests beyond the last bucket are logged as slow (default of 10 implies - # "slow" is >1 second). - # The last bucket is _not_ called 'requests_in_Xms_to_inf' on purpose, so - # we can change our minds about how many buckets we want to have and have - # their names remain consistent. - buckets: 10 - - # If request took more than specified amount of time, it will be logged as a slow request as well - slowLogThreshold: "1s" - - timeouts: - # Maximum backend request time for find requests. - find: "2s" - # Maximum backend request time for render requests. This is total one and doesn't take into account in-flight requests - render: "10s" - # Timeout to connect to the server - connect: "200ms" - - # Number of concurrent requests to any given backend - default is no limit. - # If set, you likely want >= MaxIdleConnsPerHost - concurrencyLimitPerServer: 0 - - # Configures how often keep alive packets will be sent out - keepAliveInterval: "30s" - - # Control http.MaxIdleConnsPerHost. Large values can lead to more idle - # connections on the backend servers which may bump into limits; tune with care. - maxIdleConnsPerHost: 100 - - # Only affects cases with maxBatchSize > 0. If set to `false` requests after split will be sent out one by one, otherwise in parallel - doMultipleRequestsIfSplit: false - - # "http://host:port" array of instances of carbonserver stores - # It MUST be specified. - backends: - - "http://go-carbon:8080" - - #backends section will override this one! - backendsv2: - backends: - - - groupName: "group" - protocol: "carbonapi_v3_pb" - lbMethod: "roundrobin" - servers: - - "http://go-carbon:8080" - - - # carbonsearch is not used if empty - # carbonsearch: - # # Instance of carbonsearch backend - # backend: "http://go-carbon:8070" - # # carbonsearch prefix to reserve/register - # prefix: "virt.v1.*" - # # carbonsearch is not used if empty - # # carbonsearch section will override this one! - carbonsearchv2: - # Carbonsearch instances. Follows the same syntax as backendsv2 - backends: - - - groupName: "group" - protocol: "carbonapi_v3_pb" - lbMethod: "broadcast" - servers: - - "http://go-carbon:8080" - # carbonsearch prefix to reserve/register - prefix: "virt.v1.*" - - # Enable compatibility with graphite-web 0.9 - # This will affect graphite-web 1.0+ with multiple cluster_servers - # Default: disabled - graphite09compat: false -# If not zero, enabled cache for find requests -# This parameter controls when it will expire (in seconds) -# Default: 600 (10 minutes) -# graphTemplates: /conf/graphTemplates.yaml -expireDelaySec: 10 -# Uncomment this to get the behavior of graphite-web as proposed in https://github.com/graphite-project/graphite-web/pull/2239 -# Beware this will make darkbackground graphs less readable -#defaultColors: -# "red": "ff0000" -# "green": "00ff00" -# "blue": "#0000ff" -# "darkred": "#c80032" -# "darkgreen": "00c800" -# "darkblue": "002173" -logger: - - logger: "" - file: "stderr" - level: "debug" - encoding: "console" - encodingTime: "iso8601" - encodingDuration: "seconds" - - logger: "" - file: "log/carbonapi.log" - level: "info" - encoding: "json" \ No newline at end of file diff --git a/grafanaConfig.example/carbonapi/graphTemplates.yaml b/grafanaConfig.example/carbonapi/graphTemplates.yaml deleted file mode 100644 index 1ac4a4d..0000000 --- a/grafanaConfig.example/carbonapi/graphTemplates.yaml +++ /dev/null @@ -1,99 +0,0 @@ -graphite: - bgColor: "white" - fgColor: "black" - colorList: ["blue", "green", "red", "purple", "brown", "yellow", "aqua", "grey", "magenta", "pink", "gold", "rose" ] - -graphite-dark: - colorList: ["blue", "green", "red", "purple", "brown", "yellow", "aqua", "grey", "magenta", "pink", "gold", "rose" ] - -solarized-light: - bgColor: "#fdf6e3" - fgColor: "#657b83" - majorLine: "#073642" - minorLine: "#586e75" - colorList: ["268bd2aa","859900aa","dc322faa","d33682aa","db4b16aa","b58900aa","2aa198aa","6c71c4aa"] - -solarized-dark: - bgColor: "#002b36" - fgColor: "#839496" - majorLine: "#fdf6e3" - minorLine: "#eee8d5" - colorList: ["268bd2aa","859900aa","dc322faa","d33682aa","db4b16aa","b58900aa","2aa198aa","6c71c4aa"] - fontName: "Sans" - fontSize: "10" - fontBold: flase - fontItalic: flase - -classic: - bgColor: "black" - fgColor: "white" - majorLine: "white" - minorLine: "grey" - colorList: ["blue","green","red","purple","brown","yellow","aqua","grey","magenta","pink","gold","rose"] - fontName: "Sans" - fontSize: "10" - fontBold: flase - fontItalic: flase - -noc: - bgColor: "black" - fgColor: "white" - majorLine: "white" - minorLine: "grey" - colorList: ["blue","green","red","yellow","purple","brown","aqua","grey","magenta","pink","gold","rose"] - fontName: "Sans" - fontSize: "10" - fontBold: flase - fontItalic: flase - -summary: - bgColor: "black" - colorList: ["#6666ff"," #66ff66"," #ff6666"] - -alphas: - bgColor: "white" - fgColor: "black" - majorLine: "grey" - minorLine: "rose" - colorList: ["00ff00aa","ff000077","00337799"] - -grafana: - bgColor: "white" - fgColor: "black" - minorLine: "grey" - majorLine: "rose" - colorList: ["#7eb26d","#eab839","#6ed0e0","#ef843c","#e24d42","#1f78c1","#ba43a9","#705da0","#508642","#cca300","#447ebc"] - fontName: "Sans" - fontSize: "10" - fontBold: flase - fontItalic: flase - -ocean1: - colorList: ["f7fcf0","e0f3db","ccebc5","a8ddb5","7bccc4","4eb3d3","2b8cbe","0868ac","084081"] - -ocean2: - colorList: ["084081","0868ac","2b8cbe","4eb3d3","7bccc4","a8ddb5","ccebc5","e0f3db","f7fcf0"] - -forest1: - colorList: ["f7fcf5","e5f5e0","c7e9c0","a1d99b","74c476","41ab5d","238b45","005a32"] - -forest2: - colorList: ["005a32","238b45","41ab5d","74c476","a1d99b","c7e9c0","e5f5e0","f7fcf5"] - -sunset1: - colorList: ["fff5eb","fee6ce","fdd0a2","fdae6b","fd8d3c","f16913","d94801","8c2d04"] - -sunset2: - colorList: ["8c2d04","d94801","f16913","fd8d3c","fdae6b","fdd0a2","fee6ce","fff5eb"] - -moonlight1: - colorList: ["fcfbfd","efedf5","dadaeb","bcbddc","9e9ac8","807dba","6a51a3","4a1486"] - -moonlight2: - colorList: ["4a1486","6a51a3","807dba","9e9ac8","bcbddc","dadaeb","efedf5","fcfbfd"] - -lava1: - colorList: ["fff5f0","fee0d2","fcbba1","fc9272","fb6a4a","ef3b2c","cb181d","99000d"] - -lava2: - colorList: ["99000d","cb181d","ef3b2c","fb6a4a","fc9272","fcbba1","fee0d2","fff5f0"] \ No newline at end of file diff --git a/grafanaConfig.example/carbonapi/graphiteWeb.yaml b/grafanaConfig.example/carbonapi/graphiteWeb.yaml deleted file mode 100644 index 3ef48ac..0000000 --- a/grafanaConfig.example/carbonapi/graphiteWeb.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# to maintain backward compatibility, this function is disabled by default -enabled: false -# roundrobin will be used in case multiple URLs are specified -fallbackUrls: - - http://localhost:8081 - - http://localhost:8080 -# if enabled, function will also check if supported parameters are the same -strict: false -# specify max amount of connections per each instance of graphit-web -maxConcurrentConnections: 10 -# specify request timeouts -timeout: "90s" -keepAliveInterval: "30s" -# Uncomment to make those functions force-proxied to graphite-web -# forceAdd: -# - "sumSeries" -# - "derivative" - -# This will remove those functions from list of proxied ones -# This is example and also suggestion for functions to remove. -forceSkip: - - "aliasByTags" - - "seriesByTag" - - "groupByTags" - - "events" diff --git a/grafanaConfig.example/carbonapi/moving.yaml b/grafanaConfig.example/carbonapi/moving.yaml deleted file mode 100644 index c08133a..0000000 --- a/grafanaConfig.example/carbonapi/moving.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# return NaNs when window size is smaller than data step, otherwise return data unmodified -# e.g. movingAverage(metric1, "30sec") for data with 1 minute step -# default is true for graphite-web compatibility -returnNaNsIfStepMismatch: false diff --git a/grafanaConfig.example/carbonapi/timeShift.yaml b/grafanaConfig.example/carbonapi/timeShift.yaml deleted file mode 100644 index 592791f..0000000 --- a/grafanaConfig.example/carbonapi/timeShift.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# graphite-web 1.1 assume resetEnd=true by default. However older versions of carbonapi didn't support that parameter which was equal to `false`. This forces graphite-web's behavior -resetEndDefaultValue: true diff --git a/grafanaConfig.example/go-carbon/storage-aggregation.conf b/grafanaConfig.example/go-carbon/storage-aggregation.conf deleted file mode 100644 index d91b73e..0000000 --- a/grafanaConfig.example/go-carbon/storage-aggregation.conf +++ /dev/null @@ -1,29 +0,0 @@ -# Documentation: -# http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf - -[screepsUserTracker_User_Info] -pattern = ^screeps\.userTracker\..*\.users\..*\.info.* -xFilesFactor = 0.5 -aggregationMethod = last - -[screepsUserTracker_Status_Ips_DataCount] -pattern = ^screeps\.userTracker\..*\.status\.ips\..*\.dataCount$ -xFilesFactor = 0.5 -aggregationMethod = sum - -[screepsUserTracker_Status] -pattern = ^screeps\.userTracker\..*\.status.* -xFilesFactor = 0.5 -aggregationMethod = last - -[default] -pattern = .* -xFilesFactor = 0.5 -aggregationMethod = average - -# Partialy enable or disable online config migration for the matched metrics; -# only works if whisper.online-migration is enabled (more in -# deploy/go-carbon.conf and README.md). -# -# xffMigration = false -aggregationMethodMigration = true \ No newline at end of file diff --git a/grafanaConfig.example/go-carbon/storage-quotas.conf b/grafanaConfig.example/go-carbon/storage-quotas.conf deleted file mode 100644 index 0cee225..0000000 --- a/grafanaConfig.example/go-carbon/storage-quotas.conf +++ /dev/null @@ -1,23 +0,0 @@ -# This control all the namespaces under root -[*] -metrics = 1,000,000 -logical-size = 250,000,000,000 -physical-size = 50,000,000,000 -# max means practically no limit -data-points = max -throughput = max - -[sys.app.*] -metrics = 3,000,000 -logical-size = 1,500,000,000,000 -physical-size = 100,000,000,000 -data-points = 130,000,000,000 - -# This controls the root/global limits -[/] -namespaces = 20 -metrics = 10,000,000 -logical-size = 2,500,000,000,000 -physical-size = 2,500,000,000,000 -data-points = 200,000,000,000 -dropping-policy = new \ No newline at end of file diff --git a/grafanaConfig.example/go-carbon/storage-schemas.conf b/grafanaConfig.example/go-carbon/storage-schemas.conf deleted file mode 100644 index ff122ce..0000000 --- a/grafanaConfig.example/go-carbon/storage-schemas.conf +++ /dev/null @@ -1,25 +0,0 @@ -# Documentation: -# http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf -# -# compressed if specified, will overwrite the value set in go-carbon.conf. - -[statsGetter] -pattern = ^screeps.* -retentions = 60s:1h,15m:1d,1d:1y -compressed = false - -[stats] -pattern = ^stats.* -retentions = 60s:1h,15m:1d,1d:30d -compressed = false - -[default] -pattern = .* -retentions = 1h:30d -compressed = false - -# Partialy enable or disable online config migration for the matched metrics; -# only works if whisper.online-migration is enabled (more in -# deploy/go-carbon.conf and README.md). -# -migration = true diff --git a/grafanaConfig.example/grafana/dashboards/carbon-api.json b/grafanaConfig.example/grafana/dashboards/carbon-api.json deleted file mode 100644 index f461073..0000000 --- a/grafanaConfig.example/grafana/dashboards/carbon-api.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 8, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByMetric(sumSeries(stats.carbonapi.agents.*.cache_size))" - } - ], - "title": "Cache size", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByMetric(sumSeries(stats.carbonapi.agents.*.cache_items))" - } - ], - "title": "Cache items", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByMetric(sumSeries(stats.carbonapi.agents.*.find_requests))" - } - ], - "title": "Find requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByMetric(sumSeries(stats.carbonapi.agents.*.render_requests))" - } - ], - "title": "Render requests", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Carbonapi", - "uid": "i2_hQhI4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafanaConfig.example/grafana/dashboards/carbon-relay-ng.json b/grafanaConfig.example/grafana/dashboards/carbon-relay-ng.json deleted file mode 100644 index 05fd256..0000000 --- a/grafanaConfig.example/grafana/dashboards/carbon-relay-ng.json +++ /dev/null @@ -1,1149 +0,0 @@ -{ - "annotations": { - "enable": true, - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "visualize Carbon-relay-ng's metrics", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 338, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "aliasColors": { - "blacklist": "#E24D42", - "direction_is_blacklist": "#E0752D", - "direction_is_in": "#3F6833", - "direction_is_unroutable": "#890F02", - "in": "#3F6833", - "unroutable": "#890F02" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "aliasByNode(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.unit_is_Metric.*), 4)" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "B", - "target": "alias(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.unit_is_Err.type_is_invalid), 'invalid')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "C", - "target": "alias(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.unit_is_Err.type_is_out_of_order), 'out-of-order')" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "$instance incoming", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "A", - "target": "alias(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.dest_is_$dest.unit_is_Metric.direction_is_out), 'to tcp')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "B", - "target": "alias(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.spool_is_$dest.unit_is_Metric.status_is_incomingRT), 'to spool')" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "dest metric directions", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "reason_is_bad_pickle": "#CCA300", - "reason_is_conn_down_no_spool": "#962D82", - "reason_is_slow_conn": "#E0752D", - "reason_is_slow_spool": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "aliasByNode(derivative(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.dest_is_$dest.unit_is_Metric.action_is_drop.*),6)" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "dest metric drops", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "type_is_truncated": "#890F02", - "type_is_write": "#C15C17" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "aliasByNode(perSecond(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.dest_is_$dest.unit_is_Err.*), 5)" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "dest Errors", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_ticker.stat_is_max, 'ticker max')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "B", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_ticker.stat_is_max_99, 'ticker 99')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "C", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_ticker.stat_is_mean,'ticker mean')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "D", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max, 'manual max')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "E", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_99, 'manual 99')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "F", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_mean, 'manual mean')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "G", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_overflow.stat_is_max,'overflow max')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "H", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_overflow.stat_is_max99, 'overflow 99')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "I", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_B.what_is_FlushSize.type_is_overflow.mean,\"overflow mean\")" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "conn Flush size", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "metrics-buffered": "#E0752D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 3, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 8 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.dest_is_$dest.unit_is_Metric.what_is_numBuffered, 'numBuffered')" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "conn metrics in buffer", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "\"overflow max\"": "#EF843C", - "\"overflow mean\"": "#58140C", - "\"ticker max\"": "#6ED0E0", - "\"ticker mean\"": "#0A50A1" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 8 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "A", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.unit_is_ns.dest_is_$dest.what_is_durationFlush.type_is_ticker.stat_is_max, '\"ticker max\"')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "B", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.unit_is_ns.dest_is_$dest.what_is_durationFlush.type_is_ticker.stat_is_mean, '\"ticker mean\"')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "C", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.unit_is_ns.dest_is_$dest.what_is_durationFlush.type_is_overflow.stat_is_max, '\"overflow max\"')" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "D", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_gauge.unit_is_ns.dest_is_$dest.what_is_durationFlush.type_is_overflow.stat_is_mean, '\"overflow mean\"')" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "conn flush durations", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "status_is_incomingBulk": "#3F2B5B", - "status_is_incomingRT": "#0A50A1" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "aliasByNode(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_counter.spool_is_$dest.unit_is_Metric.*, 5)" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "B", - "target": "" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "spool inputs", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 10, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 15 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "hide": false, - "refId": "A", - "target": "alias(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_gauge.spool_is_$dest.unit_is_Metric.status_is_buffered, 'buffered')" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "spool metrics in buffer", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": "", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 15 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "A", - "target": "aliasByNode(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_gauge.unit_is_ns.spool_is_$dest.operation_is*.stat_is_max_50, 5, 6)" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "B", - "target": "aliasByNode(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_gauge.unit_is_ns.spool_is_$dest.operation_is*.stat_is_max_95, 5, 6)" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "C", - "target": "aliasByNode(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_gauge.unit_is_ns.spool_is_$dest.operation_is*.stat_is_max, 5, 6)" - }, - { - "datasource": { - "uid": "${DS_GRAPHITE}" - }, - "refId": "D", - "target": "aliasByNode(stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_carbonrelay1-next.mtype_is_gauge.unit_is_ns.spool_is_$dest.operation_is*.stat_is_mean, 5, 6)" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "spool durations", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "show": true - }, - { - "format": "short", - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [ - "telemetry" - ], - "templating": { - "list": [ - { - "allFormat": "glob", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "instance", - "options": [], - "query": "stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_*", - "refresh": 1, - "refresh_on_load": true, - "regex": "/instance_is_(.*)/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "allFormat": "glob", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "dest", - "options": [], - "query": "stats.carbon-relay-ng.service_is_carbon-relay-ng.instance_is_$instance.mtype_is_counter.dest_is_*", - "refresh": 1, - "refresh_on_load": true, - "regex": "/dest_is_(.*)/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "collapse": false, - "enable": true, - "notice": false, - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "type": "timepicker" - }, - "timezone": "browser", - "title": "Carbon-relay-ng", - "uid": "HQlrCeoVk", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/grafanaConfig.example/grafana/dashboards/go-carbon.json b/grafanaConfig.example/grafana/dashboards/go-carbon.json deleted file mode 100644 index 58f6582..0000000 --- a/grafanaConfig.example/grafana/dashboards/go-carbon.json +++ /dev/null @@ -1,954 +0,0 @@ -{ - "annotations": { - "enable": false, - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 11221, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/errors/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.tcp.metricsReceived), 3)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "B", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.udp.metricsReceived), 3)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "C", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.pickle.metricsReceived), 3)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "D", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.*.errors), 4)" - } - ], - "title": "Metric Inputs", - "type": "timeseries" - }, - { - "cacheTimeout": "", - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 1, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.persister.updateOperations), 'Updates')" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "C", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.persister.committedPoints), 'Committed Points')" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "F", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.persister.created), 'Creates')" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "D", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.persister.pointsPerUpdate), 'Points per Update')" - } - ], - "title": "Persister Actions", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPU" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/queries/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/overflow/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - }, - { - "id": "custom.stacking", - "value": { - "group": true, - "mode": "normal" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/maxSize/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FA6400", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": true, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 2, - "links": [], - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.cache.size), 4)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "B", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.cache.queries), 4)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "C", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.cache.maxSize), 4)" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "D", - "target": "aliasByNode(sumSeries(stats.gocarbon.agents.$agent.cache.overflow), 4)" - } - ], - "title": "Cache Status", - "type": "timeseries" - }, - { - "cacheTimeout": "", - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Time/" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "min", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 7, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "A", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.cache.queueWriteoutTime), 'Queue Writeout Time')" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refCount": 0, - "refId": "B", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.cache.queueBuildTimeMs), 'Queue Build Time')" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "hide": false, - "refCount": 0, - "refId": "C", - "target": "alias(sumSeries(stats.gocarbon.agents.$agent.cache.queueBuildCount), 'Queue Rebuilds')" - } - ], - "title": "Queue Stats", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 16 - }, - "id": 11, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refId": "A", - "target": "alias(sumSeries(stats.gocarbon.agents.*.persister.committedPoints), 'Commited', false)" - } - ], - "title": "Comitted", - "type": "stat" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 16 - }, - "id": 9, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refId": "A", - "target": "alias(sumSeries(stats.gocarbon.agents.*.persister.updateOperations), 'Updates', false)" - } - ], - "title": "Updates", - "type": "stat" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 16 - }, - "id": 13, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refId": "A", - "target": "alias(sumSeries(stats.gocarbon.agents.*.persister.created), 'Commited', false)" - } - ], - "title": "Created files", - "type": "stat" - }, - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 16 - }, - "id": 15, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "refId": "A", - "target": "sumSeries(stats.gocarbon.agents.*.persister.throttledCreates)" - } - ], - "title": "Throttled new files", - "type": "stat" - } - ], - "refresh": "30s", - "schemaVersion": 36, - "style": "dark", - "tags": [ - "telemetry" - ], - "templating": { - "list": [ - { - "allValue": "", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "graphite", - "uid": "P1D261A8554D2DA69" - }, - "definition": "stats.gocarbon.agents.*", - "hide": 0, - "includeAll": true, - "label": "Carbon Agent", - "multi": true, - "name": "agent", - "options": [], - "query": "stats.gocarbon.agents.*", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "collapse": false, - "enable": true, - "notice": false, - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "type": "timepicker" - }, - "timezone": "", - "title": "Go-Carbon", - "uid": "hqIB4VvZk", - "version": 5, - "weekStart": "" -} \ No newline at end of file diff --git a/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml b/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml index d29c439..8931eed 100644 --- a/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml +++ b/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml @@ -18,7 +18,7 @@ datasources: # org id. will default to orgId 1 if not specified orgId: 1 # url - url: http://carbon-api:8081 + url: http://graphite:8080/ # database password, if used password: # database user, if used diff --git a/grafanaConfig.example/graphite/aggregation-rules.conf b/grafanaConfig.example/graphite/aggregation-rules.conf new file mode 100644 index 0000000..c952012 --- /dev/null +++ b/grafanaConfig.example/graphite/aggregation-rules.conf @@ -0,0 +1,35 @@ +# The form of each line in this file should be as follows: +# +# output_template (frequency) = method input_pattern +# +# This will capture any received metrics that match 'input_pattern' +# for calculating an aggregate metric. The calculation will occur +# every 'frequency' seconds and the 'method' can specify 'sum' or +# 'avg'. The name of the aggregate metric will be derived from +# 'output_template' filling in any captured fields from 'input_pattern'. +# +# For example, if you're metric naming scheme is: +# +# .applications... +# +# You could configure some aggregations like so: +# +# .applications..all.requests (60) = sum .applications..*.requests +# .applications..all.latency (60) = avg .applications..*.latency +# +# As an example, if the following metrics are received: +# +# prod.applications.apache.www01.requests +# prod.applications.apache.www01.requests +# +# They would all go into the same aggregation buffer and after 60 seconds the +# aggregate metric 'prod.applications.apache.all.requests' would be calculated +# by summing their values. +# +# Template components such as will match everything up to the next dot. +# To match metric multiple components including the dots, use <> in the +# input template: +# +# .applications..all. (60) = sum .applications..*.<> +# +# Note that any time this file is modified, it will be re-read automatically. diff --git a/grafanaConfig.example/graphite/blacklist.conf b/grafanaConfig.example/graphite/blacklist.conf new file mode 100644 index 0000000..cb95a1f --- /dev/null +++ b/grafanaConfig.example/graphite/blacklist.conf @@ -0,0 +1,5 @@ +# This file takes a single regular expression per line +# If USE_WHITELIST is set to True in carbon.conf, any metrics received which +# match one of these expressions will be dropped +# This file is reloaded automatically when changes are made +^some\.noisy\.metric\.prefix\..* diff --git a/grafanaConfig.example/graphite/brubeck.json b/grafanaConfig.example/graphite/brubeck.json new file mode 100644 index 0000000..9b6b556 --- /dev/null +++ b/grafanaConfig.example/graphite/brubeck.json @@ -0,0 +1,24 @@ +{ + "sharding" : false, + "server_name" : "brubeck", + "dumpfile" : "/tmp/brubeck.dump", + "capacity" : 15, + "backends" : [ + { + "type" : "carbon", + "address" : "localhost", + "port" : 2003, + "frequency" : 10 + } + ], + "samplers" : [ + { + "type" : "statsd", + "address" : "0.0.0.0", + "port" : 8125, + "workers" : 4, + "multisock" : true, + "multimsg" : 8 + } + ] +} diff --git a/grafanaConfig.example/graphite/carbon.amqp.conf b/grafanaConfig.example/graphite/carbon.amqp.conf new file mode 100644 index 0000000..fc36328 --- /dev/null +++ b/grafanaConfig.example/graphite/carbon.amqp.conf @@ -0,0 +1,75 @@ +# This is a configuration file with AMQP enabled + +[cache] +LOCAL_DATA_DIR = + +# Specify the user to drop privileges to +# If this is blank carbon runs as the user that invokes it +# This user must have write access to the local data directory +USER = + +# Limit the size of the cache to avoid swapping or becoming CPU bound. +# Sorts and serving cache queries gets more expensive as the cache grows. +# Use the value "inf" (infinity) for an unlimited cache size. +MAX_CACHE_SIZE = inf + +# Limits the number of whisper update_many() calls per second, which effectively +# means the number of write requests sent to the disk. This is intended to +# prevent over-utilizing the disk and thus starving the rest of the system. +# When the rate of required updates exceeds this, then carbon's caching will +# take effect and increase the overall throughput accordingly. +MAX_UPDATES_PER_SECOND = 1000 + +# Softly limits the number of whisper files that get created each minute. +# Setting this value low (like at 50) is a good way to ensure your graphite +# system will not be adversely impacted when a bunch of new metrics are +# sent to it. The trade off is that it will take much longer for those metrics' +# database files to all get created and thus longer until the data becomes usable. +# Setting this value high (like "inf" for infinity) will cause graphite to create +# the files quickly but at the risk of slowing I/O down considerably for a while. +MAX_CREATES_PER_MINUTE = inf + +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +UDP_RECEIVER_INTERFACE = 0.0.0.0 +UDP_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_QUERY_INTERFACE = 0.0.0.0 +CACHE_QUERY_PORT = 7002 + +# Enable AMQP if you want to receve metrics using you amqp broker +ENABLE_AMQP = True + +# Verbose means a line will be logged for every metric received +# useful for testing +AMQP_VERBOSE = True + +# your credentials for the amqp server +# AMQP_USER = guest +# AMQP_PASSWORD = guest + +# the network settings for the amqp server +# AMQP_HOST = localhost +# AMQP_PORT = 5672 + +# if you want to include the metric name as part of the message body +# instead of as the routing key, set this to True +# AMQP_METRIC_NAME_IN_BODY = False + +# NOTE: you cannot run both a cache and a relay on the same server +# with the default configuration, you have to specify a distinict +# interfaces and ports for the listeners. + +[relay] +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_SERVERS = server1, server2, server3 +MAX_QUEUE_SIZE = 10000 diff --git a/grafanaConfig.example/graphite/carbon.conf b/grafanaConfig.example/graphite/carbon.conf new file mode 100644 index 0000000..8cbe353 --- /dev/null +++ b/grafanaConfig.example/graphite/carbon.conf @@ -0,0 +1,646 @@ +[cache] +# Configure carbon directories. +# +# OS environment variables can be used to tell carbon where graphite is +# installed, where to read configuration from and where to write data. +# +# GRAPHITE_ROOT - Root directory of the graphite installation. +# Defaults to ../ +# GRAPHITE_CONF_DIR - Configuration directory (where this file lives). +# Defaults to $GRAPHITE_ROOT/conf/ +# GRAPHITE_STORAGE_DIR - Storage directory for whisper/rrd/log/pid files. +# Defaults to $GRAPHITE_ROOT/storage/ +# +# To change other directory paths, add settings to this file. The following +# configuration variables are available with these default values: +# +# STORAGE_DIR = $GRAPHITE_STORAGE_DIR +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# WHITELISTS_DIR = %(STORAGE_DIR)s/lists/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ +# +# For FHS style directory structures, use: +# +# STORAGE_DIR = /var/lib/carbon/ +# CONF_DIR = /etc/carbon/ +# LOG_DIR = /var/log/carbon/ +# PID_DIR = /var/run/ +# +#LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ + +# Specify the database library used to store metric data on disk. Each database +# may have configurable options to change the behaviour of how it writes to +# persistent storage. +# +# whisper - Fixed-size database, similar in design and purpose to RRD. This is +# the default storage backend for carbon and the most rigorously tested. +# +# ceres - Experimental alternative database that supports storing data in sparse +# files of arbitrary fixed-size resolutions. +DATABASE = whisper + +# Enable daily log rotation. If disabled, a new file will be opened whenever the log file path no +# longer exists (i.e. it is removed or renamed) +ENABLE_LOGROTATION = True + +# Specify the user to drop privileges to +# If this is blank carbon-cache runs as the user that invokes it +# This user must have write access to the local data directory +USER = + +# Limit the size of the cache to avoid swapping or becoming CPU bound. +# Sorts and serving cache queries gets more expensive as the cache grows. +# Use the value "inf" (infinity) for an unlimited cache size. +# value should be an integer number of metric datapoints. +MAX_CACHE_SIZE = inf + +# Limits the number of whisper update_many() calls per second, which effectively +# means the number of write requests sent to the disk. This is intended to +# prevent over-utilizing the disk and thus starving the rest of the system. +# When the rate of required updates exceeds this, then carbon's caching will +# take effect and increase the overall throughput accordingly. +MAX_UPDATES_PER_SECOND = 500 + +# If defined, this changes the MAX_UPDATES_PER_SECOND in Carbon when a +# stop/shutdown is initiated. This helps when MAX_UPDATES_PER_SECOND is +# relatively low and carbon has cached a lot of updates; it enables the carbon +# daemon to shutdown more quickly. +# MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 + +# Softly limits the number of whisper files that get created each minute. +# Setting this value low (e.g. 50) is a good way to ensure that your carbon +# system will not be adversely impacted when a bunch of new metrics are +# sent to it. The trade off is that any metrics received in excess of this +# value will be silently dropped, and the whisper file will not be created +# until such point as a subsequent metric is received and fits within the +# defined rate limit. Setting this value high (like "inf" for infinity) will +# cause carbon to create the files quickly but at the risk of increased I/O. +MAX_CREATES_PER_MINUTE = 50 + +# Set the minimum timestamp resolution supported by this instance. This allows +# internal optimisations by overwriting points with equal truncated timestamps +# in order to limit the number of updates to the database. It defaults to one +# second. +MIN_TIMESTAMP_RESOLUTION = 1 + +# Set the minimum lag in seconds for a point to be written to the database +# in order to optimize batching. This means that each point will wait at least +# the duration of this lag before being written. Setting this to 0 disable the feature. +# This currently only works when using the timesorted write strategy. +# MIN_TIMESTAMP_LAG = 0 + +# Set the interface and port for the line (plain text) listener. Setting the +# interface to 0.0.0.0 listens on all interfaces. Port can be set to 0 to +# disable this listener if it is not required. +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +# Set this to True to enable the UDP listener. By default this is off +# because it is very common to run multiple carbon daemons and managing +# another (rarely used) port for every carbon instance is not fun. +ENABLE_UDP_LISTENER = False +UDP_RECEIVER_INTERFACE = 0.0.0.0 +UDP_RECEIVER_PORT = 2003 + +# Set the interface and port for the pickle listener. Setting the interface to +# 0.0.0.0 listens on all interfaces. Port can be set to 0 to disable this +# listener if it is not required. +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +# Set the interface and port for the protobuf listener. Setting the interface to +# 0.0.0.0 listens on all interfaces. Port can be set to 0 to disable this +# listener if it is not required. +# PROTOBUF_RECEIVER_INTERFACE = 0.0.0.0 +# PROTOBUF_RECEIVER_PORT = 2005 + +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf + +# Per security concerns outlined in Bug #817247 the pickle receiver +# will use a more secure and slightly less efficient unpickler. +# Set this to True to revert to the old-fashioned insecure unpickler. +USE_INSECURE_UNPICKLER = False + +CACHE_QUERY_INTERFACE = 0.0.0.0 +CACHE_QUERY_PORT = 7002 + +# Set this to False to drop datapoints received after the cache +# reaches MAX_CACHE_SIZE. If this is True (the default) then sockets +# over which metrics are received will temporarily stop accepting +# data until the cache size falls below 95% MAX_CACHE_SIZE. +USE_FLOW_CONTROL = True + +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None + +# By default, carbon-cache will log every whisper update and cache hit. +# This can be excessive and degrade performance if logging on the same +# volume as the whisper data is stored. +LOG_UPDATES = False +LOG_CREATES = False +LOG_CACHE_HITS = False +LOG_CACHE_QUEUE_SORTS = False + +# The thread that writes metrics to disk can use one of the following strategies +# determining the order in which metrics are removed from cache and flushed to +# disk. The default option preserves the same behavior as has been historically +# available in version 0.9.10. +# +# sorted - All metrics in the cache will be counted and an ordered list of +# them will be sorted according to the number of datapoints in the cache at the +# moment of the list's creation. Metrics will then be flushed from the cache to +# disk in that order. +# +# timesorted - All metrics in the list will be looked at and sorted according +# to the timestamp of there datapoints. The metric that were the least recently +# written will be written first. This is an hybrid strategy between max and +# sorted which is particularly adapted to sets of metrics with non-uniform +# resolutions. +# +# max - The writer thread will always pop and flush the metric from cache +# that has the most datapoints. This will give a strong flush preference to +# frequently updated metrics and will also reduce random file-io. Infrequently +# updated metrics may only ever be persisted to disk at daemon shutdown if +# there are a large number of metrics which receive very frequent updates OR if +# disk i/o is very slow. +# +# naive - Metrics will be flushed from the cache to disk in an unordered +# fashion. This strategy may be desirable in situations where the storage for +# whisper files is solid state, CPU resources are very limited or deference to +# the OS's i/o scheduler is expected to compensate for the random write +# pattern. +# +CACHE_WRITE_STRATEGY = sorted + +# On some systems it is desirable for whisper to write synchronously. +# Set this option to True if you'd like to try this. Basically it will +# shift the onus of buffering writes from the kernel into carbon's cache. +WHISPER_AUTOFLUSH = False + +# By default new Whisper files are created pre-allocated with the data region +# filled with zeros to prevent fragmentation and speed up contiguous reads and +# writes (which are common). Enabling this option will cause Whisper to create +# the file sparsely instead. Enabling this option may allow a large increase of +# MAX_CREATES_PER_MINUTE but may have longer term performance implications +# depending on the underlying storage configuration. +# WHISPER_SPARSE_CREATE = False + +# Only beneficial on linux filesystems that support the fallocate system call. +# It maintains the benefits of contiguous reads/writes, but with a potentially +# much faster creation speed, by allowing the kernel to handle the block +# allocation and zero-ing. Enabling this option may allow a large increase of +# MAX_CREATES_PER_MINUTE. If enabled on an OS or filesystem that is unsupported +# this option will gracefully fallback to standard POSIX file access methods. +# If enabled, disables WHISPER_SPARSE_CREATE regardless of the value. +WHISPER_FALLOCATE_CREATE = False + +# Enabling this option will cause Whisper to lock each Whisper file it writes +# to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when +# multiple carbon-cache daemons are writing to the same files. +# WHISPER_LOCK_WRITES = False + +# On systems which has a large number of metrics, an amount of Whisper write(2)'s +# pageback sometimes cause disk thrashing due to memory shortage, so that abnormal +# disk reads occur. Enabling this option makes it possible to decrease useless +# page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. +# WHISPER_FADVISE_RANDOM = False + +# By default all nodes stored in Ceres are cached in memory to improve the +# throughput of reads and writes to underlying slices. Turning this off will +# greatly reduce memory consumption for databases with millions of metrics, at +# the cost of a steep increase in disk i/o, approximately an extra two os.stat +# calls for every read and write. Reasons to do this are if the underlying +# storage can handle stat() with practically zero cost (SSD, NVMe, zRAM). +# Valid values are: +# all - all nodes are cached +# none - node caching is disabled +# CERES_NODE_CACHING_BEHAVIOR = all + +# Ceres nodes can have many slices and caching the right ones can improve +# performance dramatically. Note that there are many trade-offs to tinkering +# with this, and unless you are a ceres developer you *really* should not +# mess with this. Valid values are: +# latest - only the most recent slice is cached +# all - all slices are cached +# none - slice caching is disabled +# CERES_SLICE_CACHING_BEHAVIOR = latest + +# If a Ceres node accumulates too many slices, performance can suffer. +# This can be caused by intermittently reported data. To mitigate +# slice fragmentation there is a tolerance for how much space can be +# wasted within a slice file to avoid creating a new one. That tolerance +# level is determined by MAX_SLICE_GAP, which is the number of consecutive +# null datapoints allowed in a slice file. +# If you set this very low, you will waste less of the *tiny* bit disk space +# that this feature wastes, and you will be prone to performance problems +# caused by slice fragmentation, which can be pretty severe. +# If you set this really high, you will waste a bit more disk space (each +# null datapoint wastes 8 bytes, but keep in mind your filesystem's block +# size). If you suffer slice fragmentation issues, you should increase this or +# run the ceres-maintenance defrag plugin more often. However you should not +# set it to be huge because then if a large but allowed gap occurs it has to +# get filled in, which means instead of a simple 8-byte write to a new file we +# could end up doing an (8 * MAX_SLICE_GAP)-byte write to the latest slice. +# CERES_MAX_SLICE_GAP = 80 + +# Enabling this option will cause Ceres to lock each Ceres file it writes to +# to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when +# multiple carbon-cache daemons are writing to the same files. +# CERES_LOCK_WRITES = False + +# Set this to True to enable whitelisting and blacklisting of metrics in +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through +# USE_WHITELIST = False + +# By default, carbon itself will log statistics (such as a count, +# metricsReceived) with the top level prefix of 'carbon' at an interval of 60 +# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation +# CARBON_METRIC_PREFIX = carbon +CARBON_METRIC_INTERVAL = 10 + +# Enable AMQP if you want to receve metrics using an amqp broker +ENABLE_AMQP = False + +# Verbose means a line will be logged for every metric received +# useful for testing +AMQP_VERBOSE = False + +AMQP_HOST = localhost +AMQP_PORT = 5672 +AMQP_VHOST = / +AMQP_USER = guest +AMQP_PASSWORD = guest +AMQP_EXCHANGE = graphite +AMQP_METRIC_NAME_IN_BODY = False + +# The manhole interface allows you to SSH into the carbon daemon +# and get a python interpreter. BE CAREFUL WITH THIS! If you do +# something like time.sleep() in the interpreter, the whole process +# will sleep! This is *extremely* helpful in debugging, assuming +# you are familiar with the code. If you are not, please don't +# mess with this, you are asking for trouble :) +# +# ENABLE_MANHOLE = False +# MANHOLE_INTERFACE = 127.0.0.1 +# MANHOLE_PORT = 7222 +# MANHOLE_USER = admin +# MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE= + +# Patterns for all of the metrics this machine will store. Read more at +# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings +# +# Example: store all sales, linux servers, and utilization metrics +# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization +# +# Example: store everything +# BIND_PATTERNS = # + +# URL of graphite-web instance, this is used to add incoming series to the tag database +GRAPHITE_URL = http://127.0.0.1:8080 + +# Tag support, when enabled carbon will make HTTP calls to graphite-web to update the tag index +ENABLE_TAGS = True + +# Tag update interval, this specifies how frequently updates to existing series will trigger +# an update to the tag index, the default setting is once every 100 updates +# TAG_UPDATE_INTERVAL = 100 + +# Tag hash filenames, this specifies whether tagged metric filenames should use the hash of the metric name +# or a human-readable name, using hashed names avoids issues with path length when using a large number of tags +# TAG_HASH_FILENAMES = True + +# Tag batch size, this specifies the maximum number of series to be sent to graphite-web in a single batch +# TAG_BATCH_SIZE = 100 + +# Tag queue size, this specifies the maximum number of series to be queued for sending to graphite-web +# There are separate queues for new series and for updates to existing series +# TAG_QUEUE_SIZE = 10000 + +# Set to enable Sentry.io exception monitoring. +# RAVEN_DSN='YOUR_DSN_HERE'. + +# To configure special settings for the carbon-cache instance 'b', uncomment this: +#[cache:b] +#LINE_RECEIVER_PORT = 2103 +#PICKLE_RECEIVER_PORT = 2104 +#CACHE_QUERY_PORT = 7102 +# and any other settings you want to customize, defaults are inherited +# from the [cache] section. +# You can then specify the --instance=b option to manage this instance +# +# In order to turn on logging of successful connections for the line +# receiver, set this to True +LOG_LISTENER_CONN_SUCCESS = False + +[relay] +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2013 +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2014 + +# Carbon-relay has several options for metric routing controlled by RELAY_METHOD +# +# Use relay-rules.conf to route metrics to destinations based on pattern rules +#RELAY_METHOD = rules +# +# Use consistent-hashing for even distribution of metrics between destinations +#RELAY_METHOD = consistent-hashing +# +# Use consistent-hashing but take into account an aggregation-rules.conf shared +# by downstream carbon-aggregator daemons. This will ensure that all metrics +# that map to a given aggregation rule are sent to the same carbon-aggregator +# instance. +# Enable this for carbon-relays that send to a group of carbon-aggregators +#RELAY_METHOD = aggregated-consistent-hashing +# +# You can also use fast-hashing and fast-aggregated-hashing which are in O(1) +# and will always redirect the metrics to the same destination but do not try +# to minimize rebalancing when the list of destinations is changing. +RELAY_METHOD = rules + +# If you use consistent-hashing you can add redundancy by replicating every +# datapoint to more than one machine. +REPLICATION_FACTOR = 1 + +# For REPLICATION_FACTOR >=2, set DIVERSE_REPLICAS to True to guarantee replicas +# across distributed hosts. With this setting disabled, it's possible that replicas +# may be sent to different caches on the same host. This has been the default +# behavior since introduction of 'consistent-hashing' relay method. +# Note that enabling this on an existing pre-0.9.14 cluster will require rebalancing +# your metrics across the cluster nodes using a tool like Carbonate. +#DIVERSE_REPLICAS = True + +# This is a list of carbon daemons we will send any relayed or +# generated metrics to. The default provided would send to a single +# carbon-cache instance on the default port. However if you +# use multiple carbon-cache instances then it would look like this: +# +# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b +# +# The general form is IP:PORT:INSTANCE where the :INSTANCE part is +# optional and refers to the "None" instance if omitted. +# +# Note that if the destinations are all carbon-caches then this should +# exactly match the webapp's CARBONLINK_HOSTS setting in terms of +# instances listed (order matters!). +# +# If using RELAY_METHOD = rules, all destinations used in relay-rules.conf +# must be defined in this list +DESTINATIONS = 127.0.0.1:2004 + +# This define the protocol to use to contact the destination. It can be +# set to one of "line", "pickle", "udp" and "protobuf". This list can be +# extended with CarbonClientFactory plugins and defaults to "pickle". +# DESTINATION_PROTOCOL = pickle + +# This defines the wire transport, either none or ssl. +# If SSL is used any TCP connection will be upgraded to TLS1. The system's +# trust authority will be used unless DESTINATION_SSL_CA is specified in +# which case an alternative certificate authority chain will be used for +# verifying the remote certificate. +# To use SSL you'll need the cryptography, service_identity, and twisted >= 14 +# DESTINATION_TRANSPORT = none +# DESTINATION_SSL_CA=/path/to/private-ca.crt + +# This allows to have multiple connections per destinations, this will +# pool all the replicas of a single host in the same queue and distribute +# points accross these replicas instead of replicating them. +# The following example will balance the load between :0 and :1. +## DESTINATIONS = foo:2001:0, foo:2001:1 +## RELAY_METHOD = rules +# Note: this is currently incompatible with USE_RATIO_RESET which gets +# disabled if this option is enabled. +# DESTINATIONS_POOL_REPLICAS = False + +# When using consistent hashing it sometime makes sense to make +# the ring dynamic when you don't want to loose points when a +# single destination is down. Replication is an answer to that +# but it can be quite expensive. +# DYNAMIC_ROUTER = False + +# Controls the number of connection attempts before marking a +# destination as down. We usually do one connection attempt per +# second. +# DYNAMIC_ROUTER_MAX_RETRIES = 5 + +# This is the maximum number of datapoints that can be queued up +# for a single destination. Once this limit is hit, we will +# stop accepting new data if USE_FLOW_CONTROL is True, otherwise +# we will drop any subsequently received datapoints. +MAX_QUEUE_SIZE = 10000 + +# This defines the maximum "message size" between carbon daemons. If +# your queue is large, setting this to a lower number will cause the +# relay to forward smaller discrete chunks of stats, which may prevent +# overloading on the receiving side after a disconnect. +MAX_DATAPOINTS_PER_MESSAGE = 500 + +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf + +# Specify the user to drop privileges to +# If this is blank carbon-relay runs as the user that invokes it +# USER = + +# This is the percentage that the queue must be empty before it will accept +# more messages. For a larger site, if the queue is very large it makes sense +# to tune this to allow for incoming stats. So if you have an average +# flow of 100k stats/minute, and a MAX_QUEUE_SIZE of 3,000,000, it makes sense +# to allow stats to start flowing when you've cleared the queue to 95% since +# you should have space to accommodate the next minute's worth of stats +# even before the relay incrementally clears more of the queue +QUEUE_LOW_WATERMARK_PCT = 0.8 + +# To allow for batch efficiency from the pickle protocol and to benefit from +# other batching advantages, all writes are deferred by putting them into a queue, +# and then the queue is flushed and sent a small fraction of a second later. +TIME_TO_DEFER_SENDING = 0.0001 + +# Set this to False to drop datapoints when any send queue (sending datapoints +# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the +# default) then sockets over which metrics are received will temporarily stop accepting +# data until the send queues fall below QUEUE_LOW_WATERMARK_PCT * MAX_QUEUE_SIZE. +USE_FLOW_CONTROL = True + +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None + +# Set this to True to enable whitelisting and blacklisting of metrics in +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through +# USE_WHITELIST = False + +# By default, carbon itself will log statistics (such as a count, +# metricsReceived) with the top level prefix of 'carbon' at an interval of 60 +# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation +# CARBON_METRIC_PREFIX = carbon +CARBON_METRIC_INTERVAL = 10 +# +# In order to turn on logging of successful connections for the line +# receiver, set this to True +LOG_LISTENER_CONN_SUCCESS = False + +# If you're connecting from the relay to a destination that's over the +# internet or similarly iffy connection, a backlog can develop because +# of internet weather conditions, e.g. acks getting lost or similar issues. +# To deal with that, you can enable USE_RATIO_RESET which will let you +# re-set the connection to an individual destination. Defaults to being off. +USE_RATIO_RESET=False + +# When there is a small number of stats flowing, it's not desirable to +# perform any actions based on percentages - it's just too "twitchy". +MIN_RESET_STAT_FLOW=1000 + +# When the ratio of stats being sent in a reporting interval is far +# enough from 1.0, we will disconnect the socket and reconnecto to +# clear out queued stats. The default ratio of 0.9 indicates that 10% +# of stats aren't being delivered within one CARBON_METRIC_INTERVAL +# (default of 60 seconds), which can lead to a queue backup. Under +# some circumstances re-setting the connection can fix this, so +# set this according to your tolerance, and look in the logs for +# "resetConnectionForQualityReasons" to observe whether this is kicking +# in when your sent queue is building up. +MIN_RESET_RATIO=0.9 + +# The minimum time between resets. When a connection is re-set, we +# need to wait before another reset is performed. +# (2*CARBON_METRIC_INTERVAL) + 1 second is the minimum time needed +# before stats for the new connection will be available. Setting this +# below (2*CARBON_METRIC_INTERVAL) + 1 second will result in a lot of +# reset connections for no good reason. +MIN_RESET_INTERVAL=121 + +# Enable TCP Keep Alive (http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html). +# Default settings will send a probe every 30s. Default is False. +# TCP_KEEPALIVE=True +# The interval between the last data packet sent (simple ACKs are not +# considered data) and the first keepalive probe; after the connection is marked +# to need keepalive, this counter is not used any further. +# TCP_KEEPIDLE=10 +# The interval between subsequential keepalive probes, regardless of what +# the connection has exchanged in the meantime. +# TCP_KEEPINTVL=30 +# The number of unacknowledged probes to send before considering the connection +# dead and notifying the application layer. +# TCP_KEEPCNT=2 + + +[aggregator] +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2023 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2024 + +# If set true, metric received will be forwarded to DESTINATIONS in addition to +# the output of the aggregation rules. If set false the carbon-aggregator will +# only ever send the output of aggregation. +FORWARD_ALL = True + +# Filenames of the configuration files to use for this instance of aggregator. +# Filenames are relative to CONF_DIR. +# +# AGGREGATION_RULES = aggregation-rules.conf +# REWRITE_RULES = rewrite-rules.conf + +# This is a list of carbon daemons we will send any relayed or +# generated metrics to. The default provided would send to a single +# carbon-cache instance on the default port. However if you +# use multiple carbon-cache instances then it would look like this: +# +# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b +# +# The format is comma-delimited IP:PORT:INSTANCE where the :INSTANCE part is +# optional and refers to the "None" instance if omitted. +# +# Note that if the destinations are all carbon-caches then this should +# exactly match the webapp's CARBONLINK_HOSTS setting in terms of +# instances listed (order matters!). +DESTINATIONS = 127.0.0.1:2004 + +# If you want to add redundancy to your data by replicating every +# datapoint to more than one machine, increase this. +REPLICATION_FACTOR = 1 + +# This is the maximum number of datapoints that can be queued up +# for a single destination. Once this limit is hit, we will +# stop accepting new data if USE_FLOW_CONTROL is True, otherwise +# we will drop any subsequently received datapoints. +MAX_QUEUE_SIZE = 10000 + +# Set this to False to drop datapoints when any send queue (sending datapoints +# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the +# default) then sockets over which metrics are received will temporarily stop accepting +# data until the send queues fall below 80% MAX_QUEUE_SIZE. +USE_FLOW_CONTROL = True + +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None + +# This defines the maximum "message size" between carbon daemons. +# You shouldn't need to tune this unless you really know what you're doing. +MAX_DATAPOINTS_PER_MESSAGE = 500 + +# This defines how many datapoints the aggregator remembers for +# each metric. Aggregation only happens for datapoints that fall in +# the past MAX_AGGREGATION_INTERVALS * intervalSize seconds. +MAX_AGGREGATION_INTERVALS = 5 + +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf + +# By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back +# aggregated data points once every rule.frequency seconds, on a per-rule basis. +# Set this (WRITE_BACK_FREQUENCY = N) to write back all aggregated data points +# every N seconds, independent of rule frequency. This is useful, for example, +# to be able to query partially aggregated metrics from carbon-cache without +# having to first wait rule.frequency seconds. +# WRITE_BACK_FREQUENCY = 0 + +# Set this to True to enable whitelisting and blacklisting of metrics in +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through +# USE_WHITELIST = False + +# By default, carbon itself will log statistics (such as a count, +# metricsReceived) with the top level prefix of 'carbon' at an interval of 60 +# seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation +# CARBON_METRIC_PREFIX = carbon +CARBON_METRIC_INTERVAL = 10 + +# In order to turn off logging of successful connections for the line +# receiver, set this to True +LOG_LISTENER_CONN_SUCCESS = False + +# In order to turn off logging of metrics with no corresponding +# aggregation rules receiver, set this to False +# LOG_AGGREGATOR_MISSES = False + +# Specify the user to drop privileges to +# If this is blank carbon-aggregator runs as the user that invokes it +# USER = + +# Part of the code, and particularly aggregator rules, need +# to cache metric names. To avoid leaking too much memory you +# can tweak the size of this cache. The default allow for 1M +# different metrics per rule (~200MiB). +# CACHE_METRIC_NAMES_MAX=1000000 + +# You can optionally set a ttl to this cache. +# CACHE_METRIC_NAMES_TTL=600 \ No newline at end of file diff --git a/grafanaConfig.example/graphite/dashboard.conf b/grafanaConfig.example/graphite/dashboard.conf new file mode 100644 index 0000000..2e1b0bc --- /dev/null +++ b/grafanaConfig.example/graphite/dashboard.conf @@ -0,0 +1,57 @@ +# This configuration file controls the behavior of the Dashboard UI, available +# at http://my-graphite-server/dashboard/. +# +# This file must contain a [ui] section that defines values for all of the +# following settings. +[ui] +default_graph_width = 400 +default_graph_height = 250 +automatic_variants = true +refresh_interval = 60 +autocomplete_delay = 375 +merge_hover_delay = 750 + +# You can set this 'default', 'white', or a custom theme name. +# To create a custom theme, copy the dashboard-default.css file +# to dashboard-myThemeName.css in the content/css directory and +# modify it to your liking. +theme = default + +[keyboard-shortcuts] +toggle_toolbar = ctrl-z +toggle_metrics_panel = ctrl-space +erase_all_graphs = alt-x +save_dashboard = alt-s +completer_add_metrics = alt-enter +completer_del_metrics = alt-backspace +give_completer_focus = shift-space + +# These settings apply to the UI as a whole, all other sections in this file +# pertain only to specific metric types. +# +# The dashboard presents only metrics that fall into specified naming schemes +# defined in this file. This creates a simpler, more targetted view of the +# data. The general form for defining a naming scheme is as follows: +# +#[Metric Type] +#scheme = basis.path... +#field1.label = Foo +#field2.label = Bar +# +# +# Where each will be displayed as a dropdown box +# in the UI and the remaining portion of the namespace +# shown in the Metric Selector panel. The .label options set the labels +# displayed for each dropdown. +# +# For example: +# +#[Sales] +#scheme = sales... +#channel.label = Channel +#type.label = Product Type +#brand.label = Brand +# +# This defines a 'Sales' metric type that uses 3 dropdowns in the Context Selector +# (the upper-left panel) while any deeper metrics (per-product counts or revenue, etc) +# will be available in the Metric Selector (upper-right panel). diff --git a/grafanaConfig.example/go-carbon/go-carbon.conf b/grafanaConfig.example/graphite/go-carbon.conf similarity index 51% rename from grafanaConfig.example/go-carbon/go-carbon.conf rename to grafanaConfig.example/graphite/go-carbon.conf index dcb0b6e..d9b65b2 100644 --- a/grafanaConfig.example/go-carbon/go-carbon.conf +++ b/grafanaConfig.example/graphite/go-carbon.conf @@ -1,9 +1,9 @@ [common] # Run as user. Works only in daemon mode -user = "carbon" +user = "root" # Prefix for store all internal go-carbon graphs. Supported macroses: {host} -graph-prefix = "stats.gocarbon.agents.{host}" -# Endpoint to store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port" +graph-prefix = "carbon.agents.{host}" +# Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port" metric-endpoint = "local" # Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL metric-interval = "1m0s" @@ -11,20 +11,17 @@ metric-interval = "1m0s" max-cpu = 4 [whisper] -data-dir = "/var/lib/graphite/whisper" +data-dir = "/opt/graphite/storage/whisper" # http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-schemas-conf. Required -schemas-file = "/etc/go-carbon/storage-schemas.conf" +schemas-file = "/opt/graphite/conf/storage-schemas.conf" # http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-aggregation-conf. Optional -aggregation-file = "/etc/go-carbon/storage-aggregation.conf" -# It's currently go-carbon only feature, not a standard graphite feature. Optional -# More details in doc/quotas.md -# quotas-file = "/conf/storage-quotas.conf" +aggregation-file = "/opt/graphite/conf/storage-aggregation.conf" # Worker threads count. Metrics sharded by "crc32(metricName) % workers" workers = 8 # Limits the number of whisper update_many() calls per second. 0 - no limit -max-updates-per-second = 0 +max-updates-per-second = 500 # Softly limits the number of whisper files that get created each second. 0 - no limit -max-creates-per-second = 0 +max-creates-per-second = 500 # Make max-creates-per-second a hard limit. Extra new metrics are dropped. A hard throttle of 0 drops all new metrics. hard-max-creates-per-second = false # Sparse file creation @@ -33,37 +30,16 @@ sparse-create = false flock = true enabled = true # Use hashed filenames for tagged metrics instead of human readable -# https://github.com/go-graphite/go-carbon/pull/225 +# https://github.com/lomik/go-carbon/pull/225 hash-filenames = true -# specify to enable/disable compressed format (EXPERIMENTAL) -# See details and limitations in https://github.com/go-graphite/go-whisper#compressed-format -# IMPORTANT: Only one process/thread could write to compressed whisper files at a time, especially when you are -# rebalancing graphite clusters (with buckytools, for example), flock needs to be enabled both in go-carbon and your tooling. +# specify to enable/disable compressed format. IMPORTANT: Only one process/thread could write to compressed whisper files at a time, especially when you are rebalancing graphite clusters (with buckytools, for example), flock needs to be enabled both in go-carbon and your tooling. compressed = false # automatically delete empty whisper file caused by edge cases like server reboot -remove-empty-file = false - -# Enable online whisper file config migration. -# -# online-migration-rate means metrics per second to migrate. -# -# To partially enable default migration for only some matched rules in -# storage-schemas.conf or storage-aggregation.conf, we can set -# online-migration-global-scope = "-" and enable the migration in the config -# files (more examples in deploy/storage-aggregation.conf and deploy/storage-schemas.conf). -# -# online-migration-global-scope can also be set any combination of the 3 rules -# (xff,aggregationMethod,schema) as a csv string -# like: "xff", "xff,aggregationMethod", "xff,schema", -# or "xff,aggregationMethod,schema". -# -online-migration = true -online-migration-rate = 5 -online-migration-global-scope = "-" +remove-empty-file = true [cache] # Limit of in-memory stored points (not metrics) -max-size = 1000000 +max-size = 100000000 # Capacity of queue between receivers and cache # Strategy to persist metrics. Values: "max","sorted","noop" # "max" - write metrics with most unwritten datapoints first @@ -73,19 +49,19 @@ max-size = 1000000 write-strategy = "max" [udp] -listen = "go-carbon:2003" +listen = ":2003" enabled = true # Optional internal queue between receiver and cache buffer-size = 0 [tcp] -listen = "go-carbon:2003" +listen = ":2003" enabled = true # Optional internal queue between receiver and cache buffer-size = 0 [pickle] -listen = "go-carbon:2004" +listen = ":2004" # Limit message size for prevent memory overflow max-message-size = 67108864 enabled = true @@ -119,7 +95,7 @@ buffer-size = 0 # [receiver.protobuf] # protocol = "protobuf" # # Same framing protocol as pickle, but message encoded in protobuf format -# # See https://github.com/go-graphite/go-carbon/blob/master/helper/carbonpb/carbon.proto +# # See https://github.com/lomik/go-carbon/blob/master/helper/carbonpb/carbon.proto # listen = ":2005" # # Limit message size for prevent memory overflow # max-message-size = 67108864 @@ -156,7 +132,7 @@ buffer-size = 0 # fetch-interval = "200ms" # # # Path to saved kafka state. Used for restarts -# state-file = "/graphite/kafka.state" +# state-file = "/var/lib/graphite/kafka.state" # # Initial offset, if there is no saved state. Can be relative time or "newest" or "oldest". # # In case offset is unavailable (in future, etc) fallback is "oldest" # initial-offset = "-30m" @@ -197,34 +173,34 @@ buffer-size = 0 # receiver_max_bytes = 500000000 # default 500MB [carbonlink] -listen = "go-carbon:7002" +listen = "127.0.0.1:7002" enabled = true # Close inactive connections after "read-timeout" read-timeout = "30s" # grpc api -# protocol: https://github.com/go-graphite/go-carbon/blob/master/helper/carbonpb/carbon.proto -# samples: https://github.com/go-graphite/go-carbon/tree/master/api/sample +# protocol: https://github.com/lomik/go-carbon/blob/master/helper/carbonpb/carbon.proto +# samples: https://github.com/lomik/go-carbon/tree/master/api/sample [grpc] -listen = "go-carbon:7003" -enabled = true +listen = "127.0.0.1:7003" +enabled = false # http://graphite.readthedocs.io/en/latest/tags.html [tags] -enabled = false +enabled = true # TagDB url. It should support /tags/tagMultiSeries endpoint -tagdb-url = "http://go-carbon:8000" +tagdb-url = "http://127.0.0.1:8080" tagdb-chunk-size = 32 tagdb-update-interval = 100 # Directory for send queue (based on leveldb) -local-dir = "/graphite/tagging/" +local-dir = "/tmp" # POST timeout tagdb-timeout = "1s" [carbonserver] # Please NOTE: carbonserver is not intended to fully replace graphite-web # It acts as a "REMOTE_STORAGE" for graphite-web or carbonzipper/carbonapi -listen = "go-carbon:8080" +listen = "127.0.0.1:8000" # Carbonserver support is still experimental and may contain bugs # Or be incompatible with github.com/grobian/carbonserver enabled = true @@ -232,16 +208,12 @@ enabled = true buckets = 10 # carbonserver-specific metrics will be sent as counters # For compatibility with grobian/carbonserver -metrics-as-counters = false +metrics-as-counters = true # Read and Write timeouts for HTTP server read-timeout = "60s" write-timeout = "60s" -# Request timeout for each API call -request-timeout = "60s" # Enable /render cache, it will cache the result for 1 minute -query-cache-enabled = true -# Enable carbonV2 gRPC streaming render cache, it will cache the result for 1 minute -streaming-query-cache-enabled = false +query-cache-enabled = false # 0 for unlimited query-cache-size-mb = 0 # Enable /metrics/find cache, it will cache the result for 5 minutes @@ -253,11 +225,9 @@ find-cache-enabled = true # Another drawback is that it will recreate index every scan-frequency interval # All new/deleted metrics will still be searchable until index is recreated trigram-index = true -# carbonserver keeps track of all available whisper files in memory. -# This determines how often it will check FS for new or deleted metrics. -# If you only use the trie index, have 'realtime-index' > 0, and delete metrics -# unfrequently you can have a very low scan frequency. -# Ex : you delete metrics only once every 24 hours, you can have a 24H scan frequency +# carbonserver keeps track of all available whisper files +# in memory. This determines how often it will check FS +# for new or deleted metrics. scan-frequency = "5m0s" # Control trie index # This index is built as an alternative to trigram index, with shorter indexing @@ -267,77 +237,6 @@ scan-frequency = "5m0s" # could be speeded up by enabling adding trigrams to trie, at the some costs of # memory usage (by setting both trie-index and trigram-index to true). trie-index = false -# Control how frequent it is to generate quota and usage metrics, and reset -# throughput counters (More details in doc/quotas.md). -# quota-usage-report-frequency = "1m" - -# Cache file list scan data in the specified path. This option speeds -# up index building after reboot by reading the last scan result in file -# system instead of scanning the whole data dir, which could takes up -# most of the indexing time if it contains a high number of metrics (10 -# - 40 millions). go-carbon only reads the cached file list once after -# reboot and the cache result is updated after every scan. (EXPERIMENTAL) -# -# file-list-cache = "/carbon/carbonserver-file-list-cache.bin" - -# Supported FLC (file list cache) version values are: 2, 1 (default value is 1 -# for backward compatibility). -# -# File list cache v2 is is more advanced of file list cache for go-carbon, with -# good quota support during restarts (by keeping physical, logical sizes, metric -# creation time in the cache). For more details, see carbonserver/flc.go. -# -# V2 file is no longer a plain text file compressed with gzip, but a semi-binary -# file. For reading flc v2 cache file, use go-carbon with flag -print-file-list-cache: -# -# go-carbon -print-file-list-cache /carbon/carbonserver-file-list-cache.bin -# -# For upgrade or downgrade to v2 or v1, you can just update the config, -# go-carbon would transparently detect the existing cache file on restart. No -# manual deletion needed. -# -# file-list-cache-version = 2 - -# Enable concurrently building index without maintaining a new copy -# index structure. More memory efficient. -# Currently only trie-index is supported. (EXPERIMENTAL) -concurrent-index = false - -# Set to larger than 0 to enable realtime indexing of new metrics, -# The value controls how many new metrics could be buffered at once. Suitable to -# adjust it higher if there are high number of new metrics being produced. -# Currently only trie-index is supported. -# (EXPERIMENTAL) -realtime-index = 0 - -# Maximum inflight requests allowed in go-carbon. Default is 0 which means no limit. -# If limit is reached, go-carbon/carbonserver returns 429 (Too Many Requests). -# This option would be handy as if there are too many long running requests piling up, -# it would slows down data ingestion in persister and slows down index building -# (especially during restart). -max-inflight-requests = 0 - -# Returns 503 (Service Unavailable) when go-carbon is still building the first -# index cache (trie/trigram) after restart. With trie or trigram index, -# carbonserver falls back to filepath.Glob when index is not ready. And -# filepath.Glob is expensive and not scaled enough to support some heavy -# queries (manifested as high usage and frequent GC). Thus it is possible that -# the read requests is so heavy that it not only slows down index building, it -# also hinders persister from flushing down data to disk and causing cache.size -# overflow. -# -# By default, it is disabled. -# -# It is recommend to enable this flag together with "file-list-cache" (depends on -# the number of metrics, indexing building with "file-list-cache" is usually -# much faster than re-scanning the whole file tree after restart). -no-service-when-index-is-not-ready = false - -# This provides the ability to query for new metrics without any wsp files -# i.e query for metrics present only in cache. Does a cache-scan and -# populates index with metrics with or without corresponding wsp files, -# but will lead to increased memory consumption. Disabled by default. -cache-scan = false # Maximum amount of globs in a single metric in index # This value is used to speed-up /find requests with @@ -347,10 +246,10 @@ max-globs = 100 fail-on-max-globs = false # Maximum metrics could be returned by glob/wildcard in find request (currently -# works only for trie index) (Default: 10,000,000) +# works only for trie index) max-metrics-globbed = 30000 # Maximum metrics could be returned in render request (works both all types of -# indexes) (Default: 1,000,000) +# indexes) max-metrics-rendered = 1000 # graphite-web-10-mode @@ -360,97 +259,21 @@ max-metrics-rendered = 1000 # That might degrade performance of the cluster # But will be compatible with both graphite-web 1.0 and 0.9.x graphite-web-10-strict-mode = true - -# Return a 404 instead of empty results. Set to true if you are serving requests to graphite-web -empty-result-ok = false - # Allows to keep track for "last time readed" between restarts, leave empty to disable internal-stats-dir = "" -# Calculate /render request time percentiles for the bucket, '95' means calculate 95th Percentile. -# To disable this feature, leave the list blank +# Calculate /render request time percentiles for the bucket, '95' means calculate 95th Percentile. To disable this feature, leave the list blank stats-percentiles = [99, 98, 95, 75, 50] -# heavy-glob-query-rate-limiters is a narrow control against queries that might -# causes high cpu and memory consumption due to matching over too many metrics -# or nodes at the same time, queries like: "*.*.*.*.*.*.*.*.*keyword*". For -# these types of queries, trigram might be able to handle it better, but for -# trie and filesystem glob, it's too expensive. -# -# pattern is a Go regular expression: https://pkg.go.dev/regexp/syntax. -# -# When max-inflight-requests is set to 0, it means instant rejection. -# When max-inflight-requests is set as a positive integer and when there are too -# many concurrent requests, it would block/delay the request until the previous ones -# are completed. -# -# The configs are order sensitive and are applied top down. The current -# implementation is in an O(n) so it's advised not to apply too many rules here -# as they are applied on all the queries. -# -# [[carbonserver.heavy-glob-query-rate-limiters]] -# pattern = "^(\*\.){5,}" -# max-inflight-requests = 1 -# -# [[carbonserver.heavy-glob-query-rate-limiters]] -# pattern = "^sys(\*\.){7,}" -# max-inflight-requests = 0 - -# api-per-path-rate-limiters are used for strict api call rate limiting. All -# registered API paths (see carbonserver.Listen for a full list) can be -# controlled separately here: -# -# "/_internal/capabilities/" -# "/metrics/find/" -# "/metrics/list/" -# "/metrics/list_query/" -# "/metrics/details/" -# "/render/" -# "/info/" -# "/forcescan" -# "/admin/quota" -# "/admin/info" -# "/robots.txt" -# ... -# -# When max-inflight-requests is set to 0, it means instant rejection. -# When max-inflight-requests is set as a positive integer and when there are too -# many concurrent requests, it would block/delay the request until the previous -# ones are completed. -# -# request-timeout would override the global request-timeout. -# -# [[carbonserver.api-per-path-rate-limiters]] -# path = "/metrics/list/" -# max-inflight-requests = 1 -# request-timeout = "600s" -# -# [[carbonserver.api-per-path-rate-limiters]] -# path = "/metrics/list_query/" -# max-inflight-requests = 3 -# -# For gRPC rpcs, path should be full method name. -# -# [[carbonserver.api-per-path-rate-limiters]] -# path = "/carbonapi_v2_grpc.CarbonV2/Render" -# max-inflight-requests = 10 -# request-timeout = "40s" - -# carbonserver.grpc is the configuration for listening for grpc clients. -# Note: currently, only CarbonV2 Render rpc is implemented. -# [carbonserver.grpc] -# enabled = true -# listen = ":7004" - [dump] # Enable dump/restore function on USR2 signal enabled = false # Directory for store dump data. Should be writeable for carbon -path = "/graphite/dump/" +path = "/tmp" # Restore speed. 0 - unlimited restore-per-second = 0 [pprof] -listen = "go-carbon:7007" +listen = "localhost:7007" enabled = false #[prometheus] @@ -468,7 +291,7 @@ enabled = false # @TODO logger = "" # Log output: filename, "stderr", "stdout", "none", "" (same as "stderr") -file = "none" +file = "/var/log/go-carbon.log" # Log level: "debug", "info", "warn", "error", "dpanic", "panic", and "fatal" level = "info" # Log format: "json", "console", "mixed" @@ -477,3 +300,13 @@ encoding = "mixed" encoding-time = "iso8601" # Log duration format: "seconds", "nanos", "string" encoding-duration = "seconds" + +# You can define multiply loggers: + +[[logging]] +logger = "" +file = "stderr" +level = "info" +encoding = "mixed" +encoding-time = "iso8601" +encoding-duration = "seconds" \ No newline at end of file diff --git a/grafanaConfig.example/graphite/graphTemplates.conf b/grafanaConfig.example/graphite/graphTemplates.conf new file mode 100644 index 0000000..a6ee77c --- /dev/null +++ b/grafanaConfig.example/graphite/graphTemplates.conf @@ -0,0 +1,38 @@ +[default] +background = black +foreground = white +majorLine = white +minorLine = grey +lineColors = blue,green,red,purple,brown,yellow,aqua,grey,magenta,pink,gold,rose +fontName = Sans +fontSize = 10 +fontBold = False +fontItalic = False + +[noc] +background = black +foreground = white +majorLine = white +minorLine = grey +lineColors = blue,green,red,yellow,purple,brown,aqua,grey,magenta,pink,gold,rose +fontName = Sans +fontSize = 10 +fontBold = False +fontItalic = False + +[plain] +background = white +foreground = black +minorLine = grey +majorLine = rose + +[summary] +background = black +lineColors = #6666ff, #66ff66, #ff6666 + +[alphas] +background = white +foreground = black +majorLine = grey +minorLine = rose +lineColors = 00ff00aa,ff000077,00337799 diff --git a/grafanaConfig.example/graphite/relay-rules.conf b/grafanaConfig.example/graphite/relay-rules.conf new file mode 100644 index 0000000..8260236 --- /dev/null +++ b/grafanaConfig.example/graphite/relay-rules.conf @@ -0,0 +1,21 @@ +# Relay destination rules for carbon-relay. Entries are scanned in order, +# and the first pattern a metric matches will cause processing to cease after sending +# unless `continue` is set to true +# +# [name] +# pattern = +# destinations = +# continue = # default: False +# +# name: Arbitrary unique name to identify the rule +# pattern: Regex pattern to match against the metric name +# destinations: Comma-separated list of destinations. +# ex: 127.0.0.1, 10.1.2.3:2004, 10.1.2.4:2004:a, myserver.mydomain.com +# continue: Continue processing rules if this rule matches (default: False) + +# You must have exactly one section with 'default = true' +# Note that all destinations listed must also exist in carbon.conf +# in the DESTINATIONS setting in the [relay] section +[default] +default = true +destinations = 127.0.0.1:2004 diff --git a/grafanaConfig.example/graphite/rewrite-rules.conf b/grafanaConfig.example/graphite/rewrite-rules.conf new file mode 100644 index 0000000..f7bd4d1 --- /dev/null +++ b/grafanaConfig.example/graphite/rewrite-rules.conf @@ -0,0 +1,18 @@ +# This file defines regular expression patterns that can be used to +# rewrite metric names in a search & replace fashion. It consists of two +# sections, [pre] and [post]. The rules in the pre section are applied to +# metric names as soon as they are received. The post rules are applied +# after aggregation has taken place. +# +# The general form of each rule is as follows: +# +# regex-pattern = replacement-text +# +# For example: +# +# [post] +# _sum$ = +# _avg$ = +# +# These rules would strip off a suffix of _sum or _avg from any metric names +# after aggregation. diff --git a/grafanaConfig.example/graphite/storage-aggregation.conf b/grafanaConfig.example/graphite/storage-aggregation.conf new file mode 100644 index 0000000..80a3842 --- /dev/null +++ b/grafanaConfig.example/graphite/storage-aggregation.conf @@ -0,0 +1,42 @@ +# Aggregation methods for whisper files. Entries are scanned in order, +# and first match wins. This file is scanned for changes every 60 seconds +# +# [name] +# pattern = +# xFilesFactor = +# aggregationMethod = +# +# name: Arbitrary unique name for the rule +# pattern: Regex pattern to match against the metric name +# xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur +# aggregationMethod: function to apply to data points for aggregation +# +[min] +pattern = \.lower$ +xFilesFactor = 0.1 +aggregationMethod = min + +[max] +pattern = \.upper(_\d+)?$ +xFilesFactor = 0.1 +aggregationMethod = max + +[sum] +pattern = \.sum$ +xFilesFactor = 0 +aggregationMethod = sum + +[count] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[count_legacy] +pattern = ^stats_counts.* +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0.3 +aggregationMethod = average diff --git a/grafanaConfig.example/graphite/storage-schemas.conf b/grafanaConfig.example/graphite/storage-schemas.conf new file mode 100644 index 0000000..f3c23bc --- /dev/null +++ b/grafanaConfig.example/graphite/storage-schemas.conf @@ -0,0 +1,26 @@ +# Schema definitions for Whisper files. Entries are scanned in order, +# and first match wins. This file is scanned for changes every 60 seconds. +# +# Definition Syntax: +# +# [name] +# pattern = regex +# retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ... +# +# Remember: To support accurate aggregation from higher to lower resolution +# archives, the precision of a longer retention archive must be +# cleanly divisible by precision of next lower retention archive. +# +# Valid: 60s:7d,300s:30d (300/60 = 5) +# Invalid: 180s:7d,300s:30d (300/180 = 3.333) +# + +# Carbon's internal metrics. This entry should match what is specified in +# CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings +[carbon] +pattern = ^carbon\. +retentions = 10s:6h,1m:90d + +[default_1min_for_1day] +pattern = .* +retentions = 10s:6h,1m:6d,10m:1800d diff --git a/grafanaConfig.example/graphite/whitelist.conf b/grafanaConfig.example/graphite/whitelist.conf new file mode 100644 index 0000000..f84895d --- /dev/null +++ b/grafanaConfig.example/graphite/whitelist.conf @@ -0,0 +1,6 @@ +# This file takes a single regular expression per line +# If USE_WHITELIST is set to True in carbon.conf, only metrics received which +# match one of these expressions will be persisted. If this file is empty or +# missing, all metrics will pass through. +# This file is reloaded automatically when changes are made +.* diff --git a/package-lock.json b/package-lock.json index 809d9fb..455223f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "screeps-grafana-go_carbon", - "version": "1.0.2", + "version": "1.0.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "screeps-grafana-go_carbon", - "version": "1.0.2", + "version": "1.0.5", "dependencies": { "axios": "^0.27.2", "dotenv": "^16.0.2", diff --git a/src/pushStats/apiFunctions.js b/src/pushStats/apiFunctions.js index c510acd..f9ff011 100644 --- a/src/pushStats/apiFunctions.js +++ b/src/pushStats/apiFunctions.js @@ -199,7 +199,13 @@ export default class { static async getMemory(info, shard, statsPath = 'stats') { const options = await getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); const res = await req(options); - if (!res) return undefined; + + if (res) { + console.log(`Got memory from ${info.username} in ${shard} `); + } else { + return undefined + } + const data = await gz(res.data); return data; } diff --git a/src/pushStats/index.js b/src/pushStats/index.js index e79546b..e615443 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -35,7 +35,7 @@ const cronTransport = new transports.DailyRotateFile({ maxFiles: '14d', }); -const client = graphite.createClient('plaintext://carbon-relay-ng:2003/'); +const client = graphite.createClient('plaintext://graphite:2003/'); const { combine, timestamp, prettyPrint } = format; const logger = createLogger({ format: combine( @@ -64,14 +64,19 @@ class ManageStats { } async handleUsers(type) { + console.log(`[${type}] Handling Users`); + const beginningOfMinute = new Date().getSeconds() < 15; const getStatsFunctions = []; users.forEach((user) => { try { + if (user.type !== type) return; + const rightMinuteForShard = new Date().getMinutes() % user.shards.length === 0; const shouldContinue = !beginningOfMinute || !rightMinuteForShard; if (user.type === 'mmo' && shouldContinue) return; if (user.type === 'season' && shouldContinue) return; + for (let y = 0; y < user.shards.length; y += 1) { const shard = user.shards[y]; getStatsFunctions.push(this.getStats(user, shard, this.message)); @@ -80,6 +85,9 @@ class ManageStats { logger.error(error.message); } }); + + console.log(`[${type}] Getting ${getStatsFunctions.length} statistics`) + await Promise.all(getStatsFunctions); const { groupedStats } = this; @@ -87,19 +95,24 @@ class ManageStats { if (type === 'mmo') { if (Object.keys(groupedStats).length > 0) { if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - this.message += `Pushed ${type} stats to graphite`; - logger.info(this.message); + + console.log(`[${type}] Pushed stats to graphite`) + return console.log(this.message); } + if (beginningOfMinute) return console.log('No stats to push'); return undefined; } if (type === 'season') { if (Object.keys(groupedStats).length > 0) { + if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - this.message += `Pushed ${type} stats to graphite`; - logger.info(this.message); + + console.log(`[${type}] Pushed stats to graphite`) + return console.log(this.message); + } if (beginningOfMinute) return console.log('No stats to push'); return undefined; @@ -112,9 +125,10 @@ class ManageStats { if (adminUtilsServerStats) { try { const groupedAdminStatsUsers = {}; - adminUtilsServerStats.users.forEach((user) => { - groupedAdminStatsUsers[user.username] = user; - }); + for (const [username, user] of Object.entries(adminUtilsServerStats)) { + groupedAdminStatsUsers[username] = user; + } + adminUtilsServerStats.users = groupedAdminStatsUsers; } catch (error) { console.log(error); @@ -163,6 +177,7 @@ class ManageStats { const stats = userinfo.segment === undefined ? await ApiFunc.getMemory(userinfo, shard) : await ApiFunc.getSegmentMemory(userinfo, shard); + await this.processStats(userinfo, shard, stats); return 'success'; } catch (error) { @@ -180,6 +195,7 @@ class ManageStats { static async reportStats(stats) { return new Promise((resolve) => { + console.log("Writing stats " + JSON.stringify(stats) + " to graphite"); client.write({ [`${process.env.PREFIX ? `${process.env.PREFIX}.` : ''}screeps`]: stats }, (err) => { if (err) { console.log(err); @@ -189,13 +205,16 @@ class ManageStats { lastUpload = new Date().getTime(); resolve(true); }); + //resolve(true); }); } pushStats(userinfo, stats, shard) { if (Object.keys(stats).length === 0) return; - this.groupedStats[(userinfo.prefix ? `${userinfo.prefix}.` : '') + userinfo.username] = userinfo.type === 'mmo' ? { [shard]: stats } : { shard: stats }; - this.message += `${userinfo.type}: Added stats object for ${userinfo.username} in ${shard}\r\n`; + let username = userinfo.replaceName !== undefined ? userinfo.replaceName : userinfo.username; + this.groupedStats[(userinfo.prefix ? `${userinfo.prefix}.` : '') + username] = { [shard]: stats }; + + console.log(`Pushing stats for ${(userinfo.prefix ? `${userinfo.prefix}.` : '') + username} in ${shard}`); } } @@ -209,7 +228,7 @@ const groupedUsers = users.reduce((group, user) => { cron.schedule('*/30 * * * * *', async () => { const message = `Cron event hit: ${new Date()}`; - console.log(`\r\n${message}`); + console.log(`\r\n${message}\n`); cronLogger.info(message); Object.keys(groupedUsers).forEach((type) => { new ManageStats(groupedUsers[type]).handleUsers(type); diff --git a/src/setup/setup.js b/src/setup/setup.js index 44c71f9..498e29c 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -31,7 +31,7 @@ function UpdateEnvFile() { let exampleEnvText = fs.readFileSync(exampleEnvFilePath, 'utf8'); exampleEnvText = exampleEnvText .replace('GRAFANA_PORT=3000', `GRAFANA_PORT=${grafanaPort}`) - .replace('COMPOSE_PROJECT_NAME=screeps-grafana', `COMPOSE_PROJECT_NAME=screeps-grafana-${!argv.traefik ? grafanaPort : 'standalone'}`) + .replace('COMPOSE_PROJECT_NAME=screeps-grafana', `COMPOSE_PROJECT_NAME=screeps-grafana-3000`) .replace('COMPOSE_FILE=./docker-compose.yml', `COMPOSE_FILE=${join(__dirname, '../../docker-compose.yml')}`); if (serverPort) exampleEnvText = exampleEnvText.replace('SERVER_PORT=21025', `SERVER_PORT=${serverPort}`); @@ -61,18 +61,7 @@ async function UpdateDockerComposeFile() { } if (argv.includePushStatusApi) exampleDockerComposeText = exampleDockerComposeText.replace('INCLUDE_PUSH_STATUS_API=false', `INCLUDE_PUSH_STATUS_API=true${regexEscape} ports:${regexEscape} - ${port}:${port}`); if (argv.prefix) exampleDockerComposeText = exampleDockerComposeText.replace('PREFIX=', `PREFIX=${argv.prefix}`); - if (argv.traefik) { - exampleDockerComposeText = exampleDockerComposeText.replace(/#t/g, ''); - exampleDockerComposeText = exampleDockerComposeText.replace(/grafana.localhost/g, argv.grafanaDomain); - } - if (argv.traefikHost) { - exampleDockerComposeText = exampleDockerComposeText.replace('#t labels:', ' labels:'); - exampleDockerComposeText = exampleDockerComposeText.replace('#t - "traefik.enable=true"', ' - "traefik.enable=true"'); - exampleDockerComposeText = exampleDockerComposeText.replace('#t - "traefik.http.routers.grafana.rule=Host(`grafana.localhost`)"', ' - "traefik.http.routers.grafana.rule=Host(`grafana.localhost`)"'); - exampleDockerComposeText = exampleDockerComposeText.replace('#t - "traefik.http.services.grafana.loadbalancer.server.port=3000"', ` - "traefik.http.services.grafana.loadbalancer.server.port=${grafanaPort}"`); - exampleDockerComposeText = exampleDockerComposeText.replace(/#t#h/g, ''); - exampleDockerComposeText = exampleDockerComposeText.replace(/grafana.localhost/g, argv.grafanaDomain); - } + fs.writeFileSync(dockerComposeFile, exampleDockerComposeText); console.log('Docker-compose file created'); @@ -109,6 +98,8 @@ function UpdateGrafanaConfigFolder() { grafanaIniText = grafanaIniText.replace(createRegexWithEscape('enable anonymous access\r\nenabled = (.*)'), `enable anonymous access${regexEscape}enabled = ${enableAnonymousAccess}`); fs.writeFileSync(grafanaIniFile, grafanaIniText); + // This can just be set manually in the config folder. + /* const storageSchemasFile = join(grafanaConfigFolder, './go-carbon/storage-schemas.conf'); let storageSchemasText = fs.readFileSync(storageSchemasFile, 'utf8'); const { defaultRetention } = argv; @@ -116,6 +107,8 @@ function UpdateGrafanaConfigFolder() { if (defaultRetention) storageSchemasText = storageSchemasText.replace(createRegexWithEscape('pattern = .*\r\nretentions = (.*)'), `pattern = .*${regexEscape}retentions = ${defaultRetention}`); fs.writeFileSync(storageSchemasFile, storageSchemasText); + */ + console.log('Grafana config folder created'); } @@ -143,9 +136,7 @@ async function Setup(mArgv) { argv = mArgv || {}; if (argv.grafanaPort) grafanaPort = argv.grafanaPort; else { - grafanaPort = !argv.traefik - ? await getPort({ portRange: [3000, 4000] }) - : 3000; + grafanaPort = 3000 } argv.grafanaPort = grafanaPort; diff --git a/users.example.json b/users.example.json index d393025..f6733dc 100644 --- a/users.example.json +++ b/users.example.json @@ -1,56 +1,17 @@ [ { - "username": "W1N1", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W6N1", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W2N5", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W5N8", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W7N3", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W9N9", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W3N9", - "type": "private", - "shards": ["screeps"], - "password": "password" - }, - { - "username": "W8N6", - "type": "private", - "shards": ["screeps"], - "password": "password" + "username": "YourUsernameHere", + "type": "mmo", + "shards": ["shard"], + "password": "YourPasswordHere" }, { - "username": "W3N3", + "username": "YourEmailHere", + "replaceName": "YourUsernameHere", + "host": "localhost", + "port": 21025, "type": "private", - "shards": ["screeps"], - "password": "password" + "shards": [""], + "password": "YourPasswordHere" } ] From 49b0a07d280c40654e4bce113c302fc11270deac Mon Sep 17 00:00:00 2001 From: InfinityDevTech <47366052+InfinityDevTech@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:35:39 -0400 Subject: [PATCH 2/2] Update README, fixed formatting added comments. --- README.md | 54 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e326d70..b129282 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,12 @@ A. MMO: ```json { -"prefix": "a.b.c", -"username": "PandaMaster", -"type": "mmo", -"shards": ["shard0"], -"token": "TOKEN_FOR_THIS_USER!", + # Prefix is entirely optional + "prefix": "a.b.c", + "username": "PandaMaster", + "type": "mmo", + "shards": ["shard0"], + "token": "TOKEN_FOR_THIS_USER!", } ``` @@ -36,12 +37,15 @@ B. Private: ```json { -"prefix": "a.b.c", -"username": "EMAIL", -"replaceName": "USERNAME HERE", -"type": "private", -"shards": ["screeps"], -"password": "password", + # Prefix is entirely optional + "prefix": "a.b.c", + "username": "EMAIL", + # If your email has a period in it, grafana displays it incorrectly + # Its reccomended to change this to the username you set for MMO. + "replaceName": "USERNAME HERE", + "type": "private", + "shards": ["screeps"], + "password": "password", } ``` @@ -49,12 +53,12 @@ If the private server is not hosted on localhost, add the host to the user: ```json { -"username": "EMAIL", -"replaceName": "USERNAME", -"type": "private", -"shards": ["screeps"], -"password": "password", -"host": "192.168.1.10", + "username": "EMAIL", + "replaceName": "USERNAME", + "type": "private", + "shards": ["screeps"], + "password": "password", + "host": "192.168.1.10", } ``` @@ -62,12 +66,12 @@ If the segment of the stats is not memory, add it to the user: ```json { -"username": "W1N1", -"type": "private", -"shards": ["screeps"], -"password": "password", -"host": "123.456.789", -"segment": 0, + "username": "W1N1", + "type": "private", + "shards": ["screeps"], + "password": "password", + "host": "123.456.789", + "segment": 0, } ``` @@ -81,7 +85,6 @@ Update all .example files and/or folders to match your needs. This step is not r * `--debug`: listen to setup Docker logs * `--username`: overwrite the username for the Grafana admin user * `--password`: overwrite the password for the Grafana admin user -* `--defaultRetention`: overwrite the default retention for the default retention polic of all not regex'd retention paths. * `--enableAnonymousAccess`: enable anonymous access to Grafana #### Network @@ -99,8 +102,7 @@ Update all .example files and/or folders to match your needs. This step is not r ## Usage * `npm run setup`: to execute setup only -* `npm run start:standalone`: to configure and start it -* `npm run start:standalone-traefik` to confgiure with traefik and start it +* `npm run start`: to configure and start it * For other run commands like eslint, check out package.json scripts object. Go to [localhost:3000](http://localhost:3000) (if you used port 3000) and login with `admin` and `password` (or your custom set login info).