diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ae7277f --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# The port where the Screeps server is listen at +SERVER_PORT=21025 + +# External port where Grafana listens to +GRAFANA_PORT=3000 + +# The port where the Push Status API will listen to +# Optional, if unset, the port won't be opened +PUSH_STATUS_PORT= + +# A prefix to use when pushing keys to Graphite +PREFIX= + +# Admin user for Grafana +ADMIN_USER=admin +ADMIN_PASSWORD=password + +# The external hostname for the server Grafana runs on +HOSTNAME=localhost + +# Email address Grafana uses for emails +EMAIL_ADDRESS=admin@grafana.localhost + +# Whether anonymous access to Grafana is allowed +ANONYMOUS_AUTH_ENABLED=false diff --git a/.eslintrc.js b/.eslintrc.js index defab5b..5b95e04 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,5 +17,6 @@ module.exports = { "no-underscore-dangle":"off", "no-param-reassign": ["error", { "props": false }], "no-restricted-syntax": "off", + "no-continue": "off", } } diff --git a/.gitignore b/.gitignore index 65df0e2..251f6b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ # Files -docker-compose.yml .env -users.json # Folders node_modules diff --git a/README.md b/README.md index 7f7ed8e..5b863ce 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,27 @@ ## Setup -1. Update all .example files and/or folders to match your needs. This step is not required if you are using the default setup. -2. Add your own Grafana variables in `grafanaConfig/.env.grafana`. This file will be updated after a volume reset. +1. Copy `.env.example` to `.env` and edit to match your needs. +2. Copy `users.example.json` to `users.json` and edit it according to [User Setup](#User-Setup). +3. The configuration files for both Grafana and Graphite are in `config/grafana` and `config/graphite` respectively. +4. If you have a dashboard you want to auto-add, you can drop their JSON files into `config/grafana/provisioning/dashboards` +and they'll be auto-added to the instance. + +## Usage + +* `npm run start`: start the containers +* `npm run logs`: check the container's logs +* `npm run stop`: stop the containers +* `npm run reset`: remove the containers +* `npm run reset:hard`: remove the containers and their volumes +* `npm run rebuild`: rebuild the pushStats container and restart it; needed if you make changes to its code. + +See the scripts section in the package.json file. + +Go to [localhost:3000](http://localhost:3000) (if you used port 3000) and login with `admin` and `password` (or your custom set login info). + +Its possible to use https for your grafana instance, check out this [tutorial](https://www.turbogeek.co.uk/grafana-how-to-configure-ssl-https-in-grafana/) for example on how to do this, enough info online about it. I dont support this (yet) + ### User Setup @@ -59,6 +78,7 @@ If the private server is not hosted on localhost, add the host to the user: "shards": ["screeps"], "password": "password", "host": "192.168.1.10", + "port": 21025, } ``` @@ -74,38 +94,3 @@ If the segment of the stats is not memory, add it to the user: "segment": 0, } ``` - -Update all .example files and/or folders to match your needs. This step is not required if you are using the default setup. - -### Run Commands - -#### Config - -* `--force`: force the non .example config files to be overwritten. -* `--debug`: listen to setup Docker logs -* `--username`: overwrite the username for the Grafana admin user -* `--password`: overwrite the password for the Grafana admin user -* `--enableAnonymousAccess`: enable anonymous access to Grafana - -#### Network - -* `--grafanaDomain`: Overwrite grafana.ini domain -* `--grafanaPort`: port for Grafana to run on -* `--relayPort`: port for relay-ng to run on (default: 2003) -* `--pushStatusPort`: port for the stats-getter push API (default: disabled) - -#### Exporting - -* `--deleteLogs`: deletes the logs folder -* `--removeWhisper`: Deletes the carbon whisper folder -* `--removeVolumes`: Remove all volumes, including the grafana database. - -## Usage - -* `npm run setup`: to execute setup only -* `npm run start`: to configure and start it -* For other run commands like eslint, check out package.json scripts object. - -Go to [localhost:3000](http://localhost:3000) (if you used port 3000) and login with `admin` and `password` (or your custom set login info). - -Its possible to use https for your grafana instance, check out this [tutorial](https://www.turbogeek.co.uk/grafana-how-to-configure-ssl-https-in-grafana/) for example on how to do this, enough info online about it. I dont support this (yet) diff --git a/bin/server.js b/bin/server.js deleted file mode 100644 index 2293d88..0000000 --- a/bin/server.js +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env node - -const { execSync } = require('child_process'); -const { join } = require('path'); -require('dotenv').config({ path: join(__dirname, '../.env') }); - -const nodeVersion = process.versions.node; -const nodeVersionMajor = Number(nodeVersion.split('.')[0]); -const { getPort } = nodeVersionMajor >= 14 ? require('get-port-please') : { getPort: async () => 3000 }; - -const minimist = require('minimist'); -const { createLogger, format, transports } = require('winston'); - -const setup = require('../src/setup/setup'); -const start = require('../src/setup/start'); - -const argv = minimist(process.argv.slice(2)); - -const { combine, timestamp, prettyPrint } = format; -const logger = createLogger({ - transports: [ - new transports.Console({ - format: combine(format.colorize(), format.simple()), - }), - new transports.File({ - filename: 'logs/setup.log', - format: combine( - timestamp(), - prettyPrint(), - ), - })], -}); - -async function main() { - argv.grafanaPort = argv.grafanaPort ?? await getPort({ portRange: [3000, 4000] }); - argv.serverPort = argv.serverPort ?? 21025; - - const cli = { - cmd: argv._.shift(), - args: argv, - logger, - }; - - switch (cli.cmd) { - case 'setup': - setup(cli); - break; - - case 'start': - start(cli); - break; - - case 'stop': - logger.info(`Stopping server from ${process.env.COMPOSE_FILE}`); - execSync('docker-compose stop'); - break; - - default: - logger.error(`expected command, got "${cli.cmd}"`); - break; - } -} - -main(); diff --git a/grafanaConfig.example/grafana/grafana.ini b/config/grafana/grafana.ini similarity index 100% rename from grafanaConfig.example/grafana/grafana.ini rename to config/grafana/grafana.ini diff --git a/config/grafana/provisioning/access-control/.gitignore b/config/grafana/provisioning/access-control/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/config/grafana/provisioning/alerting/.gitignore b/config/grafana/provisioning/alerting/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/grafanaConfig.example/grafana/provisioning/dashboards/telemetry.yaml b/config/grafana/provisioning/dashboards/dashboard.yml similarity index 76% rename from grafanaConfig.example/grafana/provisioning/dashboards/telemetry.yaml rename to config/grafana/provisioning/dashboards/dashboard.yml index 07d4e8b..fa0f40c 100644 --- a/grafanaConfig.example/grafana/provisioning/dashboards/telemetry.yaml +++ b/config/grafana/provisioning/dashboards/dashboard.yml @@ -2,23 +2,23 @@ apiVersion: 1 providers: # an unique provider name. Required - - name: 'telemetry' + - name: 'serviceInfo' # Org id. Default to 1 orgId: 1 # name of the dashboard folder. - folder: 'Telemetry' + folder: '' # folder UID. will be automatically generated if not specified folderUid: '' # provider type. Default to 'file' type: file # disable dashboard deletion - disableDeletion: true + disableDeletion: false # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 60 + updateIntervalSeconds: 10 # allow updating provisioned dashboards from the UI - allowUiUpdates: false + allowUiUpdates: true options: # path to dashboard files on disk. Required when using the 'file' type - path: /etc/grafana/dashboards + path: /etc/grafana/provisioning/dashboards # use folder names from filesystem to create folders in Grafana - foldersFromFilesStructure: false \ No newline at end of file + foldersFromFilesStructure: true \ No newline at end of file diff --git a/config/grafana/provisioning/dashboards/serviceInfo.json b/config/grafana/provisioning/dashboards/serviceInfo.json new file mode 100644 index 0000000..e959866 --- /dev/null +++ b/config/grafana/provisioning/dashboards/serviceInfo.json @@ -0,0 +1,567 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "carbon.*.*.cpuUsage" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "aliasByNode(carbon.agents.*.memUsage, 2)" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "groupByNode(aliasByNode(carbon.*.*.metricsReceived, 3), 0, 'sumSeries')" + } + ], + "title": "Metrics recieved", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "alias(stats.statsd.processing_time, 'Total')" + } + ], + "title": "Processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 7, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "stats.gauges.statsd.timestamp_lag" + } + ], + "title": "Timestamp lag", + "transformations": [], + "type": "gauge" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "groupByNode(alias(carbon.agents.*.cache.size, 'Total'), 0, 'sum')" + } + ], + "title": "Cache size", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service info", + "uid": "weVPAU7Vk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml b/config/grafana/provisioning/datasources/carbonapi.yaml similarity index 93% rename from grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml rename to config/grafana/provisioning/datasources/carbonapi.yaml index 8931eed..800710e 100644 --- a/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml +++ b/config/grafana/provisioning/datasources/carbonapi.yaml @@ -19,6 +19,8 @@ datasources: orgId: 1 # url url: http://graphite:8080/ + # uid, used to link that source with the provisioned dashboards + uid: d34EAUnVz # database password, if used password: # database user, if used diff --git a/config/grafana/provisioning/notifiers/.gitignore b/config/grafana/provisioning/notifiers/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/config/grafana/provisioning/plugins/.gitignore b/config/grafana/provisioning/plugins/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/grafanaConfig.example/graphite/aggregation-rules.conf b/config/graphite/aggregation-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/aggregation-rules.conf rename to config/graphite/aggregation-rules.conf diff --git a/grafanaConfig.example/graphite/blacklist.conf b/config/graphite/blacklist.conf similarity index 100% rename from grafanaConfig.example/graphite/blacklist.conf rename to config/graphite/blacklist.conf diff --git a/grafanaConfig.example/graphite/brubeck.json b/config/graphite/brubeck.json similarity index 100% rename from grafanaConfig.example/graphite/brubeck.json rename to config/graphite/brubeck.json diff --git a/grafanaConfig.example/graphite/carbon.amqp.conf b/config/graphite/carbon.amqp.conf similarity index 100% rename from grafanaConfig.example/graphite/carbon.amqp.conf rename to config/graphite/carbon.amqp.conf diff --git a/grafanaConfig.example/graphite/carbon.conf b/config/graphite/carbon.conf similarity index 100% rename from grafanaConfig.example/graphite/carbon.conf rename to config/graphite/carbon.conf diff --git a/grafanaConfig.example/graphite/dashboard.conf b/config/graphite/dashboard.conf similarity index 100% rename from grafanaConfig.example/graphite/dashboard.conf rename to config/graphite/dashboard.conf diff --git a/grafanaConfig.example/graphite/go-carbon.conf b/config/graphite/go-carbon.conf similarity index 100% rename from grafanaConfig.example/graphite/go-carbon.conf rename to config/graphite/go-carbon.conf diff --git a/grafanaConfig.example/graphite/graphTemplates.conf b/config/graphite/graphTemplates.conf similarity index 100% rename from grafanaConfig.example/graphite/graphTemplates.conf rename to config/graphite/graphTemplates.conf diff --git a/grafanaConfig.example/graphite/relay-rules.conf b/config/graphite/relay-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/relay-rules.conf rename to config/graphite/relay-rules.conf diff --git a/grafanaConfig.example/graphite/rewrite-rules.conf b/config/graphite/rewrite-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/rewrite-rules.conf rename to config/graphite/rewrite-rules.conf diff --git a/grafanaConfig.example/graphite/storage-aggregation.conf b/config/graphite/storage-aggregation.conf similarity index 100% rename from grafanaConfig.example/graphite/storage-aggregation.conf rename to config/graphite/storage-aggregation.conf diff --git a/grafanaConfig.example/graphite/storage-schemas.conf b/config/graphite/storage-schemas.conf similarity index 100% rename from grafanaConfig.example/graphite/storage-schemas.conf rename to config/graphite/storage-schemas.conf diff --git a/grafanaConfig.example/graphite/whitelist.conf b/config/graphite/whitelist.conf similarity index 100% rename from grafanaConfig.example/graphite/whitelist.conf rename to config/graphite/whitelist.conf diff --git a/dashboards/helper.js b/dashboards/helper.js deleted file mode 100644 index 0bd04e6..0000000 --- a/dashboards/helper.js +++ /dev/null @@ -1,46 +0,0 @@ -const fs = require('fs'); -const { join } = require('path'); - -function transformDashboard(dashboard) { - delete dashboard.id; - delete dashboard.uid; - for (let i = 0; i < dashboard.templating.list.length; i += 1) { - const { datasource } = dashboard.templating.list[i]; - if (datasource) { - delete datasource.type; - delete datasource.uid; - } - } - - for (let i = 0; i < dashboard.panels.length; i += 1) { - const panel = dashboard.panels[i]; - if (panel.panels) { - for (let y = 0; y < panel.panels.length; y += 1) { - const subPanel = panel.panels[y]; - delete subPanel.datasource.uid; - delete subPanel.datasource.type; - } - } else if (panel.type !== 'row') { - delete panel.datasource.uid; - delete panel.datasource.type; - } - } - return { dashboard, overwrite: true }; -} - -module.exports = function GetDashboards() { - const setupDashboards = {}; - - const dashboardPath = __dirname; - const dashboardFileNames = fs.readdirSync(dashboardPath).filter((d) => d.endsWith('.json')); - dashboardFileNames.forEach((name) => { - try { - const filePath = join(dashboardPath, `${name}`); - const rawDashboardText = fs.readFileSync(filePath); - setupDashboards[name] = transformDashboard(JSON.parse(rawDashboardText)); - } catch (error) { - console.log(error); - } - }); - return setupDashboards; -}; diff --git a/dashboards/serviceInfo.json b/dashboards/serviceInfo.json deleted file mode 100644 index 9823c82..0000000 --- a/dashboards/serviceInfo.json +++ /dev/null @@ -1,317 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" - }, - "refId": "A", - "target": "groupByNode(aliasByNode(openmetric.carbon.*.*.metricsReceived, 3), 0, 'sumSeries')" - } - ], - "title": "Metrics recieved", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" - }, - "refId": "A", - "target": "groupByNode(alias(openmetric.carbon.*.cache.metrics, 'Total'), 0, 'sum')" - } - ], - "title": "Metrics processed", - "type": "timeseries" - }, - { - "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" - }, - "refId": "A", - "target": "groupByNode(alias(openmetric.carbon.*.cache.size, 'Total'), 0, 'sum')" - } - ], - "title": "Cache size", - "type": "timeseries" - } - ], - "refresh": "15m", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Service info", - "uid": "weVPAU7Vk", - "version": 19, - "weekStart": "" - } \ No newline at end of file diff --git a/docker-compose.example.yml b/docker-compose.yml similarity index 56% rename from docker-compose.example.yml rename to docker-compose.yml index 54bce82..cf594be 100644 --- a/docker-compose.example.yml +++ b/docker-compose.yml @@ -16,8 +16,8 @@ services: graphite: image: graphiteapp/graphite-statsd volumes: + - ./config/graphite:/opt/graphite/conf - ./logs/graphite:/var/log - - ./grafanaConfig/graphite:/opt/graphite/conf - graphite_data:/opt/graphite/storage networks: - stats @@ -25,13 +25,20 @@ services: grafana: image: grafana/grafana-oss:9.3.6-ubuntu volumes: - - grafana_data:/var/lib/grafana - - ./grafanaConfig/grafana:/etc/grafana + - ./config/grafana/grafana.ini:/etc/grafana/grafana.ini + - ./config/grafana/provisioning:/etc/grafana/provisioning - ./logs/grafana:/var/log/grafana + - grafana_data:/var/lib/grafana ports: - - 3000:3000 + - ${GRAFANA_PORT}:3000 + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD} + - GF_DOMAIN=${HOSTNAME} + - GF_SMTP_FROM_ADDRESS=${EMAIL_ADDRESS} + - GF_AUTH_ANONYMOUS_ENABLED=${ANONYMOUS_AUTH_ENABLED} healthcheck: - test: "curl -fsSL -o /dev/null http://localhost:3000/login" + test: "curl -fsSL -o /dev/null http://localhost:${GRAFANA_PORT}/login" interval: 10s timeout: 5s retries: 3 @@ -44,12 +51,13 @@ services: dockerfile: ./src/pushStats/Dockerfile volumes: - ./logs/statsGetter:/app/logs + - ./users.json:/app/users.json depends_on: - graphite environment: - - PREFIX= - - SERVER_PORT=21025 - - INCLUDE_PUSH_STATUS_API=false + - PREFIX=${PREFIX} + - SERVER_PORT=${SERVER_PORT} + - PUSH_STATUS_PORT=${PUSH_STATUS_PORT} networks: - stats - logging: *default-logging \ No newline at end of file + logging: *default-logging diff --git a/example.env b/example.env deleted file mode 100644 index 6dccac6..0000000 --- a/example.env +++ /dev/null @@ -1,4 +0,0 @@ -SERVER_PORT=21025 -GRAFANA_PORT=3000 -COMPOSE_PROJECT_NAME=screeps-grafana -COMPOSE_FILE=./docker-compose.yml diff --git a/package-lock.json b/package-lock.json index 455223f..78c5ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "screeps-grafana-go_carbon", - "version": "1.0.5", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "screeps-grafana-go_carbon", - "version": "1.0.5", + "version": "1.1.0", "dependencies": { "axios": "^0.27.2", "dotenv": "^16.0.2", @@ -15,9 +15,6 @@ "minimist": "^1.2.7", "winston": "^3.8.1" }, - "bin": { - "screeps-grafana-go_carbon": "bin/setup.js" - }, "devDependencies": { "eslint": "^8.23.1", "eslint-config-airbnb-base": "^15.0.0", diff --git a/package.json b/package.json index 3af7bab..b3af475 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,15 @@ "name": "screeps-grafana-go_carbon", "version": "1.1.0", "scripts": { - "setup": "node bin/server.js setup", - "start": "node bin/server.js start --grafanaPort=3000", - "start:test": "node bin/server.js start --grafanaPort=3000 --force", - "lint": "eslint src/**/*.js && eslint dashboards/**/*.js", - "lint:fix": "eslint src/**/*.js --fix && eslint dashboards/**/*.js --fix", - "update-stats-getter": "docker-compose up --detach --build" + "start": "docker compose up -d", + "start:logs": "docker compose up -d && docker compose logs -ft", + "logs": "docker compose logs -ft", + "stop": "docker compose stop", + "reset": "docker compose down", + "reset:hard": "docker compose down -v", + "rebuild": "docker compose build --no-cache && docker compose down stats-getter && docker compose up -d stats-getter", + "lint": "eslint src/**/*.js", + "lint:fix": "eslint src/**/*.js --fix" }, "dependencies": { "axios": "^0.27.2", @@ -22,17 +25,12 @@ "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-import": "^2.26.0" }, - "bin": { - "screeps-grafana-go_carbon": "./bin/server.js" - }, "files": [ "src", - "dashboards", - "grafanaConfig.example", + "config", "users.example.json", - "docker-compose.example.yml", - "example.env", - ".dockerignore", - "go-carbon-storage" + "docker-compose.yml", + "env.example", + ".dockerignore" ] } diff --git a/src/deletePath.js b/src/deletePath.js deleted file mode 100644 index c3f79a9..0000000 --- a/src/deletePath.js +++ /dev/null @@ -1,41 +0,0 @@ -const fs = require('fs'); -const { join } = require('path'); - -const minimist = require('minimist'); - -const argv = minimist(process.argv.slice(2)); -console.dir(argv); - -const whisperPath = join(__dirname, '../whisper/'); -let { statsPath } = argv; -if (!statsPath) { - console.error('Please provide a path to the stats'); - process.exit(1); -} - -if (!fs.existsSync(whisperPath)) { - console.log('No whisper folder found, manually delete the stats are not working while whisper export is disabled.'); - process.exit(1); -} else if (statsPath.startsWith('.')) { - console.log('Please provide a path without a leading dot'); - process.exit(1); -} else if (statsPath.endsWith('.')) { - console.log('Please provide a path without a trailing dot'); - process.exit(1); -} -statsPath = statsPath.split('.').join('/'); - -function deletePath(path) { - if (fs.existsSync(path)) { - fs.rm(path, { recursive: true }, (err) => { - if (err) { - console.error(err); - process.exit(1); - } - console.log(`Deleted ${path}`); - }); - return; - } - console.log(`Path not found: ${path}`); -} -deletePath(join(whisperPath, statsPath)); diff --git a/src/pushStats/Dockerfile b/src/pushStats/Dockerfile index 0d43f8d..308f935 100644 --- a/src/pushStats/Dockerfile +++ b/src/pushStats/Dockerfile @@ -6,7 +6,6 @@ ENV NODE_ENV=production COPY ./src/pushStats . -COPY ./users.json ./users.json -RUN npm install +RUN npm clean-install CMD ["node", "index.js"] \ No newline at end of file diff --git a/src/pushStats/apiFunctions.js b/src/pushStats/apiFunctions.js index c7356d8..334190a 100644 --- a/src/pushStats/apiFunctions.js +++ b/src/pushStats/apiFunctions.js @@ -1,26 +1,13 @@ import http from 'http'; import https from 'https'; -import net from 'net'; import util from 'util'; import zlib from 'zlib'; -import fs from 'fs'; -// import users from './users.json' assert {type: 'json'}; -import { fileURLToPath } from 'url'; -import * as dotenv from 'dotenv'; -import { join, dirname } from 'path'; import { createLogger, format, transports } from 'winston'; // eslint-disable-next-line import/no-unresolved import 'winston-daily-rotate-file'; -const users = JSON.parse(fs.readFileSync('users.json')); - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -dotenv.config({ path: join(__dirname, './.env') }); -const needsPrivateHost = users.some((u) => u.type !== 'mmo' && !u.host); - const gunzipAsync = util.promisify(zlib.gunzip); const { combine, timestamp, prettyPrint } = format; @@ -40,6 +27,11 @@ const logger = createLogger({ transports: [transport], }); +/** + * + * @param {string} data + * @returns + */ async function gz(data) { if (!data) return {}; const buf = Buffer.from(data.slice(3), 'base64'); @@ -47,6 +39,11 @@ async function gz(data) { return JSON.parse(ret.toString()); } +/** + * + * @param {any} obj + * @returns + */ function removeNonNumbers(obj) { if (!obj) return obj; @@ -64,59 +61,17 @@ function removeNonNumbers(obj) { return obj; } -let privateHost; -let serverPort = 21025; - -function getPrivateHost() { - serverPort = process.env.SERVER_PORT || 21025; - const hosts = [ - 'localhost', - 'host.docker.internal', - '172.17.0.1', - ]; - for (let h = 0; h < hosts.length; h += 1) { - const host = hosts[h]; - const sock = new net.Socket(); - sock.setTimeout(2500); - // eslint-disable-next-line no-loop-func - sock.on('connect', () => { - sock.destroy(); - privateHost = host; - }) - .on('error', () => { - sock.destroy(); - }) - .on('timeout', () => { - sock.destroy(); - }) - .connect(serverPort, host); - } -} - -async function TryToGetPrivateHost() { - if (!privateHost && needsPrivateHost) { - getPrivateHost(); - if (!privateHost) console.log('No private host found to make connection with yet! Trying again in 60 seconds.'); - else console.log(`Private host found! Continuing with ${privateHost}.`); - - // eslint-disable-next-line - await new Promise((resolve) => setTimeout(resolve, 60 * 1000)); - TryToGetPrivateHost(); - } -} - -if (!privateHost && needsPrivateHost) { - TryToGetPrivateHost(); -} - -async function getHost(host, type) { - if (type === 'mmo') return 'screeps.com'; - if (type === 'season') return 'screeps.com/season'; - if (host) return host; - return privateHost; -} - -async function getRequestOptions(info, path, method = 'GET', body = {}) { +/** + * + * @param {Omit & { token?: string }} info + * @param {string} path + * @param {'GET'|'POST'} method + * @param {{}} body + * @returns {http.RequestOptions & {body: {}, isHTTPS: boolean}} + */ +function getRequestOptions(info, path, method = 'GET', body = {}) { + /** @type {Record} */ const headers = { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(JSON.stringify(body)), @@ -125,8 +80,8 @@ async function getRequestOptions(info, path, method = 'GET', body = {}) { if (info.username) headers['X-Username'] = info.username; if (info.token) headers['X-Token'] = info.token; return { - host: await getHost(info.host, info.type), - port: info.type === 'mmo' ? 443 : serverPort, + host: info.host, + port: info.port, path, method, headers, @@ -134,6 +89,12 @@ async function getRequestOptions(info, path, method = 'GET', body = {}) { isHTTPS: info.type === 'mmo', }; } + +/** + * + * @param {https.RequestOptions & { body?: {}, isHTTPS?: boolean }} options + * @returns + */ async function req(options) { const reqBody = JSON.stringify(options.body); const { isHTTPS } = options; @@ -171,12 +132,13 @@ async function req(options) { .then((result) => { if (result === 'Timeout') { logger.log('info', 'Timeout hit!', new Date(), JSON.stringify(options), reqBody); - return; + return undefined; + } + if (typeof result === 'string' && result.startsWith('Rate limit exceeded')) { + logger.log('error', { data: result, options }); + } else { + logger.log('info', { data: `${JSON.stringify(result).length / 1000} MB`, options }); } - // is result string - if (typeof result === 'string' && result.startsWith('Rate limit exceeded')) logger.log('error', { data: result, options }); - else logger.log('info', { data: `${JSON.stringify(result).length / 1000} MB`, options }); - // eslint-disable-next-line consistent-return return result; }) .catch((result) => { @@ -186,8 +148,13 @@ async function req(options) { } export default class { + /** + * + * @param {UserInfo} info + * @returns + */ static async getPrivateServerToken(info) { - const options = await getRequestOptions({ type: 'private', username: info.username, host: info.host }, '/api/auth/signin', 'POST', { + const options = getRequestOptions(info, '/api/auth/signin', 'POST', { email: info.username, password: info.password, }); @@ -196,13 +163,18 @@ export default class { return res.token; } + /** + * + * @param {UserInfo} info + * @param {string} shard + * @param {string} statsPath + * @returns + */ static async getMemory(info, shard, statsPath = 'stats') { - const options = await getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); + const options = getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); const res = await req(options); - if (res) { - console.log(`Got memory from ${info.username} in ${shard} `); - } else { + if (!res) { return undefined; } @@ -210,8 +182,14 @@ export default class { return data; } + /** + * + * @param {UserInfo} info + * @param {string} shard + * @returns + */ static async getSegmentMemory(info, shard) { - const options = await getRequestOptions(info, `/api/user/memory-segment?segment=${info.segment}&shard=${shard}`, 'GET'); + const options = getRequestOptions(info, `/api/user/memory-segment?segment=${info.segment}&shard=${shard}`, 'GET'); const res = await req(options); if (!res || res.data == null) return {}; try { @@ -222,21 +200,36 @@ export default class { } } + /** + * + * @param {UserInfo} info + * @returns + */ static async getUserinfo(info) { - const options = await getRequestOptions(info, '/api/auth/me', 'GET'); + const options = getRequestOptions(info, '/api/auth/me', 'GET'); const res = await req(options); return res; } + /** + * + * @param {UserInfo} info + * @returns + */ static async getLeaderboard(info) { - const options = await getRequestOptions(info, `/api/leaderboard/find?username=${info.username}&mode=world`, 'GET'); + const options = getRequestOptions(info, `/api/leaderboard/find?username=${info.username}&mode=world`, 'GET'); const res = await req(options); return res; } - static async getServerStats(host) { - const serverHost = host || privateHost; - const options = await getRequestOptions({ host: serverHost }, '/api/stats/server', 'GET'); + /** + * + * @param {string | undefined} host + * @param {number} port + * @returns + */ + static async getServerStats(host, port) { + const options = getRequestOptions(/** @type {UserInfo} */ ({ host, port }), '/api/stats/server', 'GET'); const res = await req(options); if (!res || !res.users) { logger.error(res); @@ -245,9 +238,14 @@ export default class { return removeNonNumbers(res); } - static async getAdminUtilsServerStats(host) { - const serverHost = host || privateHost; - const options = await getRequestOptions({ host: serverHost }, '/stats', 'GET'); + /** + * + * @param {string | undefined} host + * @param {number} port + * @returns + */ + static async getAdminUtilsServerStats(host, port) { + const options = getRequestOptions(/** @type {UserInfo} */ ({ host, port }), '/stats', 'GET'); const res = await req(options); if (!res || !res.gametime) { logger.error(res); @@ -255,7 +253,9 @@ export default class { } delete res.ticks.ticks; + /** @type {Record} */ const mUsers = {}; + // @ts-expect-error res.users.forEach((user) => { mUsers[user.username] = user; }); diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 06d20b8..6f6c4d9 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -5,19 +5,15 @@ import graphite from 'graphite'; import { createLogger, format, transports } from 'winston'; // eslint-disable-next-line import/no-unresolved import 'winston-daily-rotate-file'; -import fs from 'fs'; -import * as dotenv from 'dotenv'; // eslint-disable-next-line import/no-unresolved import express from 'express'; import ApiFunc from './apiFunctions.js'; +import loadUsers from './users.js'; const app = express(); -const port = 10004; +const pushStatusPort = Number(process.env.PUSH_STATUS_PORT); let lastUpload = new Date().getTime(); -const users = JSON.parse(fs.readFileSync('users.json')); -dotenv.config(); - const pushTransport = new transports.DailyRotateFile({ filename: 'logs/push-%DATE%.log', auditFile: 'logs/push-audit.json', @@ -54,108 +50,100 @@ const cronLogger = createLogger({ }); class ManageStats { + /** @type {Record} */ groupedStats; - message; - constructor() { this.groupedStats = {}; - this.message = '----------------------------------------------------------------\r\n'; } - async handleUsers(type) { - console.log(`[${type}] Handling Users`); + /** + * + * @param {string} host + * @param {UserInfo[]} hostUsers + * @returns + */ + async handleUsers(host, hostUsers) { + console.log(`[${host}] Handling Users`); const beginningOfMinute = new Date().getSeconds() < 15; + /** @type {(Promise)[]} */ const getStatsFunctions = []; - users.forEach((user) => { + for (const user of hostUsers) { try { - if (user.type !== type) return; + if (user.host !== host) continue; const rightMinuteForShard = new Date().getMinutes() % user.shards.length === 0; const shouldContinue = !beginningOfMinute || !rightMinuteForShard; - if (user.type === 'mmo' && shouldContinue) return; - if (user.type === 'season' && shouldContinue) return; + if (user.type === 'mmo' && shouldContinue) continue; + if (user.type === 'season' && shouldContinue) continue; - for (let y = 0; y < user.shards.length; y += 1) { - const shard = user.shards[y]; - getStatsFunctions.push(this.getStats(user, shard, this.message)); + for (const shard of user.shards) { + getStatsFunctions.push(this.getStats(user, shard)); } } catch (error) { - logger.error(error.message); - } - }); - - console.log(`[${type}] Getting ${getStatsFunctions.length} statistics`); - - await Promise.all(getStatsFunctions); - - const { groupedStats } = this; - - if (type === 'mmo') { - if (Object.keys(groupedStats).length > 0) { - if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - - console.log(`[${type}] Pushed stats to graphite`); - - return console.log(this.message); + logger.error(error); } - - if (beginningOfMinute) return console.log('No stats to push'); - return undefined; } - if (type === 'season') { - if (Object.keys(groupedStats).length > 0) { - if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - console.log(`[${type}] Pushed stats to graphite`); + console.log(`[${host}] Getting ${getStatsFunctions.length} statistics`); - return console.log(this.message); - } - if (beginningOfMinute) return console.log('No stats to push'); - return undefined; - } + await Promise.all(getStatsFunctions); - const privateUser = users.find((user) => user.type === 'private' && user.host); - const host = privateUser ? privateUser.host : undefined; - const serverStats = await ApiFunc.getServerStats(host); - const adminUtilsServerStats = await ApiFunc.getAdminUtilsServerStats(host); - if (adminUtilsServerStats) { - try { - const groupedAdminStatsUsers = {}; - for (const [username, user] of Object.entries(adminUtilsServerStats)) { - groupedAdminStatsUsers[username] = user; + /** @type {Record} */ + const stats = { + stats: this.groupedStats, + }; + + if (!host.startsWith('screeps.com')) { + const serverStats = await ApiFunc.getServerStats(host, hostUsers[0].port); + const adminUtilsServerStats = await ApiFunc.getAdminUtilsServerStats(host, hostUsers[0].port); + if (adminUtilsServerStats) { + try { + /** @type {Record} */ + const groupedAdminStatsUsers = {}; + for (const [username, user] of Object.entries(adminUtilsServerStats)) { + groupedAdminStatsUsers[username] = user; + } + + adminUtilsServerStats.users = groupedAdminStatsUsers; + } catch (error) { + console.log(error); } - - adminUtilsServerStats.users = groupedAdminStatsUsers; - } catch (error) { - console.log(error); } + console.log(`[${host}] Server stats: ${serverStats ? 'yes' : 'no'}, adminUtils: ${adminUtilsServerStats ? 'yes' : 'no'}`); + stats.serverStats = serverStats; + stats.adminUtilsServerStats = adminUtilsServerStats; } - if (!await ManageStats.reportStats({ stats: groupedStats, serverStats, adminUtilsServerStats })) return console.log('Error while pushing stats'); - let statsPushed = ''; - if (Object.keys(groupedStats).length > 0) { - statsPushed = `Pushed ${type} stats`; + const push = await ManageStats.reportStats(stats); + if (!push) { + console.log(`[${host}] Error while pushing stats`); + return; } - if (serverStats) { - statsPushed += statsPushed.length > 0 ? ', server stats' : 'Pushed server stats'; + /** @type {string[]} */ + const typesPushed = []; + if (Object.keys(stats.stats).length > 0) { + typesPushed.push(host); } - if (adminUtilsServerStats) { - statsPushed += statsPushed.length > 0 ? ', adminUtilsServerStats' : 'Pushed server stats'; + if (stats.serverStats) { + typesPushed.push('server stats'); } - this.message += statsPushed.length > 0 ? `> ${statsPushed} to graphite` : '> Pushed no stats to graphite'; - logger.info(this.message); - return console.log(this.message); - } - - static async getLoginInfo(userinfo) { - if (userinfo.type === 'private') { - userinfo.token = await ApiFunc.getPrivateServerToken(userinfo); + if (stats.adminUtilsServerStats) { + typesPushed.push('admin-utils stats'); + } + if (typesPushed.length) { + logger.info(`> [${host}] Pushed ${typesPushed.join(', ')}`); + } else { + logger.info(`> [${host}] Pushed no stats`); } - return userinfo.token; } + /** + * + * @param {UserInfo} userinfo + * @returns {Promise<{ rank: number, score: number }>} + */ static async addLeaderboardData(userinfo) { try { const leaderboard = await ApiFunc.getLeaderboard(userinfo); @@ -169,31 +157,51 @@ class ManageStats { } } - async getStats(userinfo, shard) { - try { - await ManageStats.getLoginInfo(userinfo); - const stats = userinfo.segment === undefined - ? await ApiFunc.getMemory(userinfo, shard) - : await ApiFunc.getSegmentMemory(userinfo, shard); - - await this.processStats(userinfo, shard, stats); - return 'success'; - } catch (error) { - return error; + /** + * + * @param {UserInfo} userinfo + * @returns + */ + static async getLoginInfo(userinfo) { + if (userinfo.type === 'private') { + userinfo.token = await ApiFunc.getPrivateServerToken(userinfo); } + return userinfo.token; } - async processStats(userinfo, shard, stats) { + /** + * + * @param {UserInfo} userinfo + * @param {string} shard + * @returns {Promise} + */ + async getStats(userinfo, shard) { + await ManageStats.getLoginInfo(userinfo); + const stats = userinfo.segment === undefined + ? await ApiFunc.getMemory(userinfo, shard) + : await ApiFunc.getSegmentMemory(userinfo, shard); + if (Object.keys(stats).length === 0) return; + + console.log(`Got memory from ${userinfo.username} in ${shard}`); + const me = await ApiFunc.getUserinfo(userinfo); if (me) stats.power = me.power || 0; stats.leaderboard = await ManageStats.addLeaderboardData(userinfo); this.pushStats(userinfo, stats, shard); } + /** + * + * @param {*} stats + * @returns + */ static async reportStats(stats) { return new Promise((resolve) => { - console.log(`Writing stats ${JSON.stringify(stats)} to graphite`); + if (Object.keys(stats).length === 0) { + resolve(false); + } + console.debug(`Writing stats ${JSON.stringify(stats)}`); client.write({ [`${process.env.PREFIX ? `${process.env.PREFIX}.` : ''}screeps`]: stats }, (err) => { if (err) { console.log(err); @@ -203,43 +211,56 @@ class ManageStats { lastUpload = new Date().getTime(); resolve(true); }); - // resolve(true); }); } + /** + * + * @param {UserInfo} userinfo + * @param {*} stats + * @param {string} shard + * @returns + */ pushStats(userinfo, stats, shard) { - if (Object.keys(stats).length === 0) return; - const username = userinfo.replaceName !== undefined ? userinfo.replaceName : userinfo.username; - this.groupedStats[(userinfo.prefix ? `${userinfo.prefix}.` : '') + username] = { [shard]: stats }; - - console.log(`Pushing stats for ${(userinfo.prefix ? `${userinfo.prefix}.` : '') + username} in ${shard}`); + const statSize = Object.keys(stats).length; + if (statSize === 0) return; + const username = userinfo.replaceName ? userinfo.replaceName : userinfo.username; + const userStatsKey = (userinfo.prefix ? `${userinfo.prefix}.` : '') + username; + + console.log(`[${userinfo.host}] Pushing ${statSize} stats for ${userStatsKey} in ${shard}`); + if (!this.groupedStats[userStatsKey]) { + this.groupedStats[userStatsKey] = { [shard]: stats }; + } else { + this.groupedStats[userStatsKey][shard] = stats; + } } } -const groupedUsers = users.reduce((group, user) => { - const { type } = user; - // eslint-disable-next-line no-param-reassign - group[type] = group[type] ?? []; - group[type].push(user); - return group; -}, {}); - cron.schedule('*/30 * * * * *', async () => { - const message = `Cron event hit: ${new Date()}`; - console.log(`\r\n${message}\n`); - cronLogger.info(message); - Object.keys(groupedUsers).forEach((type) => { - new ManageStats(groupedUsers[type]).handleUsers(type); - }); + console.log(`Cron event hit: ${new Date()}`); + cronLogger.info(`Cron event hit: ${new Date()}`); + /** @type {UserInfo[]} */ + const users = await loadUsers(); + + const usersByHost = users.reduce((group, user) => { + const { host } = user; + group[host] = group[host] ?? []; + group[host].push(user); + return group; + }, /** @type {Record} */ ({})); + + for (const [host, usersForHost] of Object.entries(usersByHost)) { + new ManageStats().handleUsers(host, usersForHost); + } }); -if (process.env.INCLUDE_PUSH_STATUS_API === 'true') { - app.listen(port, () => { - console.log(`App listening at http://localhost:${port}`); +if (pushStatusPort) { + app.listen(pushStatusPort, () => { + console.log(`App listening at http://localhost:${pushStatusPort}`); }); app.get('/', (req, res) => { const diffCompleteMinutes = Math.ceil( - Math.abs(parseInt(new Date().getTime(), 10) - parseInt(lastUpload, 10)) / (1000 * 60), + Math.abs(new Date().getTime() - lastUpload) / (1000 * 60), ); res.json({ result: diffCompleteMinutes < 300, lastUpload, diffCompleteMinutes }); }); diff --git a/src/pushStats/package-lock.json b/src/pushStats/package-lock.json index 9a00fe2..b654e04 100644 --- a/src/pushStats/package-lock.json +++ b/src/pushStats/package-lock.json @@ -6,12 +6,17 @@ "": { "dependencies": { "axios": "^0.27.2", - "dotenv": "^16.0.2", "express": "^4.18.2", "graphite": "^0.1.4", "node-cron": "^3.0.1", "winston": "^3.8.1", "winston-daily-rotate-file": "^4.7.1" + }, + "devDependencies": { + "@types/express": "^5.0.0", + "@types/graphite": "^0.1.2", + "@types/node": "^22.7.4", + "@types/node-cron": "^3.0.11" } }, "node_modules/@colors/colors": { @@ -32,6 +37,128 @@ "kuler": "^2.0.0" } }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", + "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/graphite": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@types/graphite/-/graphite-0.1.2.tgz", + "integrity": "sha512-k+esqcUwtDaAZbTXf6J06Z82ZNbndJohxh9/PqvtWwYtqJZxSzpD7HBR7oN5ADXGE67MbqDCs1+1/TpJxIz2zQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.4.tgz", + "integrity": "sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/qs": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -232,17 +359,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/dotenv": { - "version": "16.3.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", - "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/motdotla/dotenv?sponsor=1" - } - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -882,6 +998,13 @@ "node": ">= 0.6" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -987,6 +1110,115 @@ "kuler": "^2.0.0" } }, + "@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "dev": true, + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", + "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "@types/graphite": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@types/graphite/-/graphite-0.1.2.tgz", + "integrity": "sha512-k+esqcUwtDaAZbTXf6J06Z82ZNbndJohxh9/PqvtWwYtqJZxSzpD7HBR7oN5ADXGE67MbqDCs1+1/TpJxIz2zQ==", + "dev": true + }, + "@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true + }, + "@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true + }, + "@types/node": { + "version": "22.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.4.tgz", + "integrity": "sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==", + "dev": true, + "requires": { + "undici-types": "~6.19.2" + } + }, + "@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, + "@types/qs": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", + "dev": true + }, + "@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true + }, + "@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "requires": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -1152,11 +1384,6 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, - "dotenv": { - "version": "16.3.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", - "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==" - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1634,6 +1861,12 @@ "mime-types": "~2.1.24" } }, + "undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/src/pushStats/package.json b/src/pushStats/package.json index 5bff340..c87dfec 100644 --- a/src/pushStats/package.json +++ b/src/pushStats/package.json @@ -1,12 +1,17 @@ { "dependencies": { "axios": "^0.27.2", - "dotenv": "^16.0.2", "express": "^4.18.2", "graphite": "^0.1.4", "node-cron": "^3.0.1", "winston": "^3.8.1", "winston-daily-rotate-file": "^4.7.1" }, - "type": "module" + "type": "module", + "devDependencies": { + "@types/express": "^5.0.0", + "@types/graphite": "^0.1.2", + "@types/node": "^22.7.4", + "@types/node-cron": "^3.0.11" + } } diff --git a/src/pushStats/types.d.ts b/src/pushStats/types.d.ts new file mode 100644 index 0000000..2fd4cca --- /dev/null +++ b/src/pushStats/types.d.ts @@ -0,0 +1,14 @@ +type UserType = "mmo"|"season"|"private"; + +interface UserInfo { + type: UserType; + username: string; + host: string; + port: number; + replaceName: string; + password: string; + token: string; + prefix: string; + segment: number; + shards: string[]; +} diff --git a/src/pushStats/users.js b/src/pushStats/users.js new file mode 100644 index 0000000..ef3b781 --- /dev/null +++ b/src/pushStats/users.js @@ -0,0 +1,105 @@ +import fs from 'fs'; +import net from 'net'; + +const SERVER_PORT = parseInt(/** @type {string} */ (process.env.SERVER_PORT), 10) ?? 21025; + +/** + * Check whether there's a server nearby + * @returns {Promise<[string, number]>} + */ +async function checkLocalhostServer() { + const hosts = [ + 'localhost', + 'host.docker.internal', + '172.17.0.1', + ]; + /** @type {Promise<[string, number]>[]} */ + const promises = []; + for (const host of hosts) { + const p = new Promise((resolve, reject) => { + const sock = new net.Socket(); + sock.setTimeout(2500); + console.log(`[${host}:${SERVER_PORT}] attempting connection`); + sock + .on('connect', () => { + sock.destroy(); + resolve([host, SERVER_PORT]); + }) + .on('error', () => { + sock.destroy(); + reject(); + }) + .on('timeout', () => { + sock.destroy(); + reject(); + }) + .connect(SERVER_PORT, host); + }); + promises.push(p); + } + return Promise.any(promises); +} + +/** + * + * @param {UserType} type + * @returns {[string, number]} + */ +function getHostInfoFromType(type) { + switch (type) { + case 'mmo': + return ['screeps.com', 443]; + case 'season': + return ['screeps.com/season', 443]; + default: + throw new Error(`no idea what type ${type} is`); + } +} + +export default async function loadUsers() { + /** @type {UserInfo[]} */ + const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); + /** @type {UserInfo[]} */ + const validUsers = []; + const localServer = await checkLocalhostServer(); + for (const user of users) { + if (typeof user.username !== 'string' || user.username.length <= 0) { + console.log('Missing username!'); + continue; + } + if (user.username.includes('.') && !user.replaceName) { + // Just yank the dot from the name + user.replaceName = user.username.replace(/\./g, ''); + } + if (user.type && !['mmo', 'season', 'private'].includes(user.type)) { + console.log(`Invalid type for user ${user.username}, ignoring.`); + continue; + } + if (!user.host) { + try { + if (user.type === 'private') { + [user.host, user.port] = localServer; + } else { + [user.host, user.port] = getHostInfoFromType(user.type); + } + } catch { + console.log(`Cannot get host for user ${user.username}, ignoring.`); + continue; + } + } + if (!user.host || !user.port) { + console.log(`Missing host or port for user ${user.username}, ignoring.`); + continue; + } + if (!user.password && !user.token) { + console.log(`Missing password or token for user ${user.username}, ignoring.`); + continue; + } + if (!Array.isArray(user.shards) || !user.shards.every((s) => typeof s === 'string')) { + console.log(`Missing or invalid shard for user ${user.username}, ignoring.`); + continue; + } + validUsers.push(user); + } + return validUsers; +} diff --git a/src/setup/setup.js b/src/setup/setup.js deleted file mode 100644 index ef8d682..0000000 --- a/src/setup/setup.js +++ /dev/null @@ -1,181 +0,0 @@ -const fs = require('fs'); -const fse = require('fs-extra'); -const { join } = require('path'); -const { execSync } = require('child_process'); - -let argv; -/** @type {import('winston').Logger} */ -let logger; - -const isWindows = process.platform === 'win32'; -const regexEscape = isWindows ? '\r\n' : '\n'; - -function createRegexWithEscape(string) { - return new RegExp(string.replace('\r\n', regexEscape)); -} - -function UpdateEnvFile() { - const envFile = join(__dirname, '../../.env'); - if (fs.existsSync(envFile) && !argv.force) { - return logger.warn('Env file already exists, use --force to overwrite it'); - } - - const exampleEnvFilePath = join(__dirname, '../../example.env'); - let contents = fs.readFileSync(exampleEnvFilePath, 'utf8'); - contents = contents - .replace('GRAFANA_PORT=3000', `GRAFANA_PORT=${argv.grafanaPort}`) - .replace('COMPOSE_PROJECT_NAME=screeps-grafana', `COMPOSE_PROJECT_NAME=screeps-grafana-${argv.grafanaPort}`) - .replace('COMPOSE_FILE=./docker-compose.yml', `COMPOSE_FILE=${join(__dirname, '../../docker-compose.yml')}`); - if (argv.serverPort) { - contents = contents.replace('SERVER_PORT=21025', `SERVER_PORT=${argv.serverPort}`); - } - - fs.writeFileSync(envFile, contents); - logger.info('Env file created'); -} - -async function UpdateDockerComposeFile() { - const dockerComposeFile = join(__dirname, '../../docker-compose.yml'); - if (fs.existsSync(dockerComposeFile) && !argv.force) { - return logger.warn('Docker-compose file already exists, use --force to overwrite it'); - } - - const exampleDockerComposeFile = join(__dirname, '../../docker-compose.example.yml'); - let contents = fs.readFileSync(exampleDockerComposeFile, 'utf8'); - contents = contents.replace('3000:3000', `${argv.grafanaPort}:${argv.grafanaPort}`); - contents = contents.replace('http://localhost:3000/login', `http://localhost:${argv.grafanaPort}/login`); - - if (argv.relayPort) { - contents = contents.replace('2003:2003', `${argv.relayPort}:2003`); - } else { - contents = contents.replace(createRegexWithEscape('ports:\r\n - 2003:2003'), ''); - } - if (argv.serverPort) { - contents = contents - .replace('http://localhost:21025/web', `http://localhost:${argv.serverPort}/web`) - .replace('SERVER_PORT: 21025', `SERVER_PORT: ${argv.serverPort}`); - } - if (argv.pushStatusPort) { - contents = contents.replace( - 'INCLUDE_PUSH_STATUS_API=false', - `INCLUDE_PUSH_STATUS_API=true${regexEscape} ports:${regexEscape} - ${argv.pushStatusPort}:${argv.pushStatusPort}`, - ); - } - if (argv.prefix) { - contents = contents.replace('PREFIX=', `PREFIX=${argv.prefix}`); - } - - fs.writeFileSync(dockerComposeFile, contents); - logger.info('Docker-compose file created'); -} - -function UpdateUsersFile() { - const usersFile = join(__dirname, '../../users.json'); - if (fs.existsSync(usersFile) && !argv.force) { - return logger.warn('Users file already exists, use --force to overwrite it'); - } - - const exampleUsersFilePath = join(__dirname, '../../users.example.json'); - const exampleUsersText = fs.readFileSync(exampleUsersFilePath, 'utf8'); - fs.writeFileSync(usersFile, exampleUsersText); - logger.info('Users file created'); -} - -function UpdateGrafanaConfigFolder() { - const configDirPath = join(__dirname, '../../grafanaConfig'); - if (fs.existsSync(configDirPath) && !argv.force) { - return logger.warn('Grafana config folder already exists, use --force to overwrite it'); - } - - fse.copySync(join(__dirname, '../../grafanaConfig.example'), configDirPath); - const grafanaIniFile = join(configDirPath, './grafana/grafana.ini'); - let grafanaIniText = fs.readFileSync(grafanaIniFile, 'utf8'); - - if (argv.username) grafanaIniText = grafanaIniText.replace(/admin_user = (.*)/, `admin_user = ${argv.username}`); - if (argv.password) grafanaIniText = grafanaIniText.replace(/admin_password = (.*)/, `admin_password = ${argv.password}`); - if (argv.grafanaDomain) { - grafanaIniText = grafanaIniText.replace('domain = localhost', `domain = ${argv.grafanaDomain}`); - grafanaIniText = grafanaIniText.replace('from_address = admin@localhost', `from_address = admin@${argv.grafanaDomain}`); - } - if (argv.grafanaPort) { - grafanaIniText = grafanaIniText.replace('http_port = 3000', `http_port = ${argv.grafanaPort}`); - } - grafanaIniText = grafanaIniText.replace( - createRegexWithEscape('enable anonymous access\r\nenabled = (.*)'), - `enable anonymous access${regexEscape}enabled = ${argv.enableAnonymousAccess ? 'true' : 'false'}`, - ); - fs.writeFileSync(grafanaIniFile, grafanaIniText); - - // This can just be set manually in the config folder. - /* - const storageSchemasFile = join(grafanaConfigFolder, './go-carbon/storage-schemas.conf'); - let storageSchemasText = fs.readFileSync(storageSchemasFile, 'utf8'); - const { defaultRetention } = argv; - - if (defaultRetention) { - storageSchemasText = storageSchemasText.replace( - createRegexWithEscape('pattern = .*\r\nretentions = (.*)'), - `pattern = .*${regexEscape}retentions = ${defaultRetention}`, - ); - } - fs.writeFileSync(storageSchemasFile, storageSchemasText); - */ - - logger.info('Grafana config folder created'); -} - -function resetFolders() { - const carbonStoragePath = join(__dirname, '../../go-carbon-storage'); - let carbonStorageExists = fs.existsSync(carbonStoragePath); - if (carbonStorageExists && argv.removeWhisper) { - fs.rmdirSync(carbonStoragePath, { recursive: true }); - carbonStorageExists = false; - } - if (!carbonStorageExists) { - fs.mkdirSync(carbonStoragePath, { recursive: true }); - } - - const logsPath = join(__dirname, '../../logs'); - let logsExist = fs.existsSync(logsPath); - if (logsExist && argv.deleteLogs) { - fs.rmdirSync(logsPath, { recursive: true }); - logsExist = false; - } - if (!logsExist) fs.mkdirSync(logsPath, { recursive: true }); -} - -async function Setup(cli) { - argv = cli.args; - logger = cli.logger; - - UpdateUsersFile(); - UpdateEnvFile(); - await UpdateDockerComposeFile(); - UpdateGrafanaConfigFolder(); -} - -module.exports = Setup; - -module.exports.commands = async function Commands(grafanaApiUrl) { - logger.info(`Grafana API URL: ${grafanaApiUrl}, serverPort: ${argv.serverPort}`); - - const commands = [ - { command: `docker compose down ${argv.removeVolumes ? '--volumes' : ''} --remove-orphans`, name: 'docker-compose down' }, - { command: 'docker compose build --no-cache', name: 'docker-compose build' }, - { command: 'docker compose up -d', name: 'docker-compose up' }, - ]; - - logger.info('Executing start commands:'); - for (let i = 0; i < commands.length; i += 1) { - const commandInfo = commands[i]; - try { - logger.info(`Running command ${commandInfo.name}`); - execSync(commandInfo.command, { stdio: argv.debug ? 'inherit' : 'ignore' }); - if (commandInfo.name.startsWith('docker-compose down')) resetFolders(); - } catch (error) { - logger.error(`Command ${commandInfo.name} errored`, error); - logger.error('Stopping setup'); - process.exit(1); - } - } -}; diff --git a/src/setup/start.js b/src/setup/start.js deleted file mode 100644 index 72bcb4d..0000000 --- a/src/setup/start.js +++ /dev/null @@ -1,67 +0,0 @@ -const dotenv = require('dotenv'); -const axios = require('axios'); -const { join } = require('path'); -const fs = require('fs'); - -let grafanaApiUrl; - -const setup = require('./setup.js'); -const getDashboards = require('../../dashboards/helper.js'); - -/** @type {import('winston').Logger} */ -let logger; - -function sleep(milliseconds) { - // eslint-disable-next-line no-promise-executor-return - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -const dashboards = getDashboards(); -let adminLogin; - -function handleSuccess(type) { - logger.info(`${type} dashboard setup done`); -} - -function handleError(type, err) { - logger.error(`${type} dashboard error: `, err); -} - -async function SetupServiceInfoDashboard() { - const type = 'Service-Info'; - try { - const dashboard = dashboards.serviceInfo; - await axios({ - url: `${grafanaApiUrl}/dashboards/db`, - method: 'post', - auth: adminLogin, - data: dashboard, - }); - handleSuccess(type); - } catch (err) { - handleError(type, err); - } -} - -async function Start(cli) { - logger = cli.logger; - await setup(cli); - dotenv.config({ path: join(__dirname, '../../.env') }); - - const grafanaIni = fs.readFileSync(join(__dirname, '../../grafanaConfig/grafana/grafana.ini'), 'utf8'); - const username = grafanaIni.match(/admin_user = (.*)/)[1]; - const password = grafanaIni.match(/admin_password = (.*)/)[1]; - adminLogin = { username, password }; - - dotenv.config({ path: join(__dirname, '../../grafanaConfig/.env.grafana') }); - - grafanaApiUrl = `http://localhost:${cli.args.grafanaPort}/api`; - await setup.commands(grafanaApiUrl); - logger.info('Pre setup done! Waiting for Grafana to start...'); - await sleep(30 * 1000); - - await SetupServiceInfoDashboard(); - logger.info('Setup done!'); -} - -module.exports = Start; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..9f1f160 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,109 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "Node16", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + "rootDirs": [".", "src/pushStats"], /* Allow multiple folders to be treated as one when resolving modules. */ + "typeRoots": ["./node_modules/@types", "./src/pushStats/node_modules/@types"], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +}