merge: add freebind support (#490)

Dockerfile

@@ -1,14 +1,13 @@
 FROM node:18-bullseye-slim
 WORKDIR /app
 
-RUN apt-get update
-RUN apt-get install -y git
-RUN rm -rf /var/lib/apt/lists/*
-
 COPY package*.json ./
-RUN npm install
 
-RUN git clone -n https://github.com/imputnet/cobalt.git --depth 1 && mv cobalt/.git ./ && rm -rf cobalt
+RUN apt-get update && \
+    apt-get install -y git python3 build-essential && \
+    npm install && \
+    apt purge --autoremove -y python3 build-essential && \
+    rm -rf ~/.cache/ /var/lib/apt/lists/*
 
 COPY . .
 EXPOSE 9000

docs/run-an-instance.md

@@ -53,13 +53,15 @@ sudo service nscd start
 | variable name         | default   | example                 | description |
 |:----------------------|:----------|:------------------------|:------------|
 | `API_PORT`            | `9000`    |  `9000`                 | changes port from which api server is accessible. |
+| `API_LISTEN_ADDRESS`  | `0.0.0.0` |  `127.0.0.1`            | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
 | `API_URL`             | ➖        | `https://co.wuk.sh/`    | changes url from which api server is accessible. <br> ***REQUIRED TO RUN API***. |
 | `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
 | `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
 | `CORS_URL`            | not used  | `https://cobalt.tools/` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
 | `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
 | `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
-| `TIKTOK_DEVICE_INFO`  | ➖        | *see below*                    | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. |
+| `TIKTOK_DEVICE_INFO`  | ➖        | *see below*             | device info (including `iid` and `device_id`) for tiktok functionality. required for tiktok to work. |
+| `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. for more info, see below. |
 
 \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
 
@@ -85,6 +87,12 @@ you can compress the json to save space. if you're using a `.env` file then the
 TIKTOK_DEVICE_INFO='{"iid":"<install_id here>","device_id":"<device_id here>","channel":"googleplay","app_name":"musical_ly","version_code":"310503","device_platform":"android","device_type":"Redmi+7","os_version":"13"}'
 ```
 
+#### FREEBIND_CIDR
+setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
+requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
+in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
+`network_mode` for the container to `host`.
+
 ### variables for web
 | variable name        | default              | example                 | description                                                                           |
 |:---------------------|:---------------------|:------------------------|:--------------------------------------------------------------------------------------|

package.json

@@ -38,8 +38,11 @@
         "node-cache": "^5.1.2",
         "psl": "1.9.0",
         "set-cookie-parser": "2.6.0",
-        "undici": "^6.7.0",
+        "undici": "^5.19.1",
         "url-pattern": "1.0.3",
         "youtubei.js": "^9.3.0"
+    },
+    "optionalDependencies": {
+        "freebind": "^0.2.2"
     }
 }

src/core/api.js

@@ -194,7 +194,7 @@ export function runAPI(express, app, gitCommit, gitBranch, __dirname) {
         res.redirect('/api/json')
     });
 
-    app.listen(env.apiPort, () => {
+    app.listen(env.apiPort, env.listenAddress, () => {
         console.log(`\n` +
             `${Cyan("cobalt")} API ${Bright(`v.${version}-${gitCommit} (${gitBranch})`)}\n` +
             `Start time: ${Bright(`${startTime.toUTCString()} (${startTimestamp})`)}\n\n` +

src/modules/config.js

@@ -1,5 +1,6 @@
 import UrlPattern from "url-pattern";
 import { loadJSON } from "./sub/loadFromFs.js";
+
 const config = loadJSON("./src/config.json");
 const packageJson = loadJSON("./package.json");
 const servicesConfigJson = loadJSON("./src/modules/processing/servicesConfig.json");
@@ -29,13 +30,15 @@ const
     apiEnvs = {
         apiPort: process.env.API_PORT || 9000,
         apiName: process.env.API_NAME || 'unknown',
+        listenAddress: process.env.API_LISTEN_ADDRESS,
         corsWildcard: process.env.CORS_WILDCARD !== '0',
         corsURL: process.env.CORS_URL,
         cookiePath: process.env.COOKIE_PATH,
-        processingPriority: process.platform !== "win32"
+        processingPriority: process.platform !== 'win32'
                                 && process.env.PROCESSING_PRIORITY
                                 && parseInt(process.env.PROCESSING_PRIORITY),
         tiktokDeviceInfo: process.env.TIKTOK_DEVICE_INFO && JSON.parse(process.env.TIKTOK_DEVICE_INFO),
+        freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
         apiURL
     }
 
@@ -46,7 +49,7 @@ export const
     streamLifespan = config.streamLifespan,
     maxVideoDuration = config.maxVideoDuration,
     genericUserAgent = config.genericUserAgent,
-    repo = packageJson["bugs"]["url"].replace('/issues', ''),
+    repo = packageJson.bugs.url.replace('/issues', ''),
     authorInfo = config.authorInfo,
     donations = config.donations,
     ffmpegArgs = config.ffmpegArgs,

src/modules/processing/match.js

@@ -25,9 +25,21 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import { env } from '../config.js';
 
+let freebind;
 export default async function(host, patternMatch, url, lang, obj) {
     assert(url instanceof URL);
+    let dispatcher, requestIP;
+
+    if (env.freebindCIDR) {
+        if (!freebind) {
+            freebind = await import('freebind');
+        }
+
+        requestIP = freebind.ip.random(env.freebindCIDR);
+        dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false });
+    }
 
     try {
         let r, isAudioOnly = !!obj.isAudioOnly, disableMetadata = !!obj.disableMetadata;
@@ -66,7 +78,8 @@ export default async function(host, patternMatch, url, lang, obj) {
                     format: obj.vCodec,
                     isAudioOnly: isAudioOnly,
                     isAudioMuted: obj.isAudioMuted,
-                    dubLang: obj.dubLang
+                    dubLang: obj.dubLang,
+                    dispatcher
                 }
 
                 if (url.hostname === 'music.youtube.com' || isAudioOnly === true) {
@@ -122,7 +135,8 @@ export default async function(host, patternMatch, url, lang, obj) {
             case "instagram":
                 r = await instagram({
                     ...patternMatch,
-                    quality: obj.vQuality
+                    quality: obj.vQuality,
+                    dispatcher
                 })
                 break;
             case "vine":
@@ -181,7 +195,8 @@ export default async function(host, patternMatch, url, lang, obj) {
         return matchActionDecider(
             r, host, obj.aFormat, isAudioOnly,
             lang, isAudioMuted, disableMetadata,
-            obj.filenamePattern, obj.twitterGif
+            obj.filenamePattern, obj.twitterGif,
+            requestIP
         )
     } catch (e) {
         return apiJSON(0, { t: genericError(lang, host) })

src/modules/processing/matchActionDecider.js

@@ -3,15 +3,16 @@ import { apiJSON } from "../sub/utils.js";
 import loc from "../../localization/manager.js";
 import createFilename from "./createFilename.js";
 
-export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif) {
+export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
     let action,
         responseType = 2,
         defaultParams = {
             u: r.urls,
             service: host,
             filename: r.filenameAttributes ?
                     createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename,
-            fileMetadata: !disableMetadata ? r.fileMetadata : false
+            fileMetadata: !disableMetadata ? r.fileMetadata : false,
+            requestIP
         },
         params = {},
         audioFormat = String(userFormat);