From 0dfc59a972e59d4e554a61e5e70c079635429829 Mon Sep 17 00:00:00 2001 From: Snazzah Date: Mon, 15 Apr 2024 21:42:19 -0500 Subject: [PATCH 01/14] feat: snapchat support --- src/modules/processing/match.js | 10 ++++ src/modules/processing/matchActionDecider.js | 1 + src/modules/processing/services/snapchat.js | 56 +++++++++++++++++++ src/modules/processing/servicesConfig.json | 6 ++ .../processing/servicesPatternTesters.js | 5 ++ src/test/tests.json | 19 ++++++- 6 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/modules/processing/services/snapchat.js diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 6f715ef76..1a46d0cda 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -25,6 +25,7 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import snapchat from "./services/snapchat.js"; export default async function(host, patternMatch, url, lang, obj) { assert(url instanceof URL); @@ -158,6 +159,15 @@ export default async function(host, patternMatch, url, lang, obj) { case "dailymotion": r = await dailymotion(patternMatch); break; + case "snapchat": + r = await snapchat({ + url, + username: patternMatch.username, + storyId: patternMatch.storyId, + spotlightId: patternMatch.spotlightId, + shortLink: patternMatch.shortLink || false + }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 2d8840a3c..622f2bbcb 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -120,6 +120,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "tumblr": case "pinterest": case "streamable": + case "snapchat": responseType = 1; break; } diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js new file mode 100644 index 000000000..eadbd2ae3 --- /dev/null +++ b/src/modules/processing/services/snapchat.js @@ -0,0 +1,56 @@ +import { genericUserAgent } from "../../config.js"; + +const SPOTLIGHT_VIDEO_REGEX = //; + +export default async function(obj) { + let link; + if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) { + link = await fetch(`https://t.snapchat.com/${obj.shortLink}`, { redirect: "manual" }).then((r) => { + if (r.status === 303 && r.headers.get("location").startsWith("https://www.snapchat.com/")) { + return r.headers.get("location").split('?', 1)[0] + } + }).catch(() => {}); + } + + if (!link && obj.username && obj.storyId) { + link = `https://www.snapchat.com/add/${obj.username}/${obj.storyId}` + } else if (!link && obj.spotlightId) { + link = `https://www.snapchat.com/spotlight/${obj.spotlightId}` + } else if (link?.startsWith('https://www.snapchat.com/download')) { + return { error: 'ErrorCouldntFetch' }; + } + + const path = new URL(link).pathname; + + if (path.startsWith('/spotlight/')) { + const html = await fetch(link, { + headers: { "user-agent": genericUserAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + if (!html) return { error: 'ErrorCouldntFetch' }; + + const id = path.split('/')[2]; + const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1]; + if (videoURL) return { + urls: videoURL, + filename: `snapchat_${id}.mp4`, + audioFilename: `snapchat_${id}_audio` + } + } else if (path.startsWith('/add/')) { + const html = await fetch(link, { + headers: { "user-agent": genericUserAgent } + }).then((r) => { return r.text() }).catch(() => { return false }); + if (!html) return { error: 'ErrorCouldntFetch' }; + + const id = path.split('/')[3]; + const storyVideoRegex = new RegExp(`"snapId":{"value":"${id}"},"snapMediaType":1,"snapUrls":{"mediaUrl":"(https:\\/\\/bolt-gcdn\\.sc-cdn\\.net\\/3\/[^"]+)","mediaPreviewUrl"`); + const videoURL = html.match(storyVideoRegex)?.[1]; + if (videoURL) return { + urls: videoURL, + filename: `snapchat_${id}.mp4`, + audioFilename: `snapchat_${id}_audio` + } + } + + + return { error: 'ErrorCouldntFetch' }; +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 633fa2a66..afaee500d 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -116,6 +116,12 @@ "alias": "dailymotion videos", "patterns": ["video/:id"], "enabled": true + }, + "snapchat": { + "alias": "snapchat stories & spotlights", + "subdomains": ["t", "story"], + "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index f4dee15b4..3d75c4f04 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -25,6 +25,11 @@ export const testers = { (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255) || patternMatch.shortLink?.length <= 32, + "snapchat": (patternMatch) => + (patternMatch.username?.length <= 32 && patternMatch.storyId?.length <= 255) + || patternMatch.spotlightId?.length <= 255 + || patternMatch.shortLink?.length <= 16, + "streamable": (patternMatch) => patternMatch.id?.length === 6, diff --git a/src/test/tests.json b/src/test/tests.json index bd8b33c58..2f654552d 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -1115,5 +1115,22 @@ "code": 200, "status": "stream" } + }], + "snapchat": [{ + "name": "spotlight", + "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, { + "name": "shortlinked spotlight", + "url": "https://t.snapchat.com/4ZsiBLDi", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } }] -} +} \ No newline at end of file From 48c48c61de13518a8159a360168518c526bfa6ff Mon Sep 17 00:00:00 2001 From: Snazzah Date: Mon, 15 Apr 2024 21:43:36 -0500 Subject: [PATCH 02/14] chore: remove redundancy --- src/modules/processing/services/snapchat.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js index eadbd2ae3..8442198a5 100644 --- a/src/modules/processing/services/snapchat.js +++ b/src/modules/processing/services/snapchat.js @@ -16,8 +16,6 @@ export default async function(obj) { link = `https://www.snapchat.com/add/${obj.username}/${obj.storyId}` } else if (!link && obj.spotlightId) { link = `https://www.snapchat.com/spotlight/${obj.spotlightId}` - } else if (link?.startsWith('https://www.snapchat.com/download')) { - return { error: 'ErrorCouldntFetch' }; } const path = new URL(link).pathname; From 7ac9c2895b4bdb1b881f63e8368240a1fa76c750 Mon Sep 17 00:00:00 2001 From: Snazzah Date: Mon, 15 Apr 2024 21:46:52 -0500 Subject: [PATCH 03/14] chore: a bit of better matching --- src/modules/processing/services/snapchat.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js index 8442198a5..3f2f87977 100644 --- a/src/modules/processing/services/snapchat.js +++ b/src/modules/processing/services/snapchat.js @@ -1,6 +1,6 @@ import { genericUserAgent } from "../../config.js"; -const SPOTLIGHT_VIDEO_REGEX = //; +const SPOTLIGHT_VIDEO_REGEX = //; export default async function(obj) { let link; From 5dfc16b76ce93c8d46aa18b4b858630871bb497c Mon Sep 17 00:00:00 2001 From: Snazzah Date: Thu, 18 Apr 2024 13:11:43 -0500 Subject: [PATCH 04/14] chore: update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2ebd3067e..120dce81e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ this list is not final and keeps expanding over time. if support for a service y | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat stories & spotlights | ✅ | ✅ | ✅ | ➖ | ➖ | | soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | | streamable | ✅ | ✅ | ✅ | ➖ | ➖ | | tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -44,6 +45,7 @@ this list is not final and keeps expanding over time. if support for a service y | instagram | supports photos, videos, and stories. lets you pick what to save from multi-media posts. | | pinterest | supports videos and stories. | | reddit | supports gifs and videos. | +| snapchat | supports spotlights and stories. stories will need to be directly linked to a video. | | soundcloud | supports private links. | | tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. | | twitter/x | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management. | From 4b4adc4e8becc85268e00c904d0eeba178f4c5ac Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sun, 12 May 2024 22:19:10 -0500 Subject: [PATCH 05/14] refactor(snapchat): refactor story matching to use pickers --- src/modules/processing/matchActionDecider.js | 1 + src/modules/processing/services/snapchat.js | 42 +++++++++++++++---- src/modules/processing/servicesConfig.json | 2 +- .../processing/servicesPatternTesters.js | 2 +- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index 622f2bbcb..70db973b3 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -63,6 +63,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di switch (host) { case "instagram": case "twitter": + case "snapchat": params = { picker: r.picker }; break; case "douyin": diff --git a/src/modules/processing/services/snapchat.js b/src/modules/processing/services/snapchat.js index 3f2f87977..519c4f73f 100644 --- a/src/modules/processing/services/snapchat.js +++ b/src/modules/processing/services/snapchat.js @@ -1,6 +1,7 @@ import { genericUserAgent } from "../../config.js"; const SPOTLIGHT_VIDEO_REGEX = //; +const NEXT_DATA_REGEX = /