From d15e729d46ce045e94da969e4011293d74d7e049 Mon Sep 17 00:00:00 2001 From: Dawid H Date: Sat, 29 Oct 2022 02:22:45 +0200 Subject: [PATCH 1/6] feat: added custom format support --- config.toml.example | 8 ++++++-- pkg/feed/config.go | 7 +++++++ pkg/feed/xml.go | 2 ++ pkg/model/feed.go | 5 +++-- pkg/ytdl/ytdl.go | 7 ++++++- pkg/ytdl/ytdl_test.go | 27 +++++++++++++++++++-------- 6 files changed, 43 insertions(+), 13 deletions(-) diff --git a/config.toml.example b/config.toml.example index 094f744e..9e3ac6e3 100644 --- a/config.toml.example +++ b/config.toml.example @@ -51,13 +51,17 @@ vimeo = [ # Multiple keys will be rotated. update_period = "12h" quality = "high" # or "low" - format = "video" # or "audio" + format = "video" # or: "audio", "custom" + # When format = "custom" + # YouTubeDL format parameter and result file extension + custom_format = { youtube_dl_format = "bestaudio[ext=m4a]", extension = "m4a" } + playlist_sort = "asc" # or "desc", which will fetch playlist items from the end # Optional maximal height of video, example: 720, 1080, 1440, 2160, ... max_height = 720 - # Optinally include this feed in OPML file (default value: false) + # Optionally include this feed in OPML file (default value: false) opml = true # Optional cron expression format for more precise update schedule. diff --git a/pkg/feed/config.go b/pkg/feed/config.go index 2a7a7412..facfa6ff 100644 --- a/pkg/feed/config.go +++ b/pkg/feed/config.go @@ -28,6 +28,8 @@ type Config struct { MaxHeight int `toml:"max_height"` // Format to use for this feed Format model.Format `toml:"format"` + // Custom format properties + CustomFormat CustomFormat `toml:"custom_format"` // Only download episodes that match this regexp (defaults to matching anything) Filters Filters `toml:"filters"` // Clean is a cleanup policy to use for this feed @@ -44,6 +46,11 @@ type Config struct { PlaylistSort model.Sorting `toml:"playlist_sort"` } +type CustomFormat struct { + YouTubeDLFormat string `toml:"youtube_dl_format"` + Extension string `toml:"extension"` +} + type Filters struct { Title string `toml:"title"` NotTitle string `toml:"not_title"` diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index c20cc923..42ff272f 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -164,6 +164,8 @@ func EpisodeName(feedConfig *Config, episode *model.Episode) string { ext := "mp4" if feedConfig.Format == model.FormatAudio { ext = "mp3" + }else if feedConfig.Format == model.FormatCustom { + ext = feedConfig.CustomFormat.Extension } return fmt.Sprintf("%s.%s", episode.ID, ext) diff --git a/pkg/model/feed.go b/pkg/model/feed.go index 7c79e1bb..fb6cc40b 100644 --- a/pkg/model/feed.go +++ b/pkg/model/feed.go @@ -16,8 +16,9 @@ const ( type Format string const ( - FormatAudio = Format("audio") - FormatVideo = Format("video") + FormatAudio = Format("audio") + FormatVideo = Format("video") + FormatCustom = Format("custom") ) // Playlist sorting style diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index f0354617..7dfc1782 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -197,7 +197,10 @@ func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, epis ext := "mp4" if feedConfig.Format == model.FormatAudio { ext = "mp3" + } else if feedConfig.Format == model.FormatCustom { + ext = feedConfig.CustomFormat.Extension } + // filePath now with the final extension filePath = filepath.Join(tmpDir, fmt.Sprintf("%s.%s", episode.ID, ext)) f, err := os.Open(filePath) @@ -236,7 +239,7 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s } args = append(args, "--format", format) - } else { + } else if feedConfig.Format == model.FormatAudio { // Audio, mp3, high by default format := "bestaudio" if feedConfig.Quality == model.QualityLow { @@ -244,6 +247,8 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s } args = append(args, "--extract-audio", "--audio-format", "mp3", "--format", format) + } else { + args = append(args, "--audio-format", feedConfig.CustomFormat.Extension, "--format", feedConfig.CustomFormat.YouTubeDLFormat) } // Insert additional per-feed youtube-dl arguments diff --git a/pkg/ytdl/ytdl_test.go b/pkg/ytdl/ytdl_test.go index c55571ef..f1cbe915 100644 --- a/pkg/ytdl/ytdl_test.go +++ b/pkg/ytdl/ytdl_test.go @@ -11,14 +11,15 @@ import ( func TestBuildArgs(t *testing.T) { tests := []struct { - name string - format model.Format - quality model.Quality - maxHeight int - output string - videoURL string - ytdlArgs []string - expect []string + name string + format model.Format + customFormat feed.CustomFormat + quality model.Quality + maxHeight int + output string + videoURL string + ytdlArgs []string + expect []string }{ { name: "Audio unknown quality", @@ -101,6 +102,15 @@ func TestBuildArgs(t *testing.T) { ytdlArgs: []string{"--write-sub", "--embed-subs", "--sub-lang", "en,en-US,en-GB"}, expect: []string{"--format", "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--write-sub", "--embed-subs", "--sub-lang", "en,en-US,en-GB", "--output", "/tmp/2", "http://url1"}, }, + { + name: "Custom format", + format: model.FormatCustom, + customFormat: feed.CustomFormat{YouTubeDLFormat: "bestaudio[ext=m4a]", Extension: "m4a"}, + quality: model.QualityHigh, + output: "/tmp/2", + videoURL: "http://url1", + expect: []string{"--audio-format", "m4a", "--format", "bestaudio[ext=m4a]", "--output", "/tmp/2", "http://url1"}, + }, } for _, tst := range tests { @@ -108,6 +118,7 @@ func TestBuildArgs(t *testing.T) { result := buildArgs(&feed.Config{ Format: tst.format, Quality: tst.quality, + CustomFormat: tst.customFormat, MaxHeight: tst.maxHeight, YouTubeDLArgs: tst.ytdlArgs, }, &model.Episode{ From a0353e49149bc2014cb9ea0f83b8e8210ea8d3d2 Mon Sep 17 00:00:00 2001 From: Dawid H Date: Sat, 29 Oct 2022 02:30:59 +0200 Subject: [PATCH 2/6] chore: files formatting --- pkg/feed/xml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 42ff272f..0d96e8a6 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -164,7 +164,7 @@ func EpisodeName(feedConfig *Config, episode *model.Episode) string { ext := "mp4" if feedConfig.Format == model.FormatAudio { ext = "mp3" - }else if feedConfig.Format == model.FormatCustom { + } else if feedConfig.Format == model.FormatCustom { ext = feedConfig.CustomFormat.Extension } From 5f0123ddbc2921a28aa52bbb3b9b33ab5e42b1b4 Mon Sep 17 00:00:00 2001 From: Dawid H <5307341+Harnas@users.noreply.github.com> Date: Thu, 3 Nov 2022 08:48:47 +0100 Subject: [PATCH 3/6] fix: correct enclosure type on custom format --- config.toml.example | 4 ++-- pkg/feed/xml.go | 29 ++++++++++++++++++++++++++++- pkg/model/feed.go | 15 +++++++++++++++ pkg/ytdl/ytdl.go | 3 ++- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/config.toml.example b/config.toml.example index 9e3ac6e3..98cea593 100644 --- a/config.toml.example +++ b/config.toml.example @@ -50,8 +50,8 @@ vimeo = [ # Multiple keys will be rotated. # How often query for updates, examples: "60m", "4h", "2h45m" update_period = "12h" - quality = "high" # or "low" - format = "video" # or: "audio", "custom" + quality = "high" # "high" or "low" + format = "video" # "audio", "video" or "custom" # When format = "custom" # YouTubeDL format parameter and result file extension custom_format = { youtube_dl_format = "bestaudio[ext=m4a]", extension = "m4a" } diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 0d96e8a6..3b11868d 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -133,6 +133,10 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) if feed.Format == model.FormatAudio { enclosureType = itunes.MP3 } + if feed.Format == model.FormatCustom { + enclosureType = EnclosureFromExtension(cfg) + } + var ( episodeName = EpisodeName(cfg, episode) @@ -164,9 +168,32 @@ func EpisodeName(feedConfig *Config, episode *model.Episode) string { ext := "mp4" if feedConfig.Format == model.FormatAudio { ext = "mp3" - } else if feedConfig.Format == model.FormatCustom { + } + if feedConfig.Format == model.FormatCustom { ext = feedConfig.CustomFormat.Extension } return fmt.Sprintf("%s.%s", episode.ID, ext) } + +func EnclosureFromExtension(feedConfig *Config) itunes.EnclosureType { + ext := feedConfig.CustomFormat.Extension + // Use switch on the day variable. + switch { + case ext == "m4a": + return itunes.M4A + case ext == "m4v": + return itunes.M4V + case ext == "mp4": + return itunes.MP4 + case ext == "mp3": + return itunes.MP3 + case ext == "mov": + return itunes.MOV + case ext == "pdf": + return itunes.PDF + case ext == "epub": + return itunes.EPUB + } + return -1 +} diff --git a/pkg/model/feed.go b/pkg/model/feed.go index fb6cc40b..ac68af3e 100644 --- a/pkg/model/feed.go +++ b/pkg/model/feed.go @@ -21,6 +21,21 @@ const ( FormatCustom = Format("custom") ) +// Format to convert episode when downloading episodes +type Enclosure struct { + // ID of episode + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Thumbnail string `json:"thumbnail"` + Duration int64 `json:"duration"` + VideoURL string `json:"video_url"` + PubDate time.Time `json:"pub_date"` + Size int64 `json:"size"` + Order string `json:"order"` + Status EpisodeStatus `json:"status"` // Disk status +} + // Playlist sorting style type Sorting string diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index 7dfc1782..1d33a928 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -197,7 +197,8 @@ func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, epis ext := "mp4" if feedConfig.Format == model.FormatAudio { ext = "mp3" - } else if feedConfig.Format == model.FormatCustom { + } + if feedConfig.Format == model.FormatCustom { ext = feedConfig.CustomFormat.Extension } From 3a42d8249ea80b5ef872723b6166e6d58eabd0ee Mon Sep 17 00:00:00 2001 From: Dawid H <5307341+Harnas@users.noreply.github.com> Date: Thu, 3 Nov 2022 08:51:24 +0100 Subject: [PATCH 4/6] fix: correct enclosure type on custom format --- pkg/feed/xml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 3b11868d..0bd79c4c 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -178,7 +178,7 @@ func EpisodeName(feedConfig *Config, episode *model.Episode) string { func EnclosureFromExtension(feedConfig *Config) itunes.EnclosureType { ext := feedConfig.CustomFormat.Extension - // Use switch on the day variable. + switch { case ext == "m4a": return itunes.M4A From d3fe8c61e5406bdddec5dc5b97217a39d2c82f1d Mon Sep 17 00:00:00 2001 From: Dawid H <5307341+Harnas@users.noreply.github.com> Date: Fri, 4 Nov 2022 21:35:57 +0100 Subject: [PATCH 5/6] chore: correct formating --- pkg/feed/xml.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 0bd79c4c..cf64f8ac 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -137,7 +137,6 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) enclosureType = EnclosureFromExtension(cfg) } - var ( episodeName = EpisodeName(cfg, episode) downloadURL = fmt.Sprintf("%s/%s/%s", strings.TrimRight(hostname, "/"), cfg.ID, episodeName) From 7d44946e3cdcf069a3188ca9c3a2ffe1f9b19dda Mon Sep 17 00:00:00 2001 From: Dawid H <5307341+Harnas@users.noreply.github.com> Date: Fri, 4 Nov 2022 21:42:10 +0100 Subject: [PATCH 6/6] chore: remove dead code added by mistake --- pkg/model/feed.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/model/feed.go b/pkg/model/feed.go index ac68af3e..fb6cc40b 100644 --- a/pkg/model/feed.go +++ b/pkg/model/feed.go @@ -21,21 +21,6 @@ const ( FormatCustom = Format("custom") ) -// Format to convert episode when downloading episodes -type Enclosure struct { - // ID of episode - ID string `json:"id"` - Title string `json:"title"` - Description string `json:"description"` - Thumbnail string `json:"thumbnail"` - Duration int64 `json:"duration"` - VideoURL string `json:"video_url"` - PubDate time.Time `json:"pub_date"` - Size int64 `json:"size"` - Order string `json:"order"` - Status EpisodeStatus `json:"status"` // Disk status -} - // Playlist sorting style type Sorting string