-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
enhancement: better ytdlp integration
- Loading branch information
Showing
8 changed files
with
160 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package ytdlp | ||
|
|
||
| import ( | ||
| "io" | ||
| "net/url" | ||
| ) | ||
|
|
||
| func Parse(body io.ReadCloser) (streamURLs, metaURLs []*url.URL, rawJSON string, HTTPHeaders map[string]string, err error) { | ||
| // Create a temporary server to serve the body and call ytdlp on it | ||
| port, stopChan, err := serveBody(body) | ||
| if err != nil { | ||
| return streamURLs, metaURLs, rawJSON, HTTPHeaders, err | ||
| } | ||
| defer close(stopChan) | ||
|
|
||
| // Call ytdlp on the temporary server | ||
| rawStreamURLs, rawMetaURLs, rawJSON, HTTPHeaders, err := getJSON(port) | ||
| if err != nil { | ||
| return streamURLs, metaURLs, rawJSON, HTTPHeaders, err | ||
| } | ||
|
|
||
| // Range over rawStreamURLs and rawMetaURLs to parse them as url.URL in videoURLs and metaURLs | ||
| for _, urlString := range rawStreamURLs { | ||
| URL, err := url.Parse(urlString) | ||
| if err != nil { | ||
| return streamURLs, metaURLs, rawJSON, HTTPHeaders, err | ||
| } | ||
|
|
||
| streamURLs = append(streamURLs, URL) | ||
| } | ||
|
|
||
| for _, urlString := range rawMetaURLs { | ||
| URL, err := url.Parse(urlString) | ||
| if err != nil { | ||
| return streamURLs, metaURLs, rawJSON, HTTPHeaders, err | ||
| } | ||
|
|
||
| metaURLs = append(metaURLs, URL) | ||
| } | ||
|
|
||
| return streamURLs, metaURLs, rawJSON, HTTPHeaders, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,32 @@ | ||
| package youtube | ||
|
|
||
| import ( | ||
| "io" | ||
| "net/url" | ||
| "strings" | ||
|
|
||
| "github.com/internetarchive/Zeno/internal/pkg/crawl/dependencies/ytdlp" | ||
| ) | ||
|
|
||
| func IsYouTubeWatchPage(URL *url.URL) bool { | ||
| return strings.Contains(URL.Host, "youtube.com") && (strings.Contains(URL.Path, "/watch") || strings.Contains(URL.Path, "/v/")) | ||
| } | ||
|
|
||
| func Parse(body io.ReadCloser) (URLs []*url.URL, rawJSON string, HTTPHeaders map[string]string, err error) { | ||
| HTTPHeaders = make(map[string]string) | ||
|
|
||
| // Create a temporary server to serve the body and call ytdlp on it | ||
| port, stopChan, err := ytdlp.ServeBody(body) | ||
| if err != nil { | ||
| return nil, rawJSON, HTTPHeaders, err | ||
| } | ||
| defer close(stopChan) | ||
| // func Parse(body io.ReadCloser) (URLs []*url.URL, rawJSON string, HTTPHeaders map[string]string, err error) { | ||
| // HTTPHeaders = make(map[string]string) | ||
|
|
||
| // Call ytdlp on the temporary server | ||
| rawURLs, rawJSON, HTTPHeaders, err := ytdlp.GetJSON(port) | ||
| if err != nil { | ||
| return nil, rawJSON, HTTPHeaders, err | ||
| } | ||
| // // Call ytdlp on the temporary server | ||
| // rawURLs, rawJSON, HTTPHeaders, err := ytdlp.GetJSON() | ||
| // if err != nil { | ||
| // return nil, rawJSON, HTTPHeaders, err | ||
| // } | ||
|
|
||
| // Parse the URLs | ||
| for _, urlString := range rawURLs { | ||
| URL, err := url.Parse(urlString) | ||
| if err != nil { | ||
| return nil, rawJSON, HTTPHeaders, err | ||
| } | ||
| // // Parse the URLs | ||
| // for _, urlString := range rawURLs { | ||
| // URL, err := url.Parse(urlString) | ||
| // if err != nil { | ||
| // return nil, rawJSON, HTTPHeaders, err | ||
| // } | ||
|
|
||
| URLs = append(URLs, URL) | ||
| } | ||
| // URLs = append(URLs, URL) | ||
| // } | ||
|
|
||
| return URLs, rawJSON, HTTPHeaders, nil | ||
| } | ||
| // return URLs, rawJSON, HTTPHeaders, nil | ||
| // } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters