From 87be4b01b0640a7c189cf1e6d6b16d7f001c2990 Mon Sep 17 00:00:00 2001 From: ston Date: Tue, 31 Dec 2024 23:45:40 +0800 Subject: [PATCH 1/2] feat: sub file as sublink fallback --- common/subscription/subscription.go | 44 +++++++ docs/en/user-guide/persistent-subscription.md | 124 ------------------ example.dae | 5 + 3 files changed, 49 insertions(+), 124 deletions(-) delete mode 100644 docs/en/user-guide/persistent-subscription.md diff --git a/common/subscription/subscription.go b/common/subscription/subscription.go index 4af6e34747..17ede6d8ee 100644 --- a/common/subscription/subscription.go +++ b/common/subscription/subscription.go @@ -154,6 +154,9 @@ func ResolveSubscription(log *logrus.Logger, client *http.Client, configDir stri req *http.Request resp *http.Response ) + + persistToFile := false + switch u.Scheme { case "file": b, err = ResolveFile(u, configDir) @@ -161,6 +164,13 @@ func ResolveSubscription(log *logrus.Logger, client *http.Client, configDir stri return "", nil, err } goto resolve + case "http-file", "https-file": + if len(tag) == 0 { + return "", nil, fmt.Errorf("tag is required for http-file/https-file subscription") + } + persistToFile = true + subscription = strings.Replace(subscription, "-file", "", 1) + break default: } req, err = http.NewRequest("GET", subscription, nil) @@ -170,6 +180,18 @@ func ResolveSubscription(log *logrus.Logger, client *http.Client, configDir stri req.Header.Set("User-Agent", fmt.Sprintf("dae/%v (like v2rayA/1.0 WebRequestHelper) (like v2rayN/1.0 WebRequestHelper)", config.Version)) resp, err = client.Do(req) if err != nil { + if persistToFile { + log.Warnln("failed to fetch subscription, try to read from file") + u.Host = "persist.d/" + tag + ".sub" + u.Path = "" + b, err = ResolveFile(u, configDir) + + if err != nil { + return "", nil, err + } + goto resolve + } + return "", nil, err } defer resp.Body.Close() @@ -177,6 +199,28 @@ func ResolveSubscription(log *logrus.Logger, client *http.Client, configDir stri if err != nil { return "", nil, err } + + if persistToFile { + path := filepath.Join(configDir, "persist.d") + if _, err := os.Stat(path); os.IsNotExist(err) { + err := os.MkdirAll(path, 0700) + if err != nil { + return "", nil, err + } + } + + path = filepath.Join(path, tag+".sub") + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return "", nil, err + } + defer file.Close() + + _, err = file.Write(b) + if err != nil { + return "", nil, err + } + } resolve: if nodes, err = ResolveSubscriptionAsSIP008(log, b); err == nil { return tag, nodes, nil diff --git a/docs/en/user-guide/persistent-subscription.md b/docs/en/user-guide/persistent-subscription.md deleted file mode 100644 index bffaf07f7b..0000000000 --- a/docs/en/user-guide/persistent-subscription.md +++ /dev/null @@ -1,124 +0,0 @@ -# Persistent and automatically update subscriptions - -[systemd.timer](https://www.freedesktop.org/software/systemd/man/latest/systemd.timer.html) allows you to run a service periodically. - -The current behavior of dae is to pull subscriptions at startup, but this can sometimes result in an empty group if there is a network anomaly or other possible reason when pulling subscriptions. - -We'll introduce a way to achieve persistent subscription storage and automatic subscription updates through the systemd service and timer. - -## Preparations - -### Shell script - -We assume that your dae configuration file is stored in `/usr/local/etc/dae/` . - -`/usr/local/bin/update-dae-subs.sh`: - -```sh -#!/bin/bash - -# Change the path to suit your needs -cd /usr/local/etc/dae || exit 1 -version="$(dae --version | head -n 1 | sed 's/dae version //')" -UA="dae/${version} (like v2rayA/1.0 WebRequestHelper) (like v2rayN/1.0 WebRequestHelper)" -fail=false - -while IFS=':' read -r name url -do - curl --retry 3 --retry-delay 5 -fL -A "$UA" "$url" -o "${name}.sub.new" - if [[ $? -eq 0 ]]; then - mv "${name}.sub.new" "${name}.sub" - chmod 0600 "${name}.sub" - echo "Downloaded $name" - else - if [ -f "${name}.sub.new" ]; then - rm "${name}.sub.new" - fi - fail=true - echo "Failed to download $name" - fi -done < sublist - -dae reload - -if $fail; then - echo "Failed to update some subs" - exit 2 -fi -``` - -You need to give it proper permission: - -```sh -chmod +x /usr/local/bin/update-dae-subs.sh -``` - -### `systemd.timer` and `systemd.service` - -This timer will automatically update dae subscriptions every 12 hours, or 15 minutes after each boot. - -`/etc/systemd/system/update-subs.timer`: - -```systemd -[Unit] -Description=Auto-update dae subscriptions - -[Timer] -OnBootSec=15min -OnUnitActiveSec=12h - -[Install] -WantedBy=timers.target -``` - -`/etc/systemd/system/update-subs.service`: - -```systemd -[Unit] -Description=Update dae subscriptions -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/update-dae-subs.sh -Restart=on-failure -``` - -## Configurations - -Put your subscription links into `/usr/local/etc/dae/sublist`: - -```text -sub1:https://mysub1.com -sub2:https://mysub2.com -sub3:https://mysub3.com -``` - -Give the file appropriate permissions (for your privacy): - -```sh -chmod 0600 /usr/local/etc/dae/sublist -``` - -Edit `config.dae`: - -```text -subscription { - # Add your subscription links here. - sub1:'file://sub1.sub' - sub2:'file://sub2.sub' - sub3:'file://sub3.sub' -} -``` - -## Enable timer - -Execute the following command: - -```sh -systemctl enable --now update-dae-subs.timer - -# If you need to renew your subscription immediately or haven't pulled a subscription before -systemctl start update-dae-subs.service -``` diff --git a/example.dae b/example.dae index f3d600d395..a36944dd88 100644 --- a/example.dae +++ b/example.dae @@ -117,6 +117,11 @@ subscription { another_sub: 'https://example.com/another_sub' 'https://example.com/no_tag_link' 'file://relative/path/to/mysub.sub' # Put subscription content in /etc/dae/relative/path/to/mysub.sub + + # Subscriptions with '-file' will be saved to 'config_dir/persist.d/your_sub_tag.sub'. + # This file will serve as a fallback when fetching the subscription via a link fails. + # It will be updated automatically once the fetch is successful. + persist_sub: 'https-file://www.example.com/persist_sub/link' } # Nodes defined here will be merged as a part of the global node pool. From 03c409c53795a240449e3e792939e1c34b70674e Mon Sep 17 00:00:00 2001 From: ston Date: Fri, 10 Jan 2025 14:59:26 +0800 Subject: [PATCH 2/2] fix: delete unused persist file --- cmd/run.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/run.go b/cmd/run.go index 16f2fc5b4e..6fd6a4702a 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -397,6 +397,22 @@ func newControlPlane(log *logrus.Logger, bpf interface{}, dnsCache map[string]*c tagToNodeList[tag] = append(tagToNodeList[tag], nodes...) } } + + // Delete all files in persist.d that are not in tagToNodeList + files, err := os.ReadDir(filepath.Join(filepath.Dir(cfgFile), "persist.d")) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, file := range files { + tag := strings.TrimSuffix(file.Name(), ".sub") + if _, ok := tagToNodeList[tag]; !ok { + err := os.Remove(filepath.Join(filepath.Dir(cfgFile), "persist.d", file.Name())) + if err != nil { + return nil, err + } + } + } + if len(tagToNodeList) == 0 { if resolvingfailed { log.Warnln("No node found because all subscription resolving failed.")