Merge pull request #96 from zema1/new-features

New Features

grab/avd.go

@@ -8,6 +8,7 @@ import (
 	"github.com/imroc/req/v3"
 	"github.com/kataras/golog"
 	"github.com/pkg/errors"
+	"github.com/zema1/watchvuln/util"
 	"golang.org/x/net/html"
 	"net/url"
 	"regexp"
@@ -27,7 +28,7 @@ type AVDCrawler struct {
 }
 
 func NewAVDCrawler() Grabber {
-	client := NewHttpClient().AddCommonRetryCondition(func(resp *req.Response, err error) bool {
+	client := util.NewHttpClient().AddCommonRetryCondition(func(resp *req.Response, err error) bool {
 		if err != nil {
 			return !errors.Is(err, context.Canceled)
 		}

grab/grab.go

@@ -2,11 +2,7 @@ package grab
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"github.com/imroc/req/v3"
-	"github.com/kataras/golog"
-	"time"
 )
 
 type SeverityLevel string
@@ -58,48 +54,6 @@ type Grabber interface {
 	IsValuable(info *VulnInfo) bool
 }
 
-func NewHttpClient() *req.Client {
-	client := req.C()
-	client.
-		SetCommonHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.58").
-		SetTimeout(10 * time.Second).
-		SetCommonRetryCount(3).
-		SetCookieJar(nil).
-		SetCommonRetryInterval(func(resp *req.Response, attempt int) time.Duration {
-			if errors.Is(resp.Err, context.Canceled) {
-				return 0
-			}
-			return time.Second * 5
-		}).
-		SetCommonRetryHook(func(resp *req.Response, err error) {
-			if err != nil {
-				if !errors.Is(err, context.Canceled) {
-					golog.Warnf("retrying as %s", err)
-				}
-			}
-		}).SetCommonRetryCondition(func(resp *req.Response, err error) bool {
-		if err != nil {
-			return !errors.Is(err, context.Canceled)
-		}
-		return false
-	})
-	return client
-}
-
-func wrapApiClient(client *req.Client) *req.Client {
-	return client.SetCommonHeaders(map[string]string{
-		"Accept":             "application/json, text/plain, */*",
-		"Accept-Language":    "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
-		"Content-Type":       "application/json",
-		"Sec-Fetch-Dest":     "empty",
-		"Sec-Fetch-Mode":     "cors",
-		"Sec-Fetch-Site":     "same-origin",
-		"sec-ch-ua":          `"Microsoft Edge";v="111", "Not(A:Brand";v="8", "Chromium";v="111"`,
-		"sec-ch-ua-mobile":   `?0`,
-		"sec-ch-ua-platform": `"Windows"`,
-	})
-}
-
 func MergeUniqueString(s1 []string, s2 []string) []string {
 	m := make(map[string]struct{}, len(s1)+len(s2))
 	for _, s := range s1 {

grab/kev.go

@@ -2,6 +2,7 @@ package grab
 
 import (
 	"context"
+	"github.com/zema1/watchvuln/util"
 	"sort"
 	"time"
 
@@ -21,7 +22,7 @@ type KEVCrawler struct {
 }
 
 func NewKEVCrawler() Grabber {
-	client := NewHttpClient()
+	client := util.NewHttpClient()
 	client.SetCommonHeader("Referer", "Referer: https://www.cisa.gov/known-exploited-vulnerabilities-catalog")
 
 	return &KEVCrawler{

grab/oscs.go

@@ -7,6 +7,7 @@ import (
 	"github.com/imroc/req/v3"
 	"github.com/kataras/golog"
 	"github.com/pkg/errors"
+	"github.com/zema1/watchvuln/util"
 	"strings"
 	"time"
 )
@@ -19,7 +20,7 @@ type OSCSCrawler struct {
 }
 
 func NewOSCSCrawler() Grabber {
-	client := wrapApiClient(NewHttpClient())
+	client := util.WrapApiClient(util.NewHttpClient())
 	client.SetCommonHeader("Referer", "https://www.oscs1024.com/cm")
 	client.SetCommonHeader("Origin", "https://www.oscs1024.com")
 	c := &OSCSCrawler{

grab/seebug.go

@@ -10,6 +10,7 @@ import (
 	"github.com/imroc/req/v3"
 	"github.com/kataras/golog"
 	"github.com/pkg/errors"
+	"github.com/zema1/watchvuln/util"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
@@ -211,7 +212,7 @@ func (t *SeebugCrawler) IsValuable(info *VulnInfo) bool {
 
 func (t *SeebugCrawler) newClient() *req.Client {
 	jar, _ := cookiejar.New(nil)
-	client := NewHttpClient().
+	client := util.NewHttpClient().
 		SetCookieJar(jar).
 		SetCommonHeader("Referer", "https://www.seebug.org/")
 	return client

grab/struts2.go

@@ -3,6 +3,7 @@ package grab
 import (
 	"bytes"
 	"context"
+	"github.com/zema1/watchvuln/util"
 	"regexp"
 	"strings"
 	"time"
@@ -20,7 +21,7 @@ type Struts2Crawler struct {
 }
 
 func NewStruts2Crawler() Grabber {
-	client := NewHttpClient()
+	client := util.NewHttpClient()
 	client.SetCommonHeader("Referer", "https://cwiki.apache.org/")
 	client.SetCommonHeader("Origin", "https://cwiki.apache.org/")
 	client.SetCommonHeader("Accept-Language", "en-US,en;q=0.9")

grab/threatbook.go

@@ -2,6 +2,7 @@ package grab
 
 import (
 	"context"
+	"github.com/zema1/watchvuln/util"
 	"time"
 
 	"github.com/imroc/req/v3"
@@ -15,7 +16,7 @@ type ThreatBookCrawler struct {
 }
 
 func NewThreatBookCrawler() Grabber {
-	client := NewHttpClient()
+	client := util.NewHttpClient()
 	client.SetCommonHeader("Referer", "https://x.threatbook.com/v5/vulIntelligence")
 	client.SetCommonHeader("Origin", "https://mp.weixin.qq.com/")
 	client.SetCommonHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")

grab/ti.go

@@ -2,6 +2,7 @@ package grab
 
 import (
 	"context"
+	"github.com/zema1/watchvuln/util"
 	"strconv"
 	"strings"
 
@@ -15,7 +16,7 @@ type TiCrawler struct {
 }
 
 func NewTiCrawler() Grabber {
-	client := wrapApiClient(NewHttpClient())
+	client := util.WrapApiClient(util.NewHttpClient())
 	client.SetCommonHeader("Referer", "https: //ti.qianxin.com/")
 	client.SetCommonHeader("Origin", "https: //ti.qianxin.com")
 

push/bark.go

@@ -1,14 +1,9 @@
 package push
 
 import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net/http"
-
-	"github.com/pkg/errors"
-
+	"github.com/imroc/req/v3"
 	"github.com/kataras/golog"
+	"github.com/zema1/watchvuln/util"
 )
 
 var _ = TextPusher(&Bark{})
@@ -17,7 +12,7 @@ type Bark struct {
 	url       string
 	deviceKey string
 	log       *golog.Logger
-	client    *http.Client
+	client    *req.Client
 }
 
 type BarkData struct {
@@ -36,7 +31,7 @@ func NewBark(url string, deviceKey string) TextPusher {
 		url:       url,
 		deviceKey: deviceKey,
 		log:       golog.Child("[bark]"),
-		client:    &http.Client{},
+		client:    util.NewHttpClient(),
 	}
 }
 
@@ -78,27 +73,9 @@ func (m *Bark) PushMarkdown(title, content string) error {
 }
 
 func (m *Bark) postJSON(url string, params *BarkData) ([]byte, error) {
-	postBody, _ := json.Marshal(params)
-	return m.doPostRequest(url, "application/json", postBody)
-}
-
-func (m *Bark) doPostRequest(url string, contentType string, body []byte) ([]byte, error) {
-	req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
-	if err != nil {
-		return nil, errors.Wrap(err, "create request")
-	}
-
-	req.Header.Set("Content-Type", contentType)
-
-	resp, err := m.client.Do(req)
-	if err != nil {
-		return nil, errors.Wrap(err, "send request")
-	}
-	defer resp.Body.Close()
-
-	respBody, err := io.ReadAll(resp.Body) // 使用 io.ReadAll 替代 ioutil.ReadAll。
+	resp, err := m.client.R().SetBodyJsonMarshal(params).Post(url)
 	if err != nil {
-		return nil, errors.Wrap(err, "read body")
+		return nil, err
 	}
-	return respBody, nil
+	return resp.ToBytes()
 }

push/push.go

@@ -1,5 +1,7 @@
 package push
 
+import "github.com/hashicorp/go-multierror"
+
 // TextPusher is a type that can push text and markdown messages.
 type TextPusher interface {
 	PushText(s string) error
@@ -27,21 +29,23 @@ func MultiRawPusher(pushers ...RawPusher) RawPusher {
 }
 
 func (m *multiPusher) PushText(s string) error {
+	var pushErr *multierror.Error
 	for _, push := range m.textPusher {
 		if err := push.PushText(s); err != nil {
-			return err
+			pushErr = multierror.Append(pushErr, err)
 		}
 	}
-	return nil
+	return pushErr.ErrorOrNil()
 }
 
 func (m *multiPusher) PushMarkdown(title, content string) error {
+	var pushErr *multierror.Error
 	for _, push := range m.textPusher {
 		if err := push.PushMarkdown(title, content); err != nil {
-			return err
+			pushErr = multierror.Append(pushErr, err)
 		}
 	}
-	return nil
+	return pushErr.ErrorOrNil()
 }
 
 func (m *multiPusher) PushRaw(r *RawMessage) error {

push/sereverchan.go

@@ -1,28 +1,32 @@
 package push
 
 import (
+	"fmt"
+	"github.com/imroc/req/v3"
 	"github.com/kataras/golog"
 	"github.com/pkg/errors"
-	serverchan "github.com/rayepeng/serverchan"
+	"github.com/zema1/watchvuln/util"
 )
 
 var _ = TextPusher(&ServerChan{})
 
 type ServerChan struct {
-	client *serverchan.ServerChan
-	log    *golog.Logger
+	pushUrl string
+	log     *golog.Logger
+	client  *req.Client
 }
 
 func NewServerChan(botKey string) TextPusher {
 	return &ServerChan{
-		client: serverchan.NewServerChan(botKey),
-		log:    golog.Child("[pusher-server-chan]"),
+		pushUrl: fmt.Sprintf("https://sctapi.ftqq.com/%s.send", botKey),
+		log:     golog.Child("[pusher-server-chan]"),
+		client:  util.NewHttpClient(),
 	}
 }
 
 func (d *ServerChan) PushText(s string) error {
 	d.log.Infof("sending text %s", s)
-	_, err := d.client.Send("", s)
+	err := d.send("", s)
 	if err != nil {
 		return errors.Wrap(err, "server-chan")
 	}
@@ -31,9 +35,18 @@ func (d *ServerChan) PushText(s string) error {
 
 func (d *ServerChan) PushMarkdown(title, content string) error {
 	d.log.Infof("sending markdown %s", title)
-	_, err := d.client.Send(title, content)
+	err := d.send(title, content)
 	if err != nil {
 		return errors.Wrap(err, "server-chan")
 	}
 	return nil
 }
+
+func (d *ServerChan) send(text string, desp string) error {
+	body := map[string]string{
+		"text": text,
+		"desp": desp,
+	}
+	_, err := d.client.R().SetBodyJsonMarshal(body).Post(d.pushUrl)
+	return err
+}