diff --git a/grab/avd.go b/grab/avd.go index b8bf4cc..0ae6d35 100644 --- a/grab/avd.go +++ b/grab/avd.go @@ -8,6 +8,7 @@ import ( "github.com/imroc/req/v3" "github.com/kataras/golog" "github.com/pkg/errors" + "github.com/zema1/watchvuln/util" "golang.org/x/net/html" "net/url" "regexp" @@ -27,7 +28,7 @@ type AVDCrawler struct { } func NewAVDCrawler() Grabber { - client := NewHttpClient().AddCommonRetryCondition(func(resp *req.Response, err error) bool { + client := util.NewHttpClient().AddCommonRetryCondition(func(resp *req.Response, err error) bool { if err != nil { return !errors.Is(err, context.Canceled) } diff --git a/grab/grab.go b/grab/grab.go index 0279cb4..0a89d1d 100644 --- a/grab/grab.go +++ b/grab/grab.go @@ -2,11 +2,7 @@ package grab import ( "context" - "errors" "fmt" - "github.com/imroc/req/v3" - "github.com/kataras/golog" - "time" ) type SeverityLevel string @@ -58,48 +54,6 @@ type Grabber interface { IsValuable(info *VulnInfo) bool } -func NewHttpClient() *req.Client { - client := req.C() - client. - SetCommonHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.58"). - SetTimeout(10 * time.Second). - SetCommonRetryCount(3). - SetCookieJar(nil). - SetCommonRetryInterval(func(resp *req.Response, attempt int) time.Duration { - if errors.Is(resp.Err, context.Canceled) { - return 0 - } - return time.Second * 5 - }). - SetCommonRetryHook(func(resp *req.Response, err error) { - if err != nil { - if !errors.Is(err, context.Canceled) { - golog.Warnf("retrying as %s", err) - } - } - }).SetCommonRetryCondition(func(resp *req.Response, err error) bool { - if err != nil { - return !errors.Is(err, context.Canceled) - } - return false - }) - return client -} - -func wrapApiClient(client *req.Client) *req.Client { - return client.SetCommonHeaders(map[string]string{ - "Accept": "application/json, text/plain, */*", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", - "Content-Type": "application/json", - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "cors", - "Sec-Fetch-Site": "same-origin", - "sec-ch-ua": `"Microsoft Edge";v="111", "Not(A:Brand";v="8", "Chromium";v="111"`, - "sec-ch-ua-mobile": `?0`, - "sec-ch-ua-platform": `"Windows"`, - }) -} - func MergeUniqueString(s1 []string, s2 []string) []string { m := make(map[string]struct{}, len(s1)+len(s2)) for _, s := range s1 { diff --git a/grab/kev.go b/grab/kev.go index 3a32a3f..3a135d9 100644 --- a/grab/kev.go +++ b/grab/kev.go @@ -2,6 +2,7 @@ package grab import ( "context" + "github.com/zema1/watchvuln/util" "sort" "time" @@ -21,7 +22,7 @@ type KEVCrawler struct { } func NewKEVCrawler() Grabber { - client := NewHttpClient() + client := util.NewHttpClient() client.SetCommonHeader("Referer", "Referer: https://www.cisa.gov/known-exploited-vulnerabilities-catalog") return &KEVCrawler{ diff --git a/grab/oscs.go b/grab/oscs.go index 21ab4fc..f05cc12 100644 --- a/grab/oscs.go +++ b/grab/oscs.go @@ -7,6 +7,7 @@ import ( "github.com/imroc/req/v3" "github.com/kataras/golog" "github.com/pkg/errors" + "github.com/zema1/watchvuln/util" "strings" "time" ) @@ -19,7 +20,7 @@ type OSCSCrawler struct { } func NewOSCSCrawler() Grabber { - client := wrapApiClient(NewHttpClient()) + client := util.WrapApiClient(util.NewHttpClient()) client.SetCommonHeader("Referer", "https://www.oscs1024.com/cm") client.SetCommonHeader("Origin", "https://www.oscs1024.com") c := &OSCSCrawler{ diff --git a/grab/seebug.go b/grab/seebug.go index 7da6cfe..a004cfd 100644 --- a/grab/seebug.go +++ b/grab/seebug.go @@ -10,6 +10,7 @@ import ( "github.com/imroc/req/v3" "github.com/kataras/golog" "github.com/pkg/errors" + "github.com/zema1/watchvuln/util" "net/http" "net/http/cookiejar" "net/url" @@ -211,7 +212,7 @@ func (t *SeebugCrawler) IsValuable(info *VulnInfo) bool { func (t *SeebugCrawler) newClient() *req.Client { jar, _ := cookiejar.New(nil) - client := NewHttpClient(). + client := util.NewHttpClient(). SetCookieJar(jar). SetCommonHeader("Referer", "https://www.seebug.org/") return client diff --git a/grab/struts2.go b/grab/struts2.go index f6672ee..a7296b1 100644 --- a/grab/struts2.go +++ b/grab/struts2.go @@ -3,6 +3,7 @@ package grab import ( "bytes" "context" + "github.com/zema1/watchvuln/util" "regexp" "strings" "time" @@ -20,7 +21,7 @@ type Struts2Crawler struct { } func NewStruts2Crawler() Grabber { - client := NewHttpClient() + client := util.NewHttpClient() client.SetCommonHeader("Referer", "https://cwiki.apache.org/") client.SetCommonHeader("Origin", "https://cwiki.apache.org/") client.SetCommonHeader("Accept-Language", "en-US,en;q=0.9") diff --git a/grab/threatbook.go b/grab/threatbook.go index 26873e7..0a18dbb 100644 --- a/grab/threatbook.go +++ b/grab/threatbook.go @@ -2,6 +2,7 @@ package grab import ( "context" + "github.com/zema1/watchvuln/util" "time" "github.com/imroc/req/v3" @@ -15,7 +16,7 @@ type ThreatBookCrawler struct { } func NewThreatBookCrawler() Grabber { - client := NewHttpClient() + client := util.NewHttpClient() client.SetCommonHeader("Referer", "https://x.threatbook.com/v5/vulIntelligence") client.SetCommonHeader("Origin", "https://mp.weixin.qq.com/") client.SetCommonHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") diff --git a/grab/ti.go b/grab/ti.go index 1ef8574..61f1dea 100644 --- a/grab/ti.go +++ b/grab/ti.go @@ -2,6 +2,7 @@ package grab import ( "context" + "github.com/zema1/watchvuln/util" "strconv" "strings" @@ -15,7 +16,7 @@ type TiCrawler struct { } func NewTiCrawler() Grabber { - client := wrapApiClient(NewHttpClient()) + client := util.WrapApiClient(util.NewHttpClient()) client.SetCommonHeader("Referer", "https: //ti.qianxin.com/") client.SetCommonHeader("Origin", "https: //ti.qianxin.com") diff --git a/push/bark.go b/push/bark.go index 8f5ee1d..71e502f 100644 --- a/push/bark.go +++ b/push/bark.go @@ -1,14 +1,9 @@ package push import ( - "bytes" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" - + "github.com/imroc/req/v3" "github.com/kataras/golog" + "github.com/zema1/watchvuln/util" ) var _ = TextPusher(&Bark{}) @@ -17,7 +12,7 @@ type Bark struct { url string deviceKey string log *golog.Logger - client *http.Client + client *req.Client } type BarkData struct { @@ -36,7 +31,7 @@ func NewBark(url string, deviceKey string) TextPusher { url: url, deviceKey: deviceKey, log: golog.Child("[bark]"), - client: &http.Client{}, + client: util.NewHttpClient(), } } @@ -78,27 +73,9 @@ func (m *Bark) PushMarkdown(title, content string) error { } func (m *Bark) postJSON(url string, params *BarkData) ([]byte, error) { - postBody, _ := json.Marshal(params) - return m.doPostRequest(url, "application/json", postBody) -} - -func (m *Bark) doPostRequest(url string, contentType string, body []byte) ([]byte, error) { - req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) - if err != nil { - return nil, errors.Wrap(err, "create request") - } - - req.Header.Set("Content-Type", contentType) - - resp, err := m.client.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send request") - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) // 使用 io.ReadAll 替代 ioutil.ReadAll。 + resp, err := m.client.R().SetBodyJsonMarshal(params).Post(url) if err != nil { - return nil, errors.Wrap(err, "read body") + return nil, err } - return respBody, nil + return resp.ToBytes() } diff --git a/push/push.go b/push/push.go index 63dfb19..e9cc26a 100644 --- a/push/push.go +++ b/push/push.go @@ -1,5 +1,7 @@ package push +import "github.com/hashicorp/go-multierror" + // TextPusher is a type that can push text and markdown messages. type TextPusher interface { PushText(s string) error @@ -27,21 +29,23 @@ func MultiRawPusher(pushers ...RawPusher) RawPusher { } func (m *multiPusher) PushText(s string) error { + var pushErr *multierror.Error for _, push := range m.textPusher { if err := push.PushText(s); err != nil { - return err + pushErr = multierror.Append(pushErr, err) } } - return nil + return pushErr.ErrorOrNil() } func (m *multiPusher) PushMarkdown(title, content string) error { + var pushErr *multierror.Error for _, push := range m.textPusher { if err := push.PushMarkdown(title, content); err != nil { - return err + pushErr = multierror.Append(pushErr, err) } } - return nil + return pushErr.ErrorOrNil() } func (m *multiPusher) PushRaw(r *RawMessage) error { diff --git a/push/sereverchan.go b/push/sereverchan.go index d8d094c..5f94b94 100644 --- a/push/sereverchan.go +++ b/push/sereverchan.go @@ -1,28 +1,32 @@ package push import ( + "fmt" + "github.com/imroc/req/v3" "github.com/kataras/golog" "github.com/pkg/errors" - serverchan "github.com/rayepeng/serverchan" + "github.com/zema1/watchvuln/util" ) var _ = TextPusher(&ServerChan{}) type ServerChan struct { - client *serverchan.ServerChan - log *golog.Logger + pushUrl string + log *golog.Logger + client *req.Client } func NewServerChan(botKey string) TextPusher { return &ServerChan{ - client: serverchan.NewServerChan(botKey), - log: golog.Child("[pusher-server-chan]"), + pushUrl: fmt.Sprintf("https://sctapi.ftqq.com/%s.send", botKey), + log: golog.Child("[pusher-server-chan]"), + client: util.NewHttpClient(), } } func (d *ServerChan) PushText(s string) error { d.log.Infof("sending text %s", s) - _, err := d.client.Send("", s) + err := d.send("", s) if err != nil { return errors.Wrap(err, "server-chan") } @@ -31,9 +35,18 @@ func (d *ServerChan) PushText(s string) error { func (d *ServerChan) PushMarkdown(title, content string) error { d.log.Infof("sending markdown %s", title) - _, err := d.client.Send(title, content) + err := d.send(title, content) if err != nil { return errors.Wrap(err, "server-chan") } return nil } + +func (d *ServerChan) send(text string, desp string) error { + body := map[string]string{ + "text": text, + "desp": desp, + } + _, err := d.client.R().SetBodyJsonMarshal(body).Post(d.pushUrl) + return err +} diff --git a/push/webhook.go b/push/webhook.go index d41f7ba..6218677 100644 --- a/push/webhook.go +++ b/push/webhook.go @@ -1,13 +1,10 @@ package push import ( - "bytes" - "encoding/json" - "github.com/pkg/errors" - "io" - "net/http" - + "github.com/imroc/req/v3" "github.com/kataras/golog" + "github.com/pkg/errors" + "github.com/zema1/watchvuln/util" ) var _ = RawPusher(&Webhook{}) @@ -15,45 +12,23 @@ var _ = RawPusher(&Webhook{}) type Webhook struct { url string log *golog.Logger - client *http.Client + client *req.Client } func NewWebhook(url string) RawPusher { return &Webhook{ url: url, log: golog.Child("[webhook]"), - client: &http.Client{}, + client: util.NewHttpClient(), } } func (m *Webhook) PushRaw(r *RawMessage) error { m.log.Infof("sending webhook data %s, %v", r.Type, r.Content) - postBody, _ := json.Marshal(r) - resp, err := m.doPostRequest(m.url, "application/json", postBody) + resp, err := m.client.R().SetBodyJsonMarshal(r).Post(m.url) if err != nil { - return err + return errors.Wrap(err, "webhook") } - m.log.Infof("raw response from server: %s", string(resp)) + m.log.Infof("raw response from server: %s", resp.String()) return nil } - -func (m *Webhook) doPostRequest(url string, contentType string, body []byte) ([]byte, error) { - req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) - if err != nil { - return nil, errors.Wrap(err, "create request") - } - - req.Header.Set("Content-Type", contentType) - - resp, err := m.client.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send request") - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) // 使用 io.ReadAll 替代 ioutil.ReadAll。 - if err != nil { - return nil, errors.Wrap(err, "read body") - } - return respBody, nil -} diff --git a/util/http.go b/util/http.go new file mode 100644 index 0000000..50b7aa8 --- /dev/null +++ b/util/http.go @@ -0,0 +1,51 @@ +package util + +import ( + "context" + "errors" + "github.com/imroc/req/v3" + "github.com/kataras/golog" + "time" +) + +func NewHttpClient() *req.Client { + client := req.C() + client. + ImpersonateChrome(). + SetTimeout(10 * time.Second). + SetCommonRetryCount(3). + SetCookieJar(nil). + SetCommonRetryInterval(func(resp *req.Response, attempt int) time.Duration { + if errors.Is(resp.Err, context.Canceled) { + return 0 + } + return time.Second * 5 + }). + SetCommonRetryHook(func(resp *req.Response, err error) { + if err != nil { + if !errors.Is(err, context.Canceled) { + golog.Warnf("retrying as %s", err) + } + } + }).SetCommonRetryCondition(func(resp *req.Response, err error) bool { + if err != nil { + return !errors.Is(err, context.Canceled) + } + return false + }) + return client +} + +func WrapApiClient(client *req.Client) *req.Client { + return client.SetCommonHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", + "Content-Type": "application/json", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "sec-ch-ua": `"Microsoft Edge";v="111", "Not(A:Brand";v="8", "Chromium";v="111"`, + "sec-ch-ua-mobile": `?0`, + "sec-ch-ua-platform": `"Windows"`, + }) +}