diff --git a/Dockerfile b/Dockerfile index 9344c10..681e74b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,4 @@ WORKDIR /app COPY --from=builder /app/main /app/main ENV DINGDING_ACCESS_TOKEN="" DINGDING_SECRET="" WECHATWORK_KEY="" BARK_URL="" INTERVAL=30m -ENTRYPOINT ["/app/main"] +ENTRYPOINT ["/app/main"] \ No newline at end of file diff --git a/ctrl/config.go b/ctrl/config.go index a971947..a7bad11 100644 --- a/ctrl/config.go +++ b/ctrl/config.go @@ -52,7 +52,7 @@ func (c *WatchVulnAppConfig) Init() { c.Interval = "1h" } if len(c.Sources) == 0 { - c.Sources = []string{"avd", "nox", "oscs", "threatbook", "seebug", "struts2", "kev"} + c.Sources = []string{"avd", "chaitin", "nox", "oscs", "threatbook", "seebug", "struts2", "kev"} } } diff --git a/ctrl/ctrl.go b/ctrl/ctrl.go index 70f8566..6132178 100644 --- a/ctrl/ctrl.go +++ b/ctrl/ctrl.go @@ -80,6 +80,8 @@ func NewApp(config *WatchVulnAppConfig) (*WatchVulnApp, error) { for _, part := range config.Sources { part = strings.ToLower(strings.TrimSpace(part)) switch part { + case "chaitin": + grabs = append(grabs, grab.NewChaitinCrawler()) case "avd": grabs = append(grabs, grab.NewAVDCrawler()) case "nox", "ti": diff --git a/docker-compose.yaml b/docker-compose.yaml index 60ad7cd..f9ecdb5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,4 +19,4 @@ services: POSTGRES_USER: watchvuln POSTGRES_PASSWORD: watchvuln volumes: - - "./data/postgresql:/var/lib/postgresql/data" \ No newline at end of file + - "./data/postgresql:/var/lib/postgresql/data" diff --git a/grab/chaitin.go b/grab/chaitin.go new file mode 100644 index 0000000..993240e --- /dev/null +++ b/grab/chaitin.go @@ -0,0 +1,175 @@ +package grab + +import ( + "context" + "fmt" + "github.com/zema1/watchvuln/util" + "strings" + "time" + + "github.com/imroc/req/v3" + "github.com/kataras/golog" +) + +type ChaitinCrawler struct { + client *req.Client + log *golog.Logger +} + +func NewChaitinCrawler() Grabber { + client := util.WrapApiClient(util.NewHttpClient()) + client.SetCommonHeader("Referer", "https://stack.chaitin.com/vuldb/index") + client.SetCommonHeader("Origin", "https://stack.chaitin.com") + + c := &ChaitinCrawler{ + log: golog.Child("[chaitin]"), + client: client, + } + return c +} + +func (t *ChaitinCrawler) ProviderInfo() *Provider { + return &Provider{ + Name: "chaitin", + DisplayName: "长亭漏洞库", + Link: "https://stack.chaitin.com/vuldb/index", + } +} + +func (t *ChaitinCrawler) GetUpdate(ctx context.Context, pageLimit int) ([]*VulnInfo, error) { + var results []*VulnInfo + resp, err := t.client.R(). + SetContext(ctx). + Get("https://stack.chaitin.com/api/v2/vuln/list/?limit=15&offset=0&search=CT-") + if err != nil { + return nil, err + } + fmt.Print(resp.Dump()) + var body ChaitinResp + if err = resp.UnmarshalJson(&body); err != nil { + return nil, err + } + errCount := 0 + nextReqUrl := "https://stack.chaitin.com/api/v2/vuln/list/?limit=15&offset=0&search=CT-" + for i := 1; i <= pageLimit; i++ { + if errCount > 5 { + t.log.Errorf("get page %d failed more than 5 times, stop", i) + break + } + // CT- 为长亭漏洞库的标识 + t.log.Infof("req url: %s", nextReqUrl) + nextReqUrl = strings.Replace(nextReqUrl, "http://", "https://", -1) + resp, err := t.client.R(). + SetContext(ctx). + Get(nextReqUrl) + if err != nil { + t.log.Errorf("get page %d failed: %v", i, err) + i = i - 1 + errCount++ + time.Sleep(5 * time.Second) + continue + } + var body ChaitinResp + if err = resp.UnmarshalJson(&body); err != nil { + t.log.Errorf("unmarshal page %d failed: %v", i, err) + errCount++ + i = i - 1 + time.Sleep(5 * time.Second) + continue + } + nextReqUrl = body.Data.Next + for _, d := range body.Data.List { + severity := Low + switch d.Severity { + case "low": + severity = Low + case "medium": + severity = Medium + case "high": + severity = High + case "critical": + severity = Critical + } + + DisclosureDate := "" + if d.DisclosureDate != nil { + DisclosureDate = *d.DisclosureDate + } + info := &VulnInfo{ + UniqueKey: d.CtId, + Title: d.Title, + Description: d.Summary, + Severity: severity, + CVE: d.CtId, + Disclosure: DisclosureDate, + References: nil, + //Tags: tags, + Solutions: "", + From: "https://stack.chaitin.com/vuldb/detail/" + d.Id, + Creator: t, + } + results = append(results, info) + } + } + + // 根据 ID 去重 + uniqResults := make(map[string]*VulnInfo) + for _, info := range results { + uniqResults[info.UniqueKey] = info + } + // 保持顺序 + newResults := make([]*VulnInfo, 0, len(uniqResults)) + for _, info := range results { + if uniqResults[info.UniqueKey] == nil { + continue + } + newResults = append(newResults, info) + uniqResults[info.UniqueKey] = nil + } + t.log.Infof("got %d vulns from chaitin api", len(newResults)) + return newResults, nil +} + +func (t *ChaitinCrawler) IsValuable(info *VulnInfo) bool { + if info.Severity != High && info.Severity != Critical { + return false + } + return true +} + +type ChaitinResp struct { + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Next string `json:"next"` + Previous interface{} `json:"previous"` + List []struct { + Id string `json:"id"` + Title string `json:"title"` + TitleEn interface{} `json:"title_en"` + Summary string `json:"summary"` + SummaryEn *string `json:"summary_en"` + Weakness string `json:"weakness"` + Severity string `json:"severity"` + Cvss3 interface{} `json:"cvss3"` + Cvss2 interface{} `json:"cvss2"` + CtId string `json:"ct_id"` + CveId string `json:"cve_id"` + CnvdId interface{} `json:"cnvd_id"` + CnnvdId *string `json:"cnnvd_id"` + FixSteps interface{} `json:"fix_steps"` + References *string `json:"references"` + DisclosureDate *string `json:"disclosure_date"` + PocDisclosureDate interface{} `json:"poc_disclosure_date"` + ExpDisclosureDate interface{} `json:"exp_disclosure_date"` + PatchDate interface{} `json:"patch_date"` + Impact interface{} `json:"impact"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + PocId interface{} `json:"poc_id"` + BountyTime interface{} `json:"bounty_time"` + BountyRewardScore int `json:"bounty_reward_score"` + } `json:"list"` + } `json:"data"` + Code int `json:"code"` +} diff --git a/grab/chaitin_test.go b/grab/chaitin_test.go new file mode 100644 index 0000000..2560d26 --- /dev/null +++ b/grab/chaitin_test.go @@ -0,0 +1,29 @@ +package grab + +import ( + "context" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestChaitin(t *testing.T) { + assert := require.New(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + grab := NewChaitinCrawler() + vulns, err := grab.GetUpdate(ctx, 3) + assert.Nil(err) + + count := 0 + for _, v := range vulns { + t.Logf("get vuln info %s", v) + count++ + assert.NotEmpty(v.UniqueKey) + assert.NotEmpty(v.Description) + assert.NotEmpty(v.Title) + assert.NotEmpty(v.Disclosure) + assert.NotEmpty(v.From) + } + assert.Equal(count, 30) +} diff --git a/main.go b/main.go index b66f6ad..e5ecc4d 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,7 @@ func main() { Name: "sources", Aliases: []string{"s"}, Usage: "set vuln sources", - Value: "avd,nox,oscs,threatbook,seebug,struts2,kev", + Value: "avd,nox,oscs,threatbook,seebug,struts2,kev,chaitin", Category: "[Launch Options]", }, &cli.StringFlag{