feat: allow failure when initilising database

CHANGELOG.md

@@ -1,3 +1,13 @@
+## v1.4.0 (2023.11.21)
+
+## 变更
+
+- 默认启用全部数据源, `seebug` 也默认启用了
+- 某个数据源初始化失败不会再推出，而是成功几个用几个并给出提示信息
+
+## 修复
+- 修复 `threatbook` 数据源异常结束的问题 [#57](https://github.com/zema1/watchvuln/issues/57)
+
 ## v1.3.0 (2023.11.20)
 
 ### 新增

ctrl/ctrl.go

@@ -107,26 +107,28 @@ func NewApp(config *WatchVulnAppConfig, textPusher push.TextPusher, rawPusher pu
 
 func (w *WatchVulnApp) Run(ctx context.Context) error {
 	w.log.Infof("initialize local database..")
-	if err := w.initData(ctx); err != nil {
-		return err
-	}
-	w.log.Infof("grabber finished successfully")
-
+	success, fail := w.initData(ctx)
+	w.grabbers = success
 	localCount, err := w.db.VulnInformation.Query().Count(ctx)
 	if err != nil {
 		return err
 	}
 	w.log.Infof("system init finished, local database has %d vulns", localCount)
 	if !w.config.NoStartMessage {
-		providers := make([]*grab.Provider, 0, 3)
+		providers := make([]*grab.Provider, 0, 10)
+		failed := make([]*grab.Provider, 0, 10)
 		for _, p := range w.grabbers {
 			providers = append(providers, p.ProviderInfo())
 		}
+		for _, p := range fail {
+			failed = append(failed, p.ProviderInfo())
+		}
 		msg := &push.InitialMessage{
-			Version:   w.config.Version,
-			VulnCount: localCount,
-			Interval:  w.config.Interval.String(),
-			Provider:  providers,
+			Version:        w.config.Version,
+			VulnCount:      localCount,
+			Interval:       w.config.Interval.String(),
+			Provider:       providers,
+			FailedProvider: failed,
 		}
 		if err := w.textPusher.PushMarkdown("WatchVuln 初始化完成", push.RenderInitialMsg(msg)); err != nil {
 			return err
@@ -241,32 +243,37 @@ func (w *WatchVulnApp) Close() {
 	_ = w.db.Close()
 }
 
-func (w *WatchVulnApp) initData(ctx context.Context) error {
-	eg, ctx := errgroup.WithContext(ctx)
+func (w *WatchVulnApp) initData(ctx context.Context) ([]grab.Grabber, []grab.Grabber) {
+	var eg errgroup.Group
 	eg.SetLimit(len(w.grabbers))
+	var success []grab.Grabber
+	var fail []grab.Grabber
 	for _, grabber := range w.grabbers {
 		gb := grabber
 		eg.Go(func() error {
 			source := gb.ProviderInfo()
 			w.log.Infof("start to init data from %s", source.Name)
 			initVulns, err := gb.GetUpdate(ctx, InitPageLimit)
 			if err != nil {
+				fail = append(fail, gb)
 				return errors.Wrap(err, source.Name)
 			}
 
 			for _, data := range initVulns {
 				if _, err = w.createOrUpdate(ctx, source, data); err != nil {
+					fail = append(fail, gb)
 					return errors.Wrap(errors.Wrap(err, data.String()), source.Name)
 				}
 			}
+			success = append(success, gb)
 			return nil
 		})
 	}
 	err := eg.Wait()
 	if err != nil {
-		return errors.Wrap(err, "init data")
+		w.log.Error(errors.Wrap(err, "init data"))
 	}
-	return nil
+	return success, fail
 }
 
 func (w *WatchVulnApp) collectUpdate(ctx context.Context) ([]*grab.VulnInfo, error) {

examples/webhook/README.md

@@ -35,7 +35,8 @@ webhook 的数据统一使用 json 格式发送，最外层的定义为:
         "display_name": "OSCS开源安全情报预警",
         "link": "https://www.oscs1024.com/cm"
       }
-    ]
+    ],
+    "failed_provider": []
   }
 }
 ```

examples/webhook/main.go

@@ -55,7 +55,7 @@ func handleWebhookData(writer http.ResponseWriter, request *http.Request) {
 			return
 		}
 		fmt.Printf("msg: %s\n", string(wd.Content))
-		//fmt.Printf("unmarshal: %+v\n", msg)
+		fmt.Printf("unmarshal: %+v\n", msg)
 	case push.RawMessageTypeText:
 		fmt.Println("recv text data:")
 		var msg push.TextMessage

main.go

@@ -16,7 +16,7 @@ import (
 )
 
 var log = golog.Child("[main]")
-var Version = "v1.3.0"
+var Version = "v1.4.0"
 
 func main() {
 	golog.Default.SetLevel("info")
@@ -98,7 +98,7 @@ func main() {
 			Name:     "sources",
 			Aliases:  []string{"s"},
 			Usage:    "set vuln sources",
-			Value:    "avd,nox,oscs,threatbook",
+			Value:    "avd,nox,oscs,threatbook,seebug",
 			Category: "[Launch Options]",
 		},
 		&cli.StringFlag{

push/msg.go

@@ -38,11 +38,14 @@ const vulnInfoMsg = `
 
 const initialMsg = `
 数据初始化完成，当前版本 {{ .Version }}， 本地漏洞数量: {{ .VulnCount }}, 检查周期: {{ .Interval }} 
+
 启用的数据源:
+{{ range .Provider }}- [{{ .DisplayName }}]({{ .Link -}})
+{{ end }}
 
-{{ range .Provider }}
-- [{{ .DisplayName }}]({{ .Link -}})
-{{- end }}
+失败的数据源:
+{{ range .Provider }}- [{{ .DisplayName }}]({{ .Link }})
+{{ end -}}
 `
 
 var (
@@ -74,10 +77,11 @@ func RenderInitialMsg(v *InitialMessage) string {
 }
 
 type InitialMessage struct {
-	Version   string           `json:"version"`
-	VulnCount int              `json:"vuln_count"`
-	Interval  string           `json:"interval"`
-	Provider  []*grab.Provider `json:"provider"`
+	Version        string           `json:"version"`
+	VulnCount      int              `json:"vuln_count"`
+	Interval       string           `json:"interval"`
+	Provider       []*grab.Provider `json:"provider"`
+	FailedProvider []*grab.Provider `json:"failed_provider"`
 }
 
 type TextMessage struct {