diff --git a/README.md b/README.md index 128bddfb..09b79418 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,43 @@ scrape_configs: target_label: vhost # and store it in 'vhost' label ``` +## Dynamic Probes + +In addition to static probes, configured via a [configuration file](CONFIGURATION.md), blackbox exporter supports the probe configuration via HTTP +query parameter on the endpoint `/scrape/dynamic`. + +Most of the probe configuration can be configured via HTTP query parameter with the schema `.`, e.g. `http.method=POST`. +HTTP query parameter supports only the configuration of top-level parameter. The configuration of sub level parameters needs to be passed as JSON document, e.g. +`http.http_client_config={"tls_config":{"insecure_skip_verify":true}}` instead `http.http_client_config.tls_config.insecure_skip_verify=true`. + +### Examples + +- `http://localhost:9115/probe/dynamic?target=example.com&prober=http` +- `http://localhost:9115/probe/dynamic?target=expired.badssl.com&prober=http&http.http_client_config={"tls_config":{"insecure_skip_verify":true}}` + +Example config: +```yml +scrape_configs: + - job_name: 'blackbox' + metrics_path: /probe/dynamic + params: + prober: [http] + http.method: ["POST"] + http.valid_http_versions: ["200", "204"] + http.http_client_config: ['{"tls_config":{"insecure_skip_verify":true}}'] + static_configs: + - targets: + - http://prometheus.io # Target to probe with http. + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port. +``` + + ## Permissions The ICMP probe requires elevated privileges to function: diff --git a/config/binding.go b/config/binding.go new file mode 100644 index 00000000..11ddfa33 --- /dev/null +++ b/config/binding.go @@ -0,0 +1,82 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "reflect" + + "github.com/bytedance/go-tagexpr/v2/binding" + "github.com/prometheus/common/config" + "gopkg.in/yaml.v3" +) + +func InitializeBinding() { + binding.MustRegTypeUnmarshal(reflect.TypeOf(Regexp{}), func(v string, emptyAsZero bool) (reflect.Value, error) { + if v == "" && emptyAsZero { + return reflect.ValueOf(Regexp{}), nil + } + + t, err := NewRegexp(v) + if err != nil { + return reflect.ValueOf(Regexp{}), err + } + + return reflect.ValueOf(t), nil + }) + + binding.MustRegTypeUnmarshal(reflect.TypeOf(HeaderMatch{}), func(v string, emptyAsZero bool) (reflect.Value, error) { + if v == "" && emptyAsZero { + return reflect.ValueOf(HeaderMatch{}), nil + } + + var c = &HeaderMatch{} + + decoder := yaml.NewDecoder(bytes.NewBufferString(v)) + decoder.KnownFields(true) + + if err := decoder.Decode(c); err != nil { + return reflect.ValueOf(HeaderMatch{}), err + } + + if err := c.Validate(); err != nil { + return reflect.ValueOf(HeaderMatch{}), err + } + + return reflect.ValueOf(*c), nil + }) + + binding.MustRegTypeUnmarshal(reflect.TypeOf(config.HTTPClientConfig{}), binderYamlDecoder(config.HTTPClientConfig{})) + binding.MustRegTypeUnmarshal(reflect.TypeOf(config.TLSConfig{}), binderYamlDecoder(config.TLSConfig{})) + binding.MustRegTypeUnmarshal(reflect.TypeOf(DNSRRValidator{}), binderYamlDecoder(DNSRRValidator{})) +} + +func binderYamlDecoder[T any](s T) func(v string, emptyAsZero bool) (reflect.Value, error) { + return func(v string, emptyAsZero bool) (reflect.Value, error) { + if v == "" && emptyAsZero { + return reflect.ValueOf(s), nil + } + + var c = &s + + decoder := yaml.NewDecoder(bytes.NewBufferString(v)) + decoder.KnownFields(true) + + if err := decoder.Decode(c); err != nil { + return reflect.ValueOf(s), err + } + + return reflect.ValueOf(*c), nil + } +} diff --git a/config/config.go b/config/config.go index b665a6e1..1b49243f 100644 --- a/config/config.go +++ b/config/config.go @@ -193,7 +193,7 @@ func MustNewRegexp(s string) Regexp { } type Module struct { - Prober string `yaml:"prober,omitempty"` + Prober string `yaml:"prober,omitempty" query:"prober"` Timeout time.Duration `yaml:"timeout,omitempty"` HTTP HTTPProbe `yaml:"http,omitempty"` TCP TCPProbe `yaml:"tcp,omitempty"` @@ -204,33 +204,33 @@ type Module struct { type HTTPProbe struct { // Defaults to 2xx. - ValidStatusCodes []int `yaml:"valid_status_codes,omitempty"` - ValidHTTPVersions []string `yaml:"valid_http_versions,omitempty"` - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SkipResolvePhaseWithProxy bool `yaml:"skip_resolve_phase_with_proxy,omitempty"` - NoFollowRedirects *bool `yaml:"no_follow_redirects,omitempty"` - FailIfSSL bool `yaml:"fail_if_ssl,omitempty"` - FailIfNotSSL bool `yaml:"fail_if_not_ssl,omitempty"` - Method string `yaml:"method,omitempty"` - Headers map[string]string `yaml:"headers,omitempty"` - FailIfBodyMatchesRegexp []Regexp `yaml:"fail_if_body_matches_regexp,omitempty"` - FailIfBodyNotMatchesRegexp []Regexp `yaml:"fail_if_body_not_matches_regexp,omitempty"` - FailIfHeaderMatchesRegexp []HeaderMatch `yaml:"fail_if_header_matches,omitempty"` - FailIfHeaderNotMatchesRegexp []HeaderMatch `yaml:"fail_if_header_not_matches,omitempty"` - Body string `yaml:"body,omitempty"` + ValidStatusCodes []int `yaml:"valid_status_codes,omitempty" query:"http.valid_status_codes[]"` + ValidHTTPVersions []string `yaml:"valid_http_versions,omitempty" query:"http.valid_http_versions[]"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" query:"http.preferred_ip_protocol"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" query:"http.ip_protocol_fallback"` + SkipResolvePhaseWithProxy bool `yaml:"skip_resolve_phase_with_proxy,omitempty" query:"http.skip_resolve_phase_with_proxy"` + NoFollowRedirects *bool `yaml:"no_follow_redirects,omitempty" query:"http.no_follow_redirects"` + FailIfSSL bool `yaml:"fail_if_ssl,omitempty" query:"http.fail_if_ssl"` + FailIfNotSSL bool `yaml:"fail_if_not_ssl,omitempty" query:"http.fail_if_not_ssl"` + Method string `yaml:"method,omitempty" query:"http.method"` + Headers map[string]string `yaml:"headers,omitempty" query:"http.headers"` + FailIfBodyMatchesRegexp []Regexp `yaml:"fail_if_body_matches_regexp,omitempty" query:"http.fail_if_body_matches_regexp[]"` + FailIfBodyNotMatchesRegexp []Regexp `yaml:"fail_if_body_not_matches_regexp,omitempty" query:"http.fail_if_body_not_matches_regexp[]"` + FailIfHeaderMatchesRegexp []HeaderMatch `yaml:"fail_if_header_matches,omitempty" query:"http.fail_if_header_matches[]"` + FailIfHeaderNotMatchesRegexp []HeaderMatch `yaml:"fail_if_header_not_matches,omitempty" query:"http.fail_if_header_not_matches[]"` + Body string `yaml:"body,omitempty" query:"http.body"` BodyFile string `yaml:"body_file,omitempty"` - HTTPClientConfig config.HTTPClientConfig `yaml:"http_client_config,inline"` - Compression string `yaml:"compression,omitempty"` - BodySizeLimit units.Base2Bytes `yaml:"body_size_limit,omitempty"` + HTTPClientConfig config.HTTPClientConfig `yaml:"http_client_config,inline" query:"http.http_client_config"` + Compression string `yaml:"compression,omitempty" query:"http.compression"` + BodySizeLimit units.Base2Bytes `yaml:"body_size_limit,omitempty" query:"http.body_size_limit"` } type GRPCProbe struct { - Service string `yaml:"service,omitempty"` - TLS bool `yaml:"tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty"` + Service string `yaml:"service,omitempty" query:"grpc.service"` + TLS bool `yaml:"tls,omitempty" query:"grpc.tls"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" query:"grpc.tls_config"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" query:"grpc.ip_protocol_fallback"` + PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty" query:"grpc.preferred_ip_protocols"` } type HeaderMatch struct { @@ -246,38 +246,38 @@ type QueryResponse struct { } type TCPProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - QueryResponse []QueryResponse `yaml:"query_response,omitempty"` - TLS bool `yaml:"tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" query:"tcp.preferred_ip_protocol"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" query:"tcp.ip_protocol_fallback"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" query:"tcp.source_ip_address"` + QueryResponse []QueryResponse `yaml:"query_response,omitempty" query:"tcp.query_response[]"` + TLS bool `yaml:"tls,omitempty" query:"tcp.tls"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" query:"tcp.tls_config"` } type ICMPProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` // Defaults to "ip6". - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - PayloadSize int `yaml:"payload_size,omitempty"` - DontFragment bool `yaml:"dont_fragment,omitempty"` - TTL int `yaml:"ttl,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" query:"icmp.preferred_ip_protocol"` // Defaults to "ip6". + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" query:"icmp.ip_protocol_fallback"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" query:"icmp.source_ip_address"` + PayloadSize int `yaml:"payload_size,omitempty" query:"icmp.payload_size"` + DontFragment bool `yaml:"dont_fragment,omitempty" query:"icmp.dont_fragment"` + TTL int `yaml:"ttl,omitempty" query:"icmp.ttl"` } type DNSProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - DNSOverTLS bool `yaml:"dns_over_tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - TransportProtocol string `yaml:"transport_protocol,omitempty"` - QueryClass string `yaml:"query_class,omitempty"` // Defaults to IN. - QueryName string `yaml:"query_name,omitempty"` - QueryType string `yaml:"query_type,omitempty"` // Defaults to ANY. - Recursion bool `yaml:"recursion_desired,omitempty"` // Defaults to true. - ValidRcodes []string `yaml:"valid_rcodes,omitempty"` // Defaults to NOERROR. - ValidateAnswer DNSRRValidator `yaml:"validate_answer_rrs,omitempty"` - ValidateAuthority DNSRRValidator `yaml:"validate_authority_rrs,omitempty"` - ValidateAdditional DNSRRValidator `yaml:"validate_additional_rrs,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" query:"dns.preferred_ip_protocol"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" query:"dns.ip_protocol_fallback"` + DNSOverTLS bool `yaml:"dns_over_tls,omitempty" query:"dns.dns_over_tls"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" query:"dns.tls_config"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" query:"dns.source_ip_address"` + TransportProtocol string `yaml:"transport_protocol,omitempty" query:"dns.transport_protocol"` + QueryClass string `yaml:"query_class,omitempty" query:"dns.query_class"` // Defaults to IN. + QueryName string `yaml:"query_name,omitempty" query:"dns.query_name"` + QueryType string `yaml:"query_type,omitempty" query:"dns.query_type"` // Defaults to ANY. + Recursion bool `yaml:"recursion_desired,omitempty" query:"dns.recursion_desired"` // Defaults to true. + ValidRcodes []string `yaml:"valid_rcodes,omitempty" query:"dns.valid_rcodes[]"` // Defaults to NOERROR. + ValidateAnswer DNSRRValidator `yaml:"validate_answer_rrs,omitempty" query:"dns.validate_answer_rrs"` + ValidateAuthority DNSRRValidator `yaml:"validate_authority_rrs,omitempty" query:"dns.validate_authority_rrs"` + ValidateAdditional DNSRRValidator `yaml:"validate_additional_rrs,omitempty" query:"dns.validate_additional_rrs"` } type DNSRRValidator struct { @@ -314,6 +314,14 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + if err := s.Validate(); err != nil { + return err + } + + return nil +} + +func (s *HTTPProbe) Validate() error { // BodySizeLimit == 0 means no limit. By leaving it at 0 we // avoid setting up the limiter. if s.BodySizeLimit < 0 || s.BodySizeLimit == math.MaxInt64 { @@ -344,7 +352,6 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { } } } - return nil } @@ -365,6 +372,15 @@ func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(s)); err != nil { return err } + + if err := s.Validate(); err != nil { + return err + } + + return nil +} + +func (s *DNSProbe) Validate() error { if s.QueryName == "" { return errors.New("query name must be set for DNS module") } @@ -378,7 +394,6 @@ func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return fmt.Errorf("query type '%s' is not valid", s.QueryType) } } - return nil } @@ -409,6 +424,13 @@ func (s *ICMPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + if err := s.Validate(); err != nil { + return err + } + return nil +} + +func (s *ICMPProbe) Validate() error { if runtime.GOOS == "windows" && s.DontFragment { return errors.New("\"dont_fragment\" is not supported on windows platforms") } @@ -439,6 +461,14 @@ func (s *HeaderMatch) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + if err := s.Validate(); err != nil { + return err + } + + return nil +} + +func (s *HeaderMatch) Validate() error { if s.Header == "" { return errors.New("header name must be set for HTTP header matchers") } @@ -446,7 +476,6 @@ func (s *HeaderMatch) UnmarshalYAML(unmarshal func(interface{}) error) error { if s.Regexp.Regexp == nil || s.Regexp.Regexp.String() == "" { return errors.New("regexp must be set for HTTP header matchers") } - return nil } diff --git a/go.mod b/go.mod index 8d9366c5..6286dba2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/alecthomas/kingpin/v2 v2.3.2 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/andybalholm/brotli v1.0.5 + github.com/bytedance/go-tagexpr/v2 v2.9.8 github.com/go-kit/log v0.2.1 github.com/miekg/dns v1.1.55 github.com/pkg/errors v0.9.1 @@ -20,6 +21,8 @@ require ( ) require ( + github.com/andeya/ameda v1.5.3 // indirect + github.com/andeya/goutil v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -29,7 +32,10 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/nyaruka/phonenumbers v1.0.55 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.8.0 // indirect diff --git a/go.sum b/go.sum index 516bfc7a..e056bd78 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,16 @@ github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWr github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/andeya/ameda v1.5.3 h1:SvqnhQPZwwabS8HQTRGfJwWPl2w9ZIPInHAw9aE1Wlk= +github.com/andeya/ameda v1.5.3/go.mod h1:FQDHRe1I995v6GG+8aJ7UIUToEmbdTJn/U26NCPIgXQ= +github.com/andeya/goutil v1.0.1 h1:eiYwVyAnnK0dXU5FJsNjExkJW4exUGn/xefPt3k4eXg= +github.com/andeya/goutil v1.0.1/go.mod h1:jEG5/QnnhG7yGxwFUX6Q+JGMif7sjdHmmNVjn7nhJDo= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytedance/go-tagexpr/v2 v2.9.8 h1:p1PWxiUPxAdzreRBRbC9M2k7tf1cZYkds58NLypXbg4= +github.com/bytedance/go-tagexpr/v2 v2.9.8/go.mod h1:UAyKh4ZRLBPGsyTRFZoPqTni1TlojMdOJXQnEIPCX84= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -13,6 +19,7 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -20,6 +27,7 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -36,6 +44,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nyaruka/phonenumbers v1.0.55 h1:bj0nTO88Y68KeUQ/n3Lo2KgK7lM1hF7L9NFuwcCl3yg= +github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -52,8 +62,15 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -88,6 +105,7 @@ google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -95,5 +113,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 2699b6f8..c7039bec 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,8 @@ func run() int { level.Info(logger).Log("msg", "Loaded config file") + config.InitializeBinding() + // Infer or set Blackbox exporter externalURL listenAddrs := toolkitFlags.WebListenAddresses if *externalURL == "" && *toolkitFlags.WebSystemdSocket { @@ -184,6 +186,9 @@ func run() int { sc.Unlock() prober.Handler(w, r, conf, logger, rh, *timeoutOffset, nil, moduleUnknownCounter) }) + http.HandleFunc(path.Join(*routePrefix, "/probe/dynamic"), func(w http.ResponseWriter, r *http.Request) { + prober.DynamicHandler(w, r, logger, rh, *timeoutOffset, nil, moduleUnknownCounter) + }) http.HandleFunc(*routePrefix, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") w.Write([]byte(` diff --git a/main_test.go b/main_test.go index 45c9abe1..ad436734 100644 --- a/main_test.go +++ b/main_test.go @@ -13,7 +13,9 @@ package main -import "testing" +import ( + "testing" +) func TestComputeExternalURL(t *testing.T) { tests := []struct { diff --git a/prober/handler.go b/prober/handler.go index 1caa2f92..26c0c76b 100644 --- a/prober/handler.go +++ b/prober/handler.go @@ -23,6 +23,7 @@ import ( "strconv" "time" + "github.com/bytedance/go-tagexpr/v2/binding" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/blackbox_exporter/config" @@ -138,6 +139,50 @@ func Handler(w http.ResponseWriter, r *http.Request, c *config.Config, logger lo h.ServeHTTP(w, r) } +func DynamicHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, rh *ResultHistory, timeoutOffset float64, params url.Values, + moduleUnknownCounter prometheus.Counter) { + + if params == nil { + params = r.URL.Query() + } + + module := config.DefaultModule + binder := binding.New(nil) + + if err := binder.BindAndValidate(&module, r, nil); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + switch module.Prober { + case "http": + if err := module.HTTP.Validate(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + break + case "dns": + if err := module.DNS.Validate(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + break + case "icmp": + if err := module.ICMP.Validate(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + break + } + + params.Set("module", "custom") + customConfig := &config.Config{Modules: map[string]config.Module{ + "custom": module, + }} + + Handler(w, r, customConfig, logger, rh, timeoutOffset, params, moduleUnknownCounter) +} + func setHTTPHost(hostname string, module *config.Module) error { // By creating a new hashmap and copying values there we // ensure that the initial configuration remain intact. diff --git a/prober/handler_test.go b/prober/handler_test.go index 885f751d..f99b6e70 100644 --- a/prober/handler_test.go +++ b/prober/handler_test.go @@ -16,6 +16,7 @@ package prober import ( "bytes" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -257,3 +258,234 @@ func TestTCPHostnameParam(t *testing.T) { } } + +func TestDynamicProbe(t *testing.T) { + const skipTlsVerifyParam = `&http.http_client_config={"tls_config":{"insecure_skip_verify":true}}` + + mockHTTPServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Custom-Header", "value") + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprintln(w, r.Method+": Hello World") + })) + defer mockHTTPServer.Close() + + mockHTTPSServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Custom-Header", "value") + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprintln(w, r.Method+": Hello World") + })) + defer mockHTTPSServer.Close() + + tests := []struct { + name string + prober string + target string + queryParams string + error string + expectBody []string + }{ + { + name: "http default", + prober: "http", + target: mockHTTPServer.URL, + queryParams: ``, + error: "", + expectBody: []string{"probe_success 1"}, + }, + { + name: "http.valid_http_versions", + prober: "http", + target: mockHTTPSServer.URL, + queryParams: `http.valid_http_versions[]=HTTP/2` + skipTlsVerifyParam, + error: "", + expectBody: []string{"HTTP/2", "probe_http_version 1.1", "probe_success 0"}, + }, + { + name: "http.preferred_ip_protocol", + prober: "http", + target: mockHTTPSServer.URL, + queryParams: `http.preferred_ip_protocol=6&http.ip_protocol_fallback=false` + skipTlsVerifyParam, + error: "", + expectBody: []string{`preferred_ip_protocol: "6"`, "probe_ip_protocol 4", "probe_success 1"}, + }, + { + name: "http.valid_http_versions", + prober: "http", + target: mockHTTPSServer.URL, + queryParams: `http.valid_http_versions[]=204` + skipTlsVerifyParam, + error: "", + expectBody: []string{"204", "probe_success 0"}, + }, + { + name: "http.fail_if_body_matches_regexp", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.fail_if_body_matches_regexp[]=He\S.*`, + error: "", + expectBody: []string{"fail_if_body_matches_regexp", `- He\S.*`, "probe_failed_due_to_regex 1", "probe_success 0"}, + }, + { + name: "http.fail_if_body_not_matches_regexp", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.fail_if_body_not_matches_regexp[]=\d%2B`, + error: "", + expectBody: []string{"fail_if_body_not_matches_regexp:", `- \d+`, "probe_failed_due_to_regex 1", "probe_success 0"}, + }, + { + name: "http.method", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.method=POST&http.fail_if_body_not_matches_regexp[]=POST:`, + error: "", + expectBody: []string{"method: POST", "probe_failed_due_to_regex 0", "probe_success 1"}, + }, + { + name: "http.fail_if_ssl", + prober: "http", + target: mockHTTPSServer.URL + skipTlsVerifyParam, + queryParams: `http.fail_if_ssl=true`, + error: "", + expectBody: []string{"fail_if_ssl: true", "probe_success 0"}, + }, + { + name: "http.fail_if_not_ssl", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.fail_if_not_ssl=true`, + error: "", + expectBody: []string{"fail_if_not_ssl: true", "probe_success 0"}, + }, + { + name: "http.fail_if_header_matches", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.fail_if_header_matches[]={"header":"X-Custom-Header","regexp":"value"}`, + error: "", + expectBody: []string{"fail_if_header_matches", "- header: X-Custom-Header", "probe_failed_due_to_regex 1", "probe_success 0"}, + }, + { + name: "http.fail_if_header_not_matches", + prober: "http", + target: mockHTTPServer.URL, + queryParams: `http.fail_if_header_not_matches[]={"header":"X-Custom-Header","regexp":"value"}`, + error: "", + expectBody: []string{"fail_if_header_not_matches", "- header: X-Custom-Header", "probe_success 1"}, + }, + { + name: "tcp default", + prober: "tcp", + target: mockHTTPSServer.Listener.Addr().String(), + queryParams: ``, + error: "", + expectBody: []string{"probe_success 1"}, + }, + { + name: "tcp.tls", + prober: "tcp", + target: mockHTTPSServer.Listener.Addr().String(), + queryParams: `tcp.tls=true&tcp.tls_config={"insecure_skip_verify":true}`, + error: "", + expectBody: []string{"tls: true", "probe_success 1"}, + }, + { + name: "dns.query_name", + prober: "dns", + target: `8.8.8.8:53`, + queryParams: ``, + error: "query name must be set for DNS module", + expectBody: []string{}, + }, + { + name: "dns.query_name", + prober: "dns", + target: `8.8.8.8:53`, + queryParams: `dns.query_name=.&dns.query_type=SOA`, + error: "", + expectBody: []string{"query_type: SOA", "probe_success 1"}, + }, + { + name: "dns.recursion_desired", + prober: "dns", + target: `8.8.8.8:53`, + queryParams: `dns.recursion_desired=true&dns.query_name=.&dns.query_type=SOA`, + error: "", + expectBody: []string{"recursion_desired: true", "probe_success 1"}, + }, + { + name: "dns.valid_rcodes", + prober: "dns", + target: `8.8.8.8:53`, + queryParams: `dns.valid_rcodes[]=NOERROR&dns.query_name=.&dns.query_type=SOA`, + error: "", + expectBody: []string{"valid_rcodes:", "NOERROR", "probe_success 1"}, + }, + { + name: "dns.validate_answer_rrs", + prober: "dns", + target: `8.8.8.8:53`, + queryParams: `dns.validate_answer_rrs={"fail_if_matches_regexp":[".*"]}&dns.query_name=.&dns.query_type=SOA`, + error: "", + expectBody: []string{"validate_answer_rrs:", "fail_if_matches_regexp", ".*", "Answer RRs validation failed", "probe_success 0"}, + }, + { + name: "icmp.ttl", + prober: "icmp", + target: `127.0.0.1`, + queryParams: `icmp.ttl=300`, + error: `"ttl" cannot exceed 255`, + expectBody: []string{"validate_answer_rrs:", "fail_if_matches_regexp", ".*", "Answer RRs validation failed", "probe_success 0"}, + }, + { + name: "icmp", + prober: "icmp", + target: `8.8.8.8`, + queryParams: ``, + error: ``, + expectBody: []string{"probe_success 1"}, + }, + } + + config.InitializeBinding() + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + DynamicHandler(w, r, log.NewNopLogger(), &ResultHistory{MaxResults: 0}, 0.5, nil, nil) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + probeUrl := fmt.Sprintf("%s/probe/custom?prober=%s&target=%s&debug=true&%s", server.URL, tc.prober, tc.target, tc.queryParams) + + resp, err := http.Get(probeUrl) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + body := string(bodyBytes) + + if tc.error != "" { + if !strings.Contains(body, tc.error) { + t.Errorf("expected error %s not found in anser:\n%s", tc.error, body) + } + } else { + if resp.StatusCode != 200 { + t.Errorf("unexpected http status %d", resp.StatusCode) + } + + for _, expectMetric := range tc.expectBody { + if !strings.Contains(body, expectMetric) { + t.Errorf("expected metric %s not found in anser:\n%s", expectMetric, body) + } + } + } + }) + } +}