diff --git a/config.yml b/config.yml index 89f23ce..70c678e 100644 --- a/config.yml +++ b/config.yml @@ -2,7 +2,5 @@ server: bind: ":8080" timeout: 1000 #ms targets: - - "http://localhost:9100/metrics" - - "http://localhost:9100/metrics" - - "http://localhost:9100/metrics" - - "http://localhost:9100/metrics" + - "http://localhost/prom.txt" + - "http://localhost/prom2.txt" diff --git a/fixture/histogram.golden.txt b/fixture/histogram.golden.txt new file mode 100644 index 0000000..3054ba8 --- /dev/null +++ b/fixture/histogram.golden.txt @@ -0,0 +1,10 @@ +# HELP http_request_duration_seconds A histogram of the request duration. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{ae_source="foo",le="0.05"} 24054 +http_request_duration_seconds_bucket{ae_source="foo",le="0.1"} 33444 +http_request_duration_seconds_bucket{ae_source="foo",le="0.2"} 100392 +http_request_duration_seconds_bucket{ae_source="foo",le="0.5"} 129389 +http_request_duration_seconds_bucket{ae_source="foo",le="1"} 133988 +http_request_duration_seconds_bucket{ae_source="foo",le="+Inf"} 144320 +http_request_duration_seconds_sum{ae_source="foo"} 53423 +http_request_duration_seconds_count{ae_source="foo"} 144320 diff --git a/fixture/histogram.txt b/fixture/histogram.txt new file mode 100644 index 0000000..f5f4994 --- /dev/null +++ b/fixture/histogram.txt @@ -0,0 +1,9 @@ +# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{method="post",code="200"} 1027 1395066363000 +http_requests_total{method="post",code="400"} 3 1395066363000 + +# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{method="post",code="200"} 1027 1395066363000 +http_requests_total{method="post",code="400"} 3 1395066363000 \ No newline at end of file diff --git a/glide.lock b/glide.lock index bba8218..3ce7d30 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,24 @@ -hash: e04594e2b6fcae1028befdf96e159b25c0a2639cc72eb046460094fa95cb6b4c -updated: 2016-07-14T19:24:53.044987906+02:00 +hash: 9ad826b36a1c590807ac502d31efddc979622140a424bd03d3fdf055c9c5a67e +updated: 2017-04-27T20:08:02.731796344+02:00 imports: -- name: github.com/go-yaml/yaml.git - version: a83829b6f1293c91addabc89d0571c246397bbf4 +- name: github.com/golang/protobuf + version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef + subpackages: + - proto +- name: github.com/matttproud/golang_protobuf_extensions + version: c12348ce28de40eed0136aa2b644d0ee0650e56c + subpackages: + - pbutil +- name: github.com/prometheus/client_model + version: 6f3806018612930941127f2a7c6c453ba2c527d2 + subpackages: + - go +- name: github.com/prometheus/common + version: 13ba4ddd0caa9c28ca7b7bffe1dfa9ed8d5ef207 + subpackages: + - expfmt + - internal/bitbucket.org/ww/goautoneg + - model - name: gopkg.in/yaml.v2 - version: a83829b6f1293c91addabc89d0571c246397bbf4 + version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b devImports: [] diff --git a/glide.yaml b/glide.yaml index 5e8291d..e83d255 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,3 +1,4 @@ package: github.com/warmans/prometheus-aggregate-exporter import: - package: gopkg.in/yaml.v2 +- package: github.com/prometheus/common diff --git a/main.go b/main.go index 36a629b..7983289 100644 --- a/main.go +++ b/main.go @@ -13,16 +13,21 @@ import ( "net/http" "strconv" "time" + + "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" ) var ( - Version = "unknown" + Version = "unknown" ) var ( configPathFlag = flag.String("config", "config.yml", "Path to config YAML file.") - verboseFlag = flag.Bool("verbose", false, "Log more information") - versionFlag = flag.Bool("version", false, "Show version and exit") + verboseFlag = flag.Bool("verbose", false, "Log more information") + versionFlag = flag.Bool("version", false, "Show version and exit") + appendLabel = flag.Bool("label", false, "Add a label to metrics to show their origin target") + labelName = flag.String("label.name", "ae_source", "Label name to use if a target name label is appended to metrics") ) type Config struct { @@ -85,7 +90,7 @@ func main() { type Result struct { URL string SecondsTaken float64 - Payload io.ReadCloser + MetricFamily map[string]*io_prometheus_client.MetricFamily Error error } @@ -102,10 +107,14 @@ func (f *Aggregator) Aggregate(targets []string, output io.Writer) { } func(numTargets int, resultChan chan *Result) { + numResuts := 0 + + allFamilies := make(map[string]*io_prometheus_client.MetricFamily) + for { if numTargets == numResuts { - return + break } select { case result := <-resultChan: @@ -116,21 +125,31 @@ func (f *Aggregator) Aggregate(targets []string, output io.Writer) { continue } - _, err := io.Copy(output, result.Payload) - if err != nil { - log.Printf("Copy error: %s", err.Error()) + for mfName, mf := range result.MetricFamily { + if *appendLabel { + for _, m := range mf.Metric { + m.Label = append(m.Label, &io_prometheus_client.LabelPair{Name: labelName, Value: &result.URL}) + } + } + if existingMf, ok := allFamilies[mfName]; ok { + for _, m := range mf.Metric { + existingMf.Metric = append(existingMf.Metric, m) + } + } else { + allFamilies[*mf.Name] = mf + } } - - err = result.Payload.Close() - if err != nil { - log.Printf("Result body close error: %s", err.Error()) - } - if *verboseFlag { log.Printf("OK: %s was refreshed in %.3f seconds", result.URL, result.SecondsTaken) } } } + + encoder := expfmt.NewEncoder(output, expfmt.FmtText) + for _, f := range allFamilies { + encoder.Encode(f) + } + }(len(targets), resultChan) } @@ -138,12 +157,27 @@ func (f *Aggregator) fetch(target string, resultChan chan *Result) { startTime := time.Now() res, err := f.HTTP.Get(target) + result := &Result{URL: target, SecondsTaken: time.Since(startTime).Seconds(), Error: nil} if res != nil { - result.Payload = res.Body + result.MetricFamily, err = getMetricFamilies(res.Body) + if err != nil { + result.Error = fmt.Errorf("failed to add labels to target %s metrics: %s", target, err.Error()) + resultChan <- result + return + } } if err != nil { - result.Error = fmt.Errorf("Failed to fetch URL %s due to error: %s", target, err.Error()) + result.Error = fmt.Errorf("failed to fetch URL %s due to error: %s", target, err.Error()) } resultChan <- result } + +func getMetricFamilies(sourceData io.Reader) (map[string]*io_prometheus_client.MetricFamily, error) { + parser := expfmt.TextParser{} + metricFamiles, err := parser.TextToMetricFamilies(sourceData) + if err != nil { + return nil, err + } + return metricFamiles, nil +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..002db8d --- /dev/null +++ b/main_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "testing" +) + +var updateGoldenFile = flag.Bool("update.golden", false, "update golden files") + +func TestMain(m *testing.M) { + + flag.Parse() + os.Exit(m.Run()) +} + +func mustOpenFile(name string, flag int) *os.File { + file, err := os.OpenFile(fmt.Sprintf("fixture/%s", name), flag, 0666) + if err != nil { + panic(err) + } + return file +} + +func mustReadAll(r io.Reader) string { + b, err := ioutil.ReadAll(r) + if err != nil { + panic("failed to readall reader: " + err.Error()) + } + return string(b[:]) +} + +//todo: re-implement tests \ No newline at end of file diff --git a/vendor/github.com/golang/protobuf b/vendor/github.com/golang/protobuf new file mode 160000 index 0000000..2bba060 --- /dev/null +++ b/vendor/github.com/golang/protobuf @@ -0,0 +1 @@ +Subproject commit 2bba0603135d7d7f5cb73b2125beeda19c09f4ef diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions b/vendor/github.com/matttproud/golang_protobuf_extensions new file mode 160000 index 0000000..c12348c --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions @@ -0,0 +1 @@ +Subproject commit c12348ce28de40eed0136aa2b644d0ee0650e56c diff --git a/vendor/github.com/prometheus/client_model b/vendor/github.com/prometheus/client_model new file mode 160000 index 0000000..6f38060 --- /dev/null +++ b/vendor/github.com/prometheus/client_model @@ -0,0 +1 @@ +Subproject commit 6f3806018612930941127f2a7c6c453ba2c527d2 diff --git a/vendor/github.com/prometheus/common b/vendor/github.com/prometheus/common new file mode 160000 index 0000000..13ba4dd --- /dev/null +++ b/vendor/github.com/prometheus/common @@ -0,0 +1 @@ +Subproject commit 13ba4ddd0caa9c28ca7b7bffe1dfa9ed8d5ef207 diff --git a/vendor/gopkg.in/yaml.v2/LICENSE b/vendor/gopkg.in/yaml.v2/LICENSE index a68e67f..866d74a 100644 --- a/vendor/gopkg.in/yaml.v2/LICENSE +++ b/vendor/gopkg.in/yaml.v2/LICENSE @@ -1,188 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. -Copyright (c) 2011-2014 - Canonical Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -This software is licensed under the LGPLv3, included below. + http://www.apache.org/licenses/LICENSE-2.0 -As a special exception to the GNU Lesser General Public License version 3 -("LGPL3"), the copyright holders of this Library give you permission to -convey to a third party a Combined Work that links statically or dynamically -to this Library without providing any Minimal Corresponding Source or -Minimal Application Code as set out in 4d or providing the installation -information set out in section 4e, provided that you comply with the other -provisions of LGPL3 and provided that you meet, for the Application the -terms and conditions of the license(s) which apply to the Application. - -Except as stated in this special exception, the provisions of LGPL3 will -continue to comply in full to this Library. If you modify this Library, you -may apply this exception to your version of this Library, but you are not -obliged to do so. If you do not wish to do so, delete this exception -statement from your version. This exception does not (and cannot) modify any -license terms which apply to the Application, with which you must still -comply. - - - GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/gopkg.in/yaml.v2/README.md b/vendor/gopkg.in/yaml.v2/README.md index 7b8bd86..1884de6 100644 --- a/vendor/gopkg.in/yaml.v2/README.md +++ b/vendor/gopkg.in/yaml.v2/README.md @@ -42,7 +42,7 @@ The package API for yaml v2 will remain stable as described in [gopkg.in](https: License ------- -The yaml package is licensed under the LGPL with an exception that allows it to be linked statically. Please see the LICENSE file for details. +The yaml package is licensed under the Apache License 2.0. Please see the LICENSE file for details. Example diff --git a/vendor/gopkg.in/yaml.v2/decode.go b/vendor/gopkg.in/yaml.v2/decode.go index 085cddc..052ecfc 100644 --- a/vendor/gopkg.in/yaml.v2/decode.go +++ b/vendor/gopkg.in/yaml.v2/decode.go @@ -120,7 +120,6 @@ func (p *parser) parse() *node { default: panic("attempted to parse unknown event: " + strconv.Itoa(int(p.event.typ))) } - panic("unreachable") } func (p *parser) node(kind int) *node { @@ -251,7 +250,7 @@ func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { // // If n holds a null value, prepare returns before doing anything. func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { - if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "") { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "" && n.implicit) { return out, false, false } again := true diff --git a/vendor/gopkg.in/yaml.v2/decode_test.go b/vendor/gopkg.in/yaml.v2/decode_test.go index c159760..a6fea0f 100644 --- a/vendor/gopkg.in/yaml.v2/decode_test.go +++ b/vendor/gopkg.in/yaml.v2/decode_test.go @@ -551,7 +551,7 @@ var unmarshalTests = []struct { }, { "a: 2015-02-24T18:19:39Z\n", - map[string]time.Time{"a": time.Unix(1424801979, 0)}, + map[string]time.Time{"a": time.Unix(1424801979, 0).In(time.UTC)}, }, // Encode empty lists as zero-length slices. @@ -581,6 +581,15 @@ var unmarshalTests = []struct { "\xfe\xff\x00\xf1\x00o\x00\xf1\x00o\x00:\x00 \x00v\x00e\x00r\x00y\x00 \x00y\x00e\x00s\x00 \xd8=\xdf\xd4\x00\n", M{"ñoño": "very yes 🟔"}, }, + + // YAML Float regex shouldn't match this + { + "a: 123456e1\n", + M{"a": "123456e1"}, + }, { + "a: 123456E1\n", + M{"a": "123456E1"}, + }, } type M map[interface{}]interface{} @@ -660,6 +669,7 @@ var unmarshalerTests = []struct { {`_: BAR!`, "!!str", "BAR!"}, {`_: "BAR!"`, "!!str", "BAR!"}, {"_: !!foo 'BAR!'", "!!foo", "BAR!"}, + {`_: ""`, "!!str", ""}, } var unmarshalerResult = map[int]error{} diff --git a/vendor/gopkg.in/yaml.v2/emitterc.go b/vendor/gopkg.in/yaml.v2/emitterc.go index 2befd55..6ecdcb3 100644 --- a/vendor/gopkg.in/yaml.v2/emitterc.go +++ b/vendor/gopkg.in/yaml.v2/emitterc.go @@ -666,7 +666,6 @@ func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, return yaml_emitter_set_emitter_error(emitter, "expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS") } - return false } // Expect ALIAS. diff --git a/vendor/gopkg.in/yaml.v2/parserc.go b/vendor/gopkg.in/yaml.v2/parserc.go index 0a7037a..81d05df 100644 --- a/vendor/gopkg.in/yaml.v2/parserc.go +++ b/vendor/gopkg.in/yaml.v2/parserc.go @@ -166,7 +166,6 @@ func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool default: panic("invalid parser state") } - return false } // Parse the production: diff --git a/vendor/gopkg.in/yaml.v2/resolve.go b/vendor/gopkg.in/yaml.v2/resolve.go index 93a8632..232313c 100644 --- a/vendor/gopkg.in/yaml.v2/resolve.go +++ b/vendor/gopkg.in/yaml.v2/resolve.go @@ -3,6 +3,7 @@ package yaml import ( "encoding/base64" "math" + "regexp" "strconv" "strings" "unicode/utf8" @@ -80,6 +81,8 @@ func resolvableTag(tag string) bool { return false } +var yamlStyleFloat = regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+][0-9]+)?$`) + func resolve(tag string, in string) (rtag string, out interface{}) { if !resolvableTag(tag) { return tag, in @@ -135,9 +138,11 @@ func resolve(tag string, in string) (rtag string, out interface{}) { if err == nil { return yaml_INT_TAG, uintv } - floatv, err := strconv.ParseFloat(plain, 64) - if err == nil { - return yaml_FLOAT_TAG, floatv + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } } if strings.HasPrefix(plain, "0b") { intv, err := strconv.ParseInt(plain[2:], 2, 64) diff --git a/vendor/gopkg.in/yaml.v2/scannerc.go b/vendor/gopkg.in/yaml.v2/scannerc.go index 2580800..2c9d511 100644 --- a/vendor/gopkg.in/yaml.v2/scannerc.go +++ b/vendor/gopkg.in/yaml.v2/scannerc.go @@ -9,7 +9,7 @@ import ( // ************ // // The following notes assume that you are familiar with the YAML specification -// (http://yaml.org/spec/cvs/current.html). We mostly follow it, although in +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in // some cases we are less restrictive that it requires. // // The process of transforming a YAML stream into a sequence of events is