From 67719a0836fa04e6068dee50e826e422e7526202 Mon Sep 17 00:00:00 2001 From: Preetam Jinka Date: Mon, 4 Sep 2017 10:37:27 -0400 Subject: [PATCH] Release v0.2.0 --- README.md | 172 ++------------------------------------------ build.sh | 10 +++ circle.yml | 4 +- cmd/cistern/main.go | 2 +- 4 files changed, 19 insertions(+), 169 deletions(-) create mode 100755 build.sh diff --git a/README.md b/README.md index 8604d38..4f18c5d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@

- +

-Cistern is a network flow collector. +Cistern is an event aggregation and indexing system. Cistern consumes VPC Flow Logs and JSON events +from AWS CloudWatch Logs and exposes a SQL-like querying interface.

![experimental](https://img.shields.io/badge/status-experimental-orange.svg) @@ -15,175 +16,14 @@ Cistern is a network flow collector. * CloudWatch Logs * VPC Flow Logs + * JSON CloudWatch Logs events Coming soon: -* JSON CloudWatch Logs events * sFlow v5 -## Getting started +## Documentation -Cistern has no external dependencies. - -``` -Usage of ./cistern: - -api-addr string - API listen address (default "localhost:2020") - -config string - Path to config file (default "./cistern.json") - -data-dir string - Data directory (default "./data/") -``` - - - -#### Config file - -The config file has two main options: - -* cloudwatch_logs: A list of CloudWatch Logs log groups to consume. See [VPC Flow Logs](#vpc-flow-logs) for more details. -* retention: The retention of events in days. - -```json -{ - "cloudwatch_logs": [], - "retention": 3 -} -``` - -### VPC Flow Logs - -You can specify the flow log groups to consume in the config file. -In the `cloudwatch_logs` section, add an object for each log group -with the name and `flowlog: true`. - -**Example** - -```json -{ - "cloudwatch_logs": [ - { - "name": "flowlogs", - "flowlog": true - } - ], - "retention": 3 -} -``` - -#### Credentials - -Cistern will try to use AWS credentials from the following locations: - -* The environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) -* The Shared Credentials file (~/.aws/credentials) -* EC2 Instance Role Credentials - -To specify the region, set the AWS_REGION environment variable. - -## Querying - -You can query Cistern using the API or the CLI, which uses the API. -Most of the logic is built into the API, so it's not a bad idea to -learn how the API works in order to make sense of the CLI. - -### API - -**POST /collections/:collection/compact** - -This endpoint enforces the retention policy for the collection. - -**POST /collections/:collection/query** - -This endpoint queries events within a collection. - -The query endpoint accepts a **Query** object, which has the following -syntax: - -```js -Query: { - /* All of the following are optional. */ - "time_range": TimeRange - "columns": []Column - "group_by": []string - "filters": []Filter - "point_size": integer - "order_by": []string - "limit": integer - "descending": bool -} - -TimeRange : { - "start": string - "end": string -} - -Column: { - "name": string - "aggregate": string -} - -Filter: { - "column": string - "condition": string - "value": * -} -``` - -#### Filters - -Filters are applied as the first stage of query execution. A filter requires -a column name, a condition, and a value for the condition. The supported -conditions are: - -* **eq**: equal -* **neq**: not equal - -If multiple filters are specified, they are applied in an "AND" condition. - -#### Generating time series - -Time series can be generated by providing a nonzero `point_size`. This will -automatically group events by time ranges determined by the provided point size. - -### CLI - -``` -Usage of ./cistern-cli: - -address string - Cistern node address (default "http://localhost:2020") - -collection string - Collection to query - -columns string - Comma-separated list of columns to aggregate. - Example: 'sum(bytes), sum(packets)' - -descending - Sort in descending order. - -end int - End Unix timestamp - -filters string - Comma-separated list of filters. - Filters have the format ' '. - Possible conditions are [eq,neq]. - Values have to be valid JSON values. - Example: 'dest_address neq "172.31.31.192" , packets eq 3' - -group string - Comma-separated list of fields to group by. - Example: 'source_address, dest_address' - -limit int - Maximum number of events to return. - -order-by string - Comma-separated list of columns to order by. - Providing multiple columns means the results are ordered - by the first column, then the next, etc. - -point-size duration - Point size of time series. 0 means series will not be generated. - -start int - Start Unix timestamp - -version - Show version and exit. -``` - -The CLI prints the output of the API response in JSON format. +The official documentation is available on the [Cistern website](https://cistern.github.io/docs/). ## License diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..58a78ab --- /dev/null +++ b/build.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +cd ~/.go_workspace/src/github.com/Cistern/cistern +go build -o cistern-linux-amd64 ./cmd/cistern && mv cistern-linux-amd64 $CIRCLE_ARTIFACTS +GOOS=darwin GOARCH=amd64 go build -o cistern-darwin-amd64 ./cmd/cistern && mv cistern-darwin-amd64 $CIRCLE_ARTIFACTS +cd ui +npm i +npm run build +tar czvf cistern-ui-assets.tar.gz static && mv cistern-ui-assets.tar.gz $CIRCLE_ARTIFACTS diff --git a/circle.yml b/circle.yml index e3fce5e..34e84a1 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,9 @@ dependencies: compile: override: - - cd ~/.go_workspace/src/github.com/Cistern/cistern && go build ./cmd/cistern && mv cistern $CIRCLE_ARTIFACTS - - cd ui && npm i && npm run build && tar czvf static.tar.gz static && mv static.tar.gz $CIRCLE_ARTIFACTS + - ./build.sh test: override: - cd ~/.go_workspace/src/github.com/Cistern/cistern && go test $(go list ./... | grep -v /vendor/) + - cd ~/.go_workspace/src/github.com/Cistern/cistern/ui && npm test diff --git a/cmd/cistern/main.go b/cmd/cistern/main.go index 307b9bc..ab55997 100644 --- a/cmd/cistern/main.go +++ b/cmd/cistern/main.go @@ -17,7 +17,7 @@ var ( DataDir = "./data/" Collections = map[string]*EventCollection{} collectionsLock sync.Mutex - version = "0.1.1" + version = "0.2.0" ) func main() {