From d6577c2f299a1f9942ef83569c6f819a85edbd52 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Tue, 26 Dec 2023 17:16:38 +0300 Subject: [PATCH 01/11] feat: update dependencies --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 64782c5..628b8b9 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/juev/tor-relay-scanner-go go 1.20 require ( - github.com/carlmjohnson/requests v0.23.3 + github.com/carlmjohnson/requests v0.23.5 github.com/json-iterator/go v1.1.12 github.com/spf13/pflag v1.0.5 ) require ( - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index ad21dff..2f41723 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/carlmjohnson/requests v0.23.3 h1:22EEJsJqjNWprjQtqw2nLoQ1Sz+I1qJUbvhd0cHSHUg= github.com/carlmjohnson/requests v0.23.3/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= +github.com/carlmjohnson/requests v0.23.5 h1:NPANcAofwwSuC6SIMwlgmHry2V3pLrSqRiSBKYbNHHA= +github.com/carlmjohnson/requests v0.23.5/go.mod h1:zG9P28thdRnN61aD7iECFhH5iGGKX2jIjKQD9kqYH+o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -8,6 +10,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -19,3 +23,5 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= From 568ed774ba094697f65d9a7019dbc977d6316f7e Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Wed, 27 Dec 2023 15:50:42 +0300 Subject: [PATCH 02/11] feat: use additional libraries --- cmd/main.go | 33 ++++---- go.mod | 8 ++ go.sum | 29 +++++-- models.go | 17 +++- scanner.go | 237 +++++++++++++++++++++++++--------------------------- 5 files changed, 177 insertions(+), 147 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index c7e25b8..f7e9087 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,17 +1,17 @@ package main import ( - "fmt" "io" "os" "time" + "github.com/gookit/color" scanner "github.com/juev/tor-relay-scanner-go" flag "github.com/spf13/pflag" ) var Usage = func() { - fmt.Fprintf(os.Stdout, "Usage of tor-relay-scanner-go:\n") + color.Fprintln(os.Stdout, "Usage of tor-relay-scanner-go:") flag.PrintDefaults() os.Exit(0) } @@ -54,7 +54,7 @@ func main() { if outfile != "" { logFile, err := os.OpenFile(outfile, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666) if err != nil { - fmt.Fprintf(os.Stderr, "cannot create file (%s): %v", outfile, err) + color.Fprintf(os.Stderr, "cannot create file (%s): %s", outfile, err.Error()) os.Exit(3) } out = io.MultiWriter(os.Stdout, logFile) @@ -63,23 +63,24 @@ func main() { if jsonRelays { relays, err := sc.GetRelays() if err != nil { - fmt.Fprintf(os.Stderr, "No relays are reachable this try.\n") + color.Fprintln(os.Stderr, "Tor Relay information can't be downloaded!") os.Exit(4) } - fmt.Fprintf(out, "%s\n", relays) - os.Exit(0) + color.Fprintf(out, "%s\n", relays) + return } relays := sc.Grab() - if len(relays) > 0 { - fmt.Printf("All reachable relays:\n") - for _, el := range relays { - fmt.Fprintf(out, "%s%s %s\n", prefix, el.Addresses, el.Fingerprint) - } - if torrc { - fmt.Fprintf(out, "UseBridges 1\n") - } - } else { - fmt.Fprintf(os.Stderr, "No relays are reachable this try.\n") + if len(relays) == 0 { + color.Fprintf(os.Stderr, "No relays are reachable this try.\n") + os.Exit(5) + } + + color.Printf("All reachable relays:\n") + for _, el := range relays { + color.Fprintf(out, "%s%s %s\n", prefix, el.Addresses, el.Fingerprint) + } + if torrc { + color.Fprintf(out, "UseBridges 1\n") } } diff --git a/go.mod b/go.mod index 628b8b9..4cb416a 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,20 @@ go 1.20 require ( github.com/carlmjohnson/requests v0.23.5 + github.com/gookit/color v1.5.4 github.com/json-iterator/go v1.1.12 github.com/spf13/pflag v1.0.5 ) require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/schollz/progressbar/v3 v3.14.1 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index 2f41723..b3ad670 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,17 @@ -github.com/carlmjohnson/requests v0.23.3 h1:22EEJsJqjNWprjQtqw2nLoQ1Sz+I1qJUbvhd0cHSHUg= -github.com/carlmjohnson/requests v0.23.3/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= github.com/carlmjohnson/requests v0.23.5 h1:NPANcAofwwSuC6SIMwlgmHry2V3pLrSqRiSBKYbNHHA= github.com/carlmjohnson/requests v0.23.5/go.mod h1:zG9P28thdRnN61aD7iECFhH5iGGKX2jIjKQD9kqYH+o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -16,12 +19,26 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= +github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= +github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/models.go b/models.go index d56206e..4b69198 100644 --- a/models.go +++ b/models.go @@ -1,7 +1,10 @@ package scanner import ( + "os" "time" + + "github.com/schollz/progressbar/v3" ) // TorRelayScanner ... @@ -12,7 +15,6 @@ type TorRelayScanner interface { type torRelayScanner struct { relayInfo RelayInfo - relays Relays // The number of concurrent relays tested. default=30 poolSize int // Test until at least this number of working relays are found. default=5 @@ -24,7 +26,7 @@ type torRelayScanner struct { // Preferred alternative URL for onionoo relay list. Could be used multiple times urls []string // Scan for relays running on specified port number. Could be used multiple times - port []string + ports []string // Use ipv4 only nodes ipv4 bool // Use ipv6 only nodes @@ -63,3 +65,14 @@ type ResultRelay struct { Fingerprint string `json:"fingerprint"` Addresses string `json:"or_addresses"` } + +var progressbarOptions = []progressbar.Option{ + progressbar.OptionSetDescription("Testing"), + progressbar.OptionSetWidth(15), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionShowCount(), + progressbar.OptionClearOnFinish(), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionSetPredictTime(false), + progressbar.OptionSetRenderBlankState(true), +} diff --git a/scanner.go b/scanner.go index a243fbc..e178212 100644 --- a/scanner.go +++ b/scanner.go @@ -2,15 +2,18 @@ package scanner import ( "context" - "fmt" "math/rand" "net" "net/url" "os" - "sync" "time" + "errors" + "github.com/carlmjohnson/requests" + "github.com/gookit/color" + "github.com/schollz/progressbar/v3" + "github.com/sourcegraph/conc/pool" json "github.com/json-iterator/go" ) @@ -42,13 +45,12 @@ func New( } return &torRelayScanner{ - relays: Relays{}, poolSize: poolSize, goal: goal, timeout: timeout, outfile: outfile, urls: urls, - port: port, + ports: port, ipv4: ipv4, ipv6: ipv6, } @@ -56,75 +58,45 @@ func New( // Grab returns relay list from public addresses func (t *torRelayScanner) Grab() (relays []ResultRelay) { - t.loadRelays() + if err := t.loadRelays(); err != nil { + color.Fprintln(os.Stderr, "Tor Relay information can't be downloaded!") + os.Exit(1) + } - fmt.Printf("Test started...\n\n") + r := rand.New(rand.NewSource(time.Now().UnixNano())) + chanRelays := make(chan ResultRelay) + go func() { + p := pool.New().WithMaxGoroutines(t.poolSize) + for _, el := range t.relayInfo.Relays { + el := el + p.Go(func() { + if tcpSocketConnectChecker(el.OrAddresses[0], t.timeout) { + chanRelays <- ResultRelay{ + Fingerprint: el.Fingerprint, + Addresses: el.OrAddresses[r.Intn(len(el.OrAddresses))], + } + } + }) + } + p.Wait() + close(chanRelays) + }() - numTries := len(t.relays) / t.poolSize - relayPos := 0 + bar := progressbar.NewOptions( + t.goal, + progressbarOptions..., + ) - r := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 1; i <= numTries; i++ { + for el := range chanRelays { + relays = append(relays, el) + _ = bar.Add(1) if len(relays) >= t.goal { break } + } - fmt.Printf("Try %d/%d, We'll test the following %d random relays:\n", i, numTries, t.poolSize) - - relayNum := min(t.poolSize, len(t.relays)-relayPos-1) - for _, el := range t.relays[relayPos : relayPos+relayNum] { - p, _ := json.Marshal(el) - fmt.Printf("%s\n", p) - } - fmt.Println() - - mu := sync.Mutex{} - wg := sync.WaitGroup{} - var testRelays []ResultRelay - for _, el := range t.relays[relayPos : relayPos+relayNum] { - fingerprint := el.Fingerprint - var addr string - if t.ipv4 { - for _, ad := range el.OrAddresses { - if ad[0] != '[' { - addr = ad - } - } - } else if t.ipv6 { - for _, ad := range el.OrAddresses { - if ad[0] == '[' { - addr = ad - } - } - } else { - addr = el.OrAddresses[r.Intn(len(el.OrAddresses))] - } - wg.Add(1) - go func() { - defer wg.Done() - if tcpSocketConnectChecker(addr, t.timeout) { - mu.Lock() - testRelays = append(testRelays, ResultRelay{ - Fingerprint: fingerprint, - Addresses: addr, - }) - mu.Unlock() - } - }() - } - wg.Wait() - if len(testRelays) == 0 { - fmt.Fprintf(os.Stderr, "No relays are reachable this try.\n\n") - } else { - fmt.Printf("The following relays are reachable this try:\n") - for _, rel := range testRelays { - p, _ := json.Marshal(rel) - fmt.Printf("%s\n", p) - } - fmt.Println() - } - relays = append(relays, testRelays...) - relayPos += t.poolSize + if len(relays) == 0 { + return relays } return relays[:t.goal] @@ -132,29 +104,44 @@ func (t *torRelayScanner) Grab() (relays []ResultRelay) { // GetRelays returns available relays in json format func (t *torRelayScanner) GetRelays() ([]byte, error) { - t.loadRelays() + if err := t.loadRelays(); err != nil { + return nil, err + } + + chanRelays := make(chan Relay) + go func() { + p := pool.New().WithMaxGoroutines(t.poolSize) + for _, el := range t.relayInfo.Relays { + el := el + p.Go(func() { + if tcpSocketConnectChecker(el.OrAddresses[0], t.timeout) { + chanRelays <- Relay{ + Fingerprint: el.Fingerprint, + OrAddresses: el.OrAddresses, + } + } + }) + } + p.Wait() + close(chanRelays) + }() + + bar := progressbar.NewOptions( + t.goal, + progressbarOptions..., + ) - mu := sync.Mutex{} var relays Relays - sem := make(chan struct{}, 30) - for _, el := range t.relayInfo.Relays { - el := el - sem <- struct{}{} - go func() { - if tcpSocketConnectChecker(el.OrAddresses[0], t.timeout) { - mu.Lock() - relays = append(relays, Relay{ - Fingerprint: el.Fingerprint, - OrAddresses: el.OrAddresses, - }) - mu.Unlock() - } - <-sem - }() + for el := range chanRelays { + relays = append(relays, el) + _ = bar.Add(1) + if len(relays) >= t.goal { + break + } } + if len(relays) == 0 { - fmt.Fprintf(os.Stderr, "No relays are reachable this try.\n\n") - return nil, fmt.Errorf("no relays are reachable this try") + return nil, errors.New("no relays are reachable this try") } result, err := json.MarshalIndent(RelayInfo{ @@ -166,81 +153,85 @@ func (t *torRelayScanner) GetRelays() ([]byte, error) { Bridges: Bridges{}, }, "", " ") if err != nil { - fmt.Fprintf(os.Stderr, "Cannot marshal RelayInfo: %v.\n", err) + color.Fprintf(os.Stderr, "Cannot marshal RelayInfo: %v.\n", err) return nil, err } return result, nil } -func (t *torRelayScanner) loadRelays() { - fmt.Printf("Tor Relay Scanner. Will scan up to %d working relays (or till the end)\n", t.goal) - fmt.Println("Downloading Tor Relay information from Tor Metrics...") - fmt.Println() - - var ( - err error - ) - +func (t *torRelayScanner) loadRelays() (err error) { for _, addr := range t.urls { - t.relays, t.relayInfo, err = t.grab(addr) + t.relayInfo, err = t.grab(addr) if err != nil { continue } break } - if t.relays == nil { - fmt.Fprintln(os.Stderr, "Tor Relay information can't be downloaded!") - os.Exit(1) + if len(t.relayInfo.Relays) == 0 { + return errors.New("tor Relay information can't be downloaded") } - if len(t.port) > 0 { - var tmp Relays - for _, rel := range t.relays { - addr := rel.OrAddresses[0] + var filtered Relays + for _, rel := range t.relayInfo.Relays { + var orAddresses []string + for _, addr := range rel.OrAddresses { + if t.ipv4 || t.ipv6 { + if t.ipv4 && !t.ipv6 { + if addr[0] == '[' { + continue + } + } + if t.ipv6 && !t.ipv4 { + if addr[0] != '[' { + continue + } + } + } + if len(t.ports) == 0 { + orAddresses = append(orAddresses, addr) + } u, _ := url.Parse("//" + addr) - for _, p := range t.port { + for _, p := range t.ports { if u.Port() == p { - tmp = append(tmp, rel) + orAddresses = append(orAddresses, addr) } } + if len(orAddresses) > 0 { + rel.OrAddresses = orAddresses + filtered = append(filtered, rel) + } } - if len(tmp) == 0 { - fmt.Fprintf(os.Stderr, "There are no relays within specified port number constrains!\n") - fmt.Fprintf(os.Stderr, "Try changing port numbers.") - os.Exit(2) - } - t.relays = tmp } - shuffle(t.relays) + if len(filtered) == 0 { + return errors.New("there are no relays within specified port number constrains!\nTry changing port numbers.") + } + t.relayInfo.Relays = filtered + + shuffle(t.relayInfo.Relays) - fmt.Printf("Done!\n\n") + return nil } -func (t *torRelayScanner) grab(addr string) (Relays, RelayInfo, error) { +func (t *torRelayScanner) grab(addr string) (RelayInfo, error) { var relayInfo RelayInfo - u, _ := url.Parse(addr) err := requests. URL(addr). UserAgent("tor-relay-scanner"). ToJSON(&relayInfo). Fetch(context.Background()) if err != nil { - fmt.Printf("Can't download Tor Relay data from/via %s: %v\n\n", u.Hostname(), err) - return nil, RelayInfo{}, err + return RelayInfo{}, err } - fmt.Printf("Download from %s\n\n", u.Hostname()) - return relayInfo.Relays, relayInfo, nil + return relayInfo, nil } // tcpSocketConnectChecker just checked network connection with specific host:port func tcpSocketConnectChecker(addr string, timeout time.Duration) bool { _, err := net.DialTimeout("tcp", addr, timeout) - if err != nil { - return false - } - return true + + return err == nil } From 47154c008db7eeefa01745e6f5375f90dc8a3d72 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Wed, 27 Dec 2023 16:09:52 +0300 Subject: [PATCH 03/11] feat: update github actions --- .github/workflows/build.yml | 6 +++--- .github/workflows/ci.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 854c707..ea34722 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,14 +8,14 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: '>=1.19.0' + go-version: '>=1.20.0' check-latest: true - name: Install dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ecc8c9..c16a46f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,18 +11,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: '>=1.19.0' + go-version: '>=1.20.0' check-latest: true - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v5 with: distribution: goreleaser version: latest From 2549bad414bbd4a0df8a8d99bd710b8ce6033aba Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Wed, 27 Dec 2023 16:11:55 +0300 Subject: [PATCH 04/11] fix: go lint errors --- models.go | 20 ++++++++++---------- scanner.go | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models.go b/models.go index 4b69198..1a84149 100644 --- a/models.go +++ b/models.go @@ -34,21 +34,21 @@ type torRelayScanner struct { } type ( - Version string - BuildRevision string - RelaysPublished string - BridgesPublished string - Bridges []any + version string + buildRevision string + relaysPublished string + bridgesPublished string + bridges []any ) // RelayInfo struct with basics information relay lists type RelayInfo struct { - Version Version - BuildRevision BuildRevision `json:"build_revision"` - RelaysPublished RelaysPublished `json:"relays_published"` + Version version + BuildRevision buildRevision `json:"build_revision"` + RelaysPublished relaysPublished `json:"relays_published"` Relays Relays `json:"relays"` - BridgesPublished BridgesPublished `json:"bridges_published"` - Bridges Bridges `json:"bridges"` + BridgesPublished bridgesPublished `json:"bridges_published"` + Bridges bridges `json:"bridges"` } // Relays ... diff --git a/scanner.go b/scanner.go index e178212..126f15f 100644 --- a/scanner.go +++ b/scanner.go @@ -150,7 +150,7 @@ func (t *torRelayScanner) GetRelays() ([]byte, error) { RelaysPublished: t.relayInfo.RelaysPublished, Relays: relays, BridgesPublished: t.relayInfo.BridgesPublished, - Bridges: Bridges{}, + Bridges: bridges{}, }, "", " ") if err != nil { color.Fprintf(os.Stderr, "Cannot marshal RelayInfo: %v.\n", err) @@ -206,7 +206,7 @@ func (t *torRelayScanner) loadRelays() (err error) { } if len(filtered) == 0 { - return errors.New("there are no relays within specified port number constrains!\nTry changing port numbers.") + return errors.New("there are no relays within specified port number constrains!\nTry changing port numbers") } t.relayInfo.Relays = filtered From 68c8ce0f211720df346bad4d2941f81ad063bc93 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Wed, 27 Dec 2023 16:16:49 +0300 Subject: [PATCH 05/11] fix: remove unused function --- funcs.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/funcs.go b/funcs.go index bfaf0dd..917f391 100644 --- a/funcs.go +++ b/funcs.go @@ -11,10 +11,3 @@ func shuffle(relays Relays) { relays[i], relays[j] = relays[j], relays[i] }) } - -func min(x, y int) int { - if x < y { - return x - } - return y -} From fda38c540c8395b33a83a25a6d7fe350d1815937 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 10:02:55 +0300 Subject: [PATCH 06/11] fix: Addresses -> Address, make usage local func --- cmd/main.go | 16 ++++++++-------- models.go | 2 +- scanner.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f7e9087..a19ffef 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,13 +10,13 @@ import ( flag "github.com/spf13/pflag" ) -var Usage = func() { - color.Fprintln(os.Stdout, "Usage of tor-relay-scanner-go:") - flag.PrintDefaults() - os.Exit(0) -} - func main() { + usage := func() { + color.Fprintln(os.Stdout, "Usage of tor-relay-scanner-go:") + flag.PrintDefaults() + os.Exit(0) + } + flag.IntVarP(&poolSize, "num_relays", "n", 30, `The number of concurrent relays tested.`) flag.IntVarP(&goal, "working_relay_num_goal", "g", 5, `Test until at least this number of working relays are found`) flag.IntVarP(&timeout, "timeout", "t", 1, `Socket connection timeout`) @@ -27,7 +27,7 @@ func main() { flag.BoolVarP(&ipv4, "ipv4", "4", false, `Use ipv4 only nodes`) flag.BoolVarP(&ipv6, "ipv6", "6", false, `Use ipv6 only nodes`) flag.BoolVarP(&jsonRelays, "json", "j", false, `Get available relays in json format`) - flag.Usage = Usage + flag.Usage = usage flag.Parse() if timeout < 1 { @@ -78,7 +78,7 @@ func main() { color.Printf("All reachable relays:\n") for _, el := range relays { - color.Fprintf(out, "%s%s %s\n", prefix, el.Addresses, el.Fingerprint) + color.Fprintf(out, "%s%s %s\n", prefix, el.Address, el.Fingerprint) } if torrc { color.Fprintf(out, "UseBridges 1\n") diff --git a/models.go b/models.go index 1a84149..2c0291a 100644 --- a/models.go +++ b/models.go @@ -63,7 +63,7 @@ type Relay struct { // ResultRelay ... type ResultRelay struct { Fingerprint string `json:"fingerprint"` - Addresses string `json:"or_addresses"` + Address string `json:"or_addresses"` } var progressbarOptions = []progressbar.Option{ diff --git a/scanner.go b/scanner.go index 126f15f..79ae1b8 100644 --- a/scanner.go +++ b/scanner.go @@ -73,7 +73,7 @@ func (t *torRelayScanner) Grab() (relays []ResultRelay) { if tcpSocketConnectChecker(el.OrAddresses[0], t.timeout) { chanRelays <- ResultRelay{ Fingerprint: el.Fingerprint, - Addresses: el.OrAddresses[r.Intn(len(el.OrAddresses))], + Address: el.OrAddresses[r.Intn(len(el.OrAddresses))], } } }) From 85591c201580f2fa09d9ae88d1a8c5a444f1595b Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 10:18:32 +0300 Subject: [PATCH 07/11] fix: shuffle should use crypto/rand --- funcs.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/funcs.go b/funcs.go index 917f391..9e3e3be 100644 --- a/funcs.go +++ b/funcs.go @@ -1,13 +1,13 @@ package scanner import ( - "math/rand" - "time" + "crypto/rand" + "math/big" ) func shuffle(relays Relays) { - rand.New(rand.NewSource(time.Now().UnixNano())). - Shuffle(len(relays), func(i, j int) { - relays[i], relays[j] = relays[j], relays[i] - }) + for i := len(relays) - 1; i > 0; i-- { + j, _ := rand.Int(rand.Reader, big.NewInt(int64(i+1))) + relays[i], relays[j.Uint64()] = relays[j.Uint64()], relays[i] + } } From 01040185e88e0cda310fdf418d6fadcae2ba9f3a Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 10:19:01 +0300 Subject: [PATCH 08/11] fix: increase num_relays 30 -> 100 by default --- cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index a19ffef..de83fbe 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,7 +17,7 @@ func main() { os.Exit(0) } - flag.IntVarP(&poolSize, "num_relays", "n", 30, `The number of concurrent relays tested.`) + flag.IntVarP(&poolSize, "num_relays", "n", 100, `The number of concurrent relays tested.`) flag.IntVarP(&goal, "working_relay_num_goal", "g", 5, `Test until at least this number of working relays are found`) flag.IntVarP(&timeout, "timeout", "t", 1, `Socket connection timeout`) flag.StringVarP(&outfile, "outfile", "o", "", `Output reachable relays to file`) From 0c6579fff0f7f8ad5764e88e6a95078c8209b733 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 10:24:15 +0300 Subject: [PATCH 09/11] fix: use crypto/rand in scanner --- scanner.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scanner.go b/scanner.go index 79ae1b8..b4e083d 100644 --- a/scanner.go +++ b/scanner.go @@ -2,7 +2,8 @@ package scanner import ( "context" - "math/rand" + "crypto/rand" + "math/big" "net" "net/url" "os" @@ -63,7 +64,6 @@ func (t *torRelayScanner) Grab() (relays []ResultRelay) { os.Exit(1) } - r := rand.New(rand.NewSource(time.Now().UnixNano())) chanRelays := make(chan ResultRelay) go func() { p := pool.New().WithMaxGoroutines(t.poolSize) @@ -71,9 +71,10 @@ func (t *torRelayScanner) Grab() (relays []ResultRelay) { el := el p.Go(func() { if tcpSocketConnectChecker(el.OrAddresses[0], t.timeout) { + n, _ := rand.Int(rand.Reader, big.NewInt(int64(len(el.OrAddresses)))) chanRelays <- ResultRelay{ Fingerprint: el.Fingerprint, - Address: el.OrAddresses[r.Intn(len(el.OrAddresses))], + Address: el.OrAddresses[n.Uint64()], } } }) From 7cbc1b557712c4ae0a7b6edc3e9c16ec651a7e2b Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 10:53:31 +0300 Subject: [PATCH 10/11] fix: simplify loadRelays --- scanner.go | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/scanner.go b/scanner.go index b4e083d..30fd5d1 100644 --- a/scanner.go +++ b/scanner.go @@ -178,25 +178,15 @@ func (t *torRelayScanner) loadRelays() (err error) { for _, rel := range t.relayInfo.Relays { var orAddresses []string for _, addr := range rel.OrAddresses { - if t.ipv4 || t.ipv6 { - if t.ipv4 && !t.ipv6 { - if addr[0] == '[' { - continue - } - } - if t.ipv6 && !t.ipv4 { - if addr[0] != '[' { - continue - } - } + if t.shouldSkipAddr(addr) { + continue } - if len(t.ports) == 0 { - orAddresses = append(orAddresses, addr) - } - u, _ := url.Parse("//" + addr) - for _, p := range t.ports { - if u.Port() == p { - orAddresses = append(orAddresses, addr) + if len(t.ports) > 0 { + u, _ := url.Parse("//" + addr) + for _, p := range t.ports { + if u.Port() == p { + orAddresses = append(orAddresses, addr) + } } } if len(orAddresses) > 0 { @@ -216,6 +206,22 @@ func (t *torRelayScanner) loadRelays() (err error) { return nil } +func (t *torRelayScanner) shouldSkipAddr(addr string) bool { + if t.ipv4 || t.ipv6 { + if t.ipv4 && !t.ipv6 { + if addr[0] == '[' { + return true + } + } + if t.ipv6 && !t.ipv4 { + if addr[0] != '[' { + return true + } + } + } + return false +} + func (t *torRelayScanner) grab(addr string) (RelayInfo, error) { var relayInfo RelayInfo err := requests. From a9255a829c9b18365fd498b9b817568d8841245a Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Thu, 28 Dec 2023 11:11:33 +0300 Subject: [PATCH 11/11] fix: improve loadRelays --- scanner.go | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/scanner.go b/scanner.go index 30fd5d1..1947bab 100644 --- a/scanner.go +++ b/scanner.go @@ -178,21 +178,17 @@ func (t *torRelayScanner) loadRelays() (err error) { for _, rel := range t.relayInfo.Relays { var orAddresses []string for _, addr := range rel.OrAddresses { - if t.shouldSkipAddr(addr) { + if t.skipAddrType(addr) { continue } - if len(t.ports) > 0 { - u, _ := url.Parse("//" + addr) - for _, p := range t.ports { - if u.Port() == p { - orAddresses = append(orAddresses, addr) - } - } - } - if len(orAddresses) > 0 { - rel.OrAddresses = orAddresses - filtered = append(filtered, rel) + if t.checkPorts(addr) { + continue } + orAddresses = append(orAddresses, addr) + } + if len(orAddresses) > 0 { + rel.OrAddresses = orAddresses + filtered = append(filtered, rel) } } @@ -206,7 +202,23 @@ func (t *torRelayScanner) loadRelays() (err error) { return nil } -func (t *torRelayScanner) shouldSkipAddr(addr string) bool { +func (t *torRelayScanner) checkPorts(addr string) bool { + if len(t.ports) > 0 { + u, _ := url.Parse("//" + addr) + var keep bool + for _, p := range t.ports { + if u.Port() == p { + keep = true + } + } + if !keep { + return true + } + } + return false +} + +func (t *torRelayScanner) skipAddrType(addr string) bool { if t.ipv4 || t.ipv6 { if t.ipv4 && !t.ipv6 { if addr[0] == '[' {