From d94ba0bc88d21fd507f0f5d3bd626ac82f3a21d1 Mon Sep 17 00:00:00 2001 From: dotcs Date: Sat, 7 May 2022 16:45:03 +0200 Subject: [PATCH 1/4] Add capabilities to configure proxy through env vars or args --- main.go | 8 ++++- url.go | 62 ++++++++++++++++++++++++++++++++- url_test.go | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index cde814b..512cc3d 100644 --- a/main.go +++ b/main.go @@ -62,6 +62,7 @@ func init() { h += " -m, --monochrome Monochrome (don't colorize output)\n" h += " -s, --stream Treat each line of input as a separate JSON object\n" h += " -k, --insecure Disable certificate validation\n" + h += " -x, --proxy Set proxy configuration\n" h += " -j, --json Represent gron data as JSON stream\n" h += " --no-sort Don't sort output (faster)\n" h += " --version Print version information\n\n" @@ -97,6 +98,8 @@ func main() { insecureFlag bool jsonFlag bool valuesFlag bool + proxyURL string + noProxy string ) flag.BoolVar(&ungronFlag, "ungron", false, "") @@ -116,6 +119,9 @@ func main() { flag.BoolVar(&valuesFlag, "values", false, "") flag.BoolVar(&valuesFlag, "value", false, "") flag.BoolVar(&valuesFlag, "v", false, "") + flag.StringVar(&proxyURL, "x", "", "") + flag.StringVar(&proxyURL, "proxy", "", "") + flag.StringVar(&noProxy, "noproxy", "", "") flag.Parse() @@ -137,7 +143,7 @@ func main() { if filename == "" || filename == "-" { rawInput = os.Stdin } else if validURL(filename) { - r, err := getURL(filename, insecureFlag) + r, err := getURL(filename, insecureFlag, &proxyURL, &noProxy) if err != nil { fatal(exitFetchURL, err) } diff --git a/url.go b/url.go index a2e10f9..c116eaa 100644 --- a/url.go +++ b/url.go @@ -6,7 +6,10 @@ import ( "fmt" "io" "net/http" + neturl "net/url" + "os" "regexp" + "strings" "time" ) @@ -15,10 +18,67 @@ func validURL(url string) bool { return r.MatchString(url) } -func getURL(url string, insecure bool) (io.Reader, error) { +func configureProxy(url string, proxyRef *string, noProxyRef *string) func(*http.Request) (*neturl.URL, error) { + var proxy, noProxy string + + cURL, err := neturl.Parse(url) + if err != nil { + return nil + } + + // Direct arguments are superior to environment variables. + if proxyRef != nil { + proxy = *proxyRef + } else { + proxy = os.Getenv(fmt.Sprintf("%s_proxy", cURL.Scheme)) + } + if proxy == "" { + // Skip setting a proxy if no environment variable has been set. + return nil + } + + // Test if any of the hosts mentioned in the noProxy variable or the + // no_proxy env variable. Skip setting up the proxy if a match is found. + if noProxyRef != nil { + noProxy = *noProxyRef + } else { + noProxy = os.Getenv("no_proxy") + } + noProxyHosts := strings.Split(noProxy, ",") + if len(noProxyHosts) > 0 { + for _, noProxyHost := range noProxyHosts { + if len(noProxyHost) == 0 { + continue + } + // Test for direct matches of the hostname. + if cURL.Host == noProxyHost { + return nil + } + // Match through wildcard-like pattern, e.g. ".foobar.com" should + // match all subdomains of foobar.com. + if strings.HasPrefix(noProxyHost, ".") && strings.HasSuffix(cURL.Host, noProxyHost) { + return nil + } + } + } + + proxyURL, err := neturl.Parse(proxy) + if err != nil { + return nil + } + + return http.ProxyURL(proxyURL) +} + +func getURL(url string, insecure bool, proxyURL *string, noProxy *string) (io.Reader, error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, } + // Set proxy if defined. + proxy := configureProxy(url, proxyURL, noProxy) + if proxy != nil { + tr.Proxy = proxy + } client := http.Client{ Transport: tr, Timeout: 20 * time.Second, diff --git a/url_test.go b/url_test.go index fe46b69..ab8482b 100644 --- a/url_test.go +++ b/url_test.go @@ -1,6 +1,7 @@ package main import ( + "os" "testing" ) @@ -23,3 +24,101 @@ func TestValidURL(t *testing.T) { } } } + +func TestConfigureProxyHttpWithEnv(t *testing.T) { + emptyStr := "" + + tests := []struct { + url string + httpProxy *string + envHttpProxy string + noProxy *string + envNoProxy string + hasProxy bool + }{ + // http proxy via env variables + {"http://test1.com", nil, "http://localhost:1234", nil, "", true}, + {"https://test1.com", nil, "http://localhost:1234", nil, "", false}, + {"schema://test1.com", nil, "http://localhost:1234", nil, "", false}, + + // http proxy with env variables, overwritten by argument + {"http://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + {"https://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + {"schema://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + + // http proxy with env variables, domain excluded by no_proxy + {"http://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"http://foobar3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"http://test.foobar3.com", nil, "http://localhost:1234", nil, ".foobar3.com", false}, + {"https://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"schema://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + + // http proxy with env variables, domain excluded by no_proxy, overwritten by argument + {"http://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, + {"http://foobar4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, + {"http://test.foobar4.com", nil, "http://localhost:1234", &emptyStr, ".foobar4.com", true}, + {"https://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + {"schema://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + } + + for _, test := range tests { + os.Setenv("http_proxy", test.envHttpProxy) + os.Setenv("no_proxy", test.envNoProxy) + proxy := configureProxy(test.url, test.httpProxy, test.noProxy) + hasProxy := proxy != nil + if hasProxy != test.hasProxy { + t.Errorf("Want %t for configureProxy; have %t; %v", test.hasProxy, hasProxy, test) + } + os.Unsetenv("http_proxy") + os.Unsetenv("no_proxy") + } +} + +func TestConfigureProxyHttpsWithEnv(t *testing.T) { + emptyStr := "" + + tests := []struct { + url string + httpsProxy *string + envHttpsProxy string + noProxy *string + envNoProxy string + hasProxy bool + }{ + // http proxy via env variables + {"http://test1.com", nil, "http://localhost:1234", nil, "", false}, + {"https://test1.com", nil, "http://localhost:1234", nil, "", true}, + {"schema://test1.com", nil, "http://localhost:1234", nil, "", false}, + + // http proxy with env variables, overwritten by argument + {"http://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + {"https://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + {"schema://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + + // http proxy with env variables, domain excluded by no_proxy + {"http://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"http://foobar3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"http://test.foobar3.com", nil, "http://localhost:1234", nil, ".foobar3.com", false}, + {"https://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"schema://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + + // http proxy with env variables, domain excluded by no_proxy, overwritten by argument + {"http://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + {"http://foobar4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + {"http://test.foobar4.com", nil, "http://localhost:1234", &emptyStr, ".foobar4.com", false}, + {"https://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, + {"schema://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + } + + for _, test := range tests { + os.Setenv("https_proxy", test.envHttpsProxy) + os.Setenv("no_proxy", test.envNoProxy) + proxy := configureProxy(test.url, test.httpsProxy, test.noProxy) + hasProxy := proxy != nil + if hasProxy != test.hasProxy { + t.Errorf("Want %t for configureProxy; have %t; %v", test.hasProxy, hasProxy, test) + } + os.Unsetenv("https_proxy") + os.Unsetenv("no_proxy") + } +} From 8b0f6ce6553df47a899c0719525bf9a6f563dac1 Mon Sep 17 00:00:00 2001 From: dotcs Date: Sun, 8 May 2022 08:59:49 +0200 Subject: [PATCH 2/4] Use default value for undefined proxy values instead of ptr --- main.go | 12 ++++++-- url.go | 17 ++++------- url_test.go | 88 +++++++++++++++++++++++++---------------------------- 3 files changed, 57 insertions(+), 60 deletions(-) diff --git a/main.go b/main.go index 512cc3d..192f3a6 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,11 @@ var ( // time with the ldflags -X option var gronVersion = "dev" +// Default value that is set if proxy/noproxy variables is not specified. +// Cannot be an empty string, because an empty string explicitly deactivates the +// proxy. +var undefinedProxy = "-" + func init() { flag.Usage = func() { h := "Transform JSON (from a file, URL, or stdin) into discrete assignments to make it greppable\n\n" @@ -63,6 +68,7 @@ func init() { h += " -s, --stream Treat each line of input as a separate JSON object\n" h += " -k, --insecure Disable certificate validation\n" h += " -x, --proxy Set proxy configuration\n" + h += " --noproxy Comma-separated list of hosts for which not to use a proxy, if one is specified.\n" h += " -j, --json Represent gron data as JSON stream\n" h += " --no-sort Don't sort output (faster)\n" h += " --version Print version information\n\n" @@ -120,8 +126,8 @@ func main() { flag.BoolVar(&valuesFlag, "value", false, "") flag.BoolVar(&valuesFlag, "v", false, "") flag.StringVar(&proxyURL, "x", "", "") - flag.StringVar(&proxyURL, "proxy", "", "") - flag.StringVar(&noProxy, "noproxy", "", "") + flag.StringVar(&proxyURL, "proxy", undefinedProxy, "") + flag.StringVar(&noProxy, "noproxy", undefinedProxy, "") flag.Parse() @@ -143,7 +149,7 @@ func main() { if filename == "" || filename == "-" { rawInput = os.Stdin } else if validURL(filename) { - r, err := getURL(filename, insecureFlag, &proxyURL, &noProxy) + r, err := getURL(filename, insecureFlag, proxyURL, noProxy) if err != nil { fatal(exitFetchURL, err) } diff --git a/url.go b/url.go index c116eaa..bb6b6bb 100644 --- a/url.go +++ b/url.go @@ -18,30 +18,25 @@ func validURL(url string) bool { return r.MatchString(url) } -func configureProxy(url string, proxyRef *string, noProxyRef *string) func(*http.Request) (*neturl.URL, error) { - var proxy, noProxy string - +func configureProxy(url string, proxy string, noProxy string) func(*http.Request) (*neturl.URL, error) { cURL, err := neturl.Parse(url) if err != nil { return nil } // Direct arguments are superior to environment variables. - if proxyRef != nil { - proxy = *proxyRef - } else { + if proxy == undefinedProxy { proxy = os.Getenv(fmt.Sprintf("%s_proxy", cURL.Scheme)) } if proxy == "" { - // Skip setting a proxy if no environment variable has been set. + // Skip setting a proxy if no proxy has been set through env variable or + // argument. return nil } // Test if any of the hosts mentioned in the noProxy variable or the // no_proxy env variable. Skip setting up the proxy if a match is found. - if noProxyRef != nil { - noProxy = *noProxyRef - } else { + if noProxy == undefinedProxy { noProxy = os.Getenv("no_proxy") } noProxyHosts := strings.Split(noProxy, ",") @@ -70,7 +65,7 @@ func configureProxy(url string, proxyRef *string, noProxyRef *string) func(*http return http.ProxyURL(proxyURL) } -func getURL(url string, insecure bool, proxyURL *string, noProxy *string) (io.Reader, error) { +func getURL(url string, insecure bool, proxyURL string, noProxy string) (io.Reader, error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, } diff --git a/url_test.go b/url_test.go index ab8482b..10c8810 100644 --- a/url_test.go +++ b/url_test.go @@ -25,40 +25,38 @@ func TestValidURL(t *testing.T) { } } -func TestConfigureProxyHttpWithEnv(t *testing.T) { - emptyStr := "" - +func TestConfigureProxyHttp(t *testing.T) { tests := []struct { url string - httpProxy *string + httpProxy string envHttpProxy string - noProxy *string + noProxy string envNoProxy string hasProxy bool }{ // http proxy via env variables - {"http://test1.com", nil, "http://localhost:1234", nil, "", true}, - {"https://test1.com", nil, "http://localhost:1234", nil, "", false}, - {"schema://test1.com", nil, "http://localhost:1234", nil, "", false}, + {"http://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", true}, + {"https://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false}, + {"schema://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false}, // http proxy with env variables, overwritten by argument - {"http://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, - {"https://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, - {"schema://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + {"http://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, + {"https://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, + {"schema://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, // http proxy with env variables, domain excluded by no_proxy - {"http://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"http://foobar3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"http://test.foobar3.com", nil, "http://localhost:1234", nil, ".foobar3.com", false}, - {"https://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"schema://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + {"http://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"http://foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"http://test.foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, ".foobar3.com", false}, + {"https://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"schema://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, // http proxy with env variables, domain excluded by no_proxy, overwritten by argument - {"http://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, - {"http://foobar4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, - {"http://test.foobar4.com", nil, "http://localhost:1234", &emptyStr, ".foobar4.com", true}, - {"https://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, - {"schema://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + {"http://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true}, + {"http://foobar4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true}, + {"http://test.foobar4.com", undefinedProxy, "http://localhost:1234", "", ".foobar4.com", true}, + {"https://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false}, + {"schema://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false}, } for _, test := range tests { @@ -74,40 +72,38 @@ func TestConfigureProxyHttpWithEnv(t *testing.T) { } } -func TestConfigureProxyHttpsWithEnv(t *testing.T) { - emptyStr := "" - +func TestConfigureProxyHttps(t *testing.T) { tests := []struct { url string - httpsProxy *string + httpsProxy string envHttpsProxy string - noProxy *string + noProxy string envNoProxy string hasProxy bool }{ - // http proxy via env variables - {"http://test1.com", nil, "http://localhost:1234", nil, "", false}, - {"https://test1.com", nil, "http://localhost:1234", nil, "", true}, - {"schema://test1.com", nil, "http://localhost:1234", nil, "", false}, + // https proxy via env variables + {"http://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false}, + {"https://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", true}, + {"schema://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false}, - // http proxy with env variables, overwritten by argument - {"http://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, - {"https://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, - {"schema://test2.com", &emptyStr, "http://localhost:1234", nil, "", false}, + // https proxy with env variables, overwritten by argument + {"http://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, + {"https://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, + {"schema://test2.com", "", "http://localhost:1234", undefinedProxy, "", false}, - // http proxy with env variables, domain excluded by no_proxy - {"http://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"http://foobar3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"http://test.foobar3.com", nil, "http://localhost:1234", nil, ".foobar3.com", false}, - {"https://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, - {"schema://test3.com", nil, "http://localhost:1234", nil, "test3.com,foobar3.com", false}, + // https proxy with env variables, domain excluded by no_proxy + {"http://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"http://foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"http://test.foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, ".foobar3.com", false}, + {"https://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, + {"schema://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false}, - // http proxy with env variables, domain excluded by no_proxy, overwritten by argument - {"http://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, - {"http://foobar4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, - {"http://test.foobar4.com", nil, "http://localhost:1234", &emptyStr, ".foobar4.com", false}, - {"https://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", true}, - {"schema://test4.com", nil, "http://localhost:1234", &emptyStr, "test4.com,foobar4.com", false}, + // https proxy with env variables, domain excluded by no_proxy, overwritten by argument + {"http://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false}, + {"http://foobar4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false}, + {"http://test.foobar4.com", undefinedProxy, "http://localhost:1234", "", ".foobar4.com", false}, + {"https://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true}, + {"schema://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false}, } for _, test := range tests { From 587f281019b37c5d8b991d825749a3eaeb4c0afc Mon Sep 17 00:00:00 2001 From: dotcs Date: Sun, 8 May 2022 09:02:19 +0200 Subject: [PATCH 3/4] Refactor code --- url.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/url.go b/url.go index bb6b6bb..ffa6cba 100644 --- a/url.go +++ b/url.go @@ -28,17 +28,18 @@ func configureProxy(url string, proxy string, noProxy string) func(*http.Request if proxy == undefinedProxy { proxy = os.Getenv(fmt.Sprintf("%s_proxy", cURL.Scheme)) } + if noProxy == undefinedProxy { + noProxy = os.Getenv("no_proxy") + } + + // Skip setting a proxy if no proxy has been set through env variable or + // argument. if proxy == "" { - // Skip setting a proxy if no proxy has been set through env variable or - // argument. return nil } // Test if any of the hosts mentioned in the noProxy variable or the // no_proxy env variable. Skip setting up the proxy if a match is found. - if noProxy == undefinedProxy { - noProxy = os.Getenv("no_proxy") - } noProxyHosts := strings.Split(noProxy, ",") if len(noProxyHosts) > 0 { for _, noProxyHost := range noProxyHosts { From b141aa715b48a85c65df8252d41cbe211324e6f3 Mon Sep 17 00:00:00 2001 From: dotcs Date: Sun, 8 May 2022 09:12:09 +0200 Subject: [PATCH 4/4] Also set 'undefinedProxy' value as default value for '-x' option --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 192f3a6..1001aac 100644 --- a/main.go +++ b/main.go @@ -125,7 +125,7 @@ func main() { flag.BoolVar(&valuesFlag, "values", false, "") flag.BoolVar(&valuesFlag, "value", false, "") flag.BoolVar(&valuesFlag, "v", false, "") - flag.StringVar(&proxyURL, "x", "", "") + flag.StringVar(&proxyURL, "x", undefinedProxy, "") flag.StringVar(&proxyURL, "proxy", undefinedProxy, "") flag.StringVar(&noProxy, "noproxy", undefinedProxy, "")