Add proxy capabilities

main.go

@@ -48,6 +48,11 @@ var (
 // time with the ldflags -X option
 var gronVersion = "dev"
 
+// Default value that is set if proxy/noproxy variables is not specified.
+// Cannot be an empty string, because an empty string explicitly deactivates the
+// proxy.
+var undefinedProxy = "-"
+
 func init() {
 	flag.Usage = func() {
 		h := "Transform JSON (from a file, URL, or stdin) into discrete assignments to make it greppable\n\n"
@@ -62,6 +67,8 @@ func init() {
 		h += "  -m, --monochrome Monochrome (don't colorize output)\n"
 		h += "  -s, --stream     Treat each line of input as a separate JSON object\n"
 		h += "  -k, --insecure   Disable certificate validation\n"
+		h += "  -x, --proxy      Set proxy configuration\n"
+		h += "      --noproxy    Comma-separated list of hosts for which not to use a proxy, if one is specified.\n"
 		h += "  -j, --json       Represent gron data as JSON stream\n"
 		h += "      --no-sort    Don't sort output (faster)\n"
 		h += "      --version    Print version information\n\n"
@@ -97,6 +104,8 @@ func main() {
 		insecureFlag   bool
 		jsonFlag       bool
 		valuesFlag     bool
+		proxyURL       string
+		noProxy        string
 	)
 
 	flag.BoolVar(&ungronFlag, "ungron", false, "")
@@ -116,6 +125,9 @@ func main() {
 	flag.BoolVar(&valuesFlag, "values", false, "")
 	flag.BoolVar(&valuesFlag, "value", false, "")
 	flag.BoolVar(&valuesFlag, "v", false, "")
+	flag.StringVar(&proxyURL, "x", undefinedProxy, "")
+	flag.StringVar(&proxyURL, "proxy", undefinedProxy, "")
+	flag.StringVar(&noProxy, "noproxy", undefinedProxy, "")
 
 	flag.Parse()
 
@@ -137,7 +149,7 @@ func main() {
 	if filename == "" || filename == "-" {
 		rawInput = os.Stdin
 	} else if validURL(filename) {
-		r, err := getURL(filename, insecureFlag)
+		r, err := getURL(filename, insecureFlag, proxyURL, noProxy)
 		if err != nil {
 			fatal(exitFetchURL, err)
 		}

url.go

@@ -6,7 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	neturl "net/url"
+	"os"
 	"regexp"
+	"strings"
 	"time"
 )
 
@@ -15,10 +18,63 @@ func validURL(url string) bool {
 	return r.MatchString(url)
 }
 
-func getURL(url string, insecure bool) (io.Reader, error) {
+func configureProxy(url string, proxy string, noProxy string) func(*http.Request) (*neturl.URL, error) {
+	cURL, err := neturl.Parse(url)
+	if err != nil {
+		return nil
+	}
+
+	// Direct arguments are superior to environment variables.
+	if proxy == undefinedProxy {
+		proxy = os.Getenv(fmt.Sprintf("%s_proxy", cURL.Scheme))
+	}
+	if noProxy == undefinedProxy {
+		noProxy = os.Getenv("no_proxy")
+	}
+
+	// Skip setting a proxy if no proxy has been set through env variable or
+	// argument.
+	if proxy == "" {
+		return nil
+	}
+
+	// Test if any of the hosts mentioned in the noProxy variable or the
+	// no_proxy env variable. Skip setting up the proxy if a match is found.
+	noProxyHosts := strings.Split(noProxy, ",")
+	if len(noProxyHosts) > 0 {
+		for _, noProxyHost := range noProxyHosts {
+			if len(noProxyHost) == 0 {
+				continue
+			}
+			// Test for direct matches of the hostname.
+			if cURL.Host == noProxyHost {
+				return nil
+			}
+			// Match through wildcard-like pattern, e.g. ".foobar.com" should
+			// match all subdomains of foobar.com.
+			if strings.HasPrefix(noProxyHost, ".") && strings.HasSuffix(cURL.Host, noProxyHost) {
+				return nil
+			}
+		}
+	}
+
+	proxyURL, err := neturl.Parse(proxy)
+	if err != nil {
+		return nil
+	}
+
+	return http.ProxyURL(proxyURL)
+}
+
+func getURL(url string, insecure bool, proxyURL string, noProxy string) (io.Reader, error) {
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure},
 	}
+	// Set proxy if defined.
+	proxy := configureProxy(url, proxyURL, noProxy)
+	if proxy != nil {
+		tr.Proxy = proxy
+	}
 	client := http.Client{
 		Transport: tr,
 		Timeout:   20 * time.Second,

url_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"os"
 	"testing"
 )
 
@@ -23,3 +24,97 @@ func TestValidURL(t *testing.T) {
 		}
 	}
 }
+
+func TestConfigureProxyHttp(t *testing.T) {
+	tests := []struct {
+		url          string
+		httpProxy    string
+		envHttpProxy string
+		noProxy      string
+		envNoProxy   string
+		hasProxy     bool
+	}{
+		// http proxy via env variables
+		{"http://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", true},
+		{"https://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false},
+		{"schema://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false},
+
+		// http proxy with env variables, overwritten by argument
+		{"http://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+		{"https://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+		{"schema://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+
+		// http proxy with env variables, domain excluded by no_proxy
+		{"http://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"http://foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"http://test.foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, ".foobar3.com", false},
+		{"https://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"schema://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+
+		// http proxy with env variables, domain excluded by no_proxy, overwritten by argument
+		{"http://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true},
+		{"http://foobar4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true},
+		{"http://test.foobar4.com", undefinedProxy, "http://localhost:1234", "", ".foobar4.com", true},
+		{"https://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false},
+		{"schema://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false},
+	}
+
+	for _, test := range tests {
+		os.Setenv("http_proxy", test.envHttpProxy)
+		os.Setenv("no_proxy", test.envNoProxy)
+		proxy := configureProxy(test.url, test.httpProxy, test.noProxy)
+		hasProxy := proxy != nil
+		if hasProxy != test.hasProxy {
+			t.Errorf("Want %t for configureProxy; have %t; %v", test.hasProxy, hasProxy, test)
+		}
+		os.Unsetenv("http_proxy")
+		os.Unsetenv("no_proxy")
+	}
+}
+
+func TestConfigureProxyHttps(t *testing.T) {
+	tests := []struct {
+		url           string
+		httpsProxy    string
+		envHttpsProxy string
+		noProxy       string
+		envNoProxy    string
+		hasProxy      bool
+	}{
+		// https proxy via env variables
+		{"http://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false},
+		{"https://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", true},
+		{"schema://test1.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "", false},
+
+		// https proxy with env variables, overwritten by argument
+		{"http://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+		{"https://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+		{"schema://test2.com", "", "http://localhost:1234", undefinedProxy, "", false},
+
+		// https proxy with env variables, domain excluded by no_proxy
+		{"http://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"http://foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"http://test.foobar3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, ".foobar3.com", false},
+		{"https://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+		{"schema://test3.com", undefinedProxy, "http://localhost:1234", undefinedProxy, "test3.com,foobar3.com", false},
+
+		// https proxy with env variables, domain excluded by no_proxy, overwritten by argument
+		{"http://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false},
+		{"http://foobar4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false},
+		{"http://test.foobar4.com", undefinedProxy, "http://localhost:1234", "", ".foobar4.com", false},
+		{"https://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", true},
+		{"schema://test4.com", undefinedProxy, "http://localhost:1234", "", "test4.com,foobar4.com", false},
+	}
+
+	for _, test := range tests {
+		os.Setenv("https_proxy", test.envHttpsProxy)
+		os.Setenv("no_proxy", test.envNoProxy)
+		proxy := configureProxy(test.url, test.httpsProxy, test.noProxy)
+		hasProxy := proxy != nil
+		if hasProxy != test.hasProxy {
+			t.Errorf("Want %t for configureProxy; have %t; %v", test.hasProxy, hasProxy, test)
+		}
+		os.Unsetenv("https_proxy")
+		os.Unsetenv("no_proxy")
+	}
+}