From 5ea6392be17ad89a32ed336dd8a00825b7a72de9 Mon Sep 17 00:00:00 2001 From: Simon Vanhaverbeke Date: Fri, 30 Apr 2021 00:41:35 +0200 Subject: [PATCH 1/4] add gitlab as provider --- go.sum | 6 ++-- main.go | 9 ++++- plugin/plugin.go | 16 ++++++--- plugin/plugin_test.go | 14 ++++---- providers/bitbucket_server.go | 5 ++- providers/gitlab.go | 64 +++++++++++++++++++++++++++++++++++ 6 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 providers/gitlab.go diff --git a/go.sum b/go.sum index d21539f..8538e8e 100644 --- a/go.sum +++ b/go.sum @@ -22,7 +22,6 @@ github.com/drone/drone-go v1.1.0 h1:2mritc5b7PhQWvILNyzaImZMRWVbMmmZ5Q0UDwwO7SI= github.com/drone/drone-go v1.1.0/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4= github.com/drone/go-scm v1.7.1 h1:wME/n7Qdo70VJ+WXZanJHjLtNWONEfjNsO2iwHDdlkE= github.com/drone/go-scm v1.7.1/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.8.0 h1:kDHu38a11loKf6uaBu75TmY1YPwsSaZdseET738Oy0o= github.com/gfleury/go-bitbucket-v1 v0.0.0-20200810125852-15f2a16ca820 h1:WZjcz0B/K3rhvsu2HAmFRpgChrGVIBfvrmhwK2AdYYQ= github.com/gfleury/go-bitbucket-v1 v0.0.0-20200810125852-15f2a16ca820/go.mod h1:LB3osS9X2JMYmTzcCArHHLrndBAfcVLQAvUddfs+ONs= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -38,6 +37,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= @@ -86,12 +87,10 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -110,6 +109,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/main.go b/main.go index a8145df..1f5af05 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ type spec struct { Provider string `envconfig:"PROVIDER"` Token string `envconfig:"TOKEN"` BitBucketAddress string `envconfig:"BB_ADDRESS"` + GitLabAddress string `envconfig:"GITLAB_ADDRESS"` } func contains(s []string, str string) bool { @@ -61,7 +62,7 @@ func main() { if spec.Provider == "" { logrus.Fatalln("missing provider") } else { - providers := []string{"bitbucket-server", "github"} + providers := []string{"bitbucket-server", "github", "gitlab"} if !contains(providers, spec.Provider) { logrus.Fatalln("invalid provider:", spec.Provider) } @@ -69,6 +70,10 @@ func main() { if spec.BitBucketAddress == "" && spec.Provider == "bitbucket-server" { logrus.Fatalln("missing bitbucket server address") } + if spec.GitLabAddress == "" && spec.Provider == "gitlab" { + spec.GitLabAddress = "https://gitlab.com" + logrus.Info("no gitlab address provided, using 'https://gitlab.com'") + } if spec.Bind == "" { spec.Bind = ":3000" } @@ -77,6 +82,8 @@ func main() { plugin.New( spec.Token, spec.Provider, + spec.BitBucketAddress, + spec.GitLabAddress, ), spec.Secret, logrus.StandardLogger(), diff --git a/plugin/plugin.go b/plugin/plugin.go index 0c8cf01..7d5dce1 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -24,6 +24,7 @@ type ( token string provider string bitbucketAddress string + gitlabAddress string } resource struct { @@ -104,10 +105,12 @@ func marshal(in []*resource) ([]byte, error) { } // New returns a new conversion plugin. -func New(token string, provider string) converter.Plugin { +func New(token string, provider string, bitbucketAddress string, gitlabAddress string) converter.Plugin { return &plugin{ - token: token, - provider: provider, + token: token, + provider: provider, + bitbucketAddress: bitbucketAddress, + gitlabAddress: gitlabAddress, } } @@ -153,7 +156,12 @@ func (p *plugin) Convert(ctx context.Context, req *converter.Request) (*drone.Co return nil, err } case "bitbucket-server": - changedFiles, err = providers.GetBBFilesChanged(req.Repo, req.Build, p.token) + changedFiles, err = providers.GetBBFilesChanged(req.Repo, req.Build, p.token, p.bitbucketAddress) + if err != nil { + return nil, err + } + case "gitlab": + changedFiles, err = providers.GetGitLabFilesChanged(req.Repo, req.Build, p.token, p.gitlabAddress) if err != nil { return nil, err } diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 84d3722..27de146 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -34,7 +34,7 @@ func TestNewEmptyPipeline(t *testing.T) { } for _, provider := range providers { - plugin := New("invalidtoken", provider) + plugin := New("invalidtoken", provider, "", "") config, err := plugin.Convert(noContext, req) if err != nil { @@ -73,7 +73,7 @@ this_is_invalid_yaml }, } - plugin := New("invalidtoken", "") + plugin := New("invalidtoken", "", "", "") _, err := plugin.Convert(noContext, req) if err == nil { @@ -115,7 +115,7 @@ steps: }, } - plugin := New("invalidtoken", "unsupported") + plugin := New("invalidtoken", "unsupported", "", "") _, err := plugin.Convert(noContext, req) if err == nil { @@ -163,7 +163,7 @@ steps: }, } - plugin := New("invalidtoken", "github") + plugin := New("invalidtoken", "github", "", "") config, err := plugin.Convert(noContext, req) if err != nil { @@ -236,7 +236,7 @@ steps: }, } - plugin := New("invalidtoken", "github") + plugin := New("invalidtoken", "github", "", "") config, err := plugin.Convert(noContext, req) if err != nil { @@ -306,7 +306,7 @@ steps: }, } - plugin := New("invalidtoken", "github") + plugin := New("invalidtoken", "github", "", "") config, err := plugin.Convert(noContext, req) if err != nil { @@ -379,7 +379,7 @@ steps: }, } - plugin := New("invalidtoken", "github") + plugin := New("invalidtoken", "github", "", "") config, err := plugin.Convert(noContext, req) if err != nil { diff --git a/providers/bitbucket_server.go b/providers/bitbucket_server.go index 2fb5fd7..5258748 100644 --- a/providers/bitbucket_server.go +++ b/providers/bitbucket_server.go @@ -3,9 +3,9 @@ package providers import ( "context" "encoding/json" + "github.com/drone/drone-go/drone" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" - "os" ) type bitbucketDiffs struct { @@ -16,10 +16,9 @@ type bitbucketDiffs struct { } `json:"diffs"` } -func GetBBFilesChanged(repo drone.Repo, build drone.Build, token string) ([]string, error) { +func GetBBFilesChanged(repo drone.Repo, build drone.Build, token string, bitbucketAddress string) ([]string, error) { var files []string var ctx context.Context - bitbucketAddress := os.Getenv("BB_ADDRESS") params := map[string]interface{}{ "since": build.Before, } diff --git a/providers/gitlab.go b/providers/gitlab.go new file mode 100644 index 0000000..26f803c --- /dev/null +++ b/providers/gitlab.go @@ -0,0 +1,64 @@ +package providers + +import ( + "context" + "net/http" + + "github.com/drone/drone-go/drone" + "github.com/drone/go-scm/scm" + "github.com/drone/go-scm/scm/driver/gitlab" + "github.com/drone/go-scm/scm/transport" + + "github.com/prometheus/client_golang/prometheus" +) + +func GetGitLabFilesChanged(repo drone.Repo, build drone.Build, token string, uri string) ([]string, error) { + var err error + + newctx := context.Background() + client, err := gitlab.New(uri) + if err != nil { + return nil, err + } + client.Client = &http.Client{ + Transport: &transport.BearerToken{ + Token: token, + }, + } + + var changes []*scm.Change + var result *scm.Response + + if build.Before == "" || build.Before == scm.EmptyCommit { + changes, result, err = client.Git.ListChanges(newctx, repo.Slug, build.After, scm.ListOptions{}) + if err != nil { + return nil, err + } + } else { + changes, result, err = client.Git.CompareChanges(newctx, repo.Slug, build.Before, build.After, scm.ListOptions{}) + if err != nil { + return nil, err + } + } + + GitLabApiCount.Set(float64(result.Rate.Remaining)) + + var files []string + for _, c := range changes { + files = append(files, c.Path) + } + + return files, nil +} + +var ( + GitLabApiCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "gitlab_api_calls_remaining", + Help: "Total number of github api calls per hour remaining", + }) +) + +func init() { + prometheus.MustRegister(GitLabApiCount) +} From d46565110f37d205fc35d54c576c7278f86f7f18 Mon Sep 17 00:00:00 2001 From: Simon Vanhaverbeke Date: Fri, 30 Apr 2021 00:43:49 +0200 Subject: [PATCH 2/4] run go mod tidy --- go.sum | 3 --- 1 file changed, 3 deletions(-) diff --git a/go.sum b/go.sum index 8538e8e..4e6ae3f 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= @@ -109,7 +107,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From 86084b3c5528a76072a7961c0b0f61eb886d9567 Mon Sep 17 00:00:00 2001 From: Simon Vanhaverbeke Date: Fri, 30 Apr 2021 00:49:23 +0200 Subject: [PATCH 3/4] plugin_test add gitlab as provider --- plugin/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 27de146..7b645e9 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -23,7 +23,7 @@ var noContext = context.Background() func TestNewEmptyPipeline(t *testing.T) { - providers := []string{"github", "bitbucket-server"} + providers := []string{"github", "bitbucket-server", "gitlab"} req := &converter.Request{ Build: drone.Build{}, From e4537d2d5d42e60235d7b54b174e3aaa19d41d00 Mon Sep 17 00:00:00 2001 From: Simon Vanhaverbeke Date: Fri, 30 Apr 2021 01:51:16 +0200 Subject: [PATCH 4/4] Update README.md --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/README.md b/README.md index c300700..4bb2b1b 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,58 @@ $ docker run -d \ --env=DRONE_SECRET=bea26a2221fd8090ea38720fc445eca6 \ --env=TOKEN=9e6eij3ckzvpe9mrhnqcis6zf8dhopmm46e3pi96 \ --env=PROVIDER=bitbucket-server \ + --env=BB_ADDRESS=https://your-bitbucket-address \ --restart=always \ --name=converter meltwater/drone-convert-pathschanged ``` +4. Update your Drone server configuration to include the plugin address and the shared secret. + +```text +DRONE_CONVERT_PLUGIN_ENDPOINT=http://1.2.3.4:3000 +DRONE_CONVERT_PLUGIN_SECRET=bea26a2221fd8090ea38720fc445eca6 +``` + +## GitLab + +_GitLab support is currently considered experimental_ + +1. Create a BitBucket access token via https://your-bitbucket-address/plugins/servlet/access-tokens/manage with read-only rights + +2. Create a shared secret: + +```console +$ openssl rand -hex 16 +bea26a2221fd8090ea38720fc445eca6 +``` + +3. (A) Download and run the plugin (using gitlab.com): + +```console +$ docker run -d \ + --publish=3000:3000 \ + --env=DRONE_DEBUG=true \ + --env=DRONE_SECRET=bea26a2221fd8090ea38720fc445eca6 \ + --env=TOKEN=9e6eij3ckzvpe9mrhnqcis6zf8dhopmm46e3pi96 \ + --env=PROVIDER=gitlab \ + --restart=always \ + --name=converter meltwater/drone-convert-pathschanged +``` + +3. (B) Download and run the plugin (using self-hosted GitLab) +```console +$ docker run -d \ + --publish=3000:3000 \ + --env=DRONE_DEBUG=true \ + --env=DRONE_SECRET=bea26a2221fd8090ea38720fc445eca6 \ + --env=TOKEN=9e6eij3ckzvpe9mrhnqcis6zf8dhopmm46e3pi96 \ + --env=PROVIDER=gitlab \ + --env=GITLAB_ADDRESS=https://gitlab.example.com \ + --restart=always \ + --name=converter meltwater/drone-convert-pathschanged +``` + + 4. Update your Drone server configuration to include the plugin address and the shared secret. ```text