From 2330feeb9495decffedf04cc6661e6381ba79a91 Mon Sep 17 00:00:00 2001 From: Ido David <36866853+ido-namely@users.noreply.github.com> Date: Mon, 29 Aug 2022 10:41:24 -0500 Subject: [PATCH] update protoc-gen-go and split go gRPC plugin to protoc-gen-go-grpc (#325) * update protoc-gen-go and split go gRPC plugin to protoc-gen-go-grpc * add protoc-gen-go and protoc-gen-go-grpc to renovate config * move additional go and go-grpc plugin args to plug_opt flag --- .github/renovate.json5 | 20 +++++++++++ Dockerfile | 21 +++++------- all/entrypoint.sh | 77 +++++++++++++++++++++++------------------- all/test/all_test.go | 39 +++++++++++++++++++-- build.sh | 2 ++ variables.sh | 2 ++ 6 files changed, 110 insertions(+), 51 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index e20126d6..34f2d521 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -133,6 +133,26 @@ ], depNameTemplate: "ts-proto", datasourceTemplate: "npm", + }, + { + fileMatch: [ + "variables.sh$" + ], + matchStrings: [ + "GO_PROTOC_GEN_GO_VERSION=\\$\\{GO_PROTOC_GEN_GO_VERSION:-(?.*?)\\}\\n" + ], + depNameTemplate: "google.golang.org/protobuf", + datasourceTemplate: "go", + }, + { + fileMatch: [ + "variables.sh$" + ], + matchStrings: [ + "GO_PROTOC_GEN_GO_GRPC_VERSION=\\$\\{GO_PROTOC_GEN_GO_GRPC_VERSION:-(?.*?)\\}\\n" + ], + depNameTemplate: "google.golang.org/grpc/cmd/protoc-gen-go-grpc", + datasourceTemplate: "go", } ], packageRules: [ diff --git a/Dockerfile b/Dockerfile index b88d3189..aac4142c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ ARG node_protoc_gen_grpc_web_version ARG ts_proto_version ARG go_envoyproxy_pgv_version ARG go_mwitkow_gpv_version +ARG go_protoc_gen_go_version +ARG go_protoc_gen_go_grpc_version FROM golang:$go_version-$debian AS build @@ -24,6 +26,9 @@ ARG scala_pb_version ARG go_envoyproxy_pgv_version ARG go_mwitkow_gpv_version ARG uber_prototool_version +ARG go_protoc_gen_go_version +ARG go_protoc_gen_go_grpc_version + RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -42,7 +47,6 @@ RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /tmp RUN git clone --depth 1 --shallow-submodules -b v$grpc_version.x --recursive https://github.com/grpc/grpc && \ git clone --depth 1 --shallow-submodules -b v$grpc_java_version.x --recursive https://github.com/grpc/grpc-java.git && \ - git clone --depth 1 --shallow-submodules -b v$grpc_version.x --recursive https://github.com/grpc/grpc-go.git && \ git clone --depth 1 https://github.com/googleapis/googleapis && \ git clone --depth 1 https://github.com/googleapis/api-common-protos @@ -68,14 +72,8 @@ RUN curl -fsSL "https://github.com/uber/prototool/releases/download/v${uber_prot -o /usr/local/bin/prototool && \ chmod +x /usr/local/bin/prototool -# Workaround for the transition to protoc-gen-go-grpc -# https://grpc.io/docs/languages/go/quickstart/#regenerate-grpc-code -RUN ( cd ./grpc-go/cmd/protoc-gen-go-grpc && go install . ) - # Go get go-related bins -WORKDIR /tmp -RUN set -e && \ - GO111MODULE=on go get google.golang.org/grpc@v1.48.0 +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${go_protoc_gen_go_grpc_version} # install protoc-gen-grpc-gateway and protoc-gen-openapiv2 RUN set -e && \ @@ -98,10 +96,7 @@ RUN set -e && \ RUN go get -u github.com/micro/micro/v3/cmd/protoc-gen-micro -# protoc-gen-go is depended on by protoc-gen-validate, install here and then overwrite later just in case to ensure that ultimately the right version is installed -RUN go get -u github.com/golang/protobuf/protoc-gen-go -RUN GO111MODULE=on go get -d github.com/envoyproxy/protoc-gen-validate@v${go_envoyproxy_pgv_version} -RUN make -C /go/pkg/mod/github.com/envoyproxy/protoc-gen-validate@v${go_envoyproxy_pgv_version}/ build +RUN go install github.com/envoyproxy/protoc-gen-validate@v${go_envoyproxy_pgv_version} # Add Ruby Sorbet types support (rbi) RUN go get -u github.com/coinbase/protoc-gen-rbi @@ -109,7 +104,7 @@ RUN go get -u github.com/coinbase/protoc-gen-rbi RUN go get github.com/gomatic/renderizer/v2/cmd/renderizer # Origin protoc-gen-go should be installed last, for not been overwritten by any other binaries(see #210) -RUN go get -u github.com/golang/protobuf/protoc-gen-go +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@${go_protoc_gen_go_version} # Need to get these too: RUN go get -u github.com/mwitkow/go-proto-validators/@v${go_mwitkow_gpv_version} diff --git a/all/entrypoint.sh b/all/entrypoint.sh index 1f30e4dc..c325b55a 100755 --- a/all/entrypoint.sh +++ b/all/entrypoint.sh @@ -7,40 +7,41 @@ printUsage() { echo "Usage: gen-proto -f my-service.proto -l go" echo " " echo "options:" - echo " -h, --help Show help" - echo " -f FILE The proto source file to generate" - echo " -d DIR Scans the given directory for all proto files" - echo " -l LANGUAGE The language to generate (${SUPPORTED_LANGUAGES[@]})" - echo " -o DIRECTORY The output directory for generated files. Will be automatically created." - echo " -i includes Extra includes" - echo " --lint CHECKS Enable linting protoc-lint (CHECKS are optional - see https://github.com/ckaznocha/protoc-gen-lint#optional-checks)" - echo " --with-gateway Generate grpc-gateway files (experimental)." - echo " --with-docs FORMAT Generate documentation (FORMAT is optional - see https://github.com/pseudomuto/protoc-gen-doc#invoking-the-plugin)" - echo " --with-rbi Generate Sorbet type declaration files (.rbi files) - see https://github.com/coinbase/protoc-gen-rbi" - echo " --with-typescript Generate TypeScript declaration files (.d.ts files) - see https://github.com/improbable-eng/ts-protoc-gen#readme" - echo " --with-validator Generate validations for (${VALIDATOR_SUPPORTED_LANGUAGES[@]}) - see https://github.com/envoyproxy/protoc-gen-validate" - echo " --validator-source-relative Make the output dirctory for protoc-gen-validate 'source relative' - see https://github.com/envoyproxy/protoc-gen-validate#go" - echo " --go-source-relative Make go import paths 'source_relative' - see https://github.com/golang/protobuf#parameters" - echo " --go-module-prefix Specify the module prefix to remove from the import path - see https://developers.google.com/protocol-buffers/docs/reference/go-generated#invocation" - echo " --go-package-map Map proto imports to go import paths" - echo " --go-plugin-micro Replaces the Go gRPC plugin with go-micro" - echo " --go-proto-validator Generate Go proto validations - see https://github.com/mwitkow/go-proto-validators" - echo " --no-google-includes Don't include Google protobufs" - echo " --descr-include-imports When using --descriptor_set_out, also include all dependencies of the input files in the set, so that the set is self-contained" - echo " --descr-include-source-info When using --descriptor_set_out, do not strip SourceCodeInfo from the FileDescriptorProto. This results in vastly - larger descriptors that include information about the original location of each decl in the source file as well - as surrounding comments." - echo " --descr-filename The filename for the descriptor proto when used with -l descriptor_set. Default to descriptor_set.pb" - echo " --csharp_opt The options to pass to protoc to customize the csharp code generation." - echo " --scala_opt The options to pass to protoc to customize the scala code generation." - echo " --with-swagger-json-names Use with --with-gateway flag. Generated swagger file will use JSON names instead of protobuf names. - (deprecated. Please use --with-openapi-json-names)" - echo " --with-openapi-json-names Use with --with-gateway flag. Generated OpenAPI file will use JSON names instead of protobuf names." - echo " --generate-unbound-methods Use with --with-gateway flag. Produce the HTTP mapping even for methods without any HttpRule annotation." - echo " --js-out This option overrides the 'js_out=' argument in the grpc-node and grpc-web code generation. Defaults to 'import_style=commonjs'." - echo " --grpc-out This option allows overriding the left-half of the 'grpc_out=' argument (before the colon) with grpc-node and grpc-web code generation. Options are: generate_package_definition, grpc_js or grpc(depricated from April 2021). Defaults to grpc_js." - echo " --grpc-web-out This option overrides the 'grpc-web_out=' argument in the grpc-web code generation. Defaults to 'import_style=typescript'." - echo " --ts_opt The options to pass to protoc to customize the typescript code generation. See https://github.com/stephenh/ts-proto#supported-options. --ts_opt useOptionals=messages will evaluate to --ts_proto_opt=useOptionals=messages" + echo " -h, --help Show help" + echo " -f FILE The proto source file to generate" + echo " -d DIR Scans the given directory for all proto files" + echo " -l LANGUAGE The language to generate (${SUPPORTED_LANGUAGES[@]})" + echo " -o DIRECTORY The output directory for generated files. Will be automatically created." + echo " -i includes Extra includes" + echo " --lint CHECKS Enable linting protoc-lint (CHECKS are optional - see https://github.com/ckaznocha/protoc-gen-lint#optional-checks)" + echo " --with-gateway Generate grpc-gateway files (experimental)." + echo " --with-docs FORMAT Generate documentation (FORMAT is optional - see https://github.com/pseudomuto/protoc-gen-doc#invoking-the-plugin)" + echo " --with-rbi Generate Sorbet type declaration files (.rbi files) - see https://github.com/coinbase/protoc-gen-rbi" + echo " --with-typescript Generate TypeScript declaration files (.d.ts files) - see https://github.com/improbable-eng/ts-protoc-gen#readme" + echo " --with-validator Generate validations for (${VALIDATOR_SUPPORTED_LANGUAGES[@]}) - see https://github.com/envoyproxy/protoc-gen-validate" + echo " --validator-source-relative Make the output dirctory for protoc-gen-validate 'source relative' - see https://github.com/envoyproxy/protoc-gen-validate#go" + echo " --go-source-relative Make go import paths 'source_relative' - see https://github.com/golang/protobuf#parameters" + echo " --go-module-prefix Specify the module prefix to remove from the import path - see https://developers.google.com/protocol-buffers/docs/reference/go-generated#invocation" + echo " --go-package-map Map proto imports to go import paths" + echo " --go-plugin-micro Replaces the Go gRPC plugin with go-micro" + echo " --go-proto-validator Generate Go proto validations - see https://github.com/mwitkow/go-proto-validators" + echo " --go-grpc-require-unimplemented-servers Generate Go gRPC service with unimplemented server for future compatability- https://github.com/grpc/grpc-go/tree/master/cmd/protoc-gen-go-grpc#future-proofing-services" + echo " --no-google-includes Don't include Google protobufs" + echo " --descr-include-imports When using --descriptor_set_out, also include all dependencies of the input files in the set, so that the set is self-contained" + echo " --descr-include-source-info When using --descriptor_set_out, do not strip SourceCodeInfo from the FileDescriptorProto. This results in vastly + larger descriptors that include information about the original location of each decl in the source file as well + as surrounding comments." + echo " --descr-filename The filename for the descriptor proto when used with -l descriptor_set. Default to descriptor_set.pb" + echo " --csharp_opt The options to pass to protoc to customize the csharp code generation." + echo " --scala_opt The options to pass to protoc to customize the scala code generation." + echo " --with-swagger-json-names Use with --with-gateway flag. Generated swagger file will use JSON names instead of protobuf names. + (deprecated. Please use --with-openapi-json-names)" + echo " --with-openapi-json-names Use with --with-gateway flag. Generated OpenAPI file will use JSON names instead of protobuf names." + echo " --generate-unbound-methods Use with --with-gateway flag. Produce the HTTP mapping even for methods without any HttpRule annotation." + echo " --js-out This option overrides the 'js_out=' argument in the grpc-node and grpc-web code generation. Defaults to 'import_style=commonjs'." + echo " --grpc-out This option allows overriding the left-half of the 'grpc_out=' argument (before the colon) with grpc-node and grpc-web code generation. Options are: generate_package_definition, grpc_js or grpc(depricated from April 2021). Defaults to grpc_js." + echo " --grpc-web-out This option overrides the 'grpc-web_out=' argument in the grpc-web code generation. Defaults to 'import_style=typescript'." + echo " --ts_opt The options to pass to protoc to customize the typescript code generation. See https://github.com/stephenh/ts-proto#supported-options. --ts_opt useOptionals=messages will evaluate to --ts_proto_opt=useOptionals=messages" } GEN_GATEWAY=false @@ -60,6 +61,7 @@ GO_MODULE_PREFIX="" GO_PACKAGE_MAP="" GO_PLUGIN="grpc" GO_VALIDATOR=false +GO_GRPC_REQUIRE_UNIMPLEMENTED_SERVERS="require_unimplemented_servers=false" NO_GOOGLE_INCLUDES=false DESCR_INCLUDE_IMPORTS=false DESCR_INCLUDE_SOURCE_INFO=false @@ -176,6 +178,10 @@ while test $# -gt 0; do GO_VALIDATOR=true shift ;; + --go-grpc-require-unimplemented-servers) + GO_GRPC_REQUIRE_UNIMPLEMENTED_SERVERS="require_unimplemented_servers=true" + shift + ;; --no-google-includes) NO_GOOGLE_INCLUDES=true shift @@ -329,7 +335,8 @@ fi GEN_STRING='' case $GEN_LANG in "go") - GEN_STRING="--go_out=${GO_SOURCE_RELATIVE}${GO_MODULE_PREFIX}${GO_PACKAGE_MAP}plugins=grpc:$OUT_DIR" + GEN_STRING="--go_out=$OUT_DIR --go_opt=${GO_SOURCE_RELATIVE}${GO_MODULE_PREFIX}${GO_PACKAGE_MAP}\ + --go-grpc_out=$OUT_DIR --go-grpc_opt=${GO_SOURCE_RELATIVE}${GO_MODULE_PREFIX}${GO_PACKAGE_MAP}${GO_GRPC_REQUIRE_UNIMPLEMENTED_SERVERS}" if [[ ${GO_PLUGIN} == "micro" ]]; then GEN_STRING="$GEN_STRING --micro_out=$OUT_DIR" fi diff --git a/all/test/all_test.go b/all/test/all_test.go index 728a0094..92d3c6b8 100644 --- a/all/test/all_test.go +++ b/all/test/all_test.go @@ -53,14 +53,23 @@ func (s *TestSuite) TestAllCases() { lang: "go", protofileName: "all/test/test.proto", expectedOutputDir: "gen/pb-go", - fileExpectations: []FileExpectation{{fileName: "all/test.pb.go"}}, + fileExpectations: []FileExpectation{ + {fileName: "all/test.pb.go"}, + {fileName: "/all/test_grpc.pb.go", assert: func(filePath, expectedValue string) { + fileText := s.readFile(filePath) + s.Assert().False(strings.Contains(fileText, expectedValue), "contains \"%s\"", expectedValue) + }, expectedValue: "func (UnimplementedMessageServer) mustEmbedUnimplementedMessageServer() {}"}, + }, }, "go with alternative output dir": { lang: "go", protofileName: "all/test/test.proto", expectedOutputDir: "gen/foo/bar", - fileExpectations: []FileExpectation{{fileName: "all/test.pb.go"}}, - extraArgs: []string{"-o", "gen/foo/bar"}, + fileExpectations: []FileExpectation{ + {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, + }, + extraArgs: []string{"-o", "gen/foo/bar"}, }, "ruby": { lang: "ruby", @@ -309,6 +318,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/doc/index.html"}, }, extraArgs: []string{"--with-docs"}, @@ -319,6 +329,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/doc/index.md"}, }, extraArgs: []string{"--with-docs", "markdown,index.md"}, @@ -329,6 +340,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/all/test.pb.gw.go", assert: func(filePath, expectedValue string) { fileText := s.readFile(filePath) s.Assert().False(strings.Contains(fileText, expectedValue), "contains \"%s\"", expectedValue) @@ -359,6 +371,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/all/test.pb.gw.go"}, {fileName: "/all/test/test.swagger.json", assert: func(filePath, expectedValue string) { fileText := s.readFile(filePath) @@ -388,6 +401,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/all/test.pb.gw.go"}, {fileName: "/all/test/test.swagger.json", assert: func(filePath, expectedValue string) { fileText := s.readFile(filePath) @@ -402,6 +416,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "all/test.pb.go"}, + {fileName: "all/test_grpc.pb.go"}, {fileName: "/all/test.pb.gw.go", assert: func(filePath, expectedValue string) { fileText := s.readFile(filePath) s.Assert().True(strings.Contains(fileText, expectedValue), "does not contain \"%s\"", expectedValue) @@ -416,6 +431,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "/all/test/test.pb.go"}, + {fileName: "/all/test/test_grpc.pb.go"}, }, extraArgs: []string{"--go-source-relative"}, }, @@ -425,6 +441,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "/test.pb.go"}, + {fileName: "/test_grpc.pb.go"}, }, extraArgs: []string{"--go-module-prefix", "all"}, }, @@ -448,6 +465,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "/all/test.pb.go"}, + {fileName: "/all/test_grpc.pb.go"}, {fileName: "/all/test.pb.validate.go"}, }, extraArgs: []string{"--with-validator"}, @@ -468,6 +486,7 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "/all/test.pb.go"}, + {fileName: "/all/test_grpc.pb.go"}, {fileName: "/all/test/test.pb.validate.go"}, }, extraArgs: []string{"--with-validator", "--validator-source-relative"}, @@ -478,10 +497,24 @@ func (s *TestSuite) TestAllCases() { expectedOutputDir: "gen/pb-go", fileExpectations: []FileExpectation{ {fileName: "/all/test.pb.go"}, + {fileName: "/all/test_grpc.pb.go"}, {fileName: "/all/test.validator.pb.go"}, }, extraArgs: []string{"--go-proto-validator"}, }, + "go with require-unimplemented-servers": { + lang: "go", + protofileName: "all/test/test.proto", + expectedOutputDir: "gen/pb-go", + fileExpectations: []FileExpectation{ + {fileName: "/all/test.pb.go"}, + {fileName: "/all/test_grpc.pb.go", assert: func(filePath, expectedValue string) { + fileText := s.readFile(filePath) + s.Assert().True(strings.Contains(fileText, expectedValue), "does not contain \"%s\"", expectedValue) + }, expectedValue: "func (UnimplementedMessageServer) mustEmbedUnimplementedMessageServer() {}"}, + }, + extraArgs: []string{"--go-grpc-require-unimplemented-servers"}, + }, "go micro": { lang: "go", protofileName: "all/test/test.proto", diff --git a/build.sh b/build.sh index eb5462ae..49de8912 100755 --- a/build.sh +++ b/build.sh @@ -22,6 +22,8 @@ for build in ${BUILDS[@]}; do --build-arg go_envoyproxy_pgv_version="${GO_ENVOYPROXY_PGV_VERSION}" \ --build-arg go_mwitkow_gpv_version="${GO_MWITKOW_GPV_VERSION}" \ --build-arg ts_proto_version="${TS_PROTO_VERSION}" \ + --build-arg go_protoc_gen_go_version="${GO_PROTOC_GEN_GO_VERSION}" \ + --build-arg go_protoc_gen_go_grpc_version="${GO_PROTOC_GEN_GO_GRPC_VERSION}" \ --target "${build}" \ . diff --git a/variables.sh b/variables.sh index 409a534a..ec0907ff 100755 --- a/variables.sh +++ b/variables.sh @@ -20,5 +20,7 @@ NODE_PROTOC_GEN_GRPC_WEB_VERSION=${NODE_PROTOC_GEN_GRPC_WEB_VERSION:-1.4.0} TS_PROTO_VERSION=${TS_PROTO_VERSION:1.117.0} GO_ENVOYPROXY_PGV_VERSION=${GO_ENVOYPROXY_PGV_VERSION:-0.6.7} GO_MWITKOW_GPV_VERSION=${GO_MWITKOW_GPV_VERSION:-0.3.2} +GO_PROTOC_GEN_GO_VERSION=${GO_PROTOC_GEN_GO_VERSION:-v1.28.1} +GO_PROTOC_GEN_GO_GRPC_VERSION=${GO_PROTOC_GEN_GO_GRPC_VERSION:-v1.2.0} BUILD_VERSION="${BUILD_VERSION:-local}" VERSION="${VERSION:-${GRPC_VERSION}_${BUILD_VERSION}}"