From 5325945c72ae7516a47a5f3c5a6631a87cedec0b Mon Sep 17 00:00:00 2001 From: tk_sky <63036400+tksky1@users.noreply.github.com> Date: Tue, 24 Oct 2023 19:27:16 +0800 Subject: [PATCH] feat: add yaml-config support for trimmer (#131) Co-authored-by: Li2CO3 --- generator/golang/backend.go | 2 +- tool/trimmer/args.go | 4 +- tool/trimmer/dirTree.go | 81 ------------------- tool/trimmer/dirTree_test.go | 58 ------------- tool/trimmer/main.go | 41 +++++----- .../test_cases/tests/dir/dir2/test.thrift | 11 +++ .../tests/dir/dir2/trim_config.yaml | 19 +++++ .../tests/dir/dir3/dir4/another.thrift | 16 ++++ tool/trimmer/trim/config.go | 50 ++++++++++++ tool/trimmer/trim/mark.go | 5 ++ tool/trimmer/trim/traversal.go | 4 +- tool/trimmer/trim/trimmer.go | 43 ++++++++-- tool/trimmer/trim/trimmer_test.go | 47 +++++++++++ 13 files changed, 210 insertions(+), 171 deletions(-) delete mode 100644 tool/trimmer/dirTree.go delete mode 100644 tool/trimmer/dirTree_test.go create mode 100644 tool/trimmer/test_cases/tests/dir/dir2/trim_config.yaml create mode 100644 tool/trimmer/test_cases/tests/dir/dir3/dir4/another.thrift create mode 100644 tool/trimmer/trim/config.go diff --git a/generator/golang/backend.go b/generator/golang/backend.go index d96e843d..e653c1db 100644 --- a/generator/golang/backend.go +++ b/generator/golang/backend.go @@ -86,7 +86,7 @@ func (g *GoBackend) Generate(req *plugin.Request, log backend.LogFunc) *plugin.R g.log = log g.prepareUtilities() if g.utils.Features().TrimIDL { - err := trim.TrimAST(req.AST, nil, false) + err := trim.TrimAST(&trim.TrimASTArg{Ast: req.AST, TrimMethods: nil, Preserve: nil}) if err != nil { g.log.Warn("trim error:", err.Error()) } diff --git a/tool/trimmer/args.go b/tool/trimmer/args.go index eee0f637..ca3cea05 100644 --- a/tool/trimmer/args.go +++ b/tool/trimmer/args.go @@ -63,8 +63,8 @@ func (a *Arguments) BuildFlags() *flag.FlagSet { f.Var(&a.Methods, "m", "") f.Var(&a.Methods, "method", "") - f.StringVar(&a.Preserve, "p", "true", "") - f.StringVar(&a.Preserve, "preserve", "true", "") + f.StringVar(&a.Preserve, "p", "", "") + f.StringVar(&a.Preserve, "preserve", "", "") f.Usage = help return f diff --git a/tool/trimmer/dirTree.go b/tool/trimmer/dirTree.go deleted file mode 100644 index c7c39683..00000000 --- a/tool/trimmer/dirTree.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2023 CloudWeGo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" -) - -// create directory-tree before dump -func createDirTree(sourceDir, destinationDir string) { - err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - if path[len(sourceDir)-1] != filepath.Separator { - path = path + string(filepath.Separator) - } - newDir := filepath.Join(destinationDir, path[len(sourceDir):]) - err := os.MkdirAll(newDir, os.ModePerm) - if err != nil { - return errors.New("create dir tree error:" + err.Error()) - } - } - return nil - }) - if err != nil { - fmt.Printf("manage output error: %v\n", err) - os.Exit(2) - } -} - -// remove empty directory of output dir-tree -func removeEmptyDir(source string) { - files, err := os.ReadDir(source) - if err != nil { - return - } - for _, file := range files { - if file.IsDir() { - removeEmptyDir(source + string(filepath.Separator) + file.Name()) - } - } - empty, err := isDirectoryEmpty(source) - if empty || err != nil { - _ = os.RemoveAll(source) - } -} - -func isDirectoryEmpty(path string) (bool, error) { - dir, err := os.Open(path) - if err != nil { - return false, err - } - defer dir.Close() - - _, err = dir.Readdirnames(1) - if err == nil { - return false, nil - } - - if errors.Is(err, os.ErrNotExist) { - return true, nil - } - return false, err -} diff --git a/tool/trimmer/dirTree_test.go b/tool/trimmer/dirTree_test.go deleted file mode 100644 index 7ae23697..00000000 --- a/tool/trimmer/dirTree_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2023 CloudWeGo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/cloudwego/thriftgo/pkg/test" -) - -func TestDirTree(t *testing.T) { - _ = os.RemoveAll("trimmer_test") - createDirTree("test_cases", "trimmer_test") - fileCount, dirCount, err := countFilesAndSubdirectories("trimmer_test") - test.Assert(t, err == nil) - test.Assert(t, fileCount == 0) - test.Assert(t, dirCount == 4) - removeEmptyDir("trimmer_test") - _, err = os.ReadDir("trimmer_test") - test.Assert(t, err != nil) -} - -func countFilesAndSubdirectories(dirPath string) (int, int, error) { - var fileCount, dirCount int - files, err := os.ReadDir(dirPath) - if err != nil { - return 0, 0, err - } - for _, file := range files { - if file.IsDir() { - dirCount++ - subDirPath := filepath.Join(dirPath, file.Name()) - subFileCount, subDirCount, err := countFilesAndSubdirectories(subDirPath) - if err != nil { - return 0, 0, err - } - fileCount += subFileCount - dirCount += subDirCount - } else { - fileCount++ - } - } - return fileCount, dirCount, nil -} diff --git a/tool/trimmer/main.go b/tool/trimmer/main.go index b955fda0..fa97afac 100644 --- a/tool/trimmer/main.go +++ b/tool/trimmer/main.go @@ -53,13 +53,14 @@ func main() { os.Exit(0) } - preserve := true + var preserveInput *bool if a.Preserve != "" { - preserve, err = strconv.ParseBool(a.Preserve) + preserve, err := strconv.ParseBool(a.Preserve) if err != nil { help() os.Exit(2) } + preserveInput = &preserve } // parse file to ast @@ -74,7 +75,9 @@ func main() { check(semantic.ResolveSymbols(ast)) // trim ast - check(trim.TrimAST(ast, a.Methods, !preserve)) + check(trim.TrimAST(&trim.TrimASTArg{ + Ast: ast, TrimMethods: a.Methods, Preserve: preserveInput, + })) // dump the trimmed ast to idl idl, err := dump.DumpIDL(ast) @@ -117,16 +120,7 @@ func main() { println("output-dir should be set outside of -r base-dir to avoid overlay") os.Exit(2) } - createDirTree(a.Recurse, a.OutputFile) recurseDump(ast, a.Recurse, a.OutputFile) - relativePath, err := filepath.Rel(a.Recurse, a.IDL) - if err != nil { - println("-r input err, range should cover all the target IDLs;", err.Error()) - os.Exit(2) - } - outputFileUrl := filepath.Join(a.OutputFile, relativePath) - check(writeStringToFile(outputFileUrl, idl)) - removeEmptyDir(a.OutputFile) } else { check(writeStringToFile(a.OutputFile, idl)) } @@ -139,16 +133,21 @@ func recurseDump(ast *parser.Thrift, sourceDir, outDir string) { if ast == nil { return } + out, err := dump.DumpIDL(ast) + check(err) + relativeUrl, err := filepath.Rel(sourceDir, ast.Filename) + if err != nil { + println("-r input err, range should cover all the target IDLs;", err.Error()) + os.Exit(2) + } + outputFileUrl := filepath.Join(outDir, relativeUrl) + err = os.MkdirAll(filepath.Dir(outputFileUrl), os.ModePerm) + if err != nil { + println("mkdir", filepath.Dir(outputFileUrl), "error:", err.Error()) + os.Exit(2) + } + check(writeStringToFile(outputFileUrl, out)) for _, includes := range ast.Includes { - out, err := dump.DumpIDL(includes.Reference) - check(err) - relativeUrl, err := filepath.Rel(sourceDir, includes.Reference.Filename) - if err != nil { - println("-r input err, range should cover all the target IDLs;", err.Error()) - os.Exit(2) - } - outputFileUrl := filepath.Join(outDir, relativeUrl) - check(writeStringToFile(outputFileUrl, out)) recurseDump(includes.Reference, sourceDir, outDir) } } diff --git a/tool/trimmer/test_cases/tests/dir/dir2/test.thrift b/tool/trimmer/test_cases/tests/dir/dir2/test.thrift index a7359c51..804cb5d8 100644 --- a/tool/trimmer/test_cases/tests/dir/dir2/test.thrift +++ b/tool/trimmer/test_cases/tests/dir/dir2/test.thrift @@ -13,8 +13,19 @@ // limitations under the License. include "../../../sample1.thrift" +include "../dir3/dir4/another.thrift" // @preserve struct TestStruct{ 1: sample1.Person person + 2: another.AnotherStruct another +} + +service TestService{ + void func1() + another.AnotherStruct func2() + void func3() +} + +union useless{ } \ No newline at end of file diff --git a/tool/trimmer/test_cases/tests/dir/dir2/trim_config.yaml b/tool/trimmer/test_cases/tests/dir/dir2/trim_config.yaml new file mode 100644 index 00000000..f65338b1 --- /dev/null +++ b/tool/trimmer/test_cases/tests/dir/dir2/trim_config.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 CloudWeGo Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +methods: + - "TestService.func1" + - "TestService.func3" +preserve: true +preserved_structs: + - "useless" \ No newline at end of file diff --git a/tool/trimmer/test_cases/tests/dir/dir3/dir4/another.thrift b/tool/trimmer/test_cases/tests/dir/dir3/dir4/another.thrift new file mode 100644 index 00000000..20aff287 --- /dev/null +++ b/tool/trimmer/test_cases/tests/dir/dir3/dir4/another.thrift @@ -0,0 +1,16 @@ +// Copyright 2023 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +struct AnotherStruct{ +} \ No newline at end of file diff --git a/tool/trimmer/trim/config.go b/tool/trimmer/trim/config.go new file mode 100644 index 00000000..28786bab --- /dev/null +++ b/tool/trimmer/trim/config.go @@ -0,0 +1,50 @@ +// Copyright 2023 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trim + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +var DefaultYamlFileName = "trim_config.yaml" + +type YamlArguments struct { + Methods []string `yaml:"methods,omitempty"` + Preserve *bool `yaml:"preserve,omitempty"` + PreservedStructs []string `yaml:"preserved_structs,omitempty"` +} + +func ParseYamlConfig(path string) *YamlArguments { + cfg := YamlArguments{} + dataBytes, err := os.ReadFile(filepath.Join(path, DefaultYamlFileName)) + if err != nil { + return nil + } + fmt.Println("using trim config:", filepath.Join(path, DefaultYamlFileName)) + err = yaml.Unmarshal(dataBytes, &cfg) + if err != nil { + fmt.Println("unmarshal yaml config fail:", err) + return nil + } + if cfg.Preserve == nil { + t := true + cfg.Preserve = &t + } + return &cfg +} diff --git a/tool/trimmer/trim/mark.go b/tool/trimmer/trim/mark.go index 55adbb1a..8bd5e903 100644 --- a/tool/trimmer/trim/mark.go +++ b/tool/trimmer/trim/mark.go @@ -261,5 +261,10 @@ func (t *Trimmer) checkPreserve(theStruct *parser.StructLike) bool { if t.forceTrimming { return false } + for _, name := range t.preservedStructs { + if name == theStruct.Name { + return true + } + } return t.preserveRegex.MatchString(strings.ToLower(theStruct.ReservedComments)) } diff --git a/tool/trimmer/trim/traversal.go b/tool/trimmer/trim/traversal.go index ddba0cf8..bc4441e0 100644 --- a/tool/trimmer/trim/traversal.go +++ b/tool/trimmer/trim/traversal.go @@ -40,7 +40,7 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listUnion []*parser.StructLike for i := range ast.Unions { - if t.marks[filename][ast.Unions[i]] { + if t.marks[filename][ast.Unions[i]] || t.checkPreserve(ast.Unions[i]) { listUnion = append(listUnion, ast.Unions[i]) } } @@ -48,7 +48,7 @@ func (t *Trimmer) traversal(ast *parser.Thrift, filename string) { var listException []*parser.StructLike for i := range ast.Exceptions { - if t.marks[filename][ast.Exceptions[i]] { + if t.marks[filename][ast.Exceptions[i]] || t.checkPreserve(ast.Exceptions[i]) { listException = append(listException, ast.Exceptions[i]) } } diff --git a/tool/trimmer/trim/trimmer.go b/tool/trimmer/trim/trimmer.go index 39a143f2..97fe1bdd 100644 --- a/tool/trimmer/trim/trimmer.go +++ b/tool/trimmer/trim/trimmer.go @@ -32,14 +32,44 @@ type Trimmer struct { marks map[string]map[interface{}]bool outDir string // use -m - trimMethods []string - trimMethodValid []bool - preserveRegex *regexp.Regexp - forceTrimming bool + trimMethods []string + trimMethodValid []bool + preserveRegex *regexp.Regexp + forceTrimming bool + preservedStructs []string } -// TrimAST trim the single AST, pass method names if -m specified -func TrimAST(ast *parser.Thrift, trimMethods []string, forceTrimming bool) error { +type TrimASTArg struct { + Ast *parser.Thrift + TrimMethods []string + Preserve *bool +} + +// TrimAST parse the cfg and trim the single AST +func TrimAST(arg *TrimASTArg) error { + var preservedStructs []string + if wd, err := os.Getwd(); err == nil { + cfg := ParseYamlConfig(wd) + if cfg != nil { + if len(arg.TrimMethods) == 0 && len(cfg.Methods) > 0 { + arg.TrimMethods = cfg.Methods + } + if arg.Preserve == nil && !(*cfg.Preserve) { + preserve := false + arg.Preserve = &preserve + } + preservedStructs = cfg.PreservedStructs + } + } + forceTrim := false + if arg.Preserve != nil { + forceTrim = !*arg.Preserve + } + return doTrimAST(arg.Ast, arg.TrimMethods, forceTrim, preservedStructs) +} + +// doTrimAST trim the single AST, pass method names if -m specified +func doTrimAST(ast *parser.Thrift, trimMethods []string, forceTrimming bool, preservedStructs []string) error { trimmer, err := newTrimmer(nil, "") if err != nil { return err @@ -59,6 +89,7 @@ func TrimAST(ast *parser.Thrift, trimMethods []string, forceTrimming bool) error } } } + trimmer.preservedStructs = preservedStructs trimmer.markAST(ast) trimmer.traversal(ast, ast.Filename) if path := parser.CircleDetect(ast); len(path) > 0 { diff --git a/tool/trimmer/trim/trimmer_test.go b/tool/trimmer/trim/trimmer_test.go index 5687a2d1..0dd219ae 100644 --- a/tool/trimmer/trim/trimmer_test.go +++ b/tool/trimmer/trim/trimmer_test.go @@ -85,3 +85,50 @@ func TestInclude(t *testing.T) { test.Assert(t, len(ast.Includes) == 1) test.Assert(t, ast.Includes[0].Used == nil) } + +func TestTrimMethod(t *testing.T) { + filename := filepath.Join("..", "test_cases", "tests", "dir", "dir2", "test.thrift") + ast, err := parser.ParseFile(filename, nil, true) + check(err) + if path := parser.CircleDetect(ast); len(path) > 0 { + check(fmt.Errorf("found include circle:\n\t%s", path)) + } + checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) + _, err = checker.CheckAll(ast) + check(err) + check(semantic.ResolveSymbols(ast)) + + methods := make([]string, 1) + methods[0] = "func1" + + err = TrimAST(&TrimASTArg{ + Ast: ast, + TrimMethods: methods, + Preserve: nil, + }) + check(err) + test.Assert(t, len(ast.Services[0].Functions) == 1) +} + +func TestPreserve(t *testing.T) { + filename := filepath.Join("..", "test_cases", "tests", "dir", "dir2", "test.thrift") + ast, err := parser.ParseFile(filename, nil, true) + check(err) + if path := parser.CircleDetect(ast); len(path) > 0 { + check(fmt.Errorf("found include circle:\n\t%s", path)) + } + checker := semantic.NewChecker(semantic.Options{FixWarnings: true}) + _, err = checker.CheckAll(ast) + check(err) + check(semantic.ResolveSymbols(ast)) + + preserve := false + + err = TrimAST(&TrimASTArg{ + Ast: ast, + TrimMethods: nil, + Preserve: &preserve, + }) + check(err) + test.Assert(t, len(ast.Structs) == 0) +}