diff --git a/README-zh_cn.md b/README-zh_cn.md index d61b871..5ebf0e7 100644 --- a/README-zh_cn.md +++ b/README-zh_cn.md @@ -1,6 +1,6 @@ # ๐Ÿšซ go-sensitive -[![build](https://img.shields.io/badge/build-0.90-brightgreen)](https://github.com/LanshanTeam/wecqupt)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/LanshanTeam/wecqupt) +[![build](https://img.shields.io/badge/build-1.01-brightgreen)](https://github.com/StellarisW/go-sensitive)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/StellarisW/go-sensitive) [English](README.md) | ไธญๆ–‡ @@ -17,6 +17,8 @@ - `FindAllCount()` ่ฟ”ๅ›žๅŒน้…ๅˆฐ็š„ๆ‰€ๆœ‰ๆ•ๆ„Ÿ่ฏๅŠๅ‡บ็Žฐๆฌกๆ•ฐ - ๆ”ฏๆŒๅคš็งๆ•ฐๆฎๆบๅŠ ่ฝฝ, ๅŠจๆ€ไฟฎๆ”นๆ•ฐๆฎๆบ - ๆ”ฏๆŒๅ†…ๅญ˜ๅญ˜ๅ‚จ + - ๆ”ฏๆŒmysqlๅญ˜ๅ‚จ + - ๆ”ฏๆŒmongoๅญ˜ๅ‚จ - ๆ”ฏๆŒๅคš็งๅญ—ๅ…ธๅŠ ่ฝฝๆ–นๅผ - ๆ”ฏๆŒ่ฟ่กŒ่ฟ‡็จ‹ไธญๅŠจๆ€ไฟฎๆ”นๆ•ฐๆฎๆบ - ๆ”ฏๆŒๅคš็ง่ฟ‡ๆปค็ฎ—ๆณ• @@ -75,13 +77,13 @@ func main() { ## โœ” Get ``` -$ go get -u github.com/stellarisw/go-sensitive +$ go get -u github.com/StellarisW/go-sensitive ``` ## ๐Ÿ“‚ Import ```go -import "github.com/stellarisw/go-sensitive" +import "github.com/StellarisW/go-sensitive" ``` ## @@ -89,6 +91,4 @@ import "github.com/stellarisw/go-sensitive" ## ๐Ÿ“Œ TODO - [ ] add mongo data source support -- [ ] add redis data source support -- [ ] add mysql data source support - [ ] add bloom algorithm \ No newline at end of file diff --git a/README.md b/README.md index ed095b1..b235d49 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ๐Ÿšซ go-sensitive -[![build](https://img.shields.io/badge/build-0.90-brightgreen)](https://github.com/LanshanTeam/wecqupt)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/LanshanTeam/wecqupt) +[![build](https://img.shields.io/badge/build-1.01-brightgreen)](https://github.com/StellarisW/go-sensitive)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/StellarisW/go-sensitive) English | [ไธญๆ–‡](README-zh_cn.md) @@ -17,6 +17,8 @@ English | [ไธญๆ–‡](README-zh_cn.md) - `FindAllCount()` return all sensitive word with its count that has been found in the text - support multiple data sources with dynamic modification - support memory storage + - support mysql storage + - support mongo storage - support multiple ways of add dict - support dynamic add/del sensitive word while running - support multiple filter algorithms @@ -81,14 +83,12 @@ $ go get -u github.com/StellairsW/go-sensitive ## ๐Ÿ“‚ Import ```go -import "github.com/stellarisw/go-sensitive" +import "github.com/StellairsW/go-sensitive" ``` ## ## ๐Ÿ“Œ TODO -- [ ] add mongo data source support - [ ] add redis data source support -- [ ] add mysql data source support -- [ ] add bloom algorithm +- [ ] add bloom algorithm diff --git a/go.mod b/go.mod index 65aa022..6b50ce8 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,38 @@ module github.com/StellarisW/go-sensitive go 1.19 require ( + github.com/go-sql-driver/mysql v1.7.0 github.com/imroc/req/v3 v3.30.0 + github.com/jmoiron/sqlx v1.3.5 github.com/orcaman/concurrent-map/v2 v2.0.1 + go.mongodb.org/mongo-driver v1.11.1 ) require ( github.com/cheekybits/genny v1.0.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/marten-seemann/qpack v0.2.1 // indirect github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.1 // indirect + github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.11 // indirect diff --git a/go.sum b/go.sum index bee5f73..2256ca8 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,9 @@ github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmV github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -50,11 +53,15 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -74,13 +81,21 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imroc/req/v3 v3.30.0 h1:4iSXgIQfh/3N7JK9Lt7S0q3n/ZvuGICYwV3iv/MWY1M= github.com/imroc/req/v3 v3.30.0/go.mod h1:DKtNwSxMdpqZKJ6neBw8VwRioq78uwmQB4ynQEXNNUk= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= @@ -93,10 +108,14 @@ github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDA github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -116,6 +135,8 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -150,14 +171,27 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= +go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= @@ -167,8 +201,9 @@ golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -194,8 +229,9 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -207,6 +243,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -255,6 +292,7 @@ golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -282,6 +320,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -292,6 +331,9 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/manager_test.go b/manager_test.go index 6967ef6..938f681 100644 --- a/manager_test.go +++ b/manager_test.go @@ -1,8 +1,11 @@ package sensitive import ( + "fmt" + "github.com/StellarisW/go-sensitive/store" "reflect" "testing" + "time" ) func Test_NewFilter(t *testing.T) { @@ -19,7 +22,41 @@ func Test_NewFilter(t *testing.T) { args: args{ storeOption: StoreOption{ Type: StoreMemory, - Dsn: "", + }, + filterOption: FilterOption{ + Type: FilterDfa, + }, + }, + }, + { + name: "mongo+dfa", + args: args{ + storeOption: StoreOption{ + Type: StoreMongo, + MongoConfig: &store.MongoConfig{ + Address: "", + Port: "", + Username: "", + Password: "", + Database: "", + Collection: "", + }, + }, + filterOption: FilterOption{ + Type: FilterDfa, + }, + }, + }, + { + name: "mysql+dfa", + args: args{ + storeOption: StoreOption{ + Type: StoreMysql, + MysqlConfig: &store.MysqlConfig{ + Dsn: "", + Database: "", + TableName: "", + }, }, filterOption: FilterOption{ Type: FilterDfa, @@ -31,11 +68,18 @@ func Test_NewFilter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { filterManager := NewFilter(tt.args.storeOption, tt.args.filterOption) - err := filterManager.GetStore().AddWord("ๆ•ๆ„Ÿ่ฏ1", "ๆ•ๆ„Ÿ่ฏ2", "ๆ•ๆ„Ÿ่ฏ3") + err := filterManager.GetStore().LoadDictPath("./dict/default_dict.txt") + if err != nil { + fmt.Println(err) + } + + err = filterManager.GetStore().AddWord("ๆ•ๆ„Ÿ่ฏ1", "ๆ•ๆ„Ÿ่ฏ2", "ๆ•ๆ„Ÿ่ฏ3") if err != nil { t.Errorf("add sensitive word failed, err: %v", err) } + time.Sleep(1 * time.Second) + isSensitive := filterManager.GetFilter().IsSensitive("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") if !reflect.DeepEqual(isSensitive, true) { t.Errorf("IsSensitive() = %v, want %v", isSensitive, true) @@ -43,27 +87,27 @@ func Test_NewFilter(t *testing.T) { filtered := filterManager.GetFilter().Filter("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") if !reflect.DeepEqual(filtered, "่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") { - t.Errorf("IsSensitive() = %v, want %v", filtered, "่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") + t.Errorf("Filter() = %v, want %v", filtered, "่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™ๆ˜ฏ,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") } replaced := filterManager.GetFilter().Replace("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ", '*') if !reflect.DeepEqual(replaced, "่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") { - t.Errorf("IsSensitive() = %v, want %v", replaced, "่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") + t.Errorf("Replace() = %v, want %v", replaced, "่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™ๆ˜ฏ****,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") } matchedOne := filterManager.GetFilter().FindOne("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") if !reflect.DeepEqual(matchedOne, "ๆ•ๆ„Ÿ่ฏ1") { - t.Errorf("IsSensitive() = %v, want %v", matchedOne, "ๆ•ๆ„Ÿ่ฏ1") + t.Errorf("FindOne() = %v, want %v", matchedOne, "ๆ•ๆ„Ÿ่ฏ1") } matchedAll := filterManager.GetFilter().FindAll("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") if !reflect.DeepEqual(matchedAll, []string{"ๆ•ๆ„Ÿ่ฏ1", "ๆ•ๆ„Ÿ่ฏ2", "ๆ•ๆ„Ÿ่ฏ3"}) { - t.Errorf("IsSensitive() = %v, want %v", matchedAll, []string{"ๆ•ๆ„Ÿ่ฏ1", "ๆ•ๆ„Ÿ่ฏ2", "ๆ•ๆ„Ÿ่ฏ3"}) + t.Errorf("FindAll() = %v, want %v", matchedAll, []string{"ๆ•ๆ„Ÿ่ฏ1", "ๆ•ๆ„Ÿ่ฏ2", "ๆ•ๆ„Ÿ่ฏ3"}) } matchedMap := filterManager.GetFilter().FindAllCount("่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ2,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ3,่ฟ™ๆ˜ฏๆ•ๆ„Ÿ่ฏ1,่ฟ™้‡Œๆฒกๆœ‰ๆ•ๆ„Ÿ่ฏ") if !reflect.DeepEqual(matchedMap, map[string]int{"ๆ•ๆ„Ÿ่ฏ1": 2, "ๆ•ๆ„Ÿ่ฏ2": 1, "ๆ•ๆ„Ÿ่ฏ3": 1}) { - t.Errorf("IsSensitive() = %v, want %v", matchedMap, map[string]int{"ๆ•ๆ„Ÿ่ฏ1": 2, "ๆ•ๆ„Ÿ่ฏ2": 1, "ๆ•ๆ„Ÿ่ฏ3": 1}) + t.Errorf("FindAllCount() = %v, want %v", matchedMap, map[string]int{"ๆ•ๆ„Ÿ่ฏ1": 2, "ๆ•ๆ„Ÿ่ฏ2": 1, "ๆ•ๆ„Ÿ่ฏ3": 1}) } }) } diff --git a/options.go b/options.go index 0c131de..4da74da 100644 --- a/options.go +++ b/options.go @@ -1,7 +1,13 @@ package sensitive +import ( + "github.com/StellarisW/go-sensitive/store" +) + const ( StoreMemory = iota + StoreMysql + StoreMongo ) const ( @@ -9,8 +15,9 @@ const ( ) type StoreOption struct { - Type uint32 - Dsn string + Type uint32 + MysqlConfig *store.MysqlConfig + MongoConfig *store.MongoConfig } type FilterOption struct { diff --git a/store/memory.go b/store/memory.go index f82dc52..4f4f318 100644 --- a/store/memory.go +++ b/store/memory.go @@ -12,15 +12,15 @@ import ( type MemoryModel struct { store cmap.ConcurrentMap[string, struct{}] - AddChan chan string - DelChan chan string + addChan chan string + delChan chan string } func NewMemoryModel() *MemoryModel { return &MemoryModel{ store: cmap.New[struct{}](), - AddChan: make(chan string), - DelChan: make(chan string), + addChan: make(chan string), + delChan: make(chan string), } } @@ -85,7 +85,7 @@ func (m *MemoryModel) LoadDict(reader io.Reader) error { } m.store.Set(string(line), struct{}{}) - m.AddChan <- string(line) + m.addChan <- string(line) } return nil @@ -115,17 +115,17 @@ func (m *MemoryModel) ReadString() []string { } func (m *MemoryModel) GetAddChan() <-chan string { - return m.AddChan + return m.addChan } func (m *MemoryModel) GetDelChan() <-chan string { - return m.DelChan + return m.delChan } func (m *MemoryModel) AddWord(words ...string) error { for _, word := range words { m.store.Set(word, struct{}{}) - m.AddChan <- word + m.addChan <- word } return nil @@ -134,7 +134,7 @@ func (m *MemoryModel) AddWord(words ...string) error { func (m *MemoryModel) DelWord(words ...string) error { for _, word := range words { m.store.Remove(word) - m.DelChan <- word + m.delChan <- word } return nil diff --git a/store/mongo.go b/store/mongo.go new file mode 100644 index 0000000..b9bbdc2 --- /dev/null +++ b/store/mongo.go @@ -0,0 +1,274 @@ +package store + +import ( + "bufio" + "context" + "errors" + "fmt" + "github.com/imroc/req/v3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "io" + "net/http" + "os" +) + +const ( + defaultCollection = "dirties" +) + +type MongoConfig struct { + Address string + Port string + Username string + Password string + Database string + Collection string +} + +type doc struct { + Id string `bson:"_id"` + Word string `bson:"word"` +} + +type MongoModel struct { + store *mongo.Collection + addChan chan string + delChan chan string +} + +func NewMongoModel(config *MongoConfig) *MongoModel { + clientOptions := options.Client().ApplyURI( + fmt.Sprintf("mongodb://%s:%s", + config.Address, + config.Port, + ), + ) + + if config.Username != "" { + clientOptions.SetAuth(options.Credential{ + Username: config.Username, + Password: config.Password, + }) + } + + mdb, err := mongo.Connect(context.TODO(), clientOptions) + if err != nil { + return nil + } + + err = mdb.Ping(context.TODO(), nil) + if err != nil { + return nil + } + + if config.Database == "" { + return nil + } + + if config.Collection == "" { + config.Collection = defaultCollection + } + + collection := mdb.Database(config.Database).Collection(config.Collection) + + _, err = collection.Indexes().CreateOne(context.Background(), + mongo.IndexModel{ + Keys: bson.D{{"word", 1}}, + Options: options.Index().SetUnique(true), + }, + ) + if err != nil { + return nil + } + + return &MongoModel{ + store: collection, + addChan: make(chan string), + delChan: make(chan string), + } +} + +func (m *MongoModel) LoadDictPath(paths ...string) error { + for _, path := range paths { + err := func(path string) error { + f, err := os.Open(path) + defer func(f *os.File) { + _ = f.Close() + }(f) + if err != nil { + return err + } + + return m.LoadDict(f) + }(path) + if err != nil { + return err + } + } + + return nil +} + +func (m *MongoModel) LoadDictHttp(urls ...string) error { + for _, url := range urls { + err := func(url string) error { + httpRes, err := req.Get(url) + if err != nil { + return err + } + if httpRes == nil { + return errors.New("nil http response") + } + if httpRes.StatusCode != http.StatusOK { + return errors.New(httpRes.GetStatus()) + } + + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(httpRes.Body) + + return m.LoadDict(httpRes.Body) + }(url) + if err != nil { + return err + } + } + + return nil +} + +func (m *MongoModel) LoadDict(reader io.Reader) error { + buf := bufio.NewReader(reader) + var words []interface{} + + for { + line, _, err := buf.ReadLine() + if err != nil { + if err != io.EOF { + return err + } + break + } + + word := string(line) + + words = append(words, bson.D{ + {"word", word}, + }) + + m.addChan <- word + } + + ctx := context.Background() + + _, _ = m.store.InsertMany(ctx, words, options.InsertMany().SetOrdered(false)) + + return nil +} + +func (m *MongoModel) ReadChan() <-chan string { + ch := make(chan string) + + go func() { + ctx := context.Background() + cur, _ := m.store.Find(ctx, + bson.D{}, + options.Find().SetProjection( + bson.D{ + {"_id", 0}, + {"word", 1}, + }, + ), + ) + + for cur.Next(ctx) { + var word doc + + _ = cur.Decode(&word) + + ch <- word.Word + } + + close(ch) + }() + + return ch +} + +func (m *MongoModel) ReadString() []string { + ctx := context.Background() + cur, err := m.store.Find(ctx, + bson.D{}, + options.Find().SetProjection( + bson.D{ + {"_id", 0}, + {"word", 1}, + }, + ), + ) + if err != nil { + + } + + var words []*doc + + err = cur.All(ctx, &words) + + res := make([]string, 0, len(words)) + + for _, word := range words { + res = append(res, word.Word) + } + + return res +} + +func (m *MongoModel) GetAddChan() <-chan string { + return m.addChan +} + +func (m *MongoModel) GetDelChan() <-chan string { + return m.delChan +} + +func (m *MongoModel) AddWord(words ...string) error { + for _, word := range words { + _, err := m.store.UpdateOne(context.Background(), + bson.D{ + {"word", word}, + }, + bson.D{ + {"$set", bson.D{ + {"word", word}, + }}, + }, + options.Update().SetUpsert(true), + ) + if err != nil { + return err + } + + m.addChan <- word + } + + return nil +} + +func (m *MongoModel) DelWord(words ...string) error { + for _, word := range words { + _, err := m.store.DeleteOne(context.Background(), + bson.D{ + {"word", word}, + }, + ) + if err != nil { + return err + } + + m.delChan <- word + } + + return nil +} diff --git a/store/mysql.go b/store/mysql.go new file mode 100644 index 0000000..2df0aee --- /dev/null +++ b/store/mysql.go @@ -0,0 +1,257 @@ +package store + +import ( + "bufio" + "database/sql" + "errors" + "fmt" + _ "github.com/go-sql-driver/mysql" + "github.com/imroc/req/v3" + "github.com/jmoiron/sqlx" + "io" + "net/http" + "os" +) + +const ( + defaultTable = "dirties" +) + +type MysqlConfig struct { + Dsn string + Database string + TableName string +} + +type Subject struct { + Id int64 `db:"id"` + Word string `db:"word"` +} + +type MysqlModel struct { + store *sqlx.DB + TableName string + addChan chan string + delChan chan string +} + +func NewMysqlModel(config *MysqlConfig) *MysqlModel { + db, err := sqlx.Connect("mysql", config.Dsn) + if err != nil { + return nil + } + + if config.TableName == "" { + config.TableName = defaultTable + } + + var tableName string + + err = db.Get( + &tableName, + fmt.Sprintf( + "SELECT `TABLE_NAME` FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s' LIMIT 1", + config.Database, + config.TableName), + ) + if err != nil { + if err != sql.ErrNoRows { + return nil + } + } + + if tableName == "" { + _, err = db.Exec(fmt.Sprintf("CREATE TABLE `%s` "+ + "(`id` bigint(20) NOT NULL AUTO_INCREMENT, "+ + "`word` varchar(255) NOT NULL, "+ + "PRIMARY KEY (`id`) USING BTREE)", + config.TableName), + ) + if err != nil { + return nil + } + } + + return &MysqlModel{ + store: db, + TableName: config.TableName, + addChan: make(chan string), + delChan: make(chan string), + } +} + +func (m *MysqlModel) LoadDictPath(paths ...string) error { + for _, path := range paths { + err := func(path string) error { + f, err := os.Open(path) + defer func(f *os.File) { + _ = f.Close() + }(f) + if err != nil { + return err + } + + return m.LoadDict(f) + }(path) + if err != nil { + return err + } + } + + return nil +} + +func (m *MysqlModel) LoadDictHttp(urls ...string) error { + for _, url := range urls { + err := func(url string) error { + httpRes, err := req.Get(url) + if err != nil { + return err + } + if httpRes == nil { + return errors.New("nil http response") + } + if httpRes.StatusCode != http.StatusOK { + return errors.New(httpRes.GetStatus()) + } + + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(httpRes.Body) + + return m.LoadDict(httpRes.Body) + }(url) + if err != nil { + return err + } + } + + return nil +} + +func (m *MysqlModel) LoadDict(reader io.Reader) error { + buf := bufio.NewReader(reader) + var words []*Subject + set := make(map[string]struct{}) + + for { + line, _, err := buf.ReadLine() + if err != nil { + if err != io.EOF { + return err + } + break + } + + word := string(line) + + if _, ok := set[word]; !ok { + words = append(words, &Subject{ + Id: 0, + Word: word, + }) + set[word] = struct{}{} + } + + m.addChan <- word + } + + _, err := m.store.NamedExec(fmt.Sprintf("INSERT INTO `%s` (`word`) VALUES (:word)", m.TableName), words) + if err != nil { + return err + } + + _, err = m.store.Exec(fmt.Sprintf("DELETE FROM `%s` AS t1 "+ + "WHERE t1.`id` <> "+ + "(SELECT t.minid FROM "+ + "(SELECT MIN(t2.`id`) AS minid FROM `%s` AS t2 WHERE t1.`word` = t2.`word`) t )", + m.TableName, + m.TableName), + ) + if err != nil { + return err + } + + return nil +} + +func (m *MysqlModel) ReadChan() <-chan string { + ch := make(chan string) + var words []string + + go func() { + defer close(ch) + + err := m.store.Select(&words, fmt.Sprintf("SELECT `word` FROM `%s`", m.TableName)) + if err != nil { + return + } + + for _, word := range words { + ch <- word + } + }() + + return ch +} + +func (m *MysqlModel) ReadString() []string { + var words []string + + err := m.store.Select(&words, fmt.Sprintf("SELECT `word` FROM `%s`", m.TableName)) + if err != nil { + return nil + } + + return words +} + +func (m *MysqlModel) GetAddChan() <-chan string { + return m.addChan +} + +func (m *MysqlModel) GetDelChan() <-chan string { + return m.delChan +} + +func (m *MysqlModel) AddWord(words ...string) error { + insertedWords := make([]*Subject, 0, len(words)) + set := make(map[string]struct{}) + + for _, word := range words { + if _, ok := set[word]; !ok { + insertedWords = append(insertedWords, &Subject{ + Id: 0, + Word: word, + }) + set[word] = struct{}{} + } + } + + _, err := m.store.NamedExec(fmt.Sprintf("INSERT INTO `%s` (`word`) VALUES (:word)", m.TableName), insertedWords) + if err != nil { + return err + } + _, err = m.store.Exec(fmt.Sprintf("DELETE FROM `%s` AS t1 "+ + "WHERE t1.`id` <> "+ + "(SELECT t.minid FROM "+ + "(SELECT MIN(t2.`id`) AS minid FROM `%s` AS t2 WHERE t1.`word` = t2.`word`) t )", + m.TableName, + m.TableName), + ) + if err != nil { + return err + } + + return nil +} + +func (m *MysqlModel) DelWord(words ...string) error { + query, args, _ := sqlx.In(fmt.Sprintf("DELETE FROM `%s` WHERE `word` IN (?)", m.TableName), words) + _, err := m.store.Exec(query, args) + if err != nil { + return err + } + + return nil +}