From 48da6df6261404c16d75e8382fbbba266ac8810e Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Tue, 16 Jan 2024 11:24:56 +0300 Subject: [PATCH 01/14] add: iter1 completed --- .golangci.yml | 462 ++++++++++++++++++ cmd/main.go | 53 ++ docker-compose.yml | 52 ++ go.mod | 29 ++ go.sum | 138 ++++++ golangci-lint.cmd | 3 + internal/adapter/api/middleware/jwtguard.go | 38 ++ internal/adapter/api/middleware/logger.go | 40 ++ internal/adapter/api/rest/rules.go | 123 +++++ internal/adapter/api/rest/user.go | 120 +++++ internal/adapter/api/router/router.go | 33 ++ internal/adapter/api/util/util.go | 22 + .../postgres/migrations/00001_init.down.sql | 2 + .../postgres/migrations/00001_init.up.sql | 15 + .../migrations/00002_init-data.down.sql | 2 + .../migrations/00002_init-data.up.sql | 8 + internal/adapter/store/postgres/postgres.go | 186 +++++++ internal/adapter/store/store.go | 32 ++ internal/config/config.go | 34 ++ internal/core/auth/auth.go | 62 +++ internal/core/server/http/http.go | 38 ++ internal/core/worker/pool.go | 71 +++ internal/core/worker/worker.go | 206 ++++++++ internal/models/model.go | 38 ++ postgres/init.sql | 2 + 25 files changed, 1809 insertions(+) create mode 100644 .golangci.yml create mode 100644 cmd/main.go create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 golangci-lint.cmd create mode 100644 internal/adapter/api/middleware/jwtguard.go create mode 100644 internal/adapter/api/middleware/logger.go create mode 100644 internal/adapter/api/rest/rules.go create mode 100644 internal/adapter/api/rest/user.go create mode 100644 internal/adapter/api/router/router.go create mode 100644 internal/adapter/api/util/util.go create mode 100644 internal/adapter/store/postgres/migrations/00001_init.down.sql create mode 100644 internal/adapter/store/postgres/migrations/00001_init.up.sql create mode 100644 internal/adapter/store/postgres/migrations/00002_init-data.down.sql create mode 100644 internal/adapter/store/postgres/migrations/00002_init-data.up.sql create mode 100644 internal/adapter/store/postgres/postgres.go create mode 100644 internal/adapter/store/store.go create mode 100644 internal/config/config.go create mode 100644 internal/core/auth/auth.go create mode 100644 internal/core/server/http/http.go create mode 100644 internal/core/worker/pool.go create mode 100644 internal/core/worker/worker.go create mode 100644 internal/models/model.go create mode 100644 postgres/init.sql diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..97acc24 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,462 @@ +# Options for analysis running. +run: + # Settable parameters # + timeout: 5m + tests: true + build-tags: [] + # Which dirs to skip: issues from them won't be reported. + skip-dirs: + - "bin" + - "scripts" + - ".ci" + - ".gradle" + # Which files to skip: they will be analyzed, but issues from them won't be reported. + skip-files: [] + + # Invariable parameters # + + # Exit code when at least one issue was found. + issues-exit-code: 1 + # Enables skipping of directories: + # - vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: false + # If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + modules-download-mode: readonly + # Allow multiple parallel golangci-lint instances running. + # If false (default) - golangci-lint acquires file lock on start. + allow-parallel-runners: false + + # Disabled parameters # + + # The default concurrency value is the number of available CPU. + # concurrency: + + # Define the Go version limit. + # Mainly related to generics support since go1.18. + # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.18 + # go: + + +# Output configuration options +output: + format: json + # Print lines of code with issue. + print-issued-lines: true + # Print linter name in the end of issue text. + print-linter-name: true + # Make issues output unique by line. + uniq-by-line: true + # Add a prefix to the output file references. + path-prefix: "" + # Sort results by: filepath, line and column. + sort-results: true + +# Linters settings configuration +linters-settings: + + asasalint: + ignore-test: false + + dupl: + threshold: 100 + + dupword: + keywords: [] + + errcheck: + check-blank: false + check-type-assertions: true + disable-default-exclusions: false + exclude-functions: [] + + errorlint: + asserts: true + comparison: true + errorf: true + errorf-multi: true + + exhaustive: + check: + - switch + - map + check-generated: false + default-signifies-exhaustive: true + explicit-exhaustive-map: false + explicit-exhaustive-switch: false + ignore-enum-members: "" + ignore-enum-types: "" + package-scope-only: false + + goconst: + ignore-calls: false + ignore-tests: true + match-constant: true + min-len: 3 + min-occurrences: 5 + numbers: false + + gocritic: + enabled-checks: + - appendAssign + - appendCombine + - argOrder + - assignOp + - badCall + - badCond + - captLocal + - caseOrder + - commentFormatting + - defaultCaseOrder + - deprecatedComment + - dupArg + - dupBranchBody + - dupCase + - dupSubExpr + - elseif + - exitAfterDefer + - flagDeref + - flagName + - ifElseChain + - indexAlloc + - mapKey + - newDeref + - offBy1 + - rangeExprCopy + - rangeValCopy + - regexpMust + - singleCaseSwitch + - sloppyLen + - stringXbytes + - switchTrue + - typeSwitchVar + - underef + - unlambda + - unslice + - valSwap + - wrapperFunc + + godot: + capital: true + exclude: [] + period: true + scope: toplevel + + gofmt: + simplify: false + + goimports: + local-prefixes: github.com/org/project + + gomnd: + checks: + - argument + - case + - condition + - operation + - return + - assign + ignored-files: [] + ignored-functions: [] + ignored-numbers: [] + + gosec: + includes: + - G104 + - G109 + - G110 + - G201 + - G202 + - G203 + - G301 + - G302 + - G303 + - G305 + - G306 + - G307 + - G402 + - G403 + + gosimple: + checks: + - '*' + - -SA1014 + - -SA1028 + + govet: + disable: + - shadow + enable-all: true + + grouper: + const-require-grouping: false + const-require-single-const: false + import-require-grouping: false + import-require-single-import: true + type-require-grouping: false + type-require-single-type: false + var-require-grouping: false + var-require-single-var: false + + lll: + line-length: 130 + tab-width: 1 + + makezero: + always: false + + nilnil: + checked-types: + - ptr + - func + - iface + - map + - chan + + nolintlint: + allow-no-explanation: [] + allow-unused: false + require-explanation: true + require-specific: false + + prealloc: + for-loops: false + range-loops: true + simple: true + + predeclared: + ignore: "" + q: false + + reassign: + patterns: + - .* + + revive: + confidence: 0.8 + enable-all-rules: false + ignore-generated-header: true + rules: + - name: atomic + - arguments: + - allowTypesBefore: '*testing.T' + name: context-as-argument + - name: context-keys-type + - arguments: + - - recover + - return + name: defer + - name: dot-imports + - name: identical-branches + - name: increment-decrement + - name: range-val-in-closure + - name: range-val-address + - name: unconditional-recursion + - name: unnecessary-stmt + - name: unreachable-code + - name: useless-break + - name: waitgroup-by-value + severity: error + + rowserrcheck: + packages: [] + + staticcheck: + checks: + - '*' + + stylecheck: + checks: + - '*' + - -ST1000 + http-status-code-whitelist: [] + initialisms: + - ACL + - API + - ASCII + - CPU + - CSS + - DNS + - EOF + - GUID + - HTML + - HTTP + - HTTPS + - ID + - IP + - JSON + - QPS + - RAM + - RPC + - SLA + - SMTP + - SQL + - SSH + - TCP + - TLS + - TTL + - UDP + - UI + - GID + - UID + - UUID + - URI + - URL + - UTF8 + - VM + - XML + - XMPP + - XSRF + - XSS + - SIP + - RTP + - AMQP + - DB + - TS + + tenv: + all: false + + thelper: + benchmark: + begin: true + first: true + name: true + fuzz: + begin: false + first: false + name: false + tb: + begin: true + first: true + name: true + test: + begin: true + first: true + name: true + + unparam: + check-exported: false + + usestdlibvars: + constant-kind: true + crypto-hash: true + default-rpc-path: true + http-method: true + http-status-code: true + sql-isolation-level: true + time-layout: true + time-month: true + time-weekday: true + tls-signature-scheme: true + + whitespace: + multi-func: false + multi-if: false + +# Linters configuration +linters: + # Disable all linters. + disable-all: true + # Enable specific linters. + enable: + - asasalint + - asciicheck + - bodyclose + - containedctx + - dupl + - dupword + - durationcheck + - errcheck + - errname + - errorlint + - execinquery + - exhaustive + - gocheckcompilerdirectives + - goconst + - gocritic + - godot + - gofmt + - goimports + - gomnd + - goprintffuncname + - gosec + - gosimple + # - govet + - grouper + - ineffassign + - lll + - makezero + - nilerr + - nilnil + - nolintlint + - nosprintfhostport + - prealloc + - predeclared + - reassign + - revive + - rowserrcheck + - sqlclosecheck + - staticcheck + - stylecheck + - tenv + - testableexamples + - thelper + - tparallel + - typecheck + - unconvert + - unparam + - unused + - usestdlibvars + - wastedassign + - whitespace + - wrapcheck + # Run only fast linters from enabled linters set (first run won't be fast). + fast: false + +# Issues configuration +issues: + exclude: [] + exclude-case-sensitive: false + exclude-rules: + - path: "_test\\.go" + text: "fieldalignment: struct with \\d+ pointer bytes could be \\d+" + fix: false + max-issues-per-linter: 0 + max-same-issues: 0 + + # Invariable parameters # + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + new: false + # Independently of option `exclude` we use default exclude patterns, + # it can be disabled by this option. + # To list all excluded by default patterns execute `golangci-lint run --help`. + exclude-use-default: false + + # Disabled parameters # + + # Show only new issues created after git revision `REV`. + # new-from-rev: + + # Show only new issues created in git patch with set file path. + # new-from-patch: + +severity: + # Invariable parameters # + + # Set the default severity for issues. + # If severity rules are defined and the issues do not match or no severity is provided to the rule + # this will be the default severity applied. This option does not affect the exit code of the linter. + default-severity: error + # If set to true `severity-rules` regular expressions become case-sensitive. + case-sensitive: false + # When a list of severity rules are provided, severity information will be added to lint issues. + # Severity rules have the same filtering capability as exclude rules + # except you are allowed to specify one matcher per severity rule. + # Only affects out formats that support setting severity information. + rules: [] diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..12c7d8e --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "log" + + "github.com/dedpnd/unifier/internal/adapter/api/router" + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/config" + h "github.com/dedpnd/unifier/internal/core/server/http" + "github.com/dedpnd/unifier/internal/core/worker" +) + +func main() { + log.Println("Server start...") + + // Читаем конфигурацию + cfg, err := config.GetConfig() + if err != nil { + log.Fatal(err.Error()) + } + + // Создаем хранилище + str, err := store.NewStore(cfg.DatabaseDSN) + if err != nil { + log.Fatal(err.Error()) + } + + // Запускаем пул воркеров + p, err := worker.StartPool(cfg.KafkaAdress, str) + if err != nil { + log.Fatal(err.Error()) + } + + // Создаем роутер + r, err := router.Router(str, p) + if err != nil { + log.Fatal(err.Error()) + } + + // Функция для завершения работы + callback := func() { + err := str.Close() + if err != nil { + log.Fatal(err.Error()) + } + + p.StopPool() + } + + // Поднимаем сервер + addr := "localhost:8080" + h.GracefulServer(addr, r, callback) +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b8341f3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +version: "3.9" +services: + zookeeper: + image: wurstmeister/zookeeper:latest + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka:2.11-1.1.1 + ports: + - "9092:9092" + expose: + - "9093" + hostname: kafka + links: + - zookeeper + environment: + KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_DELETE_TOPIC_ENABLE: "true" + kafka-workload: + build: ./kafka-perf-test/. + links: + - kafka + environment: + KAFKA_TOPIC: "events" + NUM_RECORDS: "100" + PRODUCER_THROUGHPUT: "100" + TEST_INTERVAL_SECONDS: "30" + BOOTSTRAP_SERVERS: "kafka:9093" + command: ["./producer_test.sh"] + redpanda: + image: docker.redpanda.com/redpandadata/console:latest + links: + - kafka + ports: + - "8081:8080" + environment: + KAFKA_BROKERS: kafka:9093 + postgres: + image: postgres:16.1-alpine3.18 + environment: + POSTGRES_PASSWORD: "secret" + PGDATA: "/var/lib/postgresql/data/pgdata" + volumes: + - ./pgdata:/var/lib/postgresql/data + - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql + ports: + - "5432:5432" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a66f995 --- /dev/null +++ b/go.mod @@ -0,0 +1,29 @@ +module github.com/dedpnd/unifier + +go 1.21 + +require ( + github.com/caarlos0/env v3.5.0+incompatible + github.com/go-chi/chi/v5 v5.0.11 + github.com/golang-jwt/jwt/v5 v5.2.0 + github.com/golang-migrate/migrate/v4 v4.17.0 + github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa + github.com/jackc/pgx/v5 v5.5.2 + github.com/segmentio/kafka-go v0.4.47 + golang.org/x/crypto v0.17.0 +) + +require ( + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/klauspost/compress v1.15.11 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/pierrec/lz4/v4 v4.1.16 // indirect + github.com/stretchr/testify v1.8.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..33477bb --- /dev/null +++ b/go.sum @@ -0,0 +1,138 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= +github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.0 h1:z05UmuXZHO/bgj/ds2bGMBu8FI4WA+Ag/m3ghL+om7M= +github.com/dhui/dktest v0.4.0/go.mod h1:v/Dbz1LgCBOi2Uki2nUqLBGa83hWBGFMu5MrgMDCc78= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= +github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-migrate/migrate/v4 v4.17.0 h1:rd40H3QXU0AA4IoLllFcEAEo9dYKRHYND2gB4p7xcaU= +github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdrNx2HGs45rbWAo6OsKM= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw= +github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.2 h1:iLlpgp4Cp/gC9Xuscl7lFL1PhhW+ZLtXZcrfCt4C3tA= +github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= +github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= +github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/golangci-lint.cmd b/golangci-lint.cmd new file mode 100644 index 0000000..7e0809b --- /dev/null +++ b/golangci-lint.cmd @@ -0,0 +1,3 @@ +docker run --rm -v %cd%:/app -w /app golangci/golangci-lint:v1.53.3 golangci-lint run -c .golangci.yml > ./golangci-lint/report-unformatted.json +docker run --rm -v %cd%:/app imega/jq -c . /app/golangci-lint/report-unformatted.json > ./golangci-lint/report.json +pause \ No newline at end of file diff --git a/internal/adapter/api/middleware/jwtguard.go b/internal/adapter/api/middleware/jwtguard.go new file mode 100644 index 0000000..dd660b5 --- /dev/null +++ b/internal/adapter/api/middleware/jwtguard.go @@ -0,0 +1,38 @@ +package middleware + +import ( + "context" + "errors" + "net/http" + + "github.com/dedpnd/unifier/internal/adapter/api/util" + "github.com/dedpnd/unifier/internal/core/auth" +) + +func JWTguard() func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + c, err := req.Cookie("token") + if err != nil { + if errors.Is(err, http.ErrNoCookie) { + res.WriteHeader(http.StatusUnauthorized) + return + } + res.WriteHeader(http.StatusBadRequest) + return + } + + token := c.Value + pl, err := auth.VerifyJWTandGetPayload(token) + if err != nil { + res.WriteHeader(http.StatusUnauthorized) + return + } + + ctx := req.Context() + r := req.WithContext(context.WithValue(ctx, util.ContextKeyToken, pl)) + + h.ServeHTTP(res, r) + }) + } +} diff --git a/internal/adapter/api/middleware/logger.go b/internal/adapter/api/middleware/logger.go new file mode 100644 index 0000000..3cb20ab --- /dev/null +++ b/internal/adapter/api/middleware/logger.go @@ -0,0 +1,40 @@ +package middleware + +import ( + "log" + "net/http" + "time" +) + +type responseObserver struct { + http.ResponseWriter + status int + written int64 +} + +func (o *responseObserver) Write(p []byte) (n int, err error) { + n, err = o.ResponseWriter.Write(p) + o.written += int64(n) + return +} + +func (o *responseObserver) WriteHeader(code int) { + o.ResponseWriter.WriteHeader(code) + o.status = code +} + +func Logger() func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + start := time.Now() + + o := &responseObserver{ResponseWriter: res} + h.ServeHTTP(o, req) + + duration := time.Since(start) + + log.Printf(`Fetch URL: [ method:%v, uri:%v, status: %v, size: %v, duration:%v ]`, + req.Method, req.RequestURI, o.status, o.written, duration) + }) + } +} diff --git a/internal/adapter/api/rest/rules.go b/internal/adapter/api/rest/rules.go new file mode 100644 index 0000000..433df18 --- /dev/null +++ b/internal/adapter/api/rest/rules.go @@ -0,0 +1,123 @@ +package rest + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + + "github.com/dedpnd/unifier/internal/adapter/api/util" + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/core/worker" + "github.com/dedpnd/unifier/internal/models" + "github.com/go-chi/chi/v5" +) + +type RulesHandler struct { + Store store.Storage + Pool worker.Pool +} + +const IntServerError = "Internal server error" + +func (h RulesHandler) GetAllRules(res http.ResponseWriter, req *http.Request) { + data, err := h.Store.GetAllRules(req.Context()) + if err != nil { + log.Println(fmt.Errorf("failed get all records from database: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + resBodyBytes := new(bytes.Buffer) + if err := json.NewEncoder(resBodyBytes).Encode(&data); err != nil { + log.Println(fmt.Errorf("invalid stringify JSON: %w", err)) + http.Error(res, IntServerError, http.StatusBadRequest) + return + } + + res.Header().Set("Content-Type", "application/json") + + _, err = res.Write(resBodyBytes.Bytes()) + if err != nil { + log.Println(fmt.Errorf("failed write record to response: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } +} + +func (h RulesHandler) CreateRule(res http.ResponseWriter, req *http.Request) { + token, ok := util.GetTokenFromContext(req.Context()) + if !ok { + log.Println(fmt.Errorf("invalid jwt token")) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + pBody := models.Config{} + + if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { + log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) + http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + return + } + + id, err := h.Store.CreateRule(req.Context(), pBody, token.ID) + if err != nil { + log.Println(fmt.Errorf("failed save records: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + pID := strconv.Itoa(id) + h.Pool.AddWorker(pID, pBody) + + res.WriteHeader(http.StatusOK) +} + +func (h RulesHandler) DeleteRule(res http.ResponseWriter, req *http.Request) { + token, ok := util.GetTokenFromContext(req.Context()) + if !ok { + log.Println(fmt.Errorf("invalid jwt token")) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + id := chi.URLParam(req, "id") + + pID, err := strconv.Atoi(id) + if err != nil { + log.Println(fmt.Errorf("failed convert id: %w", err)) + http.Error(res, `Failde convert id to int`, http.StatusBadRequest) + return + } + + dr, err := h.Store.GetRuleByID(req.Context(), pID) + if err != nil { + log.Println(fmt.Errorf("failed get rule row: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + if dr.ID == 0 { + http.Error(res, "Not Found", http.StatusNotFound) + return + } + + if dr.Owner != token.ID { + http.Error(res, "Forbidden", http.StatusForbidden) + return + } + + err = h.Store.DeleteRule(req.Context(), pID) + if err != nil { + log.Println(fmt.Errorf("failed delele rules row: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + h.Pool.DeleteWorker(id) + + res.WriteHeader(http.StatusOK) +} diff --git a/internal/adapter/api/rest/user.go b/internal/adapter/api/rest/user.go new file mode 100644 index 0000000..6254e35 --- /dev/null +++ b/internal/adapter/api/rest/user.go @@ -0,0 +1,120 @@ +package rest + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/core/auth" + "github.com/dedpnd/unifier/internal/models" + "golang.org/x/crypto/bcrypt" +) + +type UserHandler struct { + Store store.Storage +} + +type LoginBody struct { + Login *string `json:"login"` + Password *string `json:"password"` +} + +func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { + pBody := LoginBody{} + + if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { + log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) + http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + return + } + + if pBody.Login == nil || pBody.Password == nil { + http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + return + } + + data, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) + if err != nil { + log.Println(fmt.Errorf("failed get user from database: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + if data.ID > 0 { + http.Error(res, `Login exist`, http.StatusConflict) + return + } + + hash, err := bcrypt.GenerateFromPassword([]byte(*pBody.Password), bcrypt.DefaultCost) + if err != nil { + log.Println(fmt.Errorf("failed get hash from password: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + id, err := h.Store.CreateUser(req.Context(), models.User{Login: *pBody.Login, Hash: string(hash)}) + if err != nil { + log.Println(fmt.Errorf("failed create user: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + token, err := auth.GetJWT(id, *pBody.Login) + if err != nil { + log.Println(fmt.Errorf("failed create jwt token: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + http.SetCookie(res, &http.Cookie{ + Name: "token", + Value: *token, + Path: "/api/", + }) + + res.WriteHeader(http.StatusOK) +} + +func (h UserHandler) Login(res http.ResponseWriter, req *http.Request) { + pBody := LoginBody{} + + if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { + log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) + http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + return + } + + if pBody.Login == nil || pBody.Password == nil { + http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + return + } + + data, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) + if err != nil { + log.Println(fmt.Errorf("failed get user from database: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + if err := bcrypt.CompareHashAndPassword([]byte(data.Hash), []byte(*pBody.Password)); err != nil { + http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + return + } + + token, err := auth.GetJWT(data.ID, *pBody.Login) + if err != nil { + log.Println(fmt.Errorf("failed create jwt token: %w", err)) + http.Error(res, IntServerError, http.StatusInternalServerError) + return + } + + http.SetCookie(res, &http.Cookie{ + Name: "token", + Value: *token, + Path: "/api/", + }) + + res.WriteHeader(http.StatusOK) +} diff --git a/internal/adapter/api/router/router.go b/internal/adapter/api/router/router.go new file mode 100644 index 0000000..49d0ceb --- /dev/null +++ b/internal/adapter/api/router/router.go @@ -0,0 +1,33 @@ +package router + +import ( + "github.com/dedpnd/unifier/internal/adapter/api/middleware" + "github.com/dedpnd/unifier/internal/adapter/api/rest" + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/core/worker" + "github.com/go-chi/chi/v5" +) + +func Router(str store.Storage, pool worker.Pool) (chi.Router, error) { + r := chi.NewRouter() + + r.Use(middleware.Logger()) + + rulesHandler := rest.RulesHandler{ + Store: str, + Pool: pool, + } + + r.With(middleware.JWTguard()).Get("/api/rules", rulesHandler.GetAllRules) + r.With(middleware.JWTguard()).Post("/api/rules", rulesHandler.CreateRule) + r.With(middleware.JWTguard()).Delete("/api/rules/{id}", rulesHandler.DeleteRule) + + userHandler := rest.UserHandler{ + Store: str, + } + + r.Post("/api/user/register", userHandler.Register) + r.Post("/api/user/login", userHandler.Login) + + return r, nil +} diff --git a/internal/adapter/api/util/util.go b/internal/adapter/api/util/util.go new file mode 100644 index 0000000..f7557a1 --- /dev/null +++ b/internal/adapter/api/util/util.go @@ -0,0 +1,22 @@ +package util + +import ( + "context" + + "github.com/dedpnd/unifier/internal/core/auth" +) + +type contextKey string + +func (c contextKey) String() string { + return string(c) +} + +var ( + ContextKeyToken = contextKey("deleteCaller") +) + +func GetTokenFromContext(ctx context.Context) (auth.Claims, bool) { + caller, ok := ctx.Value(ContextKeyToken).(auth.Claims) + return caller, ok +} diff --git a/internal/adapter/store/postgres/migrations/00001_init.down.sql b/internal/adapter/store/postgres/migrations/00001_init.down.sql new file mode 100644 index 0000000..e93f51b --- /dev/null +++ b/internal/adapter/store/postgres/migrations/00001_init.down.sql @@ -0,0 +1,2 @@ +DROP TABLE users; +DROP TABLE rules; \ No newline at end of file diff --git a/internal/adapter/store/postgres/migrations/00001_init.up.sql b/internal/adapter/store/postgres/migrations/00001_init.up.sql new file mode 100644 index 0000000..9438e8d --- /dev/null +++ b/internal/adapter/store/postgres/migrations/00001_init.up.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS users( + ID SERIAL UNIQUE PRIMARY KEY NOT NULL, + Login VARCHAR(255) UNIQUE NOT NULL, + Hash VARCHAR(1000) NOT NULL +); + +CREATE TABLE IF NOT EXISTS rules( + ID SERIAL PRIMARY KEY, + Rule JSON, + Owner INT, + CONSTRAINT fk_users + FOREIGN KEY(Owner) + REFERENCES users(ID) + ON DELETE SET NULL +); diff --git a/internal/adapter/store/postgres/migrations/00002_init-data.down.sql b/internal/adapter/store/postgres/migrations/00002_init-data.down.sql new file mode 100644 index 0000000..e17ccd3 --- /dev/null +++ b/internal/adapter/store/postgres/migrations/00002_init-data.down.sql @@ -0,0 +1,2 @@ +DELETE FROM users WHERE id=1; +DELETE FROM rules WHERE id=1; \ No newline at end of file diff --git a/internal/adapter/store/postgres/migrations/00002_init-data.up.sql b/internal/adapter/store/postgres/migrations/00002_init-data.up.sql new file mode 100644 index 0000000..434a455 --- /dev/null +++ b/internal/adapter/store/postgres/migrations/00002_init-data.up.sql @@ -0,0 +1,8 @@ +-- password is password +INSERT INTO users +(login, hash) +VALUES('admin', '$2a$10$fvuwEbdImMWCPGzjuJ7pbOAQnZ/e9VyrVK60ComfJiEJvgkORJTci'); + +INSERT INTO rules +("rule", "owner") +VALUES('{"topicFrom":"events","filter":{"regexp":"\"dstHost.ip\": \"10.10.10.10\""},"entityHash":["srcHost.ip","dstHost.port"],"unifier":[{"name":"id","type":"string","expression":"auditEventLog"},{"name":"date","type":"timestamp","expression":"datetime"},{"name":"ipaddr","type":"string","expression":"srcHost.ip"},{"name":"category","type":"string","expression":"cat"}],"extraProcess":[{"func":"__if","args":"category, /Host/Connect/Host/Accept, high","to":"category"},{"func":"__stringConstant","args":"test","to":"customString1"}],"topicTo":"test"}'::json, 1); diff --git a/internal/adapter/store/postgres/postgres.go b/internal/adapter/store/postgres/postgres.go new file mode 100644 index 0000000..b2c401a --- /dev/null +++ b/internal/adapter/store/postgres/postgres.go @@ -0,0 +1,186 @@ +package postgres + +import ( + "context" + "embed" + "errors" + "fmt" + "log" + + "github.com/dedpnd/unifier/internal/models" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + _ "github.com/golang-migrate/migrate/v4/database/postgres" +) + +type DataBase struct { + pool *pgxpool.Pool +} + +func NewDB(ctx context.Context, dsn string) (DataBase, error) { + pool, err := connection(ctx, dsn) + if err != nil { + return DataBase{}, fmt.Errorf("failed connection to postgre: %w", err) + } + + log.Println("Connection to postgre: success") + + if err := runMigrations(dsn); err != nil { + return DataBase{}, fmt.Errorf("failed to run DB migrations: %w", err) + } + + return DataBase{ + pool: pool, + }, nil +} + +func (db DataBase) Close() error { + if db.pool != nil { + db.pool.Close() + } + + return nil +} + +//nolint:dupl // This legal code +func (db DataBase) GetUserByLogin(ctx context.Context, login string) (models.User, error) { + row := db.pool.QueryRow(ctx, + `SELECT id, login, hash FROM users WHERE login=$1`, + login, + ) + + u := models.User{} + err := row.Scan(&u.ID, &u.Login, &u.Hash) + if err != nil { + var pgErr *pgconn.PgError + // Если данные не найдены возвращаем пустую структуру + if !(errors.As(err, &pgErr) && pgErr.Code == pgerrcode.NoDataFound) { + return u, nil + } + + return u, fmt.Errorf("failed scan row: %w", err) + } + + return u, nil +} + +func (db DataBase) CreateUser(ctx context.Context, user models.User) (int, error) { + row := db.pool.QueryRow(ctx, + `INSERT INTO users (Login, Hash) VALUES($1, $2) RETURNING id`, + user.Login, + user.Hash, + ) + + var id int + err := row.Scan(&id) + if err != nil { + return 0, fmt.Errorf("failed scan user record row: %w", err) + } + + return id, nil +} + +func (db DataBase) GetAllRules(ctx context.Context) ([]models.Rule, error) { + rows, err := db.pool.Query(ctx, `SELECT ID, Rule, Owner FROM Rules`) + if err != nil { + return nil, fmt.Errorf("failed rules query records: %w", err) + } + defer rows.Close() + + var rules []models.Rule + for rows.Next() { + var rule models.Rule + if err = rows.Scan(&rule.ID, &rule.Rule, &rule.Owner); err != nil { + return nil, fmt.Errorf("failed scan rules records: %w", err) + } + rules = append(rules, rule) + } + + return rules, nil +} + +//nolint:dupl // This legal code +func (db DataBase) GetRuleByID(ctx context.Context, id int) (models.Rule, error) { + row := db.pool.QueryRow(ctx, + `SELECT ID, Rule, Owner FROM Rules WHERE id=$1`, + id, + ) + + r := models.Rule{} + err := row.Scan(&r.ID, &r.Rule, &r.Owner) + if err != nil { + var pgErr *pgconn.PgError + // Если данные не найдены возвращаем пустую структуру + if !(errors.As(err, &pgErr) && pgErr.Code == pgerrcode.NoDataFound) { + return r, nil + } + + return r, fmt.Errorf("failed scan row: %w", err) + } + + return r, nil +} + +func (db DataBase) CreateRule(ctx context.Context, rule models.Config, owner int) (int, error) { + row := db.pool.QueryRow(context.Background(), + `INSERT INTO rules (Rule, Owner) VALUES($1, $2) RETURNING id`, + rule, + owner, + ) + + var id int + err := row.Scan(&id) + if err != nil { + return 0, fmt.Errorf("failed scan rule record row: %w", err) + } + + return id, nil +} + +func (db DataBase) DeleteRule(ctx context.Context, id int) error { + _, err := db.pool.Exec(context.Background(), + `DELETE FROM rules WHERE id = $1`, + id, + ) + if err != nil { + return fmt.Errorf("failed delete record in rules: %w", err) + } + + return nil +} + +// -----------------------. + +//go:embed migrations/*.sql +var migrationsDir embed.FS + +func runMigrations(dsn string) error { + d, err := iofs.New(migrationsDir, "migrations") + if err != nil { + return fmt.Errorf("failed to return an iofs driver: %w", err) + } + + m, err := migrate.NewWithSourceInstance("iofs", d, dsn) + if err != nil { + return fmt.Errorf("failed to get a new migrate instance: %w", err) + } + if err := m.Up(); err != nil { + if !errors.Is(err, migrate.ErrNoChange) { + return fmt.Errorf("failed to apply migrations to the DB: %w", err) + } + } + return nil +} + +func connection(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + return nil, fmt.Errorf("failed connection to postgre: %w", err) + } + + return pool, nil +} diff --git a/internal/adapter/store/store.go b/internal/adapter/store/store.go new file mode 100644 index 0000000..f17b204 --- /dev/null +++ b/internal/adapter/store/store.go @@ -0,0 +1,32 @@ +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/dedpnd/unifier/internal/adapter/store/postgres" + "github.com/dedpnd/unifier/internal/models" +) + +type Storage interface { + Close() error + GetUserByLogin(ctx context.Context, login string) (models.User, error) + CreateUser(ctx context.Context, user models.User) (int, error) + GetRuleByID(ctx context.Context, id int) (models.Rule, error) + GetAllRules(ctx context.Context) ([]models.Rule, error) + CreateRule(ctx context.Context, rule models.Config, owner int) (int, error) + DeleteRule(ctx context.Context, id int) error +} + +func NewStore(dsn string) (Storage, error) { + if len(dsn) != 0 { + dbs, err := postgres.NewDB(context.Background(), dsn) + if err != nil { + return nil, fmt.Errorf("failed create database storage: %w", err) + } + return dbs, nil + } + + return nil, errors.New("storage not found") +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..494faf2 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,34 @@ +package config + +import ( + "flag" + "fmt" + + "github.com/caarlos0/env" +) + +type configENV struct { + DatabaseDSN string `env:"DATABASE_DSN"` + KafkaAdress string `env:"KAFKA_ADDRESS"` +} + +func GetConfig() (*configENV, error) { + var eCfg configENV + + flag.StringVar(&eCfg.KafkaAdress, "a", + "localhost:9092", + "address where work kafka") + flag.StringVar(&eCfg.DatabaseDSN, "d", + "postgres://user:password@localhost:5432/local?sslmode=disable", + "address database connection") + flag.Parse() + + err := env.Parse(&eCfg) + if err != nil { + return nil, fmt.Errorf("failed parsing environment variables: %w", err) + } + + flag.Parse() + + return &eCfg, nil +} diff --git a/internal/core/auth/auth.go b/internal/core/auth/auth.go new file mode 100644 index 0000000..03877ea --- /dev/null +++ b/internal/core/auth/auth.go @@ -0,0 +1,62 @@ +package auth + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +type Claims struct { + ID int `json:"id"` + Login string `json:"login"` + jwt.RegisteredClaims +} + +var jwtKey = []byte("12345") + +func GetJWT(id int, login string) (*string, error) { + var DefaultSession = 15 + var DefaultExpTime = time.Now().Add(time.Duration(DefaultSession) * time.Minute) + + claims := &Claims{ + ID: id, + Login: login, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(DefaultExpTime), + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tokenString, err := token.SignedString(jwtKey) + if err != nil { + return nil, fmt.Errorf("failed signed jwt: %w", err) + } + + return &tokenString, nil +} + +func VerifyJWTandGetPayload(token string) (Claims, error) { + claims := &Claims{} + + tkn, err := jwt.ParseWithClaims(token, claims, func(token *jwt.Token) (interface{}, error) { + return jwtKey, nil + }) + + if err != nil { + if errors.Is(err, jwt.ErrSignatureInvalid) { + log.Println(fmt.Errorf("invalid jwt token: %w", err)) + return *claims, fmt.Errorf("failed signature from jwt: %w", err) + } + log.Println(fmt.Errorf("invalid jwt token: %w", err)) + return *claims, fmt.Errorf("invalid jwt token: %w", err) + } + + if !tkn.Valid { + return *claims, fmt.Errorf("jwt token not valid: %w", err) + } + + return *claims, nil +} diff --git a/internal/core/server/http/http.go b/internal/core/server/http/http.go new file mode 100644 index 0000000..e30b8a3 --- /dev/null +++ b/internal/core/server/http/http.go @@ -0,0 +1,38 @@ +package http + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + + "github.com/go-chi/chi/v5" +) + +func GracefulServer(addr string, r chi.Router, cb func()) { + var wg sync.WaitGroup + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) + + wg.Add(1) + + go func() { + defer func() { + cb() + stop() + wg.Done() + }() + + <-ctx.Done() + }() + + go func() { + if err := http.ListenAndServe(addr, r); err != nil { + log.Fatal(err.Error()) + } + }() + + wg.Wait() +} diff --git a/internal/core/worker/pool.go b/internal/core/worker/pool.go new file mode 100644 index 0000000..c2e7f6e --- /dev/null +++ b/internal/core/worker/pool.go @@ -0,0 +1,71 @@ +package worker + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/models" +) + +type Pool struct { + kafkaURL string + p map[string]workerEntity +} + +type workerEntity struct { + ID string + Config models.Config + Stop chan bool +} + +func StartPool(kAddr string, str store.Storage) (Pool, error) { + p := Pool{ + kafkaURL: kAddr, + p: make(map[string]workerEntity), + } + + rules, err := str.GetAllRules(context.Background()) + if err != nil { + return Pool{}, fmt.Errorf("failed get all rule from storage: %w", err) + } + + for i := range rules { + id := strconv.Itoa(rules[i].ID) + p.AddWorker(id, rules[i].Rule) + } + + return p, nil +} + +func (p Pool) AddWorker(id string, rule models.Config) { + p.p[id] = workerEntity{ + ID: id, + Config: rule, + Stop: make(chan bool), + } + + go func() { + if err := Start(context.Background(), p.kafkaURL, p.p[id]); err != nil { + log.Println(err.Error()) + } + }() +} + +func (p Pool) DeleteWorker(id string) { + wrk := p.p[id] + + // Останнавливаем воркер + wrk.Stop <- true + + // Удаляем конфигурацию + delete(p.p, id) +} + +func (p Pool) StopPool() { + for i := range p.p { + p.p[i].Stop <- true + } +} diff --git a/internal/core/worker/worker.go b/internal/core/worker/worker.go new file mode 100644 index 0000000..ca41800 --- /dev/null +++ b/internal/core/worker/worker.go @@ -0,0 +1,206 @@ +package worker + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "regexp" + "strings" + "time" + + "github.com/dedpnd/unifier/internal/models" + "github.com/segmentio/kafka-go" +) + +func Start(ctx context.Context, kafkaURL string, wrkConfig workerEntity) error { + var r *kafka.Reader + var p *kafka.Conn + + log.Println("Worker start:", wrkConfig.ID) + + // Создаем kafka consumer + r = kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{kafkaURL}, + GroupID: wrkConfig.ID, + Topic: wrkConfig.Config.TopicFrom, + }) + + // Создаем kafka producer + var err error + p, err = kafka.DialLeader(context.Background(), "tcp", kafkaURL, wrkConfig.Config.TopicTo, 0) + if err != nil { + return fmt.Errorf("worker:%v - failed create producer: %w", wrkConfig.ID, err) + } + // Вычитываем сообщения + for { + select { + default: + msg, err := r.ReadMessage(ctx) + if err != nil { + return fmt.Errorf("worker:%v - failed read message: %w", wrkConfig.ID, err) + } + + // Фильтруем событие по регулярному выражению + strMsg := string(msg.Value) + matched, err := regexp.Match(wrkConfig.Config.Filter.Regexp, []byte(strMsg)) + if err != nil { + return fmt.Errorf("worker:%v - failed filter message: %w", wrkConfig.ID, err) + } + + // Преобразум сообщения для удобства разбора + if matched { + var uniferEvents = make(map[string]interface{}) + var pEvent map[string]interface{} + + if err := json.Unmarshal([]byte(strMsg), &pEvent); err != nil { + return fmt.Errorf("worker:%v - invalid JSON parse: %w", wrkConfig.ID, err) + } + + // Вычисляем уникальных идентификатор для записи + hex := calculateHash(pEvent, wrkConfig.Config.EntityHash) + uniferEvents["entity"] = hex + + // Унификация полей + err = unificationFields(pEvent, wrkConfig.Config.Unifier, &uniferEvents) + if err != nil { + log.Println(err.Error()) + } + + // Допольнительная обработка + err = extraProcess(wrkConfig.Config.ExtraProcess, &uniferEvents) + if err != nil { + log.Println(err.Error()) + } + + buf, err := json.Marshal(uniferEvents) + if err != nil { + return fmt.Errorf("worker:%v - failed stringify message: %w", wrkConfig.ID, err) + } + + _, err = p.WriteMessages(kafka.Message{Value: buf}) + if err != nil { + return fmt.Errorf("worker:%v - failed to write messages: %w", wrkConfig.ID, err) + } + } + case <-wrkConfig.Stop: + log.Println("Worker stop:", wrkConfig.ID) + + err := r.Close() + if err != nil { + return fmt.Errorf("worker:%v - failed close consumer: %w", wrkConfig.ID, err) + } + + err = p.Close() + if err != nil { + return fmt.Errorf("worker:%v - failed close producer: %w", wrkConfig.ID, err) + } + + return nil + } + } +} + +func extraProcess(cfgExtraProcess []models.ExtraProcess, uEvent *map[string]interface{}) error { + for _, ep := range cfgExtraProcess { + switch ep.Func { + case "__if": + r := __if(*uEvent, ep.Args) + (*uEvent)[ep.To] = r + case "__stringConstant": + r := __stringConstant(ep.Args) + (*uEvent)[ep.To] = r + default: + return fmt.Errorf("unknown func: %v", ep.Func) + } + } + + return nil +} + +func unificationFields(event map[string]interface{}, cfgUnifier []models.Unifier, uEvent *map[string]interface{}) error { + for _, u := range cfgUnifier { + v, found := event[u.Expression] + if found { + switch u.Type { + // TODO: Логировать когда преобразование не получилось + case "string": + v, ok := v.(string) + if ok { + (*uEvent)[u.Name] = v + } + case "int": + v, ok := v.(int) + if ok { + (*uEvent)[u.Name] = v + } + case "timestamp": + v, ok := v.(string) + if ok { + v, err := time.Parse(time.RFC3339, v) + if err != nil { + return fmt.Errorf("failed date parse: %w", err) + } + (*uEvent)[u.Name] = v + } + default: + return fmt.Errorf("unknown type: %v", u.Type) + } + } + } + + return nil +} + +func calculateHash(event map[string]interface{}, cfgEntHash []string) string { + strHash := "" + for _, eh := range cfgEntHash { + v, found := event[eh] + if found { + s, ok := v.(string) + if ok { + strHash += strings.TrimSpace(s) + } + } + } + + // Вычисляем хэш + hash := md5.Sum([]byte(strHash)) + hexStr := hex.EncodeToString(hash[:]) + + return hexStr +} + +// Function fot extra process! +// +//nolint:stylecheck // This legal name +func __if(uniEvent map[string]interface{}, args string) string { + aSlice := strings.Split(args, ",") + for i, v := range aSlice { + aSlice[i] = strings.TrimSpace(v) + } + + var field, stmn, result string = aSlice[0], aSlice[1], aSlice[2] + + v, ok := uniEvent[field] + if ok { + if v == stmn { + return result + } + } + + return "" +} + +//nolint:stylecheck // This legal name +func __stringConstant(args string) string { + aSlice := strings.Split(args, ",") + for i, v := range aSlice { + aSlice[i] = strings.TrimSpace(v) + } + + var c string = aSlice[0] + return c +} diff --git a/internal/models/model.go b/internal/models/model.go new file mode 100644 index 0000000..9d86058 --- /dev/null +++ b/internal/models/model.go @@ -0,0 +1,38 @@ +package models + +type User struct { + ID int `json:"id"` + Login string `json:"login"` + Hash string +} + +type Rule struct { + ID int `json:"id"` + Rule Config `json:"rule"` + Owner int `json:"owner"` +} + +type Config struct { + TopicFrom string `json:"topicFrom"` + Filter Filter `json:"filter"` + EntityHash []string `json:"entityHash"` + Unifier []Unifier `json:"unifier"` + ExtraProcess []ExtraProcess `json:"extraProcess"` + TopicTo string `json:"topicTo"` +} + +type Filter struct { + Regexp string `json:"regexp"` +} + +type Unifier struct { + Name string `json:"name"` + Type string `json:"type"` + Expression string `json:"expression"` +} + +type ExtraProcess struct { + Func string `json:"func"` + Args string `json:"args"` + To string `json:"to"` +} diff --git a/postgres/init.sql b/postgres/init.sql new file mode 100644 index 0000000..e59af08 --- /dev/null +++ b/postgres/init.sql @@ -0,0 +1,2 @@ +CREATE USER "user" WITH PASSWORD 'password'; +CREATE DATABASE local OWNER "user" \ No newline at end of file From 343baabcf890513bb3f62c0f42253234a460293d Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Wed, 24 Jan 2024 19:56:43 +0300 Subject: [PATCH 02/14] fix: correction iter1 #1 --- cmd/main.go | 26 +++++++----- go.mod | 2 + go.sum | 6 +++ internal/adapter/api/middleware/jwtguard.go | 40 +++++++++---------- internal/adapter/api/middleware/logger.go | 14 +++++-- internal/adapter/api/rest/rules.go | 35 ++++++++-------- internal/adapter/api/rest/user.go | 39 ++++++++++-------- internal/adapter/api/router/router.go | 19 +++++---- internal/adapter/api/util/util.go | 10 ++--- .../postgres/migrations/00001_init.down.sql | 6 ++- .../postgres/migrations/00001_init.up.sql | 21 ++++++++-- .../migrations/00002_init-data.down.sql | 2 - .../migrations/00002_init-data.up.sql | 8 ---- internal/adapter/store/postgres/postgres.go | 15 +++++-- internal/adapter/store/store.go | 5 ++- internal/core/auth/auth.go | 3 -- internal/core/server/http/http.go | 6 +-- internal/core/worker/pool.go | 10 +++-- internal/core/worker/worker.go | 12 +++--- internal/logger/logger.go | 26 ++++++++++++ 20 files changed, 182 insertions(+), 123 deletions(-) delete mode 100644 internal/adapter/store/postgres/migrations/00002_init-data.down.sql delete mode 100644 internal/adapter/store/postgres/migrations/00002_init-data.up.sql create mode 100644 internal/logger/logger.go diff --git a/cmd/main.go b/cmd/main.go index 12c7d8e..20510d8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,40 +8,46 @@ import ( "github.com/dedpnd/unifier/internal/config" h "github.com/dedpnd/unifier/internal/core/server/http" "github.com/dedpnd/unifier/internal/core/worker" + "github.com/dedpnd/unifier/internal/logger" ) func main() { - log.Println("Server start...") + // Создаем логер + lg, err := logger.Init("info") + if err != nil { + log.Fatalln(err.Error()) + } + lg.Info("Server start...") // Читаем конфигурацию cfg, err := config.GetConfig() if err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } // Создаем хранилище - str, err := store.NewStore(cfg.DatabaseDSN) + str, err := store.NewStore(cfg.DatabaseDSN, lg) if err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } // Запускаем пул воркеров - p, err := worker.StartPool(cfg.KafkaAdress, str) + p, err := worker.StartPool(cfg.KafkaAdress, str, lg) if err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } // Создаем роутер - r, err := router.Router(str, p) + r, err := router.Router(lg, str, p) if err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } // Функция для завершения работы callback := func() { err := str.Close() if err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } p.StopPool() @@ -49,5 +55,5 @@ func main() { // Поднимаем сервер addr := "localhost:8080" - h.GracefulServer(addr, r, callback) + h.GracefulServer(addr, r, lg, callback) } diff --git a/go.mod b/go.mod index a66f995..7325855 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa github.com/jackc/pgx/v5 v5.5.2 github.com/segmentio/kafka-go v0.4.47 + go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 ) @@ -24,6 +25,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.16 // indirect github.com/stretchr/testify v1.8.4 // indirect go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.10.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index 33477bb..464eeb0 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,12 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= diff --git a/internal/adapter/api/middleware/jwtguard.go b/internal/adapter/api/middleware/jwtguard.go index dd660b5..f529dbd 100644 --- a/internal/adapter/api/middleware/jwtguard.go +++ b/internal/adapter/api/middleware/jwtguard.go @@ -9,30 +9,28 @@ import ( "github.com/dedpnd/unifier/internal/core/auth" ) -func JWTguard() func(h http.Handler) http.Handler { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - c, err := req.Cookie("token") - if err != nil { - if errors.Is(err, http.ErrNoCookie) { - res.WriteHeader(http.StatusUnauthorized) - return - } - res.WriteHeader(http.StatusBadRequest) - return - } - - token := c.Value - pl, err := auth.VerifyJWTandGetPayload(token) - if err != nil { +func JWTguard(h http.Handler) http.Handler { + return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + c, err := req.Cookie("token") + if err != nil { + if errors.Is(err, http.ErrNoCookie) { res.WriteHeader(http.StatusUnauthorized) return } + res.WriteHeader(http.StatusBadRequest) + return + } + + token := c.Value + pl, err := auth.VerifyJWTandGetPayload(token) + if err != nil { + res.WriteHeader(http.StatusUnauthorized) + return + } - ctx := req.Context() - r := req.WithContext(context.WithValue(ctx, util.ContextKeyToken, pl)) + ctx := req.Context() + r := req.WithContext(context.WithValue(ctx, util.ContextKeyToken, pl)) - h.ServeHTTP(res, r) - }) - } + h.ServeHTTP(res, r) + }) } diff --git a/internal/adapter/api/middleware/logger.go b/internal/adapter/api/middleware/logger.go index 3cb20ab..f5002e0 100644 --- a/internal/adapter/api/middleware/logger.go +++ b/internal/adapter/api/middleware/logger.go @@ -1,9 +1,10 @@ package middleware import ( - "log" "net/http" "time" + + "go.uber.org/zap" ) type responseObserver struct { @@ -23,7 +24,7 @@ func (o *responseObserver) WriteHeader(code int) { o.status = code } -func Logger() func(h http.Handler) http.Handler { +func Logger(lg *zap.Logger) func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { start := time.Now() @@ -33,8 +34,13 @@ func Logger() func(h http.Handler) http.Handler { duration := time.Since(start) - log.Printf(`Fetch URL: [ method:%v, uri:%v, status: %v, size: %v, duration:%v ]`, - req.Method, req.RequestURI, o.status, o.written, duration) + lg.Info("Fetch URL:", + zap.String("method", req.Method), + zap.String("uri", req.RequestURI), + zap.Int("status", o.status), + zap.Int64("size", o.written), + zap.Duration("duration", duration), + ) }) } } diff --git a/internal/adapter/api/rest/rules.go b/internal/adapter/api/rest/rules.go index 433df18..1bd6732 100644 --- a/internal/adapter/api/rest/rules.go +++ b/internal/adapter/api/rest/rules.go @@ -3,8 +3,6 @@ package rest import ( "bytes" "encoding/json" - "fmt" - "log" "net/http" "strconv" @@ -13,26 +11,27 @@ import ( "github.com/dedpnd/unifier/internal/core/worker" "github.com/dedpnd/unifier/internal/models" "github.com/go-chi/chi/v5" + "go.uber.org/zap" ) type RulesHandler struct { - Store store.Storage - Pool worker.Pool + Logger *zap.Logger + Store store.Storage + Pool worker.Pool } -const IntServerError = "Internal server error" +const IntServerError = "internal server error" func (h RulesHandler) GetAllRules(res http.ResponseWriter, req *http.Request) { data, err := h.Store.GetAllRules(req.Context()) if err != nil { - log.Println(fmt.Errorf("failed get all records from database: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed get all records from database") http.Error(res, IntServerError, http.StatusInternalServerError) return } resBodyBytes := new(bytes.Buffer) if err := json.NewEncoder(resBodyBytes).Encode(&data); err != nil { - log.Println(fmt.Errorf("invalid stringify JSON: %w", err)) http.Error(res, IntServerError, http.StatusBadRequest) return } @@ -41,7 +40,7 @@ func (h RulesHandler) GetAllRules(res http.ResponseWriter, req *http.Request) { _, err = res.Write(resBodyBytes.Bytes()) if err != nil { - log.Println(fmt.Errorf("failed write record to response: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed write record to response") http.Error(res, IntServerError, http.StatusInternalServerError) return } @@ -50,7 +49,7 @@ func (h RulesHandler) GetAllRules(res http.ResponseWriter, req *http.Request) { func (h RulesHandler) CreateRule(res http.ResponseWriter, req *http.Request) { token, ok := util.GetTokenFromContext(req.Context()) if !ok { - log.Println(fmt.Errorf("invalid jwt token")) + h.Logger.Error("invalid jwt token") http.Error(res, IntServerError, http.StatusInternalServerError) return } @@ -58,14 +57,13 @@ func (h RulesHandler) CreateRule(res http.ResponseWriter, req *http.Request) { pBody := models.Config{} if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { - log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) - http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + http.Error(res, `invalid parsing JSON`, http.StatusBadRequest) return } id, err := h.Store.CreateRule(req.Context(), pBody, token.ID) if err != nil { - log.Println(fmt.Errorf("failed save records: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed save rule") http.Error(res, IntServerError, http.StatusInternalServerError) return } @@ -79,7 +77,7 @@ func (h RulesHandler) CreateRule(res http.ResponseWriter, req *http.Request) { func (h RulesHandler) DeleteRule(res http.ResponseWriter, req *http.Request) { token, ok := util.GetTokenFromContext(req.Context()) if !ok { - log.Println(fmt.Errorf("invalid jwt token")) + h.Logger.Error("invalid jwt token") http.Error(res, IntServerError, http.StatusInternalServerError) return } @@ -88,31 +86,30 @@ func (h RulesHandler) DeleteRule(res http.ResponseWriter, req *http.Request) { pID, err := strconv.Atoi(id) if err != nil { - log.Println(fmt.Errorf("failed convert id: %w", err)) - http.Error(res, `Failde convert id to int`, http.StatusBadRequest) + http.Error(res, `failde convert id to int`, http.StatusBadRequest) return } dr, err := h.Store.GetRuleByID(req.Context(), pID) if err != nil { - log.Println(fmt.Errorf("failed get rule row: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed get rule") http.Error(res, IntServerError, http.StatusInternalServerError) return } if dr.ID == 0 { - http.Error(res, "Not Found", http.StatusNotFound) + http.Error(res, "not found", http.StatusNotFound) return } if dr.Owner != token.ID { - http.Error(res, "Forbidden", http.StatusForbidden) + http.Error(res, "forbidden", http.StatusForbidden) return } err = h.Store.DeleteRule(req.Context(), pID) if err != nil { - log.Println(fmt.Errorf("failed delele rules row: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed delete rule") http.Error(res, IntServerError, http.StatusInternalServerError) return } diff --git a/internal/adapter/api/rest/user.go b/internal/adapter/api/rest/user.go index 6254e35..8c5f9b6 100644 --- a/internal/adapter/api/rest/user.go +++ b/internal/adapter/api/rest/user.go @@ -2,18 +2,20 @@ package rest import ( "encoding/json" - "fmt" - "log" + "errors" "net/http" "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/adapter/store/postgres" "github.com/dedpnd/unifier/internal/core/auth" "github.com/dedpnd/unifier/internal/models" + "go.uber.org/zap" "golang.org/x/crypto/bcrypt" ) type UserHandler struct { - Store store.Storage + Logger *zap.Logger + Store store.Storage } type LoginBody struct { @@ -25,45 +27,49 @@ func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { pBody := LoginBody{} if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { - log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) - http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + http.Error(res, `invalid parsing JSON`, http.StatusBadRequest) return } if pBody.Login == nil || pBody.Password == nil { - http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + http.Error(res, `login or password incorrect`, http.StatusBadRequest) return } data, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) if err != nil { - log.Println(fmt.Errorf("failed get user from database: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed get user from database") http.Error(res, IntServerError, http.StatusInternalServerError) return } if data.ID > 0 { - http.Error(res, `Login exist`, http.StatusConflict) + http.Error(res, `login exist`, http.StatusConflict) return } hash, err := bcrypt.GenerateFromPassword([]byte(*pBody.Password), bcrypt.DefaultCost) if err != nil { - log.Println(fmt.Errorf("failed get hash from password: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed get hash from password") http.Error(res, IntServerError, http.StatusInternalServerError) return } id, err := h.Store.CreateUser(req.Context(), models.User{Login: *pBody.Login, Hash: string(hash)}) if err != nil { - log.Println(fmt.Errorf("failed create user: %w", err)) + if errors.Is(err, postgres.ErrUserUniq) { + http.Error(res, err.Error(), http.StatusBadRequest) + return + } + + h.Logger.With(zap.Error(err)).Error("failed create user") http.Error(res, IntServerError, http.StatusInternalServerError) return } token, err := auth.GetJWT(id, *pBody.Login) if err != nil { - log.Println(fmt.Errorf("failed create jwt token: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed create jwt token") http.Error(res, IntServerError, http.StatusInternalServerError) return } @@ -81,31 +87,30 @@ func (h UserHandler) Login(res http.ResponseWriter, req *http.Request) { pBody := LoginBody{} if err := json.NewDecoder(req.Body).Decode(&pBody); err != nil { - log.Println(fmt.Errorf("invalid parsing JSON: %w", err)) - http.Error(res, `Invalid parsing JSON`, http.StatusBadRequest) + http.Error(res, `invalid parsing JSON`, http.StatusBadRequest) return } if pBody.Login == nil || pBody.Password == nil { - http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + http.Error(res, `login or password incorrect`, http.StatusBadRequest) return } data, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) if err != nil { - log.Println(fmt.Errorf("failed get user from database: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed get user from database") http.Error(res, IntServerError, http.StatusInternalServerError) return } if err := bcrypt.CompareHashAndPassword([]byte(data.Hash), []byte(*pBody.Password)); err != nil { - http.Error(res, `Login or password incorrect`, http.StatusBadRequest) + http.Error(res, `login or password incorrect`, http.StatusBadRequest) return } token, err := auth.GetJWT(data.ID, *pBody.Login) if err != nil { - log.Println(fmt.Errorf("failed create jwt token: %w", err)) + h.Logger.With(zap.Error(err)).Error("failed create jwt token") http.Error(res, IntServerError, http.StatusInternalServerError) return } diff --git a/internal/adapter/api/router/router.go b/internal/adapter/api/router/router.go index 49d0ceb..1ac85b6 100644 --- a/internal/adapter/api/router/router.go +++ b/internal/adapter/api/router/router.go @@ -6,24 +6,27 @@ import ( "github.com/dedpnd/unifier/internal/adapter/store" "github.com/dedpnd/unifier/internal/core/worker" "github.com/go-chi/chi/v5" + "go.uber.org/zap" ) -func Router(str store.Storage, pool worker.Pool) (chi.Router, error) { +func Router(lg *zap.Logger, str store.Storage, pool worker.Pool) (chi.Router, error) { r := chi.NewRouter() - r.Use(middleware.Logger()) + r.Use(middleware.Logger(lg)) rulesHandler := rest.RulesHandler{ - Store: str, - Pool: pool, + Logger: lg, + Store: str, + Pool: pool, } - r.With(middleware.JWTguard()).Get("/api/rules", rulesHandler.GetAllRules) - r.With(middleware.JWTguard()).Post("/api/rules", rulesHandler.CreateRule) - r.With(middleware.JWTguard()).Delete("/api/rules/{id}", rulesHandler.DeleteRule) + r.With(middleware.JWTguard).Get("/api/rules", rulesHandler.GetAllRules) + r.With(middleware.JWTguard).Post("/api/rules", rulesHandler.CreateRule) + r.With(middleware.JWTguard).Delete("/api/rules/{id}", rulesHandler.DeleteRule) userHandler := rest.UserHandler{ - Store: str, + Logger: lg, + Store: str, } r.Post("/api/user/register", userHandler.Register) diff --git a/internal/adapter/api/util/util.go b/internal/adapter/api/util/util.go index f7557a1..afd34c8 100644 --- a/internal/adapter/api/util/util.go +++ b/internal/adapter/api/util/util.go @@ -6,14 +6,10 @@ import ( "github.com/dedpnd/unifier/internal/core/auth" ) -type contextKey string +type contextKey int -func (c contextKey) String() string { - return string(c) -} - -var ( - ContextKeyToken = contextKey("deleteCaller") +const ( + ContextKeyToken contextKey = iota ) func GetTokenFromContext(ctx context.Context) (auth.Claims, bool) { diff --git a/internal/adapter/store/postgres/migrations/00001_init.down.sql b/internal/adapter/store/postgres/migrations/00001_init.down.sql index e93f51b..ff1f7e0 100644 --- a/internal/adapter/store/postgres/migrations/00001_init.down.sql +++ b/internal/adapter/store/postgres/migrations/00001_init.down.sql @@ -1,2 +1,6 @@ +BEGIN TRANSACTION; + DROP TABLE users; -DROP TABLE rules; \ No newline at end of file +DROP TABLE rules; + +COMMIT; \ No newline at end of file diff --git a/internal/adapter/store/postgres/migrations/00001_init.up.sql b/internal/adapter/store/postgres/migrations/00001_init.up.sql index 9438e8d..beac309 100644 --- a/internal/adapter/store/postgres/migrations/00001_init.up.sql +++ b/internal/adapter/store/postgres/migrations/00001_init.up.sql @@ -1,15 +1,28 @@ +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS users( ID SERIAL UNIQUE PRIMARY KEY NOT NULL, - Login VARCHAR(255) UNIQUE NOT NULL, + Login VARCHAR(255) UNIQUE NOT NULL, Hash VARCHAR(1000) NOT NULL ); CREATE TABLE IF NOT EXISTS rules( - ID SERIAL PRIMARY KEY, - Rule JSON, - Owner INT, + ID SERIAL PRIMARY KEY NOT NULL, + Rule JSON NOT NULL, + Owner INT NOT NULL, CONSTRAINT fk_users FOREIGN KEY(Owner) REFERENCES users(ID) ON DELETE SET NULL ); + +-- password is password +INSERT INTO users +(login, hash) +VALUES('admin', '$2a$10$fvuwEbdImMWCPGzjuJ7pbOAQnZ/e9VyrVK60ComfJiEJvgkORJTci'); + +INSERT INTO rules +("rule", "owner") +VALUES('{"topicFrom":"events","filter":{"regexp":"\"dstHost.ip\": \"10.10.10.10\""},"entityHash":["srcHost.ip","dstHost.port"],"unifier":[{"name":"id","type":"string","expression":"auditEventLog"},{"name":"date","type":"timestamp","expression":"datetime"},{"name":"ipaddr","type":"string","expression":"srcHost.ip"},{"name":"category","type":"string","expression":"cat"}],"extraProcess":[{"func":"__if","args":"category, /Host/Connect/Host/Accept, high","to":"category"},{"func":"__stringConstant","args":"test","to":"customString1"}],"topicTo":"test"}'::json, 1); + +COMMIT; \ No newline at end of file diff --git a/internal/adapter/store/postgres/migrations/00002_init-data.down.sql b/internal/adapter/store/postgres/migrations/00002_init-data.down.sql deleted file mode 100644 index e17ccd3..0000000 --- a/internal/adapter/store/postgres/migrations/00002_init-data.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DELETE FROM users WHERE id=1; -DELETE FROM rules WHERE id=1; \ No newline at end of file diff --git a/internal/adapter/store/postgres/migrations/00002_init-data.up.sql b/internal/adapter/store/postgres/migrations/00002_init-data.up.sql deleted file mode 100644 index 434a455..0000000 --- a/internal/adapter/store/postgres/migrations/00002_init-data.up.sql +++ /dev/null @@ -1,8 +0,0 @@ --- password is password -INSERT INTO users -(login, hash) -VALUES('admin', '$2a$10$fvuwEbdImMWCPGzjuJ7pbOAQnZ/e9VyrVK60ComfJiEJvgkORJTci'); - -INSERT INTO rules -("rule", "owner") -VALUES('{"topicFrom":"events","filter":{"regexp":"\"dstHost.ip\": \"10.10.10.10\""},"entityHash":["srcHost.ip","dstHost.port"],"unifier":[{"name":"id","type":"string","expression":"auditEventLog"},{"name":"date","type":"timestamp","expression":"datetime"},{"name":"ipaddr","type":"string","expression":"srcHost.ip"},{"name":"category","type":"string","expression":"cat"}],"extraProcess":[{"func":"__if","args":"category, /Host/Connect/Host/Accept, high","to":"category"},{"func":"__stringConstant","args":"test","to":"customString1"}],"topicTo":"test"}'::json, 1); diff --git a/internal/adapter/store/postgres/postgres.go b/internal/adapter/store/postgres/postgres.go index b2c401a..fe9ed65 100644 --- a/internal/adapter/store/postgres/postgres.go +++ b/internal/adapter/store/postgres/postgres.go @@ -5,7 +5,6 @@ import ( "embed" "errors" "fmt" - "log" "github.com/dedpnd/unifier/internal/models" "github.com/golang-migrate/migrate/v4" @@ -13,6 +12,7 @@ import ( "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "go.uber.org/zap" _ "github.com/golang-migrate/migrate/v4/database/postgres" ) @@ -21,13 +21,15 @@ type DataBase struct { pool *pgxpool.Pool } -func NewDB(ctx context.Context, dsn string) (DataBase, error) { +var ErrUserUniq = errors.New("such a user exists") + +func NewDB(ctx context.Context, dsn string, lg *zap.Logger) (DataBase, error) { pool, err := connection(ctx, dsn) if err != nil { return DataBase{}, fmt.Errorf("failed connection to postgre: %w", err) } - log.Println("Connection to postgre: success") + lg.Info("Connection to postgre: success") if err := runMigrations(dsn); err != nil { return DataBase{}, fmt.Errorf("failed to run DB migrations: %w", err) @@ -78,7 +80,12 @@ func (db DataBase) CreateUser(ctx context.Context, user models.User) (int, error var id int err := row.Scan(&id) if err != nil { - return 0, fmt.Errorf("failed scan user record row: %w", err) + var pgErr *pgconn.PgError + if !(errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation) { + return 0, fmt.Errorf("unique violation: %w", err) + } + + return 0, ErrUserUniq } return id, nil diff --git a/internal/adapter/store/store.go b/internal/adapter/store/store.go index f17b204..6067ed4 100644 --- a/internal/adapter/store/store.go +++ b/internal/adapter/store/store.go @@ -7,6 +7,7 @@ import ( "github.com/dedpnd/unifier/internal/adapter/store/postgres" "github.com/dedpnd/unifier/internal/models" + "go.uber.org/zap" ) type Storage interface { @@ -19,9 +20,9 @@ type Storage interface { DeleteRule(ctx context.Context, id int) error } -func NewStore(dsn string) (Storage, error) { +func NewStore(dsn string, lg *zap.Logger) (Storage, error) { if len(dsn) != 0 { - dbs, err := postgres.NewDB(context.Background(), dsn) + dbs, err := postgres.NewDB(context.Background(), dsn, lg) if err != nil { return nil, fmt.Errorf("failed create database storage: %w", err) } diff --git a/internal/core/auth/auth.go b/internal/core/auth/auth.go index 03877ea..878882b 100644 --- a/internal/core/auth/auth.go +++ b/internal/core/auth/auth.go @@ -3,7 +3,6 @@ package auth import ( "errors" "fmt" - "log" "time" "github.com/golang-jwt/jwt/v5" @@ -47,10 +46,8 @@ func VerifyJWTandGetPayload(token string) (Claims, error) { if err != nil { if errors.Is(err, jwt.ErrSignatureInvalid) { - log.Println(fmt.Errorf("invalid jwt token: %w", err)) return *claims, fmt.Errorf("failed signature from jwt: %w", err) } - log.Println(fmt.Errorf("invalid jwt token: %w", err)) return *claims, fmt.Errorf("invalid jwt token: %w", err) } diff --git a/internal/core/server/http/http.go b/internal/core/server/http/http.go index e30b8a3..215ee63 100644 --- a/internal/core/server/http/http.go +++ b/internal/core/server/http/http.go @@ -2,7 +2,6 @@ package http import ( "context" - "log" "net/http" "os" "os/signal" @@ -10,9 +9,10 @@ import ( "syscall" "github.com/go-chi/chi/v5" + "go.uber.org/zap" ) -func GracefulServer(addr string, r chi.Router, cb func()) { +func GracefulServer(addr string, r chi.Router, lg *zap.Logger, cb func()) { var wg sync.WaitGroup ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) @@ -30,7 +30,7 @@ func GracefulServer(addr string, r chi.Router, cb func()) { go func() { if err := http.ListenAndServe(addr, r); err != nil { - log.Fatal(err.Error()) + lg.Fatal(err.Error()) } }() diff --git a/internal/core/worker/pool.go b/internal/core/worker/pool.go index c2e7f6e..5a04a9b 100644 --- a/internal/core/worker/pool.go +++ b/internal/core/worker/pool.go @@ -3,14 +3,15 @@ package worker import ( "context" "fmt" - "log" "strconv" "github.com/dedpnd/unifier/internal/adapter/store" "github.com/dedpnd/unifier/internal/models" + "go.uber.org/zap" ) type Pool struct { + logger *zap.Logger kafkaURL string p map[string]workerEntity } @@ -21,8 +22,9 @@ type workerEntity struct { Stop chan bool } -func StartPool(kAddr string, str store.Storage) (Pool, error) { +func StartPool(kAddr string, str store.Storage, lg *zap.Logger) (Pool, error) { p := Pool{ + logger: lg, kafkaURL: kAddr, p: make(map[string]workerEntity), } @@ -48,8 +50,8 @@ func (p Pool) AddWorker(id string, rule models.Config) { } go func() { - if err := Start(context.Background(), p.kafkaURL, p.p[id]); err != nil { - log.Println(err.Error()) + if err := Start(context.Background(), p.kafkaURL, p.p[id], p.logger); err != nil { + p.logger.With(zap.Error(err)).Error("Worker has error", zap.String("ID", id)) } }() } diff --git a/internal/core/worker/worker.go b/internal/core/worker/worker.go index ca41800..6d118dc 100644 --- a/internal/core/worker/worker.go +++ b/internal/core/worker/worker.go @@ -6,20 +6,20 @@ import ( "encoding/hex" "encoding/json" "fmt" - "log" "regexp" "strings" "time" "github.com/dedpnd/unifier/internal/models" "github.com/segmentio/kafka-go" + "go.uber.org/zap" ) -func Start(ctx context.Context, kafkaURL string, wrkConfig workerEntity) error { +func Start(ctx context.Context, kafkaURL string, wrkConfig workerEntity, lg *zap.Logger) error { var r *kafka.Reader var p *kafka.Conn - log.Println("Worker start:", wrkConfig.ID) + lg.Info("Worker start", zap.String("ID", wrkConfig.ID)) // Создаем kafka consumer r = kafka.NewReader(kafka.ReaderConfig{ @@ -66,13 +66,13 @@ func Start(ctx context.Context, kafkaURL string, wrkConfig workerEntity) error { // Унификация полей err = unificationFields(pEvent, wrkConfig.Config.Unifier, &uniferEvents) if err != nil { - log.Println(err.Error()) + lg.Error(err.Error()) } // Допольнительная обработка err = extraProcess(wrkConfig.Config.ExtraProcess, &uniferEvents) if err != nil { - log.Println(err.Error()) + lg.Error(err.Error()) } buf, err := json.Marshal(uniferEvents) @@ -86,7 +86,7 @@ func Start(ctx context.Context, kafkaURL string, wrkConfig workerEntity) error { } } case <-wrkConfig.Stop: - log.Println("Worker stop:", wrkConfig.ID) + lg.Info("Worker stop", zap.String("ID", wrkConfig.ID)) err := r.Close() if err != nil { diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..79c6454 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,26 @@ +package logger + +import ( + "fmt" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func Init(level string) (*zap.Logger, error) { + lvl, err := zap.ParseAtomicLevel(level) + if err != nil { + return nil, fmt.Errorf("failed parse error level %w", err) + } + + cfg := zap.NewProductionConfig() + cfg.Level = lvl + cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + zl, err := cfg.Build() + if err != nil { + return nil, fmt.Errorf("failed build zap config %w", err) + } + + return zl, nil +} From 73202dd9e64e36ce2119119b9c4d8e5116d9f3ef Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 26 Jan 2024 12:12:25 +0300 Subject: [PATCH 03/14] fix: correction iter1 #2 --- cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 20510d8..52c0d3e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -47,7 +47,7 @@ func main() { callback := func() { err := str.Close() if err != nil { - lg.Fatal(err.Error()) + lg.Info(err.Error()) } p.StopPool() From f0fad30229df19e05ec0eb0720a0c4b23b1fbd88 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Mon, 29 Jan 2024 17:24:49 +0300 Subject: [PATCH 04/14] add: added tests --- go.mod | 45 +++- go.sum | 121 ++++++++-- internal/adapter/api/router/router_test.go | 218 ++++++++++++++++++ internal/core/auth/auth_test.go | 65 ++++++ internal/core/worker/pool.go | 16 +- internal/core/worker/worker.go | 21 +- internal/core/worker/worker_test.go | 243 +++++++++++++++++++++ 7 files changed, 699 insertions(+), 30 deletions(-) create mode 100644 internal/adapter/api/router/router_test.go create mode 100644 internal/core/auth/auth_test.go create mode 100644 internal/core/worker/worker_test.go diff --git a/go.mod b/go.mod index 7325855..d51aa02 100644 --- a/go.mod +++ b/go.mod @@ -5,27 +5,64 @@ go 1.21 require ( github.com/caarlos0/env v3.5.0+incompatible github.com/go-chi/chi/v5 v5.0.11 + github.com/go-resty/resty/v2 v2.11.0 github.com/golang-jwt/jwt/v5 v5.2.0 github.com/golang-migrate/migrate/v4 v4.17.0 github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa github.com/jackc/pgx/v5 v5.5.2 + github.com/lib/pq v1.10.9 + github.com/ory/dockertest/v3 v3.10.0 github.com/segmentio/kafka-go v0.4.47 + github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.18.0 ) require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/continuity v0.4.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/cli v25.0.1+incompatible // indirect + github.com/docker/docker v25.0.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/lib/pq v1.10.9 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/runc v1.1.11 // indirect github.com/pierrec/lz4/v4 v4.1.16 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel/trace v1.22.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/sync v0.5.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.17.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 464eeb0..bfe45fb 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,56 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= +github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dhui/dktest v0.4.0 h1:z05UmuXZHO/bgj/ds2bGMBu8FI4WA+Ag/m3ghL+om7M= github.com/dhui/dktest v0.4.0/go.mod h1:v/Dbz1LgCBOi2Uki2nUqLBGa83hWBGFMu5MrgMDCc78= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= +github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v25.0.1+incompatible h1:k5TYd5rIVQRSqcTwCID+cyVA0yRg86+Pcrz1ls0/frA= +github.com/docker/docker v25.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-migrate/migrate/v4 v4.17.0 h1:rd40H3QXU0AA4IoLllFcEAEo9dYKRHYND2gB4p7xcaU= github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdrNx2HGs45rbWAo6OsKM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -40,11 +66,19 @@ github.com/jackc/pgx/v5 v5.5.2 h1:iLlpgp4Cp/gC9Xuscl7lFL1PhhW+ZLtXZcrfCt4C3tA= github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -53,6 +87,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.1.11 h1:9LjxyVlE0BPMRP2wuQDRlHV4941Jp9rc3F0+YKimopA= +github.com/opencontainers/runc v1.1.11/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= +github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -60,8 +98,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -76,7 +118,24 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= @@ -86,37 +145,50 @@ go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -131,14 +203,27 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= diff --git a/internal/adapter/api/router/router_test.go b/internal/adapter/api/router/router_test.go new file mode 100644 index 0000000..7eb070e --- /dev/null +++ b/internal/adapter/api/router/router_test.go @@ -0,0 +1,218 @@ +package router + +import ( + "database/sql" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/dedpnd/unifier/internal/adapter/store" + "github.com/dedpnd/unifier/internal/config" + "github.com/dedpnd/unifier/internal/core/auth" + "github.com/dedpnd/unifier/internal/core/worker" + "github.com/dedpnd/unifier/internal/logger" + "github.com/go-resty/resty/v2" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/assert" + + _ "github.com/lib/pq" +) + +var db *sql.DB +var databaseUrl string + +func TestMain(m *testing.M) { + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + //nolint:typecheck // dockertest is defined + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + //nolint:typecheck // dockertest is defined + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "16.1-alpine3.18", + Env: []string{ + "POSTGRES_PASSWORD=test", + "POSTGRES_USER=test", + "listen_addresses = '*'", + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + hostAndPort := resource.GetHostPort("5432/tcp") + databaseUrl = fmt.Sprintf("postgres://test:test@%s?sslmode=disable", hostAndPort) + + log.Println("Connecting to database on url: ", databaseUrl) + + // Tell docker to hard kill the container in 120 seconds + resource.Expire(120) + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + pool.MaxWait = 20 * time.Second + if err = pool.Retry(func() error { + db, err = sql.Open("postgres", databaseUrl) + if err != nil { + return err + } + return db.Ping() + }); err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + //Run tests + code := m.Run() + + // You can't defer this because os.Exit doesn't care for defer + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + + os.Exit(code) +} + +func TestRouter(t *testing.T) { + tests := []struct { + name string + method string + authorization bool + url string + body map[string]interface{} + expectedCode int + expectedBody string + }{ + { + name: "Register test user", + method: http.MethodPost, + url: "/api/user/register", + body: map[string]interface{}{ + "login": "test", + "password": "test", + }, + expectedCode: http.StatusOK, + expectedBody: "", + }, + { + name: "Login test user", + method: http.MethodPost, + url: "/api/user/login", + body: map[string]interface{}{ + "login": "test", + "password": "test", + }, + expectedCode: http.StatusOK, + expectedBody: "", + }, + { + name: "Get all rules", + method: http.MethodGet, + authorization: true, + url: "/api/rules", + expectedCode: http.StatusOK, + expectedBody: "[{\"id\":1,\"rule\":{\"topicFrom\":\"events\",\"filter\":{\"regexp\":\"\\\"dstHost.ip\\\": \\\"10.10.10.10\\\"\"},\"entityHash\":[\"srcHost.ip\",\"dstHost.port\"],\"unifier\":[{\"name\":\"id\",\"type\":\"string\",\"expression\":\"auditEventLog\"},{\"name\":\"date\",\"type\":\"timestamp\",\"expression\":\"datetime\"},{\"name\":\"ipaddr\",\"type\":\"string\",\"expression\":\"srcHost.ip\"},{\"name\":\"category\",\"type\":\"string\",\"expression\":\"cat\"}],\"extraProcess\":[{\"func\":\"__if\",\"args\":\"category, /Host/Connect/Host/Accept, high\",\"to\":\"category\"},{\"func\":\"__stringConstant\",\"args\":\"test\",\"to\":\"customString1\"}],\"topicTo\":\"test\"},\"owner\":1}]\n", + }, + { + name: "Add new rule", + method: http.MethodPost, + authorization: true, + url: "/api/rules", + body: map[string]interface{}{ + "topicFrom": "events", + }, + expectedCode: http.StatusOK, + expectedBody: "", + }, + { + name: "Remove rule", + method: http.MethodDelete, + authorization: true, + url: "/api/rules/1", + expectedCode: http.StatusOK, + expectedBody: "", + }, + } + + // Создаем логер + lg, err := logger.Init("error") + if err != nil { + assert.NoError(t, err) + } + + // Читаем конфигурацию + cfg, err := config.GetConfig() + if err != nil { + assert.NoError(t, err) + } + + // Создаем хранилище + str, err := store.NewStore(databaseUrl, lg) + if err != nil { + assert.NoError(t, err) + } + + // Запускаем пул воркеров + p, err := worker.StartPool(cfg.KafkaAdress, str, lg) + if err != nil { + assert.NoError(t, err) + } + + // Сорздаем роутер + r, err := Router(lg, str, p) + if err != nil { + assert.NoError(t, err) + } + + srv := httptest.NewServer(r) + defer srv.Close() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := resty.New().R() + req.Method = tt.method + req.URL = srv.URL + tt.url + + if tt.authorization { + token, err := auth.GetJWT(1, "test") + if err != nil { + assert.NoError(t, err) + } + + req.SetCookie(&http.Cookie{ + Name: "token", + Value: *token, + }) + } + + if len(tt.body) != 0 { + req.Body = tt.body + } + + resp, err := req.Send() + assert.NoError(t, err, "error making HTTP request") + + assert.Equal(t, tt.expectedCode, resp.StatusCode(), "Response code didn't match expected") + + if tt.expectedBody != "" { + assert.Equal(t, tt.expectedBody, string(resp.Body())) + } + }) + } +} diff --git a/internal/core/auth/auth_test.go b/internal/core/auth/auth_test.go new file mode 100644 index 0000000..f309c2e --- /dev/null +++ b/internal/core/auth/auth_test.go @@ -0,0 +1,65 @@ +package auth + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetJWT(t *testing.T) { + type args struct { + id int + login string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Get jwt token", + args: args{ + id: 1, + login: "test", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := GetJWT(tt.args.id, tt.args.login) + if (err != nil) != tt.wantErr { + t.Errorf("GetJWT() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestVerifyJWTandGetPayload(t *testing.T) { + tests := []struct { + name string + wantErr bool + }{ + { + name: "Token shoul be correct", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + token, err := GetJWT(1, "test") + if err != nil { + assert.NoError(t, err) + } + + _, err = VerifyJWTandGetPayload(*token) + if (err != nil) != tt.wantErr { + t.Errorf("VerifyJWTandGetPayload() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/internal/core/worker/pool.go b/internal/core/worker/pool.go index 5a04a9b..7ec2ab6 100644 --- a/internal/core/worker/pool.go +++ b/internal/core/worker/pool.go @@ -52,18 +52,24 @@ func (p Pool) AddWorker(id string, rule models.Config) { go func() { if err := Start(context.Background(), p.kafkaURL, p.p[id], p.logger); err != nil { p.logger.With(zap.Error(err)).Error("Worker has error", zap.String("ID", id)) + + // TODO: Перезапускать воркер ? + // Удаляем конфигурацию из пула так как она не работает + delete(p.p, id) } }() } func (p Pool) DeleteWorker(id string) { - wrk := p.p[id] + wrk, ok := p.p[id] - // Останнавливаем воркер - wrk.Stop <- true + if ok { + // Останнавливаем воркер + wrk.Stop <- true - // Удаляем конфигурацию - delete(p.p, id) + // Удаляем конфигурацию + delete(p.p, id) + } } func (p Pool) StopPool() { diff --git a/internal/core/worker/worker.go b/internal/core/worker/worker.go index 6d118dc..4397261 100644 --- a/internal/core/worker/worker.go +++ b/internal/core/worker/worker.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" "strings" "time" @@ -132,18 +133,32 @@ func unificationFields(event map[string]interface{}, cfgUnifier []models.Unifier (*uEvent)[u.Name] = v } case "int": - v, ok := v.(int) + vv, ok := v.(int) if ok { - (*uEvent)[u.Name] = v + (*uEvent)[u.Name] = vv + return nil + } + + v, ok := v.(string) + if !ok { + return fmt.Errorf("failed int parse to string: %v", v) + } + + i, err := strconv.Atoi(v) + if err != nil { + return fmt.Errorf("failed int parse: %w", err) } + + (*uEvent)[u.Name] = i case "timestamp": v, ok := v.(string) if ok { v, err := time.Parse(time.RFC3339, v) + if err != nil { return fmt.Errorf("failed date parse: %w", err) } - (*uEvent)[u.Name] = v + (*uEvent)[u.Name] = v.String() } default: return fmt.Errorf("unknown type: %v", u.Type) diff --git a/internal/core/worker/worker_test.go b/internal/core/worker/worker_test.go new file mode 100644 index 0000000..52edf4b --- /dev/null +++ b/internal/core/worker/worker_test.go @@ -0,0 +1,243 @@ +package worker + +import ( + "testing" + + "github.com/dedpnd/unifier/internal/models" + "github.com/stretchr/testify/assert" +) + +func Test_extraProcess(t *testing.T) { + type args struct { + cfgExtraProcess []models.ExtraProcess + uEvent *map[string]interface{} + } + type want struct { + key string + value string + } + + tests := []struct { + name string + args args + want + wantErr bool + }{ + { + name: "Func __if must write value", + args: args{ + cfgExtraProcess: []models.ExtraProcess{{ + Func: "__if", + Args: "testFrom, 123, 321", + To: "testTo", + }}, + uEvent: &map[string]interface{}{ + "testFrom": "123", + }, + }, + want: want{ + key: "testTo", + value: "321", + }, + wantErr: false, + }, + { + name: "Func __stringConstant must write value", + args: args{ + cfgExtraProcess: []models.ExtraProcess{{ + Func: "__stringConstant", + Args: "test", + To: "testTo", + }}, + uEvent: &map[string]interface{}{ + "testFrom": "123", + }, + }, + want: want{ + key: "testTo", + value: "test", + }, + wantErr: false, + }, + { + name: "Func __testFunc should return an error", + args: args{ + cfgExtraProcess: []models.ExtraProcess{{ + Func: "__testFunc", + Args: "test", + To: "testTo", + }}, + uEvent: &map[string]interface{}{ + "testFrom": "123", + }, + }, + want: want{ + key: "testFrom", + value: "123", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := extraProcess(tt.args.cfgExtraProcess, tt.args.uEvent); (err != nil) != tt.wantErr { + t.Errorf("extraProcess() error = %v, wantErr %v", err, tt.wantErr) + } + + v, ok := (*tt.args.uEvent)[tt.want.key] + if !ok { + t.Errorf("field = %v must be exist", tt.want.key) + } + + assert.Equal(t, tt.want.value, v) + }) + } +} + +func Test_unificationFields(t *testing.T) { + type args struct { + event map[string]interface{} + cfgUnifier []models.Unifier + uEvent *map[string]interface{} + } + type want struct { + key string + value interface{} + } + + tests := []struct { + name string + args args + want want + wantErr bool + }{ + { + name: "Field must be string", + args: args{ + event: map[string]interface{}{ + "testFrom": "qwerty", + }, + cfgUnifier: []models.Unifier{{ + Name: "testString", + Type: "string", + Expression: "testFrom", + }}, + uEvent: &map[string]interface{}{}, + }, + want: want{ + key: "testString", + value: "qwerty", + }, + wantErr: false, + }, + { + name: "Field must be int", + args: args{ + event: map[string]interface{}{ + "testFrom": "123", + }, + cfgUnifier: []models.Unifier{{ + Name: "testInt", + Type: "int", + Expression: "testFrom", + }}, + uEvent: &map[string]interface{}{}, + }, + want: want{ + key: "testInt", + value: 123, + }, + wantErr: false, + }, + { + name: "Field must be timestamp", + args: args{ + event: map[string]interface{}{ + "testFrom": "2023-07-13T13:47:43+00:00", + }, + cfgUnifier: []models.Unifier{{ + Name: "testTime", + Type: "timestamp", + Expression: "testFrom", + }}, + uEvent: &map[string]interface{}{}, + }, + want: want{ + key: "testTime", + value: "2023-07-13 13:47:43 +0000 +0000", + }, + wantErr: false, + }, + { + name: "Should return an error", + args: args{ + event: map[string]interface{}{ + "testFrom": "123", + }, + cfgUnifier: []models.Unifier{{ + Name: "test", + Type: "testType", + Expression: "testFrom", + }}, + uEvent: &map[string]interface{}{ + "testFrom": "123", + }, + }, + want: want{ + key: "testFrom", + value: "123", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := unificationFields(tt.args.event, tt.args.cfgUnifier, tt.args.uEvent); (err != nil) != tt.wantErr { + t.Errorf("unificationFields() error = %v, wantErr %v", err, tt.wantErr) + } + + v, ok := (*tt.args.uEvent)[tt.want.key] + if !ok { + t.Errorf("field = %v must be exist", tt.want.key) + } + + assert.Equal(t, tt.want.value, v) + }) + } +} + +func Test_calculateHash(t *testing.T) { + type args struct { + event map[string]interface{} + cfgEntHash []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Get hash must be correct", + args: args{ + event: map[string]interface{}{ + "testString": "qwerty", + "testInt": 123, + }, + cfgEntHash: []string{"testString", "testInt"}, + }, + want: "d8578edf8458ce06fbc5bb76a58c5ca4", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := calculateHash(tt.args.event, tt.args.cfgEntHash) + if got != tt.want { + t.Errorf("calculateHash() = %v, want %v", got, tt.want) + } + + assert.Equal(t, tt.want, got) + }) + } +} From 2d3462c32ce10b8033eaf53ff6206cf61392e546 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Tue, 30 Jan 2024 17:21:07 +0300 Subject: [PATCH 05/14] fix: correction iter1 #3 --- README.md | 2 +- golangci-lint.cmd | 2 +- internal/adapter/api/middleware/jwtguard.go | 3 +- internal/adapter/api/rest/rules.go | 2 +- internal/adapter/api/rest/user.go | 10 ++---- internal/adapter/api/router/router_test.go | 32 ++++++++++++------- internal/adapter/api/util/util.go | 4 +++ .../postgres/migrations/00001_init.up.sql | 9 ++---- internal/adapter/store/postgres/postgres.go | 21 +++++++++--- internal/models/model.go | 2 +- 10 files changed, 51 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d02cd38..61a7ec0 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ postgres - СУБД # Начало работы Пользователь postgres - user | password -В службе есть предустановленный пользователь - admin | password +Требуеться зарегистрировать пользователя Так же добавленно 1 правило унификации оно сразу настроенно на работы с тестовыми событиями ``` diff --git a/golangci-lint.cmd b/golangci-lint.cmd index 7e0809b..59d8830 100644 --- a/golangci-lint.cmd +++ b/golangci-lint.cmd @@ -1,3 +1,3 @@ -docker run --rm -v %cd%:/app -w /app golangci/golangci-lint:v1.53.3 golangci-lint run -c .golangci.yml > ./golangci-lint/report-unformatted.json +docker run --rm -v %cd%:/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run -c .golangci.yml > ./golangci-lint/report-unformatted.json docker run --rm -v %cd%:/app imega/jq -c . /app/golangci-lint/report-unformatted.json > ./golangci-lint/report.json pause \ No newline at end of file diff --git a/internal/adapter/api/middleware/jwtguard.go b/internal/adapter/api/middleware/jwtguard.go index f529dbd..e012e8c 100644 --- a/internal/adapter/api/middleware/jwtguard.go +++ b/internal/adapter/api/middleware/jwtguard.go @@ -1,7 +1,6 @@ package middleware import ( - "context" "errors" "net/http" @@ -29,7 +28,7 @@ func JWTguard(h http.Handler) http.Handler { } ctx := req.Context() - r := req.WithContext(context.WithValue(ctx, util.ContextKeyToken, pl)) + r := req.WithContext(util.SetTokenToContext(ctx, pl)) h.ServeHTTP(res, r) }) diff --git a/internal/adapter/api/rest/rules.go b/internal/adapter/api/rest/rules.go index 1bd6732..ab46579 100644 --- a/internal/adapter/api/rest/rules.go +++ b/internal/adapter/api/rest/rules.go @@ -102,7 +102,7 @@ func (h RulesHandler) DeleteRule(res http.ResponseWriter, req *http.Request) { return } - if dr.Owner != token.ID { + if *dr.Owner != token.ID { http.Error(res, "forbidden", http.StatusForbidden) return } diff --git a/internal/adapter/api/rest/user.go b/internal/adapter/api/rest/user.go index 8c5f9b6..b72b169 100644 --- a/internal/adapter/api/rest/user.go +++ b/internal/adapter/api/rest/user.go @@ -36,18 +36,13 @@ func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { return } - data, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) + _, err := h.Store.GetUserByLogin(req.Context(), *pBody.Login) if err != nil { h.Logger.With(zap.Error(err)).Error("failed get user from database") http.Error(res, IntServerError, http.StatusInternalServerError) return } - if data.ID > 0 { - http.Error(res, `login exist`, http.StatusConflict) - return - } - hash, err := bcrypt.GenerateFromPassword([]byte(*pBody.Password), bcrypt.DefaultCost) if err != nil { h.Logger.With(zap.Error(err)).Error("failed get hash from password") @@ -57,7 +52,8 @@ func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { id, err := h.Store.CreateUser(req.Context(), models.User{Login: *pBody.Login, Hash: string(hash)}) if err != nil { - if errors.Is(err, postgres.ErrUserUniq) { + var userUniqErr *postgres.ErrUserUniq + if errors.As(err, &userUniqErr) { http.Error(res, err.Error(), http.StatusBadRequest) return } diff --git a/internal/adapter/api/router/router_test.go b/internal/adapter/api/router/router_test.go index 7eb070e..9c2cf69 100644 --- a/internal/adapter/api/router/router_test.go +++ b/internal/adapter/api/router/router_test.go @@ -24,11 +24,10 @@ import ( ) var db *sql.DB -var databaseUrl string +var databaseURL string func TestMain(m *testing.M) { // uses a sensible default on windows (tcp/http) and linux/osx (socket) - //nolint:typecheck // dockertest is defined pool, err := dockertest.NewPool("") if err != nil { log.Fatalf("Could not construct pool: %s", err) @@ -40,7 +39,6 @@ func TestMain(m *testing.M) { } // pulls an image, creates a container based on it and runs it - //nolint:typecheck // dockertest is defined resource, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "postgres", Tag: "16.1-alpine3.18", @@ -59,26 +57,35 @@ func TestMain(m *testing.M) { } hostAndPort := resource.GetHostPort("5432/tcp") - databaseUrl = fmt.Sprintf("postgres://test:test@%s?sslmode=disable", hostAndPort) + databaseURL = fmt.Sprintf("postgres://test:test@%s?sslmode=disable", hostAndPort) - log.Println("Connecting to database on url: ", databaseUrl) + log.Println("Connecting to database on url: ", databaseURL) // Tell docker to hard kill the container in 120 seconds - resource.Expire(120) + err = resource.Expire(120) + if err != nil { + log.Fatalf("Expire resource has error: %s", err) + } // exponential backoff-retry, because the application in the container might not be ready to accept connections yet pool.MaxWait = 20 * time.Second if err = pool.Retry(func() error { - db, err = sql.Open("postgres", databaseUrl) + db, err = sql.Open("postgres", databaseURL) + if err != nil { + return fmt.Errorf("Connection has error: %w", err) + } + + err = db.Ping() if err != nil { - return err + return fmt.Errorf("Ping has error: %w", err) } - return db.Ping() + + return nil }); err != nil { log.Fatalf("Could not connect to docker: %s", err) } - //Run tests + // Run tests code := m.Run() // You can't defer this because os.Exit doesn't care for defer @@ -127,7 +134,8 @@ func TestRouter(t *testing.T) { authorization: true, url: "/api/rules", expectedCode: http.StatusOK, - expectedBody: "[{\"id\":1,\"rule\":{\"topicFrom\":\"events\",\"filter\":{\"regexp\":\"\\\"dstHost.ip\\\": \\\"10.10.10.10\\\"\"},\"entityHash\":[\"srcHost.ip\",\"dstHost.port\"],\"unifier\":[{\"name\":\"id\",\"type\":\"string\",\"expression\":\"auditEventLog\"},{\"name\":\"date\",\"type\":\"timestamp\",\"expression\":\"datetime\"},{\"name\":\"ipaddr\",\"type\":\"string\",\"expression\":\"srcHost.ip\"},{\"name\":\"category\",\"type\":\"string\",\"expression\":\"cat\"}],\"extraProcess\":[{\"func\":\"__if\",\"args\":\"category, /Host/Connect/Host/Accept, high\",\"to\":\"category\"},{\"func\":\"__stringConstant\",\"args\":\"test\",\"to\":\"customString1\"}],\"topicTo\":\"test\"},\"owner\":1}]\n", + //nolint:lll // This legal size + expectedBody: "[{\"id\":1,\"rule\":{\"topicFrom\":\"events\",\"filter\":{\"regexp\":\"\\\"dstHost.ip\\\": \\\"10.10.10.10\\\"\"},\"entityHash\":[\"srcHost.ip\",\"dstHost.port\"],\"unifier\":[{\"name\":\"id\",\"type\":\"string\",\"expression\":\"auditEventLog\"},{\"name\":\"date\",\"type\":\"timestamp\",\"expression\":\"datetime\"},{\"name\":\"ipaddr\",\"type\":\"string\",\"expression\":\"srcHost.ip\"},{\"name\":\"category\",\"type\":\"string\",\"expression\":\"cat\"}],\"extraProcess\":[{\"func\":\"__if\",\"args\":\"category, /Host/Connect/Host/Accept, high\",\"to\":\"category\"},{\"func\":\"__stringConstant\",\"args\":\"test\",\"to\":\"customString1\"}],\"topicTo\":\"test\"},\"owner\":1}]\n", }, { name: "Add new rule", @@ -163,7 +171,7 @@ func TestRouter(t *testing.T) { } // Создаем хранилище - str, err := store.NewStore(databaseUrl, lg) + str, err := store.NewStore(databaseURL, lg) if err != nil { assert.NoError(t, err) } diff --git a/internal/adapter/api/util/util.go b/internal/adapter/api/util/util.go index afd34c8..8c2fbad 100644 --- a/internal/adapter/api/util/util.go +++ b/internal/adapter/api/util/util.go @@ -16,3 +16,7 @@ func GetTokenFromContext(ctx context.Context) (auth.Claims, bool) { caller, ok := ctx.Value(ContextKeyToken).(auth.Claims) return caller, ok } + +func SetTokenToContext(ctx context.Context, pl auth.Claims) context.Context { + return context.WithValue(ctx, ContextKeyToken, pl) +} diff --git a/internal/adapter/store/postgres/migrations/00001_init.up.sql b/internal/adapter/store/postgres/migrations/00001_init.up.sql index beac309..8262a8b 100644 --- a/internal/adapter/store/postgres/migrations/00001_init.up.sql +++ b/internal/adapter/store/postgres/migrations/00001_init.up.sql @@ -9,20 +9,15 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS rules( ID SERIAL PRIMARY KEY NOT NULL, Rule JSON NOT NULL, - Owner INT NOT NULL, + Owner INT NULL, CONSTRAINT fk_users FOREIGN KEY(Owner) REFERENCES users(ID) ON DELETE SET NULL ); --- password is password -INSERT INTO users -(login, hash) -VALUES('admin', '$2a$10$fvuwEbdImMWCPGzjuJ7pbOAQnZ/e9VyrVK60ComfJiEJvgkORJTci'); - INSERT INTO rules ("rule", "owner") -VALUES('{"topicFrom":"events","filter":{"regexp":"\"dstHost.ip\": \"10.10.10.10\""},"entityHash":["srcHost.ip","dstHost.port"],"unifier":[{"name":"id","type":"string","expression":"auditEventLog"},{"name":"date","type":"timestamp","expression":"datetime"},{"name":"ipaddr","type":"string","expression":"srcHost.ip"},{"name":"category","type":"string","expression":"cat"}],"extraProcess":[{"func":"__if","args":"category, /Host/Connect/Host/Accept, high","to":"category"},{"func":"__stringConstant","args":"test","to":"customString1"}],"topicTo":"test"}'::json, 1); +VALUES('{"topicFrom":"events","filter":{"regexp":"\"dstHost.ip\": \"10.10.10.10\""},"entityHash":["srcHost.ip","dstHost.port"],"unifier":[{"name":"id","type":"string","expression":"auditEventLog"},{"name":"date","type":"timestamp","expression":"datetime"},{"name":"ipaddr","type":"string","expression":"srcHost.ip"},{"name":"category","type":"string","expression":"cat"}],"extraProcess":[{"func":"__if","args":"category, /Host/Connect/Host/Accept, high","to":"category"},{"func":"__stringConstant","args":"test","to":"customString1"}],"topicTo":"test"}'::json, NULL); COMMIT; \ No newline at end of file diff --git a/internal/adapter/store/postgres/postgres.go b/internal/adapter/store/postgres/postgres.go index fe9ed65..0423be7 100644 --- a/internal/adapter/store/postgres/postgres.go +++ b/internal/adapter/store/postgres/postgres.go @@ -21,7 +21,20 @@ type DataBase struct { pool *pgxpool.Pool } -var ErrUserUniq = errors.New("such a user exists") +type ErrUserUniq struct { + Login string + Err error +} + +func (e *ErrUserUniq) Error() string { + return fmt.Sprintf("unique violation for login: %s", e.Login) +} + +func (e *ErrUserUniq) Unwrap() error { + return e.Err +} + +// var ErrUserUniq = errors.New("such a user exists") func NewDB(ctx context.Context, dsn string, lg *zap.Logger) (DataBase, error) { pool, err := connection(ctx, dsn) @@ -81,11 +94,11 @@ func (db DataBase) CreateUser(ctx context.Context, user models.User) (int, error err := row.Scan(&id) if err != nil { var pgErr *pgconn.PgError - if !(errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation) { - return 0, fmt.Errorf("unique violation: %w", err) + if errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation { + return 0, &ErrUserUniq{Login: user.Login, Err: err} } - return 0, ErrUserUniq + return 0, fmt.Errorf("failed scan row: %w", err) } return id, nil diff --git a/internal/models/model.go b/internal/models/model.go index 9d86058..6533e7c 100644 --- a/internal/models/model.go +++ b/internal/models/model.go @@ -9,7 +9,7 @@ type User struct { type Rule struct { ID int `json:"id"` Rule Config `json:"rule"` - Owner int `json:"owner"` + Owner *int `json:"owner"` } type Config struct { From eed3272230be76645b72d302b016878ca468e840 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Tue, 30 Jan 2024 17:27:54 +0300 Subject: [PATCH 06/14] fix: fixed golangci-lint --- internal/adapter/api/rest/user.go | 2 +- internal/adapter/store/postgres/postgres.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/adapter/api/rest/user.go b/internal/adapter/api/rest/user.go index b72b169..0f144f2 100644 --- a/internal/adapter/api/rest/user.go +++ b/internal/adapter/api/rest/user.go @@ -52,7 +52,7 @@ func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { id, err := h.Store.CreateUser(req.Context(), models.User{Login: *pBody.Login, Hash: string(hash)}) if err != nil { - var userUniqErr *postgres.ErrUserUniq + var userUniqErr *postgres.UserUniqError if errors.As(err, &userUniqErr) { http.Error(res, err.Error(), http.StatusBadRequest) return diff --git a/internal/adapter/store/postgres/postgres.go b/internal/adapter/store/postgres/postgres.go index 0423be7..8b95b5c 100644 --- a/internal/adapter/store/postgres/postgres.go +++ b/internal/adapter/store/postgres/postgres.go @@ -21,21 +21,19 @@ type DataBase struct { pool *pgxpool.Pool } -type ErrUserUniq struct { +type UserUniqError struct { Login string Err error } -func (e *ErrUserUniq) Error() string { +func (e *UserUniqError) Error() string { return fmt.Sprintf("unique violation for login: %s", e.Login) } -func (e *ErrUserUniq) Unwrap() error { +func (e *UserUniqError) Unwrap() error { return e.Err } -// var ErrUserUniq = errors.New("such a user exists") - func NewDB(ctx context.Context, dsn string, lg *zap.Logger) (DataBase, error) { pool, err := connection(ctx, dsn) if err != nil { @@ -95,7 +93,7 @@ func (db DataBase) CreateUser(ctx context.Context, user models.User) (int, error if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation { - return 0, &ErrUserUniq{Login: user.Login, Err: err} + return 0, &UserUniqError{Login: user.Login, Err: err} } return 0, fmt.Errorf("failed scan row: %w", err) From 62907238a541a552f6cc049715b3f355a3e6e930 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 14:02:04 +0300 Subject: [PATCH 07/14] add: updated tests --- .gitignore | 1 + internal/adapter/api/rest/rules.go | 2 +- internal/adapter/api/rest/user.go | 3 +- internal/adapter/api/router/router_test.go | 100 +++++++- internal/adapter/api/util/util_test.go | 47 ++++ internal/adapter/store/postgres/postgres.go | 15 +- .../adapter/store/postgres/postgres_test.go | 230 ++++++++++++++++++ internal/adapter/store/store_test.go | 43 ++++ internal/logger/logger_test.go | 28 +++ 9 files changed, 451 insertions(+), 18 deletions(-) create mode 100644 internal/adapter/api/util/util_test.go create mode 100644 internal/adapter/store/postgres/postgres_test.go create mode 100644 internal/adapter/store/store_test.go create mode 100644 internal/logger/logger_test.go diff --git a/.gitignore b/.gitignore index 07bafad..d3164ea 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ golangci-lint # Test binary, built with `go test -c` *.test +profile.cov # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/internal/adapter/api/rest/rules.go b/internal/adapter/api/rest/rules.go index ab46579..ece7500 100644 --- a/internal/adapter/api/rest/rules.go +++ b/internal/adapter/api/rest/rules.go @@ -102,7 +102,7 @@ func (h RulesHandler) DeleteRule(res http.ResponseWriter, req *http.Request) { return } - if *dr.Owner != token.ID { + if dr.Owner == nil || *dr.Owner != token.ID { http.Error(res, "forbidden", http.StatusForbidden) return } diff --git a/internal/adapter/api/rest/user.go b/internal/adapter/api/rest/user.go index 0f144f2..ea2e95f 100644 --- a/internal/adapter/api/rest/user.go +++ b/internal/adapter/api/rest/user.go @@ -52,8 +52,7 @@ func (h UserHandler) Register(res http.ResponseWriter, req *http.Request) { id, err := h.Store.CreateUser(req.Context(), models.User{Login: *pBody.Login, Hash: string(hash)}) if err != nil { - var userUniqErr *postgres.UserUniqError - if errors.As(err, &userUniqErr) { + if errors.Is(err, postgres.ErrUserUniq) { http.Error(res, err.Error(), http.StatusBadRequest) return } diff --git a/internal/adapter/api/router/router_test.go b/internal/adapter/api/router/router_test.go index 9c2cf69..6489170 100644 --- a/internal/adapter/api/router/router_test.go +++ b/internal/adapter/api/router/router_test.go @@ -117,6 +117,25 @@ func TestRouter(t *testing.T) { expectedCode: http.StatusOK, expectedBody: "", }, + { + name: "Register test user: invalid body", + method: http.MethodPost, + url: "/api/user/register", + body: map[string]interface{}{}, + expectedCode: http.StatusBadRequest, + expectedBody: "", + }, + { + name: "Register test user: login must be exist", + method: http.MethodPost, + url: "/api/user/register", + body: map[string]interface{}{ + "login": "test", + "password": "test", + }, + expectedCode: http.StatusBadRequest, + expectedBody: "failed to create a new user with login test: user already exists\n", + }, { name: "Login test user", method: http.MethodPost, @@ -128,6 +147,25 @@ func TestRouter(t *testing.T) { expectedCode: http.StatusOK, expectedBody: "", }, + { + name: "Login test user: invalid body", + method: http.MethodPost, + url: "/api/user/login", + body: map[string]interface{}{}, + expectedCode: http.StatusBadRequest, + expectedBody: "", + }, + { + name: "Login test1 user: login not exist", + method: http.MethodPost, + url: "/api/user/login", + body: map[string]interface{}{ + "login": "test1", + "password": "test1", + }, + expectedCode: http.StatusBadRequest, + expectedBody: "login or password incorrect\n", + }, { name: "Get all rules", method: http.MethodGet, @@ -135,7 +173,15 @@ func TestRouter(t *testing.T) { url: "/api/rules", expectedCode: http.StatusOK, //nolint:lll // This legal size - expectedBody: "[{\"id\":1,\"rule\":{\"topicFrom\":\"events\",\"filter\":{\"regexp\":\"\\\"dstHost.ip\\\": \\\"10.10.10.10\\\"\"},\"entityHash\":[\"srcHost.ip\",\"dstHost.port\"],\"unifier\":[{\"name\":\"id\",\"type\":\"string\",\"expression\":\"auditEventLog\"},{\"name\":\"date\",\"type\":\"timestamp\",\"expression\":\"datetime\"},{\"name\":\"ipaddr\",\"type\":\"string\",\"expression\":\"srcHost.ip\"},{\"name\":\"category\",\"type\":\"string\",\"expression\":\"cat\"}],\"extraProcess\":[{\"func\":\"__if\",\"args\":\"category, /Host/Connect/Host/Accept, high\",\"to\":\"category\"},{\"func\":\"__stringConstant\",\"args\":\"test\",\"to\":\"customString1\"}],\"topicTo\":\"test\"},\"owner\":1}]\n", + expectedBody: "[{\"id\":1,\"rule\":{\"topicFrom\":\"events\",\"filter\":{\"regexp\":\"\\\"dstHost.ip\\\": \\\"10.10.10.10\\\"\"},\"entityHash\":[\"srcHost.ip\",\"dstHost.port\"],\"unifier\":[{\"name\":\"id\",\"type\":\"string\",\"expression\":\"auditEventLog\"},{\"name\":\"date\",\"type\":\"timestamp\",\"expression\":\"datetime\"},{\"name\":\"ipaddr\",\"type\":\"string\",\"expression\":\"srcHost.ip\"},{\"name\":\"category\",\"type\":\"string\",\"expression\":\"cat\"}],\"extraProcess\":[{\"func\":\"__if\",\"args\":\"category, /Host/Connect/Host/Accept, high\",\"to\":\"category\"},{\"func\":\"__stringConstant\",\"args\":\"test\",\"to\":\"customString1\"}],\"topicTo\":\"test\"},\"owner\":null}]\n", + }, + { + name: "Get all rules: token unauth", + method: http.MethodGet, + authorization: false, + url: "/api/rules", + expectedCode: http.StatusUnauthorized, + expectedBody: "", }, { name: "Add new rule", @@ -148,14 +194,64 @@ func TestRouter(t *testing.T) { expectedCode: http.StatusOK, expectedBody: "", }, + { + name: "Add new rule: invalid body", + method: http.MethodPost, + authorization: true, + url: "/api/rules", + body: nil, + expectedCode: http.StatusBadRequest, + expectedBody: "", + }, + { + name: "Add new rule: token unauth", + method: http.MethodPost, + authorization: false, + url: "/api/rules", + body: nil, + expectedCode: http.StatusUnauthorized, + expectedBody: "", + }, { name: "Remove rule", method: http.MethodDelete, authorization: true, - url: "/api/rules/1", + url: "/api/rules/2", expectedCode: http.StatusOK, expectedBody: "", }, + { + name: "Remove rule: token unauth", + method: http.MethodDelete, + authorization: false, + url: "/api/rules/2", + expectedCode: http.StatusUnauthorized, + expectedBody: "", + }, + { + name: "Remove rule: invalid id", + method: http.MethodDelete, + authorization: true, + url: "/api/rules/sd", + expectedCode: http.StatusBadRequest, + expectedBody: "", + }, + { + name: "Remove rule: rule id not exist", + method: http.MethodDelete, + authorization: true, + url: "/api/rules/5", + expectedCode: http.StatusNotFound, + expectedBody: "", + }, + { + name: "Remove rule: not owner rule", + method: http.MethodDelete, + authorization: true, + url: "/api/rules/1", + expectedCode: http.StatusForbidden, + expectedBody: "", + }, } // Создаем логер diff --git a/internal/adapter/api/util/util_test.go b/internal/adapter/api/util/util_test.go new file mode 100644 index 0000000..02643fa --- /dev/null +++ b/internal/adapter/api/util/util_test.go @@ -0,0 +1,47 @@ +package util + +import ( + "context" + "testing" + + "github.com/dedpnd/unifier/internal/core/auth" + "github.com/stretchr/testify/assert" +) + +func TestSetTokenToContext(t *testing.T) { + // Создаем контекст + ctx := context.Background() + + // Создаем некоторый тестовый токен + testToken := auth.Claims{ + // ваш токен + } + + // Вызываем функцию SetTokenToContext + ctxWithToken := SetTokenToContext(ctx, testToken) + + // Проверяем, что токен успешно установлен в контексте + tokenFromContext, ok := GetTokenFromContext(ctxWithToken) + assert.True(t, ok) + assert.Equal(t, testToken, tokenFromContext) +} + +func TestGetTokenFromContext(t *testing.T) { + // Создаем контекст + ctx := context.Background() + + // Создаем некоторый тестовый токен + testToken := auth.Claims{ + // ваш токен + } + + // Устанавливаем токен в контекст + ctxWithToken := context.WithValue(ctx, ContextKeyToken, testToken) + + // Вызываем функцию GetTokenFromContext + tokenFromContext, ok := GetTokenFromContext(ctxWithToken) + + // Проверяем, что токен успешно получен из контекста + assert.True(t, ok) + assert.Equal(t, testToken, tokenFromContext) +} diff --git a/internal/adapter/store/postgres/postgres.go b/internal/adapter/store/postgres/postgres.go index 8b95b5c..e771502 100644 --- a/internal/adapter/store/postgres/postgres.go +++ b/internal/adapter/store/postgres/postgres.go @@ -21,18 +21,7 @@ type DataBase struct { pool *pgxpool.Pool } -type UserUniqError struct { - Login string - Err error -} - -func (e *UserUniqError) Error() string { - return fmt.Sprintf("unique violation for login: %s", e.Login) -} - -func (e *UserUniqError) Unwrap() error { - return e.Err -} +var ErrUserUniq = errors.New("user already exists") func NewDB(ctx context.Context, dsn string, lg *zap.Logger) (DataBase, error) { pool, err := connection(ctx, dsn) @@ -93,7 +82,7 @@ func (db DataBase) CreateUser(ctx context.Context, user models.User) (int, error if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation { - return 0, &UserUniqError{Login: user.Login, Err: err} + return 0, fmt.Errorf("failed to create a new user with login %s: %w", user.Login, ErrUserUniq) } return 0, fmt.Errorf("failed scan row: %w", err) diff --git a/internal/adapter/store/postgres/postgres_test.go b/internal/adapter/store/postgres/postgres_test.go new file mode 100644 index 0000000..21829a8 --- /dev/null +++ b/internal/adapter/store/postgres/postgres_test.go @@ -0,0 +1,230 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "log" + "os" + "testing" + "time" + + "github.com/dedpnd/unifier/internal/models" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +var databaseURL string +var db DataBase + +func TestMain(m *testing.M) { + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "16.1-alpine3.18", + Env: []string{ + "POSTGRES_PASSWORD=test", + "POSTGRES_USER=test", + "listen_addresses = '*'", + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + hostAndPort := resource.GetHostPort("5432/tcp") + databaseURL = fmt.Sprintf("postgres://test:test@%s?sslmode=disable", hostAndPort) + + log.Println("Connecting to database on url: ", databaseURL) + + // Tell docker to hard kill the container in 120 seconds + err = resource.Expire(120) + if err != nil { + log.Fatalf("Expire resource has error: %s", err) + } + + var sqlDB *sql.DB + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + pool.MaxWait = 20 * time.Second + if err = pool.Retry(func() error { + sqlDB, err = sql.Open("postgres", databaseURL) + if err != nil { + return fmt.Errorf("Connection has error: %w", err) + } + + err = sqlDB.Ping() + if err != nil { + return fmt.Errorf("Ping has error: %w", err) + } + + return nil + }); err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + // Создаем подключение к тестируемой базе данных + ctx := context.Background() + db, err = NewDB(ctx, databaseURL, zap.NewNop()) + if err != nil { + log.Fatalf("Could not create new database: %s", err) + } + + // Run tests + code := m.Run() + + // You can't defer this because os.Exit doesn't care for defer + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + + err = db.Close() + if err != nil { + log.Fatalf("Could not close db: %s", err) + } + + os.Exit(code) +} + +func TestNewDB_SuccessfulConnection(t *testing.T) { + ctx := context.Background() + db, err := NewDB(ctx, databaseURL, zap.NewNop()) + assert.NoError(t, err) + assert.NotNil(t, db) +} + +func TestClose(t *testing.T) { + ctx := context.Background() + db, err := NewDB(ctx, databaseURL, zap.NewNop()) + if err != nil { + log.Fatalf("Could not create new database: %s", err) + } + + // Закрываем базу данных + err = db.Close() + + // Проверяем, что ошибки нет + assert.NoError(t, err) +} + +func TestRunMigrations_SuccessfulMigration(t *testing.T) { + err := runMigrations(databaseURL) + assert.NoError(t, err) +} + +func TestCreateUser(t *testing.T) { + ctx := context.Background() + // Создаем пользователя для теста + testUser := models.User{ + Login: "testuser", + Hash: "hash123", + } + + // Вызываем функцию, которую тестируем + id, err := db.CreateUser(ctx, testUser) + assert.NoError(t, err) + assert.NotEqual(t, 0, id) +} + +func TestGetUserByLogin(t *testing.T) { + ctx := context.Background() + + // Вызываем функцию, которую тестируем + user, err := db.GetUserByLogin(ctx, "testuser") + assert.NoError(t, err) + assert.Equal(t, "testuser", user.Login) + assert.Equal(t, "hash123", user.Hash) +} + +func TestCreateRule(t *testing.T) { + ctx := context.Background() + + // Создаем тестовое правило + testRule := models.Config{ + TopicFrom: "events", + } + + // Создаем пользователя, который будет владельцем правила + ownerID, err := db.CreateUser(ctx, models.User{Login: "testowner", Hash: "hash123"}) + assert.NoError(t, err) + + // Вызываем функцию, которую тестируем + ruleID, err := db.CreateRule(ctx, testRule, ownerID) + assert.NoError(t, err) + assert.NotEqual(t, 0, ruleID) +} + +func TestGetRuleByID(t *testing.T) { + ctx := context.Background() + + // Создаем тестовое правило + testRule := models.Config{ + TopicFrom: "events", + } + + // Создаем пользователя, который будет владельцем правила + ownerID, err := db.CreateUser(ctx, models.User{Login: "testowner1", Hash: "hash123"}) + assert.NoError(t, err) + + // Создаем правило + ruleID, err := db.CreateRule(ctx, testRule, ownerID) + assert.NoError(t, err) + + // Вызываем функцию, которую тестируем + rule, err := db.GetRuleByID(ctx, ruleID) + assert.NoError(t, err) + assert.NotEqual(t, 0, rule.ID) + assert.Equal(t, testRule, rule.Rule) + assert.Equal(t, ownerID, *rule.Owner) +} + +func TestDeleteRule(t *testing.T) { + ctx := context.Background() + + // Создаем тестовое правило + testRule := models.Config{ + TopicFrom: "events", + } + + // Создаем пользователя, который будет владельцем правила + ownerID, err := db.CreateUser(ctx, models.User{Login: "testowner3", Hash: "hash123"}) + assert.NoError(t, err) + + // Создаем правило + ruleID, err := db.CreateRule(ctx, testRule, ownerID) + assert.NoError(t, err) + + // Вызываем функцию, которую тестируем + err = db.DeleteRule(ctx, ruleID) + assert.NoError(t, err) + + // Пытаемся получить правило после удаления + r, err := db.GetRuleByID(ctx, ruleID) + assert.NoError(t, err) + assert.Equal(t, r, models.Rule{}) +} + +func TestGetAllRules(t *testing.T) { + ctx := context.Background() + + // Вызываем функцию, которую тестируем + rules, err := db.GetAllRules(ctx) + assert.NoError(t, err) + assert.NotEmpty(t, rules) +} diff --git a/internal/adapter/store/store_test.go b/internal/adapter/store/store_test.go new file mode 100644 index 0000000..e359519 --- /dev/null +++ b/internal/adapter/store/store_test.go @@ -0,0 +1,43 @@ +package store + +import ( + "reflect" + "testing" + + "go.uber.org/zap" +) + +func TestNewStore(t *testing.T) { + type args struct { + dsn string + lg *zap.Logger + } + tests := []struct { + name string + args args + want Storage + wantErr bool + }{ + { + name: "Storage must be return error", + args: args{ + dsn: "test", + lg: zap.NewNop(), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewStore(tt.args.dsn, tt.args.lg) + if (err != nil) != tt.wantErr { + t.Errorf("NewStore() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewStore() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go new file mode 100644 index 0000000..b60c7a1 --- /dev/null +++ b/internal/logger/logger_test.go @@ -0,0 +1,28 @@ +package logger + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestInit(t *testing.T) { + // Передаем уровень логирования в функцию Init + level := "info" + l, err := Init(level) + + // Проверяем, что функция не вернула ошибку + assert.NoError(t, err) + // Проверяем, что логгер был успешно создан + assert.NotNil(t, l) + + assert.Equal(t, l.Core().Enabled(zap.InfoLevel), true) + + level = "warn" + l, err = Init(level) + assert.NoError(t, err) + assert.NotNil(t, l) + assert.Equal(t, l.Core().Enabled(zap.InfoLevel), false) + assert.Equal(t, l.Core().Enabled(zap.WarnLevel), true) +} From 8bfce88133f0eb736cbd4cde2fdd0cfa48383769 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 14:35:17 +0300 Subject: [PATCH 08/14] add: added test coverage badge --- .github/workflows/coverage-badge-go.yaml | 53 ++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/coverage-badge-go.yaml diff --git a/.github/workflows/coverage-badge-go.yaml b/.github/workflows/coverage-badge-go.yaml new file mode 100644 index 0000000..9932ea2 --- /dev/null +++ b/.github/workflows/coverage-badge-go.yaml @@ -0,0 +1,53 @@ +name: Generate code coverage badge + +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + name: Update coverage badge + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token. + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + + - name: Setup go + uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + + - name: Run Test + run: | + go test -v ./... -covermode=count -coverprofile=coverage.out + go tool cover -func=coverage.out -o=coverage.out + + - name: Go Coverage Badge # Pass the `coverage.out` output to this action + uses: tj-actions/coverage-badge-go@v2 + with: + filename: coverage.out + + - name: Verify Changed files + uses: tj-actions/verify-changed-files@v16 + id: verify-changed-files + with: + files: README.md + + - name: Commit changes + if: steps.verify-changed-files.outputs.files_changed == 'true' + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add README.md + git commit -m "chore: Updated coverage badge." + + - name: Push changes + if: steps.verify-changed-files.outputs.files_changed == 'true' + uses: ad-m/github-push-action@master + with: + github_token: ${{ github.token }} + branch: ${{ github.head_ref }} \ No newline at end of file From f155ea4d7f81e7f65a3e158046cbb98ad4b704e2 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 14:41:55 +0300 Subject: [PATCH 09/14] fix: add services for github actions --- .github/workflows/coverage-badge-go.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/coverage-badge-go.yaml b/.github/workflows/coverage-badge-go.yaml index 9932ea2..1e21ac0 100644 --- a/.github/workflows/coverage-badge-go.yaml +++ b/.github/workflows/coverage-badge-go.yaml @@ -8,6 +8,11 @@ on: jobs: test: runs-on: ubuntu-latest + services: + dind: + image: docker:23.0-rc-dind-rootless + ports: + - 2375:2375 name: Update coverage badge steps: - name: Checkout From da381b02c46b2df671e0f074dffd1037bcbf2927 Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 14:56:03 +0300 Subject: [PATCH 10/14] fix: fix test for worker --- internal/core/worker/worker.go | 2 +- internal/core/worker/worker_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/core/worker/worker.go b/internal/core/worker/worker.go index 4397261..8887f8b 100644 --- a/internal/core/worker/worker.go +++ b/internal/core/worker/worker.go @@ -158,7 +158,7 @@ func unificationFields(event map[string]interface{}, cfgUnifier []models.Unifier if err != nil { return fmt.Errorf("failed date parse: %w", err) } - (*uEvent)[u.Name] = v.String() + (*uEvent)[u.Name] = v.Format("2006-01-02 15:04:05") } default: return fmt.Errorf("unknown type: %v", u.Type) diff --git a/internal/core/worker/worker_test.go b/internal/core/worker/worker_test.go index 52edf4b..2676f68 100644 --- a/internal/core/worker/worker_test.go +++ b/internal/core/worker/worker_test.go @@ -164,7 +164,7 @@ func Test_unificationFields(t *testing.T) { }, want: want{ key: "testTime", - value: "2023-07-13 13:47:43 +0000 +0000", + value: "2023-07-13 13:47:43", }, wantErr: false, }, From 1c912ce5f8653e3c8ffc6892096583f800be60ef Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 15:16:45 +0300 Subject: [PATCH 11/14] fix: fix test for worker --- internal/core/worker/worker.go | 3 ++- internal/core/worker/worker_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/core/worker/worker.go b/internal/core/worker/worker.go index 8887f8b..a3df8d5 100644 --- a/internal/core/worker/worker.go +++ b/internal/core/worker/worker.go @@ -158,7 +158,8 @@ func unificationFields(event map[string]interface{}, cfgUnifier []models.Unifier if err != nil { return fmt.Errorf("failed date parse: %w", err) } - (*uEvent)[u.Name] = v.Format("2006-01-02 15:04:05") + + (*uEvent)[u.Name] = v.Format(time.RFC3339) } default: return fmt.Errorf("unknown type: %v", u.Type) diff --git a/internal/core/worker/worker_test.go b/internal/core/worker/worker_test.go index 2676f68..4a8c2f1 100644 --- a/internal/core/worker/worker_test.go +++ b/internal/core/worker/worker_test.go @@ -164,7 +164,7 @@ func Test_unificationFields(t *testing.T) { }, want: want{ key: "testTime", - value: "2023-07-13 13:47:43", + value: "2023-07-13T13:47:43Z", }, wantErr: false, }, From fd53bcc62c7baf9081fdda374b6cc7f8f682a276 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 2 Feb 2024 12:18:10 +0000 Subject: [PATCH 12/14] chore: Updated coverage badge. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 61a7ec0..4e4c17b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Как запустить ? +![Coverage](https://img.shields.io/badge/Coverage-59.6%25-yellow) Все необходимые службы для работы находятся в docker-compose From 42b1b2eb619ddb3cc79400c76bbf90a71e7fc21b Mon Sep 17 00:00:00 2001 From: Balyuk Dmitrii Date: Fri, 2 Feb 2024 18:12:14 +0300 Subject: [PATCH 13/14] fix: fixed test actions --- .github/workflows/coverage-badge-go.yaml | 4 ++-- .gitignore | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage-badge-go.yaml b/.github/workflows/coverage-badge-go.yaml index 1e21ac0..10827b3 100644 --- a/.github/workflows/coverage-badge-go.yaml +++ b/.github/workflows/coverage-badge-go.yaml @@ -28,8 +28,8 @@ jobs: - name: Run Test run: | - go test -v ./... -covermode=count -coverprofile=coverage.out - go tool cover -func=coverage.out -o=coverage.out + go test -v -count=1 -coverpkg=./... -coverprofile=profile.cov ./... + go tool cover -func=profile.cov -o=coverage.out - name: Go Coverage Badge # Pass the `coverage.out` output to this action uses: tj-actions/coverage-badge-go@v2 diff --git a/.gitignore b/.gitignore index d3164ea..3d3966b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ golangci-lint # Test binary, built with `go test -c` *.test profile.cov +coverage.out # Output of the go coverage tool, specifically when used with LiteIDE *.out From 09bf57187897b1ae5bc185cb70765eccf2240fce Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 2 Feb 2024 15:13:39 +0000 Subject: [PATCH 14/14] chore: Updated coverage badge. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e4c17b..9ba5433 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Как запустить ? -![Coverage](https://img.shields.io/badge/Coverage-59.6%25-yellow) +![Coverage](https://img.shields.io/badge/Coverage-70.3%25-brightgreen) Все необходимые службы для работы находятся в docker-compose