From 82cc883694f3f99ddfc640c47ec25978d828f240 Mon Sep 17 00:00:00 2001 From: Kervin Hu <34271744+YikaiHu@users.noreply.github.com> Date: Wed, 23 Aug 2023 08:39:08 +0800 Subject: [PATCH] Update to version v2.0.0 (#186) * Update to version v2.0.0 * fix: fix the gh-page workflow --- .github/workflows/gh-page.yml | 10 +- CHANGELOG.md | 63 +- NOTICE.txt | 119 +- README.md | 22 +- deployment/build-s3-dist.sh | 25 - deployment/cdk-solution-helper/package.json | 2 + deployment/run-unit-tests.sh | 11 + docs/en/images | 0 docs/en/implementation-guide/alarm.md | 37 + .../applications/apache.md | 34 +- .../applications/create-applog-pipeline.md | 226 +- .../applications/create-log-ingestion.md | 70 +- .../applications/create-log-source.md | 27 +- .../applications/index.md | 28 +- .../implementation-guide/applications/json.md | 6 +- .../applications/multi-line-text.md | 28 +- .../applications/nginx.md | 31 +- .../applications/single-line-text.md | 2 +- .../applications/syslog.md | 4 +- .../services-in-this-solution.md | 18 + .../solution-components.md | 10 +- .../well-architected-pillars.md | 54 + docs/en/implementation-guide/architecture.md | 28 +- .../aws-services/cloudfront.md | 53 +- .../aws-services/cloudtrail.md | 28 +- .../aws-services/config.md | 30 +- .../implementation-guide/aws-services/elb.md | 39 +- .../aws-services/include-cfn-common.md | 53 +- .../include-cfn-plugins-common.md | 54 +- .../aws-services/include-cw-cfn-common.md | 54 +- .../aws-services/index.md | 34 + .../aws-services/lambda.md | 34 +- .../implementation-guide/aws-services/rds.md | 40 +- .../implementation-guide/aws-services/s3.md | 29 +- .../implementation-guide/aws-services/vpc.md | 40 +- .../implementation-guide/aws-services/waf.md | 36 +- docs/en/implementation-guide/cost.md | 156 - .../implementation-guide/deployment/index.md | 4 +- .../deployment/with-cognito.md | 3 +- .../deployment/with-oidc.md | 2 +- .../en/implementation-guide/domains/alarms.md | 8 +- docs/en/implementation-guide/domains/proxy.md | 10 +- docs/en/implementation-guide/faq.md | 14 +- .../{aws-services => }/include-dashboard.md | 2 +- .../link-account/index.md | 38 +- docs/en/implementation-guide/monitoring.md | 81 + .../{ => plan-deployment}/considerations.md | 3 - .../plan-deployment/cost.md | 215 + .../plan-deployment/security.md | 36 + docs/en/implementation-guide/release-notes.md | 11 +- .../resources/aws-services.md | 15 - .../resources/open-ssl.md | 4 +- docs/en/implementation-guide/security.md | 25 - .../solution-overview/features.md | 10 + .../solution-overview/use-cases.md | 13 + docs/en/implementation-guide/source.md | 5 + .../implementation-guide/trouble-shooting.md | 38 +- docs/en/index.md | 22 +- docs/images/dashboards/apache.png | Bin 0 -> 611989 bytes docs/mkdocs.en.yml | 64 +- docs/mkdocs.zh.yml | 26 +- docs/zh/images | 0 docs/zh/implementation-guide/alarm.md | 37 + .../applications/apache.md | 33 +- .../applications/create-applog-pipeline.md | 260 +- .../applications/create-log-ingestion.md | 60 +- .../applications/create-log-source.md | 16 +- .../implementation-guide/applications/json.md | 2 +- .../applications/multi-line-text.md | 22 +- .../applications/nginx.md | 33 +- .../services-in-this-solution.md | 16 + .../solution-components.md | 17 + .../well-architected-pillars.md | 53 + docs/zh/implementation-guide/architecture.md | 7 +- .../aws-services/cloudfront.md | 37 +- .../aws-services/cloudtrail.md | 29 +- .../aws-services/config.md | 30 +- .../implementation-guide/aws-services/elb.md | 37 +- .../aws-services/index.md | 33 + .../aws-services/lambda.md | 35 +- .../implementation-guide/aws-services/rds.md | 27 +- .../implementation-guide/aws-services/s3.md | 29 +- .../implementation-guide/aws-services/vpc.md | 39 +- .../implementation-guide/aws-services/waf.md | 35 +- .../implementation-guide/deployment/index.md | 4 +- .../deployment/with-oidc.md | 1 - .../zh/implementation-guide/domains/alarms.md | 8 +- .../zh/implementation-guide/domains/import.md | 8 +- docs/zh/implementation-guide/domains/proxy.md | 12 +- docs/zh/implementation-guide/faq.md | 4 +- .../implementation-guide/include-dashboard.md | 3 + .../link-account/index.md | 9 +- docs/zh/implementation-guide/monitoring.md | 63 + .../{ => plan-deployment}/considerations.md | 0 .../{ => plan-deployment}/cost.md | 8 +- .../plan-deployment/security.md | 28 + docs/zh/implementation-guide/release-notes.md | 3 +- .../resources/aws-services.md | 15 - .../resources/open-ssl.md | 8 +- docs/zh/implementation-guide/security.md | 21 - .../solution-overview/features.md | 7 + .../solution-overview/use-cases.md | 13 + docs/zh/implementation-guide/source.md | 5 + .../implementation-guide/trouble-shooting.md | 24 + docs/zh/index.md | 15 +- source/constructs/bin/main.ts | 195 +- .../ecr/s3-list-objects/.dockerignore | 23 + .../constructs/ecr/s3-list-objects/.gitignore | 1 + .../constructs/ecr/s3-list-objects/Dockerfile | 25 + .../constructs/ecr/s3-list-objects/build.sh | 9 + .../ecr/s3-list-objects/requirements.txt | 1 + .../ecr/s3-list-objects/s3_list_objects.py | 245 + .../s3-list-objects/test}/__init__.py | 0 .../s3-list-objects}/test/conftest.py | 9 +- .../test/requirements-test.txt | 2 +- .../test/test_s3_list_objects.py | 137 + source/constructs/ecr/syslog/Dockerfile | 2 +- source/constructs/graphql/schema.graphql | 828 ++- .../CreateAppLogIngestion.vtl | 13 +- .../app_log_ingestion/GetEKSDaemonSetConf.vtl | 2 - .../GetEKSDeploymentConf.vtl | 4 - .../GetK8sDeploymentContentWithDaemonSet.vtl | 3 + .../GetK8sDeploymentContentWithSidecar.vtl | 2 + .../cross_account/DeleteSubAccountLink.vtl | 5 +- .../vtl/cross_account/GetSubAccountLink.vtl | 5 + .../GetInstanceAgentStatus.vtl | 8 + .../GetInstanceAgentStatusResp.vtl | 6 + .../vtl/instance_status/ListInstances.vtl | 5 + .../RequestInstallLogAgent.vtl | 7 + ...{UpdateLogConf.vtl => CreateLogConfig.vtl} | 17 +- .../graphql/vtl/log_conf/GetLogConfResp.vtl | 15 - .../graphql/vtl/log_conf/GetLogConfigResp.vtl | 16 + .../{ListLogConfs.vtl => ListLogConfigs.vtl} | 0 .../vtl/log_conf/ListLogConfigsResp.vtl | 18 + .../graphql/vtl/log_conf/ListLogConfsResp.vtl | 17 - ...{CreateLogConf.vtl => UpdateLogConfig.vtl} | 18 +- .../api/{instance_meta => alarm}/.coveragerc | 1 + .../lambda/api/alarm/lambda_function.py | 85 + .../test}/__init__.py | 0 .../lambda/api/alarm/test/conftest.py | 32 + .../api/alarm/test/requirements-test.txt | 3 + .../api/alarm/test/test_lambda_function.py | 477 ++ .../lambda/api/alarm/util/alarm_helper.py | 527 ++ ...to_scaling_group_config_lambda_function.py | 56 +- .../api/app_log_ingestion/aws_svc_mgr.py | 176 - .../lambda/api/app_log_ingestion/common.py | 18 - .../ec2_as_source_lambda_function.py | 196 - ...tion_distribution_event_lambda_function.py | 174 + ...aemonset_sidecar_config_lambda_function.py | 65 - .../flb}/__init__.py | 0 .../api/app_log_ingestion/flb/distribution.py | 212 + .../api/app_log_ingestion/flb/flb_builder.py | 435 ++ .../api/app_log_ingestion/flb/flb_model.py | 206 + .../flb/flb_template/k8s-DaemonSet.conf | 260 + .../flb/flb_template/k8s-Sidecar.conf | 130 + .../flb/flb_template/parser.conf | 46 + .../flb/flb_template/pipeline.conf | 250 + .../lambda/api/app_log_ingestion/flb/k8s.py | 47 + ...tion_modification_event_lambda_function.py | 179 +- .../api/app_log_ingestion/lambda_function.py | 439 +- .../api/app_log_ingestion/requirements.txt | 3 + .../lambda/api/app_log_ingestion/svc/ec2.py | 313 + .../svc/ec2_attach_iam_instance_profile.py | 153 + .../lambda/api/app_log_ingestion/svc/k8s.py | 135 + .../lambda/api/app_log_ingestion/svc/s3.py | 173 + .../api/app_log_ingestion/svc/service.py | 250 + .../api/app_log_ingestion/svc/syslog.py | 213 + .../syslog_as_source_lambda_function.py | 84 - .../api/app_log_ingestion/test/conftest.py | 15 +- .../test/datafile/ddb_mock_data.py | 100 +- .../test/datafile/document_content.json | 4 +- ...bit_config_TestLogIngestionSvc_task_1.conf | 2 +- ...bit_config_TestLogIngestionSvc_task_2.conf | 4 +- ...bit_config_TestLogIngestionSvc_task_3.conf | 4 +- ...bit_config_TestLogIngestionSvc_task_4.conf | 2 +- .../test/datafile/td-agent_ground_truth.conf | 52 - ...nce_to_instance_group_duplicate_event.json | 4 +- .../add_instance_to_instance_group_event.json | 20 - ...app_log_ingestion_for_single_instance.json | 37 + ...ngestion_and_refresh_flb_config_event.json | 247 + ...e_instances_from_instance_group_event.json | 20 - .../generate_and_upload_flb_config_event.json | 247 + ...app_log_ingestion_for_single_instance.json | 26 + .../test/event/test_event.json | 40 + .../flb/flb_template/k8s-DaemonSet.conf} | 160 +- .../test/flb/flb_template/k8s-Sidecar.conf | 142 + .../test/flb/flb_template/parser.conf | 44 + .../test/flb/flb_template/pipeline.conf | 235 + .../test/requirements-test.txt | 6 +- .../test/test_agent_role_mgr.py | 197 - .../test/test_asg_config_lambda_function.py | 222 - .../test/test_assume_role.py | 19 - ...to_scaling_group_config_lambda_function.py | 59 + .../test/test_ddb_connect.py | 219 - .../test/test_distribute_config_helper.py | 68 - .../test_ec2_as_source_lambda_function.py | 442 -- .../test_ec2_attach_iam_instance_profile.py | 411 ++ ...tion_distribution_event_lambda_function.py | 230 + .../app_log_ingestion/test/test_ec2_source.py | 170 + .../test/test_eks_constants.py | 6 +- ...aemonset_sidecar_config_lambda_function.py | 490 -- .../api/app_log_ingestion/test/test_flb.py | 339 ++ .../test/test_fluentbit_config_helper.py | 216 - ..._ingestion_modification_lambda_function.py | 401 +- .../app_log_ingestion/test/test_k8s_source.py | 192 + .../test/test_lambda_function.py | 363 +- .../test/test_log_agent_helper.py | 397 -- .../test/test_log_agent_helper_v2.py | 410 -- .../app_log_ingestion/test/test_s3_source.py | 187 + .../app_log_ingestion/test/test_service.py | 624 ++ .../test_syslog_as_source_lambda_function.py | 149 - .../test/test_syslog_source.py | 123 + .../api/app_log_ingestion/test/test_utils.py | 36 + .../asg_ingestion_script.sh | 217 + .../app_log_ingestion/util/agent_role_mgr.py | 140 - .../api/app_log_ingestion/util/agent_type.py | 45 - .../util/asg_deployment_configuration.py | 55 +- .../asg_ingestion_script.sh | 12 +- .../api/app_log_ingestion/util/assume_role.py | 43 - .../distribute_config_helper.py | 64 - .../util/ec2_log_ingestion_svc.py | 212 - .../fluent-bit-configmap-aos.template.yaml | 62 - .../fluent-bit-configmap.template.yaml | 54 - .../fluent-bit-daemonset.template.docker.yaml | 81 - .../fluent-bit-grep-filter.template.yaml | 6 - ...nput-output-filter-daemonset.template.yaml | 34 - ...ent-bit-input-output-sidecar.template.yaml | 21 - .../fluent-bit-k8s-filter.yaml | 22 - .../fluent-bit-namespace.template.yaml | 6 - .../fluent-bit-output-aos.template.yaml | 20 - .../fluent-bit-output-kds.template.yaml | 14 - .../fluent-bit-output-msk.template.yaml | 11 - .../fluent-bit-output-s3.template.yaml | 16 - .../fluent-bit-role-binding.template.yaml | 33 - .../fluent-bit-service-account.template.yaml | 9 - .../fluent-bit-sidecar.template.yaml | 54 - ...ined-docker-firstline-parser.template.yaml | 4 - ...ser-defined-multiline-filter.template.yaml | 6 - ...ser-defined-multiline-parser.template.yaml | 20 - ...uent-bit-user-defined-parser.template.yaml | 8 - .../uniform-time-format.lua | 7 - .../util/eks_deployment_configuration.py | 328 -- .../api/app_log_ingestion/util/exception.py | 6 - .../fluentbit_config_helper/ddb_connect.py | 258 - .../util/fluentbit_config_helper/defaults.py | 110 - .../fluentbit_config_helper.py | 124 - .../log-agent-service.template | 2 +- .../log-agent-cross-account-kds.template | 4 - .../log-agent-input.template | 17 - .../log-agent-output.template | 17 - .../log-agent-parser-csv.template | 5 - .../log-agent-parser-json.template | 7 - .../log-agent-parser-regex.template | 5 - .../log-agent-service.template | 9 - .../util/log_agent_helper.py | 1420 ----- .../util/log_agent_helper_v2.py | 232 - .../util/log_ingestion_svc.py | 1145 ---- .../app_log_ingestion/util/sys_enum_type.py | 53 - .../api/app_log_ingestion/util/utils.py | 28 + .../lambda/api/app_pipeline/common.py | 18 - .../api/app_pipeline/lambda_function.py | 548 +- .../lambda/api/app_pipeline/test/conftest.py | 27 +- .../app_pipeline/test/requirements-test.txt | 4 +- .../app_pipeline/test/test_lambda_function.py | 642 ++- .../api/app_pipeline/test/test_utils.py | 210 + .../lambda/api/app_pipeline/util/utils.py | 132 + .../lambda/api/app_pipeline/util/validator.py | 118 - .../api/app_pipeline_flow/lambda_function.py | 119 - .../api/cluster/cluster_auto_import_mgr.py | 1016 ++-- .../lambda/api/cluster/lambda_function.py | 721 ++- .../test/test_cluster_auto_import_mgr.py | 254 + .../api/cluster/test/test_lambda_function.py | 217 +- .../util/cluster_status_check_helper.py | 122 + .../lambda/api/cluster/util/exception.py | 7 - .../lambda/api/common/aws_svc_mgr.py | 176 - source/constructs/lambda/api/common/common.py | 18 - .../constructs/lambda/api/common/exception.py | 7 - .../api/common/test/test_aws_svc_mgr.py | 116 - .../api/cross_account/lambda_function.py | 342 +- .../lambda/api/cross_account/test/conftest.py | 3 +- .../cross_account/test/requirements-test.txt | 6 +- .../test/test_lambda_function.py | 174 +- source/constructs/lambda/api/cwl/.coveragerc | 9 + .../lambda/api/cwl/lambda_function.py | 172 + .../api/{eks_cluster => cwl}/test/__init__.py | 0 .../lambda/api/cwl/test/conftest.py | 31 + .../test/requirements-test.txt | 1 - .../api/cwl/test/test_lambda_function.py | 548 ++ .../api/cwl/util/cwl_metric_data_helper.py | 907 +++ .../lambda/api/eks_cluster/aws_svc_mgr.py | 176 - .../lambda/api/eks_cluster/eks_cluster.py | 539 -- .../lambda/api/eks_cluster/exception.py | 6 - .../lambda/api/eks_cluster/lambda_function.py | 61 - ...me_role_policy_document_daemonset.template | 20 - ...sume_role_policy_document_sidecar.template | 20 - .../lambda/api/eks_cluster/test/conftest.py | 25 - .../eks_cluster/test/event/test_event.json | 7 - .../eks_cluster/test/requirements-test.txt | 5 - .../assume_role_policy_document.template | 20 - .../api/eks_cluster/test/test_eks_cluster.py | 247 - .../lambda/api/instance_group/aws_svc_mgr.py | 176 - .../api/instance_group/lambda_function.py | 438 -- .../event/add_instances_to_group_event.json | 55 - .../event/apply_instance_ingestion_event.json | 20 - .../test/event/create_group_event.json | 60 - ...eate_instance_group_base_on_asg_event.json | 53 - ...lete_exist_instances_from_group_event.json | 55 - .../test/event/delete_group_event.json | 53 - ...lete_instance_group_base_on_asg_event.json | 53 - .../delete_instances_from_group_event.json | 55 - .../test/event/list_asg_event.json | 53 - .../test/event/list_group_event.json | 55 - .../test/test_lambda_function.py | 249 - .../lambda/api/instance_meta/aws_svc_mgr.py | 176 - .../api/instance_meta/lambda_function.py | 579 -- .../lambda/api/instance_meta/test/conftest.py | 53 - .../instance_meta/test/event/test_event.json | 16 - .../test/test_lambda_function.py | 598 -- .../api/log_agent_status/aws_svc_mgr.py | 176 - .../api/log_agent_status/lambda_function.py | 526 +- .../api/log_agent_status/test/conftest.py | 4 +- .../test/event/test_event.json | 27 +- .../test/event/test_event2.json | 22 + .../test/requirements-test.txt | 1 - .../test/test_lambda_function.py | 463 +- .../lambda/api/log_conf/lambda_function.py | 286 +- .../lambda/api/log_conf/test/conftest.py | 38 + .../api/log_conf/test/event/test_event.json | 40 + .../api/log_conf/test/requirements-test.txt | 4 +- .../api/log_conf/test/test_lambda_function.py | 921 +-- .../lambda/api/log_source/lambda_function.py | 94 +- .../api/log_source/log_source_helper.py | 171 +- .../requirements.txt | 0 .../lambda/api/log_source/test/conftest.py | 26 +- .../log_source/test/datafile/ddb_mock_data.py | 19 - .../event/appsync_create_s3_source_event.json | 70 - .../event/appsync_delete_s3_source_event.json | 52 - .../event/appsync_get_s3_source_event.json | 52 - .../log_source/test/event/create_event.json | 27 + .../log_source/test/event/delete_event.json | 17 + .../api/log_source/test/event/get_event.json | 17 + .../api/log_source/test/event/list_event.json | 18 + .../api/log_source/test/requirements-test.txt | 7 +- .../lambda/api/log_source/test/test_eks.py | 65 + .../test/test_eks_constants.py | 3 +- .../log_source/test/test_lambda_function.py | 542 +- .../util/__Init__.py} | 0 .../lambda/api/log_source/util/eks.py | 159 + .../lambda/api/log_source/util/log_source.py | 289 + .../lambda/api/pipeline/aws_svc_mgr.py | 176 - .../lambda/api/pipeline/lambda_function.py | 245 +- .../api/pipeline/test/test_lambda_function.py | 22 +- .../.coveragerc | 0 .../app_ingestion_flow.py | 91 + .../pipeline_ingestion_flow/app_pipe_flow.py | 219 + .../pipeline_ingestion_flow/svc_pipe_flow.py | 199 + .../test/__init__.py | 0 .../pipeline_ingestion_flow/test/conftest.py | 20 + .../test/event/app_ingestion_test_event.json | 46 + .../test/event/app_pipe_test_event.json} | 16 +- .../test/event/svc_pipe_test_event.json | 33 + .../test/requirements-test.txt | 6 +- .../test/test_app_ingestion_flow.py | 270 + .../test/test_app_pipe_flow.py | 207 + .../test/test_svc_pipe_flow.py} | 66 +- .../util/pipeline_helper.py | 70 + .../lambda/api/resource/aws_svc_mgr.py | 176 - .../lambda/api/resource/lambda_function.py | 838 +-- .../lambda/api/resource/test/conftest.py | 5 +- .../api/resource/test/test_lambda_function.py | 48 +- .../eks_cluster => common-lib}/.coveragerc | 0 source/constructs/lambda/common-lib/README.md | 2 + .../commonlib/__init__.py} | 9 +- .../lambda/common-lib/commonlib/account.py | 202 + .../lambda/common-lib/commonlib/aws.py | 351 ++ .../lambda/common-lib/commonlib/dao.py | 733 +++ .../lambda/common-lib/commonlib/decorator.py | 123 + .../lambda/common-lib/commonlib/exception.py | 63 + .../lambda/common-lib/commonlib/model.py | 490 ++ .../lambda/common-lib/commonlib/opensearch.py | 452 ++ .../lambda/common-lib/commonlib/utils.py | 93 + .../lambda/common-lib/pyproject.toml | 3 + source/constructs/lambda/common-lib/setup.cfg | 23 + .../datafile => common-lib/test}/__init__.py | 0 .../test/conftest.py | 21 +- .../common-lib/test/requirements-test.txt | 5 + .../lambda/common-lib/test/test_account.py | 150 + .../lambda/common-lib/test/test_aws.py | 175 + .../lambda/common-lib/test/test_dao.py | 1372 +++++ .../lambda/common-lib/test/test_decoractor.py | 107 + .../lambda/common-lib/test/test_exception.py | 19 + .../lambda/common-lib/test/test_opensearch.py | 233 + .../lambda/common-lib/test/test_utils.py | 127 + .../create_service_linked_role.py | 2 +- .../lambda/custom-resource/lambda_function.py | 321 +- .../lambda/custom-resource/test/conftest.py | 8 +- .../test/test_lambda_function.py | 208 +- .../lambda/main/cfnHelper/aws_svc_mgr.py | 176 - .../lambda/main/cfnHelper/lambda_function.py | 25 +- .../lambda/main/cfnHelper/test/conftest.py | 2 +- .../cfnHelper/test/event/start_event.json | 2 +- .../main/cfnHelper/test/requirements-test.txt | 2 +- .../cfnHelper/test/test_lambda_function.py | 2 +- .../lambda/main/sfnHelper/test/conftest.py | 2 +- .../main/sfnHelper/test/requirements-test.txt | 3 +- .../app/log-processor/lambda_function.py | 52 +- .../app/log-processor/test/conftest.py | 5 +- .../test/test_lambda_function.py | 14 +- .../app/log-processor/test/test_osutil.py | 92 - .../pipeline/app/log-processor/util/helper.py | 3 + .../app/log-processor/util/log_parser.py | 149 +- .../pipeline/app/log-processor/util/osutil.py | 170 - .../app/log-processor/util/protocol.py | 150 - .../pipeline/app/log-processor/util/state.py | 59 +- .../common/custom-resource/boto3_client.py | 4 +- .../cloudfront_realtime_log_config_updater.py | 2 +- .../custom-resource/cw_subscription_filter.py | 36 +- .../log_source_s3_bucket_policy_processor.py | 5 +- .../sqs_allow_s3_to_put_event.py | 97 + .../common/custom-resource/test/conftest.py | 7 +- .../test/test_sqs_allow_s3_to_put_event.py | 45 + .../pipeline/common/layer/requirements.txt | 10 +- .../assets/index_template/cloudtrail.json | 93 + .../assets/saved_objects/cloudfront.ndjson | 4 +- .../assets/saved_objects/elb.ndjson | 4 +- .../assets/saved_objects/s3.ndjson | 32 +- .../assets/saved_objects/waf.ndjson | 46 +- .../opensearch-helper/lambda_function.py | 221 +- .../test/test_lambda_function.py | 76 +- .../opensearch-helper/test/test_osutil.py | 119 - .../opensearch-helper/test/test_state.py | 152 - .../common/opensearch-helper/util/osutil.py | 331 -- .../common/opensearch-helper/util/state.py | 152 - .../service/log-processor/boto3_client.py | 83 - .../service/log-processor/lambda_function.py | 240 +- .../service/log-processor/test/conftest.py | 4 +- .../log-processor/test/event/test_event.json | 2 +- .../test/test_lambda_function.py | 68 +- .../log-processor/test/test_log_parser.py | 143 +- .../service/log-processor/test/test_osutil.py | 92 - .../service/log-processor/test/test_state.py | 152 - .../test/test_waf_sampled_lambda_function.py | 721 +-- .../service/log-processor/util/helper.py | 75 + .../service/log-processor/util/log_parser.py | 486 +- .../service/log-processor/util/osutil.py | 170 - .../service/log-processor/util/state.py | 152 - .../waf_sampled_lambda_function.py | 191 +- .../lambda/plugin/standard/geo_ip.py | 57 +- .../lambda/plugin/standard/requirements.txt | 14 +- .../plugin/standard/test/test_geo_ip.py | 4 +- source/constructs/lib/api/api-stack.ts | 1232 +--- .../lib/api/app-log-ingestion-flow.ts | 94 +- .../lib/api/app-log-ingestion-stack.ts | 997 ++-- .../constructs/lib/api/app-pipeline-flow.ts | 160 +- .../constructs/lib/api/app-pipeline-stack.ts | 128 +- source/constructs/lib/api/app-table-stack.ts | 130 + source/constructs/lib/api/appsync-stack.ts | 211 + source/constructs/lib/api/cluster-flow.ts | 2 +- source/constructs/lib/api/cluster-stack.ts | 369 ++ .../lib/api/common-resource-stack.ts | 251 + .../constructs/lib/api/cross-account-stack.ts | 191 +- source/constructs/lib/api/cwl-stack.ts | 178 + .../lib/api/ec2-iam-instance-profile.ts | 133 + .../constructs/lib/api/eks-cluster-stack.ts | 271 - .../lib/api/instance-group-stack.ts | 255 - .../constructs/lib/api/instance-meta-stack.ts | 311 - source/constructs/lib/api/instance-stack.ts | 225 + source/constructs/lib/api/log-conf-stack.ts | 158 +- source/constructs/lib/api/log-source-stack.ts | 360 +- .../lib/api/pipeline-alarm-stack.ts | 171 + source/constructs/lib/api/pipeline-flow.ts | 200 - .../constructs/lib/api/svc-pipeline-flow.ts | 249 + .../constructs/lib/api/svc-pipeline-stack.ts | 192 + source/constructs/lib/kinesis/kds-stack.ts | 252 +- source/constructs/lib/kinesis/lambda/index.py | 492 +- .../lib/kinesis/lambda/test/kds.template | 38 + .../kinesis/lambda/test/requirements-test.txt | 5 + .../lib/kinesis/lambda/test/test_index.py | 262 + source/constructs/lib/layer/layer.ts | 57 + .../lib/layer/python/requirements.txt | 6 + source/constructs/lib/main-stack.ts | 596 +- source/constructs/lib/main/auth-stack.ts | 7 +- source/constructs/lib/main/cfn-flow-stack.ts | 781 ++- source/constructs/lib/main/cr-stack.ts | 34 +- .../constructs/lib/main/ecs-cluster-stack.ts | 4 +- source/constructs/lib/main/migration-stack.ts | 190 + source/constructs/lib/main/portal-stack.ts | 3 +- source/constructs/lib/main/vpc-stack.ts | 143 +- .../opensearch/alarm-for-opensearch-stack.ts | 307 +- .../opensearch/nginx-for-opensearch-stack.ts | 27 +- .../application/app-log-pipeline-stack.ts | 456 +- .../pipeline/application/app-log-processor.ts | 100 +- .../lib/pipeline/application/msk-stack.ts | 13 +- .../pipeline/application/s3-source-stack.ts | 450 ++ .../application/syslog-to-ecs-stack.ts | 29 +- .../lib/pipeline/common/cwl-metric-stack.ts | 245 + .../pipeline/common/opensearch-init-stack.ts | 30 +- .../lib/pipeline/common/solution-stack.ts | 24 +- .../service/cloudfront-realtime-log-stack.ts | 16 +- .../pipeline/service/cloudwatch-log-stack.ts | 11 +- .../service/cw-to-firehose-to-s3-stack.ts | 5 + .../service/s3-to-opensearch-stack.ts | 366 +- .../service/service-log-pipeline-stack.ts | 114 +- .../lib/pipeline/service/waf-sampled-stack.ts | 55 +- .../lib/subaccount/cross-account-stack.ts | 745 +-- source/constructs/lib/util/stack-helper.ts | 34 + .../instance_group => migration}/.coveragerc | 0 source/constructs/migration/main.py | 616 ++ .../util => migration/test}/__init__.py | 0 .../api/common => migration}/test/conftest.py | 13 +- .../test/requirements-test.txt | 5 +- source/constructs/migration/test/test_main.py | 192 + source/constructs/migration/utils.py | 122 + source/constructs/package.json | 63 +- source/constructs/test/app-pipeline.test.ts | 245 +- source/constructs/test/main.test.ts | 5 - .../constructs/test/service-pipeline.test.ts | 425 +- source/portal/.eslintrc.json | 3 +- source/portal/.gitignore | 4 +- source/portal/package.json | 106 +- source/portal/public/index.html | 1 - source/portal/public/locales/en/applog.json | 337 +- source/portal/public/locales/en/cluster.json | 79 +- source/portal/public/locales/en/common.json | 178 +- source/portal/public/locales/en/ekslog.json | 37 +- source/portal/public/locales/en/info.json | 27 +- source/portal/public/locales/en/resource.json | 40 +- .../portal/public/locales/en/servicelog.json | 24 +- source/portal/public/locales/zh/applog.json | 303 +- source/portal/public/locales/zh/cluster.json | 79 +- source/portal/public/locales/zh/common.json | 178 +- source/portal/public/locales/zh/ekslog.json | 13 +- source/portal/public/locales/zh/info.json | 27 +- source/portal/public/locales/zh/resource.json | 32 +- .../portal/public/locales/zh/servicelog.json | 14 +- source/portal/src/API.ts | 1823 +++--- source/portal/src/App.test.tsx | 12 +- source/portal/src/App.tsx | 273 +- .../assets/images/desc/wafSamplingArch.png | Bin 0 -> 41002 bytes .../src/assets/images/type/amazon_ec2.svg | 17 + .../assets/images/type/amazon_ec2_arch.svg | 4 + .../src/assets/images/type/amazon_eks.svg | 19 + .../assets/images/type/amazon_eks_arch.svg | 4 + .../src/assets/images/type/amazon_s3_2.svg | 8 + .../assets/images/type/s3-source-on-going.svg | 4 + .../assets/images/type/s3-source-one-time.svg | 4 + .../portal/src/assets/images/type/syslog.svg | 8 + .../src/assets/images/type/syslog_arch.svg | 4 + source/portal/src/assets/js/alert.ts | 78 +- source/portal/src/assets/js/applog.ts | 39 +- source/portal/src/assets/js/const.ts | 146 +- source/portal/src/assets/js/hooks/useAlarm.ts | 35 + .../src/assets/js/hooks/useAutoValidation.ts | 30 + source/portal/src/assets/js/hooks/useTags.ts | 35 + source/portal/src/assets/js/init.ts | 107 + source/portal/src/assets/js/request.ts | 17 +- source/portal/src/assets/js/utils.ts | 336 +- source/portal/src/assets/js/xss.ts | 32 +- .../portal/src/components/Alert/_style.scss | 33 +- source/portal/src/components/Alert/alert.tsx | 95 +- .../components/AutoComplete/autoComplete.tsx | 39 +- .../src/components/Breadcrumb/breadcrumb.tsx | 10 +- .../ButtonDropdown/ButtonDropdown.tsx | 4 +- .../portal/src/components/ButtonRefresh.tsx | 40 + .../src/components/ClickableRichTooltip.tsx | 63 + .../src/components/CodeCopy/codeCopy.tsx | 31 +- .../src/components/CopyButton/copyButton.tsx | 3 +- .../src/components/CopyText/copyText.tsx | 4 +- .../src/components/CreateStep/createStep.tsx | 2 +- .../ExpandableSection/ExpandableSection.tsx | 55 + .../components/ExpandableSection/_style.scss | 17 + .../components/ExpandableSection/index.tsx | 17 + source/portal/src/components/ExtButton.tsx | 42 + .../src/components/FormItem/formItem.tsx | 14 +- .../components/HeaderPanel/headerPanel.tsx | 9 +- .../src/components/HelpPanel/helpPanel.tsx | 27 +- .../IndicatorWithLink/IndicatorWithLink.tsx | 76 + .../components/IndicatorWithLink/_style.scss | 23 + .../components/IndicatorWithLink/index.tsx | 17 + .../src/components/InfoSpan/infoSpan.tsx | 4 +- .../components/LoadingText/loadingText.tsx | 2 +- source/portal/src/components/Modal/modal.tsx | 14 +- .../src/components/MultiSelect/_style.scss | 1 - .../components/MultiSelect/multiSelect.tsx | 14 +- .../src/components/PagePanel/pagePanel.tsx | 10 +- source/portal/src/components/Radio/index.tsx | 75 + .../{S3Select => RichTooltip}/index.tsx | 4 +- .../components/RichTooltip/richTooltip.tsx | 162 + .../src/components/S3Select/_style.scss | 20 - .../src/components/S3Select/s3Select.tsx | 126 - .../portal/src/components/Select/_style.scss | 1 - .../portal/src/components/Select/select.tsx | 95 +- .../src/components/SideMenu/sideMenu.tsx | 12 +- .../portal/src/components/Status/Status.tsx | 122 +- .../portal/src/components/Status/_style.scss | 27 + .../StatusIndicator/StatusIndicator.tsx | 66 + .../components/StatusIndicator/_style.scss | 23 + .../src/components/StatusIndicator/index.tsx | 17 + .../portal/src/components/Switch/switch.tsx | 6 +- source/portal/src/components/Tab/tab.tsx | 6 +- .../src/components/TablePanel/_style.scss | 7 + .../src/components/TablePanel/tablePanel.tsx | 78 +- .../src/components/TagFilter/tagFilter.tsx | 8 +- .../portal/src/components/TagList/tagList.tsx | 15 +- .../src/components/TextInput/textInput.tsx | 13 +- source/portal/src/components/Tiles/tiles.tsx | 12 +- .../src/components/TimeRange/TimeRange.tsx | 175 + .../ValueWithLabel/valueWithLabel.tsx | 6 +- .../portal/src/components/layout/footer.tsx | 11 +- .../portal/src/components/layout/header.tsx | 11 +- source/portal/src/graphql/mutations.ts | 318 +- source/portal/src/graphql/queries.ts | 786 +-- source/portal/src/graphql/schema.json | 5121 +++++++++++------ source/portal/src/help/LogConfigPathEKS.tsx | 10 +- source/portal/src/help/PipelineAlarms.tsx | 51 + source/portal/src/help/ProxyInstance.tsx | 4 +- source/portal/src/help/SampleDashboard.tsx | 11 +- source/portal/src/index.tsx | 21 +- .../src/pages/clusters/domain/DomainAlarm.tsx | 249 +- .../pages/clusters/domain/DomainDetail.tsx | 31 +- .../src/pages/clusters/domain/DomainList.tsx | 310 +- .../clusters/domain/NginxForOpenSearch.tsx | 49 +- .../clusters/domain/detail/AccessProxy.tsx | 8 +- .../pages/clusters/domain/detail/Alarms.tsx | 22 +- .../pages/clusters/domain/detail/Network.tsx | 49 +- .../src/pages/clusters/domain/detail/Tags.tsx | 61 - .../clusters/importcluster/ImportCluster.tsx | 100 +- .../importcluster/steps/ConfigNetwork.tsx | 11 +- .../importcluster/steps/CreateTags.tsx | 71 - .../importcluster/steps/ImportDomain.tsx | 251 + .../importcluster/steps/SelectDomain.tsx | 259 +- source/portal/src/pages/comps/Validator.tsx | 41 + .../src/pages/comps/account/AccountName.tsx | 25 +- .../comps/account/CrossAccountSelect.tsx | 7 +- .../src/pages/comps/buffer/BufferKDS.tsx | 12 +- .../src/pages/comps/buffer/BufferS3.tsx | 269 +- .../src/pages/comps/monitor/LineChart.tsx | 304 + .../pages/comps/monitor/MonitorMetrics.tsx | 61 + .../pages/containers/eksLog/EksLogDetail.tsx | 155 +- .../pages/containers/eksLog/EksLogList.tsx | 94 +- .../eksLog/create/ImportEksCluster.tsx | 163 +- .../eksLog/create/steps/CreateTags.tsx | 72 - .../eksLog/create/steps/SpecifyEKSDomain.tsx | 200 - .../eksLog/create/steps/SpecifyEksSource.tsx | 115 +- .../eksLog/createIngestion/EksLogIngest.tsx | 358 +- .../createIngestion/step/CreateTags.tsx | 71 - .../createIngestion/step/SpecifyLogConfig.tsx | 201 - .../createIngestion/step/SpecifySettings.tsx | 44 +- .../eksLog/detail/DaemonsetGuide.tsx | 83 +- .../containers/eksLog/detail/Ingestions.tsx | 118 +- .../pages/containers/eksLog/detail/Tags.tsx | 62 - .../applicationLog/ApplicationLog.tsx | 121 +- .../applicationLog/ApplicationLogDetail.tsx | 289 +- .../applicationLog/SelectLogSource.tsx | 275 + .../applicationLog/common/ApplyLogConfig.tsx | 467 -- .../common/CreateSampleDashboard.tsx | 63 +- .../applicationLog/common/IndexName.tsx | 52 + .../common/InstancePermission.tsx | 36 + .../common/LogConfigSelector.tsx | 115 + .../applicationLog/common/LogPathInput.tsx | 69 + .../common/UnmodifiableLogConfigSelector.tsx | 92 + .../applicationLog/create/CreatePipeline.tsx | 951 --- .../create/ec2/ChooseBufferLayer.tsx | 201 + .../create/ec2/ChooseInstanceGroupTable.tsx | 171 + .../applicationLog/create/ec2/CreateEC2.tsx | 957 +++ .../create/ec2/PermissionsModeSelector.tsx | 61 + .../applicationLog/create/eks/CreateEKS.tsx | 1133 ++++ .../create/eks/steps/StepChooseEKSSource.tsx | 250 + .../create/s3/CompressionFormatSelector.tsx | 59 + .../applicationLog/create/s3/CreateS3.tsx | 858 +++ .../create/s3/FilePathPrefixFilter.tsx | 53 + .../create/s3/IngestionModeSelector.tsx | 79 + .../create/s3/S3BucketSelector.tsx | 75 + .../create/steps/CreateTags.tsx | 72 - .../create/steps/IngestSetting.tsx | 12 +- .../create/steps/SpecifyDomain.tsx | 187 +- .../create/syslog/CreateSyslog.tsx | 1070 ++++ .../syslog/OnlyCreateSyslogIngestion.tsx | 394 ++ .../syslog/steps/StepChooseSyslogSource.tsx} | 35 +- .../createEKSIngestion/CreateEKSIngestion.tsx | 512 -- .../createEKSIngestion/steps/CreateTags.tsx | 71 - .../steps/StepChooseSource.tsx | 179 - .../createIngestion/CreateIngestion.tsx | 726 --- .../steps/ChooseInstanceGroup.tsx | 201 - .../createIngestion/steps/CreateTags.tsx | 71 - .../steps/StepCreateInstanceGroup.tsx | 171 - .../CreateSysLogIngestion.tsx | 649 --- .../steps/CreateTags.tsx | 72 - .../applicationLog/detail/Alarm.tsx | 50 + .../detail/AnalyticsEngineDetails.tsx | 146 + .../detail/BufferLayerDetails.tsx | 176 + .../applicationLog/detail/Ingestion.tsx | 443 +- .../applicationLog/detail/Lifecycle.tsx | 64 - .../detail/LogConfigDetails.tsx | 46 + .../applicationLog/detail/Logging.tsx | 34 + .../applicationLog/detail/Monitoring.tsx | 238 + .../applicationLog/detail/Permission.tsx | 203 +- .../applicationLog/detail/Tags.tsx | 61 - .../monitor/AppLogFlbAgentSourceMetrics.tsx | 81 + .../detail/monitor/AppLogKDSBufferMetrics.tsx | 71 + .../detail/monitor/AppLogProcessorMetrics.tsx | 86 + .../detail/monitor/AppLogSQSBufferMetrics.tsx | 66 + .../monitor/AppLogSyslogSourceMetrics.tsx | 56 + .../pages/dataInjection/common/CreateTags.tsx | 55 + .../dataInjection/common/PipelineStatus.tsx | 63 + .../common/SelectOpenSearchDomain.tsx | 182 + .../{serviceLog/detail => common}/Tags.tsx | 19 +- .../ingestiondetail/AppIngestionDetail.tsx | 547 +- .../ingestiondetail/EksIngestionDetail.tsx | 52 +- .../ingestiondetail/S3SourceDetail.tsx | 41 + .../ingestiondetail/comps/LogConfig.tsx | 6 +- .../ingestiondetail/comps/Sidecar.tsx | 15 +- .../ingestiondetail/comps/SyslogGuide.tsx | 48 +- .../ingestiondetail/comps/Tags.tsx | 62 - .../dataInjection/serviceLog/ServiceLog.tsx | 54 +- .../serviceLog/ServiceLogDetail.tsx | 371 +- .../serviceLog/create/CreateChooseType.tsx | 15 +- .../create/cloudfront/CreateCloudFront.tsx | 136 +- .../create/cloudfront/comps/SourceType.tsx | 105 +- .../cloudfront/steps/SpecifySettings.tsx | 15 +- .../create/cloudtrail/CreateCloudTrail.tsx | 126 +- .../cloudtrail/steps/SpecifySettings.tsx | 6 +- .../cloudtrail/steps/comp/SourceType.tsx | 37 +- .../create/common/AutoEnableLogging.tsx | 3 +- .../serviceLog/create/common/CreateTags.tsx | 89 - .../serviceLog/create/common/KDSSettings.tsx | 4 +- .../create/common/LogProcessing.tsx | 4 +- .../create/common/SpecifyCluster.tsx | 236 +- .../serviceLog/create/common/desc/WAFDesc.tsx | 42 +- .../serviceLog/create/config/CreateConfig.tsx | 109 +- .../create/config/steps/SpecifySettings.tsx | 8 +- .../serviceLog/create/elb/CreateELB.tsx | 174 +- .../create/elb/steps/SpecifySettings.tsx | 9 +- .../serviceLog/create/lambda/CreateLambda.tsx | 108 +- .../create/lambda/steps/SpecifySettings.tsx | 6 +- .../serviceLog/create/rds/CreateRDS.tsx | 291 +- .../create/rds/steps/SpecifySettings.tsx | 7 +- .../serviceLog/create/s3/CreateS3.tsx | 168 +- .../create/s3/steps/SpecifySettings.tsx | 9 +- .../serviceLog/create/vpc/CreateVPC.tsx | 105 +- .../create/vpc/steps/SpecifySettings.tsx | 2 + .../create/vpc/steps/comp/SourceType.tsx | 82 +- .../serviceLog/create/waf/CreateWAF.tsx | 134 +- .../create/waf/steps/IngestOptionSelect.tsx | 8 +- .../create/waf/steps/SampleSchedule.tsx | 2 +- .../create/waf/steps/SpecifySettings.tsx | 78 +- .../dataInjection/serviceLog/detail/Alarm.tsx | 51 + .../serviceLog/detail/Logging.tsx | 32 + .../serviceLog/detail/Monitoring.tsx | 341 ++ .../serviceLog/detail/Overview.tsx | 18 +- .../monitor/ServiceLogBufferMetrics.tsx | 60 + .../monitor/ServiceLogProcessorMetrics.tsx | 94 + .../CloudFrontRealTimeBufferMetrics.tsx | 65 + .../cloudtrail/CloudTrailCWLBufferMetrics.tsx | 65 + .../monitor/lambda/LambdaBufferMetrics.tsx | 75 + .../detail/monitor/rds/RDSBufferMetrics.tsx | 75 + .../vpc_flow/VPCFlowLogBufferMetrics.tsx | 65 + .../waf/WAFSampledLogProcessorMetrics.tsx | 41 + source/portal/src/pages/home/Home.tsx | 20 +- .../src/pages/loggingEvents/EventList.tsx | 401 ++ .../portal/src/pages/pipelineAlarm/Alarm.tsx | 673 +++ .../src/pages/pipelineAlarm/AlarmAndTags.tsx | 75 + .../pages/pipelineAlarm/alarm/AlarmStatus.tsx | 60 + .../pages/pipelineAlarm/alarm/AlarmTable.tsx | 73 + .../pipelineAlarm/alarm/CreateAlarms.tsx | 73 + .../alarm/KDSBufferLayerAlarm.tsx | 45 + .../pipelineAlarm/alarm/LogProcessorAlarm.tsx | 49 + .../pipelineAlarm/alarm/LogSourceAlarm.tsx | 102 + .../pipelineAlarm/alarm/PipelineAlarmTab.tsx | 87 + .../alarm/S3BufferLayerAlarm.tsx | 45 + .../src/pages/pipelineLogging/AppLogging.tsx | 28 + .../src/pages/pipelineLogging/Logging.tsx | 150 + .../pages/pipelineLogging/ServiceLogging.tsx | 32 + .../pipelineLogging/logging/LoggingHeader.tsx | 65 + .../pipelineLogging/logging/LoggingTable.tsx | 183 + .../src/pages/resources/common/ASGSelect.tsx | 16 +- .../pages/resources/common/ConfigFilter.tsx | 55 +- .../resources/common/InstanceGroupComp.tsx | 19 +- .../pages/resources/common/InstanceTable.tsx | 553 +- .../pages/resources/common/LogConfigComp.tsx | 88 +- .../resources/common/SampleLogParsing.tsx | 214 +- .../crossAccount/CrossAccountDetail.tsx | 34 +- .../crossAccount/CrossAccountList.tsx | 42 +- .../resources/crossAccount/LinkAnAccount.tsx | 598 +- .../instanceGroup/InstanceGroupDetail.tsx | 191 +- .../instanceGroup/InstanceGroupList.tsx | 85 +- .../resources/instanceGroup/SelectType.tsx | 6 +- .../instanceGroup/comps/ASGGuide.tsx | 109 +- .../instanceGroup/comps/DetailASG.tsx | 8 +- .../instanceGroup/comps/DetailEC2.tsx | 211 +- .../create/CreateInstanceGroup.tsx | 96 +- .../resources/logConfig/ConfigDetail.tsx | 113 +- .../resources/logConfig/ConfigDetailComps.tsx | 81 +- .../resources/logConfig/ConfigEditor.tsx | 494 ++ .../resources/logConfig/CreateLogConfig.tsx | 297 +- .../resources/logConfig/EditLogConfig.tsx | 350 +- .../pages/resources/logConfig/LogConfig.tsx | 64 +- source/portal/src/reducer/appReducer.ts | 10 + source/portal/src/reducer/createAlarm.ts | 226 + source/portal/src/reducer/createTags.ts | 95 + source/portal/src/reducer/reducers.ts | 45 + source/portal/src/styles/index.scss | 308 +- source/portal/src/types/index.ts | 101 +- source/run-all-tests.sh | 179 +- .../jest/solutions-constructs/clover.xml | 2122 +++++++ .../solutions-constructs/coverage-final.json | 47 + .../jest/solutions-constructs/lcov.info | 2816 +++++++++ 807 files changed, 66934 insertions(+), 43265 deletions(-) create mode 100644 deployment/run-unit-tests.sh mode change 120000 => 100644 docs/en/images create mode 100644 docs/en/implementation-guide/alarm.md create mode 100644 docs/en/implementation-guide/architecture-details/services-in-this-solution.md rename docs/en/implementation-guide/{ => architecture-details}/solution-components.md (93%) create mode 100644 docs/en/implementation-guide/architecture-overview/well-architected-pillars.md delete mode 100644 docs/en/implementation-guide/cost.md rename docs/en/implementation-guide/{aws-services => }/include-dashboard.md (58%) create mode 100644 docs/en/implementation-guide/monitoring.md rename docs/en/implementation-guide/{ => plan-deployment}/considerations.md (99%) create mode 100644 docs/en/implementation-guide/plan-deployment/cost.md create mode 100644 docs/en/implementation-guide/plan-deployment/security.md delete mode 100644 docs/en/implementation-guide/resources/aws-services.md delete mode 100644 docs/en/implementation-guide/security.md create mode 100644 docs/en/implementation-guide/solution-overview/features.md create mode 100644 docs/en/implementation-guide/solution-overview/use-cases.md create mode 100644 docs/en/implementation-guide/source.md create mode 100644 docs/images/dashboards/apache.png mode change 120000 => 100644 docs/zh/images create mode 100644 docs/zh/implementation-guide/alarm.md create mode 100644 docs/zh/implementation-guide/architecture-details/services-in-this-solution.md create mode 100644 docs/zh/implementation-guide/architecture-details/solution-components.md create mode 100644 docs/zh/implementation-guide/architecture-overview/well-architected-pillars.md create mode 100644 docs/zh/implementation-guide/include-dashboard.md create mode 100644 docs/zh/implementation-guide/monitoring.md rename docs/zh/implementation-guide/{ => plan-deployment}/considerations.md (100%) rename docs/zh/implementation-guide/{ => plan-deployment}/cost.md (93%) create mode 100644 docs/zh/implementation-guide/plan-deployment/security.md delete mode 100644 docs/zh/implementation-guide/resources/aws-services.md delete mode 100644 docs/zh/implementation-guide/security.md create mode 100644 docs/zh/implementation-guide/solution-overview/features.md create mode 100644 docs/zh/implementation-guide/solution-overview/use-cases.md create mode 100644 docs/zh/implementation-guide/source.md create mode 100644 source/constructs/ecr/s3-list-objects/.dockerignore create mode 100644 source/constructs/ecr/s3-list-objects/.gitignore create mode 100644 source/constructs/ecr/s3-list-objects/Dockerfile create mode 100755 source/constructs/ecr/s3-list-objects/build.sh create mode 100644 source/constructs/ecr/s3-list-objects/requirements.txt create mode 100755 source/constructs/ecr/s3-list-objects/s3_list_objects.py rename source/constructs/{lambda/api/app_log_ingestion/util/distribute_config_helper => ecr/s3-list-objects/test}/__init__.py (100%) rename source/constructs/{lambda/api/app_pipeline_flow => ecr/s3-list-objects}/test/conftest.py (70%) rename source/constructs/{lambda/api/instance_group => ecr/s3-list-objects}/test/requirements-test.txt (91%) create mode 100644 source/constructs/ecr/s3-list-objects/test/test_s3_list_objects.py delete mode 100644 source/constructs/graphql/vtl/app_log_ingestion/GetEKSDaemonSetConf.vtl delete mode 100644 source/constructs/graphql/vtl/app_log_ingestion/GetEKSDeploymentConf.vtl create mode 100644 source/constructs/graphql/vtl/app_log_ingestion/GetK8sDeploymentContentWithDaemonSet.vtl create mode 100644 source/constructs/graphql/vtl/app_log_ingestion/GetK8sDeploymentContentWithSidecar.vtl create mode 100644 source/constructs/graphql/vtl/cross_account/GetSubAccountLink.vtl create mode 100644 source/constructs/graphql/vtl/instance_status/GetInstanceAgentStatus.vtl create mode 100644 source/constructs/graphql/vtl/instance_status/GetInstanceAgentStatusResp.vtl create mode 100644 source/constructs/graphql/vtl/instance_status/ListInstances.vtl create mode 100644 source/constructs/graphql/vtl/instance_status/RequestInstallLogAgent.vtl rename source/constructs/graphql/vtl/log_conf/{UpdateLogConf.vtl => CreateLogConfig.vtl} (58%) delete mode 100644 source/constructs/graphql/vtl/log_conf/GetLogConfResp.vtl create mode 100644 source/constructs/graphql/vtl/log_conf/GetLogConfigResp.vtl rename source/constructs/graphql/vtl/log_conf/{ListLogConfs.vtl => ListLogConfigs.vtl} (100%) create mode 100644 source/constructs/graphql/vtl/log_conf/ListLogConfigsResp.vtl delete mode 100644 source/constructs/graphql/vtl/log_conf/ListLogConfsResp.vtl rename source/constructs/graphql/vtl/log_conf/{CreateLogConf.vtl => UpdateLogConfig.vtl} (58%) rename source/constructs/lambda/api/{instance_meta => alarm}/.coveragerc (86%) create mode 100644 source/constructs/lambda/api/alarm/lambda_function.py rename source/constructs/lambda/api/{app_log_ingestion/util/fluentbit_config_helper => alarm/test}/__init__.py (100%) create mode 100644 source/constructs/lambda/api/alarm/test/conftest.py create mode 100644 source/constructs/lambda/api/alarm/test/requirements-test.txt create mode 100644 source/constructs/lambda/api/alarm/test/test_lambda_function.py create mode 100644 source/constructs/lambda/api/alarm/util/alarm_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/common.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/ec2_as_source_lambda_function.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/ec2_ingestion_distribution_event_lambda_function.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/eks_daemonset_sidecar_config_lambda_function.py rename source/constructs/lambda/api/{app_pipeline_flow/test => app_log_ingestion/flb}/__init__.py (100%) create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/distribution.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_builder.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_model.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_template/k8s-DaemonSet.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_template/k8s-Sidecar.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_template/parser.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/flb_template/pipeline.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/flb/k8s.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/requirements.txt create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/ec2.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/ec2_attach_iam_instance_profile.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/k8s.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/s3.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/service.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/svc/syslog.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/syslog_as_source_lambda_function.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/datafile/td-agent_ground_truth.conf delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/add_instance_to_instance_group_event.json create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/apply_app_log_ingestion_for_single_instance.json create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/delete_ingestion_and_refresh_flb_config_event.json delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/delete_instances_from_instance_group_event.json create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/generate_and_upload_flb_config_event.json create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/refresh_app_log_ingestion_for_single_instance.json create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/event/test_event.json rename source/constructs/lambda/api/app_log_ingestion/{util/eks_cluster_pod_template/fluent-bit-daemonset.template.containerd.yaml => test/flb/flb_template/k8s-DaemonSet.conf} (51%) create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/flb/flb_template/k8s-Sidecar.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/flb/flb_template/parser.conf create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/flb/flb_template/pipeline.conf delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_agent_role_mgr.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_asg_config_lambda_function.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_assume_role.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_auto_scaling_group_config_lambda_function.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_ddb_connect.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_distribute_config_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_ec2_as_source_lambda_function.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_ec2_attach_iam_instance_profile.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_ec2_ingestion_distribution_event_lambda_function.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_ec2_source.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_eks_daemonset_sidecar_config_lambda_function.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_flb.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_fluentbit_config_helper.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_k8s_source.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_log_agent_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_log_agent_helper_v2.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_s3_source.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_service.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_syslog_as_source_lambda_function.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_syslog_source.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/test_utils.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/test/util/asg_ingestion_script/asg_ingestion_script.sh delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/agent_role_mgr.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/agent_type.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/assume_role.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/distribute_config_helper/distribute_config_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/ec2_log_ingestion_svc.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-configmap-aos.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-configmap.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-daemonset.template.docker.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-grep-filter.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-input-output-filter-daemonset.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-input-output-sidecar.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-k8s-filter.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-namespace.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-output-aos.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-output-kds.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-output-msk.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-output-s3.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-role-binding.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-service-account.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-sidecar.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-user-defined-docker-firstline-parser.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-user-defined-multiline-filter.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-user-defined-multiline-parser.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/fluent-bit-user-defined-parser.template.yaml delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_cluster_pod_template/uniform-time-format.lua delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/eks_deployment_configuration.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/exception.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentbit_config_helper/ddb_connect.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentbit_config_helper/defaults.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentbit_config_helper/fluentbit_config_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-cross-account-kds.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-input.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-output.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-parser-csv.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-parser-json.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-parser-regex.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/fluentd_s3_template/log-agent-service.template delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/log_agent_helper.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/log_agent_helper_v2.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/log_ingestion_svc.py delete mode 100644 source/constructs/lambda/api/app_log_ingestion/util/sys_enum_type.py create mode 100644 source/constructs/lambda/api/app_log_ingestion/util/utils.py delete mode 100644 source/constructs/lambda/api/app_pipeline/common.py create mode 100644 source/constructs/lambda/api/app_pipeline/test/test_utils.py create mode 100644 source/constructs/lambda/api/app_pipeline/util/utils.py delete mode 100644 source/constructs/lambda/api/app_pipeline/util/validator.py delete mode 100644 source/constructs/lambda/api/app_pipeline_flow/lambda_function.py create mode 100644 source/constructs/lambda/api/cluster/test/test_cluster_auto_import_mgr.py create mode 100644 source/constructs/lambda/api/cluster/util/cluster_status_check_helper.py delete mode 100644 source/constructs/lambda/api/cluster/util/exception.py delete mode 100644 source/constructs/lambda/api/common/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/api/common/common.py delete mode 100644 source/constructs/lambda/api/common/exception.py delete mode 100644 source/constructs/lambda/api/common/test/test_aws_svc_mgr.py create mode 100644 source/constructs/lambda/api/cwl/.coveragerc create mode 100644 source/constructs/lambda/api/cwl/lambda_function.py rename source/constructs/lambda/api/{eks_cluster => cwl}/test/__init__.py (100%) create mode 100644 source/constructs/lambda/api/cwl/test/conftest.py rename source/constructs/lambda/api/{app_pipeline_flow => cwl}/test/requirements-test.txt (79%) create mode 100644 source/constructs/lambda/api/cwl/test/test_lambda_function.py create mode 100644 source/constructs/lambda/api/cwl/util/cwl_metric_data_helper.py delete mode 100644 source/constructs/lambda/api/eks_cluster/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/api/eks_cluster/eks_cluster.py delete mode 100644 source/constructs/lambda/api/eks_cluster/exception.py delete mode 100644 source/constructs/lambda/api/eks_cluster/lambda_function.py delete mode 100644 source/constructs/lambda/api/eks_cluster/role_template/assume_role_policy_document_daemonset.template delete mode 100644 source/constructs/lambda/api/eks_cluster/role_template/assume_role_policy_document_sidecar.template delete mode 100644 source/constructs/lambda/api/eks_cluster/test/conftest.py delete mode 100644 source/constructs/lambda/api/eks_cluster/test/event/test_event.json delete mode 100644 source/constructs/lambda/api/eks_cluster/test/requirements-test.txt delete mode 100644 source/constructs/lambda/api/eks_cluster/test/role_template/assume_role_policy_document.template delete mode 100644 source/constructs/lambda/api/eks_cluster/test/test_eks_cluster.py delete mode 100644 source/constructs/lambda/api/instance_group/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/api/instance_group/lambda_function.py delete mode 100644 source/constructs/lambda/api/instance_group/test/event/add_instances_to_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/apply_instance_ingestion_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/create_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/create_instance_group_base_on_asg_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/delete_exist_instances_from_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/delete_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/delete_instance_group_base_on_asg_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/delete_instances_from_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/list_asg_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/event/list_group_event.json delete mode 100644 source/constructs/lambda/api/instance_group/test/test_lambda_function.py delete mode 100644 source/constructs/lambda/api/instance_meta/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/api/instance_meta/lambda_function.py delete mode 100644 source/constructs/lambda/api/instance_meta/test/conftest.py delete mode 100644 source/constructs/lambda/api/instance_meta/test/event/test_event.json delete mode 100644 source/constructs/lambda/api/instance_meta/test/test_lambda_function.py delete mode 100644 source/constructs/lambda/api/log_agent_status/aws_svc_mgr.py create mode 100644 source/constructs/lambda/api/log_agent_status/test/event/test_event2.json create mode 100644 source/constructs/lambda/api/log_conf/test/event/test_event.json rename source/constructs/lambda/api/{eks_cluster => log_source}/requirements.txt (100%) delete mode 100644 source/constructs/lambda/api/log_source/test/datafile/ddb_mock_data.py delete mode 100644 source/constructs/lambda/api/log_source/test/event/appsync_create_s3_source_event.json delete mode 100644 source/constructs/lambda/api/log_source/test/event/appsync_delete_s3_source_event.json delete mode 100644 source/constructs/lambda/api/log_source/test/event/appsync_get_s3_source_event.json create mode 100644 source/constructs/lambda/api/log_source/test/event/create_event.json create mode 100644 source/constructs/lambda/api/log_source/test/event/delete_event.json create mode 100644 source/constructs/lambda/api/log_source/test/event/get_event.json create mode 100644 source/constructs/lambda/api/log_source/test/event/list_event.json create mode 100644 source/constructs/lambda/api/log_source/test/test_eks.py rename source/constructs/lambda/api/{eks_cluster => log_source}/test/test_eks_constants.py (99%) rename source/constructs/lambda/api/{instance_group/test/__init__.py => log_source/util/__Init__.py} (100%) create mode 100644 source/constructs/lambda/api/log_source/util/eks.py create mode 100644 source/constructs/lambda/api/log_source/util/log_source.py delete mode 100644 source/constructs/lambda/api/pipeline/aws_svc_mgr.py rename source/constructs/lambda/api/{app_pipeline_flow => pipeline_ingestion_flow}/.coveragerc (100%) create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/app_ingestion_flow.py create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/app_pipe_flow.py create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/svc_pipe_flow.py rename source/constructs/lambda/api/{instance_meta => pipeline_ingestion_flow}/test/__init__.py (100%) create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/test/conftest.py create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/test/event/app_ingestion_test_event.json rename source/constructs/lambda/api/{app_pipeline_flow/test/event/test_event.json => pipeline_ingestion_flow/test/event/app_pipe_test_event.json} (64%) create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/test/event/svc_pipe_test_event.json rename source/constructs/lambda/api/{instance_meta => pipeline_ingestion_flow}/test/requirements-test.txt (50%) create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/test/test_app_ingestion_flow.py create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/test/test_app_pipe_flow.py rename source/constructs/lambda/api/{app_pipeline_flow/test/test_lambda_function.py => pipeline_ingestion_flow/test/test_svc_pipe_flow.py} (70%) create mode 100644 source/constructs/lambda/api/pipeline_ingestion_flow/util/pipeline_helper.py delete mode 100644 source/constructs/lambda/api/resource/aws_svc_mgr.py rename source/constructs/lambda/{api/eks_cluster => common-lib}/.coveragerc (100%) create mode 100644 source/constructs/lambda/common-lib/README.md rename source/constructs/lambda/{api/instance_group/util/group_enum_type.py => common-lib/commonlib/__init__.py} (52%) create mode 100644 source/constructs/lambda/common-lib/commonlib/account.py create mode 100644 source/constructs/lambda/common-lib/commonlib/aws.py create mode 100644 source/constructs/lambda/common-lib/commonlib/dao.py create mode 100644 source/constructs/lambda/common-lib/commonlib/decorator.py create mode 100644 source/constructs/lambda/common-lib/commonlib/exception.py create mode 100644 source/constructs/lambda/common-lib/commonlib/model.py create mode 100644 source/constructs/lambda/common-lib/commonlib/opensearch.py create mode 100644 source/constructs/lambda/common-lib/commonlib/utils.py create mode 100644 source/constructs/lambda/common-lib/pyproject.toml create mode 100644 source/constructs/lambda/common-lib/setup.cfg rename source/constructs/lambda/{api/log_source/test/datafile => common-lib/test}/__init__.py (100%) rename source/constructs/lambda/{api/instance_group => common-lib}/test/conftest.py (66%) create mode 100644 source/constructs/lambda/common-lib/test/requirements-test.txt create mode 100644 source/constructs/lambda/common-lib/test/test_account.py create mode 100644 source/constructs/lambda/common-lib/test/test_aws.py create mode 100644 source/constructs/lambda/common-lib/test/test_dao.py create mode 100644 source/constructs/lambda/common-lib/test/test_decoractor.py create mode 100644 source/constructs/lambda/common-lib/test/test_exception.py create mode 100644 source/constructs/lambda/common-lib/test/test_opensearch.py create mode 100644 source/constructs/lambda/common-lib/test/test_utils.py delete mode 100644 source/constructs/lambda/main/cfnHelper/aws_svc_mgr.py delete mode 100644 source/constructs/lambda/pipeline/app/log-processor/test/test_osutil.py delete mode 100644 source/constructs/lambda/pipeline/app/log-processor/util/osutil.py delete mode 100644 source/constructs/lambda/pipeline/app/log-processor/util/protocol.py create mode 100644 source/constructs/lambda/pipeline/common/custom-resource/sqs_allow_s3_to_put_event.py create mode 100644 source/constructs/lambda/pipeline/common/custom-resource/test/test_sqs_allow_s3_to_put_event.py delete mode 100644 source/constructs/lambda/pipeline/common/opensearch-helper/test/test_osutil.py delete mode 100644 source/constructs/lambda/pipeline/common/opensearch-helper/test/test_state.py delete mode 100644 source/constructs/lambda/pipeline/common/opensearch-helper/util/osutil.py delete mode 100644 source/constructs/lambda/pipeline/common/opensearch-helper/util/state.py delete mode 100644 source/constructs/lambda/pipeline/service/log-processor/boto3_client.py delete mode 100644 source/constructs/lambda/pipeline/service/log-processor/test/test_osutil.py delete mode 100644 source/constructs/lambda/pipeline/service/log-processor/test/test_state.py create mode 100644 source/constructs/lambda/pipeline/service/log-processor/util/helper.py delete mode 100644 source/constructs/lambda/pipeline/service/log-processor/util/osutil.py delete mode 100644 source/constructs/lambda/pipeline/service/log-processor/util/state.py create mode 100644 source/constructs/lib/api/app-table-stack.ts create mode 100644 source/constructs/lib/api/appsync-stack.ts create mode 100644 source/constructs/lib/api/cluster-stack.ts create mode 100644 source/constructs/lib/api/common-resource-stack.ts create mode 100644 source/constructs/lib/api/cwl-stack.ts create mode 100644 source/constructs/lib/api/ec2-iam-instance-profile.ts delete mode 100644 source/constructs/lib/api/eks-cluster-stack.ts delete mode 100644 source/constructs/lib/api/instance-group-stack.ts delete mode 100644 source/constructs/lib/api/instance-meta-stack.ts create mode 100644 source/constructs/lib/api/instance-stack.ts create mode 100644 source/constructs/lib/api/pipeline-alarm-stack.ts delete mode 100644 source/constructs/lib/api/pipeline-flow.ts create mode 100644 source/constructs/lib/api/svc-pipeline-flow.ts create mode 100644 source/constructs/lib/api/svc-pipeline-stack.ts create mode 100644 source/constructs/lib/kinesis/lambda/test/kds.template create mode 100644 source/constructs/lib/kinesis/lambda/test/requirements-test.txt create mode 100644 source/constructs/lib/kinesis/lambda/test/test_index.py create mode 100644 source/constructs/lib/layer/layer.ts create mode 100644 source/constructs/lib/layer/python/requirements.txt create mode 100644 source/constructs/lib/main/migration-stack.ts create mode 100644 source/constructs/lib/pipeline/application/s3-source-stack.ts create mode 100644 source/constructs/lib/pipeline/common/cwl-metric-stack.ts create mode 100644 source/constructs/lib/util/stack-helper.ts rename source/constructs/{lambda/api/instance_group => migration}/.coveragerc (100%) create mode 100644 source/constructs/migration/main.py rename source/constructs/{lambda/pipeline/common/opensearch-helper/util => migration/test}/__init__.py (100%) rename source/constructs/{lambda/api/common => migration}/test/conftest.py (60%) rename source/constructs/{lambda/api/common => migration}/test/requirements-test.txt (65%) create mode 100644 source/constructs/migration/test/test_main.py create mode 100644 source/constructs/migration/utils.py create mode 100644 source/portal/src/assets/images/desc/wafSamplingArch.png create mode 100644 source/portal/src/assets/images/type/amazon_ec2.svg create mode 100644 source/portal/src/assets/images/type/amazon_ec2_arch.svg create mode 100644 source/portal/src/assets/images/type/amazon_eks.svg create mode 100644 source/portal/src/assets/images/type/amazon_eks_arch.svg create mode 100644 source/portal/src/assets/images/type/amazon_s3_2.svg create mode 100644 source/portal/src/assets/images/type/s3-source-on-going.svg create mode 100644 source/portal/src/assets/images/type/s3-source-one-time.svg create mode 100644 source/portal/src/assets/images/type/syslog.svg create mode 100644 source/portal/src/assets/images/type/syslog_arch.svg create mode 100644 source/portal/src/assets/js/hooks/useAlarm.ts create mode 100644 source/portal/src/assets/js/hooks/useAutoValidation.ts create mode 100644 source/portal/src/assets/js/hooks/useTags.ts create mode 100644 source/portal/src/assets/js/init.ts create mode 100644 source/portal/src/components/ButtonRefresh.tsx create mode 100644 source/portal/src/components/ClickableRichTooltip.tsx create mode 100644 source/portal/src/components/ExpandableSection/ExpandableSection.tsx create mode 100644 source/portal/src/components/ExpandableSection/_style.scss create mode 100644 source/portal/src/components/ExpandableSection/index.tsx create mode 100644 source/portal/src/components/ExtButton.tsx create mode 100644 source/portal/src/components/IndicatorWithLink/IndicatorWithLink.tsx create mode 100644 source/portal/src/components/IndicatorWithLink/_style.scss create mode 100644 source/portal/src/components/IndicatorWithLink/index.tsx create mode 100644 source/portal/src/components/Radio/index.tsx rename source/portal/src/components/{S3Select => RichTooltip}/index.tsx (89%) create mode 100644 source/portal/src/components/RichTooltip/richTooltip.tsx delete mode 100644 source/portal/src/components/S3Select/_style.scss delete mode 100644 source/portal/src/components/S3Select/s3Select.tsx create mode 100644 source/portal/src/components/StatusIndicator/StatusIndicator.tsx create mode 100644 source/portal/src/components/StatusIndicator/_style.scss create mode 100644 source/portal/src/components/StatusIndicator/index.tsx create mode 100644 source/portal/src/components/TimeRange/TimeRange.tsx create mode 100644 source/portal/src/help/PipelineAlarms.tsx delete mode 100644 source/portal/src/pages/clusters/domain/detail/Tags.tsx delete mode 100644 source/portal/src/pages/clusters/importcluster/steps/CreateTags.tsx create mode 100644 source/portal/src/pages/clusters/importcluster/steps/ImportDomain.tsx create mode 100644 source/portal/src/pages/comps/Validator.tsx create mode 100644 source/portal/src/pages/comps/monitor/LineChart.tsx create mode 100644 source/portal/src/pages/comps/monitor/MonitorMetrics.tsx delete mode 100644 source/portal/src/pages/containers/eksLog/create/steps/CreateTags.tsx delete mode 100644 source/portal/src/pages/containers/eksLog/create/steps/SpecifyEKSDomain.tsx delete mode 100644 source/portal/src/pages/containers/eksLog/createIngestion/step/CreateTags.tsx delete mode 100644 source/portal/src/pages/containers/eksLog/createIngestion/step/SpecifyLogConfig.tsx delete mode 100644 source/portal/src/pages/containers/eksLog/detail/Tags.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/SelectLogSource.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/ApplyLogConfig.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/IndexName.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/InstancePermission.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/LogConfigSelector.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/LogPathInput.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/common/UnmodifiableLogConfigSelector.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/CreatePipeline.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/ec2/ChooseBufferLayer.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/ec2/ChooseInstanceGroupTable.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/ec2/CreateEC2.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/ec2/PermissionsModeSelector.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/eks/CreateEKS.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/eks/steps/StepChooseEKSSource.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/s3/CompressionFormatSelector.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/s3/CreateS3.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/s3/FilePathPrefixFilter.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/s3/IngestionModeSelector.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/s3/S3BucketSelector.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/steps/CreateTags.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/syslog/CreateSyslog.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/create/syslog/OnlyCreateSyslogIngestion.tsx rename source/portal/src/pages/dataInjection/applicationLog/{createSyslogIngestion/steps/StepChooseSource.tsx => create/syslog/steps/StepChooseSyslogSource.tsx} (82%) delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createEKSIngestion/CreateEKSIngestion.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createEKSIngestion/steps/CreateTags.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createEKSIngestion/steps/StepChooseSource.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createIngestion/CreateIngestion.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createIngestion/steps/ChooseInstanceGroup.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createIngestion/steps/CreateTags.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createIngestion/steps/StepCreateInstanceGroup.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createSyslogIngestion/CreateSysLogIngestion.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/createSyslogIngestion/steps/CreateTags.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/Alarm.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/AnalyticsEngineDetails.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/BufferLayerDetails.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/Lifecycle.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/LogConfigDetails.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/Logging.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/Monitoring.tsx delete mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/Tags.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/monitor/AppLogFlbAgentSourceMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/monitor/AppLogKDSBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/monitor/AppLogProcessorMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/monitor/AppLogSQSBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/applicationLog/detail/monitor/AppLogSyslogSourceMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/common/CreateTags.tsx create mode 100644 source/portal/src/pages/dataInjection/common/PipelineStatus.tsx create mode 100644 source/portal/src/pages/dataInjection/common/SelectOpenSearchDomain.tsx rename source/portal/src/pages/dataInjection/{serviceLog/detail => common}/Tags.tsx (78%) create mode 100644 source/portal/src/pages/dataInjection/ingestiondetail/S3SourceDetail.tsx delete mode 100644 source/portal/src/pages/dataInjection/ingestiondetail/comps/Tags.tsx delete mode 100644 source/portal/src/pages/dataInjection/serviceLog/create/common/CreateTags.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/Alarm.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/Logging.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/Monitoring.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/ServiceLogBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/ServiceLogProcessorMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/cloudfront/CloudFrontRealTimeBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/cloudtrail/CloudTrailCWLBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/lambda/LambdaBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/rds/RDSBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/vpc_flow/VPCFlowLogBufferMetrics.tsx create mode 100644 source/portal/src/pages/dataInjection/serviceLog/detail/monitor/waf/WAFSampledLogProcessorMetrics.tsx create mode 100644 source/portal/src/pages/loggingEvents/EventList.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/Alarm.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/AlarmAndTags.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/AlarmStatus.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/AlarmTable.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/CreateAlarms.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/KDSBufferLayerAlarm.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/LogProcessorAlarm.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/LogSourceAlarm.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/PipelineAlarmTab.tsx create mode 100644 source/portal/src/pages/pipelineAlarm/alarm/S3BufferLayerAlarm.tsx create mode 100644 source/portal/src/pages/pipelineLogging/AppLogging.tsx create mode 100644 source/portal/src/pages/pipelineLogging/Logging.tsx create mode 100644 source/portal/src/pages/pipelineLogging/ServiceLogging.tsx create mode 100644 source/portal/src/pages/pipelineLogging/logging/LoggingHeader.tsx create mode 100644 source/portal/src/pages/pipelineLogging/logging/LoggingTable.tsx create mode 100644 source/portal/src/pages/resources/logConfig/ConfigEditor.tsx create mode 100644 source/portal/src/reducer/createAlarm.ts create mode 100644 source/portal/src/reducer/createTags.ts create mode 100644 source/portal/src/reducer/reducers.ts create mode 100644 source/test/coverage-reports/jest/solutions-constructs/clover.xml create mode 100644 source/test/coverage-reports/jest/solutions-constructs/coverage-final.json create mode 100644 source/test/coverage-reports/jest/solutions-constructs/lcov.info diff --git a/.github/workflows/gh-page.yml b/.github/workflows/gh-page.yml index d2a5941a..c4a17df1 100644 --- a/.github/workflows/gh-page.yml +++ b/.github/workflows/gh-page.yml @@ -17,14 +17,14 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' - name: Install dependencies run: | - python3 -m pip install mkdocs # install mkdocs - python3 -m pip install mkdocs-material # install material theme - python3 -m pip install mkdocs-macros-plugin # install macros plugin - python3 -m pip install mkdocs-include-markdown-plugin # install include-markdown + python3 -m pip install mkdocs==1.3.1 # install mkdocs + python3 -m pip install mkdocs-material==8.5.3 # install material theme + python3 -m pip install mkdocs-macros-plugin==0.7.0 # install macros plugin + python3 -m pip install mkdocs-include-markdown-plugin==3.8.1 # install include-markdown - name: Build mkdocs run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a70ebac..7b0857a9 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,78 @@ # Change Log + All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.0.3] - 2023-06-28 +## [2.0.0] - 2023-08-22 + +### Added + +- Log ingestion from S3 bucket to support more log sources #89 +- Show logs and metrics of the log analytics pipelines #112 +- Quickly enable alarms on log ingestion pipeline #113 +- Show the AWS resource changes when importing AOS using automatic networking mode #53 +- Log Agent Installation: Support of agent installation on AL2023 instances #88 +- Support of multi-AZ standby enabled OpenSearch cluster when creating log pipelines #170 +- Instance Group: Show error message on the console when the installation of log agent fails #169 +- Support same index name in different OpenSearch clusters #166 +- Installation: refresh the aws-exports.json once update the CloudFormation input parameters #161 +- Application log pipeline: Add a step to choose Log Config during the application pipeline creation steps #159 +- Log Agent: Auto rotation of Fluent Bit log file #158 +- Instance group: Add an option to attach IAM policies to Instance Group managed EC2 instances automatically #151 +- Domain management: Check the prerequisites of OpenSearch clusters before import OpenSearch clusters #148 +- Support ingest WAF (associate with CloudFront) sampled logs to OpenSearch in other regions except us-east-1 #129 + +### Fixed + +- Log Config: Time key in Fluent Bit config for Spring Boot should be time type instead of None #71 +- EventBridge will be disabled automatically if deleting instances in instance group #164 +- Log Config should not be created without Regex/Log Format #163 +- Lack of region check before creating WAF log pipeline #162 +- The Fluent bit configuration file generated in sidecar deployment option has a wrong shared volume #160 +- S3 access log dashboard: 5xx Code description is covered #157 +- S3 access log dashboard: The Average Time Unit should be milliseconds #155 +- Cross-account: Unable to get instance list and create instance group in CN region #156 +- The OpenSearch information (e.g., version, data nodes) is not updated automatically after customer upgraded the cluster #150 +- Cannot differentiate the Lambda for different AWS Service log pipeline based on Lambda description #146 +- Fix data lost when cannot find the location with IP address using MaxMind database #126 +- Syslog: Fix port conflict when adding & deleting new log source in parallel #174 + +### Changed + +- Minimize the permissions of EC2 log ingestion IAM role #154 +- Minimize the privileges of cross-account access role #153 +- Soft delete when removing OpenSearch domain #152 +- Save ALB access logs of Nginx based proxy to S3 bucket #149 +- Code refactor: DynamoDB design optimization and GraphQL API design optimization #147 +- Minimize security group egress of the provisioned ECS #145 +- WAF dashboard: Cannot filter results using `nonTerminatingMatchingRules.action` field #144 + +### Removed + +- Domain management: Remove the support of Elasticsearch engine #176 + +## [1.0.3] - 2023-06-27 + ### Fixed -- Fix the processor Lambda function urllib3 version issue + +- Fix the processor Lambda function urllib3 version issue #138 ## [1.0.2] - 2023-06-21 + ### Fixed -- Fix the EKS Fluent-Bit deployment configuration generation issue + +- Support generation of Kubernetes YAML configuration file for EKS 1.24~1.27 #133 ## [1.0.1] - 2023-04-17 + ### Fixed + - Fix deployment failure due to S3 ACL changes ## [1.0.0] - 2023-03-16 + ### Added + - All files, initial version diff --git a/NOTICE.txt b/NOTICE.txt index cf32743b..4d91c8cf 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -11,11 +11,116 @@ THIRD PARTY COMPONENTS ********************** This software includes third party software subject to the following copyrights: -AWS SDK under the Apache License Version 2.0 -user-agents under the Apache License Version 2.0 -maxminddb under the Apache License Version 2.0 -requests-aws4auth under the Apache License Version 2.0 -urllib3 under the MIT License -requests under the Apache License Version 2.0 +awscli under the Apache License Version 2.0 boto3 under the Apache License Version 2.0 -botocore under the Apache License Version 2.0 \ No newline at end of file +botocore under the Apache License Version 2.0 +defusedxml under the Python Software Foundation License +py-serializable under the Apache License Version 2.0 +pathable under the Apache License Version 2.0 (details in this link: https://github.com/p1c2u/pathable/blob/master/LICENSE) +Jinja2 under the BSD-3-Clause +MarkupSafe under the BSD-3-Clause +Werkzeug under the BSD-3-Clause +attrs under the MIT License +cffi under the MIT License +colorama under the BSD License +coverage under the Apache License Version 2.0 +cryptography under the Apache License Version 2.0 or OR BSD-3-Clause +docker under the Apache License Version 2.0 +docutils under BSD License, GNU General Public License (GPL), Python Software Foundation License, Public Domain (public domain, Python, BSD-2-Clause, GPL 3(see https://docutils.sourceforge.io/COPYING.txt)) +flake8 under the MIT License +iniconfig under the MIT License +ipaddr under the Apache License Version 2.0 +jmespath under the MIT License +jsonschema-spec under the Apache License Version 2.0 +lazy-object-proxy under the BSD-2-Clause +maxminddb under the Apache License Version 2.0 +mccabe under the MIT License +moto under the Apache License Version 2.0 +openapi-schema-validator under the BSD-3-Clause +pluggy under the MIT License +pyOpenSSL under the Apache License Version 2.0 +pyasn1 under the BSD-2-Clause +pycodestyle under the MIT License +pycparser under the BSD License +pydantic under the MIT License +pyflakes under the MIT License +pyrsistent under the MIT License +pytest under the MIT License +pytest-cov under the MIT License +pytest-mock under the MIT License +python-dateutil under Apache Software License, BSD License (Dual License) +requests-aws4auth under the MIT License +requests-mock under the Apache License Version 2.0 +responses under the Apache License Version 2.0 +rfc3339-validator under the MIT License +rsa under the Apache License Version 2.0 +s3transfer under the Apache License Version 2.0 +types-PyYAML under the Apache License Version 2.0 +typing_extensions under Python Software Foundation License +ua-parser under the Apache License Version 2.0 +user-agents under the MIT License +websocket-client under the Apache License Version 2.0 +xmltodict under the MIT License +@aws-cdk/aws-appsync-alpha under the Apache License Version 2.0 +@aws-cdk/aws-kinesisfirehose-alpha under the Apache License Version 2.0 +@aws-cdk/aws-kinesisfirehose-destinations-alpha under the Apache License Version 2.0 +@aws-solutions-constructs/aws-cloudfront-s3 under the Apache License Version 2.0 +@typescript-eslint/eslint-plugin under the MIT License +@typescript-eslint/parser under the BSD-2-Clause +eslint under the MIT License +eslint-config-prettier under the MIT License +eslint-import-resolver-node under the MIT License +eslint-import-resolver-typescript under the ISC License +eslint-plugin-import under the MIT License +eslint-plugin-prettier under the MIT License +aws-cdk under the Apache License Version 2.0 +aws-cdk-lib under the Apache License Version 2.0 +cdk-nag under the Apache License Version 2.0 +source-map-support under the MIT License +fs under the MIT License +exceptiongroup under the MIT License +tomli under MIT License +@apollo/client under the MIT License +@aws-amplify/ui-components under the Apache License Version 2.0 +@aws-amplify/ui-react under the Apache License Version 2.0 +@material-ui/core under the MIT License +@material-ui/icons under the MIT license +@material-ui/lab under the MIT License +@testing-library/jest-dom under the MIT License +@testing-library/react under the MIT License +@testing-library/user-event under the MIT License +@types/jest under the MIT License +@types/node under the MIT License +@types/react under the MIT License +@types/react-dom under the MIT License +amplify under the MIT License +apexcharts under the MIT License +apollo-link under the MIT License +aws-amplify under the Apache License Version 2.0 +aws-appsync-auth-link under the Apache License Version 2.0 +aws-appsync-subscription-link under the Apache License Version 2.0 +axios under the MIT License +classnames under the MIT License +date-fns under the MIT License +graphql-tag under the MIT License +i18next under the MIT License +i18next-browser-languagedetector under the MIT License +i18next-http-backend under the MIT License +lodash.clonedeep under the MIT License +moment under the MIT License +node-sass under the Apache License Version 2.0 +oidc-client-ts under the Apache License Version 2.0 +react under the MIT License +react-apexcharts under the MIT License +react-copy-to-clipboard under the MIT License +react-dom under the MIT License +react-i18next under the MIT License +react-minimal-datetime-range under the MIT License +react-oidc-context under the MIT License +react-redux under the MIT License +react-router-dom under the MIT License +redux under the MIT License +sweetalert2 under the MIT License +typescript under the Apache License Version 2.0 +web-vitals under the Apache License Version 2.0 +notice-js under the MIT-0 diff --git a/README.md b/README.md index a0495911..0ae1b227 100755 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ The Centralized Logging with OpenSearch solution provides comprehensive log mana ## Table of content -- [Solution Overview](#solution-overview) -- [Architecture](#architecture) -- [Deployment](#deployment) -- [Customization](#customization) -- [License](#License) +- [Centralized Logging with OpenSearch](#centralized-logging-with-opensearch) + - [Table of content](#table-of-content) + - [Solution Overview](#solution-overview) + - [Architecture](#architecture) + - [Deployment](#deployment) + - [Customization](#customization) + - [Collection of operational metrics](#collection-of-operational-metrics) ## Solution Overview @@ -17,9 +19,9 @@ The solution has the following features: - **All-in-one log ingestion**: provides a single web console to ingest both application logs and AWS service logs into the Amazon OpenSearch (AOS) domains. -- **Codeless log processor**: supports log processor plugins developed by AWS. You are allowed to enrich the raw log data through a few clicks on the web console. +- **Codeless log processor**: supports log processor plugins developed by AWS. You are allowed to enrich the raw log data through a few clicks on the web console. -- **Out-of-box dashboard template**: offers a collection of reference designs of visualization templates, for both commonly used software such as Nginx and Apache HTTP Server, and AWS services such as Amazon S3 and Amazon CloudTrail. +- **Out-of-box dashboard template**: offers a collection of reference designs of visualization templates, for both commonly used software such as Nginx and Apache HTTP Server, and AWS services such as Amazon S3 and Amazon CloudTrail. @@ -40,7 +42,11 @@ Please follow the [Implementation Guide](https://docs.aws.amazon.com/solutions/l Please follow the [Customization Guide](CUSTOM_BUILD.md) for custom build. -## License +## Collection of operational metrics + +This solution collects anonymous operational metrics to help AWS improve the quality and features of the solution. For more information, including how to disable this capability, please see the [implementation guide](https://docs.aws.amazon.com/solutions/latest/centralized-logging-with-opensearch/collection-of-operational-metrics.html). + +*** Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh index 005b0058..4b990e10 100755 --- a/deployment/build-s3-dist.sh +++ b/deployment/build-s3-dist.sh @@ -276,31 +276,6 @@ echo "${bold}[Create] Templates${normal}" echo "------------------------------------------------------------------------------" if fn_exists create_template_${template_format}; then - rm -vf ./lambda/api/app_pipeline/common.py - rm -vf ./lambda/api/app_log_ingestion/common.py - rm -vf ./lambda/api/app_log_ingestion/util/aws_svc_mgr.py - rm -vf ./lambda/api/app_log_ingestion/aws_svc_mgr.py - rm -vf ./lambda/api/pipeline/aws_svc_mgr.py - rm -vf ./lambda/api/log_agent_status/aws_svc_mgr.py - rm -vf ./lambda/api/instance_meta/aws_svc_mgr.py - rm -vf ./lambda/api/instance_group/aws_svc_mgr.py - rm -vf ./lambda/api/resource/aws_svc_mgr.py - rm -vf ./lambda/api/eks_cluster/aws_svc_mgr.py - rm -vf ./lambda/main/cfnHelper/aws_svc_mgr.py - - cp -vf ./lambda/pipeline/service/log-processor/../../common/custom-resource/boto3_client.py ./lambda/pipeline/service/log-processor/boto3_client.py - cp -vf ./lambda/api/app_pipeline/../common/common.py ./lambda/api/app_pipeline/common.py - cp -vf ./lambda/api/app_log_ingestion/../common/common.py ./lambda/api/app_log_ingestion/common.py - cp -vf ./lambda/api/pipeline/../common/aws_svc_mgr.py ./lambda/api/pipeline/aws_svc_mgr.py - cp -vf ./lambda/api/instance_group/../common/aws_svc_mgr.py ./lambda/api/instance_group/aws_svc_mgr.py - cp -vf ./lambda/api/app_log_ingestion/../common/aws_svc_mgr.py ./lambda/api/app_log_ingestion/aws_svc_mgr.py - cp -vf ./lambda/api/log_agent_status/../common/aws_svc_mgr.py ./lambda/api/log_agent_status/aws_svc_mgr.py - cp -vf ./lambda/api/instance_meta/../common/aws_svc_mgr.py ./lambda/api/instance_meta/aws_svc_mgr.py - cp -vf ./lambda/api/resource/../common/aws_svc_mgr.py ./lambda/api/resource/aws_svc_mgr.py - cp -vf ./lambda/api/eks_cluster/../common/aws_svc_mgr.py ./lambda/api/eks_cluster/aws_svc_mgr.py - cp -vf ./lambda/main/cfnHelper/../../api/common/aws_svc_mgr.py ./lambda/main/cfnHelper/aws_svc_mgr.py - - create_template_${template_format} else echo "Invalid setting for \$template_format: $template_format" diff --git a/deployment/cdk-solution-helper/package.json b/deployment/cdk-solution-helper/package.json index 89fac67a..a523255c 100755 --- a/deployment/cdk-solution-helper/package.json +++ b/deployment/cdk-solution-helper/package.json @@ -1,6 +1,8 @@ { "name": "cdk-solution-helper", + "description": "cdk solution helper", "version": "0.1.0", + "license": "Apache-2.0", "devDependencies": { "fs": "0.0.1-security" }, diff --git a/deployment/run-unit-tests.sh b/deployment/run-unit-tests.sh new file mode 100644 index 00000000..082e4bcd --- /dev/null +++ b/deployment/run-unit-tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# You can remove this script if you do NOT have unit test. +# +# This script should be run from the repo's deployment directory +# cd deployment +# ./run-unit-tests.sh +# +source_template_dir="$PWD" +cd $source_template_dir/../source +./run-all-tests.sh \ No newline at end of file diff --git a/docs/en/images b/docs/en/images deleted file mode 120000 index 5e675731..00000000 --- a/docs/en/images +++ /dev/null @@ -1 +0,0 @@ -../images \ No newline at end of file diff --git a/docs/en/images b/docs/en/images new file mode 100644 index 00000000..5e675731 --- /dev/null +++ b/docs/en/images @@ -0,0 +1 @@ +../images \ No newline at end of file diff --git a/docs/en/implementation-guide/alarm.md b/docs/en/implementation-guide/alarm.md new file mode 100644 index 00000000..5deed572 --- /dev/null +++ b/docs/en/implementation-guide/alarm.md @@ -0,0 +1,37 @@ +There are different types of log alarms: log processor alarms, buffer layer alarms, and source alarms (only for application log pipeline). The alarms will be triggered when the defined condition is met. + +| Log alarm type | Log alarm condition | Description | +| -------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Log processor alarms | Error invocation # >= 10 for 5 minutes, 1 consecutive time | When the number of log processor Lambda error calls is greater than or equal to 10 within 5 minutes (including 5 minutes), an email alarm will be triggered. | +| Log processor alarms | Failed record # >= 1 for 1 minute, 1 consecutive time | When the number of failed records is greater than or equal to 1 within a 1-minute window, an alarm will be triggered. | +| Log processor alarms | Average execution duration in last 5 minutes >= 60000 milliseconds | In the last 5 minutes, when the average execution time of log processor Lambda is greater than or equal to 60 seconds, an email alarm will be triggered. | +| Buffer layer alarms | SQS Oldest Message Age >= 30 minutes | When the age of the oldest SQS message is greater than or equal to 30 minutes, it means that the message has not been consumed for at least 30 minutes, an email alarm will be triggered. | +| Source alarms (only for application log pipeline) | Fluent Bit output_retried_record_total >= 100 for last 5 minutes | When the total number of retry records output by Fluent Bit in the past 5 minutes is greater than or equal to 100, an email alarm will be triggered. | + +You can choose to enable log alarms or disable them according to your needs. + +## Enable log alarms + +1. Sign in to the Centralized Logging with OpenSearch console. + +2. In the left navigation bar, under **Log Analytics Pipelines**, choose **AWS Service Log** or **Application Log**. + +3. Select the log pipeline created and choose **View details**. + +4. Select the **Alarm** tab. + +5. Switch on **Alarms** if needed and select an exiting SNS topic. + +6. If you choose **Create a new SNS topic**, you need to provide email address for the newly-created SNS topic to notify. + +## Disable log alarms + +1. Sign in to the Centralized Logging with OpenSearch console. + +2. In the left navigation bar, under **Log Analytics Pipelines**, choose **AWS Service Log** or **Application Log**. + +3. Select the log pipeline created and choose **View details**. + +4. Select the **Alarm** tab. + +5. Switch off **Alarms**. \ No newline at end of file diff --git a/docs/en/implementation-guide/applications/apache.md b/docs/en/implementation-guide/applications/apache.md index 15db281e..5c2b3b70 100644 --- a/docs/en/implementation-guide/applications/apache.md +++ b/docs/en/implementation-guide/applications/apache.md @@ -45,14 +45,38 @@ For Apache HTTP server logs, Centralized Logging with OpenSearch will create a b 2. Go to **Dashboard** section in the left sidebar. 3. Find the dashboard whose name starts with ``. -## Sample Dashboard +## View dashboard + +The dashboard includes the following visualizations. + +| Visualization Name | Source Field | Description | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Total Request | log event | Displays aggregated events within a specified time interval. | +| Status Codes | status | Displays the distribution of HTTP response codes served by the Apache web server over a selected time period. | +| Access History | log event | Shows a historical log of all requests handled by the Apache web server, visualized using a bar chart. This allows administrators to analyze traffic volumes and patterns over time. | +| Unique Visitors | remote_addr | Provides an estimation of the total number of users or devices accessing the Apache server for repeated visits from the same IP. | +| Status Code Metric | status | Displays the distribution of HTTP response codes served by the Apache server over a period of time. | +| Top Access Paths | request_uri | Shows the most frequently requested URLs on the Apache server over a selected time period. | +| Top Client IPs | | Displays the 10 source IP addresses generating the highest number of requests received by the Apache server during a selected time period. | +| Bandwidth | response_size_bytes | Tracks the volume of data transferred from the Apache server to clients over a selected time period. | +| Top Agents | http_user_agent | Provides a snapshot of the visitor profile. The data guides decisions to better serve core audiences. | +| Http Methods | request_method | Presents a pie chart that shows the distribution of request methods handled by the Apache server during a selected time period. | +| Top Access URIs | request_uri | Shows the most frequently hit URI paths handled by Apache during a selected period. | +| Top Referers | http_referer | Referers are the URLs of pages that link to requests for the application. Tracking referers reveals the primary external sources of visits and engagement. | +| Bandwidth History | response_size_bytes | Shows the historical trend of the data transfer activities by the Apache server to clients. | +| Apache Error Log | | Provides a detailed record of errors encountered by the web server. | +| Apache Log Examples | | A quick entry for interpreting Apache access and error logs. | + + +### Sample Dashboard -[![nginx-1]][nginx-1] -[![nginx-2]][nginx-2] +{% +include-markdown "../include-dashboard.md" +%} +[![apache]][apache] -[nginx-1]: ../../images/dashboards/nginx-1.png -[nginx-2]: ../../images/dashboards/nginx-2.png +[apache]: ../../images/dashboards/apache.png [apache-http-logs]: https://httpd.apache.org/docs/2.4/logs.html diff --git a/docs/en/implementation-guide/applications/create-applog-pipeline.md b/docs/en/implementation-guide/applications/create-applog-pipeline.md index 95686558..fc4b1484 100644 --- a/docs/en/implementation-guide/applications/create-applog-pipeline.md +++ b/docs/en/implementation-guide/applications/create-applog-pipeline.md @@ -1,12 +1,156 @@ + +#### Instance Group as Log Source + +1. Sign in to the Centralized Logging with OpenSearch Console. + +2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + +3. Choose Instance Group as **Log Source**. + +4. Select the instance group you have created, + +5. (Auto Scaling Group only) If your instance group is created based on an Auto Scaling Group, after ingestion status become "Created", then you can find the generated Shell Script in the instance group's detail page. Copy the shell script and update the User Data of the Auto Scaling [Launch configurations](https://docs.aws.amazon.com/autoscaling/ec2/userguide/launch-configurations.html) or [Launch template](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html). + +6. Choose the **Permission grant method**. If you choose **I will manually add the below required permissions after pipeline creation**, you have to click **Expand to view required permissions** and copy the provided JSON policy. + +7. Go to **AWS Console > IAM > Policies** on the left column, and + + 1. Choose **Create Policy**, choose **JSON** and replace all the content inside the text block. Remember to substitute `` with your account id. + + 2. Choose **Next**, **Next**, then enter the name for this policy. + + 3. Attach the policy to your EC2 instance profile to grant the log agent permissions to send logs to the application log pipeline. If you are using Auto Scaling group, you need to update the IAM instance profile associated with the Auto Scaling Group. If needed, you can follow the documentation to update your [launch template][launch-template] or [launch configuration][launch-configuration]. + +8. Input **Log Path** and select the log config created in previous setup, choose **Next**. + +9. Specify **Index name** in lowercase. + +10. In the **Buffer** section, choose **S3** or **Kinesis Data Streams**. If you don't want the buffer layer, choose **None**. Refer to the [Log Buffer](./index.md#log-buffer) for more information about choosing the appropriate buffer layer. + + * S3 buffer parameters + + | Parameter | Default | Description | + | ---------------------------- | ------------------------------------------------ | ------------------------------------------------------------ | + | S3 Bucket | *A log bucket will be created by the solution.* | You can also select a bucket to store the log data. | + | S3 Bucket Prefix | `AppLogs//year=%Y/month=%m/day=%d` | The log agent appends the prefix when delivering the log files to the S3 bucket. | + | Buffer size | 50 MiB | The maximum size of log data cached at the log agent side before delivering to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Buffer interval | 60 seconds | The maximum interval of the log agent to deliver logs to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Compression for data records | `Gzip` | The log agent compresses records before delivering them to the S3 bucket. | + + * Kinesis Data Streams buffer parameters + + | Parameter | Default | Description | + | -------------------- | ------------------ | ------------------------------------------------------------ | + | Shard number | `` | The number of shards of the Kinesis Data Streams. Each shard can have up to 1,000 records per second and total data write rate of 1MB per second. | + | Enable auto scaling | `No` | This solution monitors the utilization of Kinesis Data Streams every 5 minutes, and scale in/out the number of shards automatically. The solution will scale in/out for a maximum of 8 times within 24 hours. | + | Maximum Shard number | `` | Required if auto scaling is enabled. The maximum number of shards. | + + !!! important "Important" + You may observe duplicate logs in OpenSearch if threshold error occurs in Kinesis Data Streams (KDS). This is because the Fluent Bit log agent uploads logs in [chunk](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure) (contains multiple records), and will retry the chunk if upload failed. Each + KDS shard can support up to 1,000 records per second for writes, up to a maximum total data write rate of 1 MB per second. Please estimate your log volume and choose an appropriate shard number. + +11. Choose **Next**. + +12. In the **Specify OpenSearch domain** section, select an imported domain for **Amazon OpenSearch domain**. + +13. In the **Log Lifecycle** section, enter the number of days to manage the Amazon OpenSearch Service index lifecycle. The Centralized Logging with OpenSearch will create the associated [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) policy automatically for this pipeline. + +14. Choose **Next**. + +15. Enable **Alarms** if needed and select an exiting SNS topic. If you choose **Create a new SNS topic**, please provide a name and an email address for the new SNS topic. + +16. Add tags if needed. + +17. Choose **Create**. + +18. Wait for the application pipeline turning to "Active" state. + + + + +### Amazon EKS Cluster as Log Source + +1. Sign in to the Centralized Logging with OpenSearch Console. + +2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + +3. Choose the AWS account at which the log is stored and the EKS Cluster that has been imported as Log Source during the **Prerequisites**. + +4. Enter the location of the log files. + +5. Select the log config created in previous setup, and choose **Next**. + +6. Specify **Index name** in lowercase. + +7. In the **Buffer** section, choose **S3** or **Kinesis Data Streams**. If you don't want the buffer layer, choose **None**. Refer to the [Log Buffer](./index.md#log-buffer) for more information about choosing the appropriate buffer layer. + + * S3 buffer parameters + + | Parameter | Default | Description | + | ---------------------------- | ------------------------------------------------ | ------------------------------------------------------------ | + | S3 Bucket | *A log bucket will be created by the solution.* | You can also select a bucket to store the log data. | + | S3 Bucket Prefix | `AppLogs//year=%Y/month=%m/day=%d` | The log agent appends the prefix when delivering the log files to the S3 bucket. | + | Buffer size | 50 MiB | The maximum size of log data cached at the log agent side before delivering to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Buffer interval | 60 seconds | The maximum interval of the log agent to deliver logs to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Compression for data records | `Gzip` | The log agent compresses records before delivering them to the S3 bucket. | + + * Kinesis Data Streams buffer parameters + + | Parameter | Default | Description | + | -------------------- | ------------------ | ------------------------------------------------------------ | + | Shard number | `` | The number of shards of the Kinesis Data Streams. Each shard can have up to 1,000 records per second and total data write rate of 1MB per second. | + | Enable auto scaling | `No` | This solution monitors the utilization of Kinesis Data Streams every 5 minutes, and scale in/out the number of shards automatically. The solution will scale in/out for a maximum of 8 times within 24 hours. | + | Maximum Shard number | `` | Required if auto scaling is enabled. The maximum number of shards. | + + !!! important "Important" + You may observe duplicate logs in OpenSearch if threshold error occurs in Kinesis Data Streams (KDS). This is because the Fluent Bit log agent uploads logs in [chunk](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure) (contains multiple records), and will retry the chunk if upload failed. Each + KDS shard can support up to 1,000 records per second for writes, up to a maximum total data write rate of 1 MB per second. Please estimate your log volume and choose an appropriate shard number. + +8. Choose **Next**. + +9. In the **Specify OpenSearch domain** section, select an imported domain for **Amazon OpenSearch domain**. + +10. In the **Log Lifecycle** section, enter the number of days to manage the Amazon OpenSearch Service index lifecycle. The Centralized Logging with OpenSearch will create the associated [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) policy automatically for this pipeline. + +11. Choose **Next**. + +12. Enable **Alarms** if needed and select an exiting SNS topic. If you choose **Create a new SNS topic**, please provide a name and an email address for the new SNS topic. + +13. Add tags if needed. + +14. Choose **Create** to finish creating an ingestion. + +15. Wait for the application pipeline turning to "Active" state. + +16. Deploy Fluent Bit log agent following the guide generated by Centralized Logging with OpenSearch. + + 1. Select the application pipeline created in previous setup + + 2. Select the App Log Ingestion just created. + + 3. Follow **DaemonSet** or **Sidecar** Guide to deploy the log agent. + + + + +### Amazon S3 as Log Source 1. Sign in to the Centralized Logging with OpenSearch Console. 2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. -3. Click the **Create a pipeline**. +3. Choose Amazon S3 as **Log Source**. + +4. Choose the S3 bucket where your logs are stored and enter **Prefix filter** (note that **Prefix filter** is optional). + +5. Choose **Ingestion mode** based on your need. If you want to ingest the log continuously, select **On-going**; If you only need to ingest the log once, select **One-time**. + +6. Specify **Compression format** if your log files are compressed. + +7. Select the log config created in the previous setup, and choose **Next**. -4. Specify **Index name** in lowercase. +8. Specify **Index name** in lowercase. -5. In the **Buffer** section, choose **S3** or **Kinesis Data Streams**. If you don't want the buffer layer, choose **None**. Refer to the [Log Buffer](./index.md#log-buffer) for more information about choosing the appropriate buffer layer. +9. In the **Buffer** section, choose **S3** or **Kinesis Data Streams**. If you don't want the buffer layer, choose **None**. Refer to the [Log Buffer](./index.md#log-buffer) for more information about choosing the appropriate buffer layer. * S3 buffer parameters @@ -26,21 +170,83 @@ | Enable auto scaling | `No` | This solution monitors the utilization of Kinesis Data Streams every 5 minutes, and scale in/out the number of shards automatically. The solution will scale in/out for a maximum of 8 times within 24 hours. | | Maximum Shard number | `` | Required if auto scaling is enabled. The maximum number of shards. | + !!! important "Important" + You may observe duplicate logs in OpenSearch if threshold error occurs in Kinesis Data Streams (KDS). This is because the Fluent Bit log agent uploads logs in [chunk](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure) (contains multiple records), and will retry the chunk if upload failed. Each + KDS shard can support up to 1,000 records per second for writes, up to a maximum total data write rate of 1 MB per second. Please estimate your log volume and choose an appropriate shard number. + +10. Choose **Next**. + +11. In the **Specify OpenSearch domain** section, select an imported domain for **Amazon OpenSearch domain**. + +12. In the **Log Lifecycle** section, enter the number of days to manage the Amazon OpenSearch Service index lifecycle. The Centralized Logging with OpenSearch will create the associated [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) policy automatically for this pipeline. + +13. Choose **Next**. + +14. Enable **Alarms** if needed and select an exiting SNS topic. If you choose **Create a new SNS topic**, please provide a name and an email address for the new SNS topic. + +15. Add tags if needed. + +16. Choose **Create**. + +17. Wait for the application pipeline turning to "Active" state. + + + +### Syslog as Log Source + +1. Sign in to the Centralized Logging with OpenSearch Console. + +2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + +3. Choose Syslog Endpoint as **Log Source**. + +4. You can use UDP or TCP with custom port number. Choose **Next**. + +5. Select the log config created in the previous setup, and choose **Next**. + +6. Specify **Index name** in lowercase. + +7. In the **Buffer** section, choose **S3** or **Kinesis Data Streams**. If you don't want the buffer layer, choose **None**. Refer to the [Log Buffer](./index.md#log-buffer) for more information about choosing the appropriate buffer layer. + + * S3 buffer parameters + + | Parameter | Default | Description | + | ---------------------------- | ------------------------------------------------ | ------------------------------------------------------------ | + | S3 Bucket | *A log bucket will be created by the solution.* | You can also select a bucket to store the log data. | + | S3 Bucket Prefix | `AppLogs//year=%Y/month=%m/day=%d` | The log agent appends the prefix when delivering the log files to the S3 bucket. | + | Buffer size | 50 MiB | The maximum size of log data cached at the log agent side before delivering to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Buffer interval | 60 seconds | The maximum interval of the log agent to deliver logs to S3. For more information, see [Data Delivery Frequency](https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#frequency). | + | Compression for data records | `Gzip` | The log agent compresses records before delivering them to the S3 bucket. | + + * Kinesis Data Streams buffer parameters + + | Parameter | Default | Description | + | -------------------- | ------------------ | ------------------------------------------------------------ | + | Shard number | `` | The number of shards of the Kinesis Data Streams. Each shard can have up to 1,000 records per second and total data write rate of 1MB per second. | + | Enable auto scaling | `No` | This solution monitors the utilization of Kinesis Data Streams every 5 minutes, and scale in/out the number of shards automatically. The solution will scale in/out for a maximum of 8 times within 24 hours. | + | Maximum Shard number | `` | Required if auto scaling is enabled. The maximum number of shards. | !!! important "Important" You may observe duplicate logs in OpenSearch if threshold error occurs in Kinesis Data Streams (KDS). This is because the Fluent Bit log agent uploads logs in [chunk](https://docs.fluentbit.io/manual/administration/buffering-and-storage#chunks-memory-filesystem-and-backpressure) (contains multiple records), and will retry the chunk if upload failed. Each KDS shard can support up to 1,000 records per second for writes, up to a maximum total data write rate of 1 MB per second. Please estimate your log volume and choose an appropriate shard number. -6. Choose **Next**. +8. Choose **Next**. + +9. In the **Specify OpenSearch domain** section, select an imported domain for **Amazon OpenSearch domain**. + +10. In the **Log Lifecycle** section, enter the number of days to manage the Amazon OpenSearch Service index lifecycle. The Centralized Logging with OpenSearch will create the associated [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) policy automatically for this pipeline. + +11. Choose **Next**. -7. In the **Specify OpenSearch domain** section, select an imported domain for **Amazon OpenSearch domain**. +12. Enable **Alarms** if needed and select an exiting SNS topic. If you choose **Create a new SNS topic**, please provide a name and an email address for the new SNS topic. -8. In the **Log Lifecycle** section, enter the number of days to manage the Amazon OpenSearch Service index lifecycle. The Centralized Logging with OpenSearch will create the associated [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) policy automatically for this pipeline. +13. Add tags if needed. -9. Choose **Next**. +14. Choose **Create**. -10. Add tags if needed. +15. Wait for the application pipeline turning to "Active" state. -11. Choose **Create**. + -12. Wait for the application pipeline turning to "Active" state. \ No newline at end of file +[launch-template]: https://docs.aws.amazon.com/autoscaling/ec2/userguide/create-launch-template.html#advanced-settings-for-your-launch-template +[launch-configuration]: https://docs.aws.amazon.com/autoscaling/ec2/userguide/change-launch-config.html \ No newline at end of file diff --git a/docs/en/implementation-guide/applications/create-log-ingestion.md b/docs/en/implementation-guide/applications/create-log-ingestion.md index 8d6dd444..9bddfded 100644 --- a/docs/en/implementation-guide/applications/create-log-ingestion.md +++ b/docs/en/implementation-guide/applications/create-log-ingestion.md @@ -1,23 +1,33 @@ - ### Instance Group as Log Source 1. Sign in to the Centralized Logging with OpenSearch Console. + 2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + 3. Choose the application pipeline that has been created during the **Prerequisites**. -4. Go to **Permission** tab and copy the provided JSON policy. -5. Go to **AWS Console > IAM > Policies** on the left column, and + +4. Choose the **Permission grant method**. If you choose **I will manually add the below required permissions after pipeline creation**, you have to click **Expand to view required permissions** and copy the provided JSON policy. + +5. Go to **AWS Console > IAM > Policies** on the left column, and 1. Choose **Create Policy**, choose **JSON** and replace all the content inside the text block. Remember to substitute `` with your account id. + 2. Choose **Next**, **Next**, then enter the name for this policy. + 3. Attach the policy to your EC2 instance profile to grant the log agent permissions to send logs to the application log pipeline. If you are using Auto Scaling group, you need to update the IAM instance profile associated with the Auto Scaling Group. If needed, you can follow the documentation to update your [launch template][launch-template] or [launch configuration][launch-configuration]. 6. Click the **Create an Ingestion** dropdown menu, and select **From Instance Group**. + 7. Select **Choose exists** and choose **Next**. + 8. Select the instance group you have created during the **Prerequisites** and choose **Next**. + 9. (Auto Scaling Group only) If your instance group is created based on an Auto Scaling Group, after ingestion status become "Created", then you can find the generated Shell Script in the instance group's detail page. Copy the shell script and update the User Data of the Auto Scaling [Launch configurations](https://docs.aws.amazon.com/autoscaling/ec2/userguide/launch-configurations.html) or [Launch template](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html). + 10. Select **Choose exists** and select the log config created in previous setup. -11. Choose **Next**, then choose **Create**. + +11. Choose **Create** to finish creating an ingestion. [launch-template]: https://docs.aws.amazon.com/autoscaling/ec2/userguide/create-launch-template.html#advanced-settings-for-your-launch-template [launch-configuration]: https://docs.aws.amazon.com/autoscaling/ec2/userguide/change-launch-config.html @@ -25,31 +35,69 @@ -### EKS Cluster as Log Source +### Amazon EKS Cluster as Log Source 1. Sign in to the Centralized Logging with OpenSearch Console. + 2. In the left sidebar, under **Log Source**, choose **EKS Clusters**. + 3. Choose the EKS Cluster that has been imported as Log Source during the **Prerequisites**. + 4. Go to **App Log Ingestion** tab and choose **Create an Ingestion**. + 1. Select **Choose exists** and choose the application pipeline created during the **Prerequisites**. Choose **Next**. - 2. Select the log config created in previous setup, and choose **Next**. - 3. Add tags as needed, then choose **Create** to finish creating an ingestion. -5. Deploy Fluent-bit log agent following the guide generated by Centralized Logging with OpenSearch. + + 2. Select the log config created in previous setup. + + 3. Choose **Create** to finish creating an ingestion. + +5. Deploy Fluent-bit log agent following the guide generated by Centralized Logging with OpenSearch. + 1. Select the App Log Ingestion just created. + 2. Follow **DaemonSet** or **Sidecar** Guide to deploy the log agent. + +### Amazon S3 as Log Source + +1. Sign in to the Centralized Logging with OpenSearch Console. + +2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + +3. Choose the application pipeline that has been created during the **Prerequisites**. + +4. Choose the **Create an Ingestion** dropdown menu, and select **From S3 bucket**. + +5. Choose the S3 bucket where your logs are stored and enter **Prefix filter** + +6. Choose **Ingestion mode** based on your need. If you want to ingest the log continuously, select **On-going**; If you only need to ingest the log once, select **One-time**. + +7. Specify **Compression format** if your log files are compressed. + +8. Select the log config created in the previous setup. + +9. Choose **Create** to finish creating an ingestion. + + + ### Syslog as Log Source 1. Sign in to the Centralized Logging with OpenSearch Console. + 2. In the left sidebar, under **Log Analytics Pipelines**, choose **Application Log**. + 3. Choose the application pipeline that has been created during the **Prerequisites**. + 4. Choose the **Create an Ingestion** dropdown menu, and select **From Syslog**. -5. Fill in all the form fields to specify Syslog Source. You can use UDP or TCP with custom port number. Choose **Next**. -6. Select the log config created in the previous setup, and choose **Next**. -7. Add tags as needed, then choose **Create** to finish creating an ingestion. + +5. You can use UDP or TCP with custom port number. Choose **Next**. + +6. Select the log config created in the previous setup. + +7. Choose **Create** to finish creating an ingestion. diff --git a/docs/en/implementation-guide/applications/create-log-source.md b/docs/en/implementation-guide/applications/create-log-source.md index 1b77f13f..e05e656a 100644 --- a/docs/en/implementation-guide/applications/create-log-source.md +++ b/docs/en/implementation-guide/applications/create-log-source.md @@ -2,13 +2,14 @@ You need to create a log source first before collecting application logs. Centra * [Amazon EC2 instance group](#amazon-ec2-instance-group) * [Amazon EKS cluster](#amazon-eks-cluster) +* [Amazon S3](#amazon-s3) * [Syslog](#syslog) For more information, see [concepts](./index.md#concepts). ## Amazon EC2 Instance Group -An instance group represents a group of EC2 Linux instances, which enables the solution to associate a [Log Config](./index.md#log-config) with multiple EC2 instances quickly. Centralized Logging with OpenSearch uses [Systems Manager Agent(SSM Agent)][ssm-agent]{target="_blank"} to install/configure Fluent Bit agent, and sends log data to [Kinesis Data Streams][kds]{target="_blank"}. +An instance group represents a group of EC2 Linux instances, which enables the solution to associate a [Log Config](./index.md#log-config) with multiple EC2 instances quickly. Centralized Logging with OpenSearch uses [Systems Manager Agent(SSM Agent)][ssm-agent]{target="_blank"} to install/configure Fluent Bit agent, and sends log data to [Kinesis Data Streams][kds]{target="_blank"}. ### Prerequisites @@ -40,19 +41,19 @@ Make sure the instances meet the following requirements: ### (Option 2) Select an Auto Scaling group to create an Instance Group When creating an Instance Group with Amazon EC2 Auto Scaling group, the solution will generate a shell script which you -should include in the [EC2 User Data][ec2-user-data]. +should include in the [EC2 User Data][ec2-user-data]. 1. Sign in to the Centralized Logging with OpenSearch Console. 2. In the left sidebar, under **Log Source**, choose **Instance Group**. 3. Click the **Create an instance group** button. 4. In the **Settings** section, specify a group name. 5. In the **Configuration** section, select **Auto Scaling Groups**. -6. In the **Auto Scaling groups** section, select the auto scaling group from which you want to collect logs. +6. In the **Auto Scaling groups** section, select the autoscaling group from which you want to collect logs. 7. (Optional) If you want to ingest logs from another account, select a [linked account](../link-account/index.md) in the **Account Settings** section to create an instance group log source from another account. 8. Choose **Create**. After you created a Log Ingestion using the Instance Group, you can find the generated Shell Script in the details page. 9. Copy the shell script and update the User Data of the Auto Scaling Group's [launch configurations](https://docs.aws.amazon.com/autoscaling/ec2/userguide/launch-configurations.html) or [launch template](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html). -The shell script will automatically install Fluent Bit, SSM agent if needed, and download Fluent Bit configurations. -10. Once you have updated the launch configurations or launch template, you need to start an [instance refresh][instance-refresh] to update the instances within the Auto Scaling group. +The shell script will automatically install Fluent Bit, SSM agent if needed, and download Fluent Bit configurations. +10. Once you have updated the launch configurations or launch template, you need to start an [instance refresh][instance-refresh] to update the instances within the Auto Scaling group. The newly launched instances will ingest logs to the OpenSearch cluster or the [Log Buffer](./index.md#log-buffer) layer. ## Amazon EKS cluster @@ -67,9 +68,9 @@ The [EKS Cluster][eks] in Centralized Logging with OpenSearch refers to the Amaz 1. Sign in to the Centralized Logging with OpenSearch Console. 2. In the left sidebar, under **Log Source**, choose **EKS Cluster**. 3. Click the **Import a Cluster** button. -4. Choose the **EKS Cluster** where Centralized Logging with OpenSearch collects logs from. +4. Choose the **EKS Cluster** where Centralized Logging with OpenSearch collects logs from. (Optional) If you want to ingest logs from another account, select a [linked account](../link-account/index.md) from the **Account** dropdown to import an EKS log source from another account. -5. Select **DaemonSet** or **Sidecar** as log agent's deployment pattern. +5. Select **DaemonSet** or **Sidecar** as log agent's deployment pattern. 6. Choose **Next**. 7. Specify the **Amazon OpenSearch** where Centralized Logging with OpenSearch sends the logs to. 8. Follow the guidance to establish a VPC peering connection between EKS's VPC and OpenSearch's VPC. @@ -80,19 +81,29 @@ The [EKS Cluster][eks] in Centralized Logging with OpenSearch refers to the Amaz 10. Add tags if needed. 11. Choose **Create**. +## Amazon S3 + +The [S3][s3] in Centralized Logging with OpenSearch refers to the Amazon S3 from which you want to collect application logs stored in your bucket. You can choose **On-going** or **One-time** to create your ingestion job. + +!!! important "Important" + + * On-going means that the ingestion job will run when a new file is delivered to the specified S3 location. + * One-time means that the ingestion job will run at creation and only will run once to load all files in the specified location. ## Syslog + !!! important "Important" To ingest logs, make sure your Syslog generator/sender’s subnet is connected to Centralized Logging with OpenSearch’s **two** private subnets. Refer to [VPC Connectivity][vpc-connectivity] for more details about how to connect VPCs. - You can use UDP or TCP custom port number to collect syslog in Centralized Logging with OpenSearch. Syslog refers to logs generated by Linux instance, routers or network equipment. For more information, see [Syslog][syslog] in Wikipedia. + You can use UDP or TCP custom port number to collect syslog in Centralized Logging with OpenSearch. Syslog refers to logs generated by Linux instance, routers or network equipment. For more information, see [Syslog][syslog] in Wikipedia. [kds]: https://aws.amazon.com/kinesis/data-streams/ [ssm-agent]: https://docs.aws.amazon.com/systems-manager/latest/userguide/ssm-agent.html [open-ssl]: https://www.openssl.org/source/ [eks]: https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html +[s3]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html [daemonset]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ [sidecar]: https://kubernetes.io/docs/concepts/workloads/pods/#workload-resources-for-managing-pods [syslog]: https://en.wikipedia.org/wiki/Syslog diff --git a/docs/en/implementation-guide/applications/index.md b/docs/en/implementation-guide/applications/index.md index b05ec99e..53ab5d6d 100644 --- a/docs/en/implementation-guide/applications/index.md +++ b/docs/en/implementation-guide/applications/index.md @@ -24,7 +24,7 @@ Before creating log ingestion, you need to: - [Create a log source](./create-log-source.md) (not applicable for Syslog) - [Create an application log pipeline](./create-applog-pipeline.md) - + ## Concepts The following introduce concepts that help you to understand how the application log ingestion works. @@ -37,7 +37,7 @@ A log ingestion configures the Log Source, Log Type and the Application Log Anal After that, Centralized Logging with OpenSearch will start collecting certain type of logs from the log source and sending them to Amazon OpenSearch. ### Log Agent -A log agent is a program that reads logs from one location and sends them to another location (for example, OpenSearch). +A log agent is a program that reads logs from one location and sends them to another location (for example, OpenSearch). Currently, Centralized Logging with OpenSearch only supports [Fluent Bit 1.9][fluent-bit] log agent which is installed automatically. The Fluent Bit agent has a dependency of [OpenSSL 1.1][open-ssl]. To learn how to install OpenSSL on Linux instances, refer to [OpenSSL installation](../resources/open-ssl.md). To find the supported platforms by Fluent Bit, refer to this [link][supported-platforms]. ### Log Buffer @@ -45,37 +45,39 @@ Log Buffer is a buffer layer between the Log Agent and OpenSearch clusters. The layer before being processed and delivered into the OpenSearch clusters. A buffer layer is a way to protect OpenSearch clusters from overwhelming. This solution provides the following types of buffer layers. -- **Amazon S3**. Use this option if you can bear minutes-level latency for log ingestion. The log agent periodically uploads logs to an Amazon S3 bucket. The frequency of data delivery to -Amazon S3 is determined by *Buffer size* (default value is 50 MiB) and *Buffer interval* (default value is 60 seconds) value -that you configured when creating the application log analytics pipelines. The condition satisfied first triggers data delivery to Amazon S3. +- **Amazon S3**. Use this option if you can bear minutes-level latency for log ingestion. The log agent periodically uploads logs to an Amazon S3 bucket. The frequency of data delivery to +Amazon S3 is determined by *Buffer size* (default value is 50 MiB) and *Buffer interval* (default value is 60 seconds) value +that you configured when creating the application log analytics pipelines. The condition satisfied first triggers data delivery to Amazon S3. -- **Amazon Kinesis Data Streams**. Use this option if you need real-time log ingestion. The log agent uploads logs to Amazon Kinesis Data Stream in seconds. The frequency -of data delivery to Kinesis Data Streams is determined by *Buffer size* (10 MiB) and *Buffer interval* (5 seconds). The -condition satisfied first triggers data delivery to Kinesis Data Streams. +- **Amazon Kinesis Data Streams**. Use this option if you need real-time log ingestion. The log agent uploads logs to Amazon Kinesis Data Stream in seconds. The frequency +of data delivery to Kinesis Data Streams is determined by *Buffer size* (10 MiB) and *Buffer interval* (5 seconds). The +condition satisfied first triggers data delivery to Kinesis Data Streams. -Log Buffer is optional when creating an application log analytics pipeline. For all types of application logs, this +Log Buffer is optional when creating an application log analytics pipeline. For all types of application logs, this solution allows you to ingest logs without any buffer layers. However, we only recommend this option when you have small log volume, and you are confident that the logs will not exceed the thresholds at the OpenSearch side. ### Log Source A Log Source refers to a location where you want Centralized Logging with OpenSearch to collect application logs from. Supported log sources includes: -* [Instance Group](#instances-group) +* [Instance Group](#instance-group) * [EKS Cluster](#eks-cluster) * [Syslog](#syslog) #### Instance Group -An instance group is a collection of EC2 instances from which you want to collect application logs. +An instance group is a collection of EC2 instances from which you want to collect application logs. Centralized Logging with OpenSearch can help you install the log agent in each instance within a group. You can select arbitrary instances through the user interface, or choose an [EC2 Auto Scaling Group][asg]. + #### EKS Cluster -The EKS Cluster in Centralized Logging with OpenSearch refers to the Amazon EKS from which you want to collect pod logs. Centralized Logging with OpenSearch +The EKS Cluster in Centralized Logging with OpenSearch refers to the Amazon EKS from which you want to collect pod logs. Centralized Logging with OpenSearch will guide you to deploy the log agent as a DaemonSet or Sidecar in the EKS Cluster. + #### Syslog Centralized Logging with OpenSearch supports collecting syslog logs through UDP or TCP protocol. ### Log Config -A Log Config is a configuration that is telling Centralized Logging with OpenSearch where the logs had been stored on Log Source, which types of logs you want to collect, what fields a line of log contains, and types of each field. +A Log Config is a configuration that is telling Centralized Logging with OpenSearch where the logs had been stored on Log Source, which types of logs you want to collect, what fields a line of log contains, and types of each field. [fluent-bit]: https://docs.fluentbit.io/manual/ diff --git a/docs/en/implementation-guide/applications/json.md b/docs/en/implementation-guide/applications/json.md index 05d574fb..f4ee7ed8 100644 --- a/docs/en/implementation-guide/applications/json.md +++ b/docs/en/implementation-guide/applications/json.md @@ -19,9 +19,9 @@ include-markdown "include-prerequisites.md" For example: ```json - {"host":"81.95.250.9", "user-identifier":"-", "time":"08/Mar/2022:06:28:03 +0000", "method": "PATCH", "request": "/clicks-and-mortar/24%2f7", "protocol":"HTTP/2.0", "status":502, "bytes":24337, "referer": "http://www.investorturn-key.net/functionalities/innovative/integrated"} + {"host":"81.95.250.9", "user-identifier":"-", "time":"08/Mar/2022:06:28:03 +0000", "method": "PATCH", "request": "/clicks-and-mortar/24%2f7", "protocol":"HTTP/2.0", "status":502, "bytes":24337, "referer": "https://www.investorturn-key.net/functionalities/innovative/integrated"} ``` - + 8. Check if each fields type mapping is correct. You can change the type by selecting the dropdown menu in the second column. For all supported types, see [Data Types](https://opensearch.org/docs/latest/search-plugins/sql/datatypes/). !!! Note "Note" @@ -36,7 +36,7 @@ include-markdown "include-prerequisites.md" ## Step 2: Create an application log ingestion -The steps are similar to creating an application log ingestion for single-line text. Refer to [Single-line Text](./single-line-text.md) for details. +The steps are similar to creating an application log ingestion for single-line text. Refer to [Single-line Text](./single-line-text.md#step-2-create-an-application-log-ingestion) for details. ## Step 3: View your logs in OpenSearch diff --git a/docs/en/implementation-guide/applications/multi-line-text.md b/docs/en/implementation-guide/applications/multi-line-text.md index 1170a588..32ee5e2d 100644 --- a/docs/en/implementation-guide/applications/multi-line-text.md +++ b/docs/en/implementation-guide/applications/multi-line-text.md @@ -1,6 +1,6 @@ -# Multi-line Text +# Multi-line text -You can configure Centralized Logging with OpenSearch to ingest multi-line text logs. Currently, Centralized Logging with OpenSearch supports [Spring Boot](https://spring.io/projects/spring-boot) +You can configure Centralized Logging with OpenSearch to ingest multi-line text logs. Currently, Centralized Logging with OpenSearch supports [Spring Boot](https://spring.io/projects/spring-boot) style logs or customize the log format using [Regular Expression](https://rubular.com/). ## Prerequisites @@ -53,17 +53,27 @@ include-markdown "include-prerequisites.md" 1. For other kinds of logs, you could specify the first line regex pattern. For example: ``` - (?