mimecast: try fixing logon-authentication-failed again

This only fails on stack version v8.17, so fixing it required running on that stack.
elastic · Nov 20, 2024 · 791aa85 · 791aa85
1 parent a40ebee
commit 791aa85
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 28 deletions.
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.27.4"
+  changes:
+    - description: Further fix parsing of "logon-authentication-failed" events.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/
 - version: "1.27.3"
   changes:
     - description: Fix parsing of "logon-authentication-failed" events.

diff --git a/.../mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json b/.../mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json
@@ -128,7 +128,7 @@
             },
             "event": {
                 "action": "user-logged-on",
-                "created": "2021-10-11T12:17:30.000Z",
+                "created": "2021-10-11T18:17:30.000Z",
                 "id": "eNqrVipOTS4tSs1MUbJSivD0cisuyAirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A",
                 "original": "{\"id\": \"eNqrVipOTS4tSs1MUbJSivD0cisuyAirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A\",\"auditType\": \"User Logged On\",\"user\": \"[email protected]\",\"eventTime\": \"2021-10-11T17:17:30+0000\",\"eventInfo\": \"Successful authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 18:17:30 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Two Step Auth, 2FA: TOTP\",\"category\": \"authentication_logs\"}"
             },
@@ -137,7 +137,8 @@
                 "application": "Administration Console",
                 "category": "authentication_logs",
                 "eventInfo": "Successful authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 18:17:30 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Two Step Auth, 2FA: TOTP",
-                "method": "Two Step Auth"
+                "method": "Two Step Auth",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -179,7 +180,7 @@
             },
             "event": {
                 "action": "logon-requires-challenge",
-                "created": "2021-10-11T12:17:26.000Z",
+                "created": "2021-10-11T18:17:26.000Z",
                 "id": "eNqrVipOTS4tSs1MUbJSSsos9DMJTPLyMA6NcCt2TA1OCwjLcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhsqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAC8tK60",
                 "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSSsos9DMJTPLyMA6NcCt2TA1OCwjLcwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhsqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAC8tK60\",\"auditType\":\"Logon Requires Challenge\",\"user\":\"[email protected]\",\"eventTime\":\"2021-10-11T17:17:26+0000\",\"eventInfo\":\"Intermediate authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 18:17:26 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Office 365, 2FA: TOTP\",\"category\":\"authentication_logs\"}"
             },
@@ -188,7 +189,8 @@
                 "application": "Administration Console",
                 "category": "authentication_logs",
                 "eventInfo": "Intermediate authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 18:17:26 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Office 365, 2FA: TOTP",
-                "method": "Office 365"
+                "method": "Office 365",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -230,15 +232,16 @@
             },
             "event": {
                 "action": "user-logged-on",
-                "created": "2021-10-11T11:03:38.000Z",
+                "created": "2021-10-11T17:03:38.000Z",
                 "id": "eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI",
                 "original": "{ \"id\": \"eNqrVipOTS4tSs1MUbJS8o0ILw8pL_cyqQosLi-MzKjKcvMzCwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkYmZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAIqvLHI\", \"auditType\": \"User Logged On\", \"user\": \"[email protected]\", \"eventTime\": \"2021-10-11T16:03:38+0000\", \"eventInfo\": \"Successful authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 17:03:38 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Cloud\", \"category\": \"authentication_logs\"}"
             },
             "mimecast": {
                 "application": "Administration Console",
                 "category": "authentication_logs",
                 "eventInfo": "Successful authentication for [email protected] <John Doe>, Date: 2021-10-11, Time: 17:03:38 BTT, IP: 67.43.156.15, Application: Administration Console, Method: Cloud",
-                "method": "Cloud"
+                "method": "Cloud",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -287,7 +290,8 @@
             "mimecast": {
                 "application": "Administration Console",
                 "category": "mimecast_access_logs",
-                "eventInfo": "Action Performed - [email protected] logged into this account. by [email protected]<[email protected]> Date: 2021-10-11 Time: 16:39:17 +0100 IP: 67.43.156.15 Application: Administration Console"
+                "eventInfo": "Action Performed - [email protected] logged into this account. by [email protected]<[email protected]> Date: 2021-10-11 Time: 16:39:17 +0100 IP: 67.43.156.15 Application: Administration Console",
+                "timezone": "+0100"
             },
             "related": {
                 "ip": [
@@ -336,7 +340,8 @@
             "mimecast": {
                 "application": "Administration Console",
                 "category": "mimecast_access_logs",
-                "eventInfo": "Action Performed - [email protected] logged into this account. by [email protected]<[email protected]> Date: 2021-10-19 Time: 12:46:40 +0100 IP: 67.43.156.15 Application: Administration Console"
+                "eventInfo": "Action Performed - [email protected] logged into this account. by [email protected]<[email protected]> Date: 2021-10-19 Time: 12:46:40 +0100 IP: 67.43.156.15 Application: Administration Console",
+                "timezone": "+0100"
             },
             "related": {
                 "ip": [
@@ -612,7 +617,8 @@
                 "application": "mimecast-moa",
                 "category": "authentication_logs",
                 "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2021-10-12, Time: 09:47:55 BTT, IP: 67.43.156.15, Application: mimecast-moa, Method: Office 365, Reason: Wrong password",
-                "method": "Office 365"
+                "method": "Office 365",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -764,7 +770,8 @@
             "mimecast": {
                 "application": "mimecast-matfe",
                 "category": "account_logs",
-                "eventInfo": "[Export type : Download,Name :watchlist_view,Requested By :[email protected],Export time :Tue Oct 12 03:27:18 BTT 2021,IP Address :67.43.156.15,Columns exported :Name|Email|Department|Number of Videos|,File name : export_at_watchlist_view_1634005638160.xlsx,File Size: 6864,File type : .xlsx], Date: 2021-10-12, Time: 02:27:18+0000, IP: 67.43.156.15, Application: mimecast-matfe"
+                "eventInfo": "[Export type : Download,Name :watchlist_view,Requested By :[email protected],Export time :Tue Oct 12 03:27:18 BTT 2021,IP Address :67.43.156.15,Columns exported :Name|Email|Department|Number of Videos|,File name : export_at_watchlist_view_1634005638160.xlsx,File Size: 6864,File type : .xlsx], Date: 2021-10-12, Time: 02:27:18+0000, IP: 67.43.156.15, Application: mimecast-matfe",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -813,7 +820,8 @@
             "mimecast": {
                 "application": "Administration Console",
                 "category": "reporting_logs",
-                "eventInfo": "Action Performed - Custom Report Definition Created with name \"Terri test\" and description \"all user - per email report\" by [email protected]<[email protected]> Date: 2021-10-11 Time: 20:53:41 +0100 IP: 67.43.156.15 Application: Administration Console"
+                "eventInfo": "Action Performed - Custom Report Definition Created with name \"Terri test\" and description \"all user - per email report\" by [email protected]<[email protected]> Date: 2021-10-11 Time: 20:53:41 +0100 IP: 67.43.156.15 Application: Administration Console",
+                "timezone": "+0100"
             },
             "related": {
                 "ip": [
@@ -862,7 +870,8 @@
             "mimecast": {
                 "application": "Administration Console",
                 "category": "profile_group_logs",
-                "eventInfo": "Action Performed - Deleted New Folder by [email protected]<John Doe> Date: 2021-10-11 Time: 19:23:10 +0100 IP: 67.43.156.15 Application: Administration Console"
+                "eventInfo": "Action Performed - Deleted New Folder by [email protected]<John Doe> Date: 2021-10-11 Time: 19:23:10 +0100 IP: 67.43.156.15 Application: Administration Console",
+                "timezone": "+0100"
             },
             "related": {
                 "ip": [
@@ -1232,7 +1241,8 @@
             "mimecast": {
                 "application": "POP-POP2",
                 "category": "authentication_logs",
-                "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked"
+                "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked",
+                "timezone": "GMT"
             },
             "related": {
                 "ip": [
@@ -1283,7 +1293,8 @@
                 "application": "POP-POP2",
                 "category": "authentication_logs",
                 "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password",
-                "method": "Cloud"
+                "method": "Cloud",
+                "timezone": "GMT"
             },
             "related": {
                 "ip": [
@@ -1334,7 +1345,8 @@
                 "application": "POP-POP2",
                 "category": "authentication_logs",
                 "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password",
-                "method": "Cloud"
+                "method": "Cloud",
+                "timezone": "GMT"
             },
             "related": {
                 "ip": [
@@ -1433,7 +1445,8 @@
                 "application": "API",
                 "category": "authentication_logs",
                 "eventInfo": "User [email protected] attempted to access the mimecast-matfe but does not have the required permissions to do so, Date: 2022-03-29, Time: 13:31:03+0000, IP: 67.43.156.15, Application: API, Remote IP is 67.43.156.15",
-                "remote_ip": "67.43.156.15"
+                "remote_ip": "67.43.156.15",
+                "timezone": "+0000"
             },
             "related": {
                 "ip": [
@@ -1483,7 +1496,8 @@
             "mimecast": {
                 "application": "SMTP-MTA2",
                 "category": "authentication_logs",
-                "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-03-29, Time: 19:33:05 BTT, IP: 67.43.156.15, Application: SMTP-MTA2, Reason: Account locked"
+                "eventInfo": "Failed authentication for [email protected] <John Doe>, Date: 2022-03-29, Time: 19:33:05 BTT, IP: 67.43.156.15, Application: SMTP-MTA2, Reason: Account locked",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [
@@ -1534,7 +1548,8 @@
                 "application": "MfO",
                 "category": "authentication_logs",
                 "eventInfo": "Failed authentication for [email protected] <Doe, John>, Date: 2023-05-01, Time: 13:50:07 GMT-04:00, IP: 67.43.156.3, Application: MfO, Method: SP-initiated SAML, Reason: Account disabled",
-                "method": "SP-initiated SAML"
+                "method": "SP-initiated SAML",
+                "timezone": "GMT-04:00"
             },
             "related": {
                 "ip": [
@@ -1576,15 +1591,16 @@
             },
             "event": {
                 "action": "user-logged-on",
-                "created": "2024-07-01T07:56:25.000Z",
+                "created": "2024-07-01T13:56:25.000Z",
                 "id": "eNpVj21LhEAUhf_LfN2VnRl1RpclCNirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A",
                 "original": "{\"auditType\":\"User Logged On\",\"category\":\"authentication_logs\",\"eventInfo\":\"Successful authentication for [email protected] \\u003cSmith, John\\u003e, Date: 2024-07-01, Time: 13:56:25 BTT, IP: 81.2.69.144, Application: MPP, Method: SP-initiated SAML\",\"eventTime\":\"2024-07-01T12:56:25+0000\",\"id\":\"eNpVj21LhEAUhf_LfN2VnRl1RpclCNirMgpxDy12dPNMMcn1zQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxkamhiqKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWADo9K8A\",\"user\":\"[email protected]\"}"
             },
             "mimecast": {
                 "application": "MPP",
                 "category": "authentication_logs",
                 "eventInfo": "Successful authentication for [email protected] <Smith, John>, Date: 2024-07-01, Time: 13:56:25 BTT, IP: 81.2.69.144, Application: MPP, Method: SP-initiated SAML",
-                "method": "SP-initiated SAML"
+                "method": "SP-initiated SAML",
+                "timezone": "BTT"
             },
             "related": {
                 "ip": [

diff --git a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml
@@ -97,11 +97,11 @@ processors:
       ignore_missing: true
       ignore_failure: true
   - dissect:
-       field: mimecast.eventInfo
-       pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
-       if: 'ctx.event?.action=="folder-log-entry" || ctx.event?.action=="custom-report-definition-created" || ctx.event?.action=="mimecast-support-login"'
-       ignore_missing: true
-       ignore_failure: true
+      field: mimecast.eventInfo
+      pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
+      if: 'ctx.event?.action=="folder-log-entry" || ctx.event?.action=="custom-report-definition-created" || ctx.event?.action=="mimecast-support-login"'
+      ignore_missing: true
+      ignore_failure: true
   - kv:
       field: mimecast.rest_of_event_info
       field_split: ", "
@@ -123,6 +123,11 @@ processors:
       field: mimecast.event_info_parts.Date
       target_field: mimecast.date
       ignore_missing: true
+  - dissect:
+      field: mimecast.event_info_parts.Time
+      pattern: '%{mimecast.event_info_parts.Time} %{mimecast.timezone}'
+      ignore_missing: true
+      ignore_failure: true
   - rename:
       field: mimecast.event_info_parts.Time
       target_field: mimecast.time
@@ -241,6 +246,7 @@ processors:
       value: "{{{mimecast.date}}} {{{mimecast.time}}}"
       if: 'ctx?.mimecast?.date != null && ctx?.mimecast?.time != null'
   - date:
+      tag: date_event_created
       field: event.created
       target_field: event.created
       timezone: UTC
@@ -259,7 +265,7 @@ processors:
         - yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSz
         - yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ
         - yyyy-MM-dd'T'HH:mm:ss z
-      if: 'ctx?.event?.created != null'
+      if: ctx.event?.created != null
   - geoip:
       field: client.ip
       target_field: client.geo
@@ -321,7 +327,6 @@ processors:
         - mimecast.filename
         - mimecast.criteria
         - mimecast.viewed
-        - mimecast.timezone
         - mimecast.byuser
         - mimecast.export_type
         - mimecast.export_name

diff --git a/packages/mimecast/data_stream/audit_events/fields/field.yml b/packages/mimecast/data_stream/audit_events/fields/field.yml
@@ -28,3 +28,6 @@
     - name: remote_ip
       type: ip
       description: Remote IP.
+    - name: timezone
+      type: keyword
+      description: Timezone reported in the event message.
diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md
@@ -205,6 +205,7 @@ An example event for `audit_events` looks as following:
 | mimecast.method | Method which triggers audit events. | keyword |
 | mimecast.remote | Info about remote IP trying to access the API. | keyword |
 | mimecast.remote_ip | Remote IP. | ip |
+| mimecast.timezone | Timezone reported in the event message. | keyword |
 
 
 ### DLP Logs

diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: mimecast
 title: "Mimecast"
-version: "1.27.3"
+version: "1.27.4"
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration
 categories: ["security", "email_security"]