From 316c0b7f49d5de4c21e3566f07f5752c00dd93d4 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 9 Jun 2021 11:39:30 +0200 Subject: [PATCH 1/2] update azure ECS version and adding event.original --- .../test-activitylogs-raw.log-config.yml | 3 + .../test-activitylogs-raw.log-expected.json | 11 +- .../agent/stream/azure-eventhub.yml.hbs | 12 +- .../activitylogs/agent/stream/log.yml.hbs | 12 +- .../ingest_pipeline/azure-shared-pipeline.yml | 3 - .../elasticsearch/ingest_pipeline/default.yml | 17 +- .../data_stream/activitylogs/fields/ecs.yml | 5 + .../data_stream/activitylogs/manifest.yml | 25 + .../test-auditlogs-raw.log-config.yml | 3 + .../test-auditlogs-raw.log-expected.json | 7 +- .../agent/stream/azure-eventhub.yml.hbs | 13 +- .../auditlogs/agent/stream/log.yml.hbs | 12 +- .../ingest_pipeline/azure-shared-pipeline.yml | 3 - .../elasticsearch/ingest_pipeline/default.yml | 17 +- .../data_stream/auditlogs/fields/ecs.yml | 5 + .../azure/data_stream/auditlogs/manifest.yml | 25 + .../_dev/test/pipeline/test-common-config.yml | 5 + ...st-platformlogs-invalid-raw.log-config.yml | 2 - ...platformlogs-invalid-raw.log-expected.json | 7 +- .../test-platformlogs-raw.log-config.yml | 2 - .../test-platformlogs-raw.log-expected.json | 7 +- ...est-platformlogs-remote-raw.log-config.yml | 2 - ...-platformlogs-remote-raw.log-expected.json | 7 +- .../agent/stream/azure-eventhub.yml.hbs | 12 +- .../platformlogs/agent/stream/log.yml.hbs | 12 +- .../ingest_pipeline/azure-shared-pipeline.yml | 3 - .../elasticsearch/ingest_pipeline/default.yml | 17 +- .../data_stream/platformlogs/fields/ecs.yml | 5 + .../data_stream/platformlogs/manifest.yml | 25 + .../test-signinlogs-raw.log-config.yml | 3 + .../test-signinlogs-raw.log-expected.json | 22 +- .../agent/stream/azure-eventhub.yml.hbs | 12 +- .../signinlogs/agent/stream/log.yml.hbs | 12 +- .../ingest_pipeline/azure-shared-pipeline.yml | 3 - .../elasticsearch/ingest_pipeline/default.yml | 632 +++++++++--------- .../data_stream/signinlogs/fields/ecs.yml | 5 + .../azure/data_stream/signinlogs/manifest.yml | 25 + packages/azure/docs/README.md | 4 + packages/azure/manifest.yml | 8 - 39 files changed, 595 insertions(+), 410 deletions(-) create mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml delete mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml delete mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml delete mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-config.yml diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml index c39dc386179..5622947e4b8 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml @@ -1,2 +1,5 @@ dynamic_fields: event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index 4c771194ce1..cd56ab86b9f 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -16,6 +16,9 @@ }, "ip": "51.251.141.41" }, + "tags": [ + "preserve_original_event" + ], "geo": { "continent_name": "Europe", "country_name": "United Kingdom", @@ -29,21 +32,21 @@ "provider": "azure" }, "@timestamp": "2019-10-24T00:13:46.355Z", + "ecs": { + "version": "1.10.0" + }, "related": { "ip": [ "51.251.141.41" ] }, - "ecs": { - "version": "1.9.0" - }, "client": { "ip": "51.251.141.41" }, "event": { "duration": 0, "action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "ingested": "2021-04-23T12:52:57.328609264Z", + "ingested": "2021-06-09T09:37:58.777185600Z", "original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}", "type": [ "change" diff --git a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs index a1715404263..eb16b72f707 100644 --- a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs @@ -18,14 +18,16 @@ storage_account_key: {{storage_account_key}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs index fd3b5984962..234dffcea5d 100644 --- a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs @@ -4,14 +4,16 @@ paths: {{/each}} exclude_files: [".gz$"] tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index d861c267fae..a1d0663b663 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -61,9 +61,6 @@ processors: - lowercase: field: event.outcome ignore_missing: true -- set: - field: ecs.version - value: 1.9.0 on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index 55778f210e0..fbf45aac0c6 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -4,6 +4,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: "1.10.0" - rename: field: azure target_field: azure-eventhub @@ -13,8 +16,12 @@ processors: params: empty_field_name: '"":"",' ignore_failure: true - - json: + - rename: field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original target_field: azure.activitylogs - date: field: azure.activitylogs.time @@ -22,9 +29,6 @@ processors: ignore_failure: true formats: - ISO8601 - - rename: - field: message - target_field: event.original - remove: field: azure.activitylogs.time ignore_missing: true @@ -264,6 +268,11 @@ processors: value: event - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml index 11efd619e1e..240308d6c68 100644 --- a/packages/azure/data_stream/activitylogs/fields/ecs.yml +++ b/packages/azure/data_stream/activitylogs/fields/ecs.yml @@ -242,3 +242,8 @@ type: text name: user.name type: keyword +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword \ No newline at end of file diff --git a/packages/azure/data_stream/activitylogs/manifest.yml b/packages/azure/data_stream/activitylogs/manifest.yml index 413ef80e912..fe590f45bd6 100644 --- a/packages/azure/data_stream/activitylogs/manifest.yml +++ b/packages/azure/data_stream/activitylogs/manifest.yml @@ -6,3 +6,28 @@ streams: template_path: "azure-eventhub.yml.hbs" title: "Azure activity logs" description: "Collect Azure activity logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-activitylogs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. \ No newline at end of file diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml index c39dc386179..5622947e4b8 100644 --- a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml @@ -1,2 +1,5 @@ dynamic_fields: event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json index 4b78fb9da7f..bd14334745b 100644 --- a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json @@ -6,7 +6,7 @@ }, "@timestamp": "2019-10-18T15:30:51.027Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "log": { "level": "Informational" @@ -14,11 +14,14 @@ "event": { "duration": 0, "action": "Update device", - "ingested": "2021-04-23T12:52:57.546715602Z", + "ingested": "2021-06-09T09:37:59.162398200Z", "original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}", "kind": "event", "outcome": "success" }, + "tags": [ + "preserve_original_event" + ], "azure": { "tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", diff --git a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs index e357b7debfa..87b3de53cdf 100644 --- a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs @@ -18,14 +18,17 @@ storage_account_key: {{storage_account_key}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} + diff --git a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs index fd3b5984962..234dffcea5d 100644 --- a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs @@ -4,14 +4,16 @@ paths: {{/each}} exclude_files: [".gz$"] tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index ce8dae3e1e3..09c34a8e76f 100644 --- a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -39,9 +39,6 @@ processors: - lowercase: field: event.outcome ignore_missing: true -- set: - field: ecs.version - value: 1.9.0 on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml index 68873c59fc1..a293eed1b32 100644 --- a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml @@ -4,12 +4,19 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: "1.10.0" - rename: field: azure target_field: azure-eventhub ignore_missing: true - - json: + - rename: field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original target_field: azure.auditlogs - drop: if: ctx.azure.auditlogs.category != 'AuditLogs' @@ -40,9 +47,6 @@ processors: field: azure.auditlogs.level target_field: log.level ignore_missing: true - - rename: - field: message - target_field: event.original - remove: field: azure.auditlogs.time ignore_missing: true @@ -140,6 +144,11 @@ processors: value: event - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/auditlogs/fields/ecs.yml b/packages/azure/data_stream/auditlogs/fields/ecs.yml index 6ed7adae3a8..030c503054e 100644 --- a/packages/azure/data_stream/auditlogs/fields/ecs.yml +++ b/packages/azure/data_stream/auditlogs/fields/ecs.yml @@ -239,3 +239,8 @@ type: text name: user.name type: keyword +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword \ No newline at end of file diff --git a/packages/azure/data_stream/auditlogs/manifest.yml b/packages/azure/data_stream/auditlogs/manifest.yml index 4844149fad2..309fb4c475f 100644 --- a/packages/azure/data_stream/auditlogs/manifest.yml +++ b/packages/azure/data_stream/auditlogs/manifest.yml @@ -6,3 +6,28 @@ streams: template_path: "azure-eventhub.yml.hbs" title: "Azure audit logs" description: "Collect Azure audit logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-auditlogs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..e74affa452f --- /dev/null +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml deleted file mode 100644 index c39dc386179..00000000000 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json index a78a36e8e66..69a68543881 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json @@ -6,14 +6,17 @@ }, "@timestamp": "2020-10-11T20:30:59.000Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "event": { "action": "ApplicationGatewayAccess", - "ingested": "2021-04-23T12:52:57.717406206Z", + "ingested": "2021-06-09T09:37:59.356079400Z", "original": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}", "kind": "event" }, + "tags": [ + "preserve_original_event" + ], "azure": { "resource": { "id": "/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY" diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml deleted file mode 100644 index c39dc386179..00000000000 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json index b14965b0cc1..e4d7f5ddfd3 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json @@ -7,15 +7,18 @@ }, "@timestamp": "2020-11-03T09:06:42.000Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "event": { "action": "Retreive ConsumerGroup", - "ingested": "2021-04-23T12:52:57.746877770Z", + "ingested": "2021-06-09T09:37:59.380613800Z", "original": "{\"ActivityId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8\",\"Caller\":\"Portal\",\"Environment\":\"PROD\",\"EventName\":\"Retreive ConsumerGroup\",\"EventProperties\":\"{\\\"SubscriptionId\\\":\\\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\\\",\\\"Namespace\\\":\\\"obstesteventhubs\\\",\\\"Via\\\":\\\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\\u0026$skip=0\\u0026$top=100\\\",\\\"TrackingId\\\":\\\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\\\"}\",\"EventTimeString\":\"11/3/2020 9:06:42 AM +00:00\",\"Region\":\"West Europe\",\"ScaleUnit\":\"PROD-AM3-AZ501\",\"Status\":\"Succeeded\",\"category\":\"OperationalLogs\",\"resourceId\":\"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS\"}", "kind": "event", "outcome": "succeeded" }, + "tags": [ + "preserve_original_event" + ], "azure": { "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", "resource": { diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-config.yml deleted file mode 100644 index c39dc386179..00000000000 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json index b4a2d8c5a95..3d54decb8e5 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json @@ -6,15 +6,18 @@ }, "@timestamp": "2020-11-09T10:57:31.000Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "event": { "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested": "2021-04-23T12:52:57.773132703Z", + "ingested": "2021-06-09T09:37:59.405299500Z", "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", "kind": "event" }, "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", + "tags": [ + "preserve_original_event" + ], "azure": { "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", "resource": { diff --git a/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs index 5394af3857f..8d17d8c3cd4 100644 --- a/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs @@ -18,14 +18,16 @@ storage_account_key: {{storage_account_key}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs index fd3b5984962..234dffcea5d 100644 --- a/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs @@ -4,14 +4,16 @@ paths: {{/each}} exclude_files: [".gz$"] tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index 5b512b8499c..d2b7e1e68ce 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -39,9 +39,6 @@ processors: - lowercase: field: event.outcome ignore_missing: true -- set: - field: ecs.version - value: 1.9.0 on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 2cb2c9aba33..c02ebe5e405 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -4,6 +4,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: "1.10.0" - rename: field: azure target_field: azure-eventhub @@ -13,8 +16,12 @@ processors: params: empty_field_name: '"":"",' ignore_failure: true - - json: + - rename: field: message + target_field: event.original + ignore_missing: true + - json: + field: event.original target_field: azure.platformlogs - date: field: azure.platformlogs.time @@ -29,9 +36,6 @@ processors: formats: - ISO8601 - "M/d/yyyy h:mm:ss a XXX" - - rename: - field: message - target_field: event.original - remove: field: azure.platformlogs.time ignore_missing: true @@ -200,6 +204,11 @@ processors: value: event - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/platformlogs/fields/ecs.yml b/packages/azure/data_stream/platformlogs/fields/ecs.yml index 6ed7adae3a8..030c503054e 100644 --- a/packages/azure/data_stream/platformlogs/fields/ecs.yml +++ b/packages/azure/data_stream/platformlogs/fields/ecs.yml @@ -239,3 +239,8 @@ type: text name: user.name type: keyword +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/manifest.yml b/packages/azure/data_stream/platformlogs/manifest.yml index b70a6dfbf91..cbe92715ec0 100644 --- a/packages/azure/data_stream/platformlogs/manifest.yml +++ b/packages/azure/data_stream/platformlogs/manifest.yml @@ -6,3 +6,28 @@ streams: template_path: "azure-eventhub.yml.hbs" title: "Azure platform logs" description: "Collect Azure platform logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-platformlogs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-config.yml b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-config.yml index c39dc386179..5622947e4b8 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-config.yml +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-config.yml @@ -1,2 +1,5 @@ dynamic_fields: event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json index fb61fbcec00..31ab0c15f20 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json @@ -26,6 +26,9 @@ "ip": "81.171.241.231" }, "message": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", + "tags": [ + "preserve_original_event" + ], "geo": { "country_name": "Seine-Et-Marne", "city_name": "Champs-Sur-Marne", @@ -39,20 +42,20 @@ "provider": "azure" }, "@timestamp": "2019-10-18T09:45:48.072Z", + "ecs": { + "version": "1.10.0" + }, "related": { "ip": [ "81.171.241.231" ] }, - "ecs": { - "version": "1.9.0" - }, "client": { "ip": "81.171.241.231" }, "event": { "duration": 0, - "ingested": "2021-04-23T12:52:57.895009113Z", + "ingested": "2021-06-09T09:37:59.560632100Z", "original": "{\"Level\":\"4\",\"callerIpAddress\":\"81.171.241.231\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", @@ -142,6 +145,9 @@ "ip": "8.8.8.8" }, "message": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", + "tags": [ + "preserve_original_event" + ], "geo": { "country_name": "Seine-Et-Marne", "city_name": "Champs-Sur-Marne", @@ -155,20 +161,20 @@ "provider": "azure" }, "@timestamp": "2019-10-18T09:45:48.072Z", + "ecs": { + "version": "1.10.0" + }, "related": { "ip": [ "8.8.8.8" ] }, - "ecs": { - "version": "1.9.0" - }, "client": { "ip": "8.8.8.8" }, "event": { "duration": 0, - "ingested": "2021-04-23T12:52:57.895015359Z", + "ingested": "2021-06-09T09:37:59.560652700Z", "original": "{\"Level\":\"4\",\"callerIpAddress\":\"8.8.8.8\",\"category\":\"SignInLogs\",\"correlationId\":\"a8d4eb85-90c5-740d-9af6-7a15036cd135\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"c3813493-bf92-5123-2717-8a8b2979c38b\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", diff --git a/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs index ff2589d01d7..57528f5aae4 100644 --- a/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs @@ -18,14 +18,16 @@ storage_account_key: {{storage_account_key}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs index fd3b5984962..234dffcea5d 100644 --- a/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs @@ -4,14 +4,16 @@ paths: {{/each}} exclude_files: [".gz$"] tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index ce8dae3e1e3..09c34a8e76f 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -39,9 +39,6 @@ processors: - lowercase: field: event.outcome ignore_missing: true -- set: - field: ecs.version - value: 1.9.0 on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index 2a2ca3942c2..0fc065fce39 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -1,316 +1,324 @@ --- description: Pipeline for parsing azure signin logs. processors: - - set: - field: event.ingested - value: '{{_ingest.timestamp}}' - - rename: - field: azure - target_field: azure-eventhub - ignore_missing: true - - json: - field: message - target_field: azure.signinlogs - - drop: - if: ctx.azure.signinlogs.category != 'SignInLogs' - - date: - field: azure.signinlogs.time - target_field: '@timestamp' - ignore_failure: false - formats: - - ISO8601 - - rename: - field: message - target_field: event.original - - remove: - field: azure.signinlogs.time - ignore_missing: true - - rename: - field: azure.signinlogs.resourceId - target_field: azure.resource_id - ignore_missing: true - - rename: - field: azure.signinlogs.callerIpAddress - target_field: source.ip - ignore_missing: true - - set: - field: client.ip - value: '{{source.ip}}' - ignore_empty_value: true - - append: - field: related.ip - value: '{{source.ip}}' - allow_duplicates: false - if: 'ctx.source?.ip != null' - - rename: - field: azure.signinlogs.Level - target_field: log.level - ignore_missing: true - - rename: - field: azure.signinlogs.durationMs - target_field: event.duration - ignore_missing: true - - script: - lang: painless - source: ctx.event.duration = ctx.event.duration * params.param_nano - params: - param_nano: 1000000 - - rename: - field: azure.signinlogs.location - target_field: geo.country_iso_code - ignore_missing: true - - rename: - field: azure.signinlogs.resultType - target_field: azure.signinlogs.result_type - ignore_missing: true - - rename: - field: azure.signinlogs.operationName - target_field: azure.signinlogs.operation_name - ignore_missing: true - - convert: - field: azure.signinlogs.operation_name - target_field: event.action - type: string - ignore_missing: true - - rename: - field: azure.signinlogs.resultSignature - target_field: azure.signinlogs.result_signature - ignore_missing: true - - rename: - field: azure.signinlogs.resultDescription - target_field: azure.signinlogs.result_description - ignore_missing: true - - rename: - field: azure.signinlogs.operationVersion - target_field: azure.signinlogs.operation_version - ignore_missing: true - - rename: - field: azure.signinlogs.tenantId - target_field: azure.tenant_id - ignore_missing: true - - rename: - field: azure.signinlogs.correlationId - target_field: azure.correlation_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.networkLocationDetails - target_field: azure.signinlogs.properties.network_location_details - ignore_missing: true - - rename: - field: azure.signinlogs.properties.resourceId - target_field: azure.signinlogs.properties.resource_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.appliedConditionalAccessPolicies - target_field: azure.signinlogs.properties.applied_conditional_access_policies - ignore_missing: true - - rename: - field: azure.signinlogs.properties.authenticationDetails - target_field: azure.signinlogs.properties.authentication_details - ignore_missing: true - - rename: - field: azure.signinlogs.properties.authenticationRequirementPolicies - target_field: azure.signinlogs.properties.authentication_requirement_policies - ignore_missing: true - - rename: - field: azure.signinlogs.properties.authenticationProcessingDetails - target_field: azure.signinlogs.properties.authentication_processing_details - ignore_missing: true - - rename: - field: azure.signinlogs.properties.deviceDetail - target_field: azure.signinlogs.properties.device_detail - ignore_missing: true - - rename: - field: azure.signinlogs.properties.device_detail.deviceId - target_field: azure.signinlogs.properties.device_detail.device_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.device_detail.operatingSystem - target_field: azure.signinlogs.properties.device_detail.operating_system - ignore_missing: true - - rename: - field: azure.signinlogs.properties.device_detail.displayName - target_field: azure.signinlogs.properties.device_detail.display_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.device_detail.trustType - target_field: azure.signinlogs.properties.device_detail.trust_type - ignore_missing: true - - rename: - field: azure.signinlogs.properties.createdDateTime - target_field: azure.signinlogs.properties.created_at - ignore_missing: true - - rename: - field: azure.signinlogs.properties.userDisplayName - target_field: azure.signinlogs.properties.user_display_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.correlationId - target_field: azure.signinlogs.properties.correlation_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.userPrincipalName - target_field: azure.signinlogs.properties.user_principal_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.userId - target_field: azure.signinlogs.properties.user_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.appId - target_field: azure.signinlogs.properties.app_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.appDisplayName - target_field: azure.signinlogs.properties.app_display_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.ipAddress - target_field: azure.signinlogs.properties.ip_address - ignore_missing: true - - rename: - field: azure.signinlogs.properties.clientAppUsed - target_field: azure.signinlogs.properties.client_app_used - ignore_missing: true - - rename: - field: azure.signinlogs.properties.conditionalAccessStatus - target_field: azure.signinlogs.properties.conditional_access_status - ignore_missing: true - - rename: - field: azure.signinlogs.properties.originalRequestId - target_field: azure.signinlogs.properties.original_request_id - ignore_missing: true - - rename: - field: azure.signinlogs.properties.isInteractive - target_field: azure.signinlogs.properties.is_interactive - ignore_missing: true - - rename: - field: azure.signinlogs.properties.tokenIssuerName - target_field: azure.signinlogs.properties.token_issuer_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.tokenIssuerType - target_field: azure.signinlogs.properties.token_issuer_type - ignore_missing: true - - rename: - field: azure.signinlogs.properties.processingTimeInMilliseconds - target_field: azure.signinlogs.properties.processing_time_ms - ignore_missing: true - - rename: - field: azure.signinlogs.properties.riskDetail - target_field: azure.signinlogs.properties.risk_detail - ignore_missing: true - - rename: - field: azure.signinlogs.properties.riskLevelAggregated - target_field: azure.signinlogs.properties.risk_level_aggregated - ignore_missing: true - - rename: - field: azure.signinlogs.properties.riskLevelDuringSignIn - target_field: azure.signinlogs.properties.risk_level_during_signin - ignore_missing: true - - rename: - field: azure.signinlogs.properties.riskState - target_field: azure.signinlogs.properties.risk_state - ignore_missing: true - - rename: - field: azure.signinlogs.properties.resourceDisplayName - target_field: azure.signinlogs.properties.resource_display_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.status.errorCode - target_field: azure.signinlogs.properties.status.error_code - ignore_missing: true - - rename: - field: azure.signinlogs.properties.status.failureReason - target_field: message - ignore_missing: true - - rename: - field: azure.signinlogs.properties.status.additionalDetails - target_field: message - ignore_missing: true - - rename: - field: azure.signinlogs.properties.location.city - target_field: geo.city_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.location.state - target_field: geo.country_name - ignore_missing: true - - rename: - field: azure.signinlogs.properties.location.geoCoordinates.latitude - target_field: geo.location.lat - ignore_missing: true - - rename: - field: azure.signinlogs.properties.location.geoCoordinates.longitude - target_field: geo.location.lon - ignore_missing: true - - rename: - field: azure.signinlogs.properties.servicePrincipalId - target_field: azure.signinlogs.properties.service_principal_id - ignore_missing: true - - remove: - field: - - azure.signinlogs.properties.location - ignore_missing: true - - set: - field: event.kind - value: event - - set: - field: event.category - value: - - authentication - - set: - field: event.type - value: - - info - - set: - field: event.outcome - value: success - if: "ctx?.azure?.signinlogs?.properties?.status?.error_code == null || ctx.azure.signinlogs.properties.status.error_code == 0" - - set: - field: event.outcome - value: failure - if: "ctx?.azure?.signinlogs?.properties?.status?.error_code != null && ctx.azure.signinlogs.properties.status.error_code > 0" - - grok: - field: azure.signinlogs.properties.user_principal_name - patterns: - - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' - - '%{GREEDYDATA:user.name}' - ignore_missing: true - ignore_failure: true - - convert: - field: azure.signinlogs.properties.user_display_name - target_field: user.full_name - ignore_missing: true - type: string - - convert: - field: azure.signinlogs.properties.user_id - target_field: user.id - ignore_missing: true - type: string - - geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - - geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - - rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true - - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true - - pipeline: - name: '{{ IngestPipeline "azure-shared-pipeline" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: "1.10.0" + - rename: + field: azure + target_field: azure-eventhub + ignore_missing: true + - json: + field: message + target_field: azure.signinlogs + - drop: + if: ctx.azure.signinlogs.category != 'SignInLogs' + - date: + field: azure.signinlogs.time + target_field: '@timestamp' + ignore_failure: false + formats: + - ISO8601 + - rename: + field: message + target_field: event.original + - remove: + field: azure.signinlogs.time + ignore_missing: true + - rename: + field: azure.signinlogs.resourceId + target_field: azure.resource_id + ignore_missing: true + - rename: + field: azure.signinlogs.callerIpAddress + target_field: source.ip + ignore_missing: true + - set: + field: client.ip + value: '{{source.ip}}' + ignore_empty_value: true + - append: + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + if: 'ctx.source?.ip != null' + - rename: + field: azure.signinlogs.Level + target_field: log.level + ignore_missing: true + - rename: + field: azure.signinlogs.durationMs + target_field: event.duration + ignore_missing: true + - script: + lang: painless + source: ctx.event.duration = ctx.event.duration * params.param_nano + params: + param_nano: 1000000 + - rename: + field: azure.signinlogs.location + target_field: geo.country_iso_code + ignore_missing: true + - rename: + field: azure.signinlogs.resultType + target_field: azure.signinlogs.result_type + ignore_missing: true + - rename: + field: azure.signinlogs.operationName + target_field: azure.signinlogs.operation_name + ignore_missing: true + - convert: + field: azure.signinlogs.operation_name + target_field: event.action + type: string + ignore_missing: true + - rename: + field: azure.signinlogs.resultSignature + target_field: azure.signinlogs.result_signature + ignore_missing: true + - rename: + field: azure.signinlogs.resultDescription + target_field: azure.signinlogs.result_description + ignore_missing: true + - rename: + field: azure.signinlogs.operationVersion + target_field: azure.signinlogs.operation_version + ignore_missing: true + - rename: + field: azure.signinlogs.tenantId + target_field: azure.tenant_id + ignore_missing: true + - rename: + field: azure.signinlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.networkLocationDetails + target_field: azure.signinlogs.properties.network_location_details + ignore_missing: true + - rename: + field: azure.signinlogs.properties.resourceId + target_field: azure.signinlogs.properties.resource_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.appliedConditionalAccessPolicies + target_field: azure.signinlogs.properties.applied_conditional_access_policies + ignore_missing: true + - rename: + field: azure.signinlogs.properties.authenticationDetails + target_field: azure.signinlogs.properties.authentication_details + ignore_missing: true + - rename: + field: azure.signinlogs.properties.authenticationRequirementPolicies + target_field: azure.signinlogs.properties.authentication_requirement_policies + ignore_missing: true + - rename: + field: azure.signinlogs.properties.authenticationProcessingDetails + target_field: azure.signinlogs.properties.authentication_processing_details + ignore_missing: true + - rename: + field: azure.signinlogs.properties.deviceDetail + target_field: azure.signinlogs.properties.device_detail + ignore_missing: true + - rename: + field: azure.signinlogs.properties.device_detail.deviceId + target_field: azure.signinlogs.properties.device_detail.device_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.device_detail.operatingSystem + target_field: azure.signinlogs.properties.device_detail.operating_system + ignore_missing: true + - rename: + field: azure.signinlogs.properties.device_detail.displayName + target_field: azure.signinlogs.properties.device_detail.display_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.device_detail.trustType + target_field: azure.signinlogs.properties.device_detail.trust_type + ignore_missing: true + - rename: + field: azure.signinlogs.properties.createdDateTime + target_field: azure.signinlogs.properties.created_at + ignore_missing: true + - rename: + field: azure.signinlogs.properties.userDisplayName + target_field: azure.signinlogs.properties.user_display_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.correlationId + target_field: azure.signinlogs.properties.correlation_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.userPrincipalName + target_field: azure.signinlogs.properties.user_principal_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.userId + target_field: azure.signinlogs.properties.user_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.appId + target_field: azure.signinlogs.properties.app_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.appDisplayName + target_field: azure.signinlogs.properties.app_display_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.ipAddress + target_field: azure.signinlogs.properties.ip_address + ignore_missing: true + - rename: + field: azure.signinlogs.properties.clientAppUsed + target_field: azure.signinlogs.properties.client_app_used + ignore_missing: true + - rename: + field: azure.signinlogs.properties.conditionalAccessStatus + target_field: azure.signinlogs.properties.conditional_access_status + ignore_missing: true + - rename: + field: azure.signinlogs.properties.originalRequestId + target_field: azure.signinlogs.properties.original_request_id + ignore_missing: true + - rename: + field: azure.signinlogs.properties.isInteractive + target_field: azure.signinlogs.properties.is_interactive + ignore_missing: true + - rename: + field: azure.signinlogs.properties.tokenIssuerName + target_field: azure.signinlogs.properties.token_issuer_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.tokenIssuerType + target_field: azure.signinlogs.properties.token_issuer_type + ignore_missing: true + - rename: + field: azure.signinlogs.properties.processingTimeInMilliseconds + target_field: azure.signinlogs.properties.processing_time_ms + ignore_missing: true + - rename: + field: azure.signinlogs.properties.riskDetail + target_field: azure.signinlogs.properties.risk_detail + ignore_missing: true + - rename: + field: azure.signinlogs.properties.riskLevelAggregated + target_field: azure.signinlogs.properties.risk_level_aggregated + ignore_missing: true + - rename: + field: azure.signinlogs.properties.riskLevelDuringSignIn + target_field: azure.signinlogs.properties.risk_level_during_signin + ignore_missing: true + - rename: + field: azure.signinlogs.properties.riskState + target_field: azure.signinlogs.properties.risk_state + ignore_missing: true + - rename: + field: azure.signinlogs.properties.resourceDisplayName + target_field: azure.signinlogs.properties.resource_display_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.status.errorCode + target_field: azure.signinlogs.properties.status.error_code + ignore_missing: true + - rename: + field: azure.signinlogs.properties.status.failureReason + target_field: message + ignore_missing: true + - rename: + field: azure.signinlogs.properties.status.additionalDetails + target_field: message + ignore_missing: true + - rename: + field: azure.signinlogs.properties.location.city + target_field: geo.city_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.location.state + target_field: geo.country_name + ignore_missing: true + - rename: + field: azure.signinlogs.properties.location.geoCoordinates.latitude + target_field: geo.location.lat + ignore_missing: true + - rename: + field: azure.signinlogs.properties.location.geoCoordinates.longitude + target_field: geo.location.lon + ignore_missing: true + - rename: + field: azure.signinlogs.properties.servicePrincipalId + target_field: azure.signinlogs.properties.service_principal_id + ignore_missing: true + - remove: + field: + - azure.signinlogs.properties.location + ignore_missing: true + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - authentication + - set: + field: event.type + value: + - info + - set: + field: event.outcome + value: success + if: "ctx?.azure?.signinlogs?.properties?.status?.error_code == null || ctx.azure.signinlogs.properties.status.error_code == 0" + - set: + field: event.outcome + value: failure + if: "ctx?.azure?.signinlogs?.properties?.status?.error_code != null && ctx.azure.signinlogs.properties.status.error_code > 0" + - grok: + field: azure.signinlogs.properties.user_principal_name + patterns: + - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' + - '%{GREEDYDATA:user.name}' + ignore_missing: true + ignore_failure: true + - convert: + field: azure.signinlogs.properties.user_display_name + target_field: user.full_name + ignore_missing: true + type: string + - convert: + field: azure.signinlogs.properties.user_id + target_field: user.id + ignore_missing: true + type: string + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - pipeline: + name: '{{ IngestPipeline "azure-shared-pipeline" }}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/azure/data_stream/signinlogs/fields/ecs.yml b/packages/azure/data_stream/signinlogs/fields/ecs.yml index 11efd619e1e..240308d6c68 100644 --- a/packages/azure/data_stream/signinlogs/fields/ecs.yml +++ b/packages/azure/data_stream/signinlogs/fields/ecs.yml @@ -242,3 +242,8 @@ type: text name: user.name type: keyword +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/manifest.yml b/packages/azure/data_stream/signinlogs/manifest.yml index 649207fe673..a6d2bbf7b07 100644 --- a/packages/azure/data_stream/signinlogs/manifest.yml +++ b/packages/azure/data_stream/signinlogs/manifest.yml @@ -6,3 +6,28 @@ streams: template_path: "azure-eventhub.yml.hbs" title: "Azure sign-in logs" description: "Collect Azure sign-in logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-signinlogs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/azure/docs/README.md b/packages/azure/docs/README.md index 1e02aa9dce1..2346b12a7d9 100644 --- a/packages/azure/docs/README.md +++ b/packages/azure/docs/README.md @@ -268,6 +268,7 @@ An example event for `activitylogs` looks as following: | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source. | ip | | source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | | user.domain | Domain of the user. | keyword | | user.full_name | Full name of the user. | keyword | | user.id | Unique identifier of the user. | keyword | @@ -451,6 +452,7 @@ An example event for `platformlogs` looks as following: | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source. | ip | | source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | | user.domain | Domain of the user. | keyword | | user.full_name | Full name of the user. | keyword | | user.id | Unique identifier of the user. | keyword | @@ -637,6 +639,7 @@ An example event for `auditlogs` looks as following: | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source. | ip | | source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | | user.domain | Domain of the user. | keyword | | user.full_name | Full name of the user. | keyword | | user.id | Unique identifier of the user. | keyword | @@ -839,6 +842,7 @@ An example event for `signinlogs` looks as following: | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source. | ip | | source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | | user.domain | Domain of the user. | keyword | | user.full_name | Full name of the user. | keyword | | user.id | Unique identifier of the user. | keyword | diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 3db8bf2532a..f9b0a69d9f7 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -68,14 +68,6 @@ policy_templates: multi: false required: false show_user: true - - name: tags - type: text - title: Tags - multi: true - required: true - show_user: false - default: - - forwarded title: "Collect Azure logs from Event Hub" description: "Collecting activitylogs, auditlogs and signinlogs from Azure instances (input: azure-eventhub)" owner: From f867f3516070247901bd5d3e29317a55396fc42e Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 9 Jun 2021 11:48:33 +0200 Subject: [PATCH 2/2] linting and changelog --- packages/azure/changelog.yml | 5 +++++ .../test/pipeline/test-activitylogs-raw.log-expected.json | 2 +- packages/azure/data_stream/activitylogs/fields/ecs.yml | 2 +- packages/azure/data_stream/activitylogs/manifest.yml | 4 ++-- .../_dev/test/pipeline/test-auditlogs-raw.log-expected.json | 2 +- packages/azure/data_stream/auditlogs/fields/ecs.yml | 2 +- packages/azure/data_stream/auditlogs/manifest.yml | 1 + .../platformlogs/_dev/test/pipeline/test-common-config.yml | 2 +- .../pipeline/test-platformlogs-invalid-raw.log-expected.json | 2 +- .../test/pipeline/test-platformlogs-raw.log-expected.json | 2 +- .../pipeline/test-platformlogs-remote-raw.log-expected.json | 2 +- packages/azure/data_stream/platformlogs/fields/ecs.yml | 2 +- packages/azure/data_stream/platformlogs/manifest.yml | 4 ++-- .../_dev/test/pipeline/test-signinlogs-raw.log-expected.json | 4 ++-- packages/azure/data_stream/signinlogs/fields/ecs.yml | 2 +- packages/azure/data_stream/signinlogs/manifest.yml | 1 + packages/azure/manifest.yml | 2 +- 17 files changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index a2f2a04da17..b6c2b0f4099 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.0" + changes: + - description: update to ECS 1.10.0 and adding event.original options + type: enhancement + link: https://github.com/elastic/integrations/pull/1113 - version: "0.2.3" changes: - description: update to ECS 1.9.0 diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index cd56ab86b9f..5cc56cc978f 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -46,7 +46,7 @@ "event": { "duration": 0, "action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "ingested": "2021-06-09T09:37:58.777185600Z", + "ingested": "2021-06-09T09:48:12.995258100Z", "original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}", "type": [ "change" diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml index 240308d6c68..c49c4da74ff 100644 --- a/packages/azure/data_stream/activitylogs/fields/ecs.yml +++ b/packages/azure/data_stream/activitylogs/fields/ecs.yml @@ -246,4 +246,4 @@ description: List of keywords used to tag each event. example: '["production", "env2"]' ignore_above: 1024 - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/azure/data_stream/activitylogs/manifest.yml b/packages/azure/data_stream/activitylogs/manifest.yml index fe590f45bd6..28892c409c5 100644 --- a/packages/azure/data_stream/activitylogs/manifest.yml +++ b/packages/azure/data_stream/activitylogs/manifest.yml @@ -29,5 +29,5 @@ streams: multi: false required: false show_user: false - description: > - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. \ No newline at end of file + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json index bd14334745b..4695c0fbd53 100644 --- a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json @@ -14,7 +14,7 @@ "event": { "duration": 0, "action": "Update device", - "ingested": "2021-06-09T09:37:59.162398200Z", + "ingested": "2021-06-09T09:48:13.410914700Z", "original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}", "kind": "event", "outcome": "success" diff --git a/packages/azure/data_stream/auditlogs/fields/ecs.yml b/packages/azure/data_stream/auditlogs/fields/ecs.yml index 030c503054e..0321abb5445 100644 --- a/packages/azure/data_stream/auditlogs/fields/ecs.yml +++ b/packages/azure/data_stream/auditlogs/fields/ecs.yml @@ -243,4 +243,4 @@ description: List of keywords used to tag each event. example: '["production", "env2"]' ignore_above: 1024 - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/azure/data_stream/auditlogs/manifest.yml b/packages/azure/data_stream/auditlogs/manifest.yml index 309fb4c475f..c5f03d101f2 100644 --- a/packages/azure/data_stream/auditlogs/manifest.yml +++ b/packages/azure/data_stream/auditlogs/manifest.yml @@ -31,3 +31,4 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml index e74affa452f..5622947e4b8 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml @@ -2,4 +2,4 @@ dynamic_fields: event.ingested: ".*" fields: tags: - - preserve_original_event \ No newline at end of file + - preserve_original_event diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json index 69a68543881..75867c03e09 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "action": "ApplicationGatewayAccess", - "ingested": "2021-06-09T09:37:59.356079400Z", + "ingested": "2021-06-09T09:48:13.594211900Z", "original": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}", "kind": "event" }, diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json index e4d7f5ddfd3..be8739557d3 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json @@ -11,7 +11,7 @@ }, "event": { "action": "Retreive ConsumerGroup", - "ingested": "2021-06-09T09:37:59.380613800Z", + "ingested": "2021-06-09T09:48:13.618911200Z", "original": "{\"ActivityId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8\",\"Caller\":\"Portal\",\"Environment\":\"PROD\",\"EventName\":\"Retreive ConsumerGroup\",\"EventProperties\":\"{\\\"SubscriptionId\\\":\\\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\\\",\\\"Namespace\\\":\\\"obstesteventhubs\\\",\\\"Via\\\":\\\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\\u0026$skip=0\\u0026$top=100\\\",\\\"TrackingId\\\":\\\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\\\"}\",\"EventTimeString\":\"11/3/2020 9:06:42 AM +00:00\",\"Region\":\"West Europe\",\"ScaleUnit\":\"PROD-AM3-AZ501\",\"Status\":\"Succeeded\",\"category\":\"OperationalLogs\",\"resourceId\":\"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS\"}", "kind": "event", "outcome": "succeeded" diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json index 3d54decb8e5..2eb0ee6cb8e 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested": "2021-06-09T09:37:59.405299500Z", + "ingested": "2021-06-09T09:48:13.649907200Z", "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", "kind": "event" }, diff --git a/packages/azure/data_stream/platformlogs/fields/ecs.yml b/packages/azure/data_stream/platformlogs/fields/ecs.yml index 030c503054e..0321abb5445 100644 --- a/packages/azure/data_stream/platformlogs/fields/ecs.yml +++ b/packages/azure/data_stream/platformlogs/fields/ecs.yml @@ -243,4 +243,4 @@ description: List of keywords used to tag each event. example: '["production", "env2"]' ignore_above: 1024 - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/azure/data_stream/platformlogs/manifest.yml b/packages/azure/data_stream/platformlogs/manifest.yml index cbe92715ec0..c7872f90c02 100644 --- a/packages/azure/data_stream/platformlogs/manifest.yml +++ b/packages/azure/data_stream/platformlogs/manifest.yml @@ -29,5 +29,5 @@ streams: multi: false required: false show_user: false - description: > - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. \ No newline at end of file + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json index 31ab0c15f20..b66026bc343 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json @@ -55,7 +55,7 @@ }, "event": { "duration": 0, - "ingested": "2021-06-09T09:37:59.560632100Z", + "ingested": "2021-06-09T09:48:13.797038100Z", "original": "{\"Level\":\"4\",\"callerIpAddress\":\"81.171.241.231\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", @@ -174,7 +174,7 @@ }, "event": { "duration": 0, - "ingested": "2021-06-09T09:37:59.560652700Z", + "ingested": "2021-06-09T09:48:13.797078800Z", "original": "{\"Level\":\"4\",\"callerIpAddress\":\"8.8.8.8\",\"category\":\"SignInLogs\",\"correlationId\":\"a8d4eb85-90c5-740d-9af6-7a15036cd135\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"c3813493-bf92-5123-2717-8a8b2979c38b\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", diff --git a/packages/azure/data_stream/signinlogs/fields/ecs.yml b/packages/azure/data_stream/signinlogs/fields/ecs.yml index 240308d6c68..c49c4da74ff 100644 --- a/packages/azure/data_stream/signinlogs/fields/ecs.yml +++ b/packages/azure/data_stream/signinlogs/fields/ecs.yml @@ -246,4 +246,4 @@ description: List of keywords used to tag each event. example: '["production", "env2"]' ignore_above: 1024 - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/azure/data_stream/signinlogs/manifest.yml b/packages/azure/data_stream/signinlogs/manifest.yml index a6d2bbf7b07..ca7266771c0 100644 --- a/packages/azure/data_stream/signinlogs/manifest.yml +++ b/packages/azure/data_stream/signinlogs/manifest.yml @@ -31,3 +31,4 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index f9b0a69d9f7..18b16578a45 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure -version: 0.2.3 +version: 0.3.0 release: beta description: Azure Integration type: integration