elastic · P1llus · Jun 10, 2021 · Jun 9, 2021 · Jun 9, 2021
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.3.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1113
 - version: "0.2.3"
   changes:
     - description: update to ECS 1.9.0

diff --git a/...es/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml b/...es/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-config.yml
@@ -1,2 +1,5 @@
 dynamic_fields:
   event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/...azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/...azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json
@@ -16,6 +16,9 @@
                 },
                 "ip": "51.251.141.41"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "geo": {
                 "continent_name": "Europe",
                 "country_name": "United Kingdom",
@@ -29,21 +32,21 @@
                 "provider": "azure"
             },
             "@timestamp": "2019-10-24T00:13:46.355Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "51.251.141.41"
                 ]
             },
-            "ecs": {
-                "version": "1.9.0"
-            },
             "client": {
                 "ip": "51.251.141.41"
             },
             "event": {
                 "duration": 0,
                 "action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION",
-                "ingested": "2021-04-23T12:52:57.328609264Z",
+                "ingested": "2021-06-09T09:48:12.995258100Z",
                 "original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}",
                 "type": [
                     "change"

diff --git a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs
@@ -18,14 +18,16 @@ storage_account_key: {{storage_account_key}}
 resource_manager_endpoint: {{resource_manager_endpoint}}
 {{/if}}
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
-- {{tag}}
+  - {{tag}}
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
+{{#if processors}}
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.9.0
+{{processors}}
+{{/if}}
diff --git a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs
@@ -4,14 +4,16 @@ paths:
   {{/each}}
 exclude_files: [".gz$"]
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
-- {{tag}}
+  - {{tag}}
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
+{{#if processors}}
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.9.0
+{{processors}}
+{{/if}}
diff --git a/...es/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/...es/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -61,9 +61,6 @@ processors:
 - lowercase:
     field: event.outcome
     ignore_missing: true
-- set:
-    field: ecs.version
-    value: 1.9.0
 on_failure:
 - set:
     field: error.message

diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml
@@ -4,6 +4,9 @@ processors:
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
+  - set:
+      field: ecs.version
+      value: "1.10.0"
   - rename:
       field: azure
       target_field: azure-eventhub
@@ -13,18 +16,19 @@ processors:
       params:
         empty_field_name: '"":"",'
       ignore_failure: true
-  - json:
+  - rename:
       field: message
+      target_field: event.original
+      ignore_missing: true
+  - json:
+      field: event.original
       target_field: azure.activitylogs
   - date:
       field: azure.activitylogs.time
       target_field: '@timestamp'
       ignore_failure: true
       formats:
         - ISO8601
-  - rename:
-      field: message
-      target_field: event.original
   - remove:
       field: azure.activitylogs.time
       ignore_missing: true
@@ -264,6 +268,11 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml
@@ -242,3 +242,8 @@
       type: text
   name: user.name
   type: keyword
+- name: tags
+  description: List of keywords used to tag each event.
+  example: '["production", "env2"]'
+  ignore_above: 1024
+  type: keyword
diff --git a/packages/azure/data_stream/activitylogs/manifest.yml b/packages/azure/data_stream/activitylogs/manifest.yml
@@ -6,3 +6,28 @@ streams:
     template_path: "azure-eventhub.yml.hbs"
     title: "Azure activity logs"
     description: "Collect Azure activity logs using azure-eventhub input"
+    vars:
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - azure-activitylogs
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-config.yml
@@ -1,2 +1,5 @@
 dynamic_fields:
   event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json
@@ -6,19 +6,22 @@
             },
             "@timestamp": "2019-10-18T15:30:51.027Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "log": {
                 "level": "Informational"
             },
             "event": {
                 "duration": 0,
                 "action": "Update device",
-                "ingested": "2021-04-23T12:52:57.546715602Z",
+                "ingested": "2021-06-09T09:48:13.410914700Z",
                 "original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}",
                 "kind": "event",
                 "outcome": "success"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "azure": {
                 "tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53",
                 "correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53",

diff --git a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs
@@ -18,14 +18,17 @@ storage_account_key: {{storage_account_key}}
 resource_manager_endpoint: {{resource_manager_endpoint}}
 {{/if}}
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
-- {{tag}}
+  - {{tag}}
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
+{{#if processors}}
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.9.0
+{{processors}}
+{{/if}}
+
diff --git a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs
@@ -4,14 +4,16 @@ paths:
   {{/each}}
 exclude_files: [".gz$"]
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
-- {{tag}}
+  - {{tag}}
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
+{{#if processors}}
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.9.0
+{{processors}}
+{{/if}}
diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -39,9 +39,6 @@ processors:
 - lowercase:
     field: event.outcome
     ignore_missing: true
-- set:
-    field: ecs.version
-    value: 1.9.0
 on_failure:
 - set:
     field: error.message

diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml
@@ -4,12 +4,19 @@ processors:
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
+  - set:
+      field: ecs.version
+      value: "1.10.0"
   - rename:
       field: azure
       target_field: azure-eventhub
       ignore_missing: true
-  - json:
+  - rename:
       field: message
+      target_field: event.original
+      ignore_missing: true
+  - json:
+      field: event.original
       target_field: azure.auditlogs
   - drop:
       if: ctx.azure.auditlogs.category != 'AuditLogs'
@@ -40,9 +47,6 @@ processors:
       field: azure.auditlogs.level
       target_field: log.level
       ignore_missing: true
-  - rename:
-      field: message
-      target_field: event.original
   - remove:
       field: azure.auditlogs.time
       ignore_missing: true
@@ -140,6 +144,11 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/azure/data_stream/auditlogs/fields/ecs.yml b/packages/azure/data_stream/auditlogs/fields/ecs.yml
@@ -239,3 +239,8 @@
       type: text
   name: user.name
   type: keyword
+- name: tags
+  description: List of keywords used to tag each event.
+  example: '["production", "env2"]'
+  ignore_above: 1024
+  type: keyword
diff --git a/packages/azure/data_stream/auditlogs/manifest.yml b/packages/azure/data_stream/auditlogs/manifest.yml
@@ -6,3 +6,29 @@ streams:
     template_path: "azure-eventhub.yml.hbs"
     title: "Azure audit logs"
     description: "Collect Azure audit logs using azure-eventhub input"
+    vars:
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - azure-auditlogs
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/.../data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml b/.../data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-config.yml
diff --git a/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
@@ -6,14 +6,17 @@
             },
             "@timestamp": "2020-10-11T20:30:59.000Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "event": {
                 "action": "ApplicationGatewayAccess",
-                "ingested": "2021-04-23T12:52:57.717406206Z",
+                "ingested": "2021-06-09T09:48:13.594211900Z",
                 "original": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}",
                 "kind": "event"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "azure": {
                 "resource": {
                     "id": "/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY"

diff --git a/...es/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml b/...es/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-config.yml