From f5911c083b2c262a79f999fe65141f2cfee0acdb Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 5 Sep 2025 12:40:45 +0200 Subject: [PATCH 1/7] Rename CategoryValue as category to fix routing --- .../pipeline/test-invalid-categoryvalue.log | 1 + ...st-invalid-categoryvalue.log-expected.json | 19 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 13 +++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log create mode 100644 packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log-expected.json diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log new file mode 100644 index 00000000000..02c8ed04232 --- /dev/null +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log @@ -0,0 +1 @@ +{"ActivityStatusValue":"Active","Caller":"Microsoft.Advisor","CallerIpAddress":"0.0.0.0","CategoryValue":"Recommendation","Claims":"{\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress\":\"Microsoft.Advisor\"}","Claims_d":{"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress":"Microsoft.Advisor"},"CorrelationId":"aa6ee2d0-2dd0-4dc3-8e68-f6acf9a47408","EventDataId":"fff41641-2501-407a-bbc5-2b144b1f7e99","EventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","HTTPRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","Level":"Informational","OperationNameValue":"Microsoft.Advisor/recommendations/available/action","Properties":"{\"recommendationSchemaVersion\":\"1.0\"}","Properties_d":{"activityStatusValue":"Active","caller":"Microsoft.Advisor","eventDataId":"redacted","eventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","httpRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","recommendationCategory":"Security","recommendationImpact":"Medium","recommendationName":"test","recommendationResourceLink":"redacted","recommendationSchemaVersion":"1.0","recommendationType":"9b2b7b94-321b-4868-b50f-46a3921386f1","resource":"redacted","resourceGroup":"redacted","resourceProviderValue":"MICROSOFT.SECURITY","subscriptionId":"redacted"},"ResourceGroup":"redacted","ResourceProviderValue":"MICROSOFT.SECURITY","SourceSystem":"Azure","SubscriptionId":"redacted","TenantId":"redacted","TimeGenerated":"2025-08-02T09:42:47.3766735Z","Type":"AzureActivity","_Internal_WorkspaceResourceId":"redacted","_ItemId":"redacted","_ResourceId":"redacted","_SubscriptionId":"redacted"} \ No newline at end of file diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log-expected.json b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log-expected.json new file mode 100644 index 00000000000..1c93710aea5 --- /dev/null +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log-expected.json @@ -0,0 +1,19 @@ +{ + "expected": [ + { + "data_stream": { + "dataset": "azure.activitylogs", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "dataset": "azure.activitylogs", + "kind": "event" + }, + "message": "{\"ActivityStatusValue\":\"Active\",\"Caller\":\"Microsoft.Advisor\",\"CallerIpAddress\":\"0.0.0.0\",\"CategoryValue\":\"Recommendation\",\"Claims\":\"{\\\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress\\\":\\\"Microsoft.Advisor\\\"}\",\"Claims_d\":{\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress\":\"Microsoft.Advisor\"},\"CorrelationId\":\"aa6ee2d0-2dd0-4dc3-8e68-f6acf9a47408\",\"EventDataId\":\"fff41641-2501-407a-bbc5-2b144b1f7e99\",\"EventSubmissionTimestamp\":\"2025-08-02T09:42:47.3766735Z\",\"HTTPRequest\":\"{\\\"clientIpAddress\\\":\\\"0.0.0.0\\\"}\",\"Level\":\"Informational\",\"OperationNameValue\":\"Microsoft.Advisor/recommendations/available/action\",\"Properties\":\"{\\\"recommendationSchemaVersion\\\":\\\"1.0\\\"}\",\"Properties_d\":{\"activityStatusValue\":\"Active\",\"caller\":\"Microsoft.Advisor\",\"eventDataId\":\"redacted\",\"eventSubmissionTimestamp\":\"2025-08-02T09:42:47.3766735Z\",\"httpRequest\":\"{\\\"clientIpAddress\\\":\\\"0.0.0.0\\\"}\",\"recommendationCategory\":\"Security\",\"recommendationImpact\":\"Medium\",\"recommendationName\":\"test\",\"recommendationResourceLink\":\"redacted\",\"recommendationSchemaVersion\":\"1.0\",\"recommendationType\":\"9b2b7b94-321b-4868-b50f-46a3921386f1\",\"resource\":\"redacted\",\"resourceGroup\":\"redacted\",\"resourceProviderValue\":\"MICROSOFT.SECURITY\",\"subscriptionId\":\"redacted\"},\"ResourceGroup\":\"redacted\",\"ResourceProviderValue\":\"MICROSOFT.SECURITY\",\"SourceSystem\":\"Azure\",\"SubscriptionId\":\"redacted\",\"TenantId\":\"redacted\",\"TimeGenerated\":\"2025-08-02T09:42:47.3766735Z\",\"Type\":\"AzureActivity\",\"_Internal_WorkspaceResourceId\":\"redacted\",\"_ItemId\":\"redacted\",\"_ResourceId\":\"redacted\",\"_SubscriptionId\":\"redacted\"}" + } + ] +} diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml index a350b82c371..800b08086a3 100644 --- a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -44,6 +44,19 @@ processors: if: 'ctx.tmp_json?.category == null' ignore_missing: true description: 'Rename the invalid `Category` field to `category` to apply the correct routing rules.' + # Unfortunately, some Azure services produce logs with `CategoryValue` field + # instead of `Category` field. + # + # We need to rename it to `category` to apply the correct routing rules. + # + # Refs; + # - https://github.com/elastic/integrations/issues/15083 + - rename: + field: tmp_json.CategoryValue + target_field: tmp_json.category + if: 'ctx.tmp_json?.category == null' + ignore_missing: true + description: 'Rename the invalid `CategoryValue` field to `category` to apply the correct routing rules.' # Defaults to azure.events if the `category` field is not present. - set: From 2f12a75e2b4162ebf097f9879d4976eea1fcc155 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 5 Sep 2025 12:51:54 +0200 Subject: [PATCH 2/7] Support non-standard created_date_time field Some Azure services generate log without the standard `time` field, and use the non-standard `created_date_time` field instead. This fix renames the `created_date_time` field as `time`, when this is not set. --- .../_dev/test/pipeline/test-invalid-time.log | 1 + .../test-invalid-time.log-expected.json | 37 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 8 +++- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log create mode 100644 packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log-expected.json diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log new file mode 100644 index 00000000000..65618b14f75 --- /dev/null +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log @@ -0,0 +1 @@ +{"Category":"ServicePrincipalSignInLogs","CorrelationId":"83d4a233-76a0-4cc0-bbe6-9ce7ad506fc9","CreatedDateTime":"2025-07-01T10:45:17.5824212Z","DurationMs":0,"OperationName":"Sign-in activity","TenantId":"2a0bb6ef-8a1d-4e8b-83d6-c682d5ca56db7"} \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log-expected.json new file mode 100644 index 00000000000..22e181f1c45 --- /dev/null +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log-expected.json @@ -0,0 +1,37 @@ +{ + "expected": [ + { + "@timestamp": "2025-07-01T10:45:17.582Z", + "azure": { + "correlation_id": "83d4a233-76a0-4cc0-bbe6-9ce7ad506fc9", + "signinlogs": { + "category": "ServicePrincipalSignInLogs", + "operation_name": "Sign-in activity" + }, + "tenant_id": "2a0bb6ef-8a1d-4e8b-83d6-c682d5ca56db7" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Sign-in activity", + "category": [ + "authentication" + ], + "duration": 0, + "kind": "event", + "original": "{\"Category\":\"ServicePrincipalSignInLogs\",\"CorrelationId\":\"83d4a233-76a0-4cc0-bbe6-9ce7ad506fc9\",\"CreatedDateTime\":\"2025-07-01T10:45:17.5824212Z\",\"DurationMs\":0,\"OperationName\":\"Sign-in activity\",\"TenantId\":\"2a0bb6ef-8a1d-4e8b-83d6-c682d5ca56db7\"}", + "outcome": "success", + "type": [ + "info" + ] + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index 04bd4e4b833..e179911bfcb 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -45,7 +45,13 @@ processors: ctx.azure['signinlogs'] = keysToSnakeCase(ctx.azure.signinlogs); - drop: description: Drop non-SignInLogs. - if: ctx?.azure?.signinlogs?.category == null || !ctx.azure.signinlogs.category.endsWith('SignInLogs') + if: ctx.azure?.signinlogs?.category == null || !ctx.azure.signinlogs.category.endsWith('SignInLogs') + - rename: + field: azure.signinlogs.created_date_time + target_field: azure.signinlogs.time + ignore_missing: true + if: 'ctx.azure?.signinlogs?.time == null' + description: 'Fallback to handle special cases for log category: use `created_date_time` as `time` if `time` is not already set.' - date: field: azure.signinlogs.time formats: From 6c4977d36e25917fe5a598cddd70760aa222c84c Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 5 Sep 2025 13:11:02 +0200 Subject: [PATCH 3/7] Update changelog --- packages/azure/changelog.yml | 5 +++++ packages/azure/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 9246001314a..c2cdcbebcc5 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.28.7" + changes: + - description: Interim fix to support non-standard log events. + type: bugfix + link: https://github.com/elastic/integrations/pull/15205 - version: "1.28.6" changes: - description: Add FAQ section to Azure Logs integration v2. diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index a56e3a7e8b4..21abdf335bf 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.28.6" +version: "1.28.7" description: This Elastic integration collects logs from Azure type: integration icons: From d5b829a895b11c00fafc926e0d8cb7a380129b55 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 9 Sep 2025 12:18:25 +0200 Subject: [PATCH 4/7] Add missing newlines --- .../events/_dev/test/pipeline/test-activitylogs-raw.log | 2 +- .../events/_dev/test/pipeline/test-application-gateway-raw.log | 2 +- .../events/_dev/test/pipeline/test-auditlogs-raw.log | 2 +- .../events/_dev/test/pipeline/test-firewall-logs-raw.log | 2 +- .../events/_dev/test/pipeline/test-graphactivitylogs-raw.log | 2 +- .../events/_dev/test/pipeline/test-identity-protection-raw.log | 2 +- .../_dev/test/pipeline/test-invalid-category-field-raw.log | 2 +- .../events/_dev/test/pipeline/test-invalid-categoryvalue.log | 2 +- .../events/_dev/test/pipeline/test-provisioning-raw.log | 2 +- .../events/_dev/test/pipeline/test-signinlogs-raw.log | 2 +- .../signinlogs/_dev/test/pipeline/test-invalid-time.log | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-activitylogs-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-activitylogs-raw.log index 00a8542847f..470f4415a68 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-activitylogs-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-activitylogs-raw.log @@ -5,4 +5,4 @@ {"category":"Recommendation"} {"category":"Policy"} {"category":"Autoscale"} -{"category":"ResourceHealth"} \ No newline at end of file +{"category":"ResourceHealth"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-application-gateway-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-application-gateway-raw.log index 931f30664c4..1ea9b33eee7 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-application-gateway-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-application-gateway-raw.log @@ -1,2 +1,2 @@ {"category":"ApplicationGatewayFirewallLog"} -{"category":"ApplicationGatewayAccessLog"} \ No newline at end of file +{"category":"ApplicationGatewayAccessLog"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-auditlogs-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-auditlogs-raw.log index e6c5a4c418f..222f4b25ff0 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-auditlogs-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-auditlogs-raw.log @@ -1 +1 @@ -{"category":"AuditLogs"} \ No newline at end of file +{"category":"AuditLogs"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-firewall-logs-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-firewall-logs-raw.log index d283f457c20..0fc48fffd3f 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-firewall-logs-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-firewall-logs-raw.log @@ -4,4 +4,4 @@ {"category":"AZFWApplicationRule"} {"category":"AZFWNetworkRule"} {"category":"AZFWNatRule"} -{"category":"AZFWDnsQuery"} \ No newline at end of file +{"category":"AZFWDnsQuery"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-graphactivitylogs-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-graphactivitylogs-raw.log index 6440c58fd86..33b2b65f3ca 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-graphactivitylogs-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-graphactivitylogs-raw.log @@ -1 +1 @@ -{"category":"MicrosoftGraphActivityLogs"} \ No newline at end of file +{"category":"MicrosoftGraphActivityLogs"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-identity-protection-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-identity-protection-raw.log index d092942bb4f..e6586acf972 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-identity-protection-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-identity-protection-raw.log @@ -1,2 +1,2 @@ {"category":"RiskyUsers"} -{"category":"UserRiskEvents"} \ No newline at end of file +{"category":"UserRiskEvents"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-category-field-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-category-field-raw.log index e6992db40ba..7a9404da61e 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-category-field-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-category-field-raw.log @@ -1,2 +1,2 @@ {"Category":"ServicePrincipalSignInLogs"} -{"Category":"UnsupportedCategoryName"} \ No newline at end of file +{"Category":"UnsupportedCategoryName"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log index 02c8ed04232..afc754f4f93 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-invalid-categoryvalue.log @@ -1 +1 @@ -{"ActivityStatusValue":"Active","Caller":"Microsoft.Advisor","CallerIpAddress":"0.0.0.0","CategoryValue":"Recommendation","Claims":"{\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress\":\"Microsoft.Advisor\"}","Claims_d":{"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress":"Microsoft.Advisor"},"CorrelationId":"aa6ee2d0-2dd0-4dc3-8e68-f6acf9a47408","EventDataId":"fff41641-2501-407a-bbc5-2b144b1f7e99","EventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","HTTPRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","Level":"Informational","OperationNameValue":"Microsoft.Advisor/recommendations/available/action","Properties":"{\"recommendationSchemaVersion\":\"1.0\"}","Properties_d":{"activityStatusValue":"Active","caller":"Microsoft.Advisor","eventDataId":"redacted","eventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","httpRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","recommendationCategory":"Security","recommendationImpact":"Medium","recommendationName":"test","recommendationResourceLink":"redacted","recommendationSchemaVersion":"1.0","recommendationType":"9b2b7b94-321b-4868-b50f-46a3921386f1","resource":"redacted","resourceGroup":"redacted","resourceProviderValue":"MICROSOFT.SECURITY","subscriptionId":"redacted"},"ResourceGroup":"redacted","ResourceProviderValue":"MICROSOFT.SECURITY","SourceSystem":"Azure","SubscriptionId":"redacted","TenantId":"redacted","TimeGenerated":"2025-08-02T09:42:47.3766735Z","Type":"AzureActivity","_Internal_WorkspaceResourceId":"redacted","_ItemId":"redacted","_ResourceId":"redacted","_SubscriptionId":"redacted"} \ No newline at end of file +{"ActivityStatusValue":"Active","Caller":"Microsoft.Advisor","CallerIpAddress":"0.0.0.0","CategoryValue":"Recommendation","Claims":"{\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress\":\"Microsoft.Advisor\"}","Claims_d":{"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress":"Microsoft.Advisor"},"CorrelationId":"aa6ee2d0-2dd0-4dc3-8e68-f6acf9a47408","EventDataId":"fff41641-2501-407a-bbc5-2b144b1f7e99","EventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","HTTPRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","Level":"Informational","OperationNameValue":"Microsoft.Advisor/recommendations/available/action","Properties":"{\"recommendationSchemaVersion\":\"1.0\"}","Properties_d":{"activityStatusValue":"Active","caller":"Microsoft.Advisor","eventDataId":"redacted","eventSubmissionTimestamp":"2025-08-02T09:42:47.3766735Z","httpRequest":"{\"clientIpAddress\":\"0.0.0.0\"}","recommendationCategory":"Security","recommendationImpact":"Medium","recommendationName":"test","recommendationResourceLink":"redacted","recommendationSchemaVersion":"1.0","recommendationType":"9b2b7b94-321b-4868-b50f-46a3921386f1","resource":"redacted","resourceGroup":"redacted","resourceProviderValue":"MICROSOFT.SECURITY","subscriptionId":"redacted"},"ResourceGroup":"redacted","ResourceProviderValue":"MICROSOFT.SECURITY","SourceSystem":"Azure","SubscriptionId":"redacted","TenantId":"redacted","TimeGenerated":"2025-08-02T09:42:47.3766735Z","Type":"AzureActivity","_Internal_WorkspaceResourceId":"redacted","_ItemId":"redacted","_ResourceId":"redacted","_SubscriptionId":"redacted"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-provisioning-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-provisioning-raw.log index 55b746379b8..43b16c5200a 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-provisioning-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-provisioning-raw.log @@ -1 +1 @@ -{"category":"ProvisioningLogs"} \ No newline at end of file +{"category":"ProvisioningLogs"} diff --git a/packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log b/packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log index 499c53e0364..073c2c434f8 100644 --- a/packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log +++ b/packages/azure/data_stream/events/_dev/test/pipeline/test-signinlogs-raw.log @@ -1,4 +1,4 @@ {"Category":"SignInLogs"} {"Category":"NonInteractiveUserSignInLogs"} {"Category":"ServicePrincipalSignInLogs"} -{"Category":"ManagedIdentitySignInLogs"} \ No newline at end of file +{"Category":"ManagedIdentitySignInLogs"} diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log index 65618b14f75..6e39c186b05 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-invalid-time.log @@ -1 +1 @@ -{"Category":"ServicePrincipalSignInLogs","CorrelationId":"83d4a233-76a0-4cc0-bbe6-9ce7ad506fc9","CreatedDateTime":"2025-07-01T10:45:17.5824212Z","DurationMs":0,"OperationName":"Sign-in activity","TenantId":"2a0bb6ef-8a1d-4e8b-83d6-c682d5ca56db7"} \ No newline at end of file +{"Category":"ServicePrincipalSignInLogs","CorrelationId":"83d4a233-76a0-4cc0-bbe6-9ce7ad506fc9","CreatedDateTime":"2025-07-01T10:45:17.5824212Z","DurationMs":0,"OperationName":"Sign-in activity","TenantId":"2a0bb6ef-8a1d-4e8b-83d6-c682d5ca56db7"} From f5fafe609c3e0eb4ab8cd3c43bf44b880456fe03 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 9 Sep 2025 12:19:29 +0200 Subject: [PATCH 5/7] Drop redundant single quotes --- .../events/elasticsearch/ingest_pipeline/default.yml | 2 +- .../signinlogs/elasticsearch/ingest_pipeline/default.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 800b08086a3..a4e5eb79b11 100644 --- a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -54,7 +54,7 @@ processors: - rename: field: tmp_json.CategoryValue target_field: tmp_json.category - if: 'ctx.tmp_json?.category == null' + if: ctx.tmp_json?.category == null ignore_missing: true description: 'Rename the invalid `CategoryValue` field to `category` to apply the correct routing rules.' diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index e179911bfcb..cb8e46b4d14 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -50,7 +50,7 @@ processors: field: azure.signinlogs.created_date_time target_field: azure.signinlogs.time ignore_missing: true - if: 'ctx.azure?.signinlogs?.time == null' + if: ctx.azure?.signinlogs?.time == null description: 'Fallback to handle special cases for log category: use `created_date_time` as `time` if `time` is not already set.' - date: field: azure.signinlogs.time From d0a08b4f7a85a333ce7a87b8b7ec9f4f8f167e76 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 9 Sep 2025 13:12:11 +0200 Subject: [PATCH 6/7] Marginally clarify workaround comments --- .../events/elasticsearch/ingest_pipeline/default.yml | 7 ++++--- .../signinlogs/elasticsearch/ingest_pipeline/default.yml | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml index a4e5eb79b11..7b868e62fe1 100644 --- a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -44,10 +44,11 @@ processors: if: 'ctx.tmp_json?.category == null' ignore_missing: true description: 'Rename the invalid `Category` field to `category` to apply the correct routing rules.' - # Unfortunately, some Azure services produce logs with `CategoryValue` field - # instead of `Category` field. + # Unfortunately, some Azure services generate logs with + # `CategoryValue` field instead of `Category` field. # - # We need to rename it to `category` to apply the correct routing rules. + # We need to rename `CategoryValue` as `category` to + # apply the correct routing rules. # # Refs; # - https://github.com/elastic/integrations/issues/15083 diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index cb8e46b4d14..a99c2f5d889 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -46,6 +46,14 @@ processors: - drop: description: Drop non-SignInLogs. if: ctx.azure?.signinlogs?.category == null || !ctx.azure.signinlogs.category.endsWith('SignInLogs') + # Unfortunately, some Azure services generate logs with + # `created_date_time` field instead of `time` field. + # + # We need to rename `created_date_time` as `time` to + # apply the correct timestamp processing. + # + # Refs; + # - https://github.com/elastic/integrations/issues/15083 - rename: field: azure.signinlogs.created_date_time target_field: azure.signinlogs.time From db9d3f60366e01866736484cd84a61b93de95c3b Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 12 Sep 2025 17:44:36 +0200 Subject: [PATCH 7/7] Replace ref issue with a more relevant one --- .../events/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 7b868e62fe1..a390bcc090f 100644 --- a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -51,7 +51,7 @@ processors: # apply the correct routing rules. # # Refs; - # - https://github.com/elastic/integrations/issues/15083 + # - https://github.com/elastic/integrations/issues/15206 - rename: field: tmp_json.CategoryValue target_field: tmp_json.category