From 13a735b9276fa1993b8055b1d339de520105c568 Mon Sep 17 00:00:00 2001 From: narph Date: Mon, 9 Aug 2021 16:49:49 +0200 Subject: [PATCH 1/9] add support for sprincloud logs --- packages/azure/changelog.yml | 5 ++ .../elasticsearch/ingest_pipeline/default.yml | 4 + .../ingest_pipeline/springcloudlogs.yml | 78 +++++++++++++++++++ packages/azure/manifest.yml | 2 +- 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index c3f13cd4090..b7b95d70b2d 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.8.1" + changes: + - description: Add support for springcloud logs inside the platformlogs pipeline + type: enhancement + link: https://github.com/elastic/integrations/pull/1455 - version: "0.8.0" changes: - description: Import ECS field definitions diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 07fb1dafc4c..034cb623ea8 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -245,6 +245,10 @@ processors: if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true + - pipeline: + if: "ctx?.azure_log_forwarder?.resource_type != null && ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'" + name: '{{ IngestPipeline "springcloudlogs" }}' + } on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml new file mode 100644 index 00000000000..f806cfc1ea0 --- /dev/null +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml @@ -0,0 +1,78 @@ +--- +description: Pipeline for parsing azure spring cloud logs. +processors: + - script: + source: ctx['_index'] = ctx['_index'].replace('platformlogs', 'springcloudlogs') + ignore_failure: true + - rename: + field: azure.platformlogs + target_field: azure.springcloudlogs + ignore_missing: true + - drop: + if: "ctx.azure.springcloudlogs.category != 'SystemLogs' && ctx.azure.springcloudlogs.category != 'ApplicationConsole'" + ignore_failure: true + - rename: + field: azure.springcloudlogs.LogFormat + target_field: azure.springcloudlogs.log_format + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.InstanceName + target_field: azure.springcloudlogs.properties.instance_name + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Log + target_field: azure.springcloudlogs.properties.log + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.ServiceName + target_field: azure.springcloudlogs.properties.service_name + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Stream + target_field: azure.springcloudlogs.properties.stream + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.AppName + target_field: azure.springcloudlogs.properties.app_name + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.ServiceId + target_field: azure.springcloudlogs.properties.service_id + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Type + target_field: azure.springcloudlogs.properties.type + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Level + target_field: azure.springcloudlogs.level + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Logger + target_field: azure.springcloudlogs.properties.logger + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Stack + target_field: azure.springcloudlogs.properties.stack + ignore_missing: true + - rename: + field: azure.springcloudlogs.properties.Thread + target_field: azure.springcloudlogs.properties.thread + ignore_missing: true + - rename: + field: azure.springcloudlogs.level + target_field: log.level + ignore_missing: true + - rename: + field: azure.springcloudlogs.operationName + target_field: azure.springcloudlogs.operation_name + ignore_missing: true + - convert: + field: azure.springcloudlogs.operation_name + target_field: event.action + type: string + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index fb677a2a061..512ac64af89 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: 0.8.0 +version: 0.8.1 release: beta description: This Elastic integration collects logs from Azure type: integration From 9e3c5dbc2d31e4ad6ead597eb24b95a54df985e7 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 10:25:52 +0200 Subject: [PATCH 2/9] pipeline --- packages/azure/changelog.yml | 2 +- .../ingest_pipeline/azure-shared-pipeline.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 9 ++++----- ...gcloudlogs.yml => springcloudlogs-inner-pipeline.yml} | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) rename packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/{springcloudlogs.yml => springcloudlogs-inner-pipeline.yml} (97%) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index b7b95d70b2d..d212913698f 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add support for springcloud logs inside the platformlogs pipeline type: enhancement - link: https://github.com/elastic/integrations/pull/1455 + link: https://github.com/elastic/integrations/pull/1462 - version: "0.8.0" changes: - description: Import ECS field definitions diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml index 3c33af5fd7a..f419844fcad 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for parsing azure activity logs. +description: Pipeline for parsing azure logs. processors: - set: field: cloud.provider diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 034cb623ea8..acc767d6f58 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -246,10 +246,9 @@ processors: ignore_failure: true ignore_missing: true - pipeline: - if: "ctx?.azure_log_forwarder?.resource_type != null && ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'" - name: '{{ IngestPipeline "springcloudlogs" }}' - } + if: "ctx?.azure_log_forwarder?.resource_type != null && (ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring' || ctx?.azure_log_forwarder?.resource_type == 'MICROSOFT.APPPLATFORM/SPRING')" + name: '{{ IngestPipeline "springcloudlogs-inner-pipeline.yml" }}' on_failure: - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml similarity index 97% rename from packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml rename to packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml index f806cfc1ea0..a84fb1de17b 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for parsing azure spring cloud logs. +description: Pipeline for parsing azure springcloud logs. processors: - script: source: ctx['_index'] = ctx['_index'].replace('platformlogs', 'springcloudlogs') From a4b2571750eef7c929ad56c97cbf5fcc7e520a63 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 10:38:09 +0200 Subject: [PATCH 3/9] formatting --- .../ingest_pipeline/springcloudlogs-inner-pipeline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml index a84fb1de17b..8384080cd77 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml @@ -1,8 +1,8 @@ --- -description: Pipeline for parsing azure springcloud logs. +description: Pipeline for parsing azure spring cloud logs. processors: - script: - source: ctx['_index'] = ctx['_index'].replace('platformlogs', 'springcloudlogs') + source: "ctx['_index'] = ctx['_index'].replace('platformlogs', 'springcloudlogs')" ignore_failure: true - rename: field: azure.platformlogs @@ -63,7 +63,7 @@ processors: field: azure.springcloudlogs.level target_field: log.level ignore_missing: true - - rename: + - rename: field: azure.springcloudlogs.operationName target_field: azure.springcloudlogs.operation_name ignore_missing: true @@ -74,5 +74,5 @@ processors: ignore_missing: true on_failure: - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + field: error.message + value: '{{ _ingest.on_failure_message }}' From ef0737f6283a4a4fa98b4bfad2fb704ad5fdbb86 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 12:26:28 +0200 Subject: [PATCH 4/9] fix --- .../platformlogs/elasticsearch/ingest_pipeline/default.yml | 6 +++--- .../ingest_pipeline/springcloudlogs-inner-pipeline.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index acc767d6f58..5ee3ed061f8 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -246,9 +246,9 @@ processors: ignore_failure: true ignore_missing: true - pipeline: - if: "ctx?.azure_log_forwarder?.resource_type != null && (ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring' || ctx?.azure_log_forwarder?.resource_type == 'MICROSOFT.APPPLATFORM/SPRING')" + if: "ctx?.azure_log_forwarder?.resource_type != null && ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'" name: '{{ IngestPipeline "springcloudlogs-inner-pipeline.yml" }}' on_failure: - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml index 8384080cd77..c4a7b6426c4 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml @@ -74,5 +74,5 @@ processors: ignore_missing: true on_failure: - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + field: error.message + value: '{{ _ingest.on_failure_message }}' From a3f5ae2597bb4e2d4d9ca1fa5c1bbb7c75777fc8 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 14:02:35 +0200 Subject: [PATCH 5/9] update pipeline --- .../elasticsearch/ingest_pipeline/default.yml | 4 ++-- .../ingest_pipeline/springcloudlogs-inner-pipeline.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 5ee3ed061f8..a371e4254df 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -246,8 +246,8 @@ processors: ignore_failure: true ignore_missing: true - pipeline: - if: "ctx?.azure_log_forwarder?.resource_type != null && ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'" - name: '{{ IngestPipeline "springcloudlogs-inner-pipeline.yml" }}' + if: "ctx?.azure_log_forwarder?.resource_type != null && (ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'|| ctx?.azure_log_forwarder?.resource_type == 'MICROSOFT.APPPLATFORM/SPRING')" + name: '{{ IngestPipeline "springcloudlogs-inner-pipeline" }}' on_failure: - set: field: error.message diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml index c4a7b6426c4..6b4fbdf7889 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/springcloudlogs-inner-pipeline.yml @@ -8,6 +8,14 @@ processors: field: azure.platformlogs target_field: azure.springcloudlogs ignore_missing: true + - set: + field: event.dataset + value: azure.springcloudlogs + ignore_failure: true + - set: + field: data_stream.dataset + value: azure.springcloudlogs + ignore_failure: true - drop: if: "ctx.azure.springcloudlogs.category != 'SystemLogs' && ctx.azure.springcloudlogs.category != 'ApplicationConsole'" ignore_failure: true From 5f08abb775f38fa40b90822f9005159134fb2557 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 16:05:49 +0200 Subject: [PATCH 6/9] add logs --- .../test-platformlogs-springcloudlog-raw.log | 1 + ...mlogs-springcloudlog-raw.log-expected.json | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log create mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log new file mode 100644 index 00000000000..d2d9d3c5bad --- /dev/null +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log @@ -0,0 +1 @@ +{"@timestamp":"2020-10-28T11:59:26.373Z","agent":{"name":"MicrosoftAgent","version":"1.0.0"},"data_stream":{"namespace":"default","type":"logs","dataset":"azure.platformlogs"},"input":{"type":"azure-log-forwarder"}, "azure_log_forwarder" : {"service_provider" : "Microsoft.ContainerService","category" : "kube-apiserver","resource_type": "Microsoft.AppPlatform/Spring"},"message":"""{ "time": "2021-07-01T19:30:30.535404056Z", "LogFormat": "RAW", "resourceId": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", "operationName": "Microsoft.AppPlatform/Spring/logs", "category": "ApplicationConsole", "level": "Informational", "location": "westus2", "properties": {"Log":"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\n","Stream":"stdout","AppName":"admin-server","InstanceName":"admin-server-default-12-8459d44f68-g4b5f","ServiceId":"c41fd000b1a5450eb234039376da26de","ServiceName":"hm-sc-petclinic"}}""","tags":["forwarded"],"service":{"type":"azure"},"event":{"module":"azure","dataset":"azure.platformlogs"}} \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json new file mode 100644 index 00000000000..e425e53f228 --- /dev/null +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json @@ -0,0 +1,49 @@ +{ + "expected": [ + { + "cloud": { + "provider": "azure" + }, + "@timestamp": "2020-11-09T10:57:31.000Z", + "ecs": { + "version": "1.11.0" + }, + "event": { + "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "ingested": "2021-06-14T09:02:47.965542200Z", + "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", + "kind": "event" + }, + "tags": [ + "preserve_original_event" + ], + "azure": { + "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "resource": { + "name": "OBSKUBE", + "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "group": "OBS-INFRASTRUCTURE" + }, + "platformlogs": { + "ccpNamespace": "5e4bf4baee195b00017cdbfa", + "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "Cloud": "AzureCloud", + "Environment": "prod", + "category": "kube-audit", + "event_category": "Administrative", + "properties": { + "pod": "kube-apiserver-666bd4b459-hjgdc", + "log": { + "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", + "apiVersion": "audit.k8s.io/v1", + "level": "Metadata", + "kind": "Event" + }, + "stream": "stdout" + } + } + } + } + ] +} \ No newline at end of file From 34c33f2e9ca497d7810fbb89b908facdd388e911 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 16:26:18 +0200 Subject: [PATCH 7/9] test --- .../test-platformlogs-springcloudlog-raw.log | 2 +- ...mlogs-springcloudlog-raw.log-expected.json | 103 ++++++++++-------- 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log index d2d9d3c5bad..922bce2d92d 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log @@ -1 +1 @@ -{"@timestamp":"2020-10-28T11:59:26.373Z","agent":{"name":"MicrosoftAgent","version":"1.0.0"},"data_stream":{"namespace":"default","type":"logs","dataset":"azure.platformlogs"},"input":{"type":"azure-log-forwarder"}, "azure_log_forwarder" : {"service_provider" : "Microsoft.ContainerService","category" : "kube-apiserver","resource_type": "Microsoft.AppPlatform/Spring"},"message":"""{ "time": "2021-07-01T19:30:30.535404056Z", "LogFormat": "RAW", "resourceId": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", "operationName": "Microsoft.AppPlatform/Spring/logs", "category": "ApplicationConsole", "level": "Informational", "location": "westus2", "properties": {"Log":"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\n","Stream":"stdout","AppName":"admin-server","InstanceName":"admin-server-default-12-8459d44f68-g4b5f","ServiceId":"c41fd000b1a5450eb234039376da26de","ServiceName":"hm-sc-petclinic"}}""","tags":["forwarded"],"service":{"type":"azure"},"event":{"module":"azure","dataset":"azure.platformlogs"}} \ No newline at end of file +{"@timestamp":"2020-10-28T11:59:26.373Z","agent":{"name":"MicrosoftAgent","version":"1.0.0"},"data_stream":{"namespace":"default","type":"logs","dataset":"azure.platformlogs"},"input":{"type":"azure-log-forwarder"}, "azure_log_forwarder" : {"service_provider" : "Microsoft.ContainerService","category" : "kube-apiserver","resource_type": "Microsoft.AppPlatform/Spring"},"message":"{ \"time\": \"2021-07-01T19:30:30.535404056Z\", \"LogFormat\": \"RAW\", \"resourceId\": \"\/SUBSCRIPTIONS\/EDD63B67-0BA2-4837-A4EB-CD484E9FF623\/RESOURCEGROUPS\/SA-HEMANT\/PROVIDERS\/MICROSOFT.APPPLATFORM\/SPRING\/HM-SC-PETCLINIC\", \"operationName\": \"Microsoft.AppPlatform\/Spring\/logs\", \"category\": \"ApplicationConsole\", \"level\": \"Informational\", \"location\": \"westus2\", \"properties\": {\"Log\":\"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\\n\",\"Stream\":\"stdout\",\"AppName\":\"admin-server\",\"InstanceName\":\"admin-server-default-12-8459d44f68-g4b5f\",\"ServiceId\":\"c41fd000b1a5450eb234039376da26de\",\"ServiceName\":\"hm-sc-petclinic\"}}","tags":["forwarded"],"service":{"type":"azure"},"event":{"module":"azure","dataset":"azure.platformlogs"}} \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json index e425e53f228..9284e91d7be 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json @@ -1,49 +1,58 @@ { - "expected": [ - { - "cloud": { - "provider": "azure" - }, - "@timestamp": "2020-11-09T10:57:31.000Z", - "ecs": { - "version": "1.11.0" - }, - "event": { - "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested": "2021-06-14T09:02:47.965542200Z", - "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", - "kind": "event" - }, - "tags": [ - "preserve_original_event" - ], - "azure": { - "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", - "resource": { - "name": "OBSKUBE", - "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", - "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", - "group": "OBS-INFRASTRUCTURE" - }, - "platformlogs": { - "ccpNamespace": "5e4bf4baee195b00017cdbfa", - "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "Cloud": "AzureCloud", - "Environment": "prod", - "category": "kube-audit", - "event_category": "Administrative", - "properties": { - "pod": "kube-apiserver-666bd4b459-hjgdc", - "log": { - "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", - "apiVersion": "audit.k8s.io/v1", - "level": "Metadata", - "kind": "Event" - }, - "stream": "stdout" - } - } - } - } - ] +"expected": [ +{ ++ "@timestamp": "2021-07-01T19:30:30.535Z", +"azure": { +"platformlogs": { +- "Cloud": "AzureCloud", +- "Environment": "prod", +- "category": "kube-audit", +- "ccpNamespace": "5e4bf4baee195b00017cdbfa", +- "event_category": "Administrative", +- "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", +- "properties": { +- "log": { +- "apiVersion": "audit.k8s.io/v1", +- "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", +- "kind": "Event", +- "level": "Metadata" +- }, +- "pod": "kube-apiserver-666bd4b459-hjgdc", +- "stream": "stdout" +- } ++ "category": "ApplicationConsole", ++ "event_category": "Administrative" +}, +"resource": { +- "group": "OBS-INFRASTRUCTURE", +- "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", +- "name": "OBSKUBE", +- "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS" ++ "group": "SA-HEMANT", ++ "id": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", ++ "name": "HM-SC-PETCLINIC", ++ "provider": "MICROSOFT.APPPLATFORM/SPRING" +}, + ++ "subscription_id": "EDD63B67-0BA2-4837-A4EB-CD484E9FF623" +}, +"cloud": { +"provider": "azure" +}, +"ecs": { +"version": "1.11.0" +}, ++ "error": { ++ "message": "invalid json log" ++ }, +"event": { + +"kind": "event", ++ "original": "{\"@timestamp\":\"2020-10-28T11:59:26.373Z\",\"agent\":{\"name\":\"MicrosoftAgent\",\"version\":\"1.0.0\"},\"data_stream\":{\"namespace\":\"default\",\"type\":\"logs\",\"dataset\":\"azure.platformlogs\"},\"input\":{\"type\":\"azure-log-forwarder\"}, \"azure_log_forwarder\" : {\"service_provider\" : \"Microsoft.ContainerService\",\"category\" : \"kube-apiserver\",\"resource_type\": \"Microsoft.AppPlatform/Spring\"},\"message\":\"\"\"{ \"time\": \"2021-07-01T19:30:30.535404056Z\", \"LogFormat\": \"RAW\", \"resourceId\": \"/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC\", \"operationName\": \"Microsoft.AppPlatform/Spring/logs\", \"category\": \"ApplicationConsole\", \"level\": \"Informational\", \"location\": \"westus2\", \"properties\": {\"Log\":\"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\\n\",\"Stream\":\"stdout\",\"AppName\":\"admin-server\",\"InstanceName\":\"admin-server-default-12-8459d44f68-g4b5f\",\"ServiceId\":\"c41fd000b1a5450eb234039376da26de\",\"ServiceName\":\"hm-sc-petclinic\"}}\"\"\",\"tags\":[\"forwarded\"],\"service\":{\"type\":\"azure\"},\"event\":{\"module\":\"azure\",\"dataset\":\"azure.platformlogs\"}}" +}, +"tags": [ +"preserve_original_event" +] +} +] } \ No newline at end of file From 1e708f8b700cf750d9a2fa905a88f23653cb51b2 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 16:33:22 +0200 Subject: [PATCH 8/9] test --- ...mlogs-springcloudlog-raw.log-expected.json | 103 ++++++++---------- 1 file changed, 47 insertions(+), 56 deletions(-) diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json index 9284e91d7be..e425e53f228 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json @@ -1,58 +1,49 @@ { -"expected": [ -{ -+ "@timestamp": "2021-07-01T19:30:30.535Z", -"azure": { -"platformlogs": { -- "Cloud": "AzureCloud", -- "Environment": "prod", -- "category": "kube-audit", -- "ccpNamespace": "5e4bf4baee195b00017cdbfa", -- "event_category": "Administrative", -- "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", -- "properties": { -- "log": { -- "apiVersion": "audit.k8s.io/v1", -- "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", -- "kind": "Event", -- "level": "Metadata" -- }, -- "pod": "kube-apiserver-666bd4b459-hjgdc", -- "stream": "stdout" -- } -+ "category": "ApplicationConsole", -+ "event_category": "Administrative" -}, -"resource": { -- "group": "OBS-INFRASTRUCTURE", -- "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", -- "name": "OBSKUBE", -- "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS" -+ "group": "SA-HEMANT", -+ "id": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", -+ "name": "HM-SC-PETCLINIC", -+ "provider": "MICROSOFT.APPPLATFORM/SPRING" -}, - -+ "subscription_id": "EDD63B67-0BA2-4837-A4EB-CD484E9FF623" -}, -"cloud": { -"provider": "azure" -}, -"ecs": { -"version": "1.11.0" -}, -+ "error": { -+ "message": "invalid json log" -+ }, -"event": { - -"kind": "event", -+ "original": "{\"@timestamp\":\"2020-10-28T11:59:26.373Z\",\"agent\":{\"name\":\"MicrosoftAgent\",\"version\":\"1.0.0\"},\"data_stream\":{\"namespace\":\"default\",\"type\":\"logs\",\"dataset\":\"azure.platformlogs\"},\"input\":{\"type\":\"azure-log-forwarder\"}, \"azure_log_forwarder\" : {\"service_provider\" : \"Microsoft.ContainerService\",\"category\" : \"kube-apiserver\",\"resource_type\": \"Microsoft.AppPlatform/Spring\"},\"message\":\"\"\"{ \"time\": \"2021-07-01T19:30:30.535404056Z\", \"LogFormat\": \"RAW\", \"resourceId\": \"/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC\", \"operationName\": \"Microsoft.AppPlatform/Spring/logs\", \"category\": \"ApplicationConsole\", \"level\": \"Informational\", \"location\": \"westus2\", \"properties\": {\"Log\":\"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\\n\",\"Stream\":\"stdout\",\"AppName\":\"admin-server\",\"InstanceName\":\"admin-server-default-12-8459d44f68-g4b5f\",\"ServiceId\":\"c41fd000b1a5450eb234039376da26de\",\"ServiceName\":\"hm-sc-petclinic\"}}\"\"\",\"tags\":[\"forwarded\"],\"service\":{\"type\":\"azure\"},\"event\":{\"module\":\"azure\",\"dataset\":\"azure.platformlogs\"}}" -}, -"tags": [ -"preserve_original_event" -] -} -] + "expected": [ + { + "cloud": { + "provider": "azure" + }, + "@timestamp": "2020-11-09T10:57:31.000Z", + "ecs": { + "version": "1.11.0" + }, + "event": { + "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "ingested": "2021-06-14T09:02:47.965542200Z", + "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", + "kind": "event" + }, + "tags": [ + "preserve_original_event" + ], + "azure": { + "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "resource": { + "name": "OBSKUBE", + "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "group": "OBS-INFRASTRUCTURE" + }, + "platformlogs": { + "ccpNamespace": "5e4bf4baee195b00017cdbfa", + "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "Cloud": "AzureCloud", + "Environment": "prod", + "category": "kube-audit", + "event_category": "Administrative", + "properties": { + "pod": "kube-apiserver-666bd4b459-hjgdc", + "log": { + "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", + "apiVersion": "audit.k8s.io/v1", + "level": "Metadata", + "kind": "Event" + }, + "stream": "stdout" + } + } + } + } + ] } \ No newline at end of file From 0bf63370ba7bbdac3c1a5a30ea5572dab01dc5fa Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 10 Aug 2021 16:49:41 +0200 Subject: [PATCH 9/9] remove tests --- .../test-platformlogs-springcloudlog-raw.log | 1 - ...mlogs-springcloudlog-raw.log-expected.json | 49 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log delete mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log deleted file mode 100644 index 922bce2d92d..00000000000 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log +++ /dev/null @@ -1 +0,0 @@ -{"@timestamp":"2020-10-28T11:59:26.373Z","agent":{"name":"MicrosoftAgent","version":"1.0.0"},"data_stream":{"namespace":"default","type":"logs","dataset":"azure.platformlogs"},"input":{"type":"azure-log-forwarder"}, "azure_log_forwarder" : {"service_provider" : "Microsoft.ContainerService","category" : "kube-apiserver","resource_type": "Microsoft.AppPlatform/Spring"},"message":"{ \"time\": \"2021-07-01T19:30:30.535404056Z\", \"LogFormat\": \"RAW\", \"resourceId\": \"\/SUBSCRIPTIONS\/EDD63B67-0BA2-4837-A4EB-CD484E9FF623\/RESOURCEGROUPS\/SA-HEMANT\/PROVIDERS\/MICROSOFT.APPPLATFORM\/SPRING\/HM-SC-PETCLINIC\", \"operationName\": \"Microsoft.AppPlatform\/Spring\/logs\", \"category\": \"ApplicationConsole\", \"level\": \"Informational\", \"location\": \"westus2\", \"properties\": {\"Log\":\"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\\n\",\"Stream\":\"stdout\",\"AppName\":\"admin-server\",\"InstanceName\":\"admin-server-default-12-8459d44f68-g4b5f\",\"ServiceId\":\"c41fd000b1a5450eb234039376da26de\",\"ServiceName\":\"hm-sc-petclinic\"}}","tags":["forwarded"],"service":{"type":"azure"},"event":{"module":"azure","dataset":"azure.platformlogs"}} \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json deleted file mode 100644 index e425e53f228..00000000000 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-springcloudlog-raw.log-expected.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "expected": [ - { - "cloud": { - "provider": "azure" - }, - "@timestamp": "2020-11-09T10:57:31.000Z", - "ecs": { - "version": "1.11.0" - }, - "event": { - "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested": "2021-06-14T09:02:47.965542200Z", - "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", - "kind": "event" - }, - "tags": [ - "preserve_original_event" - ], - "azure": { - "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", - "resource": { - "name": "OBSKUBE", - "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", - "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", - "group": "OBS-INFRASTRUCTURE" - }, - "platformlogs": { - "ccpNamespace": "5e4bf4baee195b00017cdbfa", - "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "Cloud": "AzureCloud", - "Environment": "prod", - "category": "kube-audit", - "event_category": "Administrative", - "properties": { - "pod": "kube-apiserver-666bd4b459-hjgdc", - "log": { - "auditID": "22af12c3-a1fe-4f2c-99a9-3cdde671dbfe", - "apiVersion": "audit.k8s.io/v1", - "level": "Metadata", - "kind": "Event" - }, - "stream": "stdout" - } - } - } - } - ] -} \ No newline at end of file