From af1a157d1bdbfe89d9e6de7723b0986aee81998e Mon Sep 17 00:00:00 2001 From: 0ccupi3R <47894266+0ccupi3R@users.noreply.github.com> Date: Tue, 17 Jan 2023 03:13:09 +0530 Subject: [PATCH 1/5] Drop header log line in CloudFront events CloudFront stores these two lines as a header in each log file. Before this change it fails the pipeline and add message to error.message field. #Version: 1.0 #Fields: date time x-edge-location (TRUNCATED) --- .../cloudfront_logs/elasticsearch/ingest_pipeline/default.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml index 54a5d91dc78..0ac7147b2ce 100644 --- a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml @@ -28,6 +28,9 @@ processors: ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' + - drop: + if: "ctx.event.original.startsWith('#')" + description: "Drop if logline contains header(s), which startswith `#`" - grok: field: event.original patterns: From e356cfc3e79cb53e1fe211fdacbb36b4b40a3767 Mon Sep 17 00:00:00 2001 From: 0ccupi3R <47894266+0ccupi3R@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:52:35 +0530 Subject: [PATCH 2/5] Adding header log lines in sample file Added header events which is available in each Cloud Front log file. `DROP` filter has added pipeline. --- .../cloudfront_logs/_dev/test/pipeline/test-cloudfront.log | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log index 155fc583193..f9f0d155017 100644 --- a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log +++ b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log @@ -1,4 +1,5 @@ -2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - +#Version: 1.0 +#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - 2019-12-04 21:02:31 LAX1 392 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit k6WGMNkEzR5BEM_SaF47gjtX9zBDO2m349OY2an0QPEaUum1ZOLrow== d111111abcdef8.cloudfront.net https 23 0.000 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.000 Hit text/html 78 - - 2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit f37nTMVvnKvV2ZSvEsivup_c2kZ7VXzYdjC-GUQZ5qNs-89BlWazbw== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - 2019-12-13 22:36:27 SEA19-C1 900 89.160.20.112 GET d111111abcdef8.cloudfront.net /favicon.ico 502 http://www.example.com/ Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Error 1pkpNfBQ39sYMnjjUQjmH2w1wdJnbHYTbag21o_3OfcQgPzdL2RSSQ== www.example.com http 675 0.102 - - - Error HTTP/1.1 - - 25260 0.102 OriginDnsError text/html 507 - - From 48869501c958e7d4d54dc65131a0094ec4cd254b Mon Sep 17 00:00:00 2001 From: 0ccupi3R <47894266+0ccupi3R@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:54:39 +0530 Subject: [PATCH 3/5] Fixed Line break --- .../cloudfront_logs/_dev/test/pipeline/test-cloudfront.log | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log index f9f0d155017..8b8546b752e 100644 --- a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log +++ b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log @@ -1,5 +1,6 @@ #Version: 1.0 -#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - +#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end +2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - 2019-12-04 21:02:31 LAX1 392 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit k6WGMNkEzR5BEM_SaF47gjtX9zBDO2m349OY2an0QPEaUum1ZOLrow== d111111abcdef8.cloudfront.net https 23 0.000 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.000 Hit text/html 78 - - 2019-12-04 21:02:31 LAX1 392 89.160.20.112 GET d111111abcdef8.cloudfront.net /index.html 200 - Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Hit f37nTMVvnKvV2ZSvEsivup_c2kZ7VXzYdjC-GUQZ5qNs-89BlWazbw== d111111abcdef8.cloudfront.net https 23 0.001 - TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Hit HTTP/2.0 - - 11040 0.001 Hit text/html 78 - - 2019-12-13 22:36:27 SEA19-C1 900 89.160.20.112 GET d111111abcdef8.cloudfront.net /favicon.ico 502 http://www.example.com/ Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36 - - Error 1pkpNfBQ39sYMnjjUQjmH2w1wdJnbHYTbag21o_3OfcQgPzdL2RSSQ== www.example.com http 675 0.102 - - - Error HTTP/1.1 - - 25260 0.102 OriginDnsError text/html 507 - - From 8dd154a537e20faee88877eded3ae514939a1ec5 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 20 Jan 2023 16:03:12 +0100 Subject: [PATCH 4/5] Add changelog entry and bump package version --- packages/aws/changelog.yml | 5 +++++ packages/aws/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 1d1e991e988..731bfb96c9a 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.29.1" + changes: + - description: Drop comments from CloudFront loglines + type: bugfix + link: https://github.com/elastic/integrations/pull/5017 - version: "1.29.0" changes: - description: Add data_granularity parameter and rename period title to Collection Period. diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index e680972a85f..426553784f8 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.29.0 +version: 1.29.1 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration From 679e459cb328b53f1622e60b0b6aef87aa51e92a Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 24 Jan 2023 16:24:51 +0100 Subject: [PATCH 5/5] Add expected drop of loglines with comments --- .../_dev/test/pipeline/test-cloudfront.log-expected.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json index 4913035dcb9..27a43c0a8a8 100644 --- a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json +++ b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json @@ -1,5 +1,7 @@ { "expected": [ + null, + null, { "@timestamp": "2019-12-04T21:02:31.000Z", "aws": {