Elastic Stack에서 nginx 로그 분석할 때 잘 알려진 UserAgent 한방에 제외하는 유용한 Query DSL
뭔가 수상한 놈이 서비스를 찝적대고 있는 건 분명한데, 이 방대한 로그의 바다에서 그 녀석을 어떻게 찾지? 노이즈가 너무 많다, 시그날만 보고 싶다!
아래 Query DSL을 이용하여 Kibana Filter를 생성해 보십시오.
Exclude Well-known Web Browsers
{
"query": {
"bool": {
"should": [
{
"match_phrase": {
"user_agent.name.keyword": "Android"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Edge"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Mobile Safari"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile WebView"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Safari"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Opera"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "AdsBot-Google"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Opera Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile iOS"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "IE"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Edge Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox iOS"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Samsung Internet"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "MiuiBrowser"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Whale"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UC Browser"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "CFNetwork"
}
}
],
"minimum_should_match": 1
}
}
}
Exclude well-known apps
{
"query": {
"bool": {
"should": [
{
"match_phrase": {
"user_agent.name.keyword": "Instagram"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Facebook"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "WhatsApp"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "mamikos"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Google"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "LINE"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "BingPreview"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "WordPress"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Apple Mail"
}
}
],
"minimum_should_match": 1
}
}
}
Exclude well-known bots
{
"query": {
"bool": {
"should": [
{
"match_phrase": {
"user_agent.name.keyword": "Googlebot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "bingbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "GooglePlusBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Pinterestbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Applebot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Twitterbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "TelegramBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UptimeRobot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UptimeBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "FacebookBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slackbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slackbot-LinkExpanding"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slack-ImgProxy"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Discordbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Googlebot-Image"
}
}
],
"minimum_should_match": 1
}
}
}
All together
{
"query": {
"bool": {
"should": [
{
"match_phrase": {
"user_agent.name.keyword": "Android"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Edge"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Mobile Safari"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile WebView"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Safari"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Opera"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "AdsBot-Google"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Opera Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Chrome Mobile iOS"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "IE"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Edge Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox Mobile"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Firefox iOS"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Samsung Internet"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "MiuiBrowser"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Whale"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UC Browser"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "CFNetwork"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Instagram"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Facebook"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "WhatsApp"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "mamikos"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Google"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "LINE"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "BingPreview"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "WordPress"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Apple Mail"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Googlebot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "bingbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "GooglePlusBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Pinterestbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Applebot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Twitterbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "TelegramBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UptimeRobot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "UptimeBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "FacebookBot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slackbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slackbot-LinkExpanding"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Slack-ImgProxy"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Discordbot"
}
},
{
"match_phrase": {
"user_agent.name.keyword": "Googlebot-Image"
}
}
],
"minimum_should_match": 1
}
}
}
일찍 퇴근하세요!