Elastic Stack에서 nginx 로그 분석할 때 잘 알려진 UserAgent 한방에 제외하는 유용한 Query DSL

뭔가 수상한 놈이 서비스를 찝적대고 있는 건 분명한데, 이 방대한 로그의 바다에서 그 녀석을 어떻게 찾지? 노이즈가 너무 많다, 시그날만 보고 싶다!

아래 Query DSL을 이용하여 Kibana Filter를 생성해 보십시오.

Exclude Well-known Web Browsers

{
  "query": {
    "bool": {
      "should": [
        {
          "match_phrase": {
            "user_agent.name.keyword": "Android"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Edge"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Mobile Safari"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile WebView"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Safari"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Opera"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "AdsBot-Google"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Opera Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile iOS"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "IE"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Edge Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox iOS"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Samsung Internet"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "MiuiBrowser"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Whale"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UC Browser"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "CFNetwork"
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}

Exclude well-known apps

{
  "query": {
    "bool": {
      "should": [
        {
          "match_phrase": {
            "user_agent.name.keyword": "Instagram"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Facebook"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "WhatsApp"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "mamikos"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Google"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "LINE"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "BingPreview"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "WordPress"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Apple Mail"
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}

Exclude well-known bots

{
  "query": {
    "bool": {
      "should": [
        {
          "match_phrase": {
            "user_agent.name.keyword": "Googlebot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "bingbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "GooglePlusBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Pinterestbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Applebot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Twitterbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "TelegramBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UptimeRobot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UptimeBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "FacebookBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slackbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slackbot-LinkExpanding"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slack-ImgProxy"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Discordbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Googlebot-Image"
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}

All together

{
  "query": {
    "bool": {
      "should": [
        {
          "match_phrase": {
            "user_agent.name.keyword": "Android"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Edge"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Mobile Safari"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile WebView"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Safari"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Opera"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "AdsBot-Google"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Opera Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Chrome Mobile iOS"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "IE"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Edge Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox Mobile"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Firefox iOS"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Samsung Internet"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "MiuiBrowser"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Whale"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UC Browser"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "CFNetwork"
          }
        },
                {
          "match_phrase": {
            "user_agent.name.keyword": "Instagram"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Facebook"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "WhatsApp"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "mamikos"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Google"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "LINE"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "BingPreview"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "WordPress"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Apple Mail"
          }
        },
                {
          "match_phrase": {
            "user_agent.name.keyword": "Googlebot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "bingbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "GooglePlusBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Pinterestbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Applebot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Twitterbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "TelegramBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UptimeRobot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "UptimeBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "FacebookBot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slackbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slackbot-LinkExpanding"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Slack-ImgProxy"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Discordbot"
          }
        },
        {
          "match_phrase": {
            "user_agent.name.keyword": "Googlebot-Image"
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}

일찍 퇴근하세요!