Properly search tokens and annotate in matched field

core/db/elastic.py

@@ -338,8 +338,15 @@ class ElasticsearchBackend(StorageBackend):
                         {"match_phrase": {"src": source_iter}}
                     )
                 add_top.append(add_top_tmp)
+        if "tokens" in data:
+            add_top_tmp = {"bool": {"should": []}}
+            for token in data["tokens"]:
+                add_top_tmp["bool"]["should"].append(
+                    {"match_phrase": {"tokens": token}}
+                )
+            add_top.append(add_top_tmp)
         for field, values in data.items():
-            if field not in ["source", "index", "tags", "query", "sentiment"]:
+            if field not in ["source", "index", "tags", "query", "sentiment", "tokens"]:
                 for value in values:
                     add_top.append({"match": {field: value}})
         # Bypass the check for query and tags membership since we can search by msg, etc

core/lib/rules.py

@@ -234,6 +234,16 @@ class NotificationRuleData(object):
                                 break
                     # Continue to next field
                     continue
+                if field == "tokens":
+                    # Allow partial matches for tokens
+                    for token in value:
+                        if "tokens" in message:
+                            if token.lower() in [x.lower() for x in message["tokens"]]:
+                                matched[field] = token
+                                # Break out of the token matching loop
+                                break
+                    # Continue to next field
+                    continue
                 if field in message and message[field] in value:
                     # Do exact matches for all other fields
                     matched[field] = message[field]