Implement searching different indexes

core/lib/opensearch.py

@@ -1,6 +1,6 @@
 from django.conf import settings
 from opensearchpy import OpenSearch
-from opensearchpy.exceptions import RequestError
+from opensearchpy.exceptions import NotFoundError, RequestError
 
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
 
@@ -44,14 +44,24 @@ def annotate_results(results_parsed):
 
     for net in nets:
         # Annotate the online attribute from Threshold
-        nicks = [
+        nicks = list(
-            x["nick"] for x in results_parsed if x["src"] == "irc" and x["net"] == net
+            set(
+                [
+                    x["nick"]
+                    for x in results_parsed
+                    if x["src"] == "irc" and x["net"] == net and "nick" in x
                 ]
-        channels = [
+            )
+        )
+        channels = list(
+            set(
+                [
                     x["channel"]
                     for x in results_parsed
-            if x["src"] == "irc" and x["net"] == net
+                    if x["src"] == "irc" and x["net"] == net and "channel" in x
                 ]
+            )
+        )
         online_info = annotate_online(net, nicks)
         # Annotate the number of users in the channel
         num_users = annotate_num_users(net, channels)
@@ -137,6 +147,9 @@ def run_main_query(client, user, query, custom_query=False, index=None, size=Non
     except RequestError as err:
         print("OpenSearch error", err)
         return err
+    except NotFoundError as err:
+        print("OpenSearch error", err)
+        return err
     filter_blacklisted(user, response)
     return response
 
@@ -254,16 +267,36 @@ def query_results(request, query_params, size=None):
                     search_query["query"]["bool"]["must_not"] = [item]
         if sort:
             search_query["sort"] = sort
+        if "index" in query_params:
+            if not request.user.is_superuser:
+                message = "How did you get here?"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            else:
+                index = query_params["index"]
+                if index == "main":
+                    index = settings.OPENSEARCH_INDEX_MAIN
+                elif index == "meta":
+                    index = settings.OPENSEARCH_INDEX_META
+                elif index == "int":
+                    index = settings.OPENSEARCH_INDEX_INT
+                else:
+                    message = "Index is not valid."
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+        else:
+            index = settings.OPENSEARCH_INDEX_MAIN
         results = run_main_query(
             client,
             request.user,  # passed through run_main_query to filter_blacklisted
             search_query,
             custom_query=True,
+            index=index,
             size=size,
         )
         if not results:
             return False
-        if isinstance(results, RequestError):
+        if isinstance(results, Exception):
             message = results.info["error"]["root_cause"][0]["reason"]
             message_class = "danger"
             return {"message": message, "class": message_class}

core/static/js/column-shifter.js

@@ -48,6 +48,9 @@ $(document).ready(function(){
                             "num": "off",
                             "exemption": "off",
                             "online": "off",
+                            "mtype": "off",
+                            "realname": "off",
+                            "server": "off",
                         },
                     };
         } else {

core/templates/manage/threshold/irc/overview/alerts.html

@@ -1,94 +0,0 @@
-<div id="alerts">
-  {% include 'manage/threshold/partials/notify.html' %}
-  {% if alerts is not None %}
-    <div class="icons">
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Conn">
-          <i class="fa-solid fa-cloud-exclamation"></i>
-        </span>
-      </button>
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Highlight">
-          <i class="fa-solid fa-square-quote"></i>
-        </span>
-      </button>
-
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="ZNC">
-          <i class="fa-brands fa-unity"></i>
-        </span>
-      </button>
-
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Query">
-          <i class="fa-solid fa-inbox"></i>
-        </span>
-      </button>
-
-      <button class="button is-small">
-        <span class="icon" data-tooltip="Self">
-          <i class="fa-solid fa-message-bot"></i>
-        </span>
-      </button>
-
-    </div>
-    <div class="content" style="max-height: 30em; overflow: auto;">
-
-      <div class="table-container">
-        <table class="table is-fullwidth is-hoverable">
-          <thead>
-            <th>ts</th>
-            <th>name</th>
-            <th>type</th>
-            <th>msg</th>
-          </thead>
-          <tbody>
-            {% for alert in alerts %}
-              <tr>
-                <td>
-                  <p>{{ alert.date }}</p>
-                  <p>{{ alert.time }}</p>
-                </td>
-                <td>
-                  {{ alert.net }}/{{ alert.num }}
-                </td>
-                <td>
-                  {% if alert.type == 'conn' %}
-                    <span class="icon" data-tooltip="Conn">
-                      <i class="fa-solid fa-cloud-exclamation"></i>
-                    </span>
-                  {% elif alert.type == 'highlight' %}
-                    <span class="icon" data-tooltip="Highlight">
-                      <i class="fa-solid fa-square-quote"></i>
-                    </span>
-                  {% elif alert.type == 'znc' %}
-                    <span class="icon" data-tooltip="ZNC">
-                      <i class="fa-brands fa-unity"></i>
-                    </span>
-                  {% elif alert.type == 'query' %}
-                    <span class="icon" data-tooltip="Query">
-                      <i class="fa-solid fa-inbox"></i>
-                    </span>
-                  {% elif alert.type == 'self' %}
-                    <span class="icon" data-tooltip="Self">
-                      <i class="fa-solid fa-message-bot"></i>
-                    </span>
-                  {% else %}
-                    {{ alert.type }}
-                  {% endif %}
-                </td>
-                <td class="wrap" style="max-width: 10em">
-                  {{ alert.msg }}
-                </td>
-              </tr>
-            {% endfor %}
-          </tbody>
-        </table>
-      </div>
-    </div>
-  {% endif %}
-</div>

core/templates/manage/threshold/irc/overview/overview.html

@@ -37,15 +37,6 @@
     hx-swap="outerHTML">
   </div>
 
-  <div
-    style="display: none;"
-    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-    hx-get="{% url 'threshold_irc_overview_alerts' %}"
-    hx-trigger="load"
-    hx-target="#alerts"
-    hx-swap="outerHTML">
-  </div>
-
   <div class="columns">
     <div class="column">
       <div class="box">
@@ -71,6 +62,7 @@
     <div class="column">
       <div class="box">
         <div id="alerts">
+          Alerts here
         </div>
       </div>
     </div>

core/templates/ui/drilldown/drilldown.html

@@ -351,6 +351,44 @@
               </div>
             </div>
           </div>
+          <div class="column is-narrow">
+            <div class="field has-addons">
+              <div class="control has-icons-left">
+                <span class="select is-warning">
+                  <select {% if not user.is_superuser %}disabled{% endif %} id="index" name="index">
+                    {% if params.index == 'main' %}
+                      <option selected value="main">Main</option>
+                    {% elif params.index == None %}
+                      <option selected value="main">Main</option>
+                    {% else %}
+                      <option value="main">Main</option>
+                    {% endif %}
+
+                    {% if params.index == 'int' %}
+                      <option selected value="int">Internal</option>
+                    {% else %}
+                      <option value="int">Internal</option>
+                    {% endif %}
+
+                    {% if params.index == 'meta' %}
+                      <option selected value="meta">Meta</option>
+                    {% else %}
+                      <option value="meta">Meta</option>
+                    {% endif %}
+                    
+                  </select>
+                  <span class="icon is-small is-left">
+                    <i class="fas fa-magnifying-glass"></i>
+                  </span>
+                </span>
+              </div>
+              <p class="control">
+                <a class="button is-static">
+                  index
+                </a>
+              </p>
+            </div>
+          </div>
         </div>
       </div>
       <div class="is-hidden"></div>

core/templates/ui/drilldown/table_results_partial.html

@@ -109,7 +109,14 @@
             <tbody {{ table.attrs.tbody.as_html }}>
               {% for row in table.paginated_rows %}
                 {% block table.tbody.row %}
-                  <tr class="{% if row.cells.exemption == True %}has-background-grey-lighter{% elif row.cells.type == 'join' %}has-background-success-light{% elif row.cells.type == 'quit' %}has-background-danger-light{% elif row.cells.type == 'kick' %}has-background-danger-light{% elif row.cells.type == 'part' %}has-background-warning-light{% elif row.cells.type == 'mode' %}has-background-info-light{% endif %}">
+                  <tr class="
+                             {% if row.cells.exemption == True %}has-background-grey-lighter
+                             {% elif cell == 'join' %}has-background-success-light
+                             {% elif cell == 'quit' %}has-background-danger-light
+                             {% elif cell == 'kick' %}has-background-danger-light
+                             {% elif cell == 'part' %}has-background-warning-light
+                             {% elif cell == 'mode' %}has-background-info-light
+                             {% endif %}">
                     {% for column, cell in row.items %}
                       {% if column.name in show %}
                         {% block table.tbody.td %}
@@ -139,56 +146,73 @@
                               <p>{{ row.cells.date }}</p>
                               <p>{{ row.cells.time }}</p>
                             </td>
-                          {% elif column.name == 'type' %}
+                          {% elif column.name == 'type' or column.name == 'mtype' %}
                             <td class="{{ column.name }}">
                               <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('type', '{{ cell|escapejs }}')">
+                                 onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
-                                {% if row.cells.type == 'msg' %}
+                                {% if cell == 'msg' %}
                                   <span class="icon" data-tooltip="Message">
                                     <i class="fa-solid fa-message"></i>
                                   </span>
-                                {% elif row.cells.type == 'join' %}
+                                {% elif cell == 'join' %}
                                   <span class="icon" data-tooltip="Join">
                                     <i class="fa-solid fa-person-to-portal"></i>
                                   </span>
-                                {% elif row.cells.type == 'part' %}
+                                {% elif cell == 'part' %}
                                   <span class="icon" data-tooltip="Part">
                                     <i class="fa-solid fa-person-from-portal"></i>
                                   </span>
-                                {% elif row.cells.type == 'quit' %}
+                                {% elif cell == 'quit' %}
                                   <span class="icon" data-tooltip="Quit">
                                     <i class="fa-solid fa-circle-xmark"></i>
                                   </span>
-                                {% elif row.cells.type == 'kick' %}
+                                {% elif cell == 'kick' %}
                                   <span class="icon" data-tooltip="Kick">
                                     <i class="fa-solid fa-user-slash"></i>
                                   </span>
-                                {% elif row.cells.type == 'nick' %}
+                                {% elif cell == 'nick' %}
                                   <span class="icon" data-tooltip="Nick">
                                     <i class="fa-solid fa-signature"></i>
                                   </span>
-                                {% elif row.cells.type == 'mode' %}
+                                {% elif cell == 'mode' %}
                                   <span class="icon" data-tooltip="Mode">
                                     <i class="fa-solid fa-gear"></i>
                                   </span>
-                                {% elif row.cells.type == 'action' %}
+                                {% elif cell == 'action' %}
                                   <span class="icon" data-tooltip="Action">
                                     <i class="fa-solid fa-exclamation"></i>
                                   </span>
+                                {% elif cell == 'notice' %}
+                                  <span class="icon" data-tooltip="Notice">
+                                    <i class="fa-solid fa-message-code"></i>
+                                  </span>
+                                {% elif cell == 'conn' %}
+                                  <span class="icon" data-tooltip="Connection">
+                                    <i class="fa-solid fa-cloud-exclamation"></i>
+                                  </span>
+                                {% elif cell == 'znc' %}
+                                  <span class="icon" data-tooltip="ZNC">
+                                    <i class="fa-brands fa-unity"></i>
+                                  </span>
+                                {% elif cell == 'query' %}
+                                  <span class="icon" data-tooltip="Query">
+                                    <i class="fa-solid fa-message"></i>
+                                  </span>
+                                {% elif cell == 'highlight' %}
+                                  <span class="icon" data-tooltip="Highlight">
+                                    <i class="fa-solid fa-exclamation"></i>
+                                  </span>
+                                {% elif cell == 'who' %}
+                                  <span class="icon" data-tooltip="Who">
+                                    <i class="fa-solid fa-passport"></i>
+                                  </span>
                                 {% else %}
-                                  {{ row.cells.type }}
+                                  {{ cell }}
                                 {% endif %}
                               </a>
                             </td>
                           {% elif column.name == 'msg' %}
                             <td class="{{ column.name }} wrap" style="max-width: 10em">{{ row.cells.msg }}</td>
-                          {% elif column.name == 'host' %}
-                            <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('host', '{{ cell|escapejs }}')">
-                                {{ cell }}
-                              </a>
-                            </td>
                           {% elif column.name == 'nick' %}
                             <td class="{{ column.name }}">
                               <div class="nowrap-parent">
@@ -252,34 +276,6 @@
                                 {{ cell }}
                               {% endif %}
                             </td>
-                          {% elif column.name == 'net' %}
-                            <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('net', '{{ cell|escapejs }}')">
-                                {{ cell }}
-                              </a>
-                            </td>
-                          {% elif column.name == 'nick_id' %}
-                            <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('nick_id', '{{ cell|escapejs }}')">
-                                {{ cell }}
-                              </a>
-                            </td>
-                          {% elif column.name == 'user_id' %}
-                            <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('user_id', '{{ cell|escapejs }}')">
-                                {{ cell }}
-                              </a>
-                            </td>
-                          {% elif column.name == 'channel_id' %}
-                            <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
-                                 onclick="populateSearch('channel_id', '{{ cell|escapejs }}')">
-                                {{ cell }}
-                              </a>
-                            </td>
                           {% elif cell is True or cell is False %}
                             <td class="{{ column.name }}">
                               {% if cell is True %}
@@ -294,7 +290,10 @@
                             </td>
                           {% else %}
                             <td class="{{ column.name }}">
+                              <a class="has-text-link is-underlined"
+                                 onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
                                 {{ cell }}
+                              </a>
                             </td>
                           {% endif %}
                         {% endblock table.tbody.td %}

core/views/ui/drilldown.py

@@ -95,6 +95,13 @@ def drilldown_search(request, return_context=False, template=None):
         context = {"sizes": sizes}
         return render(request, template_name, context)
 
+    if "index" in query_params:
+        if not request.user.is_superuser:
+            message = "You can't use the index parameter"
+            message_class = "danger"
+            context = {"message": message, "class": message_class}
+            return render(request, template_name, context)
+
     # Parse the dates
     if "dates" in query_params:
         dates = parse_dates(query_params["dates"])

core/views/ui/tables.py

@@ -61,5 +61,8 @@ class DrilldownTable(Table):
     num_chans = Column()
     num_users = Column()
     online = Column()
+    mtype = Column()
+    realname = Column()
+    server = Column()
     template_name = "ui/drilldown/table_results.html"
     paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE