Improve the filtering query, and indexes needed

The `pg_trgm` bits have been inspired by:
- https://stackoverflow.com/questions/32951793/how-to-index-a-varchar-array-in-postgresql-using-array-to-string/33016333#33016333
- https://stackoverflow.com/questions/39480580/how-to-index-a-string-array-column-for-pg-trgm-term-any-array-column-que

lib/nerves_hub/devices.ex

@@ -75,6 +75,19 @@ defmodule NervesHub.Devices do
     |> Repo.paginate(pagination)
   end
 
+  def filter(product_id, opts) do
+    pagination = Map.get(opts, :pagination, %{})
+    sorting = Map.get(opts, :sort, {:asc, :identifier})
+    filters = Map.get(opts, :filters, %{})
+
+    Device
+    |> where([d], d.product_id == ^product_id)
+    |> Repo.exclude_deleted()
+    |> filtering(filters)
+    |> order_by(^sort_devices(sorting))
+    |> Repo.paginate(pagination)
+  end
+
   def get_health_by_org_id_and_product_id(org_id, product_id, opts) do
     query =
       from(
@@ -163,7 +176,7 @@ defmodule NervesHub.Devices do
                 where(
                   query,
                   [d],
-                  fragment("array_to_string(?, ',') ILIKE ?", d.tags, ^"%#{tag}%")
+                  fragment("array_to_string(?, ' ', ' ') ILIKE ?", d.tags, ^"%#{tag}%")
                 )
               end)
 

lib/nerves_hub_web/live/devices/index.ex

@@ -353,7 +353,7 @@ defmodule NervesHubWeb.Live.Devices.Index do
       filters: socket.assigns.current_filters
     }
 
-    page = Devices.get_devices_by_org_id_and_product_id(org.id, product.id, opts)
+    page = Devices.filter(product.id, opts)
 
     statuses =
       Enum.into(page.entries, %{}, fn device ->

priv/repo/migrations/20240910012345_enable_pg_trgm_extension.exs

@@ -0,0 +1,20 @@
+defmodule NervesHub.Repo.Migrations.EnablePgTrgmExtension do
+  use Ecto.Migration
+
+  def change do
+    execute "CREATE EXTENSION pg_trgm", "DROP EXTENSION pg_trgm"
+    execute(&add_function/0, &drop_function/0)
+  end
+
+  defp add_function do
+    repo().query!("""
+      create function string_array_to_string(text[], text, text) returns text as $$
+         select array_to_string($1, $2, $3)
+      $$ language sql cost 1 immutable;
+    """, [], [log: :info])
+  end
+
+  defp drop_function do
+    repo().query!("drop function string_array_to_string(text[], text, text);", [], [log: :info])
+  end
+end

priv/repo/migrations/20240910054628_add_indexes_to_devices.exs

@@ -0,0 +1,13 @@
+defmodule NervesHub.Repo.Migrations.AddIndexesToDevices do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def change do
+    create index("devices", [:product_id], concurrently: true)
+    create index("devices", [:updates_enabled], concurrently: true)
+    create index("devices", ["(firmware_metadata->'version')"], name: :devices_firmware_index, using: "GIN", concurrently: true)
+    create index("devices", ["(firmware_metadata->'platform')"], name: :devices_platform_index, using: "GIN", concurrently: true)
+    create index("devices", ["string_array_to_string(tags, ' ', ' ') gin_trgm_ops"], name: :devices_tags_index, using: "GIN", concurrently: true)
+    create index("devices", ["identifier gin_trgm_ops"], name: :devices_identifier_trgm_index, using: "GIN", concurrently: true)
+  end
+end