refactor: render source schema from flatmap

lib/logflare_web/controllers/source_controller.ex

@@ -72,7 +72,7 @@ defmodule LogflareWeb.SourceController do
     |> maybe_put_high_rate_flash(source, avg_rate)
     |> render(
       "show.html",
-      bq_schema: get_bigquery_schema(source),
+      schema_flatmap: SourceSchemas.source_schema_flatmap_or_default(source),
       logs: get_and_encode_logs(source),
       source: source,
       public_token: source.public_token,

lib/logflare_web/live/sources/bq_schema_component.ex

@@ -1,24 +1,18 @@
 defmodule LogflareWeb.SourceBqSchemaComponent do
   @moduledoc false
   use LogflareWeb, :live_component
-  alias LogflareWeb.Helpers.BqSchema
   alias Logflare.SourceSchemas
-  alias Logflare.Sources.Source.BigQuery.SchemaBuilder
+  alias LogflareWeb.Helpers.BqSchema
 
   @impl true
   def render(%{source: source} = assigns) do
-    bq_schema =
-      SourceSchemas.Cache.get_source_schema_by(source_id: source.id)
-      |> case do
-        nil -> SchemaBuilder.initial_table_schema()
-        %_{bigquery_schema: schema} -> schema
-      end
+    schema_flatmap = SourceSchemas.source_schema_flatmap_or_default(source)
 
-    assigns = assign(assigns, :bq_schema, BqSchema.format_bq_schema(bq_schema))
+    assigns = assign(assigns, :schema, BqSchema.format_schema(schema_flatmap))
 
     ~H"""
     <div>
-      {@bq_schema}
+      {@schema}
     </div>
     """
   end

lib/logflare_web/templates/source/show.html.heex

@@ -100,7 +100,7 @@
         </button>
       </div>
       <div class="modal-body">
-        {BqSchema.format_bq_schema(@bq_schema)}
+        {BqSchema.format_schema(@schema_flatmap)}
       </div>
     </div>
   </div>

lib/logflare_web/views/helpers/bq_schema_helpers.ex

@@ -1,6 +1,5 @@
 defmodule LogflareWeb.Helpers.BqSchema do
   @moduledoc false
-  alias GoogleApi.BigQuery.V2.Model.TableFieldSchema, as: TFS
   alias GoogleApi.BigQuery.V2.Model.TableSchema, as: TS
   alias Logflare.BigQuery.SchemaTypes
   alias Logflare.Google.BigQuery.SchemaUtils
@@ -9,43 +8,40 @@ defmodule LogflareWeb.Helpers.BqSchema do
   alias LogflareWeb.SharedView
 
   @type field_and_type :: {String.t(), String.t()}
-  @type format_bq_schema_opts :: [type: :markdown | :text]
+  @type format_schema_opts :: [type: :markdown | :text]
   @type timestamp_format_opts :: [format: String.t()]
 
-  @spec format_bq_schema(nil) :: String.t()
-  def format_bq_schema(nil), do: ""
-
-  @spec format_bq_schema(TS.t()) :: Phoenix.HTML.safe()
-  def format_bq_schema(bq_schema) do
+  @spec format_schema(map()) :: Phoenix.HTML.safe()
+  def format_schema(schema_flatmap) when is_map(schema_flatmap) do
     SharedView.render("bq_schema.html",
-      fields_and_types: format_bq_schema(bq_schema, type: :text),
-      markdown_schema: format_bq_schema(bq_schema, type: :markdown)
+      fields_and_types: format_schema(schema_flatmap, type: :text),
+      markdown_schema: format_schema(schema_flatmap, type: :markdown)
     )
   end
 
-  @spec format_bq_schema(TS.t(), format_bq_schema_opts()) :: [field_and_type()] | String.t()
-  def format_bq_schema(bq_schema, type: :text) do
+  @spec format_bq_schema(TS.t(), format_schema_opts()) :: [field_and_type()] | String.t()
+  def format_bq_schema(%TS{} = bq_schema, type: type) do
     bq_schema
     |> SchemaUtils.bq_schema_to_flat_typemap()
-    |> Enum.map(fn {k, v} ->
-      v =
-        case SchemaTypes.to_schema_type(v) do
-          {type, inner_type} -> "#{type}<#{inner_type}>"
-          type -> type
-        end
-
-      {k, v}
-    end)
-    |> Enum.sort_by(fn {k, _v} -> k end)
+    |> format_schema(type: type)
+  end
+
+  @spec format_schema(map(), format_schema_opts()) :: [field_and_type()] | String.t()
+  def format_schema(schema_flatmap, type: :text) when is_map(schema_flatmap) do
+    schema_flatmap
+    |> Enum.map(fn {field, type} -> {field, format_type(type)} end)
+    |> Enum.sort_by(fn {field, _type} -> field end)
   end
 
-  def format_bq_schema(%TS{fields: fields}, type: :markdown) do
+  def format_schema(schema_flatmap, type: :markdown) when is_map(schema_flatmap) do
     [
       "# Logflare source schema",
       "",
       "Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/)",
       ""
-      | markdown_field_lines(fields || [])
+      | schema_flatmap
+        |> format_schema(type: :text)
+        |> Enum.map(fn {field, type} -> "- `#{field}` #{type}#{markdown_note(field)}" end)
     ]
     |> Enum.join("\n")
   end
@@ -74,37 +70,20 @@ defmodule LogflareWeb.Helpers.BqSchema do
 
   defp convert_timezone(%DateTime{} = datetime, _timezone), do: datetime
 
-  defp markdown_field_lines(fields, path \\ [], depth \\ 0)
-
-  defp markdown_field_lines(fields, path, depth) when is_list(fields) do
-    fields
-    |> Enum.sort_by(& &1.name)
-    |> Enum.flat_map(&markdown_field_lines(&1, path, depth))
-  end
-
-  defp markdown_field_lines(%TFS{name: name, fields: fields} = field, path, depth) do
-    current_path = path ++ [name]
-
-    [markdown_field_line(field, current_path, depth)] ++
-      markdown_field_lines(fields || [], current_path, depth + 1)
-  end
-
-  defp markdown_field_line(%TFS{name: name, type: type, mode: mode}, path, depth) do
-    "#{markdown_indent(depth)}- `#{name}` #{markdown_type(type, mode)}#{markdown_note(path)}"
+  defp format_type(type) do
+    case SchemaTypes.to_schema_type(type) do
+      {type, inner_type} -> "#{type}<#{inner_type}>"
+      type -> type
+    end
   end
 
-  defp markdown_type(type, "REPEATED"), do: "ARRAY<#{type}>"
-  defp markdown_type(type, _mode), do: type
-
-  defp markdown_note(["event_message"]), do: " Human-readable event message."
-
-  defp markdown_note(["id"]), do: " Event UUID."
+  defp markdown_note("event_message"), do: " Human-readable event message."
 
-  defp markdown_note(["timestamp"]), do: " Ingest timestamp."
+  defp markdown_note("id"), do: " Event UUID."
 
-  defp markdown_note(_path), do: ""
+  defp markdown_note("timestamp"), do: " Ingest timestamp."
 
-  defp markdown_indent(depth), do: String.duplicate("  ", depth)
+  defp markdown_note(_field), do: ""
 
   def encode_metadata(metadata) do
     metadata

test/logflare_web/live/sources/bq_schema_component_test.exs

@@ -0,0 +1,50 @@
+defmodule LogflareWeb.SourceBqSchemaComponentTest do
+  use LogflareWeb.ConnCase, async: false
+
+  alias Logflare.Google.BigQuery.SchemaUtils
+  alias LogflareWeb.SourceBqSchemaComponent
+
+  describe "render/1" do
+    setup do
+      insert(:plan, name: "Free")
+      user = insert(:user)
+      source = insert(:source, user: user)
+
+      %{source: source}
+    end
+
+    test "renders and copies schema from the flatmap schema", %{source: source} do
+      bq_schema =
+        TestUtils.build_bq_schema(%{
+          "metadata" => %{"user_id" => 123, "tags" => ["testing"]},
+          "event_message" => "message"
+        })
+
+      source_schema =
+        insert(:source_schema,
+          source: source,
+          bigquery_schema: bq_schema,
+          schema_flat_map:
+            bq_schema
+            |> SchemaUtils.bq_schema_to_flat_typemap()
+            |> Map.put("flatmap_only", :string)
+        )
+
+      html = render_component(SourceBqSchemaComponent, %{id: "schema", source: source})
+
+      assert html =~ "<kbd>flatmap_only</kbd>"
+      assert html =~ "data-clipboard-text=\"flatmap_only\""
+      assert html =~ "- `flatmap_only` STRING"
+      assert html =~ "- `metadata.tags` ARRAY&lt;STRING&gt;"
+      assert html =~ "- `metadata.user_id` INTEGER"
+      assert source_schema.schema_flat_map["flatmap_only"] == :string
+    end
+
+    test "renders the default flatmap schema when source schema is missing", %{source: source} do
+      html = render_component(SourceBqSchemaComponent, %{id: "schema", source: source})
+
+      assert html =~ "<kbd>event_message</kbd>"
+      assert html =~ "- `event_message` STRING Human-readable event message."
+    end
+  end
+end

test/logflare_web/views/helpers/bq_schema_helpers_test.exs

@@ -1,26 +1,19 @@
 defmodule LogflareWeb.Helpers.BqSchemaTest do
   use LogflareWeb.ConnCase, async: true
 
-  alias GoogleApi.BigQuery.V2.Model.TableFieldSchema, as: TFS
-  alias GoogleApi.BigQuery.V2.Model.TableSchema, as: TS
+  alias Logflare.Google.BigQuery.SchemaUtils
   alias LogflareWeb.Helpers.BqSchema
 
   describe "format_bq_schema/2" do
-    test "formats a BigQuery schema as a markdown list" do
-      schema = %TS{
-        fields: [
-          %TFS{name: "event_message", type: "STRING", mode: "REQUIRED"},
-          %TFS{
-            name: "metadata",
-            type: "RECORD",
-            mode: "NULLABLE",
-            fields: [
-              %TFS{name: "tags", type: "STRING", mode: "REPEATED"},
-              %TFS{name: "user_id", type: "INTEGER", mode: "NULLABLE"}
-            ]
+    test "formats a BigQuery schema as a flat markdown list" do
+      schema =
+        TestUtils.build_bq_schema(%{
+          "event_message" => "human-readable event message",
+          "metadata" => %{
+            "tags" => ["tag"],
+            "user_id" => 1
           }
-        ]
-      }
+        })
 
       assert BqSchema.format_bq_schema(schema, type: :markdown) ==
                """
@@ -29,9 +22,39 @@ defmodule LogflareWeb.Helpers.BqSchemaTest do
                Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/)
 
                - `event_message` STRING Human-readable event message.
+               - `id` STRING Event UUID.
                - `metadata` RECORD
-                 - `tags` ARRAY<STRING>
-                 - `user_id` INTEGER\
+               - `metadata.tags` ARRAY<STRING>
+               - `metadata.user_id` INTEGER
+               - `timestamp` DATETIME Ingest timestamp.\
+               """
+    end
+  end
+
+  describe "format_schema/2" do
+    test "formats a flatmap schema as a markdown list" do
+      schema_flatmap =
+        TestUtils.build_bq_schema(%{
+          "event_message" => "human-readable event message",
+          "metadata" => %{
+            "tags" => ["tag"],
+            "user_id" => 1
+          }
+        })
+        |> SchemaUtils.bq_schema_to_flat_typemap()
+
+      assert BqSchema.format_schema(schema_flatmap, type: :markdown) ==
+               """
+               # Logflare source schema
+
+               Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/)
+
+               - `event_message` STRING Human-readable event message.
+               - `id` STRING Event UUID.
+               - `metadata` RECORD
+               - `metadata.tags` ARRAY<STRING>
+               - `metadata.user_id` INTEGER
+               - `timestamp` DATETIME Ingest timestamp.\
                """
     end
   end