diff --git a/lib/logflare_web/controllers/source_controller.ex b/lib/logflare_web/controllers/source_controller.ex index d7d0ff532b..16195261d9 100644 --- a/lib/logflare_web/controllers/source_controller.ex +++ b/lib/logflare_web/controllers/source_controller.ex @@ -72,7 +72,7 @@ defmodule LogflareWeb.SourceController do |> maybe_put_high_rate_flash(source, avg_rate) |> render( "show.html", - bq_schema: get_bigquery_schema(source), + schema_flatmap: SourceSchemas.source_schema_flatmap_or_default(source), logs: get_and_encode_logs(source), source: source, public_token: source.public_token, diff --git a/lib/logflare_web/live/sources/bq_schema_component.ex b/lib/logflare_web/live/sources/bq_schema_component.ex index 9ffe8158a2..b773bcef43 100644 --- a/lib/logflare_web/live/sources/bq_schema_component.ex +++ b/lib/logflare_web/live/sources/bq_schema_component.ex @@ -1,24 +1,18 @@ defmodule LogflareWeb.SourceBqSchemaComponent do @moduledoc false use LogflareWeb, :live_component - alias LogflareWeb.Helpers.BqSchema alias Logflare.SourceSchemas - alias Logflare.Sources.Source.BigQuery.SchemaBuilder + alias LogflareWeb.Helpers.BqSchema @impl true def render(%{source: source} = assigns) do - bq_schema = - SourceSchemas.Cache.get_source_schema_by(source_id: source.id) - |> case do - nil -> SchemaBuilder.initial_table_schema() - %_{bigquery_schema: schema} -> schema - end + schema_flatmap = SourceSchemas.source_schema_flatmap_or_default(source) - assigns = assign(assigns, :bq_schema, BqSchema.format_bq_schema(bq_schema)) + assigns = assign(assigns, :schema, BqSchema.format_schema(schema_flatmap)) ~H"""
- {@bq_schema} + {@schema}
""" end diff --git a/lib/logflare_web/templates/source/show.html.heex b/lib/logflare_web/templates/source/show.html.heex index 77752bcf28..96c371053e 100644 --- a/lib/logflare_web/templates/source/show.html.heex +++ b/lib/logflare_web/templates/source/show.html.heex @@ -100,7 +100,7 @@
- {BqSchema.format_bq_schema(@bq_schema)} + {BqSchema.format_schema(@schema_flatmap)}
diff --git a/lib/logflare_web/views/helpers/bq_schema_helpers.ex b/lib/logflare_web/views/helpers/bq_schema_helpers.ex index e0825ec433..8c4ae6413b 100644 --- a/lib/logflare_web/views/helpers/bq_schema_helpers.ex +++ b/lib/logflare_web/views/helpers/bq_schema_helpers.ex @@ -1,6 +1,5 @@ defmodule LogflareWeb.Helpers.BqSchema do @moduledoc false - alias GoogleApi.BigQuery.V2.Model.TableFieldSchema, as: TFS alias GoogleApi.BigQuery.V2.Model.TableSchema, as: TS alias Logflare.BigQuery.SchemaTypes alias Logflare.Google.BigQuery.SchemaUtils @@ -9,43 +8,40 @@ defmodule LogflareWeb.Helpers.BqSchema do alias LogflareWeb.SharedView @type field_and_type :: {String.t(), String.t()} - @type format_bq_schema_opts :: [type: :markdown | :text] + @type format_schema_opts :: [type: :markdown | :text] @type timestamp_format_opts :: [format: String.t()] - @spec format_bq_schema(nil) :: String.t() - def format_bq_schema(nil), do: "" - - @spec format_bq_schema(TS.t()) :: Phoenix.HTML.safe() - def format_bq_schema(bq_schema) do + @spec format_schema(map()) :: Phoenix.HTML.safe() + def format_schema(schema_flatmap) when is_map(schema_flatmap) do SharedView.render("bq_schema.html", - fields_and_types: format_bq_schema(bq_schema, type: :text), - markdown_schema: format_bq_schema(bq_schema, type: :markdown) + fields_and_types: format_schema(schema_flatmap, type: :text), + markdown_schema: format_schema(schema_flatmap, type: :markdown) ) end - @spec format_bq_schema(TS.t(), format_bq_schema_opts()) :: [field_and_type()] | String.t() - def format_bq_schema(bq_schema, type: :text) do + @spec format_bq_schema(TS.t(), format_schema_opts()) :: [field_and_type()] | String.t() + def format_bq_schema(%TS{} = bq_schema, type: type) do bq_schema |> SchemaUtils.bq_schema_to_flat_typemap() - |> Enum.map(fn {k, v} -> - v = - case SchemaTypes.to_schema_type(v) do - {type, inner_type} -> "#{type}<#{inner_type}>" - type -> type - end - - {k, v} - end) - |> Enum.sort_by(fn {k, _v} -> k end) + |> format_schema(type: type) + end + + @spec format_schema(map(), format_schema_opts()) :: [field_and_type()] | String.t() + def format_schema(schema_flatmap, type: :text) when is_map(schema_flatmap) do + schema_flatmap + |> Enum.map(fn {field, type} -> {field, format_type(type)} end) + |> Enum.sort_by(fn {field, _type} -> field end) end - def format_bq_schema(%TS{fields: fields}, type: :markdown) do + def format_schema(schema_flatmap, type: :markdown) when is_map(schema_flatmap) do [ "# Logflare source schema", "", "Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/)", "" - | markdown_field_lines(fields || []) + | schema_flatmap + |> format_schema(type: :text) + |> Enum.map(fn {field, type} -> "- `#{field}` #{type}#{markdown_note(field)}" end) ] |> Enum.join("\n") end @@ -74,37 +70,20 @@ defmodule LogflareWeb.Helpers.BqSchema do defp convert_timezone(%DateTime{} = datetime, _timezone), do: datetime - defp markdown_field_lines(fields, path \\ [], depth \\ 0) - - defp markdown_field_lines(fields, path, depth) when is_list(fields) do - fields - |> Enum.sort_by(& &1.name) - |> Enum.flat_map(&markdown_field_lines(&1, path, depth)) - end - - defp markdown_field_lines(%TFS{name: name, fields: fields} = field, path, depth) do - current_path = path ++ [name] - - [markdown_field_line(field, current_path, depth)] ++ - markdown_field_lines(fields || [], current_path, depth + 1) - end - - defp markdown_field_line(%TFS{name: name, type: type, mode: mode}, path, depth) do - "#{markdown_indent(depth)}- `#{name}` #{markdown_type(type, mode)}#{markdown_note(path)}" + defp format_type(type) do + case SchemaTypes.to_schema_type(type) do + {type, inner_type} -> "#{type}<#{inner_type}>" + type -> type + end end - defp markdown_type(type, "REPEATED"), do: "ARRAY<#{type}>" - defp markdown_type(type, _mode), do: type - - defp markdown_note(["event_message"]), do: " Human-readable event message." - - defp markdown_note(["id"]), do: " Event UUID." + defp markdown_note("event_message"), do: " Human-readable event message." - defp markdown_note(["timestamp"]), do: " Ingest timestamp." + defp markdown_note("id"), do: " Event UUID." - defp markdown_note(_path), do: "" + defp markdown_note("timestamp"), do: " Ingest timestamp." - defp markdown_indent(depth), do: String.duplicate(" ", depth) + defp markdown_note(_field), do: "" def encode_metadata(metadata) do metadata diff --git a/test/logflare_web/live/sources/bq_schema_component_test.exs b/test/logflare_web/live/sources/bq_schema_component_test.exs new file mode 100644 index 0000000000..3e37049849 --- /dev/null +++ b/test/logflare_web/live/sources/bq_schema_component_test.exs @@ -0,0 +1,50 @@ +defmodule LogflareWeb.SourceBqSchemaComponentTest do + use LogflareWeb.ConnCase, async: false + + alias Logflare.Google.BigQuery.SchemaUtils + alias LogflareWeb.SourceBqSchemaComponent + + describe "render/1" do + setup do + insert(:plan, name: "Free") + user = insert(:user) + source = insert(:source, user: user) + + %{source: source} + end + + test "renders and copies schema from the flatmap schema", %{source: source} do + bq_schema = + TestUtils.build_bq_schema(%{ + "metadata" => %{"user_id" => 123, "tags" => ["testing"]}, + "event_message" => "message" + }) + + source_schema = + insert(:source_schema, + source: source, + bigquery_schema: bq_schema, + schema_flat_map: + bq_schema + |> SchemaUtils.bq_schema_to_flat_typemap() + |> Map.put("flatmap_only", :string) + ) + + html = render_component(SourceBqSchemaComponent, %{id: "schema", source: source}) + + assert html =~ "flatmap_only" + assert html =~ "data-clipboard-text=\"flatmap_only\"" + assert html =~ "- `flatmap_only` STRING" + assert html =~ "- `metadata.tags` ARRAY<STRING>" + assert html =~ "- `metadata.user_id` INTEGER" + assert source_schema.schema_flat_map["flatmap_only"] == :string + end + + test "renders the default flatmap schema when source schema is missing", %{source: source} do + html = render_component(SourceBqSchemaComponent, %{id: "schema", source: source}) + + assert html =~ "event_message" + assert html =~ "- `event_message` STRING Human-readable event message." + end + end +end diff --git a/test/logflare_web/views/helpers/bq_schema_helpers_test.exs b/test/logflare_web/views/helpers/bq_schema_helpers_test.exs index 4e935aacc8..3796ca3e9c 100644 --- a/test/logflare_web/views/helpers/bq_schema_helpers_test.exs +++ b/test/logflare_web/views/helpers/bq_schema_helpers_test.exs @@ -1,26 +1,19 @@ defmodule LogflareWeb.Helpers.BqSchemaTest do use LogflareWeb.ConnCase, async: true - alias GoogleApi.BigQuery.V2.Model.TableFieldSchema, as: TFS - alias GoogleApi.BigQuery.V2.Model.TableSchema, as: TS + alias Logflare.Google.BigQuery.SchemaUtils alias LogflareWeb.Helpers.BqSchema describe "format_bq_schema/2" do - test "formats a BigQuery schema as a markdown list" do - schema = %TS{ - fields: [ - %TFS{name: "event_message", type: "STRING", mode: "REQUIRED"}, - %TFS{ - name: "metadata", - type: "RECORD", - mode: "NULLABLE", - fields: [ - %TFS{name: "tags", type: "STRING", mode: "REPEATED"}, - %TFS{name: "user_id", type: "INTEGER", mode: "NULLABLE"} - ] + test "formats a BigQuery schema as a flat markdown list" do + schema = + TestUtils.build_bq_schema(%{ + "event_message" => "human-readable event message", + "metadata" => %{ + "tags" => ["tag"], + "user_id" => 1 } - ] - } + }) assert BqSchema.format_bq_schema(schema, type: :markdown) == """ @@ -29,9 +22,39 @@ defmodule LogflareWeb.Helpers.BqSchemaTest do Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/) - `event_message` STRING Human-readable event message. + - `id` STRING Event UUID. - `metadata` RECORD - - `tags` ARRAY - - `user_id` INTEGER\ + - `metadata.tags` ARRAY + - `metadata.user_id` INTEGER + - `timestamp` DATETIME Ingest timestamp.\ + """ + end + end + + describe "format_schema/2" do + test "formats a flatmap schema as a markdown list" do + schema_flatmap = + TestUtils.build_bq_schema(%{ + "event_message" => "human-readable event message", + "metadata" => %{ + "tags" => ["tag"], + "user_id" => 1 + } + }) + |> SchemaUtils.bq_schema_to_flat_typemap() + + assert BqSchema.format_schema(schema_flatmap, type: :markdown) == + """ + # Logflare source schema + + Use this schema when writing Logflare LQL (https://docs.logflare.app/concepts/lql/) + + - `event_message` STRING Human-readable event message. + - `id` STRING Event UUID. + - `metadata` RECORD + - `metadata.tags` ARRAY + - `metadata.user_id` INTEGER + - `timestamp` DATETIME Ingest timestamp.\ """ end end