feat: Add comprehensive metadata enhancement system

check_urls.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Array of URLs to check
+urls=(
+    "http://localhost:3000/"
+    "http://localhost:3000/blog"
+    "http://localhost:3000/blog/what-makes-olake-fast"
+    "http://localhost:3000/blog/olake-architecture-deep-dive"
+    "http://localhost:3000/blog/mongodb-cdc-using-debezium-and-kafka"
+    "http://localhost:3000/blog/debezium-vs-olake"
+    "http://localhost:3000/blog/how-to-set-up-postgresql-cdc-on-aws-rds"
+    "http://localhost:3000/docs"
+    "http://localhost:3000/docs/getting-started/quick-start"
+    "http://localhost:3000/docs/connectors/mongodb/overview"
+    "http://localhost:3000/docs/connectors/postgres/overview"
+    "http://localhost:3000/iceberg"
+    "http://localhost:3000/iceberg/apache-iceberg-vs-delta-lake-guide"
+    "http://localhost:3000/iceberg/data-lake-vs-delta-lake"
+)
+
+echo "URL,Status,Has Trailing Slash Issues,Link Count,Issues Found"
+echo "---,---,---,---,---"
+
+for url in "${urls[@]}"; do
+    # Get the page content
+    content=$(curl -s "$url" 2>/dev/null)
+
+    if [ $? -eq 0 ]; then
+        status="✅ OK"
+
+        # Extract all href attributes
+        hrefs=$(echo "$content" | grep -o 'href="[^"]*"' | sed 's/href="//g' | sed 's/"//g')
+
+        # Count total links
+        link_count=$(echo "$hrefs" | wc -l | tr -d ' ')
+
+        # Check for trailing slash issues
+        issues=""
+        has_issues="No"
+
+        # Check for problematic patterns
+        if echo "$hrefs" | grep -q "href=\"[^/][^/]*[^/]\"[^/]" 2>/dev/null; then
+            has_issues="Yes"
+            issues="Potential trailing slash issues found"
+        fi
+
+        # Check for specific problematic patterns
+        problematic_links=$(echo "$hrefs" | grep -E '^/[^/]*[^/]$' | grep -v -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$')
+        if [ ! -z "$problematic_links" ]; then
+            has_issues="Yes"
+            issues="Root-level links without trailing slash: $(echo "$problematic_links" | head -3 | tr '\n' ' ')"
+        fi
+
+        echo "$url,$status,$has_issues,$link_count,$issues"
+    else
+        echo "$url,❌ Error,Unknown,0,Page not accessible"
+    fi
+done

comprehensive_url_analysis.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+echo "=== COMPREHENSIVE URL TRAILING SLASH ANALYSIS ==="
+echo ""
+
+# Function to analyze a single URL
+analyze_url() {
+    local url="$1"
+    local page_name="$2"
+
+    echo "🔍 Analyzing: $page_name"
+    echo "URL: $url"
+    echo "---"
+
+    # Get the page content
+    content=$(curl -s "$url" 2>/dev/null)
+
+    if [ $? -eq 0 ]; then
+        # Extract all href attributes
+        hrefs=$(echo "$content" | grep -o 'href="[^"]*"' | sed 's/href="//g' | sed 's/"//g' | sort | uniq)
+
+        echo "📊 Total Links Found: $(echo "$hrefs" | wc -l | tr -d ' ')"
+        echo ""
+
+        # Categorize and analyze each link
+        echo "📋 Detailed Link Analysis:"
+        echo ""
+
+        # Root URLs
+        echo "🏠 Root URLs:"
+        echo "$hrefs" | grep -E '^https://olake\.io/?$' | while read link; do
+            if [[ "$link" == "https://olake.io/" ]]; then
+                echo "  ✅ $link (CORRECT - has trailing slash)"
+            else
+                echo "  ❌ $link (INCORRECT - missing trailing slash)"
+            fi
+        done
+        echo ""
+
+        # Directory URLs (should have trailing slash)
+        echo "📁 Directory URLs (should have trailing slash):"
+        echo "$hrefs" | grep -E '^/[^/]*/?$' | grep -v -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$' | while read link; do
+            if [[ "$link" == */ ]]; then
+                echo "  ✅ $link (CORRECT - has trailing slash)"
+            else
+                echo "  ❌ $link (INCORRECT - missing trailing slash)"
+            fi
+        done
+        echo ""
+
+        # File URLs (should NOT have trailing slash)
+        echo "📄 File URLs (should NOT have trailing slash):"
+        echo "$hrefs" | grep -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$' | while read link; do
+            if [[ "$link" != */ ]]; then
+                echo "  ✅ $link (CORRECT - no trailing slash)"
+            else
+                echo "  ❌ $link (INCORRECT - has trailing slash)"
+            fi
+        done
+        echo ""
+
+        # Navigation links (should NOT have trailing slash)
+        echo "🧭 Navigation Links (should NOT have trailing slash):"
+        echo "$hrefs" | grep -E '^/(docs|blog|iceberg|ai-lake|webinar|community|search)(/|$)' | grep -v -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$' | while read link; do
+            if [[ "$link" != */ ]]; then
+                echo "  ✅ $link (CORRECT - no trailing slash for navigation)"
+            else
+                echo "  ❌ $link (INCORRECT - has trailing slash for navigation)"
+            fi
+        done
+        echo ""
+
+        # External URLs
+        echo "🌐 External URLs:"
+        echo "$hrefs" | grep -E '^https?://' | grep -v 'olake.io' | head -5 | while read link; do
+            echo "  🔗 $link (External - format varies)"
+        done
+        echo ""
+
+    else
+        echo "❌ Error: Could not access page"
+    fi
+
+    echo "================================================"
+    echo ""
+}
+
+# Check all pages
+analyze_url "http://localhost:3000/" "Homepage"
+analyze_url "http://localhost:3000/blog" "Blog Index"
+analyze_url "http://localhost:3000/blog/what-makes-olake-fast" "Blog: What makes OLake fast?"
+analyze_url "http://localhost:3000/blog/olake-architecture-deep-dive" "Blog: Architecture Deep Dive"
+analyze_url "http://localhost:3000/blog/mongodb-cdc-using-debezium-and-kafka" "Blog: MongoDB CDC"
+analyze_url "http://localhost:3000/blog/debezium-vs-olake" "Blog: Debezium vs OLake"
+analyze_url "http://localhost:3000/blog/how-to-set-up-postgresql-cdc-on-aws-rds" "Blog: PostgreSQL CDC"
+analyze_url "http://localhost:3000/docs" "Documentation Index"
+analyze_url "http://localhost:3000/docs/getting-started/quick-start" "Docs: Quick Start"
+analyze_url "http://localhost:3000/docs/connectors/mongodb/overview" "Docs: MongoDB Connector"
+analyze_url "http://localhost:3000/docs/connectors/postgres/overview" "Docs: PostgreSQL Connector"
+analyze_url "http://localhost:3000/iceberg" "Iceberg Index"
+analyze_url "http://localhost:3000/iceberg/apache-iceberg-vs-delta-lake-guide" "Iceberg: Apache vs Delta"
+analyze_url "http://localhost:3000/iceberg/data-lake-vs-delta-lake" "Iceberg: Data Lake vs Delta"
+

detailed_check.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+echo "=== COMPREHENSIVE URL TRAILING SLASH ANALYSIS ==="
+echo ""
+
+# Function to check a single URL
+check_url() {
+    local url="$1"
+    local page_name="$2"
+
+    echo "🔍 Checking: $page_name"
+    echo "URL: $url"
+    echo "---"
+
+    # Get the page content
+    content=$(curl -s "$url" 2>/dev/null)
+
+    if [ $? -eq 0 ]; then
+        # Extract all href attributes and clean them
+        hrefs=$(echo "$content" | grep -o 'href="[^"]*"' | sed 's/href="//g' | sed 's/"//g' | sort | uniq)
+
+        echo "📊 Total Links Found: $(echo "$hrefs" | wc -l | tr -d ' ')"
+        echo ""
+
+        # Categorize links
+        echo "�� Link Analysis:"
+        echo ""
+
+        # Root URLs (should have trailing slash)
+        echo "🏠 Root URLs (should have trailing slash):"
+        echo "$hrefs" | grep -E '^https://olake\.io/?$' | while read link; do
+            if [[ "$link" == "https://olake.io/" ]]; then
+                echo "  ✅ $link (correct)"
+            else
+                echo "  ❌ $link (missing trailing slash)"
+            fi
+        done
+        echo ""
+
+        # Directory URLs (should have trailing slash)
+        echo "📁 Directory URLs (should have trailing slash):"
+        echo "$hrefs" | grep -E '^/[^/]*/?$' | grep -v -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$' | while read link; do
+            if [[ "$link" == */ ]]; then
+                echo "  ✅ $link (correct)"
+            else
+                echo "  ❌ $link (missing trailing slash)"
+            fi
+        done
+        echo ""
+
+        # File URLs (should NOT have trailing slash)
+        echo "📄 File URLs (should NOT have trailing slash):"
+        echo "$hrefs" | grep -E '\.(css|js|xml|svg|png|jpg|jpeg|gif|ico)$' | while read link; do
+            if [[ "$link" != */ ]]; then
+                echo "  ✅ $link (correct)"
+            else
+                echo "  ❌ $link (has trailing slash - incorrect)"
+            fi
+        done
+        echo ""
+
+        # External URLs
+        echo "🌐 External URLs:"
+        echo "$hrefs" | grep -E '^https?://' | grep -v 'olake.io' | head -5 | while read link; do
+            echo "  🔗 $link"
+        done
+        echo ""
+
+    else
+        echo "❌ Error: Could not access page"
+    fi
+
+    echo "================================================"
+    echo ""
+}
+
+# Check all pages
+check_url "http://localhost:3000/" "Homepage"
+check_url "http://localhost:3000/blog" "Blog Index"
+check_url "http://localhost:3000/blog/what-makes-olake-fast" "Blog: What makes OLake fast?"
+check_url "http://localhost:3000/docs" "Documentation Index"
+check_url "http://localhost:3000/docs/getting-started/quick-start" "Docs: Quick Start"
+check_url "http://localhost:3000/iceberg" "Iceberg Index"
+check_url "http://localhost:3000/iceberg/apache-iceberg-vs-delta-lake-guide" "Iceberg: Apache Iceberg vs Delta Lake"
+