Add Lightpanda browser services and automation scripts

Add lightpanda-cdp and lightpanda-mcp services for local (Tilt) and production (deploy/prod/) with Traefik routing, HPAs, and CI image publishing. Create scripts for concurrent Bing search via MCP and CDP screenshots. Update flake.nix with websocat dependency. Make webshare-secret helper tolerant of missing Doppler credentials.
2026-10-13 06:35:12 +00:00 · 2026-05-05 07:56:22 +01:00
parent fff06469a9
commit ed6415894f
20 changed files with 1009 additions and 21 deletions
@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail

+if [ -z "${DOPPLER_INJECTED:-}" ] && command -v doppler >/dev/null 2>&1 && doppler whoami >/dev/null 2>&1; then
+  exec doppler run -- bash "$0" "$@"
+fi
+
 if [ -n "${WEBSHARE_API_KEY:-}" ] && [ -n "${WEBSHARE_PROXY_USERNAME:-}" ] && [ -n "${WEBSHARE_PROXY_PASSWORD:-}" ]; then
  kubectl create secret generic webshare-api \
    --from-literal=api-key="$WEBSHARE_API_KEY" \
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CDP_URL="${CDP_URL:-http://localhost:9222}"
+QUERY="chattybox ai"
+OUTPUT="lightpanda-google-search.png"
+WAIT_SECONDS=4
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/screenshot-lightpanda-page [options]
+
+Take a screenshot with the Lightpanda CDP service.
+
+Options:
+  -q, --query TEXT      Google search query (default: "chattybox ai")
+  -u, --url URL         URL to capture instead of building a Google search URL
+  -o, --output PATH     Screenshot output path (default: lightpanda-google-search.png)
+  -w, --wait SECONDS    Seconds to wait after navigation (default: 4)
+  -h, --help            Show this help message
+
+Examples:
+  scripts/screenshot-lightpanda-page
+  scripts/screenshot-lightpanda-page --output /tmp/google-consent.png
+  scripts/screenshot-lightpanda-page --url https://example.com --output /tmp/example.png
+USAGE
+}
+
+URL=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -q|--query)
+      QUERY="$2"
+      shift 2
+      ;;
+    -u|--url)
+      URL="$2"
+      shift 2
+      ;;
+    -o|--output)
+      OUTPUT="$2"
+      shift 2
+      ;;
+    -w|--wait)
+      WAIT_SECONDS="$2"
+      shift 2
+      ;;
+    --*)
+      echo "Unknown option: $1" >&2
+      usage >&2
+      exit 1
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+if [ -z "$URL" ]; then
+  encoded_query=$(jq -rn --arg q "$QUERY" '$q|@uri')
+  URL="https://www.bing.com/search?q=$encoded_query"
+fi
+
+if ! command -v websocat >/dev/null 2>&1; then
+  echo "ERROR: websocat is required. Run through nix develop after updating the flake." >&2
+  exit 1
+fi
+
+version_json=$(curl -sf "$CDP_URL/json/version") || {
+  echo "ERROR: Lightpanda CDP not reachable at $CDP_URL" >&2
+  echo "Make sure Tilt is running: nix run .#tilt -- up" >&2
+  exit 1
+}
+
+ws_url=$(printf '%s\n' "$version_json" | jq -r '.webSocketDebuggerUrl')
+ws_url="${ws_url/0.0.0.0/localhost}"
+
+echo "Capturing: $URL"
+echo "Output: $OUTPUT"
+
+coproc CDP { websocat "$ws_url"; }
+trap 'kill "$CDP_PID" >/dev/null 2>&1 || true' EXIT
+
+send_cdp() {
+  local id="$1"
+  local method="$2"
+  local params="{}"
+  if [ "$#" -ge 3 ]; then
+    params="$3"
+  fi
+  jq -cn --argjson id "$id" --arg method "$method" --argjson params "$params" \
+    '{id: $id, method: $method, params: $params}' >&"${CDP[1]}"
+}
+
+read_cdp_id() {
+  local wanted_id="$1"
+  local line id
+  while IFS= read -r line <&"${CDP[0]}"; do
+    id=$(printf '%s\n' "$line" | jq -r '.id // empty' 2>/dev/null || true)
+    if [ "$id" = "$wanted_id" ]; then
+      printf '%s\n' "$line"
+      return 0
+    fi
+  done
+  return 1
+}
+
+send_cdp 1 "Page.enable"
+read_cdp_id 1 >/dev/null
+
+send_cdp 2 "Emulation.setDeviceMetricsOverride" '{"width":1280,"height":900,"deviceScaleFactor":1,"mobile":false}'
+read_cdp_id 2 >/dev/null
+
+navigate_params=$(jq -cn --arg url "$URL" '{url: $url}')
+send_cdp 3 "Page.navigate" "$navigate_params"
+read_cdp_id 3 >/dev/null
+
+sleep "$WAIT_SECONDS"
+
+send_cdp 4 "Page.captureScreenshot" '{"format":"png","fromSurface":true}'
+screenshot_response=$(read_cdp_id 4)
+screenshot_data=$(printf '%s\n' "$screenshot_response" | jq -r '.result.data // empty')
+
+if [ -z "$screenshot_data" ]; then
+  echo "ERROR: Screenshot response did not include image data" >&2
+  printf '%s\n' "$screenshot_response" >&2
+  exit 1
+fi
+
+printf '%s' "$screenshot_data" | base64 -d > "$OUTPUT"
+echo "Saved screenshot to $OUTPUT"
@@ -0,0 +1,289 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MCP_URL="${MCP_URL:-http://localhost:8000/mcp}"
+CONCURRENT=1
+QUERY="chattybox ai"
+MAX_RESULTS=5
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/test-lightpanda-search [options]
+
+Test Lightpanda MCP by searching Google and extracting top results.
+
+Options:
+  -c, --concurrent N    Number of concurrent searches (default: 1, max: 10)
+  -q, --query TEXT      Search query (default: "chattybox ai")
+  -r, --results N       Number of results to return per worker (default: 5)
+  -h, --help            Show this help message
+
+Examples:
+  scripts/test-lightpanda-search
+  scripts/test-lightpanda-search --concurrent 5
+  scripts/test-lightpanda-search --query "kubernetes helm" --results 3 --concurrent 5
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -c|--concurrent)
+      if [ "$#" -lt 2 ]; then
+        echo "Missing value for $1" >&2
+        usage >&2
+        exit 1
+      fi
+      CONCURRENT="$2"
+      shift 2
+      ;;
+    -q|--query)
+      if [ "$#" -lt 2 ]; then
+        echo "Missing value for $1" >&2
+        usage >&2
+        exit 1
+      fi
+      QUERY="$2"
+      shift 2
+      ;;
+    -r|--results)
+      if [ "$#" -lt 2 ]; then
+        echo "Missing value for $1" >&2
+        usage >&2
+        exit 1
+      fi
+      MAX_RESULTS="$2"
+      shift 2
+      ;;
+    --*)
+      echo "Unknown option: $1" >&2
+      usage >&2
+      exit 1
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+if [ "$CONCURRENT" -lt 1 ]; then CONCURRENT=1; fi
+if [ "$CONCURRENT" -gt 10 ]; then CONCURRENT=10; fi
+
+ENCODED_QUERY=$(jq -rn --arg q "$QUERY" '$q|@uri')
+SEARCH_URL="https://www.bing.com/search?q=$ENCODED_QUERY"
+
+health_url="${MCP_URL%/mcp}"
+if [ -z "$health_url" ]; then health_url="http://localhost:8000"; fi
+if ! curl -sf "$health_url/healthz" >/dev/null 2>&1; then
+  echo "ERROR: Lightpanda MCP not reachable at $health_url" >&2
+  echo "Make sure Tilt is running: nix run .#tilt -- up" >&2
+  exit 1
+fi
+
+# Extract JSON from SSE response body
+extract_json_from_sse() {
+  sed -n 's/^data: //p'
+}
+
+# Extract session ID from response headers file
+extract_session_id() {
+  local headers_file="$1"
+  grep -i '^Mcp-Session-Id:' "$headers_file" 2>/dev/null | awk '{print $2}' | tr -d '\r' || true
+}
+
+search_worker() {
+  local worker_id="$1"
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  local headers_file="$tmpdir/headers.txt"
+
+  echo "[$worker_id] Searching for: $QUERY"
+
+  # Initialize session
+  local init_payload
+  init_payload=$(jq -n \
+    --argjson id "$worker_id" \
+    '{
+      jsonrpc: "2.0",
+      id: $id,
+      method: "initialize",
+      params: {
+        protocolVersion: "2024-11-05",
+        capabilities: {},
+        clientInfo: {
+          name: "search-test-\($id)",
+          version: "1.0"
+        }
+      }
+    }')
+
+  local init_response
+  init_response=$(curl -sf -X POST "$MCP_URL" \
+    -D "$headers_file" \
+    -H "Content-Type: application/json" \
+    -H "Accept: application/json, text/event-stream" \
+    -d "$init_payload" 2>"$tmpdir/curl.err") || {
+    echo "[$worker_id] ERROR: Initialize failed" >&2
+    cat "$tmpdir/curl.err" >&2
+    rm -rf "$tmpdir"
+    return 1
+  }
+
+  local session_id
+  session_id=$(extract_session_id "$headers_file")
+
+  if [ -z "$session_id" ]; then
+    echo "[$worker_id] ERROR: No session ID received" >&2
+    rm -rf "$tmpdir"
+    return 1
+  fi
+
+  # Navigate to search page
+  local goto_payload
+  goto_payload=$(jq -n \
+    --argjson id "$((worker_id + 100))" \
+    --arg url "$SEARCH_URL" \
+    '{
+      jsonrpc: "2.0",
+      id: $id,
+      method: "tools/call",
+      params: {
+        name: "goto",
+        arguments: {
+          url: $url
+        }
+      }
+    }')
+
+  curl -sf -X POST "$MCP_URL" \
+    -H "Mcp-Session-Id: $session_id" \
+    -H "Content-Type: application/json" \
+    -H "Accept: application/json, text/event-stream" \
+    -d "$goto_payload" >/dev/null 2>"$tmpdir/goto.err" || {
+    echo "[$worker_id] WARNING: Goto may have issues" >&2
+    cat "$tmpdir/goto.err" >&2
+  }
+
+  sleep 2
+
+  # Extract results
+  local js_file="$tmpdir/extract.js"
+  cat > "$js_file" <<'JSEOF'
+(() => {
+  function decodeBingUrl(href) {
+    try {
+      const url = new URL(href);
+      const encoded = url.searchParams.get('u');
+      if (!encoded || !encoded.startsWith('a1')) return href;
+      const base64 = encoded.slice(2).replace(/-/g, '+').replace(/_/g, '/');
+      const padded = base64.padEnd(Math.ceil(base64.length / 4) * 4, '=');
+      return atob(padded);
+    } catch (_) {
+      return href;
+    }
+  }
+
+  const results = [];
+  const links = Array.from(document.querySelectorAll('li.b_algo h2 a[href]'));
+  for (const a of links) {
+    if (results.length >= 5) break;
+    const title = a.textContent.replace(/\s+/g, ' ').trim();
+    const href = decodeBingUrl(a.href);
+    if (title.length < 3) continue;
+    if (!href.startsWith('http')) continue;
+    results.push({title, url: href});
+  }
+  return JSON.stringify({
+    url: location.href,
+    title: document.title,
+    resultCount: results.length,
+    results,
+  });
+})()
+JSEOF
+
+  local js
+  js=$(tr '\n' ' ' < "$js_file")
+
+  local eval_payload
+  eval_payload=$(jq -n \
+    --argjson id "$((worker_id + 200))" \
+    --arg script "$js" \
+    '{
+      jsonrpc: "2.0",
+      id: $id,
+      method: "tools/call",
+      params: {
+        name: "evaluate",
+        arguments: {
+          script: $script
+        }
+      }
+    }')
+
+  local eval_response
+  eval_response=$(curl -sf -X POST "$MCP_URL" \
+    -H "Mcp-Session-Id: $session_id" \
+    -H "Content-Type: application/json" \
+    -H "Accept: application/json, text/event-stream" \
+    -d "$eval_payload" 2>"$tmpdir/eval.err") || {
+    echo "[$worker_id] ERROR: Evaluate failed" >&2
+    cat "$tmpdir/eval.err" >&2
+    rm -rf "$tmpdir"
+    return 1
+  }
+
+  local result_text
+  result_text=$(printf '%s\n' "$eval_response" | extract_json_from_sse | jq -r '.result.content[0].text // empty' 2>/dev/null)
+
+  echo "[$worker_id] Results:"
+  if [ -n "$result_text" ]; then
+    local result_count
+    result_count=$(printf '%s\n' "$result_text" | jq -r '.resultCount // 0' 2>/dev/null || printf '0')
+    if [ "$result_count" -gt 0 ]; then
+      printf '%s\n' "$result_text" | jq -r --argjson max "$MAX_RESULTS" '.results[:$max] | .[] | "  \(.title)\n    \(.url)\n"' 2>/dev/null || printf '  Raw: %s\n' "$result_text"
+    else
+      echo "  (No search results found)"
+      printf '%s\n' "$result_text" | jq -r '"  Page: \(.title) at \(.url)"' 2>/dev/null || printf '  Raw: %s\n' "$result_text"
+      rm -rf "$tmpdir"
+      return 1
+    fi
+  else
+    echo "  (No results found)"
+    local parsed
+    parsed=$(printf '%s\n' "$eval_response" | extract_json_from_sse | jq . 2>/dev/null)
+    if [ -n "$parsed" ]; then
+      printf '  Response: %s\n' "$parsed"
+    else
+      printf '  Raw response:\n%s\n' "$eval_response"
+    fi
+  fi
+
+  rm -rf "$tmpdir"
+  echo "[$worker_id] Done"
+}
+
+echo "=== Lightpanda Search Test ==="
+echo "Query: $QUERY"
+echo "URL: $SEARCH_URL"
+echo "Workers: $CONCURRENT"
+echo "Results per worker: $MAX_RESULTS"
+echo ""
+
+pids=()
+for i in $(seq 1 "$CONCURRENT"); do
+  search_worker "$i" &
+  pids+=("$!")
+  sleep 0.5
+done
+
+for pid in "${pids[@]}"; do
+  wait "$pid" || true
+done
+
+echo ""
+echo "=== All workers completed ==="