Add site analyzer service

2026-10-13 06:35:12 +00:00 · 2026-05-04 10:36:34 +01:00
parent 1bebbd6bbd
commit ee1d674713
8 changed files with 832 additions and 9 deletions
@@ -1,6 +1,6 @@
 # OpenStaticFish MicroServices

-Basic local Kubernetes development setup using Nix, Kind, Tilt, Docker, and a small Go hello-world service.
+Basic local Kubernetes development setup using Nix, Kind, Tilt, Docker, and small Go microservices.

 ## Requirements

@@ -37,10 +37,16 @@ Open the Tilt UI:
 http://localhost:10350
 ```

-Test the hello-world service:
+Test the scraper service:

 ```bash
-curl http://localhost:8080
+curl http://localhost:8080/health
+```
+
+Analyze a site once Tilt is running:
+
+```bash
+scripts/analyze-site adaptive.co.uk
 ```

 Stop Tilt resources:
@@ -154,10 +160,10 @@ kubectl get pods
 List running pods.

 ```bash
-kubectl logs -f deployment/hello-world-deployment
+kubectl logs -f deployment/site-analyzer-deployment
 ```

-Stream logs from the hello-world deployment.
+Stream logs from the site analyzer deployment.

 ```bash
 kubectl get svc
@@ -174,10 +180,52 @@ docker ps
 List running containers, including Kind node containers.

 ```bash
-docker build -t hello-world ./services/hello-world
+docker build -t site-analyzer ./services/site-analyzer
 ```

-Build the hello-world image manually outside Tilt.
+Build the site analyzer image manually outside Tilt.
+
+## Site Analyzer
+
+The `site-analyzer` service accepts a URL and returns JSON describing visible site infrastructure and technology signals.
+
+Endpoint:
+
+```text
+POST http://localhost:8090/analyze
+```
+
+Request:
+
+```json
+{
+  "url": "https://example.com"
+}
+```
+
+Wrapper script:
+
+```bash
+scripts/analyze-site https://example.com
+```
+
+Override the endpoint if needed:
+
+```bash
+scripts/analyze-site --endpoint http://localhost:8090/analyze example.com
+```
+
+The response includes:
+
+- `hosting.provider`: the visible edge/provider inferred from DNS, reverse DNS, and HTTP headers
+- `hosting.cdn`: CDN detected from headers, for example Cloudflare, Fastly, CloudFront, or Akamai
+- `hosting.origin_provider`: best-effort origin/platform inference, for example Pantheon when `pantheonsite.io` hints are exposed
+- `hosting.origin_evidence`: specific signals used for origin inference
+- `dns`: nameservers and common DNS records
+- `technologies`: CMS/framework/server signals such as Drupal, WordPress, React, Next.js, or Cloudflare
+- `security`: TLS/HSTS/security-header information
+
+CDNs can hide the real origin. If a site is proxied through Cloudflare, the public DNS and IPs usually identify Cloudflare rather than the origin host. Origin detection is therefore best-effort and depends on leaked signals such as CSP entries, headers, HTML references, or provider-specific domains.

 ## Layout

@@ -186,11 +234,17 @@ Build the hello-world image manually outside Tilt.
 ├── flake.nix
 ├── Tiltfile
 ├── k8s/
-│   └── hello-world.yaml
+│   ├── scraper.yaml
+│   └── site-analyzer.yaml
 ├── scripts/
+│   ├── analyze-site
 │   └── setup-kind
 └── services/
-    └── hello-world/
+    ├── scraper/
+    │   ├── Dockerfile
+    │   ├── go.mod
+    │   └── main.go
+    └── site-analyzer/
        ├── Dockerfile
        ├── go.mod
        └── main.go
@@ -22,8 +22,10 @@ allow_k8s_contexts('kind-openstaticfish')
 # --- Scraper Service ---
 # Builds the Docker image and deploys the manifest with live reload
 docker_build('scraper', './services/scraper')
+docker_build('site-analyzer', './services/site-analyzer')

 k8s_yaml('./k8s/scraper.yaml')
+k8s_yaml('./k8s/site-analyzer.yaml')

 local_resource(
    'webshare-secret',
@@ -39,6 +41,12 @@ k8s_resource(
    labels=['scraper'],
 )

+k8s_resource(
+    workload='site-analyzer-deployment',
+    port_forwards='8090:8090',
+    labels=['site-analyzer'],
+)
+
 # --- Local Resources ---
 # Run a simple health check as a local resource
 local_resource(
@@ -47,3 +55,10 @@ local_resource(
    resource_deps=['scraper-deployment'],
    labels=['utility'],
 )
+
+local_resource(
+    'site-analyzer-health-check',
+    cmd='curl -sf http://localhost:8090/health || echo "Site analyzer not ready"',
+    resource_deps=['site-analyzer-deployment'],
+    labels=['utility'],
+)
@@ -0,0 +1,60 @@
+# --- Site Analyzer Kubernetes Manifests ---
+# Managed by Tilt - run `tilt up` to deploy
+# Tear down with: `tilt down`
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: site-analyzer-deployment
+  labels:
+    app: site-analyzer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: site-analyzer
+  template:
+    metadata:
+      labels:
+        app: site-analyzer
+    spec:
+      containers:
+        - name: site-analyzer
+          image: site-analyzer
+          ports:
+            - containerPort: 8090
+          env:
+            - name: PORT
+              value: "8090"
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "100m"
+            limits:
+              memory: "256Mi"
+              cpu: "500m"
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8090
+            initialDelaySeconds: 2
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8090
+            initialDelaySeconds: 2
+            periodSeconds: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: site-analyzer-service
+spec:
+  selector:
+    app: site-analyzer
+  ports:
+    - protocol: TCP
+      port: 8090
+      targetPort: 8090
+  type: ClusterIP
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+endpoint="${SITE_ANALYZER_ENDPOINT:-http://localhost:8090/analyze}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/analyze-site [--endpoint URL] <site-url>
+
+Examples:
+  scripts/analyze-site https://example.com
+  scripts/analyze-site --endpoint http://localhost:8090/analyze example.com
+  SITE_ANALYZER_ENDPOINT=http://localhost:8090/analyze scripts/analyze-site example.com
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    --endpoint)
+      if [ "$#" -lt 2 ]; then
+        echo "Missing value for --endpoint" >&2
+        usage >&2
+        exit 1
+      fi
+      endpoint="$2"
+      shift 2
+      ;;
+    --endpoint=*)
+      endpoint="${1#--endpoint=}"
+      shift
+      ;;
+    --*)
+      echo "Unknown option: $1" >&2
+      usage >&2
+      exit 1
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+if [ "$#" -ne 1 ]; then
+  usage >&2
+  exit 1
+fi
+
+site_url="$1"
+payload=$(jq -cn --arg url "$site_url" '{url: $url}' 2>/dev/null || printf '{"url":"%s"}' "$site_url")
+
+response=$(curl -fsS \
+  -H 'Content-Type: application/json' \
+  -X POST \
+  --data "$payload" \
+  "$endpoint")
+
+if command -v jq >/dev/null 2>&1; then
+  jq . <<<"$response"
+else
+  printf '%s\n' "$response"
+fi
@@ -0,0 +1,15 @@
+# --- Site Analyzer Service Dockerfile ---
+# Multi-stage build for minimal image size
+
+FROM golang:1.26-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -o /site-analyzer .
+
+FROM alpine:3.19
+RUN apk --no-cache add ca-certificates
+COPY --from=builder /site-analyzer /site-analyzer
+EXPOSE 8090
+ENTRYPOINT ["/site-analyzer"]
@@ -0,0 +1,10 @@
+module github.com/openstaticfish/microservices/site-analyzer
+
+go 1.26
+
+require github.com/PuerkitoBio/goquery v1.9.2
+
+require (
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	golang.org/x/net v0.25.0 // indirect
+)
@@ -0,0 +1,40 @@
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -0,0 +1,563 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type AnalyzeRequest struct {
+	URL string `json:"url"`
+}
+
+type AnalyzeResponse struct {
+	URL          string         `json:"url"`
+	FinalURL     string         `json:"final_url,omitempty"`
+	Domain       string         `json:"domain"`
+	AnalyzedAt   time.Time      `json:"analyzed_at"`
+	Hosting      HostingInfo    `json:"hosting"`
+	DNS          DNSInfo        `json:"dns"`
+	Technologies TechnologyInfo `json:"technologies"`
+	Security     SecurityInfo   `json:"security"`
+	HTTP         HTTPInfo       `json:"http"`
+	Warnings     []string       `json:"warnings,omitempty"`
+}
+
+type HostingInfo struct {
+	Provider       string   `json:"provider,omitempty"`
+	CDN            string   `json:"cdn,omitempty"`
+	OriginProvider string   `json:"origin_provider,omitempty"`
+	OriginEvidence []string `json:"origin_evidence,omitempty"`
+	IPAddresses    []string `json:"ip_addresses,omitempty"`
+	ReverseDNS     []string `json:"reverse_dns,omitempty"`
+}
+
+type DNSInfo struct {
+	Provider    string              `json:"provider,omitempty"`
+	Nameservers []string            `json:"nameservers,omitempty"`
+	Records     map[string][]string `json:"records,omitempty"`
+}
+
+type TechnologyInfo struct {
+	CMS                 string   `json:"cms,omitempty"`
+	Framework           string   `json:"framework,omitempty"`
+	Server              string   `json:"server,omitempty"`
+	ProgrammingLanguage string   `json:"programming_language,omitempty"`
+	Detected            []string `json:"detected,omitempty"`
+}
+
+type SecurityInfo struct {
+	SSL     bool              `json:"ssl"`
+	HSTS    bool              `json:"hsts"`
+	Headers map[string]string `json:"headers,omitempty"`
+}
+
+type HTTPInfo struct {
+	StatusCode int               `json:"status_code,omitempty"`
+	Headers    map[string]string `json:"headers,omitempty"`
+}
+
+type ErrorResponse struct {
+	Error   string `json:"error"`
+	Details string `json:"details,omitempty"`
+}
+
+type fetchedSite struct {
+	FinalURL   string
+	StatusCode int
+	Header     http.Header
+	Body       []byte
+	TLS        bool
+}
+
+func main() {
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "8090"
+	}
+
+	http.HandleFunc("/analyze", handleAnalyze)
+	http.HandleFunc("/health", handleHealth)
+
+	log.Printf("site-analyzer listening on :%s", port)
+	log.Fatal(http.ListenAndServe(":"+port, nil))
+}
+
+func handleHealth(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, "ok")
+}
+
+func handleAnalyze(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		respondWithError(w, http.StatusMethodNotAllowed, "Method not allowed", "Only POST requests are supported")
+		return
+	}
+
+	var req AnalyzeRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		respondWithError(w, http.StatusBadRequest, "Invalid JSON", err.Error())
+		return
+	}
+	if strings.TrimSpace(req.URL) == "" {
+		respondWithError(w, http.StatusBadRequest, "Missing URL", "url field is required")
+		return
+	}
+
+	result, err := analyzeSite(req.URL)
+	if err != nil {
+		respondWithError(w, http.StatusInternalServerError, "Analysis failed", err.Error())
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(result)
+}
+
+func analyzeSite(rawURL string) (*AnalyzeResponse, error) {
+	normalizedURL := normalizeURL(rawURL)
+	parsed, err := url.Parse(normalizedURL)
+	if err != nil || parsed.Hostname() == "" {
+		return nil, fmt.Errorf("invalid url")
+	}
+
+	domain := strings.ToLower(parsed.Hostname())
+	response := &AnalyzeResponse{
+		URL:        normalizedURL,
+		Domain:     domain,
+		AnalyzedAt: time.Now().UTC(),
+		DNS: DNSInfo{
+			Records: make(map[string][]string),
+		},
+		Security: SecurityInfo{
+			Headers: make(map[string]string),
+		},
+		HTTP: HTTPInfo{
+			Headers: make(map[string]string),
+		},
+	}
+
+	var warnings []string
+	if err := analyzeDNS(domain, &response.DNS, &response.Hosting); err != nil {
+		warnings = append(warnings, "dns lookup failed: "+err.Error())
+	}
+
+	fetched, err := fetchSite(normalizedURL)
+	if err != nil {
+		warnings = append(warnings, "http fetch failed: "+err.Error())
+	} else {
+		response.FinalURL = fetched.FinalURL
+		response.HTTP.StatusCode = fetched.StatusCode
+		response.HTTP.Headers = selectedHeaders(fetched.Header, []string{"server", "x-powered-by", "via", "x-generator", "x-vercel-id", "cf-ray", "x-drupal-cache"})
+		response.Security = analyzeSecurity(fetched)
+		response.Technologies = detectTechnologies(fetched)
+		if provider := detectHostingFromHeaders(fetched.Header); provider != "" {
+			response.Hosting.Provider = provider
+		}
+		response.Hosting.CDN = detectCDN(fetched.Header)
+		response.Hosting.OriginProvider, response.Hosting.OriginEvidence = detectOriginProvider(fetched)
+	}
+
+	if response.Hosting.Provider == "" {
+		response.Hosting.Provider = detectHostingFromDNS(response.Hosting.ReverseDNS, response.DNS.Nameservers)
+	}
+	response.Warnings = warnings
+	return response, nil
+}
+
+func analyzeDNS(domain string, dnsInfo *DNSInfo, hosting *HostingInfo) error {
+	var firstErr error
+
+	if ips, err := net.LookupHost(domain); err == nil {
+		hosting.IPAddresses = sortedUnique(ips)
+		dnsInfo.Records["A"] = filterIPs(ips, false)
+		dnsInfo.Records["AAAA"] = filterIPs(ips, true)
+		for _, ip := range ips {
+			names, err := net.LookupAddr(ip)
+			if err == nil {
+				hosting.ReverseDNS = append(hosting.ReverseDNS, names...)
+			}
+		}
+		hosting.ReverseDNS = sortedUnique(hosting.ReverseDNS)
+	} else {
+		firstErr = err
+	}
+
+	if nsRecords, err := net.LookupNS(domain); err == nil {
+		for _, ns := range nsRecords {
+			dnsInfo.Nameservers = append(dnsInfo.Nameservers, strings.TrimSuffix(strings.ToLower(ns.Host), "."))
+		}
+		dnsInfo.Nameservers = sortedUnique(dnsInfo.Nameservers)
+		dnsInfo.Provider = detectDNSProvider(dnsInfo.Nameservers)
+	} else if firstErr == nil {
+		firstErr = err
+	}
+
+	if mxRecords, err := net.LookupMX(domain); err == nil {
+		for _, mx := range mxRecords {
+			dnsInfo.Records["MX"] = append(dnsInfo.Records["MX"], strings.TrimSuffix(strings.ToLower(mx.Host), "."))
+		}
+		dnsInfo.Records["MX"] = sortedUnique(dnsInfo.Records["MX"])
+	}
+
+	for recordType, values := range dnsInfo.Records {
+		if len(values) == 0 {
+			delete(dnsInfo.Records, recordType)
+		}
+	}
+
+	return firstErr
+}
+
+func fetchSite(targetURL string) (*fetchedSite, error) {
+	client := &http.Client{Timeout: 20 * time.Second}
+	req, err := http.NewRequest(http.MethodGet, targetURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
+	if err != nil {
+		return nil, err
+	}
+
+	return &fetchedSite{
+		FinalURL:   resp.Request.URL.String(),
+		StatusCode: resp.StatusCode,
+		Header:     resp.Header,
+		Body:       body,
+		TLS:        resp.TLS != nil,
+	}, nil
+}
+
+func analyzeSecurity(site *fetchedSite) SecurityInfo {
+	securityHeaders := []string{"strict-transport-security", "content-security-policy", "x-frame-options", "x-content-type-options", "referrer-policy", "permissions-policy"}
+	return SecurityInfo{
+		SSL:     site.TLS,
+		HSTS:    site.Header.Get("Strict-Transport-Security") != "",
+		Headers: selectedHeaders(site.Header, securityHeaders),
+	}
+}
+
+func detectTechnologies(site *fetchedSite) TechnologyInfo {
+	detected := map[string]bool{}
+	tech := TechnologyInfo{}
+	lowerHTML := strings.ToLower(string(site.Body))
+
+	if server := site.Header.Get("Server"); server != "" {
+		tech.Server = server
+		detected[server] = true
+	}
+	if poweredBy := site.Header.Get("X-Powered-By"); poweredBy != "" {
+		detected[poweredBy] = true
+		tech.ProgrammingLanguage = detectLanguageFromPoweredBy(poweredBy)
+	}
+
+	if strings.Contains(lowerHTML, "wp-content/") || strings.Contains(lowerHTML, "wp-includes/") || strings.Contains(lowerHTML, "/wp-json/") {
+		tech.CMS = "WordPress"
+		detected["WordPress"] = true
+	}
+	if strings.Contains(lowerHTML, "drupal-settings-json") || strings.Contains(lowerHTML, "/sites/default/") || site.Header.Get("X-Drupal-Cache") != "" {
+		tech.CMS = "Drupal"
+		detected["Drupal"] = true
+	}
+	if strings.Contains(lowerHTML, "content=\"joomla!") || strings.Contains(lowerHTML, "/media/system/js/") {
+		tech.CMS = "Joomla"
+		detected["Joomla"] = true
+	}
+	if strings.Contains(lowerHTML, "/cdn/shop/") || strings.Contains(lowerHTML, "shopify.theme") {
+		tech.CMS = "Shopify"
+		detected["Shopify"] = true
+	}
+
+	if strings.Contains(lowerHTML, "__next_data__") || strings.Contains(lowerHTML, "/_next/static/") || site.Header.Get("X-Nextjs-Cache") != "" {
+		tech.Framework = "Next.js"
+		detected["Next.js"] = true
+		detected["React"] = true
+	} else if strings.Contains(lowerHTML, "data-reactroot") || strings.Contains(lowerHTML, "react-dom") || strings.Contains(lowerHTML, "react.production.min.js") {
+		tech.Framework = "React"
+		detected["React"] = true
+	} else if strings.Contains(lowerHTML, "__nuxt") || strings.Contains(lowerHTML, "/_nuxt/") {
+		tech.Framework = "Nuxt"
+		detected["Nuxt"] = true
+		detected["Vue"] = true
+	} else if strings.Contains(lowerHTML, "vue.js") || strings.Contains(lowerHTML, "data-v-") {
+		tech.Framework = "Vue"
+		detected["Vue"] = true
+	} else if strings.Contains(lowerHTML, "ng-version") || strings.Contains(lowerHTML, "angular.js") {
+		tech.Framework = "Angular"
+		detected["Angular"] = true
+	} else if strings.Contains(lowerHTML, "svelte") || strings.Contains(lowerHTML, "/_app/immutable/") {
+		tech.Framework = "SvelteKit"
+		detected["SvelteKit"] = true
+	}
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(site.Body)))
+	if err == nil {
+		doc.Find("meta[name='generator']").Each(func(_ int, s *goquery.Selection) {
+			content := strings.TrimSpace(s.AttrOr("content", ""))
+			if content != "" {
+				detected[content] = true
+				if tech.CMS == "" {
+					tech.CMS = detectCMSFromGenerator(content)
+				}
+			}
+		})
+	}
+
+	for _, header := range []string{"CF-Ray", "X-Vercel-ID", "X-GitHub-Request-Id", "Fly-Request-Id"} {
+		if site.Header.Get(header) != "" {
+			detected[detectTechnologyFromHeader(header)] = true
+		}
+	}
+
+	tech.Detected = mapKeys(detected)
+	return tech
+}
+
+func detectDNSProvider(nameservers []string) string {
+	return firstPatternMatch(nameservers, map[string]string{
+		"cloudflare.com":        "Cloudflare",
+		"awsdns":                "Amazon Route 53",
+		"domaincontrol.com":     "GoDaddy",
+		"googledomains.com":     "Google Domains",
+		"google.com":            "Google Cloud DNS",
+		"dnsimple.com":          "DNSimple",
+		"digitalocean.com":      "DigitalOcean",
+		"dnsmadeeasy.com":       "DNS Made Easy",
+		"namecheap.com":         "Namecheap",
+		"registrar-servers.com": "Namecheap",
+		"nsone.net":             "NS1",
+		"azure-dns":             "Azure DNS",
+		"linode.com":            "Linode",
+		"he.net":                "Hurricane Electric",
+	})
+}
+
+func detectHostingFromHeaders(header http.Header) string {
+	switch {
+	case header.Get("CF-Ray") != "":
+		return "Cloudflare"
+	case header.Get("X-Vercel-ID") != "":
+		return "Vercel"
+	case header.Get("X-GitHub-Request-Id") != "":
+		return "GitHub Pages"
+	case header.Get("Fly-Request-Id") != "":
+		return "Fly.io"
+	case strings.Contains(strings.ToLower(header.Get("Server")), "netlify"):
+		return "Netlify"
+	default:
+		return ""
+	}
+}
+
+func detectCDN(header http.Header) string {
+	switch {
+	case header.Get("CF-Ray") != "" || strings.EqualFold(header.Get("Server"), "cloudflare"):
+		return "Cloudflare"
+	case header.Get("X-Served-By") != "" && strings.Contains(strings.ToLower(header.Get("Via")), "varnish"):
+		return "Fastly"
+	case header.Get("X-Cache") != "" && strings.Contains(strings.ToLower(header.Get("Via")), "cloudfront"):
+		return "Amazon CloudFront"
+	case header.Get("X-Akamai-Transformed") != "":
+		return "Akamai"
+	default:
+		return ""
+	}
+}
+
+func detectOriginProvider(site *fetchedSite) (string, []string) {
+	lowerHTML := strings.ToLower(string(site.Body))
+	lowerCSP := strings.ToLower(site.Header.Get("Content-Security-Policy"))
+	lowerHeaders := strings.ToLower(flattenHeaders(site.Header))
+
+	if strings.Contains(lowerHTML, "pantheonsite.io") || strings.Contains(lowerCSP, "pantheonsite.io") || strings.Contains(lowerHeaders, "pantheon") {
+		var evidence []string
+		if strings.Contains(lowerHTML, "pantheonsite.io") {
+			evidence = append(evidence, "html references pantheonsite.io")
+		}
+		if strings.Contains(lowerCSP, "pantheonsite.io") {
+			evidence = append(evidence, "content-security-policy allows pantheonsite.io")
+		}
+		if strings.Contains(lowerHeaders, "pantheon") {
+			evidence = append(evidence, "response headers mention pantheon")
+		}
+		if strings.Contains(strings.ToLower(site.Header.Get("Via")), "varnish") {
+			evidence = append(evidence, "via header includes varnish")
+		}
+		return "Pantheon", evidence
+	}
+
+	return "", nil
+}
+
+func detectHostingFromDNS(reverseDNS []string, nameservers []string) string {
+	provider := firstPatternMatch(reverseDNS, map[string]string{
+		"amazonaws.com":         "AWS",
+		"googleusercontent.com": "Google Cloud",
+		"azure.com":             "Azure",
+		"cloudapp.net":          "Azure",
+		"digitalocean.com":      "DigitalOcean",
+		"linode.com":            "Linode",
+		"ovh.net":               "OVHcloud",
+		"hetzner":               "Hetzner",
+		"github.io":             "GitHub Pages",
+		"netlify":               "Netlify",
+		"vercel":                "Vercel",
+	})
+	if provider != "" {
+		return provider
+	}
+	return firstPatternMatch(nameservers, map[string]string{"cloudflare.com": "Cloudflare"})
+}
+
+func detectLanguageFromPoweredBy(value string) string {
+	lower := strings.ToLower(value)
+	switch {
+	case strings.Contains(lower, "php"):
+		return "PHP"
+	case strings.Contains(lower, "express"):
+		return "Node.js"
+	case strings.Contains(lower, "next.js"):
+		return "Node.js"
+	case strings.Contains(lower, "asp.net"):
+		return ".NET"
+	default:
+		return ""
+	}
+}
+
+func detectCMSFromGenerator(value string) string {
+	lower := strings.ToLower(value)
+	switch {
+	case strings.Contains(lower, "wordpress"):
+		return "WordPress"
+	case strings.Contains(lower, "drupal"):
+		return "Drupal"
+	case strings.Contains(lower, "joomla"):
+		return "Joomla"
+	case strings.Contains(lower, "shopify"):
+		return "Shopify"
+	default:
+		return ""
+	}
+}
+
+func detectTechnologyFromHeader(header string) string {
+	switch header {
+	case "CF-Ray":
+		return "Cloudflare"
+	case "X-Vercel-ID":
+		return "Vercel"
+	case "X-GitHub-Request-Id":
+		return "GitHub Pages"
+	case "Fly-Request-Id":
+		return "Fly.io"
+	default:
+		return header
+	}
+}
+
+func firstPatternMatch(values []string, patterns map[string]string) string {
+	for _, value := range values {
+		lower := strings.ToLower(value)
+		for pattern, provider := range patterns {
+			if strings.Contains(lower, pattern) {
+				return provider
+			}
+		}
+	}
+	return ""
+}
+
+func selectedHeaders(header http.Header, names []string) map[string]string {
+	selected := make(map[string]string)
+	for _, name := range names {
+		if value := header.Get(name); value != "" {
+			selected[strings.ToLower(name)] = value
+		}
+	}
+	return selected
+}
+
+func flattenHeaders(header http.Header) string {
+	var values []string
+	for name, headerValues := range header {
+		values = append(values, name)
+		values = append(values, headerValues...)
+	}
+	return strings.Join(values, " ")
+}
+
+func filterIPs(values []string, wantIPv6 bool) []string {
+	var ips []string
+	for _, value := range values {
+		ip := net.ParseIP(value)
+		if ip == nil {
+			continue
+		}
+		isIPv6 := ip.To4() == nil
+		if isIPv6 == wantIPv6 {
+			ips = append(ips, value)
+		}
+	}
+	return sortedUnique(ips)
+}
+
+func normalizeURL(rawURL string) string {
+	rawURL = strings.TrimSpace(rawURL)
+	if strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://") {
+		return rawURL
+	}
+	return "https://" + rawURL
+}
+
+func sortedUnique(values []string) []string {
+	seen := make(map[string]bool)
+	var result []string
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" || seen[value] {
+			continue
+		}
+		seen[value] = true
+		result = append(result, value)
+	}
+	sort.Strings(result)
+	return result
+}
+
+func mapKeys(values map[string]bool) []string {
+	keys := make([]string, 0, len(values))
+	for value := range values {
+		if value != "" {
+			keys = append(keys, value)
+		}
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func respondWithError(w http.ResponseWriter, status int, err string, details string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(ErrorResponse{Error: err, Details: details})
+}