OpenStaticFish/MicroServices
1
0
Fork 0
mirror of https://github.com/OpenStaticFish/MicroServices.git synced 2026-10-13 06:35:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add site analyzer service

Browse Source
This commit is contained in:
micqdf
2026-05-04 10:36:34 +01:00
parent 1bebbd6bbd
commit ee1d674713
8 changed files with 832 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/site-analyzer/Dockerfile
+15
View File
@@ -0,0 +1,15 @@
# --- Site Analyzer Service Dockerfile ---
# Multi-stage build for minimal image size
FROM golang:1.26-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o /site-analyzer .
FROM alpine:3.19
RUN apk --no-cache add ca-certificates
COPY --from=builder /site-analyzer /site-analyzer
EXPOSE 8090
ENTRYPOINT ["/site-analyzer"]
services/site-analyzer/go.mod
+10
View File
@@ -0,0 +1,10 @@
module github.com/openstaticfish/microservices/site-analyzer
go 1.26
require github.com/PuerkitoBio/goquery v1.9.2
require (
github.com/andybalholm/cascadia v1.3.2 // indirect
golang.org/x/net v0.25.0 // indirect
)
services/site-analyzer/go.sum
+40
View File
@@ -0,0 +1,40 @@
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
services/site-analyzer/main.go
+563
View File
@@ -0,0 +1,563 @@
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
"sort"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
type AnalyzeRequest struct {
URL string `json:"url"`
}
type AnalyzeResponse struct {
URL string `json:"url"`
FinalURL string `json:"final_url,omitempty"`
Domain string `json:"domain"`
AnalyzedAt time.Time `json:"analyzed_at"`
Hosting HostingInfo `json:"hosting"`
DNS DNSInfo `json:"dns"`
Technologies TechnologyInfo `json:"technologies"`
Security SecurityInfo `json:"security"`
HTTP HTTPInfo `json:"http"`
Warnings []string `json:"warnings,omitempty"`
}
type HostingInfo struct {
Provider string `json:"provider,omitempty"`
CDN string `json:"cdn,omitempty"`
OriginProvider string `json:"origin_provider,omitempty"`
OriginEvidence []string `json:"origin_evidence,omitempty"`
IPAddresses []string `json:"ip_addresses,omitempty"`
ReverseDNS []string `json:"reverse_dns,omitempty"`
}
type DNSInfo struct {
Provider string `json:"provider,omitempty"`
Nameservers []string `json:"nameservers,omitempty"`
Records map[string][]string `json:"records,omitempty"`
}
type TechnologyInfo struct {
CMS string `json:"cms,omitempty"`
Framework string `json:"framework,omitempty"`
Server string `json:"server,omitempty"`
ProgrammingLanguage string `json:"programming_language,omitempty"`
Detected []string `json:"detected,omitempty"`
}
type SecurityInfo struct {
SSL bool `json:"ssl"`
HSTS bool `json:"hsts"`
Headers map[string]string `json:"headers,omitempty"`
}
type HTTPInfo struct {
StatusCode int `json:"status_code,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
}
type ErrorResponse struct {
Error string `json:"error"`
Details string `json:"details,omitempty"`
}
type fetchedSite struct {
FinalURL string
StatusCode int
Header http.Header
Body []byte
TLS bool
}
func main() {
port := os.Getenv("PORT")
if port == "" {
port = "8090"
}
http.HandleFunc("/analyze", handleAnalyze)
http.HandleFunc("/health", handleHealth)
log.Printf("site-analyzer listening on :%s", port)
log.Fatal(http.ListenAndServe(":"+port, nil))
}
func handleHealth(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, "ok")
}
func handleAnalyze(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
respondWithError(w, http.StatusMethodNotAllowed, "Method not allowed", "Only POST requests are supported")
return
}
var req AnalyzeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
respondWithError(w, http.StatusBadRequest, "Invalid JSON", err.Error())
return
}
if strings.TrimSpace(req.URL) == "" {
respondWithError(w, http.StatusBadRequest, "Missing URL", "url field is required")
return
}
result, err := analyzeSite(req.URL)
if err != nil {
respondWithError(w, http.StatusInternalServerError, "Analysis failed", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(result)
}
func analyzeSite(rawURL string) (*AnalyzeResponse, error) {
normalizedURL := normalizeURL(rawURL)
parsed, err := url.Parse(normalizedURL)
if err != nil || parsed.Hostname() == "" {
return nil, fmt.Errorf("invalid url")
}
domain := strings.ToLower(parsed.Hostname())
response := &AnalyzeResponse{
URL: normalizedURL,
Domain: domain,
AnalyzedAt: time.Now().UTC(),
DNS: DNSInfo{
Records: make(map[string][]string),
},
Security: SecurityInfo{
Headers: make(map[string]string),
},
HTTP: HTTPInfo{
Headers: make(map[string]string),
},
}
var warnings []string
if err := analyzeDNS(domain, &response.DNS, &response.Hosting); err != nil {
warnings = append(warnings, "dns lookup failed: "+err.Error())
}
fetched, err := fetchSite(normalizedURL)
if err != nil {
warnings = append(warnings, "http fetch failed: "+err.Error())
} else {
response.FinalURL = fetched.FinalURL
response.HTTP.StatusCode = fetched.StatusCode
response.HTTP.Headers = selectedHeaders(fetched.Header, []string{"server", "x-powered-by", "via", "x-generator", "x-vercel-id", "cf-ray", "x-drupal-cache"})
response.Security = analyzeSecurity(fetched)
response.Technologies = detectTechnologies(fetched)
if provider := detectHostingFromHeaders(fetched.Header); provider != "" {
response.Hosting.Provider = provider
}
response.Hosting.CDN = detectCDN(fetched.Header)
response.Hosting.OriginProvider, response.Hosting.OriginEvidence = detectOriginProvider(fetched)
}
if response.Hosting.Provider == "" {
response.Hosting.Provider = detectHostingFromDNS(response.Hosting.ReverseDNS, response.DNS.Nameservers)
}
response.Warnings = warnings
return response, nil
}
func analyzeDNS(domain string, dnsInfo *DNSInfo, hosting *HostingInfo) error {
var firstErr error
if ips, err := net.LookupHost(domain); err == nil {
hosting.IPAddresses = sortedUnique(ips)
dnsInfo.Records["A"] = filterIPs(ips, false)
dnsInfo.Records["AAAA"] = filterIPs(ips, true)
for _, ip := range ips {
names, err := net.LookupAddr(ip)
if err == nil {
hosting.ReverseDNS = append(hosting.ReverseDNS, names...)
}
}
hosting.ReverseDNS = sortedUnique(hosting.ReverseDNS)
} else {
firstErr = err
}
if nsRecords, err := net.LookupNS(domain); err == nil {
for _, ns := range nsRecords {
dnsInfo.Nameservers = append(dnsInfo.Nameservers, strings.TrimSuffix(strings.ToLower(ns.Host), "."))
}
dnsInfo.Nameservers = sortedUnique(dnsInfo.Nameservers)
dnsInfo.Provider = detectDNSProvider(dnsInfo.Nameservers)
} else if firstErr == nil {
firstErr = err
}
if mxRecords, err := net.LookupMX(domain); err == nil {
for _, mx := range mxRecords {
dnsInfo.Records["MX"] = append(dnsInfo.Records["MX"], strings.TrimSuffix(strings.ToLower(mx.Host), "."))
}
dnsInfo.Records["MX"] = sortedUnique(dnsInfo.Records["MX"])
}
for recordType, values := range dnsInfo.Records {
if len(values) == 0 {
delete(dnsInfo.Records, recordType)
}
}
return firstErr
}
func fetchSite(targetURL string) (*fetchedSite, error) {
client := &http.Client{Timeout: 20 * time.Second}
req, err := http.NewRequest(http.MethodGet, targetURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
if err != nil {
return nil, err
}
return &fetchedSite{
FinalURL: resp.Request.URL.String(),
StatusCode: resp.StatusCode,
Header: resp.Header,
Body: body,
TLS: resp.TLS != nil,
}, nil
}
func analyzeSecurity(site *fetchedSite) SecurityInfo {
securityHeaders := []string{"strict-transport-security", "content-security-policy", "x-frame-options", "x-content-type-options", "referrer-policy", "permissions-policy"}
return SecurityInfo{
SSL: site.TLS,
HSTS: site.Header.Get("Strict-Transport-Security") != "",
Headers: selectedHeaders(site.Header, securityHeaders),
}
}
func detectTechnologies(site *fetchedSite) TechnologyInfo {
detected := map[string]bool{}
tech := TechnologyInfo{}
lowerHTML := strings.ToLower(string(site.Body))
if server := site.Header.Get("Server"); server != "" {
tech.Server = server
detected[server] = true
}
if poweredBy := site.Header.Get("X-Powered-By"); poweredBy != "" {
detected[poweredBy] = true
tech.ProgrammingLanguage = detectLanguageFromPoweredBy(poweredBy)
}
if strings.Contains(lowerHTML, "wp-content/") || strings.Contains(lowerHTML, "wp-includes/") || strings.Contains(lowerHTML, "/wp-json/") {
tech.CMS = "WordPress"
detected["WordPress"] = true
}
if strings.Contains(lowerHTML, "drupal-settings-json") || strings.Contains(lowerHTML, "/sites/default/") || site.Header.Get("X-Drupal-Cache") != "" {
tech.CMS = "Drupal"
detected["Drupal"] = true
}
if strings.Contains(lowerHTML, "content=\"joomla!") || strings.Contains(lowerHTML, "/media/system/js/") {
tech.CMS = "Joomla"
detected["Joomla"] = true
}
if strings.Contains(lowerHTML, "/cdn/shop/") || strings.Contains(lowerHTML, "shopify.theme") {
tech.CMS = "Shopify"
detected["Shopify"] = true
}
if strings.Contains(lowerHTML, "__next_data__") || strings.Contains(lowerHTML, "/_next/static/") || site.Header.Get("X-Nextjs-Cache") != "" {
tech.Framework = "Next.js"
detected["Next.js"] = true
detected["React"] = true
} else if strings.Contains(lowerHTML, "data-reactroot") || strings.Contains(lowerHTML, "react-dom") || strings.Contains(lowerHTML, "react.production.min.js") {
tech.Framework = "React"
detected["React"] = true
} else if strings.Contains(lowerHTML, "__nuxt") || strings.Contains(lowerHTML, "/_nuxt/") {
tech.Framework = "Nuxt"
detected["Nuxt"] = true
detected["Vue"] = true
} else if strings.Contains(lowerHTML, "vue.js") || strings.Contains(lowerHTML, "data-v-") {
tech.Framework = "Vue"
detected["Vue"] = true
} else if strings.Contains(lowerHTML, "ng-version") || strings.Contains(lowerHTML, "angular.js") {
tech.Framework = "Angular"
detected["Angular"] = true
} else if strings.Contains(lowerHTML, "svelte") || strings.Contains(lowerHTML, "/_app/immutable/") {
tech.Framework = "SvelteKit"
detected["SvelteKit"] = true
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(site.Body)))
if err == nil {
doc.Find("meta[name='generator']").Each(func(_ int, s *goquery.Selection) {
content := strings.TrimSpace(s.AttrOr("content", ""))
if content != "" {
detected[content] = true
if tech.CMS == "" {
tech.CMS = detectCMSFromGenerator(content)
}
}
})
}
for _, header := range []string{"CF-Ray", "X-Vercel-ID", "X-GitHub-Request-Id", "Fly-Request-Id"} {
if site.Header.Get(header) != "" {
detected[detectTechnologyFromHeader(header)] = true
}
}
tech.Detected = mapKeys(detected)
return tech
}
func detectDNSProvider(nameservers []string) string {
return firstPatternMatch(nameservers, map[string]string{
"cloudflare.com": "Cloudflare",
"awsdns": "Amazon Route 53",
"domaincontrol.com": "GoDaddy",
"googledomains.com": "Google Domains",
"google.com": "Google Cloud DNS",
"dnsimple.com": "DNSimple",
"digitalocean.com": "DigitalOcean",
"dnsmadeeasy.com": "DNS Made Easy",
"namecheap.com": "Namecheap",
"registrar-servers.com": "Namecheap",
"nsone.net": "NS1",
"azure-dns": "Azure DNS",
"linode.com": "Linode",
"he.net": "Hurricane Electric",
})
}
func detectHostingFromHeaders(header http.Header) string {
switch {
case header.Get("CF-Ray") != "":
return "Cloudflare"
case header.Get("X-Vercel-ID") != "":
return "Vercel"
case header.Get("X-GitHub-Request-Id") != "":
return "GitHub Pages"
case header.Get("Fly-Request-Id") != "":
return "Fly.io"
case strings.Contains(strings.ToLower(header.Get("Server")), "netlify"):
return "Netlify"
default:
return ""
}
}
func detectCDN(header http.Header) string {
switch {
case header.Get("CF-Ray") != "" || strings.EqualFold(header.Get("Server"), "cloudflare"):
return "Cloudflare"
case header.Get("X-Served-By") != "" && strings.Contains(strings.ToLower(header.Get("Via")), "varnish"):
return "Fastly"
case header.Get("X-Cache") != "" && strings.Contains(strings.ToLower(header.Get("Via")), "cloudfront"):
return "Amazon CloudFront"
case header.Get("X-Akamai-Transformed") != "":
return "Akamai"
default:
return ""
}
}
func detectOriginProvider(site *fetchedSite) (string, []string) {
lowerHTML := strings.ToLower(string(site.Body))
lowerCSP := strings.ToLower(site.Header.Get("Content-Security-Policy"))
lowerHeaders := strings.ToLower(flattenHeaders(site.Header))
if strings.Contains(lowerHTML, "pantheonsite.io") || strings.Contains(lowerCSP, "pantheonsite.io") || strings.Contains(lowerHeaders, "pantheon") {
var evidence []string
if strings.Contains(lowerHTML, "pantheonsite.io") {
evidence = append(evidence, "html references pantheonsite.io")
}
if strings.Contains(lowerCSP, "pantheonsite.io") {
evidence = append(evidence, "content-security-policy allows pantheonsite.io")
}
if strings.Contains(lowerHeaders, "pantheon") {
evidence = append(evidence, "response headers mention pantheon")
}
if strings.Contains(strings.ToLower(site.Header.Get("Via")), "varnish") {
evidence = append(evidence, "via header includes varnish")
}
return "Pantheon", evidence
}
return "", nil
}
func detectHostingFromDNS(reverseDNS []string, nameservers []string) string {
provider := firstPatternMatch(reverseDNS, map[string]string{
"amazonaws.com": "AWS",
"googleusercontent.com": "Google Cloud",
"azure.com": "Azure",
"cloudapp.net": "Azure",
"digitalocean.com": "DigitalOcean",
"linode.com": "Linode",
"ovh.net": "OVHcloud",
"hetzner": "Hetzner",
"github.io": "GitHub Pages",
"netlify": "Netlify",
"vercel": "Vercel",
})
if provider != "" {
return provider
}
return firstPatternMatch(nameservers, map[string]string{"cloudflare.com": "Cloudflare"})
}
func detectLanguageFromPoweredBy(value string) string {
lower := strings.ToLower(value)
switch {
case strings.Contains(lower, "php"):
return "PHP"
case strings.Contains(lower, "express"):
return "Node.js"
case strings.Contains(lower, "next.js"):
return "Node.js"
case strings.Contains(lower, "asp.net"):
return ".NET"
default:
return ""
}
}
func detectCMSFromGenerator(value string) string {
lower := strings.ToLower(value)
switch {
case strings.Contains(lower, "wordpress"):
return "WordPress"
case strings.Contains(lower, "drupal"):
return "Drupal"
case strings.Contains(lower, "joomla"):
return "Joomla"
case strings.Contains(lower, "shopify"):
return "Shopify"
default:
return ""
}
}
func detectTechnologyFromHeader(header string) string {
switch header {
case "CF-Ray":
return "Cloudflare"
case "X-Vercel-ID":
return "Vercel"
case "X-GitHub-Request-Id":
return "GitHub Pages"
case "Fly-Request-Id":
return "Fly.io"
default:
return header
}
}
func firstPatternMatch(values []string, patterns map[string]string) string {
for _, value := range values {
lower := strings.ToLower(value)
for pattern, provider := range patterns {
if strings.Contains(lower, pattern) {
return provider
}
}
}
return ""
}
func selectedHeaders(header http.Header, names []string) map[string]string {
selected := make(map[string]string)
for _, name := range names {
if value := header.Get(name); value != "" {
selected[strings.ToLower(name)] = value
}
}
return selected
}
func flattenHeaders(header http.Header) string {
var values []string
for name, headerValues := range header {
values = append(values, name)
values = append(values, headerValues...)
}
return strings.Join(values, " ")
}
func filterIPs(values []string, wantIPv6 bool) []string {
var ips []string
for _, value := range values {
ip := net.ParseIP(value)
if ip == nil {
continue
}
isIPv6 := ip.To4() == nil
if isIPv6 == wantIPv6 {
ips = append(ips, value)
}
}
return sortedUnique(ips)
}
func normalizeURL(rawURL string) string {
rawURL = strings.TrimSpace(rawURL)
if strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://") {
return rawURL
}
return "https://" + rawURL
}
func sortedUnique(values []string) []string {
seen := make(map[string]bool)
var result []string
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" || seen[value] {
continue
}
seen[value] = true
result = append(result, value)
}
sort.Strings(result)
return result
}
func mapKeys(values map[string]bool) []string {
keys := make([]string, 0, len(values))
for value := range values {
if value != "" {
keys = append(keys, value)
}
}
sort.Strings(keys)
return keys
}
func respondWithError(w http.ResponseWriter, status int, err string, details string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(ErrorResponse{Error: err, Details: details})
}