Integrate Doppler secrets

README.md

@@ -16,6 +16,7 @@ All project CLI dependencies are provided by `flake.nix`, including:
 - `docker`
 - `helm`
 - `go`
+- `doppler`
 
 ## Quick Start
 
@@ -71,6 +72,7 @@ tilt up
 tilt down
 kubectl get pods
 docker ps
+doppler secrets
 ```
 
 ## Project Commands
@@ -227,11 +229,31 @@ The response includes:
 
 CDNs can hide the real origin. If a site is proxied through Cloudflare, the public DNS and IPs usually identify Cloudflare rather than the origin host. Origin detection is therefore best-effort and depends on leaked signals such as CSP entries, headers, HTML references, or provider-specific domains.
 
+## Secrets
+
+Runtime secrets are managed with Doppler in the `openstaticfish-microservices` project using the `dev` config.
+
+Initial setup after cloning:
+
+```bash
+doppler login
+doppler setup --no-interactive
+```
+
+Run commands with secrets injected:
+
+```bash
+doppler run -- your-command
+```
+
+Tilt uses Doppler to create the local Kubernetes `webshare-api` secret for the scraper service. Do not commit local `.env` files; `.env` and `.env.*` are ignored by Git.
+
 ## Layout
 
 ```text
 .
 ├── flake.nix
+├── doppler.yaml
 ├── Tiltfile
 ├── k8s/
 │   ├── scraper.yaml

Tiltfile

@@ -29,7 +29,7 @@ k8s_yaml('./k8s/site-analyzer.yaml')
 
 local_resource(
     'webshare-secret',
-    cmd='set -a; [ ! -f .env ] || . ./.env; set +a; if [ -n "$WEBSHARE_API_KEY" ] && [ -n "$WEBSHARE_PROXY_USERNAME" ] && [ -n "$WEBSHARE_PROXY_PASSWORD" ]; then kubectl create secret generic webshare-api --from-literal=api-key="$WEBSHARE_API_KEY" --from-literal=proxy-username="$WEBSHARE_PROXY_USERNAME" --from-literal=proxy-password="$WEBSHARE_PROXY_PASSWORD" --dry-run=client -o yaml | kubectl apply -f -; else echo "Webshare runtime config missing; country scraping unavailable"; fi',
+    cmd='doppler run -- bash ./scripts/apply-webshare-secret',
     labels=['utility'],
 )
 

doppler.yaml

@@ -0,0 +1,3 @@
+setup:
+  - project: openstaticfish-microservices
+    config: dev

flake.nix

@@ -36,6 +36,9 @@
             docker
             docker-compose
 
+            # --- Secrets Management ---
+            doppler
+
             # --- Language Toolchain ---
             go
 
@@ -46,12 +49,6 @@
           ];
 
           shellHook = ''
-            if [ -f .env ]; then
-              set -a
-              . ./.env
-              set +a
-            fi
-
             if [ "''${OPENSTATICFISH_QUIET_SHELL:-}" != "1" ]; then
               echo "🐟 OpenStaticFish MicroServices Dev Shell"
               echo "-----------------------------------------"
@@ -101,6 +98,7 @@
               echo "  tilt down      - Stop dev environment"
               echo "  kubectl ...    - Interact with k8s cluster"
               echo "  docker ...     - Interact with containers"
+              echo "  doppler ...    - Manage project secrets"
               echo ""
             fi
           '';

scripts/apply-webshare-secret

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -n "${WEBSHARE_API_KEY:-}" ] && [ -n "${WEBSHARE_PROXY_USERNAME:-}" ] && [ -n "${WEBSHARE_PROXY_PASSWORD:-}" ]; then
+  kubectl create secret generic webshare-api \
+    --from-literal=api-key="$WEBSHARE_API_KEY" \
+    --from-literal=proxy-username="$WEBSHARE_PROXY_USERNAME" \
+    --from-literal=proxy-password="$WEBSHARE_PROXY_PASSWORD" \
+    --dry-run=client \
+    -o yaml | kubectl apply -f -
+else
+  echo "Webshare runtime config missing; country scraping unavailable"
+fi

services/site-analyzer/main.go

@@ -193,12 +193,16 @@ func analyzeDNS(domain string, dnsInfo *DNSInfo, hosting *HostingInfo) error {
 		firstErr = err
 	}
 
-	if nsRecords, err := net.LookupNS(domain); err == nil {
+	nsDomain, nsRecords, err := lookupNameservers(domain)
+	if err == nil {
 		for _, ns := range nsRecords {
 			dnsInfo.Nameservers = append(dnsInfo.Nameservers, strings.TrimSuffix(strings.ToLower(ns.Host), "."))
 		}
 		dnsInfo.Nameservers = sortedUnique(dnsInfo.Nameservers)
 		dnsInfo.Provider = detectDNSProvider(dnsInfo.Nameservers)
+		if nsDomain != domain {
+			dnsInfo.Records["NS_DOMAIN"] = []string{nsDomain}
+		}
 	} else if firstErr == nil {
 		firstErr = err
 	}
@@ -219,6 +223,20 @@ func analyzeDNS(domain string, dnsInfo *DNSInfo, hosting *HostingInfo) error {
 	return firstErr
 }
 
+func lookupNameservers(domain string) (string, []*net.NS, error) {
+	labels := strings.Split(domain, ".")
+	var lastErr error
+	for i := 0; i <= len(labels)-2; i++ {
+		candidate := strings.Join(labels[i:], ".")
+		records, err := net.LookupNS(candidate)
+		if err == nil && len(records) > 0 {
+			return candidate, records, nil
+		}
+		lastErr = err
+	}
+	return "", nil, lastErr
+}
+
 func fetchSite(targetURL string) (*fetchedSite, error) {
 	client := &http.Client{Timeout: 20 * time.Second}
 	req, err := http.NewRequest(http.MethodGet, targetURL, nil)
@@ -287,8 +305,19 @@ func detectTechnologies(site *fetchedSite) TechnologyInfo {
 		tech.CMS = "Shopify"
 		detected["Shopify"] = true
 	}
+	if strings.Contains(lowerHTML, "/umbraco/") || strings.Contains(lowerHTML, "umb-client") || strings.Contains(lowerHTML, "umbracoforms") {
+		tech.CMS = "Umbraco"
+		tech.ProgrammingLanguage = ".NET"
+		detected["Umbraco"] = true
+		detected[".NET"] = true
+	}
 
-	if strings.Contains(lowerHTML, "__next_data__") || strings.Contains(lowerHTML, "/_next/static/") || site.Header.Get("X-Nextjs-Cache") != "" {
+	if strings.Contains(lowerHTML, "<!--blazor:") || strings.Contains(lowerHTML, "_framework/blazor") || strings.Contains(lowerHTML, "blazor.server.js") {
+		tech.Framework = "Blazor"
+		tech.ProgrammingLanguage = ".NET"
+		detected["Blazor"] = true
+		detected[".NET"] = true
+	} else if strings.Contains(lowerHTML, "__next_data__") || strings.Contains(lowerHTML, "/_next/static/") || site.Header.Get("X-Nextjs-Cache") != "" {
 		tech.Framework = "Next.js"
 		detected["Next.js"] = true
 		detected["React"] = true
@@ -309,6 +338,9 @@ func detectTechnologies(site *fetchedSite) TechnologyInfo {
 		tech.Framework = "SvelteKit"
 		detected["SvelteKit"] = true
 	}
+	if strings.Contains(lowerHTML, "_content/mudblazor/") || strings.Contains(lowerHTML, "mudblazor") {
+		detected["MudBlazor"] = true
+	}
 
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(site.Body)))
 	if err == nil {
@@ -323,7 +355,7 @@ func detectTechnologies(site *fetchedSite) TechnologyInfo {
 		})
 	}
 
-	for _, header := range []string{"CF-Ray", "X-Vercel-ID", "X-GitHub-Request-Id", "Fly-Request-Id"} {
+	for _, header := range []string{"CF-Ray", "X-Vercel-ID", "X-GitHub-Request-Id", "Fly-Request-Id", "X-Coolify"} {
 		if site.Header.Get(header) != "" {
 			detected[detectTechnologyFromHeader(header)] = true
 		}
@@ -405,6 +437,16 @@ func detectOriginProvider(site *fetchedSite) (string, []string) {
 		}
 		return "Pantheon", evidence
 	}
+	if strings.Contains(lowerHTML, "coolify") || strings.Contains(lowerHeaders, "coolify") {
+		var evidence []string
+		if strings.Contains(lowerHTML, "coolify") {
+			evidence = append(evidence, "html mentions coolify")
+		}
+		if strings.Contains(lowerHeaders, "coolify") {
+			evidence = append(evidence, "response headers mention coolify")
+		}
+		return "Coolify", evidence
+	}
 
 	return "", nil
 }
@@ -471,6 +513,8 @@ func detectTechnologyFromHeader(header string) string {
 		return "GitHub Pages"
 	case "Fly-Request-Id":
 		return "Fly.io"
+	case "X-Coolify":
+		return "Coolify"
 	default:
 		return header
 	}