OpenStaticFish/MicroServices
1
0
Fork 0
mirror of https://github.com/OpenStaticFish/MicroServices.git synced 2026-10-13 06:35:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Route country scrapes through Webshare

Browse Source
This commit is contained in:
micqdf
2026-05-04 09:42:24 +01:00
parent cef13cfbc4
commit 1bebbd6bbd
6 changed files with 303 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3
View File
@@ -0,0 +1,3 @@
WEBSHARE_API_KEY=
WEBSHARE_PROXY_USERNAME=
WEBSHARE_PROXY_PASSWORD=
Tiltfile
+7
View File
@@ -25,10 +25,17 @@ docker_build('scraper', './services/scraper')
k8s_yaml('./k8s/scraper.yaml')
local_resource(
'webshare-secret',
cmd='set -a; [ ! -f .env ] || . ./.env; set +a; if [ -n "$WEBSHARE_API_KEY" ] && [ -n "$WEBSHARE_PROXY_USERNAME" ] && [ -n "$WEBSHARE_PROXY_PASSWORD" ]; then kubectl create secret generic webshare-api --from-literal=api-key="$WEBSHARE_API_KEY" --from-literal=proxy-username="$WEBSHARE_PROXY_USERNAME" --from-literal=proxy-password="$WEBSHARE_PROXY_PASSWORD" --dry-run=client -o yaml | kubectl apply -f -; else echo "Webshare runtime config missing; country scraping unavailable"; fi',
labels=['utility'],
)
# Watch source files for live reload
k8s_resource(
workload='scraper-deployment',
port_forwards='8080:8080',
resource_deps=['webshare-secret'],
labels=['scraper'],
)
flake.nix
+8
View File
@@ -46,6 +46,13 @@
];
shellHook = ''
if [ -f .env ]; then
set -a
. ./.env
set +a
fi
if [ "''${OPENSTATICFISH_QUIET_SHELL:-}" != "1" ]; then
echo "🐟 OpenStaticFish MicroServices Dev Shell"
echo "-----------------------------------------"
@@ -95,6 +102,7 @@
echo " kubectl ... - Interact with k8s cluster"
echo " docker ... - Interact with containers"
echo ""
fi
'';
};
k8s/scraper.yaml
+18
View File
@@ -26,6 +26,24 @@ spec:
env:
- name: PORT
value: "8080"
- name: WEBSHARE_API_KEY
valueFrom:
secretKeyRef:
name: webshare-api
key: api-key
optional: true
- name: WEBSHARE_PROXY_USERNAME
valueFrom:
secretKeyRef:
name: webshare-api
key: proxy-username
optional: true
- name: WEBSHARE_PROXY_PASSWORD
valueFrom:
secretKeyRef:
name: webshare-api
key: proxy-password
optional: true
resources:
requests:
memory: "64Mi"
scripts/test-scrape
+34 -4
View File
@@ -2,18 +2,20 @@
set -euo pipefail
# --- OpenStaticFish Scraper Test Script ---
# Usage: ./scripts/test-scrape <url> [md|json]
# Usage: ./scripts/test-scrape <url> [md|json] [country]
# url The URL to scrape
# format Output format: md or json (default: json)
# country Optional Webshare proxy country code (for example: us, gb, nl, pl)
SCRAPE_URL="${1:-}"
FORMAT="${2:-json}"
EGRESS="${3:-}"
API_ENDPOINT="http://localhost:8080/scrape"
if [[ -z "$SCRAPE_URL" ]]; then
echo "Error: URL is required"
echo "Usage: $0 <url> [md|json]"
echo "Example: $0 https://example.com md"
echo "Usage: $0 <url> [md|json] [country]"
echo "Example: $0 https://example.com md gb"
exit 1
fi
@@ -24,12 +26,40 @@ fi
echo "Scraping: $SCRAPE_URL"
echo "Format: $FORMAT"
if [[ -n "$EGRESS" ]]; then
echo "Country: $EGRESS"
if [[ -f .env ]]; then
set -a
# shellcheck disable=SC1091
source .env
set +a
fi
if [[ -n "${WEBSHARE_API_KEY:-}" && -n "${WEBSHARE_PROXY_USERNAME:-}" && -n "${WEBSHARE_PROXY_PASSWORD:-}" ]]; then
if command -v kubectl >/dev/null 2>&1; then
kubectl create secret generic webshare-api \
--from-literal=api-key="$WEBSHARE_API_KEY" \
--from-literal=proxy-username="$WEBSHARE_PROXY_USERNAME" \
--from-literal=proxy-password="$WEBSHARE_PROXY_PASSWORD" \
--dry-run=client \
-o yaml | kubectl apply -f - >/dev/null
kubectl rollout restart deployment/scraper-deployment >/dev/null 2>&1 || true
kubectl rollout status deployment/scraper-deployment --timeout=120s >/dev/null 2>&1 || true
else
OPENSTATICFISH_QUIET_SHELL=1 nix --quiet --no-warn-dirty --option eval-cache false develop --command bash -c 'kubectl create secret generic webshare-api --from-literal=api-key="$WEBSHARE_API_KEY" --from-literal=proxy-username="$WEBSHARE_PROXY_USERNAME" --from-literal=proxy-password="$WEBSHARE_PROXY_PASSWORD" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; kubectl rollout restart deployment/scraper-deployment >/dev/null 2>&1 || true; kubectl rollout status deployment/scraper-deployment --timeout=120s >/dev/null 2>&1 || true'
fi
fi
fi
echo "---"
PAYLOAD="{\"url\": \"$SCRAPE_URL\", \"format\": \"$FORMAT\"}"
if [[ -n "$EGRESS" ]]; then
PAYLOAD="{\"url\": \"$SCRAPE_URL\", \"format\": \"$FORMAT\", \"country\": \"$EGRESS\"}"
fi
# Send request
RESPONSE=$(curl -s -X POST "$API_ENDPOINT" \
-H "Content-Type: application/json" \
-d "{\"url\": \"$SCRAPE_URL\", \"format\": \"$FORMAT\"}")
-d "$PAYLOAD")
if [[ "$FORMAT" == "md" ]]; then
# Raw markdown output
services/scraper/main.go
+216 -4
View File
@@ -3,10 +3,14 @@ package main
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
@@ -17,6 +21,7 @@ import (
type ScrapeRequest struct {
URL string `json:"url"`
Format string `json:"format"` // "md" or "json"
Country string `json:"country"` // optional Webshare proxy country code, for example "us" or "gb"
}
// ScrapeResponse represents the response data
@@ -29,6 +34,7 @@ type ScrapeResponse struct {
Metadata map[string]string `json:"metadata,omitempty"`
Links []string `json:"links,omitempty"`
Images []string `json:"images,omitempty"`
Country string `json:"country,omitempty"`
ScrapedAt time.Time `json:"scraped_at"`
}
@@ -38,6 +44,31 @@ type ErrorResponse struct {
Details string `json:"details,omitempty"`
}
const webshareProxyCacheTTL = 5 * time.Minute
type webshareProxy struct {
Username string `json:"username"`
Password string `json:"password"`
ProxyAddress string `json:"proxy_address"`
Port int `json:"port"`
Valid bool `json:"valid"`
CountryCode string `json:"country_code"`
CityName string `json:"city_name"`
LastVerification string `json:"last_verification"`
}
type webshareProxyListResponse struct {
Count int `json:"count"`
Next string `json:"next"`
Results []webshareProxy `json:"results"`
}
var webshareCache = struct {
sync.Mutex
proxies []webshareProxy
fetchedAt time.Time
}{}
func main() {
port := os.Getenv("PORT")
if port == "" {
@@ -82,8 +113,14 @@ func handleScrape(w http.ResponseWriter, r *http.Request) {
return
}
country := normalizeCountryCode(req.Country)
if err := validateCountry(country); err != nil {
respondWithError(w, http.StatusBadRequest, "Invalid country", err.Error())
return
}
req.URL = normalizeURL(req.URL)
result, err := scrapeURL(req.URL, format)
result, err := scrapeURL(req.URL, format, country)
if err != nil {
respondWithError(w, http.StatusInternalServerError, "Scraping failed", err.Error())
return
@@ -98,7 +135,23 @@ func handleScrape(w http.ResponseWriter, r *http.Request) {
}
}
func scrapeURL(url, format string) (*ScrapeResponse, error) {
func scrapeURL(url, format string, country string) (*ScrapeResponse, error) {
if country != "" {
proxy, err := selectWebshareProxy(country)
if err != nil {
return nil, err
}
transport, err := webshareProxyTransport(proxy)
if err != nil {
return nil, err
}
return scrapeURLWithTransport(url, format, country, transport)
}
return scrapeURLWithTransport(url, format, country, nil)
}
func scrapeURLWithTransport(url, format string, country string, transport *http.Transport) (*ScrapeResponse, error) {
var htmlContent string
var title string
var description string
@@ -107,8 +160,13 @@ func scrapeURL(url, format string) (*ScrapeResponse, error) {
metadata := make(map[string]string)
c := colly.NewCollector(
colly.UserAgent("OpenStaticFish-Scraper/1.0"),
colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"),
)
c.SetRequestTimeout(30 * time.Second)
if transport != nil {
c.WithTransport(transport)
}
c.OnResponse(func(r *colly.Response) {
htmlContent = string(r.Body)
@@ -173,6 +231,7 @@ func scrapeURL(url, format string) (*ScrapeResponse, error) {
Metadata: metadata,
Links: deduplicate(links),
Images: deduplicate(images),
Country: country,
ScrapedAt: time.Now().UTC(),
}, nil
}
@@ -191,6 +250,159 @@ func normalizeURL(rawURL string) string {
return "https://" + rawURL
}
func validateCountry(country string) error {
if country == "" {
return nil
}
if len(country) != 2 || country[0] < 'a' || country[0] > 'z' || country[1] < 'a' || country[1] > 'z' {
return fmt.Errorf("country must be a two-letter country code, for example us, gb, nl, pl")
}
return nil
}
func normalizeCountryCode(country string) string {
country = strings.ToLower(strings.TrimSpace(country))
if country == "uk" {
return "gb"
}
return country
}
func webshareProxyTransport(proxy webshareProxy) (*http.Transport, error) {
username := proxy.Username
password := proxy.Password
if username == "" {
username = os.Getenv("WEBSHARE_PROXY_USERNAME")
}
if password == "" {
password = os.Getenv("WEBSHARE_PROXY_PASSWORD")
}
if username == "" || password == "" {
return nil, fmt.Errorf("Webshare proxy credentials are missing for selected proxy")
}
proxyURL := &url.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", proxy.ProxyAddress, proxy.Port),
User: url.UserPassword(username, password),
}
if _, err := url.Parse(proxyURL.String()); err != nil {
return nil, fmt.Errorf("failed to configure Webshare proxy: %w", err)
}
return &http.Transport{
Proxy: http.ProxyURL(proxyURL),
}, nil
}
func selectWebshareProxy(country string) (webshareProxy, error) {
proxies, err := getWebshareProxies(false)
if err != nil {
return webshareProxy{}, err
}
if proxy, ok := findWebshareProxy(proxies, country); ok {
return proxy, nil
}
proxies, err = getWebshareProxies(true)
if err != nil {
return webshareProxy{}, err
}
if proxy, ok := findWebshareProxy(proxies, country); ok {
return proxy, nil
}
return webshareProxy{}, fmt.Errorf("no valid Webshare proxy available for country %q; available countries: %s", country, strings.Join(availableWebshareCountries(proxies), ", "))
}
func findWebshareProxy(proxies []webshareProxy, country string) (webshareProxy, bool) {
for _, proxy := range proxies {
if proxy.Valid && strings.EqualFold(proxy.CountryCode, country) {
return proxy, true
}
}
return webshareProxy{}, false
}
func getWebshareProxies(forceRefresh bool) ([]webshareProxy, error) {
webshareCache.Lock()
defer webshareCache.Unlock()
if !forceRefresh && len(webshareCache.proxies) > 0 && time.Since(webshareCache.fetchedAt) < webshareProxyCacheTTL {
return webshareCache.proxies, nil
}
proxies, err := fetchWebshareProxies()
if err != nil {
return nil, err
}
webshareCache.proxies = proxies
webshareCache.fetchedAt = time.Now()
return proxies, nil
}
func fetchWebshareProxies() ([]webshareProxy, error) {
apiKey := os.Getenv("WEBSHARE_API_KEY")
if apiKey == "" {
return nil, fmt.Errorf("WEBSHARE_API_KEY is required when country is set")
}
var proxies []webshareProxy
listURL := "https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=1&page_size=100"
client := &http.Client{Timeout: 20 * time.Second}
for listURL != "" {
req, err := http.NewRequest(http.MethodGet, listURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create Webshare request: %w", err)
}
req.Header.Set("Authorization", "Token "+apiKey)
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch Webshare proxies: %w", err)
}
body, readErr := io.ReadAll(resp.Body)
resp.Body.Close()
if readErr != nil {
return nil, fmt.Errorf("failed to read Webshare response: %w", readErr)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("Webshare proxy list request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var result webshareProxyListResponse
if err := json.Unmarshal(body, &result); err != nil {
return nil, fmt.Errorf("failed to parse Webshare proxy list: %w", err)
}
proxies = append(proxies, result.Results...)
listURL = result.Next
}
return proxies, nil
}
func availableWebshareCountries(proxies []webshareProxy) []string {
seen := make(map[string]bool)
var countries []string
for _, proxy := range proxies {
if !proxy.Valid || proxy.CountryCode == "" {
continue
}
country := strings.ToLower(proxy.CountryCode)
if seen[country] {
continue
}
seen[country] = true
countries = append(countries, country)
}
if len(countries) == 0 {
return []string{"none"}
}
sort.Strings(countries)
return countries
}
func convertToMarkdown(html string, title string, description string) string {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
@@ -519,8 +731,8 @@ func deduplicate(slice []string) []string {
}
func respondWithError(w http.ResponseWriter, status int, err string, details string) {
w.WriteHeader(status)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(ErrorResponse{
Error: err,
Details: details,