mirror of
https://github.com/OpenStaticFish/MicroServices.git
synced 2026-10-12 22:25:14 +00:00
Replace hello-world with scraper service
This commit is contained in:
@@ -19,17 +19,17 @@
|
||||
# This creates/selects the supported local Kind context below.
|
||||
allow_k8s_contexts('kind-openstaticfish')
|
||||
|
||||
# --- Hello World Service ---
|
||||
# --- Scraper Service ---
|
||||
# Builds the Docker image and deploys the manifest with live reload
|
||||
docker_build('hello-world', './services/hello-world')
|
||||
docker_build('scraper', './services/scraper')
|
||||
|
||||
k8s_yaml('./k8s/hello-world.yaml')
|
||||
k8s_yaml('./k8s/scraper.yaml')
|
||||
|
||||
# Watch source files for live reload
|
||||
k8s_resource(
|
||||
workload='hello-world-deployment',
|
||||
workload='scraper-deployment',
|
||||
port_forwards='8080:8080',
|
||||
labels=['hello-world'],
|
||||
labels=['scraper'],
|
||||
)
|
||||
|
||||
# --- Local Resources ---
|
||||
@@ -37,6 +37,6 @@ k8s_resource(
|
||||
local_resource(
|
||||
'health-check',
|
||||
cmd='curl -sf http://localhost:8080/health || echo "Service not ready"',
|
||||
resource_deps=['hello-world-deployment'],
|
||||
resource_deps=['scraper-deployment'],
|
||||
labels=['utility'],
|
||||
)
|
||||
|
||||
@@ -78,11 +78,11 @@
|
||||
# kubectl get pods
|
||||
#
|
||||
# View service logs:
|
||||
# kubectl logs -f deployment/hello-world
|
||||
# kubectl logs -f deployment/scraper
|
||||
|
||||
# --- Docker Commands ---
|
||||
# Build images manually:
|
||||
# docker build -t hello-world ./services/hello-world
|
||||
# docker build -t scraper ./services/scraper
|
||||
#
|
||||
# List running containers:
|
||||
# docker ps
|
||||
|
||||
@@ -1,26 +1,26 @@
|
||||
# --- Hello World Kubernetes Manifests ---
|
||||
# --- Scraper Kubernetes Manifests ---
|
||||
# Managed by Tilt - run `tilt up` to deploy
|
||||
# Tear down with: `tilt down`
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hello-world-deployment
|
||||
name: scraper-deployment
|
||||
labels:
|
||||
app: hello-world
|
||||
app: scraper
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hello-world
|
||||
app: scraper
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hello-world
|
||||
app: scraper
|
||||
spec:
|
||||
containers:
|
||||
- name: hello-world
|
||||
image: hello-world
|
||||
- name: scraper
|
||||
image: scraper
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
@@ -28,11 +28,11 @@ spec:
|
||||
value: "8080"
|
||||
resources:
|
||||
requests:
|
||||
memory: "32Mi"
|
||||
cpu: "50m"
|
||||
limits:
|
||||
memory: "64Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "256Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
@@ -49,10 +49,10 @@ spec:
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hello-world-service
|
||||
name: scraper-service
|
||||
spec:
|
||||
selector:
|
||||
app: hello-world
|
||||
app: scraper
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# --- OpenStaticFish Scraper Test Script ---
|
||||
# Usage: ./scripts/test-scrape <url> [md|json]
|
||||
# url The URL to scrape
|
||||
# format Output format: md or json (default: json)
|
||||
|
||||
SCRAPE_URL="${1:-}"
|
||||
FORMAT="${2:-json}"
|
||||
API_ENDPOINT="http://localhost:8080/scrape"
|
||||
|
||||
if [[ -z "$SCRAPE_URL" ]]; then
|
||||
echo "Error: URL is required"
|
||||
echo "Usage: $0 <url> [md|json]"
|
||||
echo "Example: $0 https://example.com md"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$FORMAT" != "md" && "$FORMAT" != "json" ]]; then
|
||||
echo "Error: format must be 'md' or 'json'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Scraping: $SCRAPE_URL"
|
||||
echo "Format: $FORMAT"
|
||||
echo "---"
|
||||
|
||||
# Send request
|
||||
RESPONSE=$(curl -s -X POST "$API_ENDPOINT" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"url\": \"$SCRAPE_URL\", \"format\": \"$FORMAT\"}")
|
||||
|
||||
if [[ "$FORMAT" == "md" ]]; then
|
||||
# Raw markdown output
|
||||
echo "$RESPONSE"
|
||||
else
|
||||
# Pretty-print JSON
|
||||
if echo "$RESPONSE" | jq . > /dev/null 2>&1; then
|
||||
echo "$RESPONSE" | jq .
|
||||
else
|
||||
echo "Error: Invalid response"
|
||||
echo "$RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -1 +0,0 @@
|
||||
hello-world
|
||||
@@ -1,14 +0,0 @@
|
||||
# --- Hello World Service Dockerfile ---
|
||||
# Multi-stage build for minimal image size
|
||||
|
||||
FROM golang:1.22-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY go.mod ./
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -o /hello-world .
|
||||
|
||||
FROM alpine:3.19
|
||||
RUN apk --no-cache add ca-certificates
|
||||
COPY --from=builder /hello-world /hello-world
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/hello-world"]
|
||||
@@ -1,3 +0,0 @@
|
||||
module github.com/openstaticfish/microservices/hello-world
|
||||
|
||||
go 1.22
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]string{
|
||||
"message": "Hello from OpenStaticFish!",
|
||||
"service": "hello-world",
|
||||
})
|
||||
})
|
||||
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, "ok")
|
||||
})
|
||||
|
||||
log.Printf("hello-world listening on :%s", port)
|
||||
log.Fatal(http.ListenAndServe(":"+port, nil))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
# Binary
|
||||
/scraper
|
||||
@@ -0,0 +1,15 @@
|
||||
# --- Scraper Service Dockerfile ---
|
||||
# Multi-stage build for minimal image size
|
||||
|
||||
FROM golang:1.26-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -o /scraper .
|
||||
|
||||
FROM alpine:3.19
|
||||
RUN apk --no-cache add ca-certificates
|
||||
COPY --from=builder /scraper /scraper
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/scraper"]
|
||||
@@ -0,0 +1,27 @@
|
||||
module github.com/openstaticfish/microservices/scraper
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
github.com/gocolly/colly/v2 v2.1.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/antchfx/htmlquery v1.3.0 // indirect
|
||||
github.com/antchfx/xmlquery v1.3.18 // indirect
|
||||
github.com/antchfx/xpath v1.2.5 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/stretchr/testify v1.4.0 // indirect
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
golang.org/x/net v0.25.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/protobuf v1.32.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,178 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
|
||||
github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
|
||||
github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
|
||||
github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
|
||||
github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0=
|
||||
github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA=
|
||||
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY=
|
||||
github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
|
||||
github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
|
||||
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
|
||||
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
@@ -0,0 +1,528 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gocolly/colly/v2"
|
||||
)
|
||||
|
||||
// ScrapeRequest represents the incoming request body
|
||||
type ScrapeRequest struct {
|
||||
URL string `json:"url"`
|
||||
Format string `json:"format"` // "md" or "json"
|
||||
}
|
||||
|
||||
// ScrapeResponse represents the response data
|
||||
type ScrapeResponse struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Content string `json:"content"`
|
||||
Format string `json:"format"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Links []string `json:"links,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
ScrapedAt time.Time `json:"scraped_at"`
|
||||
}
|
||||
|
||||
// ErrorResponse represents an error response
|
||||
type ErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
Details string `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
http.HandleFunc("/scrape", handleScrape)
|
||||
http.HandleFunc("/health", handleHealth)
|
||||
|
||||
log.Printf("scraper listening on :%s", port)
|
||||
log.Fatal(http.ListenAndServe(":"+port, nil))
|
||||
}
|
||||
|
||||
func handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, "ok")
|
||||
}
|
||||
|
||||
func handleScrape(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
respondWithError(w, http.StatusMethodNotAllowed, "Method not allowed", "Only POST requests are supported")
|
||||
return
|
||||
}
|
||||
|
||||
var req ScrapeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondWithError(w, http.StatusBadRequest, "Invalid JSON", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if req.URL == "" {
|
||||
respondWithError(w, http.StatusBadRequest, "Missing URL", "url field is required")
|
||||
return
|
||||
}
|
||||
|
||||
format := strings.ToLower(req.Format)
|
||||
if format == "" {
|
||||
format = "json"
|
||||
}
|
||||
if format != "md" && format != "json" {
|
||||
respondWithError(w, http.StatusBadRequest, "Invalid format", "format must be 'md' or 'json'")
|
||||
return
|
||||
}
|
||||
|
||||
req.URL = normalizeURL(req.URL)
|
||||
result, err := scrapeURL(req.URL, format)
|
||||
if err != nil {
|
||||
respondWithError(w, http.StatusInternalServerError, "Scraping failed", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if format == "md" {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
fmt.Fprint(w, result.Content)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}
|
||||
}
|
||||
|
||||
func scrapeURL(url, format string) (*ScrapeResponse, error) {
|
||||
var htmlContent string
|
||||
var title string
|
||||
var description string
|
||||
var links []string
|
||||
var images []string
|
||||
metadata := make(map[string]string)
|
||||
|
||||
c := colly.NewCollector(
|
||||
colly.UserAgent("OpenStaticFish-Scraper/1.0"),
|
||||
)
|
||||
|
||||
c.OnResponse(func(r *colly.Response) {
|
||||
htmlContent = string(r.Body)
|
||||
})
|
||||
|
||||
c.OnHTML("title", func(e *colly.HTMLElement) {
|
||||
title = strings.TrimSpace(e.Text)
|
||||
})
|
||||
|
||||
c.OnHTML("meta[name='description']", func(e *colly.HTMLElement) {
|
||||
description = e.Attr("content")
|
||||
})
|
||||
|
||||
c.OnHTML("meta[property='og:title']", func(e *colly.HTMLElement) {
|
||||
metadata["og_title"] = e.Attr("content")
|
||||
})
|
||||
|
||||
c.OnHTML("meta[property='og:description']", func(e *colly.HTMLElement) {
|
||||
metadata["og_description"] = e.Attr("content")
|
||||
})
|
||||
|
||||
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
|
||||
link := e.Attr("href")
|
||||
if link != "" && !strings.HasPrefix(link, "#") {
|
||||
links = append(links, link)
|
||||
}
|
||||
})
|
||||
|
||||
c.OnHTML("img[src]", func(e *colly.HTMLElement) {
|
||||
img := e.Attr("src")
|
||||
if img != "" {
|
||||
images = append(images, img)
|
||||
}
|
||||
})
|
||||
|
||||
if err := c.Visit(url); err != nil {
|
||||
return nil, fmt.Errorf("failed to visit URL: %w", err)
|
||||
}
|
||||
|
||||
var content string
|
||||
if format == "md" {
|
||||
content = convertToMarkdown(htmlContent, title, description)
|
||||
} else {
|
||||
// For JSON, extract main content text
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTML: %w", err)
|
||||
}
|
||||
// Remove script and style elements
|
||||
doc.Find("script, style, nav, header, footer, aside").Remove()
|
||||
content = strings.TrimSpace(doc.Find("body").Text())
|
||||
// Clean up whitespace
|
||||
content = strings.Join(strings.Fields(content), " ")
|
||||
}
|
||||
|
||||
return &ScrapeResponse{
|
||||
URL: url,
|
||||
Title: title,
|
||||
Description: description,
|
||||
Content: content,
|
||||
Format: format,
|
||||
Metadata: metadata,
|
||||
Links: deduplicate(links),
|
||||
Images: deduplicate(images),
|
||||
ScrapedAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeURL(rawURL string) string {
|
||||
if strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://") {
|
||||
return rawURL
|
||||
}
|
||||
|
||||
// Check if it's a domain-like string (no spaces, has a dot)
|
||||
if strings.Contains(rawURL, ".") && !strings.Contains(rawURL, " ") {
|
||||
return "https://" + rawURL
|
||||
}
|
||||
|
||||
// Otherwise, treat as a path or relative URL
|
||||
return "https://" + rawURL
|
||||
}
|
||||
|
||||
func convertToMarkdown(html string, title string, description string) string {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
return html
|
||||
}
|
||||
|
||||
// Remove script/style only - keep semantic structure
|
||||
doc.Find("script, style").Remove()
|
||||
|
||||
var md strings.Builder
|
||||
|
||||
// Add title as H1 if present
|
||||
if title != "" {
|
||||
md.WriteString("# ")
|
||||
md.WriteString(title)
|
||||
md.WriteString("\n\n")
|
||||
}
|
||||
|
||||
// Add description if present
|
||||
if description != "" {
|
||||
md.WriteString(description)
|
||||
md.WriteString("\n\n")
|
||||
}
|
||||
|
||||
// Process the body
|
||||
doc.Find("body").Each(func(i int, s *goquery.Selection) {
|
||||
processNode(&md, s)
|
||||
})
|
||||
|
||||
return cleanMarkdown(md.String())
|
||||
}
|
||||
|
||||
func processNode(md *strings.Builder, s *goquery.Selection) {
|
||||
s.Children().Each(func(i int, child *goquery.Selection) {
|
||||
nodeType := goquery.NodeName(child)
|
||||
|
||||
// Add spacing before block elements (except first child)
|
||||
isBlock := isBlockElement(nodeType)
|
||||
if isBlock && i > 0 {
|
||||
md.WriteString("\n")
|
||||
}
|
||||
|
||||
switch nodeType {
|
||||
case "h1":
|
||||
md.WriteString("\n# ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "h2":
|
||||
md.WriteString("\n## ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "h3":
|
||||
md.WriteString("\n### ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "h4":
|
||||
md.WriteString("\n#### ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "h5":
|
||||
md.WriteString("\n##### ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "h6":
|
||||
md.WriteString("\n###### ")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("\n")
|
||||
case "p":
|
||||
md.WriteString("\n")
|
||||
processInline(md, child)
|
||||
md.WriteString("\n")
|
||||
case "div":
|
||||
// Check if div contains only inline content
|
||||
if isInlineContainer(child) {
|
||||
processInline(md, child)
|
||||
md.WriteString("\n")
|
||||
} else {
|
||||
processNode(md, child)
|
||||
}
|
||||
case "section", "article", "main":
|
||||
processNode(md, child)
|
||||
case "ul", "ol":
|
||||
md.WriteString("\n")
|
||||
child.Children().Each(func(j int, li *goquery.Selection) {
|
||||
if goquery.NodeName(li) == "li" {
|
||||
md.WriteString("- ")
|
||||
processInline(md, li)
|
||||
md.WriteString("\n")
|
||||
}
|
||||
})
|
||||
case "br":
|
||||
md.WriteString("\n")
|
||||
case "hr":
|
||||
md.WriteString("\n---\n")
|
||||
case "blockquote":
|
||||
md.WriteString("\n> ")
|
||||
processInline(md, child)
|
||||
md.WriteString("\n")
|
||||
case "pre":
|
||||
md.WriteString("\n```\n")
|
||||
md.WriteString(strings.TrimSpace(child.Text()))
|
||||
md.WriteString("\n```\n")
|
||||
case "img":
|
||||
src, _ := child.Attr("src")
|
||||
alt, _ := child.Attr("alt")
|
||||
if src != "" {
|
||||
md.WriteString(fmt.Sprintf("", alt, src))
|
||||
}
|
||||
case "a":
|
||||
href, _ := child.Attr("href")
|
||||
text := strings.TrimSpace(child.Text())
|
||||
if href != "" && text != "" {
|
||||
md.WriteString(fmt.Sprintf("[%s](%s)", text, href))
|
||||
}
|
||||
case "nav", "header", "footer":
|
||||
// Process but add visual separation
|
||||
processNode(md, child)
|
||||
default:
|
||||
// For inline elements, process their children
|
||||
if isInlineElement(nodeType) {
|
||||
processInline(md, child)
|
||||
} else {
|
||||
processNode(md, child)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func isBlockElement(tag string) bool {
|
||||
blockTags := map[string]bool{
|
||||
"div": true, "p": true, "h1": true, "h2": true, "h3": true,
|
||||
"h4": true, "h5": true, "h6": true, "ul": true, "ol": true,
|
||||
"li": true, "blockquote": true, "pre": true, "hr": true,
|
||||
"section": true, "article": true, "main": true, "nav": true,
|
||||
"header": true, "footer": true, "aside": true,
|
||||
}
|
||||
return blockTags[tag]
|
||||
}
|
||||
|
||||
func isInlineContainer(s *goquery.Selection) bool {
|
||||
isInline := true
|
||||
s.Children().Each(func(i int, child *goquery.Selection) {
|
||||
nodeType := goquery.NodeName(child)
|
||||
if isBlockElement(nodeType) {
|
||||
isInline = false
|
||||
}
|
||||
})
|
||||
return isInline
|
||||
}
|
||||
|
||||
func processInline(md *strings.Builder, s *goquery.Selection) {
|
||||
s.Contents().Each(func(i int, child *goquery.Selection) {
|
||||
if goquery.NodeName(child) == "#text" {
|
||||
text := strings.TrimSpace(child.Text())
|
||||
if text != "" {
|
||||
md.WriteString(text)
|
||||
md.WriteString(" ")
|
||||
}
|
||||
} else {
|
||||
nodeType := goquery.NodeName(child)
|
||||
switch nodeType {
|
||||
case "a":
|
||||
href, _ := child.Attr("href")
|
||||
text := strings.TrimSpace(child.Text())
|
||||
if href != "" && text != "" {
|
||||
writeSpaceIfNeeded(md)
|
||||
md.WriteString(fmt.Sprintf("[%s](%s)", cleanInlineText(text), href))
|
||||
md.WriteString(" ")
|
||||
}
|
||||
case "img":
|
||||
src, _ := child.Attr("src")
|
||||
alt, _ := child.Attr("alt")
|
||||
if src != "" {
|
||||
md.WriteString(fmt.Sprintf(" ", alt, src))
|
||||
}
|
||||
case "strong", "b":
|
||||
writeSpaceIfNeeded(md)
|
||||
md.WriteString("**")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("** ")
|
||||
case "em", "i":
|
||||
writeSpaceIfNeeded(md)
|
||||
md.WriteString("*")
|
||||
md.WriteString(cleanInlineText(child.Text()))
|
||||
md.WriteString("* ")
|
||||
case "code":
|
||||
md.WriteString("`")
|
||||
md.WriteString(strings.TrimSpace(child.Text()))
|
||||
md.WriteString("` ")
|
||||
case "br":
|
||||
md.WriteString("\n")
|
||||
default:
|
||||
processInline(md, child)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func writeSpaceIfNeeded(md *strings.Builder) {
|
||||
value := md.String()
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
last := value[len(value)-1]
|
||||
if last != ' ' && last != '\n' {
|
||||
md.WriteString(" ")
|
||||
}
|
||||
}
|
||||
|
||||
func cleanInlineText(input string) string {
|
||||
return strings.Join(strings.Fields(input), " ")
|
||||
}
|
||||
|
||||
func isInlineElement(tag string) bool {
|
||||
inlineTags := map[string]bool{
|
||||
"span": true, "em": true, "i": true, "strong": true, "b": true,
|
||||
"a": true, "code": true, "small": true, "sub": true, "sup": true,
|
||||
"mark": true, "del": true, "ins": true, "abbr": true, "time": true,
|
||||
}
|
||||
return inlineTags[tag]
|
||||
}
|
||||
|
||||
func cleanMarkdown(input string) string {
|
||||
// Remove email obfuscation links
|
||||
input = removeEmailObfuscation(input)
|
||||
input = strings.ReplaceAll(input, "\u00a0", " ")
|
||||
input = strings.ReplaceAll(input, "[email protected]", "email protected")
|
||||
input = strings.ReplaceAll(input, "email protected", "[email protected]")
|
||||
input = strings.ReplaceAll(input, "[email protected]]", "[email protected]")
|
||||
input = strings.ReplaceAll(input, "[email protected]", "")
|
||||
input = strings.ReplaceAll(input, "](", "](")
|
||||
input = strings.ReplaceAll(input, ")[", ") [")
|
||||
input = strings.ReplaceAll(input, ")![", ") ![")
|
||||
input = strings.ReplaceAll(input, " .", ".")
|
||||
input = splitDenseLinkLines(input)
|
||||
input = separateSectionLabels(input)
|
||||
|
||||
// Fix multiple consecutive blank lines
|
||||
for strings.Contains(input, "\n\n\n") {
|
||||
input = strings.ReplaceAll(input, "\n\n\n", "\n\n")
|
||||
}
|
||||
|
||||
// Clean up extra spaces around markdown
|
||||
input = strings.ReplaceAll(input, " ", " ")
|
||||
input = strings.ReplaceAll(input, " ", " ")
|
||||
|
||||
// Trim each line
|
||||
lines := strings.Split(input, "\n")
|
||||
for i := range lines {
|
||||
lines[i] = strings.TrimRight(lines[i], " ")
|
||||
}
|
||||
input = strings.Join(lines, "\n")
|
||||
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
|
||||
func splitDenseLinkLines(input string) string {
|
||||
lines := strings.Split(input, "\n")
|
||||
for i, line := range lines {
|
||||
if strings.Count(line, "](") > 1 && len(line) > 100 {
|
||||
line = strings.ReplaceAll(line, ") [", ")\n[")
|
||||
}
|
||||
lines[i] = line
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func separateSectionLabels(input string) string {
|
||||
labels := []string{"Navigation", "More", "Legal", "What I Offer", "Selected Work", "Why Staticfish", "Next Step"}
|
||||
lines := strings.Split(input, "\n")
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
for _, label := range labels {
|
||||
if trimmed == label {
|
||||
lines[i] = "## " + label
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func removeEmailObfuscation(input string) string {
|
||||
// Match [[anything]](/cdn-cgi/l/email-protection) and replace with just the inner text
|
||||
var result strings.Builder
|
||||
i := 0
|
||||
for i < len(input) {
|
||||
if i < len(input)-1 && input[i] == '[' && input[i+1] == '[' {
|
||||
// Find ]( after the opening [[
|
||||
end := strings.Index(input[i:], "](")
|
||||
if end != -1 {
|
||||
// Check if it's the email protection URL
|
||||
urlStart := i + end + 2
|
||||
if strings.HasPrefix(input[urlStart:], "/cdn-cgi/l/email-protection") {
|
||||
// Extract text between [[ and ]]
|
||||
emailStart := i + 2
|
||||
emailEnd := i + end
|
||||
if emailEnd > emailStart {
|
||||
result.WriteString(input[emailStart:emailEnd])
|
||||
// Skip past the closing )
|
||||
closeParen := strings.Index(input[i+end:], ")")
|
||||
if closeParen != -1 {
|
||||
i = i + end + closeParen + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.WriteByte(input[i])
|
||||
i++
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func deduplicate(slice []string) []string {
|
||||
seen := make(map[string]bool)
|
||||
result := []string{}
|
||||
for _, s := range slice {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func respondWithError(w http.ResponseWriter, status int, err string, details string) {
|
||||
w.WriteHeader(status)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(ErrorResponse{
|
||||
Error: err,
|
||||
Details: details,
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user