{ "openapi": "3.1.0", "info": { "title": "webscrapingapi.dev API", "version": "1.0.0", "summary": "Free web scraping API: any page as HTML, text, Markdown or JSON, plus JSON endpoints for sites the service already knows.", "description": "Free: 50,000 credits per account per day, 60 requests per minute per key. A plain fetch costs 1 credit, a JavaScript render 5, metadata and site endpoints 1; answers served from the shared store cost 0. Human documentation: https://webscrapingapi.dev/docs. Machine-readable overview: https://webscrapingapi.dev/llms.txt.", "termsOfService": "https://webscrapingapi.dev/terms", "contact": { "name": "webscrapingapi.dev", "url": "https://webscrapingapi.dev/about" } }, "servers": [ { "url": "https://api.webscrapingapi.dev" } ], "security": [ { "ApiKeyAuth": [] }, { "BearerAuth": [] } ], "tags": [ { "name": "site endpoints", "description": "Ready-made JSON for known sites. 1 credit live, 0 from the shared store." } ], "paths": { "/v1/health": { "get": { "operationId": "health", "summary": "Health check (no key needed)", "security": [], "responses": { "200": { "description": "ok", "content": { "application/json": { "schema": { "type": "object", "properties": { "status": { "type": "string" }, "service": { "type": "string" }, "version": { "type": "string" }, "time": { "type": "string", "format": "date-time" } } } } } } } } }, "/v1/scrape": { "get": { "operationId": "scrape", "summary": "Fetch any page as HTML, text, Markdown or JSON", "parameters": [ { "name": "url", "in": "query", "required": true, "schema": { "type": "string", "format": "uri" }, "description": "The page to fetch. Private and reserved networks are refused (URL_BLOCKED)." }, { "name": "format", "in": "query", "schema": { "type": "string", "enum": [ "html", "text", "markdown", "json" ], "default": "html" }, "description": "markdown keeps headings, lists, tables, code and absolute links; json requires extract." }, { "name": "render", "in": "query", "schema": { "type": "boolean", "default": false }, "description": "Run the page in headless Chrome first. Costs 5 credits instead of 1." }, { "name": "selector", "in": "query", "schema": { "type": "string" }, "description": "Return only this element (html, text and markdown formats)." }, { "name": "extract", "in": "query", "schema": { "type": "string" }, "description": "JSON object of field → CSS selector or { selector, attr, all, html }. Up to 50 fields." }, { "name": "wait", "in": "query", "schema": { "type": "integer", "maximum": 10000 }, "description": "render only: extra milliseconds to wait after load." }, { "name": "waitFor", "in": "query", "schema": { "type": "string" }, "description": "render only: CSS selector to wait for." }, { "name": "timeout", "in": "query", "schema": { "type": "integer", "default": 15000, "maximum": 30000 }, "description": "Fetch timeout in milliseconds." }, { "name": "raw", "in": "query", "schema": { "type": "boolean", "default": false }, "description": "Return the body alone (text/html, text/markdown or text/plain) instead of the JSON envelope." } ], "responses": { "200": { "description": "The page, in the requested format. The target's own status code is in statusCode; a 404 from the target is still a 200 here.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } }, "post": { "operationId": "scrapePost", "summary": "Same as GET /v1/scrape with a JSON body (easier for extract rules and headers)", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeRequest" } } } }, "responses": { "200": { "description": "The page, in the requested format. The target's own status code is in statusCode; a 404 from the target is still a 200 here.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/metadata": { "get": { "operationId": "metadata", "summary": "Title, description, canonical, favicon, language, Open Graph, Twitter card and JSON-LD from the head", "parameters": [ { "name": "url", "in": "query", "required": true, "schema": { "type": "string", "format": "uri" } } ], "responses": { "200": { "description": "Metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MetadataResponse" } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured": { "get": { "operationId": "listSites", "summary": "Public registry of site-specific endpoints", "security": [], "responses": { "200": { "description": "Registry", "content": { "application/json": { "schema": { "type": "object", "properties": { "sites": { "type": "array", "items": { "type": "object" } } } } } } } } } }, "/v1/structured/amazon/product": { "get": { "operationId": "amazon_product", "summary": "Amazon product: Product by ASIN", "description": "The product detail page as JSON. Results are stored for 86400 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { product: { name, brand, price, regularPrice, currency, availability, aggregateRating, mainImage, images[], features[], additionalProperties[], breadcrumbs[], sku }, extra: { asin, tld, bestSellersRank[], soldBy, shipsFrom, boughtInPastMonth, availabilityText } }", "tags": [ "site endpoints" ], "parameters": [ { "name": "asin", "in": "query", "required": true, "schema": { "type": "string" }, "description": "The 10-character Amazon product id.", "example": "B0B44XTV71" }, { "name": "tld", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Marketplace: com, co.uk, de, fr, it, es, ca, co.jp, in, com.au, com.mx, com.br, nl, ae, sa, se, pl. Default com.", "example": "com" }, { "name": "currency", "in": "query", "required": false, "schema": { "type": "string" }, "description": "ISO 4217 code for prices, e.g. USD, EUR, JPY. Default USD.", "example": "USD" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 86400 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": { "site": "amazon", "action": "product", "data": { "product": { "name": "CUCKOO Twin Pressure Rice Cooker, 12-Cup, White", "brand": { "name": "CUCKOO", "url": null }, "price": "339.99", "currency": "USD", "availability": "InStock", "aggregateRating": { "ratingValue": "4.4", "reviewCount": 608 }, "images": [ "https://m.media-amazon.com/images/I/…jpg" ], "features": [ "…", "…" ], "sku": "B0B44XTV71" }, "extra": { "asin": "B0B44XTV71", "tld": "com" } }, "source": "cache", "age": 412, "credits": 0 } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/youtube/video": { "get": { "operationId": "youtube_video", "summary": "YouTube video: Video by id", "description": "The watch page metadata as JSON. Results are stored for 3600 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { video: { id, url, title, description, channel { id, name, url }, durationSeconds, viewCount, likeCount, publishedAt, category, thumbnails[], keywords[], isLive, familySafe } }", "tags": [ "site endpoints" ], "parameters": [ { "name": "id", "in": "query", "required": true, "schema": { "type": "string" }, "description": "The 11-character video id (the v= parameter).", "example": "dQw4w9WgXcQ" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 3600 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": "{\n \"site\": \"youtube\",\n \"action\": \"video\",\n \"data\": {\n \"video\": {\n \"id\": \"dQw4w9WgXcQ\",\n \"title\": \"Rick Astley - Never Gonna Give You Up (Official Video) (4K Remaster)\",\n \"channel\": { \"id\": \"UCuAXFkgsw1L7xaCfnd5JJOw\", \"name\": \"Rick Astley\", \"url\": \"http://www.youtube.com/@RickAstleyYT\" },\n \"durationSeconds\": 213,\n \"viewCount\": 1818228078,\n \"likeCount\": 19384777,\n \"publishedAt\": \"2009-10-24T23:57:33-07:00\",\n \"category\": \"Music\",\n \"thumbnails\": [ { \"url\": \"https://i.ytimg.com/vi/dQw4w9WgXcQ/maxresdefault.jpg\", \"width\": 1280, \"height\": 720 } ],\n \"keywords\": [ \"rick astley\", \"Never Gonna Give You Up\", … ]\n }\n },\n \"source\": \"live\", \"age\": 0, \"credits\": 1\n}" } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/appstore/app": { "get": { "operationId": "appstore_app", "summary": "App Store app: App by id", "description": "The listing page as JSON. Results are stored for 86400 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { app: { name, url, platform: \"ios\", id, developer { name, url }, description, category, price, currency, free, rating { value, count }, icon, screenshots[], contentRating, osRequirement, installs: null, updatedAt, version } }", "tags": [ "site endpoints" ], "parameters": [ { "name": "id", "in": "query", "required": true, "schema": { "type": "string" }, "description": "The numeric app id (the id… part of the URL).", "example": "284882215" }, { "name": "country", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Storefront country code. Default us.", "example": "us" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 86400 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": { "site": "appstore", "action": "app", "data": { "app": { "name": "Facebook", "platform": "ios", "id": "284882215", "developer": { "name": "Meta Platforms, Inc.", "url": "https://apps.apple.com/us/developer/…" }, "category": "Social Networking", "price": "0", "currency": "USD", "free": true, "rating": { "value": 4.5, "count": 28420068 }, "icon": "https://is1-ssl.mzstatic.com/image/thumb/…png", "osRequirement": "Requires iOS 15.1 or later." } }, "source": "live", "age": 0, "credits": 1 } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/googleplay/app": { "get": { "operationId": "googleplay_app", "summary": "Google Play app: App by package", "description": "The listing page as JSON. Results are stored for 86400 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { app: { name, url, platform: \"android\", id, developer { name, url }, description, category, price, currency, free, rating { value, count }, icon, screenshots[], contentRating, osRequirement: null, installs, updatedAt, version } }", "tags": [ "site endpoints" ], "parameters": [ { "name": "id", "in": "query", "required": true, "schema": { "type": "string" }, "description": "The package name (the id= parameter).", "example": "com.spotify.music" }, { "name": "hl", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Language, e.g. en, ko. Default en.", "example": "en" }, { "name": "gl", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Country, e.g. US, KR. Default US.", "example": "US" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 86400 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": { "site": "googleplay", "action": "app", "data": { "app": { "name": "Spotify: Music and Podcasts", "platform": "android", "id": "com.spotify.music", "developer": { "name": "Spotify AB", "url": "https://www.spotify.com" }, "category": "MUSIC_AND_AUDIO", "price": "0", "currency": "USD", "free": true, "rating": { "value": 4.35, "count": 36339801 }, "installs": "1B+", "contentRating": "Teen", "updatedAt": "Sep 18, 2026" } }, "source": "live", "age": 0, "credits": 1 } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/hackernews/front": { "get": { "operationId": "hackernews_front", "summary": "Hacker News: Front page", "description": "Stories on the front page, in rank order. Results are stored for 600 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: stories[] { id, rank, title, url, site, points, author, commentCount, age }", "tags": [ "site endpoints" ], "parameters": [ { "name": "page", "in": "query", "required": false, "schema": { "type": "number" }, "description": "Page number (1-based).", "example": "1" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 600 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": "{\n \"site\": \"hackernews\",\n \"action\": \"front\",\n \"data\": {\n \"count\": 30,\n \"stories\": [\n {\n \"rank\": 1,\n \"title\": \"Samsung is expected to more than double …\",\n \"points\": 125,\n \"commentCount\": 84\n },\n // 29 more\n ]\n },\n \"source\": \"live\", \"age\": 0, \"credits\": 1\n}" } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/hackernews/item": { "get": { "operationId": "hackernews_item", "summary": "Hacker News: Item thread", "description": "A story with its comment thread (flattened, with depth). Results are stored for 600 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { id, title, url, points, author, age, comments[] { id, author, age, depth, text } }", "tags": [ "site endpoints" ], "parameters": [ { "name": "id", "in": "query", "required": true, "schema": { "type": "number" }, "description": "Hacker News item id.", "example": "1" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 600 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": { "site": "hackernews", "action": "item", "data": { "id": 1, "title": "Y Combinator", "commentsParsed": 3, "comments": [ { "id": 15, "author": "sama", "depth": 0, "text": "…" } ] }, "source": "live", "age": 0, "credits": 1 } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/wikipedia/summary": { "get": { "operationId": "wikipedia_summary", "summary": "Wikipedia: Article summary", "description": "Lead section of an article plus infobox key/values. Results are stored for 604800 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { title, url, lang, summary, paragraphs[], thumbnail, infobox {} }", "tags": [ "site endpoints" ], "parameters": [ { "name": "title", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Article title (spaces or underscores).", "example": "Web scraping" }, { "name": "lang", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Language edition (default en).", "example": "en" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 604800 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": { "site": "wikipedia", "action": "summary", "data": { "title": "Web scraping", "lang": "en", "summary": "Web scraping, web harvesting, or web data extraction is data scraping …", "paragraphs": [ "…", "…", "…", "…", "…" ], "infobox": {} }, "source": "cache", "age": 3120, "credits": 0 } } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/v1/structured/github/repo": { "get": { "operationId": "github_repo", "summary": "GitHub: Repository", "description": "Public repository overview. Results are stored for 3600 seconds by default and shared; a stored answer costs 0 credits and carries source: \"cache\". Returns data shaped like: { owner, repo, url, description, homepage, stars, forks, watchers, topics[], license, defaultBranch, archived, fork, releaseCount, language (null on the current GitHub layout) }", "tags": [ "site endpoints" ], "parameters": [ { "name": "owner", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Owner login.", "example": "withastro" }, { "name": "repo", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Repository name.", "example": "astro" }, { "name": "max_age", "in": "query", "required": false, "schema": { "type": "integer", "default": 3600 }, "description": "Accept a stored copy up to this many seconds old (credits: 0). Defaults to the endpoint's cache window; 0 forces a fresh fetch." } ], "responses": { "200": { "description": "Structured result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StructuredResponse" }, "example": "{\n \"site\": \"github\",\n \"action\": \"repo\",\n \"data\": {\n \"owner\": \"withastro\",\n \"repo\": \"astro\",\n \"description\": \"The web framework for content-driven websites.\",\n \"stars\": 62720,\n \"forks\": 3798,\n \"watchers\": 221,\n \"topics\": [ \"astro\", \"blog\", \"islands\", … ],\n \"license\": \"Other\",\n \"defaultBranch\": \"main\",\n \"releaseCount\": 3251,\n \"homepage\": \"https://astro.build\"\n },\n \"source\": \"live\", \"age\": 0, \"credits\": 1\n}" } } }, "400": { "description": "Bad request or blocked URL (BAD_REQUEST, URL_BLOCKED). Not charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "Missing or unknown API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Unknown site/action, or the item does not exist on the site.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "RATE_LIMITED (per key, per minute) or QUOTA_EXCEEDED (per account, per day).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "FETCH_FAILED (target unreachable, charged) or SITE_PARSE_FAILED (layout changed, refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "SITE_BLOCKED (target served a bot wall, refunded) or RENDER_UNAVAILABLE (refunded).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "TIMEOUT. Charged.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } } }, "components": { "securitySchemes": { "ApiKeyAuth": { "type": "apiKey", "in": "header", "name": "X-API-Key", "description": "Keys start with wsa_ and are created in the dashboard." }, "BearerAuth": { "type": "http", "scheme": "bearer", "description": "The same key as a Bearer token." } }, "schemas": { "Error": { "type": "object", "properties": { "error": { "type": "object", "properties": { "code": { "type": "string", "enum": [ "BAD_REQUEST", "URL_BLOCKED", "UNAUTHORIZED", "FORBIDDEN", "NOT_FOUND", "RATE_LIMITED", "QUOTA_EXCEEDED", "FETCH_FAILED", "SITE_PARSE_FAILED", "SITE_BLOCKED", "RENDER_UNAVAILABLE", "TIMEOUT", "INTERNAL_ERROR" ] }, "message": { "type": "string" } }, "required": [ "code", "message" ] } }, "required": [ "error" ] }, "ExtractRule": { "oneOf": [ { "type": "string", "description": "CSS selector; returns the text of the first match" }, { "type": "object", "properties": { "selector": { "type": "string" }, "attr": { "type": "string", "description": "return this attribute; href and src become absolute URLs" }, "all": { "type": "boolean", "description": "return every match as an array" }, "html": { "type": "boolean", "description": "return inner HTML instead of text" } }, "required": [ "selector" ] } ] }, "ScrapeRequest": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "format": "uri" }, "format": { "type": "string", "enum": [ "html", "text", "markdown", "json" ] }, "render": { "type": "boolean" }, "selector": { "type": "string" }, "extract": { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/ExtractRule" } }, "wait": { "type": "integer" }, "waitFor": { "type": "string" }, "timeout": { "type": "integer" }, "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Allowed: user-agent, accept, accept-language, cookie, referer" }, "raw": { "type": "boolean" } } }, "ScrapeResponse": { "type": "object", "properties": { "url": { "type": "string" }, "finalUrl": { "type": "string" }, "statusCode": { "type": "integer" }, "contentType": { "type": [ "string", "null" ] }, "title": { "type": [ "string", "null" ] }, "rendered": { "type": "boolean" }, "html": { "type": "string" }, "text": { "type": "string" }, "markdown": { "type": "string" }, "data": { "type": "object" }, "truncated": { "type": "boolean" }, "elapsedMs": { "type": "integer" }, "credits": { "type": "integer" } } }, "MetadataResponse": { "type": "object", "properties": { "url": { "type": "string" }, "finalUrl": { "type": "string" }, "title": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "canonical": { "type": [ "string", "null" ] }, "favicon": { "type": [ "string", "null" ] }, "lang": { "type": [ "string", "null" ] }, "og": { "type": "object", "additionalProperties": { "type": "string" } }, "twitter": { "type": "object", "additionalProperties": { "type": "string" } }, "jsonLd": { "type": "array", "items": {} }, "elapsedMs": { "type": "integer" }, "credits": { "type": "integer" } } }, "StructuredResponse": { "type": "object", "properties": { "site": { "type": "string" }, "action": { "type": "string" }, "params": { "type": "object" }, "data": { "type": "object", "description": "Shape depends on the endpoint; see its description." }, "source": { "type": "string", "enum": [ "live", "cache" ] }, "age": { "type": "integer", "description": "seconds since the data was fetched" }, "fetchedAt": { "type": "string", "format": "date-time" }, "credits": { "type": "integer" }, "elapsedMs": { "type": "integer" } } } } } }