curl --request POST \
--url https://api.example.com/scrape_from_html \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"selector": "<string>",
"scrape_links": true,
"scrape_images": false,
"ignored_tags": [
"<string>"
],
"only_main_content": false,
"only_images": false,
"response_format": "<unknown>",
"instructions": "",
"use_link_placeholders": false,
"frames": [
{
"frameUrl": "<string>",
"frameData": "<string>"
}
]
}
'import requests
url = "https://api.example.com/scrape_from_html"
payload = {
"selector": "<string>",
"scrape_links": True,
"scrape_images": False,
"ignored_tags": ["<string>"],
"only_main_content": False,
"only_images": False,
"response_format": "<unknown>",
"instructions": "",
"use_link_placeholders": False,
"frames": [
{
"frameUrl": "<string>",
"frameData": "<string>"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
selector: '<string>',
scrape_links: true,
scrape_images: false,
ignored_tags: ['<string>'],
only_main_content: false,
only_images: false,
response_format: '<unknown>',
instructions: '',
use_link_placeholders: false,
frames: [{frameUrl: '<string>', frameData: '<string>'}]
})
};
fetch('https://api.example.com/scrape_from_html', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/scrape_from_html",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'selector' => '<string>',
'scrape_links' => true,
'scrape_images' => false,
'ignored_tags' => [
'<string>'
],
'only_main_content' => false,
'only_images' => false,
'response_format' => '<unknown>',
'instructions' => '',
'use_link_placeholders' => false,
'frames' => [
[
'frameUrl' => '<string>',
'frameData' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/scrape_from_html"
payload := strings.NewReader("{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/scrape_from_html")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/scrape_from_html")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"model_schema": {
"success": true,
"model_schema": {},
"error": "<string>"
},
"scrape": {
"success": true,
"error": "<string>",
"data": {}
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Scrape From Html
curl --request POST \
--url https://api.example.com/scrape_from_html \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"selector": "<string>",
"scrape_links": true,
"scrape_images": false,
"ignored_tags": [
"<string>"
],
"only_main_content": false,
"only_images": false,
"response_format": "<unknown>",
"instructions": "",
"use_link_placeholders": false,
"frames": [
{
"frameUrl": "<string>",
"frameData": "<string>"
}
]
}
'import requests
url = "https://api.example.com/scrape_from_html"
payload = {
"selector": "<string>",
"scrape_links": True,
"scrape_images": False,
"ignored_tags": ["<string>"],
"only_main_content": False,
"only_images": False,
"response_format": "<unknown>",
"instructions": "",
"use_link_placeholders": False,
"frames": [
{
"frameUrl": "<string>",
"frameData": "<string>"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
selector: '<string>',
scrape_links: true,
scrape_images: false,
ignored_tags: ['<string>'],
only_main_content: false,
only_images: false,
response_format: '<unknown>',
instructions: '',
use_link_placeholders: false,
frames: [{frameUrl: '<string>', frameData: '<string>'}]
})
};
fetch('https://api.example.com/scrape_from_html', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.example.com/scrape_from_html",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'selector' => '<string>',
'scrape_links' => true,
'scrape_images' => false,
'ignored_tags' => [
'<string>'
],
'only_main_content' => false,
'only_images' => false,
'response_format' => '<unknown>',
'instructions' => '',
'use_link_placeholders' => false,
'frames' => [
[
'frameUrl' => '<string>',
'frameData' => '<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.example.com/scrape_from_html"
payload := strings.NewReader("{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.example.com/scrape_from_html")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.example.com/scrape_from_html")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"selector\": \"<string>\",\n \"scrape_links\": true,\n \"scrape_images\": false,\n \"ignored_tags\": [\n \"<string>\"\n ],\n \"only_main_content\": false,\n \"only_images\": false,\n \"response_format\": \"<unknown>\",\n \"instructions\": \"\",\n \"use_link_placeholders\": false,\n \"frames\": [\n {\n \"frameUrl\": \"<string>\",\n \"frameData\": \"<string>\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"model_schema": {
"success": true,
"model_schema": {},
"error": "<string>"
},
"scrape": {
"success": true,
"error": "<string>",
"data": {}
}
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
The access token received from the authorization server in the OAuth 2.0 flow.
Body
Playwright selector to scope the scrape to. Only content inside this selector will be scraped.
Whether to scrape links from the page. Links are scraped by default.
Whether to scrape images from the page. Images are scraped by default.
HTML tags to ignore from the page
Whether to only scrape the main content of the page. If True, navbars, footers, etc. are excluded.
Whether to only scrape images from the page. If True, the page content is excluded.
The response format to use for the scrape. You can use a Pydantic model or a JSON Schema dict (cf. https://docs.pydantic.dev/latest/concepts/json_schema/#generating-json-schema.)
User description as to what needs to be scraped
Whether to use link/image placeholders to reduce the number of tokens in the prompt and hallucinations. However this is an experimental feature and might not work as expected.
Frame data for all frames found in the page. The main frame has to be the first one
Show child attributes
Show child attributes
Was this page helpful?

