Using Crawl4AI in LangFlow
Prerequisites
Steps
Step 1: Create a Custom Component in Langflow
import httpx
from langflow.custom import CustomComponent
from langflow.inputs import StrInput, SecretStrInput, BoolInput
from langflow.io import Output
from langflow.schema import Data
class Crawl4AIComponent(CustomComponent):
display_name = "Crawl4AI Web Scraper"
description = "Crawl web pages and convert them to clean Markdown using Crawl4AI."
icon = "globe"
inputs = [
StrInput(
name="url",
display_name="Target URL",
info="The web address you want to crawl and scrape.",
value="https://example.com"
),
StrInput(
name="base_url",
display_name="Crawl4AI Base URL",
info="The base URL of your self-hosted Crawl4AI API.",
value="http://localhost:11235"
),
SecretStrInput(
name="api_token",
display_name="API Bearer Token",
info="Authentication token for your Crawl4AI instance if required.",
required=False
),
BoolInput(
name="only_text",
display_name="Only Text",
info="Strip images, layouts, and non-text elements.",
value=True
),
BoolInput(
name="fit_markdown",
display_name="Fit Markdown",
info="Extract only the most semantically relevant content (heavily pruned).",
value=False
)
]
outputs = [
Output(name="markdown", display_name="Markdown Output", method="crawl_page")
]
def crawl_page(self) -> Data:
endpoint = f"{self.base_url.rstrip('/')}/crawl"
headers = {}
if self.api_token:
headers["Authorization"] = f"Bearer {self.api_token}"
payload = {
"urls": [self.url],
"browser_config": {
"text_mode": True,
"light_mode": True
},
"crawler_config": {
"only_text": self.only_text,
"word_count_threshold": 15,
"remove_overlay_elements": True,
"excluded_tags": [
"nav", "footer", "header", "aside", "form", "button",
"iframe", "noscript", "style", "script", "svg"
]
}
}
try:
with httpx.Client(timeout=60.0) as client:
response = client.post(endpoint, json=payload, headers=headers)
response.raise_for_status()
result_data = response.json()
# Crawl4AI returns results in a nested list
first_result = result_data.get("results", [{}])[0]
# Check whether to return standard markdown or high-density "fit" markdown
raw_markdown = first_result.get("markdown", "")
if self.fit_markdown and isinstance(raw_markdown, dict):
output_text = raw_markdown.get("fit_markdown", "")
elif isinstance(raw_markdown, dict):
output_text = raw_markdown.get("raw_markdown", "")
else:
output_text = raw_markdown
return Data(value=output_text)
except Exception as e:
return Data(value=f"Error crawling page: {str(e)}")Step 2: Configure and Test the Component
Step 3: Handle the Crawler Output in your Flow
Tips for Production Flows
Need More Information?
Last updated
Was this helpful?