Web Access for Your CrewAI Crews

from crewai.tools import tool
import requests

@tool("Web Scraper")
def scrape_website(url: str) -> str:
    """Scrape a website and extract structured content.
    Use this to gather information from any URL.
    Input should be a valid URL."""
    response = requests.post(
        "https://api.mantisapi.com/v1/scrape",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
        json={
            "url": url,
            "extract": {
                "schema": {
                    "title": "string",
                    "main_content": "string",
                    "key_points": ["string"]
                }
            }
        }
    )
    return str(response.json())

from crewai import Agent, Task, Crew

# Researcher agent with web access
researcher = Agent(
    role="Senior Research Analyst",
    goal="Find and analyze information from the web",
    backstory="Expert at gathering intelligence from web sources",
    tools=[scrape_website],
    verbose=True
)

# Writer agent that uses research
writer = Agent(
    role="Content Strategist",
    goal="Create compelling reports from research",
    backstory="Skilled at synthesizing data into actionable insights",
    verbose=True
)

# Define tasks
research_task = Task(
    description="Research the top AI agent frameworks in 2026. Visit their websites and gather feature comparisons.",
    agent=researcher,
    expected_output="Detailed comparison of top 5 frameworks"
)

report_task = Task(
    description="Write a competitive analysis report based on the research.",
    agent=writer,
    expected_output="Professional competitive analysis report"
)

# Assemble and run
crew = Crew(
    agents=[researcher, writer],
    tasks=[research_task, report_task],
    verbose=True
)

result = crew.kickoff()
print(result)