OpenAI Function Calling + Web Scraping

import openai, requests, json

# Define web scraping as an OpenAI function
tools = [{
    "type": "function",
    "function": {
        "name": "scrape_website",
        "description": "Scrape a website URL and extract structured content including title, main text, and links.",
        "parameters": {
            "type": "object",
            "properties": {
                "url": {
                    "type": "string",
                    "description": "The URL to scrape"
                }
            },
            "required": ["url"]
        }
    }
}]

def scrape_website(url):
    response = requests.post(
        "https://api.mantisapi.com/v1/scrape",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
        json={
            "url": url,
            "extract": {
                "schema": {
                    "title": "string",
                    "main_content": "string",
                    "links": ["string"]
                }
            }
        }
    )
    return json.dumps(response.json())

client = openai.OpenAI()

messages = [
    {"role": "user", "content": "What are the top 3 stories on Hacker News right now?"}
]

# First call — GPT decides to use the scraping tool
response = client.chat.completions.create(
    model="gpt-4",
    messages=messages,
    tools=tools
)

# Handle tool calls
if response.choices[0].message.tool_calls:
    tool_call = response.choices[0].message.tool_calls[0]
    args = json.loads(tool_call.function.arguments)
    
    # Execute the scrape
    result = scrape_website(args["url"])
    
    # Send result back to GPT
    messages.append(response.choices[0].message)
    messages.append({
        "role": "tool",
        "tool_call_id": tool_call.id,
        "content": result
    })
    
    # Final response with web data
    final = client.chat.completions.create(
        model="gpt-4",
        messages=messages,
        tools=tools
    )
    print(final.choices[0].message.content)