You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
39 lines
1.1 KiB
39 lines
1.1 KiB
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
from swarms.agents.tools.base import BaseToolSet, tool
|
|
from swarms.utils.logger import logger
|
|
|
|
|
|
class RequestsGet(BaseToolSet):
|
|
@tool(
|
|
name="Requests Get",
|
|
description="A portal to the internet. "
|
|
"Use this when you need to get specific content from a website."
|
|
"Input should be a url (i.e. https://www.google.com)."
|
|
"The output will be the text response of the GET request.",
|
|
)
|
|
def get(self, url: str) -> str:
|
|
"""Run the tool."""
|
|
html = requests.get(url).text
|
|
soup = BeautifulSoup(html)
|
|
non_readable_tags = soup.find_all(
|
|
["script", "style", "header", "footer", "form"]
|
|
)
|
|
|
|
for non_readable_tag in non_readable_tags:
|
|
non_readable_tag.extract()
|
|
|
|
content = soup.get_text("\n", strip=True)
|
|
|
|
if len(content) > 300:
|
|
content = content[:300] + "..."
|
|
|
|
logger.debug(
|
|
f"\nProcessed RequestsGet, Input Url: {url} " f"Output Contents: {content}"
|
|
)
|
|
|
|
return content
|
|
|