"""
Paper Swarm
1. Scrape https://huggingface.co/papers for all papers, by search for all links on the paper with a /papers/, then clicks, gets the header, and then the abstract.
and various links and then adds them to a txt file for each paper on https://huggingface.co/papers

2. Feed prompts iteratively into Anthropic for summarizations + value score on impact, reliability, and novel, and other paper ranking mechanisms

3. Store papers in a database with metadata. Agents can use retrieval

4. Discord Bot // Twitter Bot
"""


import requests
from bs4 import BeautifulSoup
import os

class Paper:
    def __init__(self, title, date, authors, abstract):
        self.title = title
        self.date = date
        self.authors = authors
        self.abstract = abstract

class Scraper:
    def __init__(self, url):
        self.url = url

    def get_paper_links(self):
        response = requests.get(self.url)
        soup = BeautifulSoup(response.text, 'html.parser')
        links = [a['href'] for a in soup.find_all('a', href=True) if '/papers/' in a['href']]
        return links

    def get_paper_details(self, link):
        response = requests.get(self.url + link)
        soup = BeautifulSoup(response.text, 'html.parser')
        title = soup.find('h1').text
        date_tag = soup.find('time')
        date = date_tag.text if date_tag else 'Unknown'
        authors = [author.text for author in soup.find_all('span', class_='author')]
        abstract_tag = soup.find('div', class_='abstract')
        abstract = abstract_tag.text if abstract_tag else 'Abstract not found'
        return Paper(title, date, authors, abstract)

class FileWriter:
    def __init__(self, directory):
        self.directory = directory

    def write_paper(self, paper):
        with open(os.path.join(self.directory, paper.title + '.txt'), 'w') as f:
            f.write(f"h1: {paper.title}\n")
            f.write(f"Published on {paper.date}\n")
            f.write("Authors:\n")
            for author in paper.authors:
                f.write(f"{author}\n")
            f.write("Abstract\n")
            f.write(paper.abstract)

scraper = Scraper('https://huggingface.co/papers')
file_writer = FileWriter('images')

links = scraper.get_paper_links()
for link in links:
    paper = scraper.get_paper_details(link)
    file_writer.write_paper(paper)