You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
66 lines
2.3 KiB
66 lines
2.3 KiB
1 year ago
|
"""
|
||
|
|
Paper Swarm
|
||
|
|
1. Scrape https://huggingface.co/papers for all papers, by search for all links on the paper with a /papers/, then clicks, gets the header, and then the abstract.
|
||
|
|
and various links and then adds them to a txt file for each paper on https://huggingface.co/papers
|
||
|
|
|
||
|
|
2. Feed prompts iteratively into Anthropic for summarizations + value score on impact, reliability, and novel, and other paper ranking mechanisms
|
||
|
|
|
||
|
|
3. Store papers in a database with metadata. Agents can use retrieval
|
||
|
|
|
||
|
|
4. Discord Bot // Twitter Bot
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
import requests
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
import os
|
||
|
|
|
||
|
|
class Paper:
|
||
|
|
def __init__(self, title, date, authors, abstract):
|
||
|
|
self.title = title
|
||
|
|
self.date = date
|
||
|
|
self.authors = authors
|
||
|
|
self.abstract = abstract
|
||
|
|
|
||
|
|
class Scraper:
|
||
|
|
def __init__(self, url):
|
||
|
|
self.url = url
|
||
|
|
|
||
|
|
def get_paper_links(self):
|
||
|
|
response = requests.get(self.url)
|
||
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||
|
|
links = [a['href'] for a in soup.find_all('a', href=True) if '/papers/' in a['href']]
|
||
|
|
return links
|
||
|
|
|
||
|
|
def get_paper_details(self, link):
|
||
|
|
response = requests.get(self.url + link)
|
||
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||
|
|
title = soup.find('h1').text
|
||
|
|
date_tag = soup.find('time')
|
||
|
|
date = date_tag.text if date_tag else 'Unknown'
|
||
|
|
authors = [author.text for author in soup.find_all('span', class_='author')]
|
||
|
|
abstract_tag = soup.find('div', class_='abstract')
|
||
|
|
abstract = abstract_tag.text if abstract_tag else 'Abstract not found'
|
||
|
|
return Paper(title, date, authors, abstract)
|
||
|
|
|
||
|
|
class FileWriter:
|
||
|
|
def __init__(self, directory):
|
||
|
|
self.directory = directory
|
||
|
|
|
||
|
|
def write_paper(self, paper):
|
||
|
|
with open(os.path.join(self.directory, paper.title + '.txt'), 'w') as f:
|
||
|
|
f.write(f"h1: {paper.title}\n")
|
||
|
|
f.write(f"Published on {paper.date}\n")
|
||
|
|
f.write("Authors:\n")
|
||
|
|
for author in paper.authors:
|
||
|
|
f.write(f"{author}\n")
|
||
|
|
f.write("Abstract\n")
|
||
|
|
f.write(paper.abstract)
|
||
|
|
|
||
|
|
scraper = Scraper('https://huggingface.co/papers')
|
||
|
|
file_writer = FileWriter('images')
|
||
|
|
|
||
|
|
links = scraper.get_paper_links()
|
||
|
|
for link in links:
|
||
|
|
paper = scraper.get_paper_details(link)
|
||
|
|
file_writer.write_paper(paper)
|