pdf/snowscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

nvm, medium has a built-in tag filter

Browse source
This commit is contained in:
hackish 2023-08-14 15:56:40 -07:00
parent 652b4b6acf
commit 8310befc76
1 changed files with 1 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
snowscraper/scrapers/medium.py
View file

@ -10,7 +10,7 @@ from ..scraper import BaseScraper
@register_scraper @register_scraper
class MediumScraper(BaseScraper): class MediumScraper(BaseScraper):
url = "https://medium.com/feed/snowflake" url = "https://medium.com/feed/snowflake/tagged/snowflake"
def __init__(self, after, *args, **kwargs): def __init__(self, after, *args, **kwargs):
super(MediumScraper, self).__init__(*args, **kwargs) super(MediumScraper, self).__init__(*args, **kwargs)
@ -20,8 +20,6 @@ class MediumScraper(BaseScraper):
def scrape(self): def scrape(self):
print("Scraping Medium") print("Scraping Medium")
for entry in feedparser.parse(MediumScraper.url)["entries"]: for entry in feedparser.parse(MediumScraper.url)["entries"]:
if not any(tag["term"] == "snowflake" for tag in entry["tags"]):
continue
updated = string_to_datetime(entry["updated"]) updated = string_to_datetime(entry["updated"])
if updated > self.after: if updated > self.after:
self.data[entry["link"]] = { self.data[entry["link"]] = {