pdf/snowscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add datetime_handler

Browse source
This commit is contained in:
Marc Nixon 2023-09-01 10:39:33 -07:00
parent ba466810ef
commit c146dbb674
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
snowscraper/controller.py
View file

@ -6,12 +6,10 @@ from pathlib import Path
SCRAPERS = {} SCRAPERS = {}
def register_scraper(cls): def register_scraper(cls):
SCRAPERS[cls.__name__] = cls SCRAPERS[cls.__name__] = cls
return cls return cls
def run_all(args: argparse.Namespace): def run_all(args: argparse.Namespace):
results = {} results = {}
for scraper_cls in SCRAPERS.values(): for scraper_cls in SCRAPERS.values():
@ -19,10 +17,14 @@ def run_all(args: argparse.Namespace):
results.update(scraper.scrape()) results.update(scraper.scrape())
print(results) print(results)
def datetime_handler(obj):
if isinstance(obj, datetime.datetime):
return obj.isoformat()
raise TypeError("Unknown type")
# Save the results to a JSON file # Save the results to a JSON file
with open('results.json', 'w') as json_file: with open('results.json', 'w') as json_file:
json.dump(results, json_file, indent=4) json.dump(results, json_file, indent=4, default=datetime_handler)
def import_scrapers(): def import_scrapers():
directory = Path(__file__).resolve().parent / "scrapers" directory = Path(__file__).resolve().parent / "scrapers"