pdf/snowscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add datetime_handler

Browse source
This commit is contained in:
Marc Nixon 2023-09-01 10:39:33 -07:00
parent ba466810ef
commit c146dbb674
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
snowscraper/controller.py
View file

@ -6,12 +6,10 @@ from pathlib import Path
SCRAPERS = {} SCRAPERS = {}
def register_scraper(cls): def register_scraper(cls):
SCRAPERS[cls.__name__] = cls SCRAPERS[cls.__name__] = cls
return cls return cls
def run_all(args: argparse.Namespace): def run_all(args: argparse.Namespace):
results = {} results = {}
for scraper_cls in SCRAPERS.values(): for scraper_cls in SCRAPERS.values():
@ -19,12 +17,16 @@ def run_all(args: argparse.Namespace):
results.update(scraper.scrape()) results.update(scraper.scrape())
print(results) print(results)
# Save the results to a JSON file def datetime_handler(obj):
with open('results.json', 'w') as json_file: if isinstance(obj, datetime.datetime):
json.dump(results, json_file, indent=4) return obj.isoformat()
raise TypeError("Unknown type")
# Save the results to a JSON file
with open('results.json', 'w') as json_file:
json.dump(results, json_file, indent=4, default=datetime_handler)
def import_scrapers(): def import_scrapers():
directory = Path(__file__).resolve().parent / "scrapers" directory = Path(__file__).resolve().parent / "scrapers"
for module_loader, name, ispkg in pkgutil.iter_modules([directory]): for module_loader, name, ispkg in pkgutil.iter_modules([directory]):
importlib.import_module(f".scrapers.{name}", __package__) importlib.import_module(f".scrapers.{name}", __package__)