pdf/melamine
1
0
Fork 0
mirror of https://github.com/DarrylNixon/melamine.git synced 2024-04-22 06:27:20 -07:00
Code Issues Projects Releases Packages Wiki Activity

awaits instead of a huuuge task queue

Browse source
This commit is contained in:
Darryl Nixon 2023-07-16 16:31:23 -07:00
parent 4eb95d64ea
commit 2a773f6f1b
2 changed files with 8 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
melamine/fileops.py
View file

@ -35,8 +35,8 @@ async def mount_bound_rglob(path: AsyncPath, mount: AsyncPath, pattern: str, ign
logger.info(f"Skipping ignored subdir: {path}") logger.info(f"Skipping ignored subdir: {path}")
return return
if await path.is_dir(): if await path.is_dir():
if await find_mount(path) == mount: if await find_mount(path) != mount:
logger.info(f"Skipping differently mounted subdir: {path} (wanted {mount}))") logger.info(f"Skipping differently mounted subdir: {path} (wanted {mount})")
return return
async for subpath in path.glob(pattern): async for subpath in path.glob(pattern):
async for subitem in mount_bound_rglob(subpath, mount, pattern, ignoredirs): async for subitem in mount_bound_rglob(subpath, mount, pattern, ignoredirs):

18
melamine/shred.py
View file

@ -10,7 +10,7 @@ from .classes import ShredFile
from .fileops import mount_bound_rglob from .fileops import mount_bound_rglob
from .logs import logger from .logs import logger
IGNORE_GLOBAL = ("/proc", "/dev", "/sys") IGNORE_GLOBAL = ("/proc", "/dev", "/sys", "/run")
async def main(job: argparse.Namespace) -> bool: async def main(job: argparse.Namespace) -> bool:
@ -79,11 +79,10 @@ async def main(job: argparse.Namespace) -> bool:
for mount_point, inodes in inodes_in_mount_points.items(): for mount_point, inodes in inodes_in_mount_points.items():
# checking for . and .. should not be neccessary w/ rglob # checking for . and .. should not be neccessary w/ rglob
tasks = []
# scandir/glob/rglob doesn't play nice with FileNotFound errors, # scandir/glob/rglob doesn't play nice with FileNotFound errors,
# so let's avoid them entirely for now in /proc, /dev, and /sys # so let's avoid them in dynamic fs areas
if str(mount_point) == "/": if str(mount_point) == "/":
logger.info("Root filesystem mount seen, skipping /proc, /dev, and /sys") logger.info("Root filesystem mount processing")
async for item in mount_point.glob("*"): async for item in mount_point.glob("*"):
if await item.is_dir(): if await item.is_dir():
if str(item) in IGNORE_GLOBAL: if str(item) in IGNORE_GLOBAL:
@ -96,17 +95,12 @@ async def main(job: argparse.Namespace) -> bool:
no = await item.stat() no = await item.stat()
logger.warning(f"{abso} inode is {no.st_ino}") logger.warning(f"{abso} inode is {no.st_ino}")
logger.warning(f"is {abso} in inodes? {str(bool(no.st_ino in inodes))}") logger.warning(f"is {abso} in inodes? {str(bool(no.st_ino in inodes))}")
tasks.append(check_inode_and_unlink(subitem, inodes)) await check_inode_and_unlink(subitem, inodes)
else: else:
tasks.append(check_inode_and_unlink(item, inodes)) await check_inode_and_unlink(item, inodes)
else: else:
logger.info(f"Checking non-root filesystem mount: {str(mount_point)}") logger.info(f"Checking non-root filesystem mount: {str(mount_point)}")
async for item in mount_bound_rglob(mount_point, mount_point, "*", job.ignoredir): async for item in mount_bound_rglob(mount_point, mount_point, "*", job.ignoredir):
tasks.append(check_inode_and_unlink(item, inodes)) await check_inode_and_unlink(item, inodes)
done, _ = await asyncio.wait(tasks)
for task in done:
e = task.exception()
if e:
logger.trace(f"Error during find: {e}")
logger.info("Done") logger.info("Done")