From 4016bb096d03b041d02fe06aaae2bb350516eca8 Mon Sep 17 00:00:00 2001 From: Darryl Nixon Date: Fri, 16 Jun 2023 13:17:41 -0700 Subject: [PATCH] Add env file support, by-tab processing --- README.md | 5 +- crowdtls/apis/v1.py | 210 +++++++++++++++++++++++++++++++------------- crowdtls/main.py | 37 ++++++-- crowdtls/models.py | 1 + pyproject.toml | 1 + 5 files changed, 187 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 435b2d8..ada6498 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,13 @@ Below is an enumeration of analytics that are run on the resulting data set to t | Analytic Name | Description | Completeness | | --- | --- | --- | | Multiple Active Certificates | Flag an unusually high number of active certificates for a single FQDN, especially if they're from multiple CAs. | ❌ | -| Short Lifespan Certificates | Flag certificates with a very short lifespan, which could indicate malicious activity. | ❌ | +| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ | | Changes in Certificate Details | Track historical data of certificates for each FQDN and flag abrupt changes. | ❌ | | Certificates from Untrusted CAs | Flag certificates issued by untrusted or less common CAs. | ❌ | | Uncommon SAN Usage | Flag certificates with an unusually high number of SAN entries. | ❌ | | Use of Deprecated or Weak Encryption | Flag certificates that use deprecated or weak cryptographic algorithms. | ❌ | | New Certificate Detection | Alert users when a certificate for a known domain changes unexpectedly. | ❌ | -| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ | -| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields do not match. | ❌ | +| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields match (self-signed) and are not trusted roots. | ❌ | | Geographical Inconsistencies | Flag when the certificate's registration or issuing CA's country doesn't match the usual location of the website. | ❌ | | Suspicious Domains | Flag when the domain in the certificate doesn't match the actual domain of the website. | ❌ | | Unusual Certificate Attributes | Flag deviations in terms of certificate attributes, like too short public key lengths or unusual signature algorithms. | ❌ | diff --git a/crowdtls/apis/v1.py b/crowdtls/apis/v1.py index 03e9d8d..afb3673 100644 --- a/crowdtls/apis/v1.py +++ b/crowdtls/apis/v1.py @@ -1,6 +1,5 @@ from typing import Dict from typing import List -from typing import Union from fastapi import APIRouter from fastapi import Depends @@ -39,83 +38,176 @@ async def insert_certificate(hostname: str, certificate: Certificate, session: A logger.error(f"Failed to insert certificate into database for domain {domain.fqdn}: {certificate.fingerprint}") +# @app.post("/check") +# async def check_fingerprints( +# fingerprints: Dict[str, Union[str, List[str]]], +# request: Request = None, +# session: AsyncSession = Depends(get_session), +# ): +# logger.info("Received request to check fingerprints from client {request.client.host}") +# hostname = parse_hostname(fingerprints.get("host")) +# fps = fingerprints.get("fps") +# logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}") + +# subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery() + +# stmt = ( +# select(Certificate) +# .join(DomainCertificateLink) +# .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn) +# .options(selectinload(Certificate.domains)) +# .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True) +# ) +# try: +# result = await session.execute(stmt) +# except Exception: +# logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") +# raise_HTTPException() + +# certificates = result.scalars().all() +# logger.info( +# f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}" +# ) + +# if len(certificates) == len(fps): +# return {"send": False} + +# for certificate in certificates: +# if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]: +# certificate.domains.append(hostname) +# session.add(certificate) + +# await session.commit() +# logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.") +# return {"send": True} + + @app.post("/check") async def check_fingerprints( - fingerprints: Dict[str, Union[str, List[str]]], + fingerprints: Dict[str, List[str]], request: Request = None, session: AsyncSession = Depends(get_session), ): - logger.info("Received request to check fingerprints from client {request.client.host}") - hostname = parse_hostname(fingerprints.get("host")) - fps = fingerprints.get("fps") - logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}") + logger.info(f"Received request to check fingerprints from client {request.client.host}") - subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery() + response_dict = {} - stmt = ( - select(Certificate) - .join(DomainCertificateLink) - .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn) - .options(selectinload(Certificate.domains)) - .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True) - ) - try: - result = await session.execute(stmt) - except Exception: - logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") - raise_HTTPException() + for hostname, fps in fingerprints.items(): + parsed_hostname = parse_hostname(hostname) + logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host} for host {hostname}") - certificates = result.scalars().all() - logger.info( - f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}" - ) + subquery = ( + select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery() + ) - if len(certificates) == len(fps): - return {"send": False} + stmt = ( + select(Certificate) + .join(DomainCertificateLink) + .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn) + .options(selectinload(Certificate.domains)) + .where(DomainCertificateLink.fqdn == parsed_hostname.fqdn if parsed_hostname else True) + ) - for certificate in certificates: - if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]: - certificate.domains.append(hostname) - session.add(certificate) + try: + result = await session.execute(stmt) + except Exception: + logger.error( + f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" + ) + raise_HTTPException() - await session.commit() - logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.") - return {"send": True} + certificates = result.scalars().all() + logger.info( + f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}" + ) + + if len(certificates) != len(fps): + for certificate in certificates: + if parsed_hostname and parsed_hostname.fqdn not in [domain.fqdn for domain in certificate.domains]: + certificate.domains.append(parsed_hostname) + session.add(certificate) + + await session.commit() + logger.info(f"Added mappings between {parsed_hostname.fqdn} up to {len(fps)} certificates in the database.") + response_dict[hostname] = True + + return response_dict @app.post("/new") async def new_fingerprints( - fingerprints: Dict[str, Union[str, Dict[str, List[int]]]], + fingerprints: Dict[str, Dict[str, List[int]]], request: Request = None, session: AsyncSession = Depends(get_session), ): - try: - hostname = parse_hostname(fingerprints.get("host")) - certs = fingerprints.get("certs") - fps = certs.keys() - stmt = select(Certificate).where(Certificate.fingerprint.in_(fps)) - result = await session.execute(stmt) - except Exception: - logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") - raise_HTTPException() + # Iterate over each hostname and its fingerprints + for hostname, certs in fingerprints.items(): + try: + parsed_hostname = parse_hostname(hostname) + fps = certs.keys() + stmt = select(Certificate).where(Certificate.fingerprint.in_(fps)) + result = await session.execute(stmt) + except Exception: + logger.error( + f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" + ) + raise_HTTPException() - logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}") - existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()} + logger.info(f"Received {len(certs)} fingerprints to add from client {request.client.host} for host {hostname}") + existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()} - certificates_to_add = [] - for fp, rawDER in certs.items(): - if fp not in existing_fingerprints: - decoded = decode_der(fp, rawDER) - certificate = Certificate.from_orm(decoded) - certificate.domains.append(hostname) - certificates_to_add.append(certificate) + certificates_to_add = [] + for fp, rawDER in certs.items(): + if fp not in existing_fingerprints: + decoded = decode_der(fp, rawDER) + certificate = Certificate.from_orm(decoded) + certificate.domains.append(parsed_hostname) + certificates_to_add.append(certificate) - try: - session.add_all(certificates_to_add) - await session.commit() - except Exception: - logger.error( - f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" - ) - raise_HTTPException() + try: + session.add_all(certificates_to_add) + await session.commit() + except Exception: + logger.error( + f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" + ) + raise_HTTPException() return {"status": "OK"} + + +# @app.post("/new") +# async def new_fingerprints( +# fingerprints: Dict[str, Union[str, Dict[str, List[int]]]], +# request: Request = None, +# session: AsyncSession = Depends(get_session), +# ): +# try: +# hostname = parse_hostname(fingerprints.get("host")) +# certs = fingerprints.get("certs") +# fps = certs.keys() +# stmt = select(Certificate).where(Certificate.fingerprint.in_(fps)) +# result = await session.execute(stmt) +# except Exception: +# logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") +# raise_HTTPException() + +# logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}") +# existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()} + +# certificates_to_add = [] +# for fp, rawDER in certs.items(): +# if fp not in existing_fingerprints: +# decoded = decode_der(fp, rawDER) +# certificate = Certificate.from_orm(decoded) +# certificate.domains.append(hostname) +# certificates_to_add.append(certificate) + +# try: +# session.add_all(certificates_to_add) +# await session.commit() +# except Exception: +# logger.error( +# f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" +# ) +# raise_HTTPException() +# return {"status": "OK"} diff --git a/crowdtls/main.py b/crowdtls/main.py index ee2520c..e72772a 100644 --- a/crowdtls/main.py +++ b/crowdtls/main.py @@ -1,13 +1,17 @@ +import argparse import asyncio import sys +from pathlib import Path from types import FrameType import uvicorn import uvloop +from dotenv import load_dotenv from crowdtls.logs import logger -from crowdtls.scheduler import app as app_rocketry -from crowdtls.webserver import app as app_fastapi + +app_fastapi = None +app_rocketry = None class CrowdTLS(uvicorn.Server): @@ -16,7 +20,7 @@ class CrowdTLS(uvicorn.Server): return super().handle_exit(sig, frame) -async def start_server(): +async def start_server() -> None: logger.info("Starting CrowdTLS") server = CrowdTLS(config=uvicorn.Config(app=app_fastapi, workers=1, loop="uvloop")) @@ -26,7 +30,20 @@ async def start_server(): await asyncio.wait([rocket, fastapi], return_when=asyncio.FIRST_COMPLETED) -def run(): +def run(env: Path) -> None: + global app_rocketry + global app_fastapi + + if env.exists(): + try: + load_dotenv(env) + except Exception as e: + logger.error(f"Could not load env file {env}: {e}") + sys.exit(1) + + from crowdtls.scheduler import app as app_rocketry + from crowdtls.webserver import app as app_fastapi + if sys.version_info >= (3, 11): with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner: runner.run(start_server()) @@ -36,4 +53,14 @@ def run(): if __name__ == "__main__": - run() + parser = argparse.ArgumentParser(description="CrowdTLS Server") + parser.add_argument( + "--env", + type=lambda p: Path(p).absolute(), + default=Path(__file__).absolute().parent / ".env", + required=False, + help="Path to specific env file", + ) + print(Path(__file__).absolute().parent / ".env") + args = parser.parse_args() + run(args.env) diff --git a/crowdtls/models.py b/crowdtls/models.py index 3b0051d..3f066cf 100644 --- a/crowdtls/models.py +++ b/crowdtls/models.py @@ -69,6 +69,7 @@ def certificate_loaded(target, context): class AnomalyTypes(SQLModel, table=True): id: int = Field(primary_key=True) + response_code: int anomalyString: str diff --git a/pyproject.toml b/pyproject.toml index b4aa11d..b4e0c06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "tldextract>=3.4.4", "rocketry>=2.5.1", "uvloop>=0.17.0", + "python-dotenv>=1.0.0", ] [project.scripts]