Add env file support, by-tab processing

README.md

@@ -34,14 +34,13 @@ Below is an enumeration of analytics that are run on the resulting data set to t
 | Analytic Name | Description | Completeness |
 | --- | --- | --- |
 | Multiple Active Certificates | Flag an unusually high number of active certificates for a single FQDN, especially if they're from multiple CAs. | ❌ |
-| Short Lifespan Certificates | Flag certificates with a very short lifespan, which could indicate malicious activity. | ❌ |
+| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ |
 | Changes in Certificate Details | Track historical data of certificates for each FQDN and flag abrupt changes. | ❌ |
 | Certificates from Untrusted CAs | Flag certificates issued by untrusted or less common CAs. | ❌ |
 | Uncommon SAN Usage | Flag certificates with an unusually high number of SAN entries. | ❌ |
 | Use of Deprecated or Weak Encryption | Flag certificates that use deprecated or weak cryptographic algorithms. | ❌ |
 | New Certificate Detection | Alert users when a certificate for a known domain changes unexpectedly. | ❌ |
-| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ |
+| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields match (self-signed) and are not trusted roots. | ❌ |
-| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields do not match. | ❌ |
 | Geographical Inconsistencies | Flag when the certificate's registration or issuing CA's country doesn't match the usual location of the website. | ❌ |
 | Suspicious Domains | Flag when the domain in the certificate doesn't match the actual domain of the website. | ❌ |
 | Unusual Certificate Attributes | Flag deviations in terms of certificate attributes, like too short public key lengths or unusual signature algorithms. | ❌ |

crowdtls/apis/v1.py

@@ -1,6 +1,5 @@
 from typing import Dict
 from typing import List
-from typing import Union
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -39,83 +38,176 @@ async def insert_certificate(hostname: str, certificate: Certificate, session: A
         logger.error(f"Failed to insert certificate into database for domain {domain.fqdn}: {certificate.fingerprint}")
 
 
+# @app.post("/check")
+# async def check_fingerprints(
+#     fingerprints: Dict[str, Union[str, List[str]]],
+#     request: Request = None,
+#     session: AsyncSession = Depends(get_session),
+# ):
+#     logger.info("Received request to check fingerprints from client {request.client.host}")
+#     hostname = parse_hostname(fingerprints.get("host"))
+#     fps = fingerprints.get("fps")
+#     logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}")
+
+#     subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
+
+#     stmt = (
+#         select(Certificate)
+#         .join(DomainCertificateLink)
+#         .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
+#         .options(selectinload(Certificate.domains))
+#         .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True)
+#     )
+#     try:
+#         result = await session.execute(stmt)
+#     except Exception:
+#         logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
+#         raise_HTTPException()
+
+#     certificates = result.scalars().all()
+#     logger.info(
+#         f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
+#     )
+
+#     if len(certificates) == len(fps):
+#         return {"send": False}
+
+#     for certificate in certificates:
+#         if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
+#             certificate.domains.append(hostname)
+#             session.add(certificate)
+
+#     await session.commit()
+#     logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.")
+#     return {"send": True}
+
+
 @app.post("/check")
 async def check_fingerprints(
-    fingerprints: Dict[str, Union[str, List[str]]],
+    fingerprints: Dict[str, List[str]],
     request: Request = None,
     session: AsyncSession = Depends(get_session),
 ):
-    logger.info("Received request to check fingerprints from client {request.client.host}")
+    logger.info(f"Received request to check fingerprints from client {request.client.host}")
-    hostname = parse_hostname(fingerprints.get("host"))
-    fps = fingerprints.get("fps")
-    logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}")
 
-    subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
+    response_dict = {}
 
-    stmt = (
+    for hostname, fps in fingerprints.items():
-        select(Certificate)
+        parsed_hostname = parse_hostname(hostname)
-        .join(DomainCertificateLink)
+        logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host} for host {hostname}")
-        .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
-        .options(selectinload(Certificate.domains))
-        .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True)
-    )
-    try:
-        result = await session.execute(stmt)
-    except Exception:
-        logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
-        raise_HTTPException()
 
-    certificates = result.scalars().all()
+        subquery = (
-    logger.info(
+            select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
-        f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
+        )
-    )
 
-    if len(certificates) == len(fps):
+        stmt = (
-        return {"send": False}
+            select(Certificate)
+            .join(DomainCertificateLink)
+            .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
+            .options(selectinload(Certificate.domains))
+            .where(DomainCertificateLink.fqdn == parsed_hostname.fqdn if parsed_hostname else True)
+        )
 
-    for certificate in certificates:
+        try:
-        if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
+            result = await session.execute(stmt)
-            certificate.domains.append(hostname)
+        except Exception:
-            session.add(certificate)
+            logger.error(
+                f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
+            )
+            raise_HTTPException()
 
-    await session.commit()
+        certificates = result.scalars().all()
-    logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.")
+        logger.info(
-    return {"send": True}
+            f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
+        )
+
+        if len(certificates) != len(fps):
+            for certificate in certificates:
+                if parsed_hostname and parsed_hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
+                    certificate.domains.append(parsed_hostname)
+                    session.add(certificate)
+
+            await session.commit()
+            logger.info(f"Added mappings between {parsed_hostname.fqdn} up to {len(fps)} certificates in the database.")
+            response_dict[hostname] = True
+
+    return response_dict
 
 
 @app.post("/new")
 async def new_fingerprints(
-    fingerprints: Dict[str, Union[str, Dict[str, List[int]]]],
+    fingerprints: Dict[str, Dict[str, List[int]]],
     request: Request = None,
     session: AsyncSession = Depends(get_session),
 ):
-    try:
+    # Iterate over each hostname and its fingerprints
-        hostname = parse_hostname(fingerprints.get("host"))
+    for hostname, certs in fingerprints.items():
-        certs = fingerprints.get("certs")
+        try:
-        fps = certs.keys()
+            parsed_hostname = parse_hostname(hostname)
-        stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
+            fps = certs.keys()
-        result = await session.execute(stmt)
+            stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
-    except Exception:
+            result = await session.execute(stmt)
-        logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
+        except Exception:
-        raise_HTTPException()
+            logger.error(
+                f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
+            )
+            raise_HTTPException()
 
-    logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}")
+        logger.info(f"Received {len(certs)} fingerprints to add from client {request.client.host} for host {hostname}")
-    existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
+        existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
 
-    certificates_to_add = []
+        certificates_to_add = []
-    for fp, rawDER in certs.items():
+        for fp, rawDER in certs.items():
-        if fp not in existing_fingerprints:
+            if fp not in existing_fingerprints:
-            decoded = decode_der(fp, rawDER)
+                decoded = decode_der(fp, rawDER)
-            certificate = Certificate.from_orm(decoded)
+                certificate = Certificate.from_orm(decoded)
-            certificate.domains.append(hostname)
+                certificate.domains.append(parsed_hostname)
-            certificates_to_add.append(certificate)
+                certificates_to_add.append(certificate)
 
-    try:
+        try:
-        session.add_all(certificates_to_add)
+            session.add_all(certificates_to_add)
-        await session.commit()
+            await session.commit()
-    except Exception:
+        except Exception:
-        logger.error(
+            logger.error(
-            f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
+                f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
-        )
+            )
-        raise_HTTPException()
+            raise_HTTPException()
     return {"status": "OK"}
+
+
+# @app.post("/new")
+# async def new_fingerprints(
+#     fingerprints: Dict[str, Union[str, Dict[str, List[int]]]],
+#     request: Request = None,
+#     session: AsyncSession = Depends(get_session),
+# ):
+#     try:
+#         hostname = parse_hostname(fingerprints.get("host"))
+#         certs = fingerprints.get("certs")
+#         fps = certs.keys()
+#         stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
+#         result = await session.execute(stmt)
+#     except Exception:
+#         logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
+#         raise_HTTPException()
+
+#     logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}")
+#     existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
+
+#     certificates_to_add = []
+#     for fp, rawDER in certs.items():
+#         if fp not in existing_fingerprints:
+#             decoded = decode_der(fp, rawDER)
+#             certificate = Certificate.from_orm(decoded)
+#             certificate.domains.append(hostname)
+#             certificates_to_add.append(certificate)
+
+#     try:
+#         session.add_all(certificates_to_add)
+#         await session.commit()
+#     except Exception:
+#         logger.error(
+#             f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
+#         )
+#         raise_HTTPException()
+#     return {"status": "OK"}

crowdtls/main.py

@@ -1,13 +1,17 @@
+import argparse
 import asyncio
 import sys
+from pathlib import Path
 from types import FrameType
 
 import uvicorn
 import uvloop
+from dotenv import load_dotenv
 
 from crowdtls.logs import logger
-from crowdtls.scheduler import app as app_rocketry
+
-from crowdtls.webserver import app as app_fastapi
+app_fastapi = None
+app_rocketry = None
 
 
 class CrowdTLS(uvicorn.Server):
@@ -16,7 +20,7 @@ class CrowdTLS(uvicorn.Server):
         return super().handle_exit(sig, frame)
 
 
-async def start_server():
+async def start_server() -> None:
     logger.info("Starting CrowdTLS")
     server = CrowdTLS(config=uvicorn.Config(app=app_fastapi, workers=1, loop="uvloop"))
 
@@ -26,7 +30,20 @@ async def start_server():
     await asyncio.wait([rocket, fastapi], return_when=asyncio.FIRST_COMPLETED)
 
 
-def run():
+def run(env: Path) -> None:
+    global app_rocketry
+    global app_fastapi
+
+    if env.exists():
+        try:
+            load_dotenv(env)
+        except Exception as e:
+            logger.error(f"Could not load env file {env}: {e}")
+            sys.exit(1)
+
+    from crowdtls.scheduler import app as app_rocketry
+    from crowdtls.webserver import app as app_fastapi
+
     if sys.version_info >= (3, 11):
         with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner:
             runner.run(start_server())
@@ -36,4 +53,14 @@ def run():
 
 
 if __name__ == "__main__":
-    run()
+    parser = argparse.ArgumentParser(description="CrowdTLS Server")
+    parser.add_argument(
+        "--env",
+        type=lambda p: Path(p).absolute(),
+        default=Path(__file__).absolute().parent / ".env",
+        required=False,
+        help="Path to specific env file",
+    )
+    print(Path(__file__).absolute().parent / ".env")
+    args = parser.parse_args()
+    run(args.env)

crowdtls/models.py

@@ -69,6 +69,7 @@ def certificate_loaded(target, context):
 
 class AnomalyTypes(SQLModel, table=True):
     id: int = Field(primary_key=True)
+    response_code: int
     anomalyString: str
 
 

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "tldextract>=3.4.4",
     "rocketry>=2.5.1",
     "uvloop>=0.17.0",
+    "python-dotenv>=1.0.0",
 ]
 
 [project.scripts]