Add adminer and auto-trimming of cert keys

README.md

@@ -26,6 +26,30 @@ PW=$(/usr/bin/env python3 -c "import secrets; print(secrets.token_urlsafe(32))")
 docker-compose up --detach --build;
 ```
 
+## Analytics
+
+Below is an enumeration of analytics that are run on the resulting data set to trigger alerts for client extensions. For the "Completeness" column, the emoji symbols are used to represent the status: ❌ for "not started", ⌛ for "partial", and ✅ for "done".
+
+| Analytic Name | Description | Completeness |
+| --- | --- | --- |
+| Multiple Active Certificates | Flag an unusually high number of active certificates for a single FQDN, especially if they're from multiple CAs. | ❌ |
+| Short Lifespan Certificates | Flag certificates with a very short lifespan, which could indicate malicious activity. | ❌ |
+| Changes in Certificate Details | Track historical data of certificates for each FQDN and flag abrupt changes. | ❌ |
+| Certificates from Untrusted CAs | Flag certificates issued by untrusted or less common CAs. | ❌ |
+| Uncommon SAN Usage | Flag certificates with an unusually high number of SAN entries. | ❌ |
+| Use of Deprecated or Weak Encryption | Flag certificates that use deprecated or weak cryptographic algorithms. | ❌ |
+| New Certificate Detection | Alert users when a certificate for a known domain changes unexpectedly. | ❌ |
+| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ |
+| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields do not match. | ❌ |
+| Geographical Inconsistencies | Flag when the certificate's registration or issuing CA's country doesn't match the usual location of the website. | ❌ |
+| Suspicious Domains | Flag when the domain in the certificate doesn't match the actual domain of the website. | ❌ |
+| Unusual Certificate Attributes | Flag deviations in terms of certificate attributes, like too short public key lengths or unusual signature algorithms. | ❌ |
+| Wildcard Certificates | Flag unexpected uses of wildcard certificates. | ❌ |
+| Chain of Trust Verification | Flag if the certificate doesn't chain up correctly to a trusted root. | ❌ |
+| SAN Anomalies | Flag if the SAN field includes unusual or suspicious domains. | ❌ |
+| Frequency of Certificate Change | Flag if a certificate for a domain changes more frequently than the norm. | ❌ |
+| Compare with Public CT Logs | Detect anomalies if the certificate presented doesn't match what's found in public CT logs. | ❌ |
+
 ## FAQ
 
 **What is this? I'm looking for the browser extension!**

adminer.Dockerfile

@@ -0,0 +1,3 @@
+FROM adminer:latest
+WORKDIR /var/www/html
+COPY ./adminer.css /var/www/html

crowdtls/main.py

@@ -5,9 +5,9 @@ from types import FrameType
 import uvicorn
 import uvloop
 
-from crowdtls.api import app as app_fastapi
 from crowdtls.logs import logger
 from crowdtls.scheduler import app as app_rocketry
+from crowdtls.webserver import app as app_fastapi
 
 
 class CrowdTLS(uvicorn.Server):

crowdtls/models.py

@@ -2,6 +2,7 @@ import datetime
 from typing import List
 from typing import Optional
 
+from sqlalchemy import event
 from sqlalchemy import LargeBinary
 from sqlmodel import Field
 from sqlmodel import Relationship
@@ -11,12 +12,21 @@ from sqlmodel import SQLModel
 class DomainCertificateLink(SQLModel, table=True):
     fqdn: Optional[str] = Field(default=None, foreign_key="domain.fqdn", primary_key=True)
     fingerprint: Optional[str] = Field(default=None, foreign_key="certificate.fingerprint", primary_key=True)
+    first_linked: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
+
+
+class CertificateAnomalyFlagsLink(SQLModel, table=True):
+    certificate_fingerprint: str = Field(foreign_key="certificate.fingerprint", primary_key=True)
+    anomaly_flag_id: int = Field(foreign_key="anomalyflags.id", primary_key=True)
+    first_linked: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
 
 
 class Domain(SQLModel, table=True):
     fqdn: str = Field(primary_key=True)
     root: str
     tld: str
+    first_seen: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
+    last_seen: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
     certificates: Optional[List["Certificate"]] = Relationship(
         back_populates="domains", link_model=DomainCertificateLink
     )
@@ -33,4 +43,40 @@ class Certificate(SQLModel, table=True):
     subject: str
     subject_public_key_info: str
     raw_der_certificate: bytes = Field(default_factory=LargeBinary)
+    first_seen: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
+    last_seen: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
+    seen_count: int = Field(default=1)
     domains: Optional[List[Domain]] = Relationship(back_populates="certificates", link_model=DomainCertificateLink)
+    anomalies: Optional[List["AnomalyFlags"]] = Relationship(
+        back_populates="certificates", link_model=CertificateAnomalyFlagsLink
+    )
+
+
+@event.listens_for(Certificate, "before_insert")
+@event.listens_for(Certificate, "before_update")
+def certificate_written(mapper, connection, target):
+    target.subject_public_key_info = target.subject_public_key_info.lstrip("-----BEGIN PUBLIC KEY-----\n").rstrip(
+        "\n-----END PUBLIC KEY-----"
+    )
+
+
+@event.listens_for(Certificate, "load")
+def certificate_loaded(target, context):
+    target.subject_public_key_info = (
+        f"-----BEGIN PUBLIC KEY-----\n{target.subject_public_key_info}\n-----END PUBLIC KEY-----"
+    )
+
+
+class AnomalyTypes(SQLModel, table=True):
+    id: int = Field(primary_key=True)
+    anomalyString: str
+
+
+class AnomalyFlags(SQLModel, table=True):
+    id: int = Field(primary_key=True)
+    details: str
+    anomaly_type_id: int = Field(foreign_key=AnomalyTypes.id)
+    date_flagged: datetime.datetime = Field(default_factory=datetime.datetime.utcnow)
+    certificates: Optional[List[Certificate]] = Relationship(
+        back_populates="anomalies", link_model=CertificateAnomalyFlagsLink
+    )

docker-compose.yml

@@ -1,5 +1,6 @@
 version: '3.8'
 services:
+
   crowdtls-db:
     image: postgres:15.3
     container_name: crowdtls-db
@@ -13,10 +14,11 @@ services:
       - crowdtls-db-data:/var/lib/postgresql/data
     ports: [ "5432:5432" ]
     env_file: [ .env ]
+
   crowdtls:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: crowdtls.Dockerfile
       args:
         - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
         - CROWDTLS_INTERNAL_WEB_PORT=${CROWDTLS_INTERNAL_WEB_PORT}
@@ -31,6 +33,20 @@ services:
       crowdtls-db:
         condition: service_healthy
     env_file: [ .env ]
+
+  crowdtls-adminer:
+    image: adminer:latest
+    container_name: crowdtls-adminer
+    build:
+      context: .
+      dockerfile: adminer.Dockerfile
+    restart: unless-stopped
+    ports:
+      - 8001:8080
+    depends_on:
+      crowdtls-db:
+        condition: service_healthy
+
 volumes:
   crowdtls-db-data:
     name: crowdtls-db-data