Initial MVP

.dockerignore

@@ -0,0 +1,19 @@
+.gitignore
+.git/
+.env
+.env.sample
+.flake8
+.pre-commit-config.yaml
+.dockerignore
+docker-compose.yml
+Dockerfile
+build/
+**/.DS_Store
+**/__pycache__/
+**/*.md
+**/*.pyc
+**/*.pyo
+**/*.mo
+**/*.log
+*.egg-info/
+*.png

.env.sample

@@ -0,0 +1,19 @@
+# POSTGRES_* variables are used both to initially configure the
+# crowdtls postgresql database and to later access the data from
+# crowdtls execution.
+#
+# POSTGRES_PASSWORD is used for database authentication and should
+# be secure or randomized (optionally using the method in README.md).
+POSTGRES_CONTAINER=crowdtls-db
+POSTGRES_USER=crowdtls
+POSTGRES_DB=crowdtls
+POSTGRES_PASSWORD=
+
+# CROWDTLS_*_WEB_PORT variables are used to determine what
+# port the web interface is served on within the container (INTERNAL)
+# and what port this will map to on the host (HOST).
+#
+# If you're using a dockerized reverse proxy, you may want to remove
+# the port mapping entirely within docker-compose.yml.
+CROWDTLS_HOST_WEB_PORT=1337
+CROWDTLS_INTERNAL_WEB_PORT=1337

.gitignore

@@ -56,7 +56,7 @@ cover/
 *.pot
 
 # Django stuff:
-*.log
+**/*.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+**/.DS_Store

Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.11-alpine
+
+# Enforcement to ensure passwords environment variables are not left blank.
+# This won't stop bad passwords from being used, but at least won't cause
+# errors or, worse, weaker crypt.
+ARG POSTGRES_PASSWORD
+ENV ENV_POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+RUN [ ! -z "${ENV_POSTGRES_PASSWORD}" ] || { echo "CrowdTLS-server build error: Set POSTGRES_PASSWORD in .env."; exit 1; }
+
+# Copy project into Docker image, skipping entries in .dockerignore.
+WORKDIR /crowdtls
+COPY . .
+
+# Install ghostforge from the work directory.
+RUN pip install .
+
+# Expose the web "serve" port specific in the environment variables.
+ARG CROWDTLS_INTERNAL_WEB_PORT
+ENV ENV_CROWDTLS_INTERNAL_WEB_PORT=${CROWDTLS_INTERNAL_WEB_PORT}
+EXPOSE ${ENV_CROWDTLS_INTERNAL_WEB_PORT}
+
+ENV PYTHONPATH=/ghostforge
+
+# TODO: Replace with ghostforge_serve when it works.
+# This currently just keeps the container running for development.
+CMD ["sh", "-c", "uvicorn crowdtls.cli:app --host 0.0.0.0 --port $CROWDTLS_INTERNAL_WEB_PORT"]

README.md

@@ -1,5 +1,5 @@
 <div align="center">
-<img src="icons/crowdtls.png" alt="CrowdTLS Logo">
+<img src="crowdtls.png" alt="CrowdTLS Logo">
 
 # CrowdTLS-server
 
@@ -13,7 +13,18 @@ This is the backend server repository for it.<br/>
 
 ## Installation
 
-TODO
+I recommend that you deploy this with Docker or within a Python virtual environment.
+
+## Deployment with Docker
+
+Run the following command on your Linux system:
+```bash
+git clone https://github.com/darrylnixon/CrowdTLS-server.git && \
+cd CrowdTLS-server && \
+cp .env.sample .env && \
+PW=$(/usr/bin/env python3 -c "import secrets; print(secrets.token_urlsafe(32))") /bin/bash -c 'sed -i "" "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$PW/" .env' && \
+docker-compose up --detach --build;
+```
 
 ## FAQ
 

crowdtls/api/v1/api.py

@@ -0,0 +1,121 @@
+from typing import Dict
+from typing import List
+from typing import Union
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Request
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+from sqlmodel import select
+
+from crowdtls.db import get_session
+from crowdtls.helpers import decode_der
+from crowdtls.helpers import parse_hostname
+from crowdtls.helpers import raise_HTTPException
+from crowdtls.logs import logger
+from crowdtls.models import Certificate
+from crowdtls.models import Domain
+from crowdtls.models import DomainCertificateLink
+
+app = APIRouter()
+
+
+async def insert_certificate(hostname: str, certificate: Certificate, session: AsyncSession = Depends(get_session)):
+    domain = parse_hostname(hostname)
+    if domain:
+        existing_domain = await session.get(Domain, domain.fqdn)
+        if existing_domain:
+            logger.info("Found existing domain in database: {existing_domain.fqdn}")
+            existing_domain.certificates.append(certificate)
+            session.add(existing_domain)
+        else:
+            logger.info("Did not find existing domain in database. Creating new domain: {domain.fqdn}")
+            domain.certificates.append(certificate)
+            session.add(domain)
+    try:
+        await session.commit()
+    except Exception:
+        logger.error(f"Failed to insert certificate into database for domain {domain.fqdn}: {certificate.fingerprint}")
+
+
+@app.post("/check")
+async def check_fingerprints(
+    fingerprints: Dict[str, Union[str, List[str]]],
+    request: Request = None,
+    session: AsyncSession = Depends(get_session),
+):
+    logger.info("Received request to check fingerprints from client {request.client.host}")
+    hostname = parse_hostname(fingerprints.get("host"))
+    fps = fingerprints.get("fps")
+    logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}")
+
+    subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
+
+    stmt = (
+        select(Certificate)
+        .join(DomainCertificateLink)
+        .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
+        .options(selectinload(Certificate.domains))
+        .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True)
+    )
+    try:
+        result = await session.execute(stmt)
+    except Exception:
+        logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
+        raise_HTTPException()
+
+    certificates = result.scalars().all()
+    logger.info(
+        f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
+    )
+
+    if len(certificates) == len(fps):
+        return {"send": False}
+
+    for certificate in certificates:
+        if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
+            certificate.domains.append(hostname)
+            session.add(certificate)
+
+    await session.commit()
+    logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.")
+    return {"send": True}
+
+
+@app.post("/new")
+async def new_fingerprints(
+    fingerprints: Dict[str, Union[str, Dict[str, List[int]]]],
+    request: Request = None,
+    session: AsyncSession = Depends(get_session),
+):
+    try:
+        hostname = parse_hostname(fingerprints.get("host"))
+        certs = fingerprints.get("certs")
+        fps = certs.keys()
+        stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
+        result = await session.execute(stmt)
+    except Exception:
+        logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
+        raise_HTTPException()
+
+    logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}")
+    existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
+
+    certificates_to_add = []
+    for fp, rawDER in certs.items():
+        if fp not in existing_fingerprints:
+            decoded = decode_der(fp, rawDER)
+            certificate = Certificate.from_orm(decoded)
+            certificate.domains.append(hostname)
+            certificates_to_add.append(certificate)
+
+    try:
+        session.add_all(certificates_to_add)
+        await session.commit()
+    except Exception:
+        logger.error(
+            f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
+        )
+        raise_HTTPException()
+    return {"status": "OK"}

crowdtls/cli.py

@@ -1,50 +1,22 @@
-from typing import Dict
-from typing import List
-
 from fastapi import FastAPI
-from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.ext.asyncio import create_async_engine
-from sqlalchemy.future import select
-from sqlmodel import SQLModel
+from fastapi.middleware.cors import CORSMiddleware
 
-from crowdtls.helpers import decode_der
-from db import CertificateChain
-
-DATABASE_URL = "postgresql+asyncpg://user:password@localhost:5432/database"
-engine = create_async_engine(DATABASE_URL, echo=True)
+from crowdtls.api.v1.api import app as api_v1_app
+from crowdtls.db import create_db_and_tables
+from crowdtls.logs import logger
 
 app = FastAPI()
 
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["POST"], allow_headers=["*"])
+
+app.include_router(api_v1_app, prefix="/api/v1")
+
 
 @app.on_event("startup")
 async def startup_event():
-    async with engine.begin() as connection:
-        await connection.run_sync(SQLModel.metadata.create_all)
-
-
-@app.post("/check")
-async def check_fingerprints(fingerprints: Dict[str, List[int]]):
-    fps = fingerprints.get("fps")
-    async with AsyncSession(engine) as session:
-        for fp in fps:
-            stmt = select(CertificateChain).where(CertificateChain.fingerprint == fp)
-            result = await session.execute(stmt)
-            certificate = result.scalars().first()
-            if not certificate:
-                return {"send": True}
-    return {"send": False}
-
-
-@app.post("/new")
-async def new_fingerprints(fingerprints: Dict[str, List[int]]):
-    async with AsyncSession(engine) as session:
-        for fp, _ in fingerprints.items():
-            stmt = select(CertificateChain).where(CertificateChain.fingerprint == fp)
-            result = await session.execute(stmt)
-            certificate = result.scalars().first()
-            if not certificate:
-                new_certificate = decode_der(fp)
-                session.add(new_certificate)
-                pass
-        await session.commit()
-    return {"status": "OK"}
+    logger.info("Creating database and tables")
+    try:
+        await create_db_and_tables()
+    except Exception:
+        logger.error("Failed to create database and tables")
+        raise

crowdtls/db.py

@@ -1,22 +1,28 @@
-from sqlalchemy import Integer
-from sqlalchemy import LargeBinary
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlmodel import Field
+import os
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.orm import sessionmaker
 from sqlmodel import SQLModel
 
 
-class CertificateChain(SQLModel, table=True):
-    id: int = Field(default=None, primary_key=True)
-    fingerprint: str = Field(index=True)
-    domain_name: str
-    raw_der_certificate: LargeBinary
-    version: int
-    serial_number: str
-    signature: LargeBinary
-    issuer: JSONB
-    validity: JSONB
-    subject: JSONB
-    subject_public_key_info: JSONB
-    issuer_unique_id: Integer
-    subject_unique_id: Integer
-    extensions: JSONB
+DATABASE_URL = (
+    f'postgresql+asyncpg://{os.environ.get("POSTGRES_USER")}:'
+    + f'{os.environ.get("POSTGRES_PASSWORD")}@'
+    + os.environ.get("POSTGRES_CONTAINER")
+    + f':5432/{os.environ.get("POSTGRES_DB")}'
+)
+
+engine = create_async_engine(DATABASE_URL, echo=True, future=True)
+
+
+async def get_session() -> AsyncSession:
+    async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    async with async_session() as session:
+        yield session
+
+
+async def create_db_and_tables():
+    async with engine.begin() as conn:
+        await conn.run_sync(SQLModel.metadata.drop_all)
+        await conn.run_sync(SQLModel.metadata.create_all)

crowdtls/helpers.py

@@ -2,29 +2,46 @@ from typing import List
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from fastapi import HTTPException
+from tld import get_tld
 
-from crowdtls.db import CertificateChain
+from crowdtls.logs import logger
+from crowdtls.models import Certificate
+from crowdtls.models import Domain
 
 
-def decode_der(raw_der_certificate: List[int]) -> CertificateChain:
-    # Convert list of integers to bytes
+def decode_der(fingerprint: str, raw_der_certificate: List[int]) -> Certificate:
     der_cert_bytes = bytes(raw_der_certificate)
-
-    # Parse the DER certificate
     cert = x509.load_der_x509_certificate(der_cert_bytes, default_backend())
 
-    certificate_chain = CertificateChain(
-        raw_der_certificate=der_cert_bytes,
+    public_key_bytes = cert.public_key().public_bytes(
+        encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo
+    )
+
+    return Certificate(
+        fingerprint=fingerprint,
         version=cert.version.value,
         serial_number=cert.serial_number,
         signature=cert.signature,
         issuer=cert.issuer.rfc4514_string(),
-        validity={"not_valid_before": cert.not_valid_before, "not_valid_after": cert.not_valid_after},
+        not_valid_before=cert.not_valid_before,
+        not_valid_after=cert.not_valid_after,
         subject=cert.subject.rfc4514_string(),
-        subject_public_key_info=cert.public_key().public_bytes(),
-        issuer_unique_id=cert.issuer_unique_id,
-        subject_unique_id=cert.subject_unique_id,
-        extensions=[str(ext) for ext in cert.extensions],
+        subject_public_key_info=public_key_bytes,
+        raw_der_certificate=der_cert_bytes,
     )
 
-    return certificate_chain
+
+def parse_hostname(hostname: str) -> Domain:
+    try:
+        parsed_domain = get_tld(f"https://{hostname}", as_object=True)
+        return Domain(fqdn=hostname, root=parsed_domain.domain, tld=parsed_domain.tld)
+    except Exception:
+        logger.error(f"Failed to parse hostname: {hostname}")
+
+
+def raise_HTTPException(
+    status_code: int = 500, detail: str = "Error encountered and reported. Please try again later."
+) -> None:
+    raise HTTPException(status_code=status_code, detail=detail)

crowdtls/logs.py

@@ -0,0 +1,35 @@
+from loguru import logger
+
+
+logger.add(
+    "app.log",
+    format="<level><light-blue>{time:YYYY-MM-DD HH:mm:ss} | {message}</light-blue></level>",
+    level="INFO",
+    rotation="1 day",
+    retention="30 days",
+)
+
+logger.add(
+    "errors.log",
+    format="<level><yellow>ℹ️ {time:YYYY-MM-DD HH:mm:ss} | {message}</yellow></level>",
+    level="WARNING",
+    rotation="1 day",
+    retention="30 days",
+)
+
+logger.add(
+    "error.log",
+    format="<level><red>⛔️ {time:YYYY-MM-DD HH:mm:ss} | {message}</red></level>",
+    level="ERROR",
+    rotation="1 day",
+    retention="30 days",
+)
+
+
+logger.add(
+    "critical.log",
+    format="<level><magenta>🚨 {time:YYYY-MM-DD HH:mm:ss} | {message}</magenta></level>",
+    level="CRITICAL",
+    rotation="1 day",
+    retention="30 days",
+)

crowdtls/models.py

@@ -0,0 +1,36 @@
+import datetime
+from typing import List
+from typing import Optional
+
+from sqlalchemy import LargeBinary
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+
+class DomainCertificateLink(SQLModel, table=True):
+    fqdn: Optional[str] = Field(default=None, foreign_key="domain.fqdn", primary_key=True)
+    fingerprint: Optional[str] = Field(default=None, foreign_key="certificate.fingerprint", primary_key=True)
+
+
+class Domain(SQLModel, table=True):
+    fqdn: str = Field(primary_key=True)
+    root: str
+    tld: str
+    certificates: Optional[List["Certificate"]] = Relationship(
+        back_populates="domains", link_model=DomainCertificateLink
+    )
+
+
+class Certificate(SQLModel, table=True):
+    fingerprint: str = Field(index=True, primary_key=True)
+    version: int
+    serial_number: str
+    signature: bytes = Field(default_factory=LargeBinary)
+    issuer: str
+    not_valid_before: datetime.datetime
+    not_valid_after: datetime.datetime
+    subject: str
+    subject_public_key_info: str
+    raw_der_certificate: bytes = Field(default_factory=LargeBinary)
+    domains: Optional[List[Domain]] = Relationship(back_populates="certificates", link_model=DomainCertificateLink)

crowdtls/types.py

@@ -0,0 +1,90 @@
+import json
+from typing import Generic
+from typing import TypeVar
+
+from fastapi.encoders import jsonable_encoder
+from pydantic import parse_obj_as
+from pydantic.main import ModelMetaclass
+from sqlmodel import JSON
+from sqlmodel import TypeDecorator
+
+# Adapted from: https://github.com/tiangolo/sqlmodel/issues/63#issuecomment-1081555082
+
+T = TypeVar("T")
+
+
+def pydantic_column_type(pydantic_type):
+    class PydanticJSONType(TypeDecorator, Generic[T]):
+        impl = JSON()
+
+        def __init__(
+            self,
+            json_encoder=json,
+        ):
+            self.json_encoder = json_encoder
+            super(PydanticJSONType, self).__init__()
+
+        def bind_processor(self, dialect):
+            impl_processor = self.impl.bind_processor(dialect)
+            dumps = self.json_encoder.dumps
+            if impl_processor:
+
+                def process(value: T):
+                    if value is not None:
+                        if isinstance(pydantic_type, ModelMetaclass):
+                            # This allows to assign non-InDB models and if they're
+                            # compatible, they're directly parsed into the InDB
+                            # representation, thus hiding the implementation in the
+                            # background. However, the InDB model will still be returned
+                            value_to_dump = pydantic_type.from_orm(value)
+                        else:
+                            value_to_dump = value
+                        value = jsonable_encoder(value_to_dump)
+                    return impl_processor(value)
+
+            else:
+
+                def process(value):
+                    if isinstance(pydantic_type, ModelMetaclass):
+                        # This allows to assign non-InDB models and if they're
+                        # compatible, they're directly parsed into the InDB
+                        # representation, thus hiding the implementation in the
+                        # background. However, the InDB model will still be returned
+                        value_to_dump = pydantic_type.from_orm(value)
+                    else:
+                        value_to_dump = value
+                    value = dumps(jsonable_encoder(value_to_dump))
+                    return value
+
+            return process
+
+        def result_processor(self, dialect, coltype) -> T:
+            impl_processor = self.impl.result_processor(dialect, coltype)
+            if impl_processor:
+
+                def process(value):
+                    value = impl_processor(value)
+                    if value is None:
+                        return None
+
+                    data = value
+                    # Explicitly use the generic directly, not type(T)
+                    full_obj = parse_obj_as(pydantic_type, data)
+                    return full_obj
+
+            else:
+
+                def process(value):
+                    if value is None:
+                        return None
+
+                    # Explicitly use the generic directly, not type(T)
+                    full_obj = parse_obj_as(pydantic_type, value)
+                    return full_obj
+
+            return process
+
+        def compare_values(self, x, y):
+            return x == y
+
+    return PydanticJSONType

docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+services:
+  crowdtls-db:
+    image: postgres:15.3
+    container_name: crowdtls-db
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - crowdtls-db-data:/var/lib/postgresql/data
+    ports: [ "5432:5432" ]
+    env_file: [ .env ]
+  crowdtls:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+        - CROWDTLS_INTERNAL_WEB_PORT=${CROWDTLS_INTERNAL_WEB_PORT}
+    ports:
+      [
+        "${CROWDTLS_HOST_WEB_PORT}:${CROWDTLS_INTERNAL_WEB_PORT}"
+      ]
+    image: crowdtls:latest
+    container_name: crowdtls
+    restart: unless-stopped
+    depends_on:
+      crowdtls-db:
+        condition: service_healthy
+    env_file: [ .env ]
+volumes:
+  crowdtls-db-data:
+    name: crowdtls-db-data

pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
     "greenlet==2.0.2",
     "sqlmodel==0.0.8",
     "sqlalchemy==1.4.41",
+    "tld>=0.13",
 ]
 
 [project.urls]