diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fca47d6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.gitignore +.git/ +.env +.env.sample +.flake8 +.pre-commit-config.yaml +.dockerignore +docker-compose.yml +Dockerfile +build/ +**/.DS_Store +**/__pycache__/ +**/*.md +**/*.pyc +**/*.pyo +**/*.mo +**/*.log +*.egg-info/ +*.png diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..72476be --- /dev/null +++ b/.env.sample @@ -0,0 +1,19 @@ +# POSTGRES_* variables are used both to initially configure the +# crowdtls postgresql database and to later access the data from +# crowdtls execution. +# +# POSTGRES_PASSWORD is used for database authentication and should +# be secure or randomized (optionally using the method in README.md). +POSTGRES_CONTAINER=crowdtls-db +POSTGRES_USER=crowdtls +POSTGRES_DB=crowdtls +POSTGRES_PASSWORD= + +# CROWDTLS_*_WEB_PORT variables are used to determine what +# port the web interface is served on within the container (INTERNAL) +# and what port this will map to on the host (HOST). +# +# If you're using a dockerized reverse proxy, you may want to remove +# the port mapping entirely within docker-compose.yml. +CROWDTLS_HOST_WEB_PORT=1337 +CROWDTLS_INTERNAL_WEB_PORT=1337 diff --git a/.gitignore b/.gitignore index 68bc17f..4d2ff84 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,7 @@ cover/ *.pot # Django stuff: -*.log +**/*.log local_settings.py db.sqlite3 db.sqlite3-journal @@ -158,3 +158,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +**/.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a3535e1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11-alpine + +# Enforcement to ensure passwords environment variables are not left blank. +# This won't stop bad passwords from being used, but at least won't cause +# errors or, worse, weaker crypt. +ARG POSTGRES_PASSWORD +ENV ENV_POSTGRES_PASSWORD=${POSTGRES_PASSWORD} +RUN [ ! -z "${ENV_POSTGRES_PASSWORD}" ] || { echo "CrowdTLS-server build error: Set POSTGRES_PASSWORD in .env."; exit 1; } + +# Copy project into Docker image, skipping entries in .dockerignore. +WORKDIR /crowdtls +COPY . . + +# Install ghostforge from the work directory. +RUN pip install . + +# Expose the web "serve" port specific in the environment variables. +ARG CROWDTLS_INTERNAL_WEB_PORT +ENV ENV_CROWDTLS_INTERNAL_WEB_PORT=${CROWDTLS_INTERNAL_WEB_PORT} +EXPOSE ${ENV_CROWDTLS_INTERNAL_WEB_PORT} + +ENV PYTHONPATH=/ghostforge + +# TODO: Replace with ghostforge_serve when it works. +# This currently just keeps the container running for development. +CMD ["sh", "-c", "uvicorn crowdtls.cli:app --host 0.0.0.0 --port $CROWDTLS_INTERNAL_WEB_PORT"] diff --git a/README.md b/README.md index 5025718..a52b0b9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
-CrowdTLS Logo +CrowdTLS Logo # CrowdTLS-server @@ -13,7 +13,18 @@ This is the backend server repository for it.
## Installation -TODO +I recommend that you deploy this with Docker or within a Python virtual environment. + +## Deployment with Docker + +Run the following command on your Linux system: +```bash +git clone https://github.com/darrylnixon/CrowdTLS-server.git && \ +cd CrowdTLS-server && \ +cp .env.sample .env && \ +PW=$(/usr/bin/env python3 -c "import secrets; print(secrets.token_urlsafe(32))") /bin/bash -c 'sed -i "" "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$PW/" .env' && \ +docker-compose up --detach --build; +``` ## FAQ diff --git a/crowdtls/api/v1/api.py b/crowdtls/api/v1/api.py new file mode 100644 index 0000000..03e9d8d --- /dev/null +++ b/crowdtls/api/v1/api.py @@ -0,0 +1,121 @@ +from typing import Dict +from typing import List +from typing import Union + +from fastapi import APIRouter +from fastapi import Depends +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload +from sqlmodel import select + +from crowdtls.db import get_session +from crowdtls.helpers import decode_der +from crowdtls.helpers import parse_hostname +from crowdtls.helpers import raise_HTTPException +from crowdtls.logs import logger +from crowdtls.models import Certificate +from crowdtls.models import Domain +from crowdtls.models import DomainCertificateLink + +app = APIRouter() + + +async def insert_certificate(hostname: str, certificate: Certificate, session: AsyncSession = Depends(get_session)): + domain = parse_hostname(hostname) + if domain: + existing_domain = await session.get(Domain, domain.fqdn) + if existing_domain: + logger.info("Found existing domain in database: {existing_domain.fqdn}") + existing_domain.certificates.append(certificate) + session.add(existing_domain) + else: + logger.info("Did not find existing domain in database. Creating new domain: {domain.fqdn}") + domain.certificates.append(certificate) + session.add(domain) + try: + await session.commit() + except Exception: + logger.error(f"Failed to insert certificate into database for domain {domain.fqdn}: {certificate.fingerprint}") + + +@app.post("/check") +async def check_fingerprints( + fingerprints: Dict[str, Union[str, List[str]]], + request: Request = None, + session: AsyncSession = Depends(get_session), +): + logger.info("Received request to check fingerprints from client {request.client.host}") + hostname = parse_hostname(fingerprints.get("host")) + fps = fingerprints.get("fps") + logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}") + + subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery() + + stmt = ( + select(Certificate) + .join(DomainCertificateLink) + .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn) + .options(selectinload(Certificate.domains)) + .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True) + ) + try: + result = await session.execute(stmt) + except Exception: + logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") + raise_HTTPException() + + certificates = result.scalars().all() + logger.info( + f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}" + ) + + if len(certificates) == len(fps): + return {"send": False} + + for certificate in certificates: + if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]: + certificate.domains.append(hostname) + session.add(certificate) + + await session.commit() + logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.") + return {"send": True} + + +@app.post("/new") +async def new_fingerprints( + fingerprints: Dict[str, Union[str, Dict[str, List[int]]]], + request: Request = None, + session: AsyncSession = Depends(get_session), +): + try: + hostname = parse_hostname(fingerprints.get("host")) + certs = fingerprints.get("certs") + fps = certs.keys() + stmt = select(Certificate).where(Certificate.fingerprint.in_(fps)) + result = await session.execute(stmt) + except Exception: + logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}") + raise_HTTPException() + + logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}") + existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()} + + certificates_to_add = [] + for fp, rawDER in certs.items(): + if fp not in existing_fingerprints: + decoded = decode_der(fp, rawDER) + certificate = Certificate.from_orm(decoded) + certificate.domains.append(hostname) + certificates_to_add.append(certificate) + + try: + session.add_all(certificates_to_add) + await session.commit() + except Exception: + logger.error( + f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}" + ) + raise_HTTPException() + return {"status": "OK"} diff --git a/crowdtls/cli.py b/crowdtls/cli.py index 3016ee0..6816e13 100644 --- a/crowdtls/cli.py +++ b/crowdtls/cli.py @@ -1,50 +1,22 @@ -from typing import Dict -from typing import List - from fastapi import FastAPI -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.ext.asyncio import create_async_engine -from sqlalchemy.future import select -from sqlmodel import SQLModel +from fastapi.middleware.cors import CORSMiddleware -from crowdtls.helpers import decode_der -from db import CertificateChain - -DATABASE_URL = "postgresql+asyncpg://user:password@localhost:5432/database" -engine = create_async_engine(DATABASE_URL, echo=True) +from crowdtls.api.v1.api import app as api_v1_app +from crowdtls.db import create_db_and_tables +from crowdtls.logs import logger app = FastAPI() +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["POST"], allow_headers=["*"]) + +app.include_router(api_v1_app, prefix="/api/v1") + @app.on_event("startup") async def startup_event(): - async with engine.begin() as connection: - await connection.run_sync(SQLModel.metadata.create_all) - - -@app.post("/check") -async def check_fingerprints(fingerprints: Dict[str, List[int]]): - fps = fingerprints.get("fps") - async with AsyncSession(engine) as session: - for fp in fps: - stmt = select(CertificateChain).where(CertificateChain.fingerprint == fp) - result = await session.execute(stmt) - certificate = result.scalars().first() - if not certificate: - return {"send": True} - return {"send": False} - - -@app.post("/new") -async def new_fingerprints(fingerprints: Dict[str, List[int]]): - async with AsyncSession(engine) as session: - for fp, _ in fingerprints.items(): - stmt = select(CertificateChain).where(CertificateChain.fingerprint == fp) - result = await session.execute(stmt) - certificate = result.scalars().first() - if not certificate: - new_certificate = decode_der(fp) - session.add(new_certificate) - pass - await session.commit() - return {"status": "OK"} + logger.info("Creating database and tables") + try: + await create_db_and_tables() + except Exception: + logger.error("Failed to create database and tables") + raise diff --git a/crowdtls/db.py b/crowdtls/db.py index e750f04..31b674f 100644 --- a/crowdtls/db.py +++ b/crowdtls/db.py @@ -1,22 +1,28 @@ -from sqlalchemy import Integer -from sqlalchemy import LargeBinary -from sqlalchemy.dialects.postgresql import JSONB -from sqlmodel import Field +import os + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.orm import sessionmaker from sqlmodel import SQLModel -class CertificateChain(SQLModel, table=True): - id: int = Field(default=None, primary_key=True) - fingerprint: str = Field(index=True) - domain_name: str - raw_der_certificate: LargeBinary - version: int - serial_number: str - signature: LargeBinary - issuer: JSONB - validity: JSONB - subject: JSONB - subject_public_key_info: JSONB - issuer_unique_id: Integer - subject_unique_id: Integer - extensions: JSONB +DATABASE_URL = ( + f'postgresql+asyncpg://{os.environ.get("POSTGRES_USER")}:' + + f'{os.environ.get("POSTGRES_PASSWORD")}@' + + os.environ.get("POSTGRES_CONTAINER") + + f':5432/{os.environ.get("POSTGRES_DB")}' +) + +engine = create_async_engine(DATABASE_URL, echo=True, future=True) + + +async def get_session() -> AsyncSession: + async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with async_session() as session: + yield session + + +async def create_db_and_tables(): + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.drop_all) + await conn.run_sync(SQLModel.metadata.create_all) diff --git a/crowdtls/helpers.py b/crowdtls/helpers.py index c865f55..02e6ce0 100644 --- a/crowdtls/helpers.py +++ b/crowdtls/helpers.py @@ -2,29 +2,46 @@ from typing import List from cryptography import x509 from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from fastapi import HTTPException +from tld import get_tld -from crowdtls.db import CertificateChain +from crowdtls.logs import logger +from crowdtls.models import Certificate +from crowdtls.models import Domain -def decode_der(raw_der_certificate: List[int]) -> CertificateChain: - # Convert list of integers to bytes +def decode_der(fingerprint: str, raw_der_certificate: List[int]) -> Certificate: der_cert_bytes = bytes(raw_der_certificate) - - # Parse the DER certificate cert = x509.load_der_x509_certificate(der_cert_bytes, default_backend()) - certificate_chain = CertificateChain( - raw_der_certificate=der_cert_bytes, + public_key_bytes = cert.public_key().public_bytes( + encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + + return Certificate( + fingerprint=fingerprint, version=cert.version.value, serial_number=cert.serial_number, signature=cert.signature, issuer=cert.issuer.rfc4514_string(), - validity={"not_valid_before": cert.not_valid_before, "not_valid_after": cert.not_valid_after}, + not_valid_before=cert.not_valid_before, + not_valid_after=cert.not_valid_after, subject=cert.subject.rfc4514_string(), - subject_public_key_info=cert.public_key().public_bytes(), - issuer_unique_id=cert.issuer_unique_id, - subject_unique_id=cert.subject_unique_id, - extensions=[str(ext) for ext in cert.extensions], + subject_public_key_info=public_key_bytes, + raw_der_certificate=der_cert_bytes, ) - return certificate_chain + +def parse_hostname(hostname: str) -> Domain: + try: + parsed_domain = get_tld(f"https://{hostname}", as_object=True) + return Domain(fqdn=hostname, root=parsed_domain.domain, tld=parsed_domain.tld) + except Exception: + logger.error(f"Failed to parse hostname: {hostname}") + + +def raise_HTTPException( + status_code: int = 500, detail: str = "Error encountered and reported. Please try again later." +) -> None: + raise HTTPException(status_code=status_code, detail=detail) diff --git a/crowdtls/logs.py b/crowdtls/logs.py new file mode 100644 index 0000000..b6006ad --- /dev/null +++ b/crowdtls/logs.py @@ -0,0 +1,35 @@ +from loguru import logger + + +logger.add( + "app.log", + format="{time:YYYY-MM-DD HH:mm:ss} | {message}", + level="INFO", + rotation="1 day", + retention="30 days", +) + +logger.add( + "errors.log", + format="ℹī¸ {time:YYYY-MM-DD HH:mm:ss} | {message}", + level="WARNING", + rotation="1 day", + retention="30 days", +) + +logger.add( + "error.log", + format="⛔ī¸ {time:YYYY-MM-DD HH:mm:ss} | {message}", + level="ERROR", + rotation="1 day", + retention="30 days", +) + + +logger.add( + "critical.log", + format="🚨 {time:YYYY-MM-DD HH:mm:ss} | {message}", + level="CRITICAL", + rotation="1 day", + retention="30 days", +) diff --git a/crowdtls/models.py b/crowdtls/models.py new file mode 100644 index 0000000..6708eea --- /dev/null +++ b/crowdtls/models.py @@ -0,0 +1,36 @@ +import datetime +from typing import List +from typing import Optional + +from sqlalchemy import LargeBinary +from sqlmodel import Field +from sqlmodel import Relationship +from sqlmodel import SQLModel + + +class DomainCertificateLink(SQLModel, table=True): + fqdn: Optional[str] = Field(default=None, foreign_key="domain.fqdn", primary_key=True) + fingerprint: Optional[str] = Field(default=None, foreign_key="certificate.fingerprint", primary_key=True) + + +class Domain(SQLModel, table=True): + fqdn: str = Field(primary_key=True) + root: str + tld: str + certificates: Optional[List["Certificate"]] = Relationship( + back_populates="domains", link_model=DomainCertificateLink + ) + + +class Certificate(SQLModel, table=True): + fingerprint: str = Field(index=True, primary_key=True) + version: int + serial_number: str + signature: bytes = Field(default_factory=LargeBinary) + issuer: str + not_valid_before: datetime.datetime + not_valid_after: datetime.datetime + subject: str + subject_public_key_info: str + raw_der_certificate: bytes = Field(default_factory=LargeBinary) + domains: Optional[List[Domain]] = Relationship(back_populates="certificates", link_model=DomainCertificateLink) diff --git a/crowdtls/types.py b/crowdtls/types.py new file mode 100644 index 0000000..9703af4 --- /dev/null +++ b/crowdtls/types.py @@ -0,0 +1,90 @@ +import json +from typing import Generic +from typing import TypeVar + +from fastapi.encoders import jsonable_encoder +from pydantic import parse_obj_as +from pydantic.main import ModelMetaclass +from sqlmodel import JSON +from sqlmodel import TypeDecorator + +# Adapted from: https://github.com/tiangolo/sqlmodel/issues/63#issuecomment-1081555082 + +T = TypeVar("T") + + +def pydantic_column_type(pydantic_type): + class PydanticJSONType(TypeDecorator, Generic[T]): + impl = JSON() + + def __init__( + self, + json_encoder=json, + ): + self.json_encoder = json_encoder + super(PydanticJSONType, self).__init__() + + def bind_processor(self, dialect): + impl_processor = self.impl.bind_processor(dialect) + dumps = self.json_encoder.dumps + if impl_processor: + + def process(value: T): + if value is not None: + if isinstance(pydantic_type, ModelMetaclass): + # This allows to assign non-InDB models and if they're + # compatible, they're directly parsed into the InDB + # representation, thus hiding the implementation in the + # background. However, the InDB model will still be returned + value_to_dump = pydantic_type.from_orm(value) + else: + value_to_dump = value + value = jsonable_encoder(value_to_dump) + return impl_processor(value) + + else: + + def process(value): + if isinstance(pydantic_type, ModelMetaclass): + # This allows to assign non-InDB models and if they're + # compatible, they're directly parsed into the InDB + # representation, thus hiding the implementation in the + # background. However, the InDB model will still be returned + value_to_dump = pydantic_type.from_orm(value) + else: + value_to_dump = value + value = dumps(jsonable_encoder(value_to_dump)) + return value + + return process + + def result_processor(self, dialect, coltype) -> T: + impl_processor = self.impl.result_processor(dialect, coltype) + if impl_processor: + + def process(value): + value = impl_processor(value) + if value is None: + return None + + data = value + # Explicitly use the generic directly, not type(T) + full_obj = parse_obj_as(pydantic_type, data) + return full_obj + + else: + + def process(value): + if value is None: + return None + + # Explicitly use the generic directly, not type(T) + full_obj = parse_obj_as(pydantic_type, value) + return full_obj + + return process + + def compare_values(self, x, y): + return x == y + + return PydanticJSONType diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..78be37c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.8' +services: + crowdtls-db: + image: postgres:15.3 + container_name: crowdtls-db + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - crowdtls-db-data:/var/lib/postgresql/data + ports: [ "5432:5432" ] + env_file: [ .env ] + crowdtls: + build: + context: . + dockerfile: Dockerfile + args: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - CROWDTLS_INTERNAL_WEB_PORT=${CROWDTLS_INTERNAL_WEB_PORT} + ports: + [ + "${CROWDTLS_HOST_WEB_PORT}:${CROWDTLS_INTERNAL_WEB_PORT}" + ] + image: crowdtls:latest + container_name: crowdtls + restart: unless-stopped + depends_on: + crowdtls-db: + condition: service_healthy + env_file: [ .env ] +volumes: + crowdtls-db-data: + name: crowdtls-db-data diff --git a/pyproject.toml b/pyproject.toml index b4fc738..861a277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "greenlet==2.0.2", "sqlmodel==0.0.8", "sqlalchemy==1.4.41", + "tld>=0.13", ] [project.urls]