pdf/CrowdTLS-server
pdf/CrowdTLS-server
1
0
Fork 0
mirror of https://github.com/DarrylNixon/CrowdTLS-server.git synced 2024-09-22 18:19:43 -07:00
Code Issues Projects Releases Packages Wiki Activity

Add env file support, by-tab processing

Browse source
This commit is contained in:
Darryl Nixon 2023-06-16 13:17:41 -07:00
parent 75b7f2427f
commit 4016bb096d
5 changed files with 187 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
README.md
View file

@ -34,14 +34,13 @@ Below is an enumeration of analytics that are run on the resulting data set to t
| Analytic Name | Description | Completeness |
| --- | --- | --- |
| Multiple Active Certificates | Flag an unusually high number of active certificates for a single FQDN, especially if they're from multiple CAs. | ❌ |
| Short Lifespan Certificates | Flag certificates with a very short lifespan, which could indicate malicious activity. | ❌ |
| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ |
| Changes in Certificate Details | Track historical data of certificates for each FQDN and flag abrupt changes. | ❌ |
| Certificates from Untrusted CAs | Flag certificates issued by untrusted or less common CAs. | ❌ |
| Uncommon SAN Usage | Flag certificates with an unusually high number of SAN entries. | ❌ |
| Use of Deprecated or Weak Encryption | Flag certificates that use deprecated or weak cryptographic algorithms. | ❌ |
| New Certificate Detection | Alert users when a certificate for a known domain changes unexpectedly. | ❌ |
| Certificate Lifespan Analysis | Flag certificates with unusually short or long lifespans. | ❌ |
| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields do not match. | ❌ |
| Mismatched Issuer and Subject | Flag certificates where the issuer and subject fields match (self-signed) and are not trusted roots. | ❌ |
| Geographical Inconsistencies | Flag when the certificate's registration or issuing CA's country doesn't match the usual location of the website. | ❌ |
| Suspicious Domains | Flag when the domain in the certificate doesn't match the actual domain of the website. | ❌ |
| Unusual Certificate Attributes | Flag deviations in terms of certificate attributes, like too short public key lengths or unusual signature algorithms. | ❌ |

210
crowdtls/apis/v1.py
View file

@ -1,6 +1,5 @@
from typing import Dict
from typing import List
from typing import Union
from fastapi import APIRouter
from fastapi import Depends
@ -39,83 +38,176 @@ async def insert_certificate(hostname: str, certificate: Certificate, session: A
logger.error(f"Failed to insert certificate into database for domain {domain.fqdn}: {certificate.fingerprint}")
# @app.post("/check")
# async def check_fingerprints(
# fingerprints: Dict[str, Union[str, List[str]]],
# request: Request = None,
# session: AsyncSession = Depends(get_session),
# ):
# logger.info("Received request to check fingerprints from client {request.client.host}")
# hostname = parse_hostname(fingerprints.get("host"))
# fps = fingerprints.get("fps")
# logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}")
# subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
# stmt = (
# select(Certificate)
# .join(DomainCertificateLink)
# .join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
# .options(selectinload(Certificate.domains))
# .where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True)
# )
# try:
# result = await session.execute(stmt)
# except Exception:
# logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
# raise_HTTPException()
# certificates = result.scalars().all()
# logger.info(
# f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
# )
# if len(certificates) == len(fps):
# return {"send": False}
# for certificate in certificates:
# if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
# certificate.domains.append(hostname)
# session.add(certificate)
# await session.commit()
# logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.")
# return {"send": True}
@app.post("/check")
async def check_fingerprints(
fingerprints: Dict[str, Union[str, List[str]]],
fingerprints: Dict[str, List[str]],
request: Request = None,
session: AsyncSession = Depends(get_session),
):
logger.info("Received request to check fingerprints from client {request.client.host}")
hostname = parse_hostname(fingerprints.get("host"))
fps = fingerprints.get("fps")
logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host}")
logger.info(f"Received request to check fingerprints from client {request.client.host}")
subquery = select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
response_dict = {}
stmt = (
select(Certificate)
.join(DomainCertificateLink)
.join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
.options(selectinload(Certificate.domains))
.where(DomainCertificateLink.fqdn == hostname.fqdn if hostname else True)
)
try:
result = await session.execute(stmt)
except Exception:
logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
raise_HTTPException()
for hostname, fps in fingerprints.items():
parsed_hostname = parse_hostname(hostname)
logger.info(f"Received {len(fps)} fingerprints to check from client {request.client.host} for host {hostname}")
certificates = result.scalars().all()
logger.info(
f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
)
subquery = (
select(DomainCertificateLink.fqdn).join(Certificate).where(Certificate.fingerprint.in_(fps)).subquery()
)
if len(certificates) == len(fps):
return {"send": False}
stmt = (
select(Certificate)
.join(DomainCertificateLink)
.join(subquery, DomainCertificateLink.fqdn == subquery.c.fqdn)
.options(selectinload(Certificate.domains))
.where(DomainCertificateLink.fqdn == parsed_hostname.fqdn if parsed_hostname else True)
)
for certificate in certificates:
if hostname and hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
certificate.domains.append(hostname)
session.add(certificate)
try:
result = await session.execute(stmt)
except Exception:
logger.error(
f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
)
raise_HTTPException()
await session.commit()
logger.info(f"Added mappings between {hostname.fqdn} up to {len(fps)} certificates in the database.")
return {"send": True}
certificates = result.scalars().all()
logger.info(
f"Found {len(certificates)} certificates (of {len(fps)} requested) in the database for client {request.client.host}"
)
if len(certificates) != len(fps):
for certificate in certificates:
if parsed_hostname and parsed_hostname.fqdn not in [domain.fqdn for domain in certificate.domains]:
certificate.domains.append(parsed_hostname)
session.add(certificate)
await session.commit()
logger.info(f"Added mappings between {parsed_hostname.fqdn} up to {len(fps)} certificates in the database.")
response_dict[hostname] = True
return response_dict
@app.post("/new")
async def new_fingerprints(
fingerprints: Dict[str, Union[str, Dict[str, List[int]]]],
fingerprints: Dict[str, Dict[str, List[int]]],
request: Request = None,
session: AsyncSession = Depends(get_session),
):
try:
hostname = parse_hostname(fingerprints.get("host"))
certs = fingerprints.get("certs")
fps = certs.keys()
stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
result = await session.execute(stmt)
except Exception:
logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
raise_HTTPException()
# Iterate over each hostname and its fingerprints
for hostname, certs in fingerprints.items():
try:
parsed_hostname = parse_hostname(hostname)
fps = certs.keys()
stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
result = await session.execute(stmt)
except Exception:
logger.error(
f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
)
raise_HTTPException()
logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}")
existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
logger.info(f"Received {len(certs)} fingerprints to add from client {request.client.host} for host {hostname}")
existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
certificates_to_add = []
for fp, rawDER in certs.items():
if fp not in existing_fingerprints:
decoded = decode_der(fp, rawDER)
certificate = Certificate.from_orm(decoded)
certificate.domains.append(hostname)
certificates_to_add.append(certificate)
certificates_to_add = []
for fp, rawDER in certs.items():
if fp not in existing_fingerprints:
decoded = decode_der(fp, rawDER)
certificate = Certificate.from_orm(decoded)
certificate.domains.append(parsed_hostname)
certificates_to_add.append(certificate)
try:
session.add_all(certificates_to_add)
await session.commit()
except Exception:
logger.error(
f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
)
raise_HTTPException()
try:
session.add_all(certificates_to_add)
await session.commit()
except Exception:
logger.error(
f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
)
raise_HTTPException()
return {"status": "OK"}
# @app.post("/new")
# async def new_fingerprints(
# fingerprints: Dict[str, Union[str, Dict[str, List[int]]]],
# request: Request = None,
# session: AsyncSession = Depends(get_session),
# ):
# try:
# hostname = parse_hostname(fingerprints.get("host"))
# certs = fingerprints.get("certs")
# fps = certs.keys()
# stmt = select(Certificate).where(Certificate.fingerprint.in_(fps))
# result = await session.execute(stmt)
# except Exception:
# logger.error(f"Failed to execute stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}")
# raise_HTTPException()
# logger.info(f"Received {len(fingerprints)} fingerprints to add from client {request.client.host}")
# existing_fingerprints = {certificate.fingerprint for certificate in result.scalars().all()}
# certificates_to_add = []
# for fp, rawDER in certs.items():
# if fp not in existing_fingerprints:
# decoded = decode_der(fp, rawDER)
# certificate = Certificate.from_orm(decoded)
# certificate.domains.append(hostname)
# certificates_to_add.append(certificate)
# try:
# session.add_all(certificates_to_add)
# await session.commit()
# except Exception:
# logger.error(
# f"Failed to add certificates to db: {certificates_to_add} after stmt: {stmt} (req body {request.body}) and IP address: {request.client.host}"
# )
# raise_HTTPException()
# return {"status": "OK"}

37
crowdtls/main.py
View file

@ -1,13 +1,17 @@
import argparse
import asyncio
import sys
from pathlib import Path
from types import FrameType
import uvicorn
import uvloop
from dotenv import load_dotenv
from crowdtls.logs import logger
from crowdtls.scheduler import app as app_rocketry
from crowdtls.webserver import app as app_fastapi
app_fastapi = None
app_rocketry = None
class CrowdTLS(uvicorn.Server):
@ -16,7 +20,7 @@ class CrowdTLS(uvicorn.Server):
return super().handle_exit(sig, frame)
async def start_server():
async def start_server() -> None:
logger.info("Starting CrowdTLS")
server = CrowdTLS(config=uvicorn.Config(app=app_fastapi, workers=1, loop="uvloop"))
@ -26,7 +30,20 @@ async def start_server():
await asyncio.wait([rocket, fastapi], return_when=asyncio.FIRST_COMPLETED)
def run():
def run(env: Path) -> None:
global app_rocketry
global app_fastapi
if env.exists():
try:
load_dotenv(env)
except Exception as e:
logger.error(f"Could not load env file {env}: {e}")
sys.exit(1)
from crowdtls.scheduler import app as app_rocketry
from crowdtls.webserver import app as app_fastapi
if sys.version_info >= (3, 11):
with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner:
runner.run(start_server())
@ -36,4 +53,14 @@ def run():
if __name__ == "__main__":
run()
parser = argparse.ArgumentParser(description="CrowdTLS Server")
parser.add_argument(
"--env",
type=lambda p: Path(p).absolute(),
default=Path(__file__).absolute().parent / ".env",
required=False,
help="Path to specific env file",
)
print(Path(__file__).absolute().parent / ".env")
args = parser.parse_args()
run(args.env)

1
crowdtls/models.py
View file

@ -69,6 +69,7 @@ def certificate_loaded(target, context):
class AnomalyTypes(SQLModel, table=True):
id: int = Field(primary_key=True)
response_code: int
anomalyString: str

1
pyproject.toml
View file

@ -22,6 +22,7 @@ dependencies = [
"tldextract>=3.4.4",
"rocketry>=2.5.1",
"uvloop>=0.17.0",
"python-dotenv>=1.0.0",
]
[project.scripts]