Add Rocketry for analytics processing, add uvloop

crowdtls/analytics/main.py

@@ -0,0 +1,11 @@
+from fastapi import Depends
+from rocketry.conds import hourly
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from crowdtls.db import get_session
+from crowdtls.scheduler import app as schedule
+
+
+@schedule.task(hourly)
+async def find_anomalies(session: AsyncSession = Depends(get_session)):
+    pass

crowdtls/api.py

@@ -1,14 +1,12 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
-from crowdtls.api.v1.api import app as api_v1_app
 from crowdtls.db import create_db_and_tables
 from crowdtls.logs import logger
+from crowdtls.v1.api import app as api_v1_app
 
 app = FastAPI()
-
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["POST"], allow_headers=["*"])
-
 app.include_router(api_v1_app, prefix="/api/v1")
 
 
@@ -20,3 +18,7 @@ async def startup_event():
     except Exception:
         logger.error("Failed to create database and tables")
         raise
+
+
+if __name__ == "__main__":
+    app.run()

crowdtls/helpers.py

@@ -1,10 +1,10 @@
 from typing import List
 
+import tldextract
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from fastapi import HTTPException
-from tld import get_tld
 
 from crowdtls.logs import logger
 from crowdtls.models import Certificate
@@ -35,8 +35,8 @@ def decode_der(fingerprint: str, raw_der_certificate: List[int]) -> Certificate:
 
 def parse_hostname(hostname: str) -> Domain:
     try:
-        parsed_domain = get_tld(f"https://{hostname}", as_object=True)
+        parsed_domain = tldextract.extract(hostname)
-        return Domain(fqdn=hostname, root=parsed_domain.domain, tld=parsed_domain.tld)
+        return Domain(fqdn=hostname, root=parsed_domain.domain, tld=parsed_domain.suffix)
     except Exception:
         logger.error(f"Failed to parse hostname: {hostname}")
 

crowdtls/main.py

@@ -0,0 +1,39 @@
+import asyncio
+import sys
+from types import FrameType
+
+import uvicorn
+import uvloop
+
+from crowdtls.api import app as app_fastapi
+from crowdtls.logs import logger
+from crowdtls.scheduler import app as app_rocketry
+
+
+class CrowdTLS(uvicorn.Server):
+    def handle_exit(self, sig: int, frame: FrameType) -> None:
+        logger.info("Shutting down CrowdTLS")
+        return super().handle_exit(sig, frame)
+
+
+async def start_server():
+    logger.info("Starting CrowdTLS")
+    server = CrowdTLS(config=uvicorn.Config(app=app_fastapi, workers=1, loop="uvloop"))
+
+    fastapi = asyncio.create_task(server.serve())
+    rocket = asyncio.create_task(app_rocketry.serve())
+
+    await asyncio.wait([rocket, fastapi], return_when=asyncio.FIRST_COMPLETED)
+
+
+def run():
+    if sys.version_info >= (3, 11):
+        with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner:
+            runner.run(start_server())
+    else:
+        uvloop.install()
+        asyncio.run(start_server())
+
+
+if __name__ == "__main__":
+    run()

crowdtls/scheduler.py

@@ -0,0 +1,6 @@
+from rocketry import Rocketry
+
+app = Rocketry(execution="async")
+
+if __name__ == "__main__":
+    app.run()

pyproject.toml

@@ -19,9 +19,14 @@ dependencies = [
     "greenlet==2.0.2",
     "sqlmodel==0.0.8",
     "sqlalchemy==1.4.41",
-    "tld>=0.13",
+    "tldextract>=3.4.4",
+    "rocketry>=2.5.1",
+    "uvloop>=0.17.0",
 ]
 
+[project.scripts]
+crowdtls = "crowdtls:main.run"
+
 [project.urls]
 homepage = "https://github.com/DarrylNixon/CrowdTLS"
 repository = "https://github.com/DarrylNixon/CrowdTLS-server"