diff --git a/.DEBIAN/control b/.DEBIAN/control index 1bab5e2..c2f6ccd 100644 --- a/.DEBIAN/control +++ b/.DEBIAN/control @@ -2,7 +2,7 @@ Package: fastapi-dls Version: 0.0 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de -Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl +Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl Recommends: curl Installed-Size: 10240 Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls diff --git a/.PKGBUILD/PKGBUILD b/.PKGBUILD/PKGBUILD index 820d8cc..09f606b 100644 --- a/.PKGBUILD/PKGBUILD +++ b/.PKGBUILD/PKGBUILD @@ -48,7 +48,6 @@ package() { install -Dm755 "$srcdir/$pkgname/app/main.py" "$pkgdir/opt/$pkgname/main.py" install -Dm755 "$srcdir/$pkgname/app/orm.py" "$pkgdir/opt/$pkgname/orm.py" install -Dm755 "$srcdir/$pkgname/app/util.py" "$pkgdir/opt/$pkgname/util.py" - install -Dm755 "$srcdir/$pkgname/app/middleware.py" "$pkgdir/opt/$pkgname/middleware.py" install -Dm644 "$srcdir/$pkgname.default" "$pkgdir/etc/default/$pkgname" install -Dm644 "$srcdir/$pkgname.service" "$pkgdir/usr/lib/systemd/system/$pkgname.service" install -Dm644 "$srcdir/$pkgname.tmpfiles" "$pkgdir/usr/lib/tmpfiles.d/$pkgname.conf" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8aa3bd..69c3c8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,7 +127,7 @@ build:pacman: - "*.pkg.tar.zst" test: - image: $IMAGE + image: python:3.12-slim-bookworm stage: test interruptible: true rules: @@ -142,14 +142,11 @@ test: DATABASE: sqlite:///../app/db.sqlite parallel: matrix: - - IMAGE: [ 'python:3.12-slim-bookworm' ] - REQUIREMENTS: [ 'requirements.txt' ] - - IMAGE: [ 'debian:bookworm' ] # EOL: June 06, 2026 - REQUIREMENTS: [ '.DEBIAN/requirements-bookworm-12.txt' ] - - IMAGE: [ 'ubuntu:24.04' ] # EOL: April 2036 - REQUIREMENTS: [ '.DEBIAN/requirements-ubuntu-24.04.txt' ] - - IMAGE: [ 'ubuntu:24.10' ] - REQUIREMENTS: [ '.DEBIAN/requirements-ubuntu-24.10.txt' ] + - REQUIREMENTS: + - 'requirements.txt' +# - '.DEBIAN/requirements-bookworm-12.txt' +# - '.DEBIAN/requirements-ubuntu-24.04.txt' +# - '.DEBIAN/requirements-ubuntu-24.10.txt' before_script: - apt-get update && apt-get install -y python3-dev python3-pip python3-venv gcc - python3 -m venv venv @@ -207,13 +204,15 @@ test: - apt-get purge -qq -y fastapi-dls - apt-get autoremove -qq -y && apt-get clean -qq -test:apt:debian: +test:apt: extends: .test:apt - image: debian:bookworm-slim - -test:apt:ubuntu: - extends: .test:apt - image: ubuntu:24.04 + image: $IMAGE + parallel: + matrix: + - IMAGE: + - debian:bookworm-slim # EOL: June 06, 2026 + - ubuntu:24.04 # EOL: April 2036 + - ubuntu:24.10 test:pacman:archlinux: image: archlinux:base diff --git a/README.md b/README.md index ed8c646..bc0cdf1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Minimal Delegated License Service (DLS). -Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0, 3.3.1. For Driver compatibility +Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0, 3.3.1, 3.4.0. For Driver compatibility see [compatibility matrix](#vgpu-software-compatibility-matrix). This service can be used without internet connection. @@ -393,6 +393,10 @@ Now you have to edit `/etc/default/fastapi-dls` as needed. Continue [here](#unraid-guest) for docker guest setup. +## NixOS + +Tanks to [@mrzenc](https://github.com/mrzenc) for [fastapi-dls-nixos](https://github.com/mrzenc/fastapi-dls-nixos). + ## Let's Encrypt Certificate (optional) If you're using installation via docker, you can use `traefik`. Please refer to their documentation. @@ -426,7 +430,6 @@ After first success you have to replace `--issue` with `--renew`. | `ALLOTMENT_REF` | `20000000-0000-0000-0000-000000000001` | Allotment identification uuid | | `INSTANCE_KEY_RSA` | `/cert/instance.private.pem` | Site-wide private RSA key for singing JWTs \*3 | | `INSTANCE_KEY_PUB` | `/cert/instance.public.pem` | Site-wide public key \*3 | -| `SUPPORT_MALFORMED_JSON` | `false` | Support parsing for mal formatted "mac_address_list" ([Issue](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/issues/1)) | \*1 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license every 4.8 hours. If network connectivity is lost, the loss of connectivity is detected during license renewal and the @@ -767,5 +770,6 @@ Special thanks to: - @DualCoder who creates the `vgpu_unlock` functionality [vgpu_unlock](https://github.com/DualCoder/vgpu_unlock) - Krutav Shah who wrote the [vGPU_Unlock Wiki](https://docs.google.com/document/d/1pzrWJ9h-zANCtyqRgS7Vzla0Y8Ea2-5z2HEi4X75d2Q/) - Wim van 't Hoog for the [Proxmox All-In-One Installer Script](https://wvthoog.nl/proxmox-vgpu-v3/) +- @mrzenc who wrote [fastapi-dls-nixos](https://github.com/mrzenc/fastapi-dls-nixos) And thanks to all people who contributed to all these libraries! diff --git a/app/main.py b/app/main.py index 3ad688f..d0a79f6 100644 --- a/app/main.py +++ b/app/main.py @@ -2,7 +2,7 @@ import logging from base64 import b64encode as b64enc from calendar import timegm from contextlib import asynccontextmanager -from datetime import datetime, timedelta +from datetime import datetime, timedelta, UTC from hashlib import sha256 from json import loads as json_loads from os import getenv as env @@ -96,11 +96,6 @@ app.add_middleware( allow_methods=['*'], allow_headers=['*'], ) -if bool(env('SUPPORT_MALFORMED_JSON', False)): - from middleware import PatchMalformedJsonMiddleware - - logger.info(f'Enabled "PatchMalformedJsonMiddleware"!') - app.add_middleware(PatchMalformedJsonMiddleware, enabled=True) # Helper @@ -243,7 +238,7 @@ async def _lease_delete(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_core_service_instance/service_instance_token_manager.py @app.get('/-/client-token', summary='* Client-Token', description='creates a new messenger token for this service instance') async def _client_token(): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) exp_time = cur_time + CLIENT_TOKEN_EXPIRE_DELTA payload = { @@ -289,7 +284,7 @@ async def _client_token(): # venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py @app.post('/auth/v1/origin', description='find or create an origin') async def auth_v1_origin(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) origin_ref = j.get('candidate_origin_ref') logging.info(f'> [ origin ]: {origin_ref}: {j}') @@ -319,7 +314,7 @@ async def auth_v1_origin(request: Request): # venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py @app.post('/auth/v1/origin/update', description='update an origin evidence') async def auth_v1_origin_update(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) origin_ref = j.get('origin_ref') logging.info(f'> [ update ]: {origin_ref}: {j}') @@ -346,7 +341,7 @@ async def auth_v1_origin_update(request: Request): # venv/lib/python3.9/site-packages/nls_core_auth/auth.py - CodeResponse @app.post('/auth/v1/code', description='get an authorization code') async def auth_v1_code(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) origin_ref = j.get('origin_ref') logging.info(f'> [ code ]: {origin_ref}: {j}') @@ -378,10 +373,10 @@ async def auth_v1_code(request: Request): # venv/lib/python3.9/site-packages/nls_core_auth/auth.py - TokenResponse @app.post('/auth/v1/token', description='exchange auth code and verifier for token') async def auth_v1_token(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) try: - payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key) + payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key, algorithms=ALGORITHMS.RS256) except JWTError as e: return JSONr(status_code=400, content={'status': 400, 'title': 'invalid token', 'detail': str(e)}) @@ -420,7 +415,7 @@ async def auth_v1_token(request: Request): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.post('/leasing/v1/lessor', description='request multiple leases (borrow) for current origin') async def leasing_v1_lessor(request: Request): - j, token, cur_time = json_loads((await request.body()).decode('utf-8')), __get_token(request), datetime.utcnow() + j, token, cur_time = json_loads((await request.body()).decode('utf-8')), __get_token(request), datetime.now(UTC) try: token = __get_token(request) @@ -468,7 +463,7 @@ async def leasing_v1_lessor(request: Request): # venv/lib/python3.9/site-packages/nls_dal_service_instance_dls/schema/service_instance/V1_0_21__product_mapping.sql @app.get('/leasing/v1/lessor/leases', description='get active leases for current origin') async def leasing_v1_lessor_lease(request: Request): - token, cur_time = __get_token(request), datetime.utcnow() + token, cur_time = __get_token(request), datetime.now(UTC) origin_ref = token.get('origin_ref') @@ -488,7 +483,7 @@ async def leasing_v1_lessor_lease(request: Request): # venv/lib/python3.9/site-packages/nls_core_lease/lease_single.py @app.put('/leasing/v1/lease/{lease_ref}', description='renew a lease') async def leasing_v1_lease_renew(request: Request, lease_ref: str): - token, cur_time = __get_token(request), datetime.utcnow() + token, cur_time = __get_token(request), datetime.now(UTC) origin_ref = token.get('origin_ref') logging.info(f'> [ renew ]: {origin_ref}: renew {lease_ref}') @@ -515,7 +510,7 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_single_controller.py @app.delete('/leasing/v1/lease/{lease_ref}', description='release (return) a lease') async def leasing_v1_lease_delete(request: Request, lease_ref: str): - token, cur_time = __get_token(request), datetime.utcnow() + token, cur_time = __get_token(request), datetime.now(UTC) origin_ref = token.get('origin_ref') logging.info(f'> [ return ]: {origin_ref}: return {lease_ref}') @@ -541,7 +536,7 @@ async def leasing_v1_lease_delete(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.delete('/leasing/v1/lessor/leases', description='release all leases') async def leasing_v1_lessor_lease_remove(request: Request): - token, cur_time = __get_token(request), datetime.utcnow() + token, cur_time = __get_token(request), datetime.now(UTC) origin_ref = token.get('origin_ref') @@ -561,7 +556,7 @@ async def leasing_v1_lessor_lease_remove(request: Request): @app.post('/leasing/v1/lessor/shutdown', description='shutdown all leases') async def leasing_v1_lessor_shutdown(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) token = j.get('token') token = jwt.decode(token=token, key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) diff --git a/app/middleware.py b/app/middleware.py deleted file mode 100644 index db97bb0..0000000 --- a/app/middleware.py +++ /dev/null @@ -1,63 +0,0 @@ -import json -import logging -import re - -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request - -logger = logging.getLogger(__name__) - - -class PatchMalformedJsonMiddleware(BaseHTTPMiddleware): - # see oscar.krause/fastapi-dls#1 - - REGEX = '(\"mac_address_list\"\:\s?\[)([\w\d])' - - def __init__(self, app, enabled: bool): - super().__init__(app) - self.enabled = enabled - - async def dispatch(self, request: Request, call_next): - body = await request.body() - content_type = request.headers.get('Content-Type') - - if self.enabled and content_type == 'application/json': - logger.debug(f'Using Request-Patch because "PatchMalformedJsonMiddleware" is enabled!') - - # try to fix json - body = body.decode() - try: - j = json.loads(body) - self.fix_mac_address_list_length(j=j, size=1) - except json.decoder.JSONDecodeError: - logger.warning(f'Malformed json received! Try to fix it.') - s = PatchMalformedJsonMiddleware.fix_json(body) - logger.debug(f'Fixed JSON: "{s}"') - j = json.loads(s) # ensure json is now valid - j = self.fix_mac_address_list_length(j=j, size=1) - # set new body - request._body = json.dumps(j).encode('utf-8') - - response = await call_next(request) - return response - - def fix_mac_address_list_length(self, j: dict, size: int = 1) -> dict: - if not self.enabled: - return j - - # reduce "mac_address_list" to - environment = j.get('environment', {}) - fingerprint = environment.get('fingerprint', {}) - mac_address = fingerprint.get('mac_address_list', []) - - if len(mac_address) > 0: - logger.info(f'Transforming "mac_address_list" to length of {size}.') - j['environment']['fingerprint']['mac_address_list'] = mac_address[:size] - - return j - - @staticmethod - def fix_json(s: str) -> str: - s = s.replace('\t', '') - s = s.replace('\n', '') - return re.sub(PatchMalformedJsonMiddleware.REGEX, r'\1"\2', s) diff --git a/app/orm.py b/app/orm.py index bc902df..e2dd0bc 100644 --- a/app/orm.py +++ b/app/orm.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta, timezone, UTC from dateutil.relativedelta import relativedelta from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text @@ -178,7 +178,7 @@ class Lease(Base): @staticmethod def delete_expired(engine: Engine) -> int: session = sessionmaker(bind=engine)() - deletions = session.query(Lease).filter(Lease.lease_expires <= datetime.utcnow()).delete() + deletions = session.query(Lease).filter(Lease.lease_expires <= datetime.now(UTC)).delete() session.commit() session.close() return deletions diff --git a/test/main.py b/test/main.py index 8f2e2c7..a10f2c7 100644 --- a/test/main.py +++ b/test/main.py @@ -1,7 +1,7 @@ import sys from base64 import b64encode as b64enc from calendar import timegm -from datetime import datetime +from datetime import datetime, UTC from hashlib import sha256 from os.path import dirname, join from uuid import uuid4, UUID @@ -18,7 +18,6 @@ sys.path.append('../app') from app import main from app.util import load_key -# main.app.add_middleware(PatchMalformedJsonMiddleware, enabled=True) client = TestClient(main.app) ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-000000000001', 'HelloWorld' @@ -107,14 +106,6 @@ def test_auth_v1_origin(): assert response.json().get('origin_ref') == ORIGIN_REF -def test_auth_v1_origin_malformed_json(): # see oscar.krause/fastapi-dls#1 - from middleware import PatchMalformedJsonMiddleware - - # test regex (temporary, until this section is merged into main.py - s = '{"environment": {"fingerprint": {"mac_address_list": [ff:ff:ff:ff:ff:ff"]}}' - replaced = PatchMalformedJsonMiddleware.fix_json(s) - assert replaced == '{"environment": {"fingerprint": {"mac_address_list": ["ff:ff:ff:ff:ff:ff"]}}' - def auth_v1_origin_update(): payload = { @@ -151,7 +142,7 @@ def test_auth_v1_code(): def test_auth_v1_token(): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) access_expires_on = cur_time + relativedelta(hours=1) payload = { @@ -163,8 +154,7 @@ def test_auth_v1_token(): "kid": "00000000-0000-0000-0000-000000000000" } payload = { - "auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, - algorithm=ALGORITHMS.RS256), + "auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256), "code_verifier": SECRET, }