mirror of
https://git.collinwebdesigns.de/oscar.krause/fastapi-dls.git
synced 2024-12-25 21:39:33 +03:00
Merge branch 'dev' into 'main'
Dev See merge request oscar.krause/fastapi-dls!41
This commit is contained in:
Oscar Krause
commit
c57d76c74c
@ -2,7 +2,7 @@ Package: fastapi-dls
|
||||
Version: 0.0
|
||||
Architecture: all
|
||||
Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de
|
||||
Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl
|
||||
Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl
|
||||
Recommends: curl
|
||||
Installed-Size: 10240
|
||||
Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls
|
||||
|
||||
@ -48,7 +48,6 @@ package() {
|
||||
install -Dm755 "$srcdir/$pkgname/app/main.py" "$pkgdir/opt/$pkgname/main.py"
|
||||
install -Dm755 "$srcdir/$pkgname/app/orm.py" "$pkgdir/opt/$pkgname/orm.py"
|
||||
install -Dm755 "$srcdir/$pkgname/app/util.py" "$pkgdir/opt/$pkgname/util.py"
|
||||
install -Dm755 "$srcdir/$pkgname/app/middleware.py" "$pkgdir/opt/$pkgname/middleware.py"
|
||||
install -Dm644 "$srcdir/$pkgname.default" "$pkgdir/etc/default/$pkgname"
|
||||
install -Dm644 "$srcdir/$pkgname.service" "$pkgdir/usr/lib/systemd/system/$pkgname.service"
|
||||
install -Dm644 "$srcdir/$pkgname.tmpfiles" "$pkgdir/usr/lib/tmpfiles.d/$pkgname.conf"
|
||||
|
||||
@ -127,7 +127,7 @@ build:pacman:
|
||||
- "*.pkg.tar.zst"
|
||||
|
||||
test:
|
||||
image: $IMAGE
|
||||
image: python:3.12-slim-bookworm
|
||||
stage: test
|
||||
interruptible: true
|
||||
rules:
|
||||
@ -142,14 +142,11 @@ test:
|
||||
DATABASE: sqlite:///../app/db.sqlite
|
||||
parallel:
|
||||
matrix:
|
||||
- IMAGE: [ 'python:3.12-slim-bookworm' ]
|
||||
REQUIREMENTS: [ 'requirements.txt' ]
|
||||
- IMAGE: [ 'debian:bookworm' ] # EOL: June 06, 2026
|
||||
REQUIREMENTS: [ '.DEBIAN/requirements-bookworm-12.txt' ]
|
||||
- IMAGE: [ 'ubuntu:24.04' ] # EOL: April 2036
|
||||
REQUIREMENTS: [ '.DEBIAN/requirements-ubuntu-24.04.txt' ]
|
||||
- IMAGE: [ 'ubuntu:24.10' ]
|
||||
REQUIREMENTS: [ '.DEBIAN/requirements-ubuntu-24.10.txt' ]
|
||||
- REQUIREMENTS:
|
||||
- 'requirements.txt'
|
||||
# - '.DEBIAN/requirements-bookworm-12.txt'
|
||||
# - '.DEBIAN/requirements-ubuntu-24.04.txt'
|
||||
# - '.DEBIAN/requirements-ubuntu-24.10.txt'
|
||||
before_script:
|
||||
- apt-get update && apt-get install -y python3-dev python3-pip python3-venv gcc
|
||||
- python3 -m venv venv
|
||||
@ -207,13 +204,15 @@ test:
|
||||
- apt-get purge -qq -y fastapi-dls
|
||||
- apt-get autoremove -qq -y && apt-get clean -qq
|
||||
|
||||
test:apt:debian:
|
||||
test:apt:
|
||||
extends: .test:apt
|
||||
image: debian:bookworm-slim
|
||||
|
||||
test:apt:ubuntu:
|
||||
extends: .test:apt
|
||||
image: ubuntu:24.04
|
||||
image: $IMAGE
|
||||
parallel:
|
||||
matrix:
|
||||
- IMAGE:
|
||||
- debian:bookworm-slim # EOL: June 06, 2026
|
||||
- ubuntu:24.04 # EOL: April 2036
|
||||
- ubuntu:24.10
|
||||
|
||||
test:pacman:archlinux:
|
||||
image: archlinux:base
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
Minimal Delegated License Service (DLS).
|
||||
|
||||
Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0, 3.3.1. For Driver compatibility
|
||||
Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0, 3.3.1, 3.4.0. For Driver compatibility
|
||||
see [compatibility matrix](#vgpu-software-compatibility-matrix).
|
||||
|
||||
This service can be used without internet connection.
|
||||
@ -393,6 +393,10 @@ Now you have to edit `/etc/default/fastapi-dls` as needed.
|
||||
|
||||
Continue [here](#unraid-guest) for docker guest setup.
|
||||
|
||||
## NixOS
|
||||
|
||||
Tanks to [@mrzenc](https://github.com/mrzenc) for [fastapi-dls-nixos](https://github.com/mrzenc/fastapi-dls-nixos).
|
||||
|
||||
## Let's Encrypt Certificate (optional)
|
||||
|
||||
If you're using installation via docker, you can use `traefik`. Please refer to their documentation.
|
||||
@ -426,7 +430,6 @@ After first success you have to replace `--issue` with `--renew`.
|
||||
| `ALLOTMENT_REF` | `20000000-0000-0000-0000-000000000001` | Allotment identification uuid |
|
||||
| `INSTANCE_KEY_RSA` | `<app-dir>/cert/instance.private.pem` | Site-wide private RSA key for singing JWTs \*3 |
|
||||
| `INSTANCE_KEY_PUB` | `<app-dir>/cert/instance.public.pem` | Site-wide public key \*3 |
|
||||
| `SUPPORT_MALFORMED_JSON` | `false` | Support parsing for mal formatted "mac_address_list" ([Issue](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/issues/1)) |
|
||||
|
||||
\*1 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license
|
||||
every 4.8 hours. If network connectivity is lost, the loss of connectivity is detected during license renewal and the
|
||||
@ -767,5 +770,6 @@ Special thanks to:
|
||||
- @DualCoder who creates the `vgpu_unlock` functionality [vgpu_unlock](https://github.com/DualCoder/vgpu_unlock)
|
||||
- Krutav Shah who wrote the [vGPU_Unlock Wiki](https://docs.google.com/document/d/1pzrWJ9h-zANCtyqRgS7Vzla0Y8Ea2-5z2HEi4X75d2Q/)
|
||||
- Wim van 't Hoog for the [Proxmox All-In-One Installer Script](https://wvthoog.nl/proxmox-vgpu-v3/)
|
||||
- @mrzenc who wrote [fastapi-dls-nixos](https://github.com/mrzenc/fastapi-dls-nixos)
|
||||
|
||||
And thanks to all people who contributed to all these libraries!
|
||||
|
||||
31
app/main.py
31
app/main.py
@ -2,7 +2,7 @@ import logging
|
||||
from base64 import b64encode as b64enc
|
||||
from calendar import timegm
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, UTC
|
||||
from hashlib import sha256
|
||||
from json import loads as json_loads
|
||||
from os import getenv as env
|
||||
@ -96,11 +96,6 @@ app.add_middleware(
|
||||
allow_methods=['*'],
|
||||
allow_headers=['*'],
|
||||
)
|
||||
if bool(env('SUPPORT_MALFORMED_JSON', False)):
|
||||
from middleware import PatchMalformedJsonMiddleware
|
||||
|
||||
logger.info(f'Enabled "PatchMalformedJsonMiddleware"!')
|
||||
app.add_middleware(PatchMalformedJsonMiddleware, enabled=True)
|
||||
|
||||
|
||||
# Helper
|
||||
@ -243,7 +238,7 @@ async def _lease_delete(request: Request, lease_ref: str):
|
||||
# venv/lib/python3.9/site-packages/nls_core_service_instance/service_instance_token_manager.py
|
||||
@app.get('/-/client-token', summary='* Client-Token', description='creates a new messenger token for this service instance')
|
||||
async def _client_token():
|
||||
cur_time = datetime.utcnow()
|
||||
cur_time = datetime.now(UTC)
|
||||
exp_time = cur_time + CLIENT_TOKEN_EXPIRE_DELTA
|
||||
|
||||
payload = {
|
||||
@ -289,7 +284,7 @@ async def _client_token():
|
||||
# venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py
|
||||
@app.post('/auth/v1/origin', description='find or create an origin')
|
||||
async def auth_v1_origin(request: Request):
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC)
|
||||
|
||||
origin_ref = j.get('candidate_origin_ref')
|
||||
logging.info(f'> [ origin ]: {origin_ref}: {j}')
|
||||
@ -319,7 +314,7 @@ async def auth_v1_origin(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py
|
||||
@app.post('/auth/v1/origin/update', description='update an origin evidence')
|
||||
async def auth_v1_origin_update(request: Request):
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC)
|
||||
|
||||
origin_ref = j.get('origin_ref')
|
||||
logging.info(f'> [ update ]: {origin_ref}: {j}')
|
||||
@ -346,7 +341,7 @@ async def auth_v1_origin_update(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_core_auth/auth.py - CodeResponse
|
||||
@app.post('/auth/v1/code', description='get an authorization code')
|
||||
async def auth_v1_code(request: Request):
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC)
|
||||
|
||||
origin_ref = j.get('origin_ref')
|
||||
logging.info(f'> [ code ]: {origin_ref}: {j}')
|
||||
@ -378,10 +373,10 @@ async def auth_v1_code(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_core_auth/auth.py - TokenResponse
|
||||
@app.post('/auth/v1/token', description='exchange auth code and verifier for token')
|
||||
async def auth_v1_token(request: Request):
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC)
|
||||
|
||||
try:
|
||||
payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key)
|
||||
payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key, algorithms=ALGORITHMS.RS256)
|
||||
except JWTError as e:
|
||||
return JSONr(status_code=400, content={'status': 400, 'title': 'invalid token', 'detail': str(e)})
|
||||
|
||||
@ -420,7 +415,7 @@ async def auth_v1_token(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py
|
||||
@app.post('/leasing/v1/lessor', description='request multiple leases (borrow) for current origin')
|
||||
async def leasing_v1_lessor(request: Request):
|
||||
j, token, cur_time = json_loads((await request.body()).decode('utf-8')), __get_token(request), datetime.utcnow()
|
||||
j, token, cur_time = json_loads((await request.body()).decode('utf-8')), __get_token(request), datetime.now(UTC)
|
||||
|
||||
try:
|
||||
token = __get_token(request)
|
||||
@ -468,7 +463,7 @@ async def leasing_v1_lessor(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_dal_service_instance_dls/schema/service_instance/V1_0_21__product_mapping.sql
|
||||
@app.get('/leasing/v1/lessor/leases', description='get active leases for current origin')
|
||||
async def leasing_v1_lessor_lease(request: Request):
|
||||
token, cur_time = __get_token(request), datetime.utcnow()
|
||||
token, cur_time = __get_token(request), datetime.now(UTC)
|
||||
|
||||
origin_ref = token.get('origin_ref')
|
||||
|
||||
@ -488,7 +483,7 @@ async def leasing_v1_lessor_lease(request: Request):
|
||||
# venv/lib/python3.9/site-packages/nls_core_lease/lease_single.py
|
||||
@app.put('/leasing/v1/lease/{lease_ref}', description='renew a lease')
|
||||
async def leasing_v1_lease_renew(request: Request, lease_ref: str):
|
||||
token, cur_time = __get_token(request), datetime.utcnow()
|
||||
token, cur_time = __get_token(request), datetime.now(UTC)
|
||||
|
||||
origin_ref = token.get('origin_ref')
|
||||
logging.info(f'> [ renew ]: {origin_ref}: renew {lease_ref}')
|
||||
@ -515,7 +510,7 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str):
|
||||
# venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_single_controller.py
|
||||
@app.delete('/leasing/v1/lease/{lease_ref}', description='release (return) a lease')
|
||||
async def leasing_v1_lease_delete(request: Request, lease_ref: str):
|
||||
token, cur_time = __get_token(request), datetime.utcnow()
|
||||
token, cur_time = __get_token(request), datetime.now(UTC)
|
||||
|
||||
origin_ref = token.get('origin_ref')
|
||||
logging.info(f'> [ return ]: {origin_ref}: return {lease_ref}')
|
||||
@ -541,7 +536,7 @@ async def leasing_v1_lease_delete(request: Request, lease_ref: str):
|
||||
# venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py
|
||||
@app.delete('/leasing/v1/lessor/leases', description='release all leases')
|
||||
async def leasing_v1_lessor_lease_remove(request: Request):
|
||||
token, cur_time = __get_token(request), datetime.utcnow()
|
||||
token, cur_time = __get_token(request), datetime.now(UTC)
|
||||
|
||||
origin_ref = token.get('origin_ref')
|
||||
|
||||
@ -561,7 +556,7 @@ async def leasing_v1_lessor_lease_remove(request: Request):
|
||||
|
||||
@app.post('/leasing/v1/lessor/shutdown', description='shutdown all leases')
|
||||
async def leasing_v1_lessor_shutdown(request: Request):
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
|
||||
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC)
|
||||
|
||||
token = j.get('token')
|
||||
token = jwt.decode(token=token, key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False})
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PatchMalformedJsonMiddleware(BaseHTTPMiddleware):
|
||||
# see oscar.krause/fastapi-dls#1
|
||||
|
||||
REGEX = '(\"mac_address_list\"\:\s?\[)([\w\d])'
|
||||
|
||||
def __init__(self, app, enabled: bool):
|
||||
super().__init__(app)
|
||||
self.enabled = enabled
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
body = await request.body()
|
||||
content_type = request.headers.get('Content-Type')
|
||||
|
||||
if self.enabled and content_type == 'application/json':
|
||||
logger.debug(f'Using Request-Patch because "PatchMalformedJsonMiddleware" is enabled!')
|
||||
|
||||
# try to fix json
|
||||
body = body.decode()
|
||||
try:
|
||||
j = json.loads(body)
|
||||
self.fix_mac_address_list_length(j=j, size=1)
|
||||
except json.decoder.JSONDecodeError:
|
||||
logger.warning(f'Malformed json received! Try to fix it.')
|
||||
s = PatchMalformedJsonMiddleware.fix_json(body)
|
||||
logger.debug(f'Fixed JSON: "{s}"')
|
||||
j = json.loads(s) # ensure json is now valid
|
||||
j = self.fix_mac_address_list_length(j=j, size=1)
|
||||
# set new body
|
||||
request._body = json.dumps(j).encode('utf-8')
|
||||
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
def fix_mac_address_list_length(self, j: dict, size: int = 1) -> dict:
|
||||
if not self.enabled:
|
||||
return j
|
||||
|
||||
# reduce "mac_address_list" to
|
||||
environment = j.get('environment', {})
|
||||
fingerprint = environment.get('fingerprint', {})
|
||||
mac_address = fingerprint.get('mac_address_list', [])
|
||||
|
||||
if len(mac_address) > 0:
|
||||
logger.info(f'Transforming "mac_address_list" to length of {size}.')
|
||||
j['environment']['fingerprint']['mac_address_list'] = mac_address[:size]
|
||||
|
||||
return j
|
||||
|
||||
@staticmethod
|
||||
def fix_json(s: str) -> str:
|
||||
s = s.replace('\t', '')
|
||||
s = s.replace('\n', '')
|
||||
return re.sub(PatchMalformedJsonMiddleware.REGEX, r'\1"\2', s)
|
||||
@ -1,4 +1,4 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timedelta, timezone, UTC
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text
|
||||
@ -178,7 +178,7 @@ class Lease(Base):
|
||||
@staticmethod
|
||||
def delete_expired(engine: Engine) -> int:
|
||||
session = sessionmaker(bind=engine)()
|
||||
deletions = session.query(Lease).filter(Lease.lease_expires <= datetime.utcnow()).delete()
|
||||
deletions = session.query(Lease).filter(Lease.lease_expires <= datetime.now(UTC)).delete()
|
||||
session.commit()
|
||||
session.close()
|
||||
return deletions
|
||||
|
||||
16
test/main.py
16
test/main.py
@ -1,7 +1,7 @@
|
||||
import sys
|
||||
from base64 import b64encode as b64enc
|
||||
from calendar import timegm
|
||||
from datetime import datetime
|
||||
from datetime import datetime, UTC
|
||||
from hashlib import sha256
|
||||
from os.path import dirname, join
|
||||
from uuid import uuid4, UUID
|
||||
@ -18,7 +18,6 @@ sys.path.append('../app')
|
||||
from app import main
|
||||
from app.util import load_key
|
||||
|
||||
# main.app.add_middleware(PatchMalformedJsonMiddleware, enabled=True)
|
||||
client = TestClient(main.app)
|
||||
|
||||
ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-000000000001', 'HelloWorld'
|
||||
@ -107,14 +106,6 @@ def test_auth_v1_origin():
|
||||
assert response.json().get('origin_ref') == ORIGIN_REF
|
||||
|
||||
|
||||
def test_auth_v1_origin_malformed_json(): # see oscar.krause/fastapi-dls#1
|
||||
from middleware import PatchMalformedJsonMiddleware
|
||||
|
||||
# test regex (temporary, until this section is merged into main.py
|
||||
s = '{"environment": {"fingerprint": {"mac_address_list": [ff:ff:ff:ff:ff:ff"]}}'
|
||||
replaced = PatchMalformedJsonMiddleware.fix_json(s)
|
||||
assert replaced == '{"environment": {"fingerprint": {"mac_address_list": ["ff:ff:ff:ff:ff:ff"]}}'
|
||||
|
||||
|
||||
def auth_v1_origin_update():
|
||||
payload = {
|
||||
@ -151,7 +142,7 @@ def test_auth_v1_code():
|
||||
|
||||
|
||||
def test_auth_v1_token():
|
||||
cur_time = datetime.utcnow()
|
||||
cur_time = datetime.now(UTC)
|
||||
access_expires_on = cur_time + relativedelta(hours=1)
|
||||
|
||||
payload = {
|
||||
@ -163,8 +154,7 @@ def test_auth_v1_token():
|
||||
"kid": "00000000-0000-0000-0000-000000000000"
|
||||
}
|
||||
payload = {
|
||||
"auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')},
|
||||
algorithm=ALGORITHMS.RS256),
|
||||
"auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256),
|
||||
"code_verifier": SECRET,
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user