From 38e1a1725c5cbaad3cfe74f966c854c5da55034a Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 12:40:10 +0200 Subject: [PATCH 01/69] code styling --- app/main.py | 10 +++++++++- app/orm.py | 2 -- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index f5388c8..f4ab600 100644 --- a/app/main.py +++ b/app/main.py @@ -24,16 +24,20 @@ from orm import Origin, Lease, init as db_init, migrate load_dotenv('../version.env') +# get local timezone TZ = datetime.now().astimezone().tzinfo +# fetch version info VERSION, COMMIT, DEBUG = env('VERSION', 'unknown'), env('COMMIT', 'unknown'), bool(env('DEBUG', False)) config = dict(openapi_url=None, docs_url=None, redoc_url=None) # dict(openapi_url='/-/openapi.json', docs_url='/-/docs', redoc_url='/-/redoc') app = FastAPI(title='FastAPI-DLS', description='Minimal Delegated License Service (DLS).', version=VERSION, **config) + +# database setup db = create_engine(str(env('DATABASE', 'sqlite:///db.sqlite'))) db_init(db), migrate(db) -# everything prefixed with "INSTANCE_*" is used as "SERVICE_INSTANCE_*" or "SI_*" in official dls service +# DLS setup (static) DLS_URL = str(env('DLS_URL', 'localhost')) DLS_PORT = int(env('DLS_PORT', '443')) SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) @@ -51,6 +55,7 @@ CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) +# fastapi middleware app.debug = DEBUG app.add_middleware( CORSMiddleware, @@ -60,6 +65,7 @@ app.add_middleware( allow_headers=['*'], ) +# logging logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) @@ -108,6 +114,8 @@ async def _config(): @app.get('/-/readme', summary='* Readme') async def _readme(): from markdown import markdown + from util import load_file + content = load_file('../README.md').decode('utf-8') return HTMLr(markdown(text=content, extensions=['tables', 'fenced_code', 'md_in_html', 'nl2br', 'toc'])) diff --git a/app/orm.py b/app/orm.py index c78ca95..d803672 100644 --- a/app/orm.py +++ b/app/orm.py @@ -12,7 +12,6 @@ class Origin(Base): __tablename__ = "origin" origin_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4 - # service_instance_xid = Column(CHAR(length=36), nullable=False, index=True) # uuid4 # not necessary, we only support one service_instance_xid ('INSTANCE_REF') hostname = Column(VARCHAR(length=256), nullable=True) guest_driver_version = Column(VARCHAR(length=10), nullable=True) @@ -71,7 +70,6 @@ class Lease(Base): __tablename__ = "lease" lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) # uuid4 - origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref, ondelete='CASCADE'), nullable=False, index=True) # uuid4 # scope_ref = Column(CHAR(length=36), nullable=False, index=True) # uuid4 # not necessary, we only support one scope_ref ('ALLOTMENT_REF') lease_created = Column(DATETIME(), nullable=False) From b4150fa527b6a098e2382bc29087167477fda275 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 12:42:13 +0200 Subject: [PATCH 02/69] implemented Site and Instance orm models including initialization --- app/orm.py | 235 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 211 insertions(+), 24 deletions(-) diff --git a/app/orm.py b/app/orm.py index d803672..587b8d6 100644 --- a/app/orm.py +++ b/app/orm.py @@ -1,13 +1,134 @@ +import logging from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta - -from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text, BLOB, INT, FLOAT from sqlalchemy.engine import Engine -from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy.orm import sessionmaker, declarative_base, Session + +from app.util import parse_key + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) Base = declarative_base() +class Site(Base): + __tablename__ = "site" + + INITIAL_SITE_KEY_XID = '00000000-0000-0000-0000-000000000000' + INITIAL_SITE_NAME = 'default' + + site_key = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4, SITE_KEY_XID + name = Column(VARCHAR(length=256), nullable=False) + + def __str__(self): + return f'SITE_KEY_XID: {self.site_key}' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Site.__table__).compile(engine) + + @staticmethod + def get_default_site(engine: Engine) -> "Site": + session = sessionmaker(bind=engine)() + entity = session.query(Site).filter(Site.site_key == Site.INITIAL_SITE_KEY_XID).first() + session.close() + return entity + + +class Instance(Base): + __tablename__ = "instance" + + DEFAULT_INSTANCE_REF = '10000000-0000-0000-0000-000000000001' + DEFAULT_TOKEN_EXPIRE_DELTA = 86_400 # 1 day + DEFAULT_LEASE_EXPIRE_DELTA = 7_776_000 # 90 days + DEFAULT_LEASE_RENEWAL_PERIOD = 0.15 + DEFAULT_CLIENT_TOKEN_EXPIRE_DELTA = 378_432_000 # 12 years + # 1 day = 86400 (min. in production setup, max 90 days), 1 hour = 3600 + + instance_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4, INSTANCE_REF + site_key = Column(CHAR(length=36), ForeignKey(Site.site_key, ondelete='CASCADE'), nullable=False, index=True) # uuid4 + private_key = Column(BLOB(length=2048), nullable=False) + public_key = Column(BLOB(length=512), nullable=False) + token_expire_delta = Column(INT(), nullable=False, default=DEFAULT_TOKEN_EXPIRE_DELTA, comment='in seconds') + lease_expire_delta = Column(INT(), nullable=False, default=DEFAULT_LEASE_EXPIRE_DELTA, comment='in seconds') + lease_renewal_period = Column(FLOAT(precision=2), nullable=False, default=DEFAULT_LEASE_RENEWAL_PERIOD) + client_token_expire_delta = Column(INT(), nullable=False, default=DEFAULT_CLIENT_TOKEN_EXPIRE_DELTA, comment='in seconds') + + def __str__(self): + return f'INSTANCE_REF: {self.instance_ref} (SITE_KEY_XID: {self.site_key})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Instance.__table__).compile(engine) + + @staticmethod + def create_or_update(engine: Engine, instance: "Instance"): + session = sessionmaker(bind=engine)() + entity = session.query(Instance).filter(Instance.instance_ref == instance.instance_ref).first() + if entity is None: + session.add(instance) + else: + x = dict( + site_key=instance.site_key, + private_key=instance.private_key, + public_key=instance.public_key, + token_expire_delta=instance.token_expire_delta, + lease_expire_delta=instance.lease_expire_delta, + lease_renewal_period=instance.lease_renewal_period, + client_token_expire_delta=instance.client_token_expire_delta, + ) + session.execute(update(Instance).where(Instance.instance_ref == instance.instance_ref).values(**x)) + session.commit() + session.flush() + session.close() + + # todo: validate on startup that "lease_expire_delta" is between 1 day and 90 days + + @staticmethod + def get_default_instance(engine: Engine) -> "Instance": + session = sessionmaker(bind=engine)() + site = Site.get_default_site(engine) + entity = session.query(Instance).filter(Instance.site_key == site.site_key).first() + session.close() + return entity + + def get_token_expire_delta(self) -> "dateutil.relativedelta.relativedelta": + return relativedelta(seconds=self.token_expire_delta) + + def get_lease_expire_delta(self) -> "dateutil.relativedelta.relativedelta": + return relativedelta(seconds=self.lease_expire_delta) + + def get_lease_renewal_delta(self) -> "datetime.timedelta": + return timedelta(seconds=self.lease_expire_delta) + + def get_client_token_expire_delta(self) -> "dateutil.relativedelta.relativedelta": + return relativedelta(seconds=self.client_token_expire_delta) + + def get_public_key(self) -> "RsaKey": + return parse_key(self.public_key) + + def get_jwt_encode_key(self) -> "jose.jkw": + from jose import jwk + from jose.constants import ALGORITHMS + return jwk.construct(self.private_key, algorithm=ALGORITHMS.RS256) + + def get_jwt_decode_key(self) -> "jose.jwt": + from jose import jwk + from jose.constants import ALGORITHMS + return jwk.construct(self.public_key, algorithm=ALGORITHMS.RS256) + + def get_private_key_str(self, encoding: str = 'utf-8'): + return self.private_key.decode(encoding) + + def get_public_key_str(self, encoding: str = 'utf-8'): + return self.private_key.decode(encoding) + + class Origin(Base): __tablename__ = "origin" @@ -181,38 +302,104 @@ class Lease(Base): return renew +def init_default_site(session: Session): + from uuid import uuid4 + from app.util import generate_key + + private_key = generate_key() + public_key = private_key.public_key() + + site = Site( + site_key=Site.INITIAL_SITE_KEY_XID, + name=Site.INITIAL_SITE_NAME + ) + session.add(site) + session.commit() + + instance = Instance( + instance_ref=str(uuid4()), + site_key=site.site_key, + private_key=private_key.export_key(), + public_key=public_key.export_key(), + ) + session.add(instance) + session.commit() + + def init(engine: Engine): - tables = [Origin, Lease] + tables = [Site, Instance, Origin, Lease] db = inspect(engine) session = sessionmaker(bind=engine)() for table in tables: - if not db.dialect.has_table(engine.connect(), table.__tablename__): + exists = db.dialect.has_table(engine.connect(), table.__tablename__) + logger.info(f'> Table "{table.__tablename__:<16}" exists: {exists}') + if not exists: session.execute(text(str(table.create_statement(engine)))) session.commit() + + # create default site + cnt = session.query(Site).count() + if cnt == 0: + init_default_site(session) + + session.flush() session.close() def migrate(engine: Engine): + from os import getenv as env + from os.path import join, dirname, isfile + from util import load_key + db = inspect(engine) - def upgrade_1_0_to_1_1(): - x = db.dialect.get_columns(engine.connect(), Lease.__tablename__) - x = next(_ for _ in x if _['name'] == 'origin_ref') - if x['primary_key'] > 0: - print('Found old database schema with "origin_ref" as primary-key in "lease" table. Dropping table!') - print(' Your leases are recreated on next renewal!') - print(' If an error message appears on the client, you can ignore it.') - Lease.__table__.drop(bind=engine) - init(engine) + # todo: add update guide to use 1.LATEST to 2.0 + def upgrade_1_x_to_2_0(): + site = Site.get_default_site(engine) + logger.info(site) + instance = Instance.get_default_instance(engine) + logger.info(instance) - # def upgrade_1_2_to_1_3(): - # x = db.dialect.get_columns(engine.connect(), Lease.__tablename__) - # x = next((_ for _ in x if _['name'] == 'scope_ref'), None) - # if x is None: - # Lease.scope_ref.compile() - # column_name = Lease.scope_ref.name - # column_type = Lease.scope_ref.type.compile(engine.dialect) - # engine.execute(f'ALTER TABLE "{Lease.__tablename__}" ADD COLUMN "{column_name}" {column_type}') + # SITE_KEY_XID + if site_key := env('SITE_KEY_XID', None) is not None: + site.site_key = str(site_key) - upgrade_1_0_to_1_1() - # upgrade_1_2_to_1_3() + # INSTANCE_REF + if instance_ref := env('INSTANCE_REF', None) is not None: + instance.instance_ref = str(instance_ref) + + # ALLOTMENT_REF + if allotment_ref := env('ALLOTMENT_REF', None) is not None: + pass # todo + + # INSTANCE_KEY_RSA, INSTANCE_KEY_PUB + default_instance_private_key_path = str(join(dirname(__file__), 'cert/instance.private.pem')) + if instance_private_key := env('INSTANCE_KEY_RSA', None) is not None: + instance.private_key = load_key(str(instance_private_key)) + elif isfile(default_instance_private_key_path): + instance.private_key = load_key(default_instance_private_key_path) + default_instance_public_key_path = str(join(dirname(__file__), 'cert/instance.public.pem')) + if instance_public_key := env('INSTANCE_KEY_PUB', None) is not None: + instance.public_key = load_key(str(instance_public_key)) + elif isfile(default_instance_public_key_path): + instance.public_key = load_key(default_instance_public_key_path) + + # TOKEN_EXPIRE_DELTA + if token_expire_delta := env('TOKEN_EXPIRE_DAYS', None) not in (None, 0): + instance.token_expire_delta = token_expire_delta * 86_400 + if token_expire_delta := env('TOKEN_EXPIRE_HOURS', None) not in (None, 0): + instance.token_expire_delta = token_expire_delta * 3_600 + + # LEASE_EXPIRE_DELTA, LEASE_RENEWAL_DELTA + if lease_expire_delta := env('LEASE_EXPIRE_DAYS', None) not in (None, 0): + instance.lease_expire_delta = lease_expire_delta * 86_400 + if lease_expire_delta := env('LEASE_EXPIRE_HOURS', None) not in (None, 0): + instance.lease_expire_delta = lease_expire_delta * 3_600 + + # LEASE_RENEWAL_PERIOD + if lease_renewal_period := env('LEASE_RENEWAL_PERIOD', None) is not None: + instance.lease_renewal_period = lease_renewal_period + + # todo: update site, instance + + upgrade_1_x_to_2_0() From 5e945bc43a7d46c57217f4649e22ea717786f558 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 14:47:32 +0200 Subject: [PATCH 03/69] code styling --- test/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/main.py b/test/main.py index 67856c9..f0f11d8 100644 --- a/test/main.py +++ b/test/main.py @@ -153,8 +153,7 @@ def test_auth_v1_token(): "kid": "00000000-0000-0000-0000-000000000000" } payload = { - "auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, - algorithm=ALGORITHMS.RS256), + "auth_code": jwt.encode(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256), "code_verifier": SECRET, } From 20448bc587fbd4fe2190dd453ded829111da14db Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 15:13:29 +0200 Subject: [PATCH 04/69] improved relationships --- app/orm.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/orm.py b/app/orm.py index 587b8d6..f395498 100644 --- a/app/orm.py +++ b/app/orm.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text, BLOB, INT, FLOAT from sqlalchemy.engine import Engine -from sqlalchemy.orm import sessionmaker, declarative_base, Session +from sqlalchemy.orm import sessionmaker, declarative_base, Session, relationship from app.util import parse_key @@ -58,6 +58,8 @@ class Instance(Base): lease_renewal_period = Column(FLOAT(precision=2), nullable=False, default=DEFAULT_LEASE_RENEWAL_PERIOD) client_token_expire_delta = Column(INT(), nullable=False, default=DEFAULT_CLIENT_TOKEN_EXPIRE_DELTA, comment='in seconds') + __origin = relationship(Site, foreign_keys=[site_key]) + def __str__(self): return f'INSTANCE_REF: {self.instance_ref} (SITE_KEY_XID: {self.site_key})' @@ -197,6 +199,9 @@ class Lease(Base): lease_expires = Column(DATETIME(), nullable=False) lease_updated = Column(DATETIME(), nullable=False) + __instance = relationship(Instance, foreign_keys=[instance_ref]) + __origin = relationship(Origin, foreign_keys=[origin_ref]) + def __repr__(self): return f'Lease(origin_ref={self.origin_ref}, lease_ref={self.lease_ref}, expires={self.lease_expires})' From 5e47ad772972d1c05f6d9c83c1af397ad616f92a Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 15:13:53 +0200 Subject: [PATCH 05/69] fastapi openapi url --- app/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index f4ab600..5cc14e4 100644 --- a/app/main.py +++ b/app/main.py @@ -30,7 +30,8 @@ TZ = datetime.now().astimezone().tzinfo # fetch version info VERSION, COMMIT, DEBUG = env('VERSION', 'unknown'), env('COMMIT', 'unknown'), bool(env('DEBUG', False)) -config = dict(openapi_url=None, docs_url=None, redoc_url=None) # dict(openapi_url='/-/openapi.json', docs_url='/-/docs', redoc_url='/-/redoc') +# fastapi setup +config = dict(openapi_url='/-/openapi.json', docs_url=None, redoc_url=None) app = FastAPI(title='FastAPI-DLS', description='Minimal Delegated License Service (DLS).', version=VERSION, **config) # database setup From 18807401e496ff7df41d8b39084d0e634f531ee4 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 15:14:12 +0200 Subject: [PATCH 06/69] orm improvements & fixes --- app/orm.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/app/orm.py b/app/orm.py index f395498..0eaf511 100644 --- a/app/orm.py +++ b/app/orm.py @@ -111,23 +111,26 @@ class Instance(Base): def get_client_token_expire_delta(self) -> "dateutil.relativedelta.relativedelta": return relativedelta(seconds=self.client_token_expire_delta) + def __get_private_key(self) -> "RsaKey": + return parse_key(self.private_key) + def get_public_key(self) -> "RsaKey": return parse_key(self.public_key) def get_jwt_encode_key(self) -> "jose.jkw": from jose import jwk from jose.constants import ALGORITHMS - return jwk.construct(self.private_key, algorithm=ALGORITHMS.RS256) + return jwk.construct(self.__get_private_key().export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) def get_jwt_decode_key(self) -> "jose.jwt": from jose import jwk from jose.constants import ALGORITHMS - return jwk.construct(self.public_key, algorithm=ALGORITHMS.RS256) + return jwk.construct(self.get_public_key().export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) - def get_private_key_str(self, encoding: str = 'utf-8'): + def get_private_key_str(self, encoding: str = 'utf-8') -> str: return self.private_key.decode(encoding) - def get_public_key_str(self, encoding: str = 'utf-8'): + def get_public_key_str(self, encoding: str = 'utf-8') -> str: return self.private_key.decode(encoding) @@ -192,6 +195,7 @@ class Origin(Base): class Lease(Base): __tablename__ = "lease" + instance_ref = Column(CHAR(length=36), ForeignKey(Instance.instance_ref, ondelete='CASCADE'), nullable=False, index=True) # uuid4 lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) # uuid4 origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref, ondelete='CASCADE'), nullable=False, index=True) # uuid4 # scope_ref = Column(CHAR(length=36), nullable=False, index=True) # uuid4 # not necessary, we only support one scope_ref ('ALLOTMENT_REF') @@ -205,7 +209,10 @@ class Lease(Base): def __repr__(self): return f'Lease(origin_ref={self.origin_ref}, lease_ref={self.lease_ref}, expires={self.lease_expires})' - def serialize(self, renewal_period: float, renewal_delta: timedelta) -> dict: + def serialize(self) -> dict: + renewal_period = self.__instance.lease_renewal_period + renewal_delta = self.__instance.get_lease_renewal_delta + lease_renewal = int(Lease.calculate_renewal(renewal_period, renewal_delta).total_seconds()) lease_renewal = self.lease_updated + relativedelta(seconds=lease_renewal) @@ -322,7 +329,7 @@ def init_default_site(session: Session): session.commit() instance = Instance( - instance_ref=str(uuid4()), + instance_ref=Instance.DEFAULT_INSTANCE_REF, site_key=site.site_key, private_key=private_key.export_key(), public_key=public_key.export_key(), From 39a2408d8df54895c1cf2b5e3f0a89b354b48a9c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 12 Jun 2023 15:19:06 +0200 Subject: [PATCH 07/69] migrated api to database-config (Site, Instance) --- app/main.py | 156 ++++++++++++++++++++++++++++++++++----------------- app/util.py | 12 ++++ test/main.py | 32 +++++++---- 3 files changed, 138 insertions(+), 62 deletions(-) diff --git a/app/main.py b/app/main.py index 5cc14e4..9713f20 100644 --- a/app/main.py +++ b/app/main.py @@ -9,18 +9,17 @@ from dotenv import load_dotenv from fastapi import FastAPI from fastapi.requests import Request from json import loads as json_loads -from datetime import datetime, timedelta +from datetime import datetime from dateutil.relativedelta import relativedelta from calendar import timegm -from jose import jws, jwk, jwt, JWTError +from jose import jws, jwt, JWTError from jose.constants import ALGORITHMS from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from util import load_key, load_file -from orm import Origin, Lease, init as db_init, migrate +from orm import init as db_init, migrate, Site, Instance, Origin, Lease load_dotenv('../version.env') @@ -41,20 +40,9 @@ db_init(db), migrate(db) # DLS setup (static) DLS_URL = str(env('DLS_URL', 'localhost')) DLS_PORT = int(env('DLS_PORT', '443')) -SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) -INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001')) -ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001')) -INSTANCE_KEY_RSA = load_key(str(env('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) -INSTANCE_KEY_PUB = load_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) -TOKEN_EXPIRE_DELTA = relativedelta(days=int(env('TOKEN_EXPIRE_DAYS', 1)), hours=int(env('TOKEN_EXPIRE_HOURS', 0))) -LEASE_EXPIRE_DELTA = relativedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0))) -LEASE_RENEWAL_PERIOD = float(env('LEASE_RENEWAL_PERIOD', 0.15)) -LEASE_RENEWAL_DELTA = timedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0))) -CLIENT_TOKEN_EXPIRE_DELTA = relativedelta(years=12) CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) else [f'https://{DLS_URL}'] -jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) -jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) +ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001')) # todo # fastapi middleware app.debug = DEBUG @@ -72,7 +60,19 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG if DEBUG else logging.INFO) -def __get_token(request: Request) -> dict: +def validate_settings(): + session = sessionmaker(bind=db)() + + lease_expire_delta_min, lease_expire_delta_max = 86_400, 7_776_000 + for instance in session.query(Instance).all(): + lease_expire_delta = instance.lease_expire_delta + if lease_expire_delta < 86_400 or lease_expire_delta > 7_776_000: + logging.warning(f'> [ instance ]: {instance.instance_ref}: "lease_expire_delta" should be between {lease_expire_delta_min} and {lease_expire_delta_max}') + + session.close() + + +def __get_token(request: Request, jwt_decode_key: "jose.jwt") -> dict: authorization_header = request.headers.get('authorization') token = authorization_header.split(' ')[1] return jwt.decode(token=token, key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) @@ -95,18 +95,20 @@ async def _health(): @app.get('/-/config', summary='* Config', description='returns environment variables.') async def _config(): + default_site, default_instance = Site.get_default_site(db), Instance.get_default_instance(db) + return JSONr({ 'VERSION': str(VERSION), 'COMMIT': str(COMMIT), 'DEBUG': str(DEBUG), 'DLS_URL': str(DLS_URL), 'DLS_PORT': str(DLS_PORT), - 'SITE_KEY_XID': str(SITE_KEY_XID), - 'INSTANCE_REF': str(INSTANCE_REF), + 'SITE_KEY_XID': str(default_site.site_key), + 'INSTANCE_REF': str(default_instance.instance_ref), 'ALLOTMENT_REF': [str(ALLOTMENT_REF)], - 'TOKEN_EXPIRE_DELTA': str(TOKEN_EXPIRE_DELTA), - 'LEASE_EXPIRE_DELTA': str(LEASE_EXPIRE_DELTA), - 'LEASE_RENEWAL_PERIOD': str(LEASE_RENEWAL_PERIOD), + 'TOKEN_EXPIRE_DELTA': str(default_instance.get_token_expire_delta()), + 'LEASE_EXPIRE_DELTA': str(default_instance.get_lease_expire_delta()), + 'LEASE_RENEWAL_PERIOD': str(default_instance.lease_renewal_period), 'CORS_ORIGINS': str(CORS_ORIGINS), 'TZ': str(TZ), }) @@ -166,8 +168,7 @@ async def _origins(request: Request, leases: bool = False): for origin in session.query(Origin).all(): x = origin.serialize() if leases: - serialize = dict(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA) - x['leases'] = list(map(lambda _: _.serialize(**serialize), Lease.find_by_origin_ref(db, origin.origin_ref))) + x['leases'] = list(map(lambda _: _.serialize(), Lease.find_by_origin_ref(db, origin.origin_ref))) response.append(x) session.close() return JSONr(response) @@ -184,8 +185,7 @@ async def _leases(request: Request, origin: bool = False): session = sessionmaker(bind=db)() response = [] for lease in session.query(Lease).all(): - serialize = dict(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA) - x = lease.serialize(**serialize) + x = lease.serialize() if origin: lease_origin = session.query(Origin).filter(Origin.origin_ref == lease.origin_ref).first() if lease_origin is not None: @@ -206,7 +206,13 @@ async def _lease_delete(request: Request, lease_ref: str): @app.get('/-/client-token', summary='* Client-Token', description='creates a new messenger token for this service instance') async def _client_token(): cur_time = datetime.utcnow() - exp_time = cur_time + CLIENT_TOKEN_EXPIRE_DELTA + + default_instance = Instance.get_default_instance(db) + public_key = default_instance.get_public_key() + # todo: implemented request parameter to support different instances + jwt_encode_key = default_instance.get_jwt_encode_key() + + exp_time = cur_time + default_instance.get_client_token_expire_delta() payload = { "jti": str(uuid4()), @@ -219,7 +225,7 @@ async def _client_token(): "scope_ref_list": [ALLOTMENT_REF], "fulfillment_class_ref_list": [], "service_instance_configuration": { - "nls_service_instance_ref": INSTANCE_REF, + "nls_service_instance_ref": default_instance.instance_ref, "svc_port_set_list": [ { "idx": 0, @@ -231,10 +237,10 @@ async def _client_token(): }, "service_instance_public_key_configuration": { "service_instance_public_key_me": { - "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], - "exp": int(INSTANCE_KEY_PUB.public_key().e), + "mod": hex(public_key.public_key().n)[2:], + "exp": int(public_key.public_key().e), }, - "service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), + "service_instance_public_key_pem": public_key.export_key().decode('utf-8'), "key_retention_mode": "LATEST_ONLY" }, } @@ -316,13 +322,16 @@ async def auth_v1_code(request: Request): delta = relativedelta(minutes=15) expires = cur_time + delta + default_site = Site.get_default_site(db) + jwt_encode_key = Instance.get_default_instance(db).get_jwt_encode_key() + payload = { 'iat': timegm(cur_time.timetuple()), 'exp': timegm(expires.timetuple()), 'challenge': j.get('code_challenge'), 'origin_ref': j.get('origin_ref'), - 'key_ref': SITE_KEY_XID, - 'kid': SITE_KEY_XID + 'key_ref': default_site.site_key, + 'kid': default_site.site_key, } auth_code = jws.sign(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256) @@ -342,8 +351,11 @@ async def auth_v1_code(request: Request): async def auth_v1_token(request: Request): j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + default_site, default_instance = Site.get_default_site(db), Instance.get_default_instance(db) + jwt_encode_key, jwt_decode_key = default_instance.get_jwt_encode_key(), default_instance.get_jwt_decode_key() + try: - payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key) + payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key, algorithms=[ALGORITHMS.RS256]) except JWTError as e: return JSONr(status_code=400, content={'status': 400, 'title': 'invalid token', 'detail': str(e)}) @@ -355,7 +367,7 @@ async def auth_v1_token(request: Request): if payload.get('challenge') != challenge: return JSONr(status_code=401, content={'status': 401, 'detail': 'expected challenge did not match verifier'}) - access_expires_on = cur_time + TOKEN_EXPIRE_DELTA + access_expires_on = cur_time + default_instance.get_token_expire_delta() new_payload = { 'iat': timegm(cur_time.timetuple()), @@ -364,8 +376,8 @@ async def auth_v1_token(request: Request): 'aud': 'https://cls.nvidia.org', 'exp': timegm(access_expires_on.timetuple()), 'origin_ref': origin_ref, - 'key_ref': SITE_KEY_XID, - 'kid': SITE_KEY_XID, + 'key_ref': default_site.site_key, + 'kid': default_site.site_key, } auth_token = jwt.encode(new_payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256) @@ -382,10 +394,13 @@ async def auth_v1_token(request: Request): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.post('/leasing/v1/lessor', description='request multiple leases (borrow) for current origin') async def leasing_v1_lessor(request: Request): - j, token, cur_time = json_loads((await request.body()).decode('utf-8')), __get_token(request), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + + default_instance = Instance.get_default_instance(db) + jwt_decode_key = default_instance.get_jwt_decode_key() try: - token = __get_token(request) + token = __get_token(request, jwt_decode_key) except JWTError: return JSONr(status_code=401, content={'status': 401, 'detail': 'token is not valid'}) @@ -399,7 +414,7 @@ async def leasing_v1_lessor(request: Request): # return JSONr(status_code=500, detail=f'no service instances found for scopes: ["{scope_ref}"]') lease_ref = str(uuid4()) - expires = cur_time + LEASE_EXPIRE_DELTA + expires = cur_time + default_instance.get_lease_expire_delta() lease_result_list.append({ "ordinal": 0, # https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html @@ -407,13 +422,13 @@ async def leasing_v1_lessor(request: Request): "ref": lease_ref, "created": cur_time.isoformat(), "expires": expires.isoformat(), - "recommended_lease_renewal": LEASE_RENEWAL_PERIOD, + "recommended_lease_renewal": default_instance.lease_renewal_period, "offline_lease": "true", "license_type": "CONCURRENT_COUNTED_SINGLE" } }) - data = Lease(origin_ref=origin_ref, lease_ref=lease_ref, lease_created=cur_time, lease_expires=expires) + data = Lease(instance_ref=default_instance.instance_ref, origin_ref=origin_ref, lease_ref=lease_ref, lease_created=cur_time, lease_expires=expires) Lease.create_or_update(db, data) response = { @@ -430,7 +445,14 @@ async def leasing_v1_lessor(request: Request): # venv/lib/python3.9/site-packages/nls_dal_service_instance_dls/schema/service_instance/V1_0_21__product_mapping.sql @app.get('/leasing/v1/lessor/leases', description='get active leases for current origin') async def leasing_v1_lessor_lease(request: Request): - token, cur_time = __get_token(request), datetime.utcnow() + cur_time = datetime.utcnow() + + jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() + + try: + token = __get_token(request, jwt_decode_key) + except JWTError: + return JSONr(status_code=401, content={'status': 401, 'detail': 'token is not valid'}) origin_ref = token.get('origin_ref') @@ -450,7 +472,15 @@ async def leasing_v1_lessor_lease(request: Request): # venv/lib/python3.9/site-packages/nls_core_lease/lease_single.py @app.put('/leasing/v1/lease/{lease_ref}', description='renew a lease') async def leasing_v1_lease_renew(request: Request, lease_ref: str): - token, cur_time = __get_token(request), datetime.utcnow() + cur_time = datetime.utcnow() + + default_instance = Instance.get_default_instance(db) + jwt_decode_key = default_instance.get_jwt_decode_key() + + try: + token = __get_token(request, jwt_decode_key) + except JWTError: + return JSONr(status_code=401, content={'status': 401, 'detail': 'token is not valid'}) origin_ref = token.get('origin_ref') logging.info(f'> [ renew ]: {origin_ref}: renew {lease_ref}') @@ -459,11 +489,11 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): if entity is None: return JSONr(status_code=404, content={'status': 404, 'detail': 'requested lease not available'}) - expires = cur_time + LEASE_EXPIRE_DELTA + expires = cur_time + default_instance.get_lease_expire_delta() response = { "lease_ref": lease_ref, "expires": expires.isoformat(), - "recommended_lease_renewal": LEASE_RENEWAL_PERIOD, + "recommended_lease_renewal": default_instance.lease_renewal_period, "offline_lease": True, "prompts": None, "sync_timestamp": cur_time.isoformat(), @@ -477,7 +507,14 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_single_controller.py @app.delete('/leasing/v1/lease/{lease_ref}', description='release (return) a lease') async def leasing_v1_lease_delete(request: Request, lease_ref: str): - token, cur_time = __get_token(request), datetime.utcnow() + cur_time = datetime.utcnow() + + jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() + + try: + token = __get_token(request, jwt_decode_key) + except JWTError: + return JSONr(status_code=401, content={'status': 401, 'detail': 'token is not valid'}) origin_ref = token.get('origin_ref') logging.info(f'> [ return ]: {origin_ref}: return {lease_ref}') @@ -503,7 +540,14 @@ async def leasing_v1_lease_delete(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.delete('/leasing/v1/lessor/leases', description='release all leases') async def leasing_v1_lessor_lease_remove(request: Request): - token, cur_time = __get_token(request), datetime.utcnow() + cur_time = datetime.utcnow() + + jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() + + try: + token = __get_token(request, jwt_decode_key) + except JWTError: + return JSONr(status_code=401, content={'status': 401, 'detail': 'token is not valid'}) origin_ref = token.get('origin_ref') @@ -525,6 +569,8 @@ async def leasing_v1_lessor_lease_remove(request: Request): async def leasing_v1_lessor_shutdown(request: Request): j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() + token = j.get('token') token = jwt.decode(token=token, key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) origin_ref = token.get('origin_ref') @@ -545,15 +591,23 @@ async def leasing_v1_lessor_shutdown(request: Request): @app.on_event('startup') async def app_on_startup(): + default_instance = Instance.get_default_instance(db) + + lease_renewal_period = default_instance.lease_renewal_period + lease_renewal_delta = default_instance.get_lease_renewal_delta() + client_token_expire_delta = default_instance.get_client_token_expire_delta() + logger.info(f''' Using timezone: {str(TZ)}. Make sure this is correct and match your clients! - Your clients renew their license every {str(Lease.calculate_renewal(LEASE_RENEWAL_PERIOD, LEASE_RENEWAL_DELTA))}. - If the renewal fails, the license is {str(LEASE_RENEWAL_DELTA)} valid. + Your clients will renew their license every {str(Lease.calculate_renewal(lease_renewal_period, lease_renewal_delta))}. + If the renewal fails, the license is valid for {str(lease_renewal_delta)}. - Your client-token file (.tok) is valid for {str(CLIENT_TOKEN_EXPIRE_DELTA)}. + Your client-token file (.tok) is valid for {str(client_token_expire_delta)}. ''') + validate_settings() + if __name__ == '__main__': import uvicorn diff --git a/app/util.py b/app/util.py index ca3eec6..fbf51fb 100644 --- a/app/util.py +++ b/app/util.py @@ -16,6 +16,18 @@ def load_key(filename) -> "RsaKey": return RSA.import_key(extern_key=load_file(filename), passphrase=None) +def parse_key(content: bytes) -> "RsaKey": + try: + # Crypto | Cryptodome on Debian + from Crypto.PublicKey import RSA + from Crypto.PublicKey.RSA import RsaKey + except ModuleNotFoundError: + from Cryptodome.PublicKey import RSA + from Cryptodome.PublicKey.RSA import RsaKey + + return RSA.import_key(extern_key=content, passphrase=None) + + def generate_key() -> "RsaKey": try: # Crypto | Cryptodome on Debian diff --git a/test/main.py b/test/main.py index f0f11d8..ad9072f 100644 --- a/test/main.py +++ b/test/main.py @@ -1,14 +1,15 @@ +from os import getenv as env from base64 import b64encode as b64enc from hashlib import sha256 from calendar import timegm from datetime import datetime -from os.path import dirname, join -from uuid import uuid4, UUID +from uuid import UUID, uuid4 from dateutil.relativedelta import relativedelta -from jose import jwt, jwk +from jose import jwt from jose.constants import ALGORITHMS from starlette.testclient import TestClient +from sqlalchemy import create_engine import sys # add relative path to use packages as they were in the app/ dir @@ -16,20 +17,23 @@ sys.path.append('../') sys.path.append('../app') from app import main -from app.util import load_key +from app.orm import init as db_init, migrate, Site, Instance -client = TestClient(main.app) ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-000000000001', 'HelloWorld' -# INSTANCE_KEY_RSA = generate_key() -# INSTANCE_KEY_PUB = INSTANCE_KEY_RSA.public_key() +# fastapi setup +client = TestClient(main.app) -INSTANCE_KEY_RSA = load_key(str(join(dirname(__file__), '../app/cert/instance.private.pem'))) -INSTANCE_KEY_PUB = load_key(str(join(dirname(__file__), '../app/cert/instance.public.pem'))) +# database setup +db = create_engine(str(env('DATABASE', 'sqlite:///db.sqlite'))) +db_init(db), migrate(db) -jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) -jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) +# test vars +DEFAULT_SITE, DEFAULT_INSTANCE = Site.get_default_site(db), Instance.get_default_instance(db) + +SITE_KEY = DEFAULT_SITE.site_key +jwt_encode_key, jwt_decode_key = DEFAULT_INSTANCE.get_jwt_encode_key(), DEFAULT_INSTANCE.get_jwt_decode_key() def __bearer_token(origin_ref: str) -> str: @@ -38,6 +42,12 @@ def __bearer_token(origin_ref: str) -> str: return token +def test_initial_default_site_and_instance(): + default_site, default_instance = Site.get_default_site(db), Instance.get_default_instance(db) + assert default_site.site_key == Site.INITIAL_SITE_KEY_XID + assert default_instance.instance_ref == Instance.DEFAULT_INSTANCE_REF + + def test_index(): response = client.get('/') assert response.status_code == 200 From 418109579138250c0561995e14b4415678537f7c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:17:12 +0200 Subject: [PATCH 08/69] requirements.txt updated --- Dockerfile | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99c76bd..e4e387b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN apk update \ && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev \ && apk add --no-cache curl postgresql postgresql-dev mariadb-connector-c-dev sqlite-dev \ && pip install --no-cache-dir --upgrade uvicorn \ - && pip install --no-cache-dir psycopg2==2.9.5 mysqlclient==2.1.1 pysqlite3==0.5.0 \ + && pip install --no-cache-dir psycopg2==3.1.9 mysqlclient==2.2.0 pysqlite3==0.5.1 \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && apk del build-deps diff --git a/requirements.txt b/requirements.txt index a460548..de8c05b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -fastapi==0.97.0 +fastapi==0.99.1 uvicorn[standard]==0.22.0 python-jose==3.3.0 pycryptodome==3.18.0 python-dateutil==2.8.2 -sqlalchemy==2.0.16 +sqlalchemy==2.0.17 markdown==3.4.3 python-dotenv==1.0.0 From b1620154db4885d3859431afbdabac40b40f5705 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:17:34 +0200 Subject: [PATCH 09/69] docker-compose.yml - added note for TZ --- docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f02cdc..f561a3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,10 @@ version: '3.9' x-dls-variables: &dls-variables - DLS_URL: localhost # REQUIRED, change to your ip or hostname - DLS_PORT: 443 # must match nginx listen & exposed port - LEASE_EXPIRE_DAYS: 90 + TZ: Europe/Berlin # REQUIRED, set your timezone correctly on fastapi-dls AND YOUR CLIENTS !!! + DLS_URL: localhost # REQUIRED, change to your ip or hostname + DLS_PORT: 443 + LEASE_EXPIRE_DAYS: 90 # 90 days is maximum DATABASE: sqlite:////app/database/db.sqlite DEBUG: false From d116eec6260eb07d9883ba8ad1cf45dc4838cdc5 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:17:45 +0200 Subject: [PATCH 10/69] updated compatibility list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d78b1e..21871ad 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Minimal Delegated License Service (DLS). -Compatibility tested with official DLS 2.0.1. +Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0. This service can be used without internet connection. Only the clients need a connection to this service on configured port. From c9e36759e30b8ba51b84f84481ef453aaec78ecc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:18:07 +0200 Subject: [PATCH 11/69] added 15.3 to supported drivers list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 21871ad..f6768b4 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,7 @@ Successfully tested with this package versions: | vGPU Suftware | vGPU Manager | Linux Driver | Windows Driver | Release Date | |---------------|--------------|--------------|----------------|---------------| +| `15.3` | `525.125.03` | `525.125.06` | `529.11` | June 2023 | | `15.2` | `525.105.14` | `525.105.17` | `528.89` | March 2023 | | `15.1` | `525.85.07` | `525.85.05` | `528.24` | January 2023 | | `15.0` | `525.60.12` | `525.60.13` | `527.41` | December 2022 | From 05cad95c2ab452db1b00b34a66692c11a80bd81c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:24:11 +0200 Subject: [PATCH 12/69] refactored docker-compose.yml so very simple example, and moved proxy to "examples" directory --- README.md | 4 +- docker-compose.yml | 110 ++----------------- examples/docker-compose-http-and-https.yml | 120 +++++++++++++++++++++ 3 files changed, 131 insertions(+), 103 deletions(-) create mode 100644 examples/docker-compose-http-and-https.yml diff --git a/README.md b/README.md index f6768b4..698dae2 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ docker run -e DLS_URL=`hostname -i` -e DLS_PORT=443 -p 443:443 -v $WORKING_DIR:/ **Docker-Compose / Deploy stack** -Goto [`docker-compose.yml`](docker-compose.yml) for more advanced example (with reverse proxy usage). +See [`examples`](examples) directory for more advanced examples (with reverse proxy usage). ```yaml version: '3.9' @@ -682,7 +682,7 @@ The error message can safely be ignored (since we have no license limitation :P) <0>:End Logging ``` -#### log with nginx as reverse proxy (see [docker-compose.yml](docker-compose.yml)) +#### log with nginx as reverse proxy (see [docker-compose-http-and-https.yml](examples/docker-compose-http-and-https.yml)) ``` <1>:NLS initialized diff --git a/docker-compose.yml b/docker-compose.yml index f561a3d..506a1a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,108 +14,16 @@ services: restart: always environment: <<: *dls-variables - volumes: - - /etc/timezone:/etc/timezone:ro - - /opt/docker/fastapi-dls/cert:/app/cert # instance.private.pem, instance.public.pem - - db:/app/database - entrypoint: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--app-dir", "/app", "--proxy-headers"] - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8000/-/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s - proxy: - image: nginx ports: - # thees are ports where nginx (!) is listen to - - "80:80" # for "/leasing/v1/lessor/shutdown" used by windows guests, can't be changed! - - "443:443" # first part must match "DLS_PORT" + - "443:443" volumes: - - /etc/timezone:/etc/timezone:ro - - /opt/docker/fastapi-dls/cert:/opt/cert - healthcheck: - test: ["CMD", "curl", "--insecure", "--fail", "https://localhost/-/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s - command: | - bash -c "bash -s <<\"EOF\" - cat > /etc/nginx/nginx.conf <<\"EON\" - daemon off; - user root; - worker_processes auto; - - events { - worker_connections 1024; - } - - http { - gzip on; - gzip_disable "msie6"; - include /etc/nginx/mime.types; - - upstream dls-backend { - server dls:8000; # must match dls listen port - } - - server { - listen 443 ssl http2 default_server; - listen [::]:443 ssl http2 default_server; - - root /var/www/html; - index index.html; - server_name _; - - ssl_certificate "/opt/cert/webserver.crt"; - ssl_certificate_key "/opt/cert/webserver.key"; - ssl_session_cache shared:SSL:1m; - ssl_session_timeout 10m; - ssl_protocols TLSv1.3 TLSv1.2; - # ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305"; - # ssl_ciphers PROFILE=SYSTEM; - ssl_prefer_server_ciphers on; - - location / { - proxy_set_header Host $$http_host; - proxy_set_header X-Real-IP $$remote_addr; - proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $$scheme; - proxy_pass http://dls-backend$$request_uri; - } - - location = /-/health { - access_log off; - add_header 'Content-Type' 'application/json'; - return 200 '{\"status\":\"up\",\"service\":\"nginx\"}'; - } - } - - server { - listen 80; - listen [::]:80; - - root /var/www/html; - index index.html; - server_name _; - - location /leasing/v1/lessor/shutdown { - proxy_set_header Host $$http_host; - proxy_set_header X-Real-IP $$remote_addr; - proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $$scheme; - proxy_pass http://dls-backend/leasing/v1/lessor/shutdown; - } - - location / { - return 301 https://$$host$$request_uri; - } - } - } - EON - nginx - EOF" + - /opt/docker/fastapi-dls/cert:/app/cert + - dls-db:/app/database + logging: # optional, for those who do not need logs + driver: "json-file" + options: + max-file: 5 + max-size: 10m volumes: - db: + dls-db: diff --git a/examples/docker-compose-http-and-https.yml b/examples/docker-compose-http-and-https.yml new file mode 100644 index 0000000..3f02cdc --- /dev/null +++ b/examples/docker-compose-http-and-https.yml @@ -0,0 +1,120 @@ +version: '3.9' + +x-dls-variables: &dls-variables + DLS_URL: localhost # REQUIRED, change to your ip or hostname + DLS_PORT: 443 # must match nginx listen & exposed port + LEASE_EXPIRE_DAYS: 90 + DATABASE: sqlite:////app/database/db.sqlite + DEBUG: false + +services: + dls: + image: collinwebdesigns/fastapi-dls:latest + restart: always + environment: + <<: *dls-variables + volumes: + - /etc/timezone:/etc/timezone:ro + - /opt/docker/fastapi-dls/cert:/app/cert # instance.private.pem, instance.public.pem + - db:/app/database + entrypoint: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--app-dir", "/app", "--proxy-headers"] + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8000/-/health"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + proxy: + image: nginx + ports: + # thees are ports where nginx (!) is listen to + - "80:80" # for "/leasing/v1/lessor/shutdown" used by windows guests, can't be changed! + - "443:443" # first part must match "DLS_PORT" + volumes: + - /etc/timezone:/etc/timezone:ro + - /opt/docker/fastapi-dls/cert:/opt/cert + healthcheck: + test: ["CMD", "curl", "--insecure", "--fail", "https://localhost/-/health"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + command: | + bash -c "bash -s <<\"EOF\" + cat > /etc/nginx/nginx.conf <<\"EON\" + daemon off; + user root; + worker_processes auto; + + events { + worker_connections 1024; + } + + http { + gzip on; + gzip_disable "msie6"; + include /etc/nginx/mime.types; + + upstream dls-backend { + server dls:8000; # must match dls listen port + } + + server { + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + + root /var/www/html; + index index.html; + server_name _; + + ssl_certificate "/opt/cert/webserver.crt"; + ssl_certificate_key "/opt/cert/webserver.key"; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_protocols TLSv1.3 TLSv1.2; + # ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305"; + # ssl_ciphers PROFILE=SYSTEM; + ssl_prefer_server_ciphers on; + + location / { + proxy_set_header Host $$http_host; + proxy_set_header X-Real-IP $$remote_addr; + proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $$scheme; + proxy_pass http://dls-backend$$request_uri; + } + + location = /-/health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{\"status\":\"up\",\"service\":\"nginx\"}'; + } + } + + server { + listen 80; + listen [::]:80; + + root /var/www/html; + index index.html; + server_name _; + + location /leasing/v1/lessor/shutdown { + proxy_set_header Host $$http_host; + proxy_set_header X-Real-IP $$remote_addr; + proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $$scheme; + proxy_pass http://dls-backend/leasing/v1/lessor/shutdown; + } + + location / { + return 301 https://$$host$$request_uri; + } + } + } + EON + nginx + EOF" + +volumes: + db: From acbe889fd9082a84755e9f0bee389452a75727e5 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:33:30 +0200 Subject: [PATCH 13/69] fixed versions --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e4e387b..2d1c745 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN apk update \ && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev \ && apk add --no-cache curl postgresql postgresql-dev mariadb-connector-c-dev sqlite-dev \ && pip install --no-cache-dir --upgrade uvicorn \ - && pip install --no-cache-dir psycopg2==3.1.9 mysqlclient==2.2.0 pysqlite3==0.5.1 \ + && pip install --no-cache-dir psycopg2==2.9.6 mysqlclient==2.2.0 pysqlite3==0.5.1 \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && apk del build-deps From 092e6186ab484f25ee37688f9f6d425b96cb3ae0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 10:46:51 +0200 Subject: [PATCH 14/69] added missing "pkg-config" for "mysqlclient==2.2.0" ref. https://stackoverflow.com/questions/76533384/docker-alpine-build-fails-on-mysqlclient-installation-with-error-exception-can --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d1c745..4022871 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN echo -e "VERSION=$VERSION\nCOMMIT=$COMMIT" > /version.env COPY requirements.txt /tmp/requirements.txt RUN apk update \ - && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev \ + && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev pkgconf \ && apk add --no-cache curl postgresql postgresql-dev mariadb-connector-c-dev sqlite-dev \ && pip install --no-cache-dir --upgrade uvicorn \ && pip install --no-cache-dir psycopg2==2.9.6 mysqlclient==2.2.0 pysqlite3==0.5.1 \ From 50eddeecfcbd960cb6238fd0244b01bdf8167bdb Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 11:06:00 +0200 Subject: [PATCH 15/69] fixed mariadb-client installation ref. https://github.com/PyMySQL/mysqlclient/discussions/624 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4022871..14d53b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,8 @@ RUN echo -e "VERSION=$VERSION\nCOMMIT=$COMMIT" > /version.env COPY requirements.txt /tmp/requirements.txt RUN apk update \ - && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev pkgconf \ - && apk add --no-cache curl postgresql postgresql-dev mariadb-connector-c-dev sqlite-dev \ + && apk add --no-cache --virtual build-deps gcc g++ python3-dev musl-dev pkgconfig \ + && apk add --no-cache curl postgresql postgresql-dev mariadb-dev sqlite-dev \ && pip install --no-cache-dir --upgrade uvicorn \ && pip install --no-cache-dir psycopg2==2.9.6 mysqlclient==2.2.0 pysqlite3==0.5.1 \ && pip install --no-cache-dir -r /tmp/requirements.txt \ From 32806e5ccaff3db8ae842f744e30ff3e0a9a0cce Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 17:47:10 +0200 Subject: [PATCH 16/69] push multiarch image to docker-hub --- .gitlab-ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eac63af..1ee7f07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,9 @@ include: cache: key: one-key-to-rule-them-all +variables: + DOCKER_BUILDX_PLATFORM: "linux/amd64,linux/arm64" + build:docker: image: docker:dind interruptible: true @@ -25,7 +28,7 @@ build:docker: script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - IMAGE=$CI_REGISTRY/$CI_PROJECT_PATH/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHA - - docker buildx build --progress=plain --platform linux/amd64,linux/arm64 --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE --push . + - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE --push . - docker buildx imagetools inspect $IMAGE - echo "CS_IMAGE=$IMAGE" > container_scanning.env artifacts: @@ -270,17 +273,13 @@ deploy:docker: - echo "========== GitLab-Registry ==========" - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - IMAGE=$CI_REGISTRY/$CI_PROJECT_PATH/$CI_COMMIT_REF_NAME - - docker build . --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:$CI_COMMIT_REF_NAME - - docker build . --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:latest - - docker push $IMAGE:$CI_COMMIT_REF_NAME - - docker push $IMAGE:latest + - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:$CI_COMMIT_REF_NAME --push . + - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:latest --push . - echo "========== Docker-Hub ==========" - docker login -u $PUBLIC_REGISTRY_USER -p $PUBLIC_REGISTRY_TOKEN - IMAGE=$PUBLIC_REGISTRY_USER/$CI_PROJECT_NAME - - docker build . --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:$CI_COMMIT_REF_NAME - - docker build . --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:latest - - docker push $IMAGE:$CI_COMMIT_REF_NAME - - docker push $IMAGE:latest + - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:$CI_COMMIT_REF_NAME --push . + - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:latest --push . deploy:apt: # doc: https://git.collinwebdesigns.de/help/user/packages/debian_repository/index.md#install-a-package From fce0eb6d74084ac3f03a8b9648ca51d864923eca Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 11:55:26 +0200 Subject: [PATCH 17/69] fixed "deploy:pacman" --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ee7f07..37ac17b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -330,7 +330,6 @@ deploy:pacman: artifacts: true script: - source .PKGBUILD/PKGBUILD - - source version.env # fastapi-dls-1.0-1-any.pkg.tar.zst - BUILD_NAME=${pkgname}-${CI_COMMIT_REF_NAME}-${pkgrel}-any.pkg.tar.zst - PACKAGE_NAME=${pkgname} From aa746feb13aa901b8933fc70d9c327502eefc351 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:19:07 +0200 Subject: [PATCH 18/69] improvements thanks to @AbsolutelyFree (https://gitea.publichub.eu/oscar.krause/fastapi-dls/issues/1) --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 698dae2..6102fb1 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ docker run -e DLS_URL=`hostname -i` -e DLS_PORT=443 -p 443:443 -v $WORKING_DIR:/ See [`examples`](examples) directory for more advanced examples (with reverse proxy usage). +> Adjust *REQUIRED* variables as needed + ```yaml version: '3.9' @@ -152,6 +154,8 @@ su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fast **Create config file** +> Adjust `DLS_URL` as needed (accessing from LAN won't work with 127.0.0.1) + ```shell mkdir /etc/fastapi-dls cat </etc/fastapi-dls/env @@ -254,10 +258,11 @@ su - ${SERVICE_USER} -c "${BASE_DIR}/venv/bin/uvicorn main:app --app-dir=${BASE_ **Create config file** +> Adjust `DLS_URL` as needed (accessing from LAN won't work with 127.0.0.1) + ```shell BASE_DIR=/opt/fastapi-dls cat </etc/fastapi-dls/env -# Adjust DSL_URL as needed (accessing from LAN won't work with 127.0.0.1) DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 @@ -332,6 +337,7 @@ apt-get install -f --fix-missing ``` Start with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`. +Now you have to edit `/etc/fastapi-dls/env` as needed. ## ArchLinux (using `pacman`) @@ -353,6 +359,7 @@ pacman -U --noconfirm fastapi-dls.pkg.tar.zst ``` Start with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`. +Now you have to edit `/etc/default/fastapi-dls` as needed. ## unRAID From aa4ebfce739041c754a9a47486ce570b9472251e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:22:22 +0200 Subject: [PATCH 19/69] added docker command to logging section thanks to @libreshare (https://gitea.publichub.eu/oscar.krause/fastapi-dls/issues/2) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6102fb1..92f7365 100644 --- a/README.md +++ b/README.md @@ -566,6 +566,10 @@ There are many other internal api endpoints for handling authentication and leas **Please make sure that fastapi-dls and your guests are on the same timezone!** +## Docker + +Logs are available with `docker logs `. To get the correct container-id use `docker container ls` or `docker ps`. + ## Linux Logs are available with `journalctl -u nvidia-gridd -f`. From a3baaab26fd617777a6ac2af84c181ed132a22e4 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:38:54 +0200 Subject: [PATCH 20/69] removed mysql from included docker drivers --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92f7365..3927bee 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Docker-Images are available here: - [Docker-Hub](https://hub.docker.com/repository/docker/collinwebdesigns/fastapi-dls): `collinwebdesigns/fastapi-dls:latest` - [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls/main:latest` -The images include database drivers for `postgres`, `mysql`, `mariadb` and `sqlite`. +The images include database drivers for `postgres`, `mariadb` and `sqlite`. **Run this on the Docker-Host** From 83f4b42f014408a4fa56b23b5adbdb1f43073e6f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:39:13 +0200 Subject: [PATCH 21/69] added information about ipv6 may be must disabled --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 3927bee..c11a149 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Only the clients need a connection to this service on configured port. - 256mb ram - 4gb hdd +- *maybe IPv6 must be disabled* Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb ram and 750mb hdd. @@ -566,6 +567,8 @@ There are many other internal api endpoints for handling authentication and leas **Please make sure that fastapi-dls and your guests are on the same timezone!** +Maybe you have to disable IPv6 on the machine you are running FastAPI-DLS. + ## Docker Logs are available with `docker logs `. To get the correct container-id use `docker container ls` or `docker ps`. From 5a1b1a5950798c7ae8d481997749d5cda0560e63 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:42:19 +0200 Subject: [PATCH 22/69] typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c11a149..9c3ed9a 100644 --- a/README.md +++ b/README.md @@ -630,7 +630,7 @@ only gets a valid local license.
- Log + Log example **Display-Container-LS** From 1ef7dd82f6148dc11e7b860b75bc911beca5e6fd Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 12:42:31 +0200 Subject: [PATCH 23/69] toggle api endpoints --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 9c3ed9a..191ced4 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,9 @@ Done. For more information check [troubleshoot section](#troubleshoot). # Endpoints +
+ show + ### `GET /` Redirect to `/-/readme`. @@ -562,6 +565,7 @@ Generate client token, (see [installation](#installation)). ### Others There are many other internal api endpoints for handling authentication and lease process. +
# Troubleshoot From b07b7da2f3d98cb65020d9c6af2ea753fff4e13e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 13:43:15 +0200 Subject: [PATCH 24/69] fixed new docker registry image path --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 191ced4..99b4ab7 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb Docker-Images are available here: - [Docker-Hub](https://hub.docker.com/repository/docker/collinwebdesigns/fastapi-dls): `collinwebdesigns/fastapi-dls:latest` -- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls/main:latest` +- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls:latest` The images include database drivers for `postgres`, `mariadb` and `sqlite`. From cad81ad1d6345a920b55fa513dfc9311760116e8 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 18:58:13 +0200 Subject: [PATCH 25/69] fixed deploy docker --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37ac17b..b0d67b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -266,9 +266,13 @@ gemnasium-python-dependency_scanning: deploy:docker: extends: .deploy + image: docker:dind stage: deploy + tags: [ docker ] before_script: - echo "Building docker image for commit $CI_COMMIT_SHA with version $CI_COMMIT_REF_NAME" + - docker buildx inspect + - docker buildx create --use script: - echo "========== GitLab-Registry ==========" - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY From f9341cdab478bb753611305ea12bc64bc607aba7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 4 Jul 2023 19:39:06 +0200 Subject: [PATCH 26/69] fixed docker image name (gitlab registry) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0d67b4..ae98336 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,7 +276,7 @@ deploy:docker: script: - echo "========== GitLab-Registry ==========" - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - IMAGE=$CI_REGISTRY/$CI_PROJECT_PATH/$CI_COMMIT_REF_NAME + - IMAGE=$CI_REGISTRY/$CI_PROJECT_PATH - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:$CI_COMMIT_REF_NAME --push . - docker buildx build --progress=plain --platform $DOCKER_BUILDX_PLATFORM --build-arg VERSION=$CI_COMMIT_REF_NAME --build-arg COMMIT=$CI_COMMIT_SHA --tag $IMAGE:latest --push . - echo "========== Docker-Hub ==========" From 23488f94d4061e1fcaf55e3d6e6847ec1e79d7cc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 10 Jul 2023 13:32:23 +0200 Subject: [PATCH 27/69] added support for 16.0 drivers to readme --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 99b4ab7..c015f78 100644 --- a/README.md +++ b/README.md @@ -423,14 +423,15 @@ client has 19.2 hours in which to re-establish connectivity before its license e Successfully tested with this package versions: -| vGPU Suftware | vGPU Manager | Linux Driver | Windows Driver | Release Date | -|---------------|--------------|--------------|----------------|---------------| -| `15.3` | `525.125.03` | `525.125.06` | `529.11` | June 2023 | -| `15.2` | `525.105.14` | `525.105.17` | `528.89` | March 2023 | -| `15.1` | `525.85.07` | `525.85.05` | `528.24` | January 2023 | -| `15.0` | `525.60.12` | `525.60.13` | `527.41` | December 2022 | -| `14.4` | `510.108.03` | `510.108.03` | `514.08` | December 2022 | -| `14.3` | `510.108.03` | `510.108.03` | `513.91` | November 2022 | +| vGPU Suftware | Linux vGPU Manager | Linux Driver | Windows Driver | Release Date | +|---------------|--------------------|--------------|----------------|---------------| +| `16.0` | `535.54.06` | `535.54.03` | `536.25` | July 2023 | +| `15.3` | `525.125.03` | `525.125.06` | `529.11` | June 2023 | +| `15.2` | `525.105.14` | `525.105.17` | `528.89` | March 2023 | +| `15.1` | `525.85.07` | `525.85.05` | `528.24` | January 2023 | +| `15.0` | `525.60.12` | `525.60.13` | `527.41` | December 2022 | +| `14.4` | `510.108.03` | `510.108.03` | `514.08` | December 2022 | +| `14.3` | `510.108.03` | `510.108.03` | `513.91` | November 2022 | - https://docs.nvidia.com/grid/index.html From 765a994d83af4ab21ae73c66a5cff71e08485ec1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 10 Jul 2023 18:47:46 +0200 Subject: [PATCH 28/69] requirements.txt updated --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index de8c05b..76e9181 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -fastapi==0.99.1 +fastapi==0.100.0 uvicorn[standard]==0.22.0 python-jose==3.3.0 pycryptodome==3.18.0 python-dateutil==2.8.2 -sqlalchemy==2.0.17 +sqlalchemy==2.0.18 markdown==3.4.3 python-dotenv==1.0.0 From 3dda920a52261a14b8fbfe1396265458a294a7cc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 13 Jul 2023 06:46:27 +0200 Subject: [PATCH 29/69] added linkt to driver compatibility section --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c015f78..ece6be6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Minimal Delegated License Service (DLS). -Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0. +Compatibility tested with official NLS 2.0.1, 2.1.0, 3.1.0. For Driver compatibility see [here](#setup-client). This service can be used without internet connection. Only the clients need a connection to this service on configured port. From 2e942f4553e6e433f6f8c2270669e2253275ab3d Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 26 Sep 2023 07:08:12 +0200 Subject: [PATCH 30/69] added Docker supported system architectures --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ece6be6..9f60b16 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb ## Docker -Docker-Images are available here: +Docker-Images are available here for Intel (x86), AMD (amd64) and ARM (arm64): - [Docker-Hub](https://hub.docker.com/repository/docker/collinwebdesigns/fastapi-dls): `collinwebdesigns/fastapi-dls:latest` - [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls:latest` From 1887cbc534c9235ad9492149118b873ffc2167b6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 26 Sep 2023 07:08:41 +0200 Subject: [PATCH 31/69] added macOS as supported host (using python-venv) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f60b16..b4a6147 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,10 @@ volumes: dls-db: ``` -## Debian/Ubuntu (manual method using `git clone` and python virtual environment) +## Debian/Ubuntu/macOS (manual method using `git clone` and python virtual environment) -Tested on `Debian 11 (bullseye)`, Ubuntu may also work. +Tested on `Debian 11 (bullseye)` and `macOS Ventura (13.6)`, Ubuntu may also work. **Please note that setup on macOS +differs from Debian based systems.** **Make sure you are logged in as root.** From 72965cc879e8d8426f2dbbb99461a40a259a7c2b Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 26 Sep 2023 07:08:58 +0200 Subject: [PATCH 32/69] added 16.1 as supported nvidia driver release --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4a6147..9fc8f03 100644 --- a/README.md +++ b/README.md @@ -426,7 +426,8 @@ Successfully tested with this package versions: | vGPU Suftware | Linux vGPU Manager | Linux Driver | Windows Driver | Release Date | |---------------|--------------------|--------------|----------------|---------------| -| `16.0` | `535.54.06` | `535.54.03` | `536.25` | July 2023 | +| `16.1` | `535.54.06` | `535.54.03` | `536.25` | August 2023 | +| `16.0` | `535.104.06` | `535.104.05` | `537.13` | July 2023 | | `15.3` | `525.125.03` | `525.125.06` | `529.11` | June 2023 | | `15.2` | `525.105.14` | `525.105.17` | `528.89` | March 2023 | | `15.1` | `525.85.07` | `525.85.05` | `528.24` | January 2023 | From 30979fd18eedad3fbdd49467b76d65001eb6af5e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 26 Sep 2023 07:09:06 +0200 Subject: [PATCH 33/69] requirements.txt updated --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 76e9181..4178b56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -fastapi==0.100.0 -uvicorn[standard]==0.22.0 +fastapi==0.103.1 +uvicorn[standard]==0.23.2 python-jose==3.3.0 -pycryptodome==3.18.0 +pycryptodome==3.19.0 python-dateutil==2.8.2 -sqlalchemy==2.0.18 -markdown==3.4.3 +sqlalchemy==2.0.21 +markdown==3.4.4 python-dotenv==1.0.0 From 7f8752a93d1e29b4f8ef0f7b443961b72f49ece3 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 16 Oct 2023 09:50:24 +0200 Subject: [PATCH 34/69] updated ubuntu from 22.10 (EOL) to 23.04 --- .gitlab-ci.yml | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae98336..f065e6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,7 +193,7 @@ test:debian: test:ubuntu: extends: .test:linux - image: ubuntu:22.10 + image: ubuntu:23.04 test:archlinux: image: archlinux:base diff --git a/README.md b/README.md index 9fc8f03..34b02fc 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Only the clients need a connection to this service on configured port. - 4gb hdd - *maybe IPv6 must be disabled* -Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb ram and 750mb hdd. +Tested with Ubuntu 22.10 (EOL!) (from Proxmox templates), actually its consuming 100mb ram and 750mb hdd. **Prepare your system** @@ -318,7 +318,8 @@ Packages are available here: Successful tested with: - Debian 12 (Bookworm) -- Ubuntu 22.10 (Kinetic Kudu) +- Ubuntu 22.10 (Kinetic Kudu) (EOL!) +- Ubuntu 23.04 (Lunar) Not working with: From 966b421dad93c7db5c04cea4e84c5090fb1b325e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 25 Oct 2023 07:30:57 +0200 Subject: [PATCH 35/69] README.md updated --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 34b02fc..5eacbaa 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,8 @@ volumes: ## Debian/Ubuntu/macOS (manual method using `git clone` and python virtual environment) -Tested on `Debian 11 (bullseye)` and `macOS Ventura (13.6)`, Ubuntu may also work. **Please note that setup on macOS -differs from Debian based systems.** +Tested on `Debian 11 (bullseye)`, `Debian 12 (bookworm)` and `macOS Ventura (13.6)`, Ubuntu may also work. +**Please note that setup on macOS differs from Debian based systems.** **Make sure you are logged in as root.** From 9c22628b4e3c3c88027c6b4407c26f5761816ef1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 25 Oct 2023 07:31:29 +0200 Subject: [PATCH 36/69] implemented python test matrix for different python dependencies on different os releases --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f065e6d..e4dc902 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,8 +134,14 @@ test: - if: $CI_PIPELINE_SOURCE == "merge_request_event" variables: DATABASE: sqlite:///../app/db.sqlite + parallel: + matrix: + - REQUIREMENTS: + - requirements.txt + - .DEBIAN/requirements-bookworm-12.txt + - .DEBIAN/requirements-ubuntu-23.04.txt before_script: - - pip install -r requirements.txt + - pip install -r $REQUIREMENTS - pip install pytest httpx - mkdir -p app/cert - openssl genrsa -out app/cert/instance.private.pem 2048 From 021c0ac38d4675c61dc3f25ab2c1798fe82b595c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 25 Oct 2023 07:36:17 +0200 Subject: [PATCH 37/69] added os specific requirements.txt --- .DEBIAN/requirements-bookworm-12.txt | 11 +++++++++++ .DEBIAN/requirements-ubuntu-23.04.txt | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 .DEBIAN/requirements-bookworm-12.txt create mode 100644 .DEBIAN/requirements-ubuntu-23.04.txt diff --git a/.DEBIAN/requirements-bookworm-12.txt b/.DEBIAN/requirements-bookworm-12.txt new file mode 100644 index 0000000..4819f80 --- /dev/null +++ b/.DEBIAN/requirements-bookworm-12.txt @@ -0,0 +1,11 @@ +# https://packages.debian.org/hu/ +fastapi==0.92.0 +uvicorn[standard]==0.17.6 +python-jose[pycryptodome]==3.3.0 +pycryptodome==3.11.0 +python-dateutil==2.8.2 +sqlalchemy==1.3.22 +markdown==3.4.1 +python-dotenv==0.21.0 +jinja2==3.0.3 +httpx==0.23.3 diff --git a/.DEBIAN/requirements-ubuntu-23.04.txt b/.DEBIAN/requirements-ubuntu-23.04.txt new file mode 100644 index 0000000..f2d4b05 --- /dev/null +++ b/.DEBIAN/requirements-ubuntu-23.04.txt @@ -0,0 +1,10 @@ +# https://packages.ubuntu.com +fastapi==0.91.0 +uvicorn[standard]==0.15.0 +python-jose[pycryptodome]==3.3.0 +pycryptodome==3.11.0 +python-dateutil==2.8.2 +sqlalchemy==1.4.46 +markdown==3.4.3 +python-dotenv==0.21.0 +jinja2==3.1.2 From 40cb5518cbf60e3ce4469a628977647c648bdaad Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 08:23:07 +0100 Subject: [PATCH 38/69] fixed versions & added 16.2 as supported --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5eacbaa..86c7b34 100644 --- a/README.md +++ b/README.md @@ -427,8 +427,9 @@ Successfully tested with this package versions: | vGPU Suftware | Linux vGPU Manager | Linux Driver | Windows Driver | Release Date | |---------------|--------------------|--------------|----------------|---------------| -| `16.1` | `535.54.06` | `535.54.03` | `536.25` | August 2023 | -| `16.0` | `535.104.06` | `535.104.05` | `537.13` | July 2023 | +| `16.2` | `535.129.03` | `535.129.03` | `537.70` | October 2023 | +| `16.1` | `535.104.06` | `535.104.05` | `537.13` | August 2023 | +| `16.0` | `535.54.06` | `535.54.03` | `536.22` | July 2023 | | `15.3` | `525.125.03` | `525.125.06` | `529.11` | June 2023 | | `15.2` | `525.105.14` | `525.105.17` | `528.89` | March 2023 | | `15.1` | `525.85.07` | `525.85.05` | `528.24` | January 2023 | From f2721c7663a5f65cae8de1694982fbdc9624c0fa Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 09:28:23 +0100 Subject: [PATCH 39/69] fixed debian package versions --- .DEBIAN/requirements-bookworm-12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.DEBIAN/requirements-bookworm-12.txt b/.DEBIAN/requirements-bookworm-12.txt index 4819f80..6b107fe 100644 --- a/.DEBIAN/requirements-bookworm-12.txt +++ b/.DEBIAN/requirements-bookworm-12.txt @@ -4,7 +4,7 @@ uvicorn[standard]==0.17.6 python-jose[pycryptodome]==3.3.0 pycryptodome==3.11.0 python-dateutil==2.8.2 -sqlalchemy==1.3.22 +sqlalchemy==1.4.46 markdown==3.4.1 python-dotenv==0.21.0 jinja2==3.0.3 From f77d3feee10b5bbdc2a24fb7b4d74fc0b2952d6a Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 10:49:06 +0100 Subject: [PATCH 40/69] fixes --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4dc902..df045b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -141,6 +141,7 @@ test: - .DEBIAN/requirements-bookworm-12.txt - .DEBIAN/requirements-ubuntu-23.04.txt before_script: + - apt-get update && apt-get install -y python3-dev - pip install -r $REQUIREMENTS - pip install pytest httpx - mkdir -p app/cert From 085186f82aa3e913e738bdde3b50f79c5abdba56 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 11:41:44 +0100 Subject: [PATCH 41/69] added gcc as dependency --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df045b3..03b53d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -141,7 +141,7 @@ test: - .DEBIAN/requirements-bookworm-12.txt - .DEBIAN/requirements-ubuntu-23.04.txt before_script: - - apt-get update && apt-get install -y python3-dev + - apt-get update && apt-get install -y python3-dev gcc - pip install -r $REQUIREMENTS - pip install pytest httpx - mkdir -p app/cert From cb6a0896781d7b74a62086cfa210bef4a0661532 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 12:53:50 +0100 Subject: [PATCH 42/69] fixed testing dependency --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03b53d2..4174f16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,6 +142,7 @@ test: - .DEBIAN/requirements-ubuntu-23.04.txt before_script: - apt-get update && apt-get install -y python3-dev gcc + - pip install cython - pip install -r $REQUIREMENTS - pip install pytest httpx - mkdir -p app/cert From ff9e85e32b5b4b4cc86a34d224b4c8adff1852b4 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 3 Nov 2023 14:03:48 +0100 Subject: [PATCH 43/69] updated test to debian bookworm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4174f16..a05d904 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,7 +126,7 @@ build:pacman: - "*.pkg.tar.zst" test: - image: python:3.11-slim-bullseye + image: python:3.11-slim-bookworm stage: test rules: - if: $CI_COMMIT_BRANCH From 11138c21913be1ef9647c2e7f4f4b5f2bc6c16c5 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 12 Jan 2024 14:25:03 +0100 Subject: [PATCH 44/69] updated debian bookworm 12 dependencies --- .DEBIAN/requirements-bookworm-12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.DEBIAN/requirements-bookworm-12.txt b/.DEBIAN/requirements-bookworm-12.txt index 6b107fe..223c64c 100644 --- a/.DEBIAN/requirements-bookworm-12.txt +++ b/.DEBIAN/requirements-bookworm-12.txt @@ -7,5 +7,5 @@ python-dateutil==2.8.2 sqlalchemy==1.4.46 markdown==3.4.1 python-dotenv==0.21.0 -jinja2==3.0.3 +jinja2==3.1.2 httpx==0.23.3 From 36d5b83fb891987cf36228c8d1e31163998b3faa Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 12 Jan 2024 14:53:17 +0100 Subject: [PATCH 45/69] requirements.txt updated --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4178b56..68afc4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -fastapi==0.103.1 -uvicorn[standard]==0.23.2 +fastapi==0.109.0 +uvicorn[standard]==0.25.0 python-jose==3.3.0 -pycryptodome==3.19.0 +pycryptodome==3.20.0 python-dateutil==2.8.2 -sqlalchemy==2.0.21 -markdown==3.4.4 +sqlalchemy==2.0.25 +markdown==3.5.2 python-dotenv==1.0.0 From e41084f5c5131aacde9bff6e65641024045984c6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 08:08:37 +0100 Subject: [PATCH 46/69] added tests for Ubuntu "Mantic Minotaur" --- .DEBIAN/requirements-ubuntu-23.10.txt | 10 ++++++++++ .gitlab-ci.yml | 3 ++- README.md | 5 +++-- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 .DEBIAN/requirements-ubuntu-23.10.txt diff --git a/.DEBIAN/requirements-ubuntu-23.10.txt b/.DEBIAN/requirements-ubuntu-23.10.txt new file mode 100644 index 0000000..4cab03f --- /dev/null +++ b/.DEBIAN/requirements-ubuntu-23.10.txt @@ -0,0 +1,10 @@ +# https://packages.ubuntu.com +fastapi==0.101.0 +uvicorn[standard]==0.23.2 +python-jose[pycryptodome]==3.3.0 +pycryptodome==3.11.0 +python-dateutil==2.8.2 +sqlalchemy==1.4.47 +markdown==3.4.4 +python-dotenv==1.0.0 +jinja2==3.1.2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a05d904..a5e48f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,6 +140,7 @@ test: - requirements.txt - .DEBIAN/requirements-bookworm-12.txt - .DEBIAN/requirements-ubuntu-23.04.txt + - .DEBIAN/requirements-ubuntu-23.10.txt before_script: - apt-get update && apt-get install -y python3-dev gcc - pip install cython @@ -201,7 +202,7 @@ test:debian: test:ubuntu: extends: .test:linux - image: ubuntu:23.04 + image: ubuntu:23.10 test:archlinux: image: archlinux:base diff --git a/README.md b/README.md index 86c7b34..5bd6af9 100644 --- a/README.md +++ b/README.md @@ -318,8 +318,9 @@ Packages are available here: Successful tested with: - Debian 12 (Bookworm) -- Ubuntu 22.10 (Kinetic Kudu) (EOL!) -- Ubuntu 23.04 (Lunar) +- Ubuntu 22.10 (Kinetic Kudu) (EOL: July 20, 2023) +- Ubuntu 23.04 (Lunar Lobster) (EOL: January 2024) +- Ubuntu 23.10 (Mantic Minotaur) (EOL: July 2024) Not working with: From 4c0f65faec43d28e7493cb6cc67dcec60e8b62bc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 09:33:58 +0100 Subject: [PATCH 47/69] removed tests for "23.04" > gcc -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/tmp/pip-install-sazb8fvo/httptools_694f06fa2e354ed9ba9f5c167df7fce4/vendor/llhttp/include -I/tmp/pip-install-sazb8fvo/httptools_694f06fa2e354ed9ba9f5c167df7fce4/vendor/llhttp/src -I/usr/local/include/python3.11 -c httptools/parser/parser.c -o build/temp.linux-x86_64-cpython-311/httptools/parser/parser.o -O2 httptools/parser/parser.c:212:12: fatal error: longintrepr.h: No such file or directory --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5e48f2..c9cc95b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,7 +139,6 @@ test: - REQUIREMENTS: - requirements.txt - .DEBIAN/requirements-bookworm-12.txt - - .DEBIAN/requirements-ubuntu-23.04.txt - .DEBIAN/requirements-ubuntu-23.10.txt before_script: - apt-get update && apt-get install -y python3-dev gcc From eecb59e2e4ed2d216dd3f3a6ca68e63464a9394d Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 11:33:22 +0100 Subject: [PATCH 48/69] removed "cython" from "test" --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c9cc95b..80a2bdc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,7 +142,6 @@ test: - .DEBIAN/requirements-ubuntu-23.10.txt before_script: - apt-get update && apt-get install -y python3-dev gcc - - pip install cython - pip install -r $REQUIREMENTS - pip install pytest httpx - mkdir -p app/cert From 90e0cb8e8493d95e6973b47bedc0ef129e596a07 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 22:37:33 +0100 Subject: [PATCH 49/69] =?UTF-8?q?added=20code=5Fquality=20=E2=80=9CSOURCE?= =?UTF-8?q?=5FCODE=E2=80=9D=20variable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80a2bdc..ce4a870 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,8 @@ test:archlinux: - pacman -U --noconfirm *.pkg.tar.zst code_quality: + variables: + SOURCE_CODE: app rules: - if: $CODE_QUALITY_DISABLED when: never From a54411a95761a59349775f43dc142016c048f159 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 22:43:47 +0100 Subject: [PATCH 50/69] added code_quality debug --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce4a870..7371bb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,6 +220,8 @@ test:archlinux: code_quality: variables: SOURCE_CODE: app + CODECLIMATE_DEBUG: 1 + REPORT_STDOUT: 1 rules: - if: $CODE_QUALITY_DISABLED when: never From 6c1b05c66a000bb84bd07c3d4bd54fd0847ed1c8 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 23:05:57 +0100 Subject: [PATCH 51/69] fixed test_coverage (fail on matrix) --- .gitlab-ci.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7371bb7..cca2e06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,10 +242,22 @@ semgrep-sast: - if: $CI_PIPELINE_SOURCE == "merge_request_event" test_coverage: - extends: test +# extends: test + image: python:3.11-slim-bookworm allow_failure: true + stage: test rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" + variables: + DATABASE: sqlite:///../app/db.sqlite + before_script: + - apt-get update && apt-get install -y python3-dev gcc + - pip install -r requirements.txt + - pip install pytest httpx + - mkdir -p app/cert + - openssl genrsa -out app/cert/instance.private.pem 2048 + - openssl rsa -in app/cert/instance.private.pem -outform PEM -pubout -out app/cert/instance.public.pem + - cd test script: - pip install pytest pytest-cov - coverage run -m pytest main.py From 77be5772c49ab94266e0d367a31f874c0b187012 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 17 Jan 2024 23:13:20 +0100 Subject: [PATCH 52/69] Update .codeclimate.yml --- .codeclimate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index 09c810c..2f7e959 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,7 +1,11 @@ +version: "2" plugins: bandit: enabled: true sonar-python: enabled: true + config: + tests_patterns: + - test/** pylint: enabled: true From e70f70d806654c5d9893c92a7680cd587510ddb9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 08:34:43 +0100 Subject: [PATCH 53/69] disabled code_quality debug --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cca2e06..8c0c56e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,8 +220,6 @@ test:archlinux: code_quality: variables: SOURCE_CODE: app - CODECLIMATE_DEBUG: 1 - REPORT_STDOUT: 1 rules: - if: $CODE_QUALITY_DISABLED when: never From a6ea8241c2d2d02b29fb7b5129d7341edd38e118 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 12:46:51 +0100 Subject: [PATCH 54/69] disabled pylint --- .codeclimate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 2f7e959..931aa68 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -7,5 +7,5 @@ plugins: config: tests_patterns: - test/** - pylint: - enabled: true +# pylint: +# enabled: true From 416df311b878ed6d6a6492bf71928660177ecafa Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 12:58:43 +0100 Subject: [PATCH 55/69] removed pylint --- .codeclimate.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 931aa68..fd7ed1f 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -7,5 +7,3 @@ plugins: config: tests_patterns: - test/** -# pylint: -# enabled: true From f82d73bb01184c35f4f77ff3dcc762af681f5bf6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 12:59:06 +0100 Subject: [PATCH 56/69] run different jobs on "$CI_DEFAULT_BRANCH" --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c0c56e..4e939c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -224,6 +224,7 @@ code_quality: - if: $CODE_QUALITY_DISABLED when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH secret_detection: rules: @@ -238,6 +239,7 @@ semgrep-sast: - if: $SAST_DISABLED when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH test_coverage: # extends: test @@ -280,6 +282,7 @@ gemnasium-python-dependency_scanning: - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH .deploy: rules: From 78b6fe52c7e9cdcb6f254fde3fb81a0b6cc39428 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 13:06:45 +0100 Subject: [PATCH 57/69] fixed CI/CD path from "/builds" to "/tmp/builds" --- .PKGBUILD/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.PKGBUILD/PKGBUILD b/.PKGBUILD/PKGBUILD index 3913360..f1ed955 100644 --- a/.PKGBUILD/PKGBUILD +++ b/.PKGBUILD/PKGBUILD @@ -12,7 +12,7 @@ depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastap provider=("$pkgname") install="$pkgname.install" backup=('etc/default/fastapi-dls') -source=('git+file:///builds/oscar.krause/fastapi-dls' # https://gitea.publichub.eu/oscar.krause/fastapi-dls.git +source=('git+file:///tmp/builds/oscar.krause/fastapi-dls' # https://gitea.publichub.eu/oscar.krause/fastapi-dls.git "$pkgname.default" "$pkgname.service" "$pkgname.tmpfiles") From 00dc848083cd4883f187c30d17f6768ec3f1d88c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 13:09:30 +0100 Subject: [PATCH 58/69] only run test matrix when "app" or "test" changes --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e939c8..3e1c299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -129,9 +129,13 @@ test: image: python:3.11-slim-bookworm stage: test rules: - - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_COMMIT_TAG - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + changes: + - app/**/* + - test/**/* variables: DATABASE: sqlite:///../app/db.sqlite parallel: From 72054d30c479559953ed36cef031302257ffecc0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 13:10:12 +0100 Subject: [PATCH 59/69] make tests interruptible --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e1c299..95debb0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,6 +128,7 @@ build:pacman: test: image: python:3.11-slim-bookworm stage: test + interruptible: true rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_COMMIT_TAG From a927e291b503e9262098b575417023d8b43bc083 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 13:30:30 +0100 Subject: [PATCH 60/69] fixes --- .PKGBUILD/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.PKGBUILD/PKGBUILD b/.PKGBUILD/PKGBUILD index f1ed955..85a35cc 100644 --- a/.PKGBUILD/PKGBUILD +++ b/.PKGBUILD/PKGBUILD @@ -12,7 +12,7 @@ depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastap provider=("$pkgname") install="$pkgname.install" backup=('etc/default/fastapi-dls') -source=('git+file:///tmp/builds/oscar.krause/fastapi-dls' # https://gitea.publichub.eu/oscar.krause/fastapi-dls.git +source=("git+file://${CI_PROJECT_DIR}" # todo: support dynamic CI/CD paths "$pkgname.default" "$pkgname.service" "$pkgname.tmpfiles") From 3e87820f6324f2eb24faff3a5d76d11f1ea6b461 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 18 Jan 2024 17:02:09 +0100 Subject: [PATCH 61/69] removed todo --- .PKGBUILD/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.PKGBUILD/PKGBUILD b/.PKGBUILD/PKGBUILD index 85a35cc..09f606b 100644 --- a/.PKGBUILD/PKGBUILD +++ b/.PKGBUILD/PKGBUILD @@ -12,7 +12,7 @@ depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastap provider=("$pkgname") install="$pkgname.install" backup=('etc/default/fastapi-dls') -source=("git+file://${CI_PROJECT_DIR}" # todo: support dynamic CI/CD paths +source=("git+file://${CI_PROJECT_DIR}" "$pkgname.default" "$pkgname.service" "$pkgname.tmpfiles") From 07aec5378784d742a4de8a15bda7b024af5319db Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 26 Feb 2024 20:53:33 +0100 Subject: [PATCH 62/69] requirements.txt updated --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 68afc4f..6eea61c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -fastapi==0.109.0 -uvicorn[standard]==0.25.0 +fastapi==0.110.0 +uvicorn[standard]==0.27.1 python-jose==3.3.0 pycryptodome==3.20.0 python-dateutil==2.8.2 -sqlalchemy==2.0.25 +sqlalchemy==2.0.27 markdown==3.5.2 -python-dotenv==1.0.0 +python-dotenv==1.0.1 From 16f80cd78b8380b22a4e40cad08a3a8f12176cf0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Mon, 26 Feb 2024 20:53:47 +0100 Subject: [PATCH 63/69] added "16.3" support --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 5bd6af9..8f7ffea 100644 --- a/README.md +++ b/README.md @@ -428,6 +428,7 @@ Successfully tested with this package versions: | vGPU Suftware | Linux vGPU Manager | Linux Driver | Windows Driver | Release Date | |---------------|--------------------|--------------|----------------|---------------| +| `16.3` | `535.154.02` | `535.154.05` | `538.15` | January 2024 | | `16.2` | `535.129.03` | `535.129.03` | `537.70` | October 2023 | | `16.1` | `535.104.06` | `535.104.05` | `537.13` | August 2023 | | `16.0` | `535.54.06` | `535.54.03` | `536.22` | July 2023 | @@ -440,6 +441,11 @@ Successfully tested with this package versions: - https://docs.nvidia.com/grid/index.html +*To get the latest drivers, visit Nvidia or search in Discord-Channel `GPU Unlocking` (Server-ID: `829786927829745685`) on channel `licensing` `biggerthanshit` + + +https://archive.biggerthanshit.com/NVIDIA/ (nvidia / b1gg3rth4nsh1t) + ## Linux Download *client-token* and place it into `/etc/nvidia/ClientConfigToken`: From cd4674caad43064156e6869e94e64dc93acf0ba0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 21 Jun 2024 19:35:42 +0200 Subject: [PATCH 64/69] fixes --- app/main.py | 9 +++++---- app/orm.py | 24 +++++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/app/main.py b/app/main.py index 321fd34..91557e7 100644 --- a/app/main.py +++ b/app/main.py @@ -2,7 +2,7 @@ import logging from base64 import b64encode as b64enc from calendar import timegm from contextlib import asynccontextmanager -from datetime import datetime, timedelta +from datetime import datetime from hashlib import sha256 from json import loads as json_loads from os import getenv as env @@ -13,12 +13,13 @@ from dateutil.relativedelta import relativedelta from dotenv import load_dotenv from fastapi import FastAPI from fastapi.requests import Request -from jose import jws, jwk, jwt, JWTError +from jose import jws, jwt, JWTError from jose.constants import ALGORITHMS from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from starlette.middleware.cors import CORSMiddleware -from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse +from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, \ + RedirectResponse from orm import init as db_init, migrate, Site, Instance, Origin, Lease @@ -94,7 +95,7 @@ logging.getLogger('NV').setLevel(LOG_LEVEL) # Helper -def __get_token(request: Request) -> dict: +def __get_token(request: Request, jwt_decode_key: "jose.jwt") -> dict: authorization_header = request.headers.get('authorization') token = authorization_header.split(' ')[1] return jwt.decode(token=token, key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) diff --git a/app/orm.py b/app/orm.py index 79cc421..a63186c 100644 --- a/app/orm.py +++ b/app/orm.py @@ -2,7 +2,7 @@ import logging from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta -from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text, BLOB, INT, FLOAT from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker, declarative_base, Session, relationship @@ -327,7 +327,6 @@ class Lease(Base): def init_default_site(session: Session): - from uuid import uuid4 from app.util import generate_key private_key = generate_key() @@ -398,30 +397,37 @@ def migrate(engine: Engine): # INSTANCE_KEY_RSA, INSTANCE_KEY_PUB default_instance_private_key_path = str(join(dirname(__file__), 'cert/instance.private.pem')) - if instance_private_key := env('INSTANCE_KEY_RSA', None) is not None: + instance_private_key = env('INSTANCE_KEY_RSA', None) + if instance_private_key is not None: instance.private_key = load_key(str(instance_private_key)) elif isfile(default_instance_private_key_path): instance.private_key = load_key(default_instance_private_key_path) default_instance_public_key_path = str(join(dirname(__file__), 'cert/instance.public.pem')) - if instance_public_key := env('INSTANCE_KEY_PUB', None) is not None: + instance_public_key = env('INSTANCE_KEY_PUB', None) + if instance_public_key is not None: instance.public_key = load_key(str(instance_public_key)) elif isfile(default_instance_public_key_path): instance.public_key = load_key(default_instance_public_key_path) # TOKEN_EXPIRE_DELTA - if token_expire_delta := env('TOKEN_EXPIRE_DAYS', None) not in (None, 0): + token_expire_delta = env('TOKEN_EXPIRE_DAYS', None) + if token_expire_delta not in (None, 0): instance.token_expire_delta = token_expire_delta * 86_400 - if token_expire_delta := env('TOKEN_EXPIRE_HOURS', None) not in (None, 0): + token_expire_delta = env('TOKEN_EXPIRE_HOURS', None) + if token_expire_delta not in (None, 0): instance.token_expire_delta = token_expire_delta * 3_600 # LEASE_EXPIRE_DELTA, LEASE_RENEWAL_DELTA - if lease_expire_delta := env('LEASE_EXPIRE_DAYS', None) not in (None, 0): + lease_expire_delta = env('LEASE_EXPIRE_DAYS', None) + if lease_expire_delta not in (None, 0): instance.lease_expire_delta = lease_expire_delta * 86_400 - if lease_expire_delta := env('LEASE_EXPIRE_HOURS', None) not in (None, 0): + lease_expire_delta = env('LEASE_EXPIRE_HOURS', None) + if lease_expire_delta not in (None, 0): instance.lease_expire_delta = lease_expire_delta * 3_600 # LEASE_RENEWAL_PERIOD - if lease_renewal_period := env('LEASE_RENEWAL_PERIOD', None) is not None: + lease_renewal_period = env('LEASE_RENEWAL_PERIOD', None) + if lease_renewal_period is not None: instance.lease_renewal_period = lease_renewal_period # todo: update site, instance From 20cdaefa1c9bab0f71ea665871e006cf01226ed1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 8 Apr 2025 13:52:09 +0200 Subject: [PATCH 65/69] code refactorings after merge from main --- app/main.py | 22 +++++++++++++--------- app/orm.py | 51 ++++++++++++++++++++++----------------------------- test/main.py | 4 ++-- 3 files changed, 37 insertions(+), 40 deletions(-) diff --git a/app/main.py b/app/main.py index 6ffbc19..3b619e2 100644 --- a/app/main.py +++ b/app/main.py @@ -1,8 +1,9 @@ import logging +import sys from base64 import b64encode as b64enc from calendar import timegm from contextlib import asynccontextmanager -from datetime import datetime, timedelta, UTC +from datetime import datetime, UTC from hashlib import sha256 from json import loads as json_loads from os import getenv as env @@ -21,8 +22,11 @@ from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, \ RedirectResponse -from orm import Origin, Lease, init as db_init, migrate -from util import PrivateKey, PublicKey, load_file +# add relative path to use packages as they were in the app/ dir +sys.path.append('../') +sys.path.append('../app') + +from orm import Origin, Lease, init as db_init, migrate, Instance, Site # Load variables load_dotenv('../version.env') @@ -249,7 +253,7 @@ async def _lease_delete(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_core_service_instance/service_instance_token_manager.py @app.get('/-/client-token', summary='* Client-Token', description='creates a new messenger token for this service instance') async def _client_token(): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) default_instance = Instance.get_default_instance(db) public_key = default_instance.get_public_key() @@ -438,7 +442,7 @@ async def auth_v1_token(request: Request): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.post('/leasing/v1/lessor', description='request multiple leases (borrow) for current origin') async def leasing_v1_lessor(request: Request): - j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow() + j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.now(UTC) default_instance = Instance.get_default_instance(db) jwt_decode_key = default_instance.get_jwt_decode_key() @@ -489,7 +493,7 @@ async def leasing_v1_lessor(request: Request): # venv/lib/python3.9/site-packages/nls_dal_service_instance_dls/schema/service_instance/V1_0_21__product_mapping.sql @app.get('/leasing/v1/lessor/leases', description='get active leases for current origin') async def leasing_v1_lessor_lease(request: Request): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() @@ -516,7 +520,7 @@ async def leasing_v1_lessor_lease(request: Request): # venv/lib/python3.9/site-packages/nls_core_lease/lease_single.py @app.put('/leasing/v1/lease/{lease_ref}', description='renew a lease') async def leasing_v1_lease_renew(request: Request, lease_ref: str): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) default_instance = Instance.get_default_instance(db) jwt_decode_key = default_instance.get_jwt_decode_key() @@ -551,7 +555,7 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_single_controller.py @app.delete('/leasing/v1/lease/{lease_ref}', description='release (return) a lease') async def leasing_v1_lease_delete(request: Request, lease_ref: str): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() @@ -584,7 +588,7 @@ async def leasing_v1_lease_delete(request: Request, lease_ref: str): # venv/lib/python3.9/site-packages/nls_services_lease/test/test_lease_multi_controller.py @app.delete('/leasing/v1/lessor/leases', description='release all leases') async def leasing_v1_lessor_lease_remove(request: Request): - cur_time = datetime.utcnow() + cur_time = datetime.now(UTC) jwt_decode_key = Instance.get_default_instance(db).get_jwt_decode_key() diff --git a/app/orm.py b/app/orm.py index 87ec623..da9d977 100644 --- a/app/orm.py +++ b/app/orm.py @@ -1,11 +1,17 @@ +import logging from datetime import datetime, timedelta, timezone, UTC +from os import getenv as env +from os.path import join, dirname, isfile from dateutil.relativedelta import relativedelta -from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text +from jose import jwk +from jose.constants import ALGORITHMS +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text, BLOB, INT, FLOAT from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker, declarative_base, Session, relationship +from sqlalchemy.schema import CreateTable -from util import NV +from util import NV, PrivateKey, PublicKey logging.basicConfig() logger = logging.getLogger(__name__) @@ -28,7 +34,6 @@ class Site(Base): @staticmethod def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable return CreateTable(Site.__table__).compile(engine) @staticmethod @@ -65,7 +70,6 @@ class Instance(Base): @staticmethod def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable return CreateTable(Instance.__table__).compile(engine) @staticmethod @@ -111,21 +115,18 @@ class Instance(Base): def get_client_token_expire_delta(self) -> "dateutil.relativedelta.relativedelta": return relativedelta(seconds=self.client_token_expire_delta) - def __get_private_key(self) -> "RsaKey": - return parse_key(self.private_key) + def __get_private_key(self) -> "PrivateKey": + return PrivateKey(self.private_key) - def get_public_key(self) -> "RsaKey": - return parse_key(self.public_key) + def get_public_key(self) -> "PublicKey": + return PublicKey(self.public_key) def get_jwt_encode_key(self) -> "jose.jkw": - from jose import jwk - from jose.constants import ALGORITHMS - return jwk.construct(self.__get_private_key().export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) + + return jwk.construct(self.__get_private_key().pem().decode('utf-8'), algorithm=ALGORITHMS.RS256) def get_jwt_decode_key(self) -> "jose.jwt": - from jose import jwk - from jose.constants import ALGORITHMS - return jwk.construct(self.get_public_key().export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) + return jwk.construct(self.get_public_key().pem().decode('utf-8'), algorithm=ALGORITHMS.RS256) def get_private_key_str(self, encoding: str = 'utf-8') -> str: return self.private_key.decode(encoding) @@ -162,7 +163,6 @@ class Origin(Base): @staticmethod def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable return CreateTable(Origin.__table__).compile(engine) @staticmethod @@ -241,7 +241,6 @@ class Lease(Base): @staticmethod def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable return CreateTable(Lease.__table__).compile(engine) @staticmethod @@ -336,9 +335,7 @@ class Lease(Base): def init_default_site(session: Session): - from app.util import generate_key - - private_key = generate_key() + private_key = PrivateKey.generate() public_key = private_key.public_key() site = Site( @@ -351,8 +348,8 @@ def init_default_site(session: Session): instance = Instance( instance_ref=Instance.DEFAULT_INSTANCE_REF, site_key=site.site_key, - private_key=private_key.export_key(), - public_key=public_key.export_key(), + private_key=private_key.pem(), + public_key=public_key.pem(), ) session.add(instance) session.commit() @@ -379,10 +376,6 @@ def init(engine: Engine): def migrate(engine: Engine): - from os import getenv as env - from os.path import join, dirname, isfile - from util import load_key - db = inspect(engine) # todo: add update guide to use 1.LATEST to 2.0 @@ -408,15 +401,15 @@ def migrate(engine: Engine): default_instance_private_key_path = str(join(dirname(__file__), 'cert/instance.private.pem')) instance_private_key = env('INSTANCE_KEY_RSA', None) if instance_private_key is not None: - instance.private_key = load_key(str(instance_private_key)) + instance.private_key = PrivateKey(instance_private_key.encode('utf-8')) elif isfile(default_instance_private_key_path): - instance.private_key = load_key(default_instance_private_key_path) + instance.private_key = PrivateKey.from_file(default_instance_private_key_path) default_instance_public_key_path = str(join(dirname(__file__), 'cert/instance.public.pem')) instance_public_key = env('INSTANCE_KEY_PUB', None) if instance_public_key is not None: - instance.public_key = load_key(str(instance_public_key)) + instance.public_key = PublicKey(instance_public_key.encode('utf-8')) elif isfile(default_instance_public_key_path): - instance.public_key = load_key(default_instance_public_key_path) + instance.public_key = PublicKey.from_file(default_instance_public_key_path) # TOKEN_EXPIRE_DELTA token_expire_delta = env('TOKEN_EXPIRE_DAYS', None) diff --git a/test/main.py b/test/main.py index 556ff09..6442148 100644 --- a/test/main.py +++ b/test/main.py @@ -2,7 +2,7 @@ from os import getenv as env from base64 import b64encode as b64enc from hashlib import sha256 from calendar import timegm -from datetime import datetime +from datetime import datetime, UTC from uuid import UUID, uuid4 from dateutil.relativedelta import relativedelta @@ -17,7 +17,7 @@ sys.path.append('../') sys.path.append('../app') from app import main -from app.orm import init as db_init, migrate, Site, Instance +from orm import init as db_init, migrate, Site, Instance ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-000000000001', 'HelloWorld' From ddf5f124095eff7777b0b9f9ae627f92919dd738 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 8 Apr 2025 14:00:15 +0200 Subject: [PATCH 66/69] fixes --- app/main.py | 6 +++--- app/orm.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/main.py b/app/main.py index 3b619e2..b31c5ec 100644 --- a/app/main.py +++ b/app/main.py @@ -285,10 +285,10 @@ async def _client_token(): }, "service_instance_public_key_configuration": { "service_instance_public_key_me": { - "mod": hex(public_key.public_key().n)[2:], - "exp": int(public_key.public_key().e), + "mod": hex(public_key.raw().public_numbers().n)[2:], + "exp": int(public_key.raw().public_numbers().e), }, - "service_instance_public_key_pem": public_key.export_key().decode('utf-8'), + "service_instance_public_key_pem": public_key.pem().decode('utf-8'), "key_retention_mode": "LATEST_ONLY" }, } diff --git a/app/orm.py b/app/orm.py index da9d977..6c66b30 100644 --- a/app/orm.py +++ b/app/orm.py @@ -122,7 +122,6 @@ class Instance(Base): return PublicKey(self.public_key) def get_jwt_encode_key(self) -> "jose.jkw": - return jwk.construct(self.__get_private_key().pem().decode('utf-8'), algorithm=ALGORITHMS.RS256) def get_jwt_decode_key(self) -> "jose.jwt": From a31c80465afa198a9a0bfdb2bca46fd595676ca0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 8 Apr 2025 14:05:54 +0200 Subject: [PATCH 67/69] code styling --- app/main.py | 9 ++------- test/main.py | 10 +++++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/main.py b/app/main.py index b31c5ec..9f30ba0 100644 --- a/app/main.py +++ b/app/main.py @@ -19,12 +19,7 @@ from jose.constants import ALGORITHMS from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from starlette.middleware.cors import CORSMiddleware -from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, \ - RedirectResponse - -# add relative path to use packages as they were in the app/ dir -sys.path.append('../') -sys.path.append('../app') +from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse from orm import Origin, Lease, init as db_init, migrate, Instance, Site @@ -403,7 +398,7 @@ async def auth_v1_token(request: Request): jwt_encode_key, jwt_decode_key = default_instance.get_jwt_encode_key(), default_instance.get_jwt_decode_key() try: - payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key, algorithms=[ALGORITHMS.RS256]) + payload = jwt.decode(token=j.get('auth_code'), key=jwt_decode_key, algorithms=ALGORITHMS.RS256) except JWTError as e: return JSONr(status_code=400, content={'status': 400, 'title': 'invalid token', 'detail': str(e)}) diff --git a/test/main.py b/test/main.py index 6442148..a2adc51 100644 --- a/test/main.py +++ b/test/main.py @@ -1,16 +1,16 @@ -from os import getenv as env +import sys from base64 import b64encode as b64enc -from hashlib import sha256 from calendar import timegm from datetime import datetime, UTC -from uuid import UUID, uuid4 +from hashlib import sha256 +from os import getenv as env +from uuid import uuid4, UUID from dateutil.relativedelta import relativedelta from jose import jwt from jose.constants import ALGORITHMS -from starlette.testclient import TestClient from sqlalchemy import create_engine -import sys +from starlette.testclient import TestClient # add relative path to use packages as they were in the app/ dir sys.path.append('../') From a12d05281c2a47e6deb914ad0d0b017b2470e548 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 8 Apr 2025 14:30:59 +0200 Subject: [PATCH 68/69] updated default data --- app/orm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/orm.py b/app/orm.py index 6c66b30..bb2e855 100644 --- a/app/orm.py +++ b/app/orm.py @@ -23,8 +23,8 @@ Base = declarative_base() class Site(Base): __tablename__ = "site" - INITIAL_SITE_KEY_XID = '00000000-0000-0000-0000-000000000000' - INITIAL_SITE_NAME = 'default' + INITIAL_SITE_KEY_XID = '10000000-0000-0000-0000-000000000000' + INITIAL_SITE_NAME = 'default-site' site_key = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4, SITE_KEY_XID name = Column(VARCHAR(length=256), nullable=False) From 5209ece1cd7db14bb8a94b9fe5e27d098e95f5fb Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 16 Apr 2025 14:56:26 +0200 Subject: [PATCH 69/69] merge fixes --- app/main.py | 2 +- app/orm.py | 4 ++-- app/util.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/main.py b/app/main.py index 9f30ba0..09ec4d1 100644 --- a/app/main.py +++ b/app/main.py @@ -49,7 +49,7 @@ logging.basicConfig(format='[{levelname:^7}] [{module:^15}] {message}', style='{ logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL) logging.getLogger('util').setLevel(LOG_LEVEL) -logging.getLogger('NV').setLevel(LOG_LEVEL) +logging.getLogger('DriverMatrix').setLevel(LOG_LEVEL) # FastAPI diff --git a/app/orm.py b/app/orm.py index bb2e855..0f50fb0 100644 --- a/app/orm.py +++ b/app/orm.py @@ -11,7 +11,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker, declarative_base, Session, relationship from sqlalchemy.schema import CreateTable -from util import NV, PrivateKey, PublicKey +from util import DriverMatrix, PrivateKey, PublicKey, DriverMatrix logging.basicConfig() logger = logging.getLogger(__name__) @@ -148,7 +148,7 @@ class Origin(Base): return f'Origin(origin_ref={self.origin_ref}, hostname={self.hostname})' def serialize(self) -> dict: - _ = NV().find(self.guest_driver_version) + _ = DriverMatrix().find(self.guest_driver_version) return { 'origin_ref': self.origin_ref, diff --git a/app/util.py b/app/util.py index c6c93d0..7d4694d 100644 --- a/app/util.py +++ b/app/util.py @@ -104,7 +104,7 @@ class DriverMatrix: self.log.debug(f'Successfully loaded "{DriverMatrix.__DRIVER_MATRIX_FILENAME}".') except Exception as e: DriverMatrix.__DRIVER_MATRIX = {} # init empty dict to not try open file everytime, just when restarting app - # self.log.warning(f'Failed to load "{NV.__DRIVER_MATRIX_FILENAME}": {e}') + # self.log.warning(f'Failed to load "{DriverMatrix.__DRIVER_MATRIX_FILENAME}": {e}') @staticmethod def find(version: str) -> dict | None: