Skip to content

Improve admin webui #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Feb 25, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[flake8]
max-line-length = 88
extend-ignore = E203
per-file-ignores = __init__.py:F401 spkrepo/app.py:F841
per-file-ignores = __init__.py:F401
exclude =
docs/*
migrations/*
2 changes: 1 addition & 1 deletion migrations/alembic.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[alembic]
script_location = .
script_location = ./migrations

[loggers]
keys = root,sqlalchemy,alembic
1 change: 0 additions & 1 deletion migrations/env.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,6 @@
config = context.config
fileConfig(config.config_file_name)


config.set_main_option(
"sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
)
4 changes: 0 additions & 4 deletions migrations/versions/dc7687894ba7_increase_field_sizes.py
Original file line number Diff line number Diff line change
@@ -13,7 +13,6 @@


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"version",
"conf_dependencies",
@@ -42,11 +41,9 @@ def upgrade():
type_=sa.UnicodeText(),
existing_nullable=True,
)
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"version",
"conf_resource",
@@ -75,4 +72,3 @@ def downgrade():
type_=sa.VARCHAR(length=255),
existing_nullable=True,
)
# ### end Alembic commands ###
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""Add firmware type and increase version length
Revision ID: f95855ce9471
Revises: 76d559b4e873
Create Date: 2024-01-15 13:58:34.160242
"""
revision = "f95855ce9471"
down_revision = "76d559b4e873"

import sqlalchemy as sa
from alembic import op


def upgrade():
op.add_column("firmware", sa.Column("type", sa.Unicode(length=4)))
# Set type based on version
op.execute(
"""
UPDATE firmware
SET type = CASE
WHEN version LIKE '1.%' THEN 'srm'
ELSE 'dsm'
END
"""
)
# Modify the column to be NOT NULL after setting the values
op.alter_column("firmware", "type", nullable=False)

op.alter_column(
"firmware",
"version",
existing_type=sa.VARCHAR(length=3),
type_=sa.Unicode(length=4),
existing_nullable=False,
)


def downgrade():
op.alter_column(
"firmware",
"version",
existing_type=sa.Unicode(length=4),
type_=sa.VARCHAR(length=3),
existing_nullable=False,
)
op.drop_column("firmware", "type")
11 changes: 6 additions & 5 deletions spkrepo/app.py
Original file line number Diff line number Diff line change
@@ -2,12 +2,11 @@
import jinja2
from flask import Flask
from flask_admin import Admin
from flask_babel import Babel
from wtforms import HiddenField

from . import config as default_config
from .cli import spkrepo as spkrepo_cli
from .ext import cache, db, debug_toolbar, mail, migrate, security
from .ext import babel, cache, db, debug_toolbar, mail, migrate, security
from .models import user_datastore
from .views import (
ArchitectureView,
@@ -16,6 +15,7 @@
IndexView,
PackageView,
ScreenshotView,
ServiceView,
SpkrepoConfirmRegisterForm,
UserView,
VersionView,
@@ -53,18 +53,19 @@ def create_app(config=None, register_blueprints=True, init_admin=True):
admin.add_view(UserView())
admin.add_view(ArchitectureView())
admin.add_view(FirmwareView())
admin.add_view(ServiceView())
admin.add_view(ScreenshotView())
admin.add_view(PackageView())
admin.add_view(VersionView())
admin.add_view(BuildView())
admin.init_app(app)

# Initialize Flask-Babel
babel = Babel(app)

# Commands
app.cli.add_command(spkrepo_cli)

# Flask-Babel
babel.init_app(app)

# SQLAlchemy
db.init_app(app)

9 changes: 8 additions & 1 deletion spkrepo/cli.py
Original file line number Diff line number Diff line change
@@ -180,8 +180,15 @@ def depopulate_db():
from spkrepo.models import Package

for package in Package.query.all():
shutil.rmtree(os.path.join(current_app.config["DATA_PATH"], package.name))
# Delete the package and its associated versions and builds
db.session.delete(package)

# Remove the directory associated with the package (if it exists)
shutil.rmtree(
os.path.join(current_app.config["DATA_PATH"], package.name),
ignore_errors=True,
)

db.session.commit()


4 changes: 4 additions & 0 deletions spkrepo/ext.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
# -*- coding: utf-8 -*-
from flask_babel import Babel
from flask_caching import Cache
from flask_debugtoolbar import DebugToolbarExtension
from flask_mail import Mail
from flask_migrate import Migrate
from flask_security import Security
from flask_sqlalchemy import SQLAlchemy

# Flask-Babel
babel = Babel()

# Cache
cache = Cache()

30 changes: 27 additions & 3 deletions spkrepo/models.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
import hashlib
import io
import os
import shutil
@@ -133,8 +134,9 @@ class Firmware(db.Model):

# Columns
id = db.Column(db.Integer, primary_key=True)
version = db.Column(db.Unicode(3), nullable=False)
version = db.Column(db.Unicode(4), nullable=False)
build = db.Column(db.Integer, unique=True, nullable=False)
type = db.Column(db.Unicode(4), nullable=False)

@classmethod
def find(cls, build):
@@ -328,7 +330,7 @@ class Build(db.Model):
publisher_user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
checksum = db.Column(db.Unicode(32))
extract_size = db.Column(db.Integer)
path = db.Column(db.Unicode(100))
path = db.Column(db.Unicode(2048))
md5 = db.Column(db.Unicode(32))
insert_date = db.Column(db.DateTime, default=db.func.now(), nullable=False)
active = db.Column(db.Boolean(), default=False, nullable=False)
@@ -343,7 +345,11 @@ class Build(db.Model):
)
firmware = db.relationship("Firmware", lazy=False)
publisher = db.relationship("User", foreign_keys=[publisher_user_id])
downloads = db.relationship("Download", back_populates="build")
downloads = db.relationship(
"Download",
back_populates="build",
cascade="save-update, merge, delete, delete-orphan",
)

@classmethod
def generate_filename(cls, package, version, firmware, architectures):
@@ -360,6 +366,24 @@ def save(self, stream):
) as f:
f.write(stream.read())

def calculate_md5(self):
if not self.path:
raise ValueError("Path cannot be empty.")

file_path = os.path.join(current_app.config["DATA_PATH"], self.path)

if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found at path: {file_path}")

if self.md5 is None:
with io.open(file_path, "rb") as f:
md5_hash = hashlib.md5()
for chunk in iter(lambda: f.read(4096), b""):
md5_hash.update(chunk)
return md5_hash.hexdigest()

return self.md5

def _after_insert(self):
assert os.path.exists(os.path.join(current_app.config["DATA_PATH"], self.path))

3 changes: 1 addition & 2 deletions spkrepo/tests/common.py
Original file line number Diff line number Diff line change
@@ -281,8 +281,7 @@ def create_spk(self, create, extracted, **kwargs):
with create_spk(self) as spk_stream:
self.save(spk_stream)
if self.md5 is None:
spk_stream.seek(0)
self.md5 = hashlib.md5(spk_stream.read()).hexdigest()
self.md5 = self.calculate_md5()
spk_stream.close()

@classmethod
21 changes: 19 additions & 2 deletions spkrepo/tests/test_nas.py
Original file line number Diff line number Diff line change
@@ -227,11 +227,11 @@ def test_stable_build_active_stable(self):
catalog[0], build, data, dict(arch="88f628x", build="1594")
)

def test_stable_build_active_stable_5004(self):
def test_stable_noarch_build_active_stable_5004(self):
build = BuildFactory(
active=True,
version__report_url=None,
architectures=[Architecture.find("88f6281", syno=True)],
architectures=[Architecture.find("noarch", syno=True)],
firmware=Firmware.find(1594),
)
db.session.commit()
@@ -247,6 +247,23 @@ def test_stable_build_active_stable_5004(self):
catalog["packages"][0], build, data, dict(arch="88f628x", build="5004")
)

def test_stable_arch_build_active_stable_5004(self):
BuildFactory(
active=True,
version__report_url=None,
architectures=[Architecture.find("88f6281", syno=True)],
firmware=Firmware.find(1594),
)
db.session.commit()
data = dict(arch="88f6281", build="5004", language="enu")
response = self.client.post(url_for("nas.catalog"), data=data)
self.assert200(response)
self.assertHeader(response, "Content-Type", "application/json")
catalog = json.loads(response.data.decode())
self.assertIn("packages", catalog)
self.assertIn("keyrings", catalog)
self.assertEqual(len(catalog["packages"]), 0)

def test_stable_build_active_stable_download_count(self):
package = PackageFactory()
build = BuildFactory(
21 changes: 20 additions & 1 deletion spkrepo/utils.py
Original file line number Diff line number Diff line change
@@ -58,6 +58,10 @@ class SPK(object):
#: Regex for files in conf
conf_filename_re = re.compile(r"^conf/.+$")

#: Regex for firmware input
firmware_version_re = re.compile(r"^\d+\.\d$")
firmware_type_re = re.compile(r"^([a-z]){3,}$")

def __init__(self, stream):
self.info = {}
self.icons = {}
@@ -345,6 +349,18 @@ def unsign(self):
self.stream.truncate()
self.stream.seek(0)

def calculate_md5(self):
md5_hash = hashlib.md5()

# Ensure the stream position is at the beginning
self.stream.seek(0)

# Update MD5 hash directly from the stream
for chunk in iter(lambda: self.stream.read(4096), b""):
md5_hash.update(chunk)

return md5_hash.hexdigest()

def _generate_signature(self, stream, timestamp_url, gnupghome): # pragma: no cover
# generate the signature
gpg = gnupg.GPG(gnupghome=gnupghome)
@@ -386,7 +402,10 @@ def populate_db():
)
db.session.execute(
Firmware.__table__.insert().values(
[{"version": "3.1", "build": 1594}, {"version": "5.0", "build": 4458}]
[
{"version": "3.1", "build": 1594, "type": "dsm"},
{"version": "5.0", "build": 4458, "type": "dsm"},
]
)
)
db.session.execute(
1 change: 1 addition & 0 deletions spkrepo/views/__init__.py
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
IndexView,
PackageView,
ScreenshotView,
ServiceView,
UserView,
VersionView,
)
116 changes: 98 additions & 18 deletions spkrepo/views/admin.py
Original file line number Diff line number Diff line change
@@ -16,7 +16,16 @@
from wtforms.validators import Regexp

from ..ext import db
from ..models import Architecture, Build, Firmware, Package, Screenshot, User, Version
from ..models import (
Architecture,
Build,
Firmware,
Package,
Screenshot,
Service,
User,
Version,
)
from ..utils import SPK


@@ -35,6 +44,12 @@ def is_accessible(self):
# View
column_list = ("username", "email", "roles", "active", "confirmed_at")

column_formatters = {
"confirmed_at": lambda v, c, m, p: (
m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") if m.confirmed_at else None
)
}

# Form
form_columns = ("username", "roles", "active")
form_overrides = {"password": PasswordField}
@@ -91,6 +106,9 @@ def is_accessible(self):

can_delete = False

# Form
form_excluded_columns = "builds"


class FirmwareView(ModelView):
"""View for :class:`~spkrepo.models.Firmware`"""
@@ -106,6 +124,28 @@ def is_accessible(self):

can_delete = False

# Form
form_columns = ("version", "build", "type")
form_args = {
"version": {"validators": [Regexp(SPK.firmware_version_re)]},
"type": {"validators": [Regexp(SPK.firmware_type_re)]},
}


class ServiceView(ModelView):
"""View for :class:`~spkrepo.models.Service`"""

def __init__(self, **kwargs):
super(ServiceView, self).__init__(Service, db.session, **kwargs)

# Permissions
def is_accessible(self):
return current_user.is_authenticated and current_user.has_role("package_admin")

can_edit = False

can_delete = False


def screenshot_namegen(obj, file_data):
pattern = "screenshot_%0d%s"
@@ -120,18 +160,6 @@ def screenshot_namegen(obj, file_data):
return os.path.join(obj.package.name, pattern % (i, ext))


# TODO: Not necessary with Flask-Admin>1.0.8
# see https://github.com/mrjoes/flask-admin/pull/705
class SpkrepoImageUploadField(ImageUploadField):
def _get_path(self, filename):
if not self.base_path: # pragma: no cover
raise ValueError("FileUploadField field requires base_path to be set.")

if callable(self.base_path):
return os.path.join(self.base_path(), filename)
return os.path.join(self.base_path, filename) # pragma: no cover


class ScreenshotView(ModelView):
"""View for :class:`~spkrepo.models.Screenshot`"""

@@ -142,7 +170,14 @@ def __init__(self, **kwargs):
def is_accessible(self):
return current_user.is_authenticated and current_user.has_role("package_admin")

can_edit = False

# View
column_labels = {
"package.name": "Package Name",
"path": "Screenshot",
}

def _display(view, context, model, name):
return Markup(
'<img src="%s" alt="screenshot" height="100" width="100">'
@@ -151,11 +186,17 @@ def _display(view, context, model, name):

column_formatters = {"path": _display}
column_sortable_list = (("package", "package.name"),)
column_default_sort = (Package.name, True)
column_default_sort = "package.name"
column_filters = ("package.name",)

# Hooks
def on_model_delete(self, model):
build_path = os.path.join(current_app.config["DATA_PATH"], model.path)
if os.path.exists(build_path):
os.remove(build_path)

# Form
form_overrides = {"path": SpkrepoImageUploadField}
form_overrides = {"path": ImageUploadField}
form_args = {
"path": {
"label": "Screenshot",
@@ -221,6 +262,10 @@ def on_model_delete(self, model):
("insert_date", "insert_date"),
)

column_formatters = {
"insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
}

# Form
form_columns = ("name", "author", "maintainers")
form_args = {"name": {"validators": [Regexp(SPK.package_re)]}}
@@ -276,21 +321,33 @@ def on_model_delete(self, model):
"startable",
)
column_labels = {
"package.name": "Package Name",
"version_string": "Version",
"dependencies": "Dependencies",
"service_dependencies": "Services",
}
column_filters = ("package.name", "version", "upstream_version")
column_filters = (
"package.name",
"upstream_version",
"version",
"beta",
"all_builds_active",
)
column_sortable_list = (
("package", "package.name"),
("upstream_version", "upstream_version"),
("version", "version"),
("beta", "beta"),
("insert_date", "insert_date"),
("all_builds_active", "all_builds_active"),
("install_wizard", "install_wizard"),
("upgrade_wizard", "upgrade_wizard"),
("startable", "startable"),
)
# TODO: Add beta and all_builds_active with Flask-Admin>1.0.8

column_formatters = {
"insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
}
column_default_sort = (Version.insert_date, True)

# Custom queries
@@ -383,8 +440,11 @@ def action_sign(self, ids):
current_app.config["GNUPG_TIMESTAMP_URL"],
current_app.config["GNUPG_PATH"],
)
build.md5 = spk.calculate_md5()
self.session.commit()
success.append(filename)
except Exception:
self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -434,8 +494,11 @@ def action_unsign(self, ids):
continue
try:
spk.unsign()
build.md5 = spk.calculate_md5()
self.session.commit()
success.append(filename)
except Exception:
self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -516,24 +579,35 @@ def can_unsign(self):
)
column_labels = {
"version.package": "Package",
"version.package.name": "Package Name",
"version.upstream_version": "Upstream Version",
"version.version": "Version",
"architectures.code": "Architecture",
"firmware.version": "Firmware Version",
"publisher.username": "Publisher Username",
}
column_filters = (
"version.package.name",
"version.upstream_version",
"version.version",
"architectures.code",
"firmware.version",
"publisher.username",
"active",
)
column_sortable_list = (
("version.package", "version.package.name"),
("version.upstream_version", "version.upstream_version"),
("version.version", "version.version"),
("firmware", "firmware.build"),
("publisher", "publisher.username"),
("insert_date", "insert_date"),
("active", "active"),
)
# TODO: Add version.package with Flask-Admin>1.0.8

column_formatters = {
"insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
}
column_default_sort = (Build.insert_date, True)

# Custom queries
@@ -627,8 +701,11 @@ def action_sign(self, ids):
current_app.config["GNUPG_TIMESTAMP_URL"],
current_app.config["GNUPG_PATH"],
)
build.md5 = spk.calculate_md5()
self.session.commit()
success.append(filename)
except Exception:
self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -677,8 +754,11 @@ def action_unsign(self, ids):
continue
try:
spk.unsign()
build.md5 = spk.calculate_md5()
self.session.commit()
success.append(filename)
except Exception:
self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
12 changes: 12 additions & 0 deletions spkrepo/views/api.py
Original file line number Diff line number Diff line change
@@ -130,6 +130,16 @@ def post(self):
if firmware is None:
abort(422, message="Unknown firmware")

# Services
input_install_dep_services = spk.info.get("install_dep_services", None)
if input_install_dep_services:
for info_dep_service in input_install_dep_services.split():
service_name = Service.find(info_dep_service)
if service_name is None:
abort(
422, message="Unknown dependent service: %s" % info_dep_service
)

# Package
create_package = False
package = Package.find(spk.info["package"])
@@ -268,6 +278,8 @@ def post(self):
for size, icon in build.version.icons.items():
icon.save(spk.icons[size])
build.save(spk.stream)
# generate md5 hash
build.md5 = build.calculate_md5()
except Exception as e: # pragma: no cover
if create_package:
shutil.rmtree(os.path.join(data_path, package.name), ignore_errors=True)
31 changes: 30 additions & 1 deletion spkrepo/views/nas.py
Original file line number Diff line number Diff line change
@@ -44,19 +44,48 @@ def is_valid_language(language):

@cache.memoize(timeout=600)
def get_catalog(arch, build, language, beta):
# latest version per package
# Find the closest matching firmware for the provided build
closest_firmware = (
Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm")
.order_by(Firmware.build.desc())
.first()
)

# Extract major version from the closest matching firmware
major_version = (
int(closest_firmware.version.split(".")[0])
if closest_firmware and closest_firmware.version
else None
)

# latest version per package and major version
latest_version = db.session.query(
Version.package_id, db.func.max(Version.version).label("latest_version")
).select_from(Version)

if not beta:
latest_version = latest_version.filter(Version.report_url.is_(None))

latest_version = (
latest_version.join(Build)
.filter(Build.active)
.join(Build.architectures)
.filter(Architecture.code.in_(["noarch", arch]))
.join(Build.firmware)
.filter(Firmware.build <= build)
.filter(
db.or_(
# Check if major_version is not None before applying the filter
(major_version is not None)
and Firmware.version.startswith(f"{major_version}."),
# Include earlier "noarch" version when major_version < 6
db.and_(
Architecture.code == "noarch",
(major_version is not None) and (major_version < 6),
Firmware.version.startswith("3."),
),
)
)
.group_by(Version.package_id)
).subquery()