quality/testdays-web
5
0
Fork 0
Code Issues 16 Pull requests Projects Releases Packages Wiki Activity Actions

Update local dev environment to use PostgreSQL db, add developer docs

Browse source
This commit is contained in:
Jaroslav Groman 2025-11-28 15:42:03 +01:00
commit 9a8fb8cb37
15 changed files with 295 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

9
.gitignore vendored
View file

@ -3,14 +3,17 @@
*.swp
*__pycache__*
.ropeproject/
conf/settings.py
*.sqlite
*.sass-cache
*build/
*dist/
*.egg*
/env
*.tar.gz
*.rpm
.python-version
.venv
.vscode
conf/client_secrets.json
/env
conf/client_secrets.json
conf/settings.py

94
Makefile Normal file
View file

@ -0,0 +1,94 @@
# testdays-web helper Makefile
SHELL := /bin/bash
APP_NAME := testdays-web
DB_DRIVER := postgresql+psycopg2
DB_DATABASE := testdays
DB_USER := tester
DB_PASSWORD := fedora
DB_HOST := localhost
DB_PORT := 5432
SETTINGS_TEMPLATE := ./conf/settings.py.example
SETTINGS_FILE := ./conf/settings.py
# Check if running inside Toolbox
IS_TOOLBOX := $(shell test -f /run/.toolboxenv && echo true || echo false)
ifeq ($(IS_TOOLBOX), true)
PODMAN_CMD := flatpak-spawn --host podman
else
PODMAN_CMD := podman
endif
.PHONY: all help create_db start_db stop_db remove_db setup_db create_mock_data config_app run_app cli
all: help
# HELP
# This will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "%-25s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
# Check if DB container is already running
IS_DB_RUNNING := $(shell $(PODMAN_CMD) ps -q -f name=$(DB_DATABASE)db | grep -q . && echo true || echo false)
create_db: ## Create and start PostgreSQL DB container
@echo 'Creating PostgreSQL container for DB_URI: $(DB_DRIVER)://$(DB_USER):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_DATABASE)'
ifneq ($(IS_DB_RUNNING), true)
@$(PODMAN_CMD) run --detach --name $(DB_DATABASE)db \
--env POSTGRESQL_USER=$(DB_USER) \
--env POSTGRESQL_PASSWORD=$(DB_PASSWORD) \
--env POSTGRESQL_DATABASE=$(DB_DATABASE) \
--publish $(DB_PORT):5432 \
quay.io/fedora/postgresql-16:latest
@echo "Wait for DB to finish initializing..."
@sleep 10
@python run_cli.py init_db
else
@echo "PostgreSQL DB Container '$(DB_DATABASE)db' is running"
endif
start_db: create_db ## Start PostgreSQL DB container
@echo "Starting PostgreSQL DB Container '$(DB_DATABASE)db'"
@$(PODMAN_CMD) start $(DB_DATABASE)db
stop_db: ## Stop PostgreSQL DB container
@echo "Stopping PostgreSQL DB Container '$(DB_DATABASE)db'"
@$(PODMAN_CMD) stop $(DB_DATABASE)db
remove_db: stop_db ## Stop and remove PostgreSQL DB container
@echo "Removing PostgreSQL DB Container '$(DB_DATABASE)db'"
@$(PODMAN_CMD) container rm $(DB_DATABASE)db
setup_db: start_db ## Initialize DB tables
@python run_cli.py init_db
create_mock_data: setup_db ## Preload mock DB users and data
@python run_cli.py mock_db_users -d
@python run_cli.py mock_data -d
config_app: ## Create app settings file
ifeq ("$(wildcard $(SETTINGS_FILE))", "")
@cp $(SETTINGS_TEMPLATE) $(SETTINGS_FILE)
@sed -i 's/PRODUCTION = True/PRODUCTION = False/g' $(SETTINGS_FILE)
@sed -i 's/OIDC_ENABLED = True/OIDC_ENABLED = False/g' $(SETTINGS_FILE)
@echo "Please review app settings in '$(SETTINGS_FILE)' and update as necessary"
else
@echo "App configuration: '$(SETTINGS_FILE)'"
endif
run_app: config_app start_db ## Start the testdays-web app web server in local development mode
@echo "Starting $(APP_NAME) by 'python runapp.py'"
@RUNMODE=dev python runapp.py
CLI_ARGS = $(filter-out $@,$(MAKECMDGOALS))
cli: ## Execute app CLI runner. Use `make cli -- --help` for help.
@echo "Note: Separate arguments from cli target by '--' like this: 'make cli -- --help'."
@echo "Executing CLI: 'python run_cli.py $(CLI_ARGS)'"
@python run_cli.py $(CLI_ARGS)
%: # match all unprocessed targets
@: # do-nothing

6
README.md
View file

@ -1,3 +1,5 @@
# testdays-web
## TestdayApp Metadata
The new TestdayApp's metadata define the testdays' structure, testcases and possibly the `Profile` column name override.
@ -49,3 +51,7 @@ For example:
Here, we moved the `pm-suspendr` testcase identified by ULID `01JR2VGYFAWA5F7EHCZ3C6FXK2` from the `Basic Tests` section to the `Other Tests section`, changed the name to `Another name` and the url to `https://another.url`, but since we kept the ULID reference intact, alle the results previously entered into the `pm-suspendr` column will be correctly shown in the new location.
## App Development Notes
Please see developer notes in [docs/DEVELOPER.md](docs/DEVELOPER.md).

13
alembic/versions/36c87c86c7b1_add_creator_role.py
View file

@ -16,7 +16,18 @@ from alembic import op
import sqlalchemy as sa
def upgrade():
op.execute("ALTER TYPE userroles ADD VALUE 'creator'")
bind = op.get_bind()
if bind.dialect.name == 'postgresql':
# PostgreSQL: Use native ALTER TYPE
op.execute("ALTER TYPE userroles ADD VALUE 'creator'")
else:
# SQLite and others: Use batch mode to recreate the column
with op.batch_alter_table('users', schema=None) as batch_op:
batch_op.alter_column('role',
existing_type=sa.Enum('none', 'user', 'admin', name='userroles'),
type_=sa.Enum('none', 'user', 'creator', 'admin', name='userroles'),
existing_nullable=True)
def downgrade():
pass

18
alembic/versions/379f1fcae26d_add_column_username_to_user.py
View file

@ -17,12 +17,20 @@ import sqlalchemy as sa
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
bind = op.get_bind()
op.add_column('users', sa.Column('username', sa.String(length=80), nullable=True))
conn = op.get_bind()
conn.execute(sa.text("UPDATE users SET username=displayname;"));
op.alter_column('users', 'username', nullable=False)
# ### end Alembic commands ###
bind.execute(sa.text("UPDATE users SET username=displayname"))
if bind.dialect.name == 'postgresql':
# PostgreSQL: Direct alter column works
op.alter_column('users', 'username', nullable=False)
else:
# SQLite: Use batch mode to alter column nullable constraint
with op.batch_alter_table('users', schema=None) as batch_op:
batch_op.alter_column('username',
existing_type=sa.String(length=80),
nullable=False)
def downgrade():

26
alembic/versions/460871bfa69e_rename_username_to_sub.py
View file

@ -17,11 +17,27 @@ import sqlalchemy as sa
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('users', 'username', new_column_name='sub')
op.drop_constraint('users_username_key', 'users', type_='unique')
op.create_unique_constraint(None, 'users', ['sub'])
# ### end Alembic commands ###
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = [col['name'] for col in inspector.get_columns('users')]
# Check if migration already applied
if 'sub' in columns and 'username' not in columns:
return # Migration already applied
if bind.dialect.name == 'postgresql':
# PostgreSQL: Direct operations work fine
op.alter_column('users', 'username', new_column_name='sub')
op.drop_constraint('users_username_key', 'users', type_='unique')
op.create_unique_constraint(None, 'users', ['sub'])
else:
# SQLite: Use batch mode for column renaming and constraint handling
with op.batch_alter_table('users', schema=None) as batch_op:
batch_op.alter_column('username',
existing_type=sa.String(length=265),
new_column_name='sub',
nullable=False)
# Batch mode automatically handles constraint recreation
def downgrade():

10
alembic/versions/615057ee3945_revamp_testdays_step_1.py
View file

@ -17,8 +17,14 @@ import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
#op.drop_index('category_fk_event_id', table_name='category')
op.drop_constraint('testcase_category_id_fkey', 'testcase', type_='foreignkey')
bind = op.get_bind()
# Only PostgreSQL requires explicit constraint dropping before table drop
# SQLite drops all constraints automatically when dropping the table
if bind.dialect.name == 'postgresql':
op.drop_constraint('testcase_category_id_fkey', 'testcase', type_='foreignkey')
# These work on both PostgreSQL and SQLite
op.drop_table('category')
op.drop_table('user')
op.drop_table('event')

8
conf/settings.py.example
View file

@ -9,3 +9,11 @@ FILE_LOGGING = False
SYSLOG_LOGGING = False
STREAM_LOGGING = True
LOGFILE = '/var/log/testdays/testdays.log'
OIDC_ENABLED = True
OIDC_TESTING_PROFILE = {
"sub": "AABBCCDD",
"nickname": "tester",
"email": "tester@example.com",
"groups": ["none"],
}

96
docs/DEVELOPER.md Normal file
View file

@ -0,0 +1,96 @@
# `testdays-web` app Developer Notes
## Development `Makefile`
There is a `Makefile` provided with common tasks related to application development and operation.
Run `make help` to get a list of supported tasks.
### Create initial DB and start local app server using `Makefile` tasks
```
make create_db
make create_mock_data
make run_app
```
## Customized setup
### Setting up PostgreSQL database
#### DB Container Start
The most convenient local dev database setup is probably using containers. The command below will download the latest
PostgreSQL 16 image and start a container named `testdaysdb`:
```
podman run --detach --name testdaysdb \
--env POSTGRESQL_USER=tester \
--env POSTGRESQL_PASSWORD=fedora \
--env POSTGRESQL_DATABASE=testdays \
--publish 5432:5432 \
quay.io/fedora/postgresql-16:latest
```
You can find more detailes regarding this image here: <https://quay.io/repository/fedora/postgresql-13>
Please note that the database set up this way will not be persistent as db data will be stored inside
container only and will be lost when the container is destroyed. On the other hand this configuration
will allow you to use `podman commit` to back up current state and experiment with database schema
and contents. Db data will also stay intact during simple container restarts.
If you wish for database contents to persist between container complete teardown and re-creation, please
add external db data volume mount like this:
```
podman run --detach --name testdaysdb \
--env POSTGRESQL_USER=tester \
--env POSTGRESQL_PASSWORD=fedora \
--env POSTGRESQL_DATABASE=testdays \
--volume /your/data/dir:/var/lib/pgsql/data:Z \
--publish 5432:5432 \
quay.io/fedora/postgresql-16:latest
```
#### DB Container Operation
```
# Start container
podman start testdaysdb
# Stop container
podman stop testdaysdb
# Check PostgreSQL version
podman exec -it testdaysdb psql --host=localhost --username=postgres --command='select version();'
```
### Python virtualenv
#### Create virtualenv
Create a new virtualenv in the base of the source tree:
```
python -m venv env_testdays
```
In order to install python packages into the virtualenv or use the packages
inside the virtualenv, it must be activated:
```
source env_testdays/bin/activate
```
#### Install project dependencies into virtualenv
While the virtualenv is active, install the project dependencies:
```
pip install --upgrade pip
pip install -r requirements.txt
```
### App settings file
1. Copy `conf/settings.py.example` to `conf/settings.py`
2. Edit `conf/settings.py` as necessary
3. If using OICD you have to also supply your own `client_secrets.json` and store it info conf directory.

0
init_db.sh Normal file → Executable file
View file

2
requirements.txt
View file

@ -7,7 +7,7 @@ python-fedora
alembic==1.13.1
Flask==3.0.3
Flask-Login==0.6.3
flask-oidc==2.1.1
flask-oidc==2.4.0
flask-simple-captcha==5.5.5
Flask-SQLAlchemy==3.1.1
Flask-WTF==1.2.1

2
testdays/__init__.py
View file

@ -37,7 +37,7 @@ elif os.getenv("RUNMODE") == "test" or os.getenv("TEST") == "true":
config_file = None
config_file = os.environ.get("FORCE_CONFIG_FILE", config_file)
if os.path.exists(config_file):
if config_file and os.path.exists(config_file):
config_obj.update_from_pyfile(config_file)

33
testdays/cli.py
View file

@ -2,6 +2,7 @@ import json
import os
import random
import sys
from datetime import datetime, timedelta
from optparse import OptionParser
from alembic import command as al_command
@ -86,9 +87,9 @@ def CMD_mock_data(destructive, **kwargs):
print("Must be ran as destructive")
return
print("Mocking data")
Result.query.delete()
Testday.query.delete()
Testcase.query.delete()
Result.query.delete()
structure = {
"Sections": [
{
@ -163,6 +164,8 @@ def CMD_mock_data(destructive, **kwargs):
}
td = Testday("Mock Testday #1", structure)
td.is_draft = False
td.start = datetime.datetime.now()
td.end = datetime.datetime.now() + timedelta(days=7)
db.session.add(td)
tcs = {}
for s in structure["Sections"]:
@ -177,8 +180,11 @@ def CMD_mock_data(destructive, **kwargs):
db.session.commit()
for i in range(3):
u = User(f"anon:Rando_{i}", f"Rando {i}")
db.session.add(u)
sub = f"anon:Rando_{i}"
u = User.query.filter(User.sub == sub).first()
if not u:
u = User(sub, f"Rando {i}")
db.session.add(u)
for tc in random.sample(list(tcs.keys()), 3):
r = Result(
td.id,
@ -196,11 +202,17 @@ def CMD_mock_db_users(destructive, **kwargs):
if destructive or not db.session.query(User).count():
print(" - User")
data_users = [
("admin", "admin", UserRoles.admin),
("user", "user", UserRoles.none),
("fedora:admin", "admin", "admin", UserRoles.admin),
("fedora:user", "user", "user", UserRoles.none),
]
for d in data_users:
sub = d[0]
existing_user = db.session.query(User).filter(User.sub == sub).first()
if existing_user:
print(f" - User '{sub}' already exists, skipping")
continue
u = User(*d)
db.session.add(u)
@ -211,15 +223,18 @@ def CMD_mock_db_users(destructive, **kwargs):
def CMD_set_user_role(value, **kwargs):
"""Accepts sub:rolename as value"""
sub, role = value.rsplit(":", 1)
if value is None:
print("Please add '-v user_sub:rolename' parameter.")
return
sub, role_name = value.rsplit(":", 1)
user = User.query.filter(User.sub == sub).first()
if not user:
print("User not found")
print(f"User '{sub}' not found")
return
role = UserRoles.get_by_name(role)
role = UserRoles.get_by_name(role_name)
if not role:
print("Role not found")
print(f"Role '{role_name}' not found")
return
user.role = role

2
testdays/config.py
View file

@ -102,7 +102,7 @@ class Development(Config):
TRAP_BAD_REQUEST_ERRORS = True
DB_URI = "sqlite:////tmp/testdays_db.sqlite"
DB_URI = "postgresql+psycopg2://tester:fedora@localhost:5432/testdays"
RUN_HOST = "0.0.0.0"
RUN_PORT = 5050

5
testdays/models/testday.py
View file

@ -56,7 +56,10 @@ class Testday(db.Model):
@structure.setter
def structure(self, value):
value["Timestamp"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
value["Author"] = {"id": current_user.id, "username": current_user.sub}
if current_user and current_user.id:
value["Author"] = {"id": current_user.id, "username": current_user.sub}
else:
value["Author"] = {"id": None, "username": "system"}
if self._history_col is None:
self._history_col = "[]"