diff --git a/cookiecutter.json b/cookiecutter.json index bbc74e70..6a0457e3 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -6,12 +6,6 @@ "author_name": "CERN", "author_email": "info@{{ cookiecutter.project_site }}", "year": "{% now 'local', '%Y' %}", - "database": [ - "postgresql" - ], - "search": [ - "opensearch2" - ], "file_storage": [ "local", "S3" @@ -24,8 +18,11 @@ "yes", "no" ], + "base_image": [ + "debian" + ], "use_reduced_vocabs": [ - "no", - "yes" + "yes", + "no" ] } diff --git a/hooks/post_gen_project.sh b/hooks/post_gen_project.sh index aaa2d43a..0ce3dae0 100644 --- a/hooks/post_gen_project.sh +++ b/hooks/post_gen_project.sh @@ -19,7 +19,7 @@ touch data/default/.gitkeep {%- if cookiecutter.site_code == 'no'%} DIR="site" -if [ -d "$DIR" ]; then +if [ -d "${DIR}" ]; then rm -r site fi {% endif %} diff --git a/run-tests.sh b/run-tests.sh index 69509891..93965a2f 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2019 CERN. # SPDX-FileCopyrightText: 2019 Northwestern University. -# SPDX-FileCopyrightText: 2024 KTH Royal Institute of Technology. +# SPDX-FileCopyrightText: 2024-2025 KTH Royal Institute of Technology. # SPDX-License-Identifier: MIT # quit on errors: @@ -10,13 +10,13 @@ set -o errexit # quit on unbound symbols: set -o nounset -WORKDIR=$(mktemp -d) -TESTDIR=$(pwd) +WORKDIR="$(mktemp -d)" +TESTDIR="$(pwd)" finish() { echo "Cleaning up." - docker compose -f ${PROJECTDIR}/docker-compose.full.yml down --volumes --remove-orphans & - pipenv --rm || true + docker compose -f "${PROJECTDIR}/docker-compose.full.yml" down --volumes --remove-orphans & + rm -rf "${PROJECTDIR}/.venv" || true rm -rf "${WORKDIR}" echo "Test setup cleaned." } @@ -26,33 +26,33 @@ trap finish EXIT PROJECT_NAME="my-site" export PROJECT_NAME -cookiecutter --no-input -o "$WORKDIR" . \ - project_name=${PROJECT_NAME} \ - database=${COOKIECUTTER_DATABASE:-postgresql} \ - opensearch=${COOKIECUTTER_OPENSEARCH:-opensearch2} \ - file_storage=${COOKIECUTTER_FILE_STORAGE:-local} +cookiecutter --no-input -o "${WORKDIR}" . \ + project_name="${PROJECT_NAME}" \ + base_image="${COOKIECUTTER_BASE_IMAGE:-debian}" \ + opensearch="${COOKIECUTTER_OPENSEARCH:-opensearch2}" \ + file_storage="${COOKIECUTTER_FILE_STORAGE:-local}" -PROJECTDIR=${WORKDIR}/${PROJECT_NAME} +PROJECTDIR="${WORKDIR}/${PROJECT_NAME}" export PROJECTDIR -# Check local installation (this also generates the Pipfile.lock) -${TESTDIR}/scripts/bootstrap +# Check local installation (this also generates the uv.lock) +"${TESTDIR}/scripts/bootstrap" -cd ${PROJECTDIR} +cd "${PROJECTDIR}" # Initialize git in the repository for 'check-manifest' to work git init git add -A # Update MANIFEST.in -pipenv run check-manifest -u || true +uv run --with check-manifest check-manifest -u || true # Build application docker images -${TESTDIR}/scripts/build-images.sh +"${TESTDIR}/scripts/build-images.sh" # Fire up a full instance via docker-compose.full.yml # We will use the services (DB, OS/ES, etc) for running the tests locally docker compose -f docker-compose.full.yml up -d -${TESTDIR}/scripts/wait-for-services.sh --full +"${TESTDIR}/scripts/wait-for-services.sh" --full echo "All services are up." docker compose -f docker-compose.full.yml down @@ -60,6 +60,6 @@ echo "Services successfully stopped" # Fire up the services we need for testing docker compose up -d -${TESTDIR}/scripts/wait-for-services.sh +"${TESTDIR}/scripts/wait-for-services.sh" # Run the instance tests # REQUIREMENTS=prod ./run-tests.sh diff --git a/scripts/bootstrap b/scripts/bootstrap index b60105d3..417645ad 100755 --- a/scripts/bootstrap +++ b/scripts/bootstrap @@ -1,24 +1,26 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2019 CERN. # SPDX-FileCopyrightText: 2019 Northwestern University. +# Copyright (C) 2025 KTH Royal Institute of Technology. # SPDX-License-Identifier: MIT set -e -cd ${PROJECTDIR} -pipfile_lock_path="./Pipfile.lock" +cd "${PROJECTDIR}" +uv_lock_path="./uv.lock" -if [ ! -f $pipfile_lock_path ]; then - echo "'Pipfile.lock' not found. Generating via 'pipenv lock'..." - pipenv lock +if [ ! -f "${uv_lock_path}" ]; then + echo "'uv.lock' not found. Generating via 'uv lock'..." + uv lock fi -# Installs all packages specified in Pipfile.lock -pipenv sync +# Install the project with its test extra declared in pyproject.toml. +uv sync --extra tests +export INVENIO_WEBPACKEXT_NPM_PKG_CLS=pynpm:PNPMPackage # Build assets -INVENIO_LOCAL_INSTANCE_PATH=$(pipenv run invenio shell --no-term-title -c "print(app.instance_path)") -cp -r ./static/. ${INVENIO_LOCAL_INSTANCE_PATH}/static/ -cp -r ./assets/. ${INVENIO_LOCAL_INSTANCE_PATH}/assets/ +INVENIO_LOCAL_INSTANCE_PATH="$(uv run invenio shell --no-term-title -c "print(app.instance_path)")" +cp -r ./static/. "${INVENIO_LOCAL_INSTANCE_PATH}/static/" +cp -r ./assets/. "${INVENIO_LOCAL_INSTANCE_PATH}/assets/" -pipenv run invenio collect -v -pipenv run invenio webpack buildall +uv run invenio collect -v +uv run invenio webpack buildall diff --git a/scripts/build-images.sh b/scripts/build-images.sh index 848145bd..a31067a5 100755 --- a/scripts/build-images.sh +++ b/scripts/build-images.sh @@ -3,7 +3,10 @@ # SPDX-FileCopyrightText: 2019-2020 Northwestern University. # SPDX-License-Identifier: MIT -cd ${PROJECTDIR} +cd "${PROJECTDIR}" # Build application image -docker build . --tag ${PROJECT_NAME} --build-arg include_assets=true +docker build . --tag "${PROJECT_NAME}" --build-arg include_assets=true + +# Build frontend image used by docker-compose.full.yml +docker build ./docker/nginx --tag "${PROJECT_NAME}-frontend" diff --git a/scripts/server b/scripts/server index 3b1db717..d5457345 100755 --- a/scripts/server +++ b/scripts/server @@ -1,22 +1,23 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: 2019 CERN. # SPDX-FileCopyrightText: 2019 Northwestern University. +# Copyright (C) 2025 KTH Royal Institute of Technology. # SPDX-License-Identifier: MIT set -e -cd ${PROJECTDIR} +cd "${PROJECTDIR}" export FLASK_ENV=development export INVENIO_SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@localhost/{{cookiecutter.project_shortname}}" # Start Worker and Server -pipenv run celery -A invenio_app.celery worker -l INFO & pid_celery=$! +uv run celery -A invenio_app.celery worker -l INFO & pid_celery=$! -pipenv run invenio run \ - --cert "$script_path"/../docker/nginx/test.crt \ - --key "$script_path"/../docker/nginx/test.key & pid_server=$! +uv run invenio run \ + --cert "${script_path}/../docker/nginx/test.crt" \ + --key "${script_path}/../docker/nginx/test.key" & pid_server=$! -trap 'kill $pid_celery $pid_server &>/dev/null' EXIT +trap 'kill "${pid_celery}" "${pid_server}" &>/dev/null' EXIT -wait $pid_celery $pid_server +wait "${pid_celery}" "${pid_server}" diff --git a/scripts/setup b/scripts/setup index de12b05f..e6de47bb 100755 --- a/scripts/setup +++ b/scripts/setup @@ -2,26 +2,27 @@ # SPDX-FileCopyrightText: 2019 CERN. # SPDX-FileCopyrightText: 2019 Northwestern University. # SPDX-FileCopyrightText: 2021 Esteban J. G. Gabancho. +# Copyright (C) 2025 KTH Royal Institute of Technology. # SPDX-License-Identifier: MIT set -e -cd ${PROJECTDIR} +cd "${PROJECTDIR}" # Clean redis -pipenv run invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')" -pipenv run invenio db destroy --yes-i-know -pipenv run invenio db init create -pipenv run invenio index destroy --force --yes-i-know -pipenv run invenio index init --force -pipenv run invenio index queue init purge -if [ $COOKIECUTTER_FILE_STORAGE -eq "S3" ] +uv run invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')" +uv run invenio db destroy --yes-i-know +uv run invenio db init create +uv run invenio index destroy --force --yes-i-know +uv run invenio index init --force +uv run invenio index queue init purge +if [ "${COOKIECUTTER_FILE_STORAGE}" = "S3" ] then -pipenv run invenio files location s3-default s3://default --default +uv run invenio files location s3-default s3://default --default else -pipenv run invenio files location --default 'default-location' $(pipenv run invenio shell --no-term-title -c "print(app.instance_path)") +uv run invenio files location --default 'default-location' "$(uv run invenio shell --no-term-title -c "print(app.instance_path)")" fi # Create admin role to rectict access -pipenv run invenio roles create admin -pipenv run invenio access allow superuser-access role admin +uv run invenio roles create admin +uv run invenio access allow superuser-access role admin diff --git a/scripts/wait-for-services.sh b/scripts/wait-for-services.sh index 7afb982c..a1c3be05 100755 --- a/scripts/wait-for-services.sh +++ b/scripts/wait-for-services.sh @@ -7,32 +7,26 @@ # Verify that all services are running before continuing check_ready() { RETRIES=20 - while ! $2 + while ! "${2}" do - echo "Waiting for $1, $((RETRIES--)) remaining attempts..." + echo "Waiting for ${1}, $((RETRIES--)) remaining attempts..." sleep 15 - if [ $RETRIES -eq 0 ] + if [ "${RETRIES}" -eq 0 ] then - echo "Couldn't reach $1" + echo "Couldn't reach ${1}" exit 1 fi done } -if [ $COOKIECUTTER_FILE_STORAGE -eq "S3" ] +if [ "${COOKIECUTTER_FILE_STORAGE}" = "S3" ] then _s3_check(){ curl --output /dev/null --silent --head --fail http://localhost:9000/minio/health/live &>/dev/null;} check_ready "S3" _s3_check fi -if [[ ${COOKIECUTTER_DATABASE} == "mysql" ]] -then - _db_check(){ docker compose exec db bash -c "mysql -p${PROJECT_NAME} -e \"select Version();\"" &>/dev/null; } - check_ready "MySQL" _db_check -else - _db_check(){ docker compose exec --user postgres db bash -c "pg_isready" &>/dev/null; } - check_ready "Postgres" _db_check -fi +_db_check(){ docker compose exec --user postgres db bash -c "pg_isready" &>/dev/null; } +check_ready "Postgres" _db_check _search_check(){ curl --output /dev/null --silent --head --fail http://localhost:9200 &>/dev/null; } check_ready "OpenSearch" _search_check diff --git a/{{cookiecutter.project_shortname}}/Dockerfile b/{{cookiecutter.project_shortname}}/Dockerfile index e6685cff..7a5978b8 100644 --- a/{{cookiecutter.project_shortname}}/Dockerfile +++ b/{{cookiecutter.project_shortname}}/Dockerfile @@ -1,29 +1,127 @@ -# Dockerfile that builds a fully functional image of your app. -# -# This image installs all Python dependencies for your application. It's based -# on Almalinux (https://github.com/inveniosoftware/docker-invenio) -# and includes Pip, Pipenv, Node.js, NPM and some few standard libraries -# Invenio usually needs. -# -# Note: It is important to keep the commands in this file in sync with your -# bootstrap script located in ./scripts/bootstrap. - -FROM registry.cern.ch/inveniosoftware/almalinux:1 +{%- if cookiecutter.base_image == "debian" %} +# syntax=docker/dockerfile:1 +# SPDX-FileCopyrightText: {{ cookiecutter.year }} Northwestern University. +# SPDX-FileCopyrightText: {{ cookiecutter.year }} Frontmatter. +# SPDX-FileCopyrightText: {{ cookiecutter.year }} KTH Royal Institute of Technology. + + +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# TODO delete once the test is successful +ARG BUILD_PLATFORM=linux/amd64 +ARG PYTHON_VERSION=3.14 +ARG OS_VERSION=trixie +ARG NODE_VERSION=24 +ARG PNPM_VERSION=latest-10 +ARG INVENIO_WEBPACKEXT_NPM_PKG_CLS=pynpm:PNPMPackage + +FROM --platform=${BUILD_PLATFORM} python:${PYTHON_VERSION}-${OS_VERSION} AS base + +LABEL org.opencontainers.image.source=https://github.com/inveniosoftware/docker-invenio + +# ARGs declared before FROM are only global defaults. Redeclare build args in +# the stage where they are used so passed values are in scope: +# https://docs.docker.com/build/building/variables/#scoping +ARG NODE_VERSION +ARG PNPM_VERSION +ARG INVENIO_WEBPACKEXT_NPM_PKG_CLS + +ENV \ + WORKING_DIR=/opt/invenio \ + INVENIO_INSTANCE_PATH=/opt/invenio/var/instance \ + INVENIO_USER_ID=1000 \ + PATH=/opt/invenio/src/.venv/bin:${PATH} + +# Create relevant application directories +RUN mkdir -p \ + ${WORKING_DIR}/src \ + ${INVENIO_INSTANCE_PATH}/data \ + ${INVENIO_INSTANCE_PATH}/archive \ + ${INVENIO_INSTANCE_PATH}/static + +# Invenio file will be in /src +WORKDIR ${WORKING_DIR}/src + +# Set folder permissions +RUN chgrp -R 0 ${WORKING_DIR} && \ + chmod -R g=u ${WORKING_DIR} && \ + useradd invenio --uid ${INVENIO_USER_ID} --gid 0 && \ + chown -R invenio:root ${WORKING_DIR} + +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get upgrade --yes \ + && \ + # Install Node.js deb source + curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ + && \ + # Install system dependencies + apt-get install --yes --no-install-recommends \ + build-essential \ + fonts-dejavu \ + locales \ + nodejs \ + && \ + # Install pnpm + npm install -g pnpm@${PNPM_VERSION} + +# Setup uv +# From https://docs.astral.sh/uv/guides/integration/docker/ +# but using docker.io/astral/uv because failure to fetch oauth token +COPY --from=docker.io/astral/uv:0.11 /uv /uvx /bin/ + +# Set locale +RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +# Delete the above code and replace it with: +# FROM AS base +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +FROM base AS builder + +COPY README.md pyproject.toml uv.lock ./ COPY site ./site -COPY Pipfile Pipfile.lock ./ -RUN pipenv install --deploy --system +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev --no-editable COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH} COPY ./invenio.cfg ${INVENIO_INSTANCE_PATH} COPY ./templates/ ${INVENIO_INSTANCE_PATH}/templates/ COPY ./app_data/ ${INVENIO_INSTANCE_PATH}/app_data/ COPY ./translations/ ${INVENIO_INSTANCE_PATH}/translations/ -COPY ./ . +COPY ./assets/ ./assets/ +COPY ./static/ ./static/ RUN cp -r ./static/. ${INVENIO_INSTANCE_PATH}/static/ && \ cp -r ./assets/. ${INVENIO_INSTANCE_PATH}/assets/ && \ - invenio collect --verbose && \ - invenio webpack buildall + invenio collect --verbose && \ + invenio webpack buildall && \ + rm -rf ${INVENIO_INSTANCE_PATH}/assets && \ + rm -rf \ + /root/.cache \ + /tmp/* \ + ${INVENIO_INSTANCE_PATH}/assets/node_modules \ + ${INVENIO_INSTANCE_PATH}/assets/.cache + +FROM base AS app-base + +COPY . . +COPY --from=builder ${WORKING_DIR}/src/.venv ./.venv +COPY --from=builder ${INVENIO_INSTANCE_PATH}/static/ ${INVENIO_INSTANCE_PATH}/static/ +COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH} +COPY ./invenio.cfg ${INVENIO_INSTANCE_PATH} +COPY ./templates/ ${INVENIO_INSTANCE_PATH}/templates/ +COPY ./app_data/ ${INVENIO_INSTANCE_PATH}/app_data/ +COPY ./translations/ ${INVENIO_INSTANCE_PATH}/translations/ -ENTRYPOINT [ "bash", "-c"] +ENTRYPOINT ["bash", "-c"] +{%- endif %} diff --git a/{{cookiecutter.project_shortname}}/Pipfile b/{{cookiecutter.project_shortname}}/Pipfile deleted file mode 100644 index 35827d0c..00000000 --- a/{{cookiecutter.project_shortname}}/Pipfile +++ /dev/null @@ -1,22 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] -check-manifest = ">=0.25" - -[packages] -invenio-app-rdm = {extras = ["{{ cookiecutter.search }}"{% if cookiecutter.file_storage == 'S3' %}, "s3"{% endif %}], version = "~=13.0.0"} -{%- if cookiecutter.site_code == 'yes'%} -{{cookiecutter.project_shortname}} = {editable = true, path="./site"} -{% endif %} -uwsgi = ">=2.0" -uwsgitop = ">=0.11" -uwsgi-tools = ">=1.1.1" - -[requires] -python_version = "3.12" - -[pipenv] -allow_prereleases = false diff --git a/{{cookiecutter.project_shortname}}/README.md b/{{cookiecutter.project_shortname}}/README.md index cde692c7..b346ae02 100644 --- a/{{cookiecutter.project_shortname}}/README.md +++ b/{{cookiecutter.project_shortname}}/README.md @@ -27,9 +27,9 @@ Following is an overview of the generated files and folders: | Name | Description | |---|---| -| ``Dockerfile`` | Dockerfile used to build your application image. | -| ``Pipfile`` | Python requirements installed via [pipenv](https://pipenv.pypa.io) | -| ``Pipfile.lock`` | Locked requirements (generated on first install). | +| ``Dockerfile`` | Dockerfile used to build your application image on {{ cookiecutter.base_image }}. | +| ``pyproject.toml`` | Python requirements installed via [uv](https://docs.astral.sh/uv/). | +| ``uv.lock`` | Locked requirements (generated on first install). | | ``app_data`` | Application data such as vocabularies. | | ``assets`` | Web assets (CSS, JavaScript, LESS, JSX templates) used in the Webpack build. | | ``docker`` | Example configuration for NGINX and uWSGI. | diff --git a/{{cookiecutter.project_shortname}}/docker-compose.full.yml b/{{cookiecutter.project_shortname}}/docker-compose.full.yml index 02721798..8616ae2f 100644 --- a/{{cookiecutter.project_shortname}}/docker-compose.full.yml +++ b/{{cookiecutter.project_shortname}}/docker-compose.full.yml @@ -13,7 +13,7 @@ # - UI application: UWSGI (not exposed) # - API application: UWSGI (not exposed) # - Cache: Redis (exposed port: 6379) -# - DB: (PostgresSQL/MySQL) (exposed port: 5432 or 3306) +# - DB: PostgreSQL (exposed port: 5432) # - Message queue: RabbitMQ (exposed ports: 5672, 15672) # - Search platform: (OpenSearch) (exposed ports: 9200, 9600) # - OpenSearch Dashboard/Kibana (view OS/ES indexes) (exposed ports: 5601) @@ -47,17 +47,10 @@ services: extends: file: docker-services.yml service: opensearch-dashboards - {%- if cookiecutter.database == 'postgresql'%} pgadmin: extends: file: docker-services.yml service: pgadmin - {%- elif cookiecutter.database == 'mysql'%} - phpmyadmin: - extends: - file: docker-services.yml - service: phpmyadmin - {%- endif %} {%- endif %} {%- if cookiecutter.file_storage == 'S3'%} s3: @@ -84,6 +77,9 @@ services: file: docker-services.yml service: app command: ["uwsgi /opt/invenio/var/instance/uwsgi_ui.ini"] + depends_on: + search: + condition: service_healthy image: {{cookiecutter.project_shortname}}:latest ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:5000:5000" @@ -100,6 +96,9 @@ services: file: docker-services.yml service: app command: ["uwsgi /opt/invenio/var/instance/uwsgi_rest.ini"] + depends_on: + search: + condition: service_healthy image: {{cookiecutter.project_shortname}}:latest ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:5001:5000" @@ -122,7 +121,7 @@ services: {%- endif %} depends_on: search: - condition: service_started + condition: service_healthy cache: condition: service_started db: @@ -145,10 +144,16 @@ services: {%- if cookiecutter.file_storage == 'S3'%} volumes: + db_data: + mq_data: + search_data: static_data: - data: + s3_data: {%- else %} volumes: + db_data: + mq_data: + search_data: static_data: uploaded_data: archived_data: diff --git a/{{cookiecutter.project_shortname}}/docker-compose.yml b/{{cookiecutter.project_shortname}}/docker-compose.yml index 15b5aef2..895eae3b 100644 --- a/{{cookiecutter.project_shortname}}/docker-compose.yml +++ b/{{cookiecutter.project_shortname}}/docker-compose.yml @@ -9,10 +9,10 @@ # # Following services are included: # - Cache: Redis (exposed port: 6379) -# - DB: (PostgresSQL/MySQL) (exposed port: 5432 or 3306) +# - DB: PostgreSQL (exposed port: 5432) # - Message queue: RabbitMQ (exposed ports: 5672, 15672) # - OpenSearch (exposed ports: 9200, 9600) -# - Kibana (view ES indexes) (exposed ports: 5601) +# - Kibana (view OS indexes) (exposed ports: 5601) # services: cache: @@ -43,23 +43,21 @@ services: extends: file: docker-services.yml service: opensearch-dashboards - {%- if cookiecutter.database == 'postgresql'%} pgadmin: extends: file: docker-services.yml service: pgadmin - {%- elif cookiecutter.database == 'mysql'%} - phpmyadmin: - extends: - file: docker-services.yml - service: phpmyadmin - {%- endif %} {%- endif %} {%- if cookiecutter.file_storage == 'S3'%} s3: extends: file: docker-services.yml service: s3 +{%- endif %} volumes: - data: + db_data: + mq_data: + search_data: +{%- if cookiecutter.file_storage == 'S3'%} + s3_data: {%- endif %} diff --git a/{{cookiecutter.project_shortname}}/docker-services.yml b/{{cookiecutter.project_shortname}}/docker-services.yml index e0259e15..0558dc82 100644 --- a/{{cookiecutter.project_shortname}}/docker-services.yml +++ b/{{cookiecutter.project_shortname}}/docker-services.yml @@ -4,9 +4,12 @@ services: context: ./ args: - ENVIRONMENT=DEV + - WEBPACKEXT_NPM_PKG_CLS=${WEBPACKEXT_NPM_PKG_CLS:-pynpm:PNPMPackage} image: {{cookiecutter.project_shortname}} restart: "unless-stopped" environment: + - "INVENIO_SITE_UI_URL=https://127.0.0.1" + - "INVENIO_SITE_API_URL=https://127.0.0.1/api" - "INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://cache:6379/1" - "INVENIO_BROKER_URL=amqp://guest:guest@mq:5672/" - "INVENIO_CACHE_REDIS_URL=redis://cache:6379/0" @@ -16,11 +19,7 @@ services: - "INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=redis://cache:6379/4" - "INVENIO_SEARCH_HOSTS=['search:9200']" - "INVENIO_SECRET_KEY=CHANGE_ME" - {%- if cookiecutter.database == 'postgresql'%} - "INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@db/{{cookiecutter.project_shortname}}" - {%- elif cookiecutter.database == 'mysql'%} - - "INVENIO_SQLALCHEMY_DATABASE_URI=mysql+pymysql://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@db/{{cookiecutter.project_shortname}}" - {%- endif %} - "INVENIO_PROXYFIX_CONFIG={'x_for': 1, 'x_proto': 1}" - "INVENIO_RATELIMIT_STORAGE_URL=redis://cache:6379/3" frontend: @@ -36,9 +35,8 @@ services: read_only: true ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:6379:6379" - {%- if cookiecutter.database == 'postgresql'%} db: - image: postgres:14.13 + image: postgres:17.10 restart: "unless-stopped" environment: - "POSTGRES_USER={{cookiecutter.project_shortname}}" @@ -46,6 +44,8 @@ services: - "POSTGRES_DB={{cookiecutter.project_shortname}}" ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:5432:5432" + volumes: + - db_data:/var/lib/postgresql/data pgadmin: image: dpage/pgadmin4:6 restart: "unless-stopped" @@ -56,26 +56,6 @@ services: PGADMIN_DEFAULT_PASSWORD: "{{cookiecutter.project_shortname}}" volumes: - ./docker/pgadmin/servers.json:/pgadmin4/servers.json - {%- elif cookiecutter.database == 'mysql'%} - db: - image: mysql - restart: "unless-stopped" - environment: - - "MYSQL_ROOT_PASSWORD={{cookiecutter.project_shortname}}" - - "MYSQL_DATABAE={{cookiecutter.project_shortname}}" - - "MYSQL_USER={{cookiecutter.project_shortname}}" - - "MYSQL_PASSWORD={{cookiecutter.project_shortname}}" - ports: - - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:3306:3306" - phpmyadmin: - image: phpmyadmin/phpmyadmin - restart: "unless-stopped" - ports: - - '${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:8080:80' - environment: - PMA_HOST: db - MYSQL_ROOT_PASSWORD: {{cookiecutter.project_shortname}} - {%- endif %} # note: RabbitMQ uses the hostname to build the Mnesia data directory path (cf. https://www.rabbitmq.com/docs/configure) # using a randomly generated hostname "forgets" about old queued messages (e.g. celery tasks) with new builds, # and hard-coding the Mnesia path with random hostnames leads to connection issues on startup for old names @@ -90,6 +70,8 @@ services: ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:15672:15672" - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:5672:5672" + volumes: + - mq_data:/var/lib/rabbitmq flower: image: mher/flower:2.0.1 restart: "unless-stopped" @@ -100,7 +82,7 @@ services: mq: condition: service_started search: - image: opensearchproject/opensearch:2.17.1 + image: opensearchproject/opensearch:2.19.5 restart: "unless-stopped" environment: # Settings only for development. DO NOT use in production! @@ -123,8 +105,16 @@ services: ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:9200:9200" - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:9600:9600" + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200 >/dev/null || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 30s + volumes: + - search_data:/usr/share/opensearch/data opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:2.17.1 + image: opensearchproject/opensearch-dashboards:2.19.5 ports: - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:5601:5601" expose: @@ -144,7 +134,7 @@ services: MINIO_ROOT_USER: CHANGE_ME MINIO_ROOT_PASSWORD: CHANGE_ME volumes: - - ./data:/data + - s3_data:/data command: server /data --console-address :9001 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] @@ -152,3 +142,11 @@ services: timeout: 20s retries: 3 {%- endif %} + +volumes: + db_data: + mq_data: + search_data: +{%- if cookiecutter.file_storage == 'S3'%} + s3_data: +{%- endif %} diff --git a/{{cookiecutter.project_shortname}}/invenio.cfg b/{{cookiecutter.project_shortname}}/invenio.cfg index 6b686819..2eacf04a 100644 --- a/{{cookiecutter.project_shortname}}/invenio.cfg +++ b/{{cookiecutter.project_shortname}}/invenio.cfg @@ -9,7 +9,7 @@ https://inveniordm.docs.cern.ch/operate/customize/configuration/. from datetime import datetime from invenio_i18n import lazy_gettext as _ - +from invenio_app_rdm import __version__ def _(x): # needed to avoid start time failure with lazy strings return x @@ -41,11 +41,7 @@ TRUSTED_HOSTS = ['0.0.0.0', 'localhost', '127.0.0.1'] # See https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ # TODO: Set -{%- if cookiecutter.database == 'postgresql'%} SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@localhost/{{cookiecutter.project_shortname}}" -{%- elif cookiecutter.database == 'mysql'%} -SQLALCHEMY_DATABASE_URI="mysql+pymysql://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@localhost/{{cookiecutter.project_shortname}}" -{%- endif %} # Invenio-App @@ -228,12 +224,15 @@ SEARCH_INDEX_PREFIX = "{{ cookiecutter.project_shortname }}-" # Invenio-Administration # ---------------------- -from invenio_app_rdm import __version__ ADMINISTRATION_DISPLAY_VERSIONS = [ ("invenio-app-rdm", f"v{__version__}"), ("{{ cookiecutter.project_shortname }}", "v1.0.0"), ] +# Asset bundling configuration +# --------------------------- +WEBPACKEXT_PROJECT = "invenio_assets.webpack:rspack_project" +"""Project asset bundling configuration. See https://invenio-assets.readthedocs.io/en/latest/configuration.html.""" # Invenio-Stats # ---------------------- diff --git a/{{cookiecutter.project_shortname}}/pyproject.toml b/{{cookiecutter.project_shortname}}/pyproject.toml new file mode 100644 index 00000000..ad5eb20d --- /dev/null +++ b/{{cookiecutter.project_shortname}}/pyproject.toml @@ -0,0 +1,49 @@ +{%- set project_shortname = cookiecutter.project_shortname -%} +{%- set app_rdm_version = "~=14.0.0b11.dev1" -%} +{%- set app_rdm_base = "invenio-app-rdm" ~ app_rdm_version -%} +{%- set app_rdm_extras = ['opensearch2'] -%} +{%- if cookiecutter.file_storage == 'S3' -%} + {%- set app_rdm_extras = app_rdm_extras + ['s3'] -%} +{%- endif -%} +{%- set instance_dist_name = (cookiecutter.project_name | slugify()) + '-instance' -%} +{%- if cookiecutter.site_code == 'yes' -%} + {%- set site_dist_name = cookiecutter.project_shortname -%} +{%- endif -%} + +[project] +name = "{{ instance_dist_name }}" +description = "Add your description here" +readme = "README.md" +version = "{{ app_rdm_version | replace('~=', '') }}" +requires-python = ">= 3.14" + +dependencies = [ + "invenio-app-rdm{% if app_rdm_extras %}[{{ app_rdm_extras | join(',') }}]{% endif %}{{ app_rdm_version }}", + "uwsgi>=2.0", + "uwsgitop>=0.11", + "uwsgi-tools>=1.1.1", + "lxml-html-clean>=0.1.1", + {% if cookiecutter.site_code == 'yes' -%} + "{{ site_dist_name }}", + {%- endif %} +] + +{% if cookiecutter.site_code == 'yes' -%} +[tool.uv.sources] +"{{ site_dist_name }}" = { workspace = true } + +[tool.uv.workspace] +members = ["site"] +{%- endif %} + +[project.optional-dependencies] +tests = [ + "pytest-invenio>=4.0.0,<5.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +bypass-selection = true diff --git a/{{cookiecutter.project_shortname}}/site/pyproject.toml b/{{cookiecutter.project_shortname}}/site/pyproject.toml index ec4e1ea4..d57124a4 100644 --- a/{{cookiecutter.project_shortname}}/site/pyproject.toml +++ b/{{cookiecutter.project_shortname}}/site/pyproject.toml @@ -1,3 +1,30 @@ +{%- set dist_name = cookiecutter.project_shortname -%} +[project] +name = "{{dist_name}}" +version = "0.1.0" +description = "Add your description here" +requires-python = ">= 3.14" + +[project.entry-points."invenio_base.blueprints"] +{{cookiecutter.package_name}}_views = "{{cookiecutter.package_name}}.views:create_blueprint" + +[project.entry-points."invenio_assets.webpack"] +{{cookiecutter.package_name}}_theme = "{{cookiecutter.package_name}}.webpack:theme" + [build-system] -requires = ["setuptools", "wheel", "babel>2.8"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["{{cookiecutter.package_name}}"] + +[tool.isort] +profile = "black" + +[tool.pydocstyle] +add_ignore = ["D401", "D403"] + +[tool.pytest.ini_options] +addopts = ["--isort", "--pydocstyle", "--doctest-glob=\"*.rst\"", "--doctest-modules", "--cov={{cookiecutter.package_name}}", "--cov-report=term-missing", "--exclude-warning-annotations"] +testpaths = "tests {{cookiecutter.package_name}}" +live_server_scope = "module" diff --git a/{{cookiecutter.project_shortname}}/site/setup.cfg b/{{cookiecutter.project_shortname}}/site/setup.cfg deleted file mode 100644 index 226c143f..00000000 --- a/{{cookiecutter.project_shortname}}/site/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ - -[metadata] -name = {{cookiecutter.project_shortname}} - -[options.extras_require] -tests = - pytest-invenio>=2.1.0,<3.0.0 - -[options.entry_points] -invenio_base.blueprints = - {{cookiecutter.package_name}}_views = {{cookiecutter.package_name}}.views:create_blueprint -invenio_assets.webpack = - {{cookiecutter.package_name}}_theme = {{cookiecutter.package_name}}.webpack:theme diff --git a/{{cookiecutter.project_shortname}}/site/setup.py b/{{cookiecutter.project_shortname}}/site/setup.py deleted file mode 100644 index 60684932..00000000 --- a/{{cookiecutter.project_shortname}}/site/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/{{cookiecutter.project_shortname}}/wipe_recreate.sh b/{{cookiecutter.project_shortname}}/wipe_recreate.sh new file mode 100755 index 00000000..25e481b4 --- /dev/null +++ b/{{cookiecutter.project_shortname}}/wipe_recreate.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env sh +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020-2026 CERN. +# +# Demo-InvenioRDM is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +# Quit on errors +set -o errexit + +# Quit on unbound symbols +set -o nounset + +# Prompt to confirm action +printf "Are you sure you want to wipe everything and create a new empty instance? [y/N] " +read -r response +case "${response}" in + [yY]|[yY][eE][sS]) + ;; + *) + exit 0 + ;; +esac + +# Wipe +# ---- +invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')" +# NOTE: db destroy is not needed since DB keeps being created +# Just need to drop all tables from it. +invenio db drop --yes-i-know +invenio index destroy --force --yes-i-know +invenio index queue init purge + +# Recreate +# -------- +# NOTE: db init is not needed since DB keeps being created +# Just need to create all tables from it. +invenio db create +invenio files location create --default 'default-location' "$(invenio shell --no-term-title -c "print(app.instance_path)")/data" +# +# Create roles +# +# Superuser role +invenio roles create admin +invenio access allow superuser-access role admin +# Administration access role +invenio roles create administration +invenio access allow administration-access role administration +# Administration moderation role +invenio roles create administration-moderation +invenio access allow administration-moderation role administration-moderation + +invenio index init --force +invenio rdm-records custom-fields init +invenio communities custom-fields init + +# Add demo and fixtures data +# ------------- +invenio rdm-records fixtures +invenio rdm-records demo + +# Enable admin user +invenio users activate admin@inveniosoftware.org