Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions cookiecutter.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,6 @@
"author_name": "CERN",
"author_email": "info@{{ cookiecutter.project_site }}",
"year": "{% now 'local', '%Y' %}",
"database": [
"postgresql"
],
"search": [
"opensearch2"
],
"file_storage": [
"local",
"S3"
Expand All @@ -24,8 +18,11 @@
"yes",
"no"
],
"base_image": [
"debian"
Comment thread
Samk13 marked this conversation as resolved.
],
"use_reduced_vocabs": [
"no",
"yes"
"yes",
"no"
Comment on lines +25 to +26

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would change defaults. I don't have a strong opinion, but worth making it clear to other reviewers.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, for other reviewers, this intentionally changes the old default.

This cookiecutter is mostly used for development and demos, where you typically want a running instance fast without long time indexing. I found myself selecting yes way more often than not, and occasionally missing the prompt and having to start over, which is why I changed this default.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

✔️

]
}
2 changes: 1 addition & 1 deletion hooks/post_gen_project.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ touch data/default/.gitkeep

{%- if cookiecutter.site_code == 'no'%}
DIR="site"
if [ -d "$DIR" ]; then
if [ -d "${DIR}" ]; then
rm -r site
fi
{% endif %}
36 changes: 18 additions & 18 deletions run-tests.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2019 CERN.
# SPDX-FileCopyrightText: 2019 Northwestern University.
# SPDX-FileCopyrightText: 2024 KTH Royal Institute of Technology.
# SPDX-FileCopyrightText: 2024-2025 KTH Royal Institute of Technology.
# SPDX-License-Identifier: MIT

# quit on errors:
Expand All @@ -10,13 +10,13 @@ set -o errexit
# quit on unbound symbols:
set -o nounset

WORKDIR=$(mktemp -d)
TESTDIR=$(pwd)
WORKDIR="$(mktemp -d)"
TESTDIR="$(pwd)"

finish() {
echo "Cleaning up."
docker compose -f ${PROJECTDIR}/docker-compose.full.yml down --volumes --remove-orphans &
pipenv --rm || true
docker compose -f "${PROJECTDIR}/docker-compose.full.yml" down --volumes --remove-orphans &
rm -rf "${PROJECTDIR}/.venv" || true
rm -rf "${WORKDIR}"
echo "Test setup cleaned."
}
Expand All @@ -26,40 +26,40 @@ trap finish EXIT
PROJECT_NAME="my-site"
export PROJECT_NAME

cookiecutter --no-input -o "$WORKDIR" . \
project_name=${PROJECT_NAME} \
database=${COOKIECUTTER_DATABASE:-postgresql} \
opensearch=${COOKIECUTTER_OPENSEARCH:-opensearch2} \
file_storage=${COOKIECUTTER_FILE_STORAGE:-local}
cookiecutter --no-input -o "${WORKDIR}" . \
project_name="${PROJECT_NAME}" \
base_image="${COOKIECUTTER_BASE_IMAGE:-debian}" \
opensearch="${COOKIECUTTER_OPENSEARCH:-opensearch2}" \
file_storage="${COOKIECUTTER_FILE_STORAGE:-local}"

PROJECTDIR=${WORKDIR}/${PROJECT_NAME}
PROJECTDIR="${WORKDIR}/${PROJECT_NAME}"
export PROJECTDIR

# Check local installation (this also generates the Pipfile.lock)
${TESTDIR}/scripts/bootstrap
# Check local installation (this also generates the uv.lock)
"${TESTDIR}/scripts/bootstrap"

cd ${PROJECTDIR}
cd "${PROJECTDIR}"

# Initialize git in the repository for 'check-manifest' to work
git init
git add -A

# Update MANIFEST.in
pipenv run check-manifest -u || true
uv run --with check-manifest check-manifest -u || true

# Build application docker images
${TESTDIR}/scripts/build-images.sh
"${TESTDIR}/scripts/build-images.sh"
# Fire up a full instance via docker-compose.full.yml
# We will use the services (DB, OS/ES, etc) for running the tests locally
docker compose -f docker-compose.full.yml up -d
${TESTDIR}/scripts/wait-for-services.sh --full
"${TESTDIR}/scripts/wait-for-services.sh" --full
echo "All services are up."

docker compose -f docker-compose.full.yml down
echo "Services successfully stopped"

# Fire up the services we need for testing
docker compose up -d
${TESTDIR}/scripts/wait-for-services.sh
"${TESTDIR}/scripts/wait-for-services.sh"
# Run the instance tests
# REQUIREMENTS=prod ./run-tests.sh
26 changes: 14 additions & 12 deletions scripts/bootstrap
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2019 CERN.
# SPDX-FileCopyrightText: 2019 Northwestern University.
# Copyright (C) 2025 KTH Royal Institute of Technology.
# SPDX-License-Identifier: MIT

set -e

cd ${PROJECTDIR}
pipfile_lock_path="./Pipfile.lock"
cd "${PROJECTDIR}"
uv_lock_path="./uv.lock"

if [ ! -f $pipfile_lock_path ]; then
echo "'Pipfile.lock' not found. Generating via 'pipenv lock'..."
pipenv lock
if [ ! -f "${uv_lock_path}" ]; then
echo "'uv.lock' not found. Generating via 'uv lock'..."
uv lock
fi

# Installs all packages specified in Pipfile.lock
pipenv sync
# Install the project with its test extra declared in pyproject.toml.
uv sync --extra tests
export INVENIO_WEBPACKEXT_NPM_PKG_CLS=pynpm:PNPMPackage
Comment thread
Samk13 marked this conversation as resolved.
# Build assets
INVENIO_LOCAL_INSTANCE_PATH=$(pipenv run invenio shell --no-term-title -c "print(app.instance_path)")
cp -r ./static/. ${INVENIO_LOCAL_INSTANCE_PATH}/static/
cp -r ./assets/. ${INVENIO_LOCAL_INSTANCE_PATH}/assets/
INVENIO_LOCAL_INSTANCE_PATH="$(uv run invenio shell --no-term-title -c "print(app.instance_path)")"
cp -r ./static/. "${INVENIO_LOCAL_INSTANCE_PATH}/static/"
cp -r ./assets/. "${INVENIO_LOCAL_INSTANCE_PATH}/assets/"

pipenv run invenio collect -v
pipenv run invenio webpack buildall
uv run invenio collect -v
uv run invenio webpack buildall
7 changes: 5 additions & 2 deletions scripts/build-images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
# SPDX-FileCopyrightText: 2019-2020 Northwestern University.
# SPDX-License-Identifier: MIT

cd ${PROJECTDIR}
cd "${PROJECTDIR}"

# Build application image
docker build . --tag ${PROJECT_NAME} --build-arg include_assets=true
docker build . --tag "${PROJECT_NAME}" --build-arg include_assets=true

# Build frontend image used by docker-compose.full.yml
docker build ./docker/nginx --tag "${PROJECT_NAME}-frontend"
15 changes: 8 additions & 7 deletions scripts/server
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2019 CERN.
# SPDX-FileCopyrightText: 2019 Northwestern University.
# Copyright (C) 2025 KTH Royal Institute of Technology.
# SPDX-License-Identifier: MIT

set -e

cd ${PROJECTDIR}
cd "${PROJECTDIR}"

export FLASK_ENV=development
export INVENIO_SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://{{cookiecutter.project_shortname}}:{{cookiecutter.project_shortname}}@localhost/{{cookiecutter.project_shortname}}"

# Start Worker and Server
pipenv run celery -A invenio_app.celery worker -l INFO & pid_celery=$!
uv run celery -A invenio_app.celery worker -l INFO & pid_celery=$!

pipenv run invenio run \
--cert "$script_path"/../docker/nginx/test.crt \
--key "$script_path"/../docker/nginx/test.key & pid_server=$!
uv run invenio run \
--cert "${script_path}/../docker/nginx/test.crt" \
--key "${script_path}/../docker/nginx/test.key" & pid_server=$!

trap 'kill $pid_celery $pid_server &>/dev/null' EXIT
trap 'kill "${pid_celery}" "${pid_server}" &>/dev/null' EXIT

wait $pid_celery $pid_server
wait "${pid_celery}" "${pid_server}"
25 changes: 13 additions & 12 deletions scripts/setup
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,27 @@
# SPDX-FileCopyrightText: 2019 CERN.
# SPDX-FileCopyrightText: 2019 Northwestern University.
# SPDX-FileCopyrightText: 2021 Esteban J. G. Gabancho.
# Copyright (C) 2025 KTH Royal Institute of Technology.
# SPDX-License-Identifier: MIT

set -e

cd ${PROJECTDIR}
cd "${PROJECTDIR}"

# Clean redis
pipenv run invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')"
pipenv run invenio db destroy --yes-i-know
pipenv run invenio db init create
pipenv run invenio index destroy --force --yes-i-know
pipenv run invenio index init --force
pipenv run invenio index queue init purge
if [ $COOKIECUTTER_FILE_STORAGE -eq "S3" ]
uv run invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')"
uv run invenio db destroy --yes-i-know
uv run invenio db init create
uv run invenio index destroy --force --yes-i-know
uv run invenio index init --force
uv run invenio index queue init purge
if [ "${COOKIECUTTER_FILE_STORAGE}" = "S3" ]
then
pipenv run invenio files location s3-default s3://default --default
uv run invenio files location s3-default s3://default --default
else
pipenv run invenio files location --default 'default-location' $(pipenv run invenio shell --no-term-title -c "print(app.instance_path)")
uv run invenio files location --default 'default-location' "$(uv run invenio shell --no-term-title -c "print(app.instance_path)")"
fi

# Create admin role to rectict access
pipenv run invenio roles create admin
pipenv run invenio access allow superuser-access role admin
uv run invenio roles create admin
uv run invenio access allow superuser-access role admin
20 changes: 7 additions & 13 deletions scripts/wait-for-services.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,32 +7,26 @@
# Verify that all services are running before continuing
check_ready() {
RETRIES=20
while ! $2
while ! "${2}"
do
echo "Waiting for $1, $((RETRIES--)) remaining attempts..."
echo "Waiting for ${1}, $((RETRIES--)) remaining attempts..."
sleep 15
if [ $RETRIES -eq 0 ]
if [ "${RETRIES}" -eq 0 ]
then
echo "Couldn't reach $1"
echo "Couldn't reach ${1}"
exit 1
fi
done
}

if [ $COOKIECUTTER_FILE_STORAGE -eq "S3" ]
if [ "${COOKIECUTTER_FILE_STORAGE}" = "S3" ]
then
_s3_check(){ curl --output /dev/null --silent --head --fail http://localhost:9000/minio/health/live &>/dev/null;}
check_ready "S3" _s3_check
fi

if [[ ${COOKIECUTTER_DATABASE} == "mysql" ]]
then
_db_check(){ docker compose exec db bash -c "mysql -p${PROJECT_NAME} -e \"select Version();\"" &>/dev/null; }
check_ready "MySQL" _db_check
else
_db_check(){ docker compose exec --user postgres db bash -c "pg_isready" &>/dev/null; }
check_ready "Postgres" _db_check
fi
_db_check(){ docker compose exec --user postgres db bash -c "pg_isready" &>/dev/null; }
check_ready "Postgres" _db_check

_search_check(){ curl --output /dev/null --silent --head --fail http://localhost:9200 &>/dev/null; }
check_ready "OpenSearch" _search_check
Expand Down
Loading