From 58e89728eee8f264b1cb95d27d19a6b054040f23 Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 11:20:47 +0100 Subject: [PATCH 01/11] Initial change for astral. --- .github/workflows/bare.yaml | 206 ------------------ .github/workflows/ci.yaml | 385 ++++++++++++++++++++++++++++++++++ .github/workflows/docker.yaml | 203 ------------------ README.rst | 76 +++---- astral-to-pythia.sh | 111 ++++++++++ build.conf | 63 +----- build.sh | 330 ----------------------------- functions.sh | 83 -------- functions_build.sh | 324 ---------------------------- os_quirks.sh | 130 ------------ publish_dist.sh | 18 +- pythia.conf | 4 +- requirements.txt | 7 +- test_compat.sh | 50 +++++ 14 files changed, 609 insertions(+), 1381 deletions(-) delete mode 100644 .github/workflows/bare.yaml create mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/docker.yaml create mode 100755 astral-to-pythia.sh delete mode 100755 build.sh delete mode 100644 functions.sh delete mode 100644 functions_build.sh delete mode 100644 os_quirks.sh create mode 100644 test_compat.sh diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml deleted file mode 100644 index dce9ada2..00000000 --- a/.github/workflows/bare.yaml +++ /dev/null @@ -1,206 +0,0 @@ -# -# GitHub actions for building and testing on bare hardware. -# Typically, Windows and macOS builds are done this way. -# Look for the Linux builds in the "docker" workflow. - -name: Bare - -on: - push: - branches: [ master ] - pull_request: - -concurrency: - group: bare-${{ github.ref }} - cancel-in-progress: true - -env: - CI: 'true' - -jobs: - windows: - # The type of runner that the job will run on - runs-on: windows-latest - timeout-minutes: 45 - - permissions: - # Give the default GITHUB_TOKEN write permission to commit requirements.txt - contents: write - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - # Explicitly run our scripts with Bash, not PowerShell (GitHub's default). - - name: Build Pythia - timeout-minutes: 10 - run: bash ./build.sh build - - - name: Test Pythia - timeout-minutes: 5 - run: bash ./build.sh test - - - name: Test compat - timeout-minutes: 10 - env: - USER: runneradmin - CODECOV_TOKEN: local - # FIXME:72: - # Re-enable compat tests. - run: true bash ./build.sh compat - - # Commit changed requirements.txt back to the repository - - uses: chevah/git-auto-commit-action@HEAD - with: - commit_message: Automated update of requirements.txt from Windows build. - file_pattern: 'requirements.txt' - - # To use an RSA key with SFTPPlus, install upstream OpenSSH package, - # which is more finicky in regards to file permissions. - # Beware the commands in this step run under PowerShell. - - name: Prepare SFTP upload - run: | - mkdir -p ~/.ssh/ - touch priv_key - icacls .\priv_key /inheritance:r - icacls .\priv_key /grant:r runneradmin:"(F)" - echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key - echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts - choco install --yes --no-progress openssh - - # Upload using a (per-OS selected) sftp command, then show final links. - # Remove key in same step to avoid leaving it on disk if publishing fails. - - name: Upload testing package - timeout-minutes: 5 - run: bash -c './publish_dist.sh ; rm priv_key' - - - name: Tmate debug on failure - if: ${{ !cancelled() && runner.debug }} - uses: chevah/action-tmate@v3 - with: - limit-access-to-actor: true - - # To access the Windows desktop for advanced debugging, as per - # https://github.com/nelsonjchen/reverse-rdp-windows-github-actions, - # but using the ngrok token as password for the runnneradmin user. - # Basically use the ngrok token and the ngrok URL (from ngrok's dashboard). - - name: Ngrok debugging on failure - if: ${{ failure() && !cancelled() && runner.debug }} - env: - NGROK_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} - run: | - choco install --yes --no-progress ngrok - ngrok.exe authtoken $env:NGROK_TOKEN - Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server'-name "fDenyTSConnections" -Value 0 - Enable-NetFirewallRule -DisplayGroup "Remote Desktop" - Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 1 - Set-LocalUser -Name "runneradmin" -Password (ConvertTo-SecureString -AsPlainText "$env:NGROK_TOKEN" -Force) - ngrok.exe tcp 3389 - - - macos: - # For macOS on Intel at GitHub. - runs-on: macos-13 - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - # Some Homebrew libs pollute the build. - # Also make sure binaries depending on these libs are out of the way. - - name: Hack Homebrew - run: | - sudo mv -v /usr/local/bin/git{,.saved} - sudo chmod -v a-r /usr/local/include/libintl.h - sudo chmod -v a-r /usr/local/include/gdbm.h - sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* - sudo chmod -v a-r /usr/local/opt/gdbm/lib/libgdbm* - sudo chmod -v a-r /usr/local/opt/mpdecimal/lib/libmpdec* - - - name: Build Pythia - timeout-minutes: 30 - run: ./build.sh build - - # Fix back Homebrew, for working Shellcheck tests and tmate debugging. - - name: Unhack Homebrew - run: | - sudo chmod -v a+r /usr/local/include/libintl.h - sudo chmod -v a-r /usr/local/include/gdbm.h - sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl* - sudo chmod -v a+r /usr/local/opt/gdbm/lib/libgdbm* - sudo chmod -v a+r /usr/local/opt/mpdecimal/lib/libmpdec* - sudo mv -v /usr/local/bin/git{.saved,} - - - name: Test Pythia - timeout-minutes: 5 - run: ./build.sh test - - - name: Test compat - timeout-minutes: 10 - env: - CODECOV_TOKEN: local - # FIXME:72: - # Re-enable compat tests. - run: true ./build.sh compat - - - name: Upload testing package - timeout-minutes: 5 - run: | - mkdir -pv ~/.ssh/ - touch priv_key - chmod 600 priv_key - echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key - echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts - ./publish_dist.sh ; rm priv_key - - - name: Tmate debug on failure - if: ${{ !cancelled() && runner.debug }} - uses: chevah/action-tmate@v3 - with: - limit-access-to-actor: true - - - macos-m1: - # For macOS on Apple Silicon at GitHub. - runs-on: macos-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Build Pythia - timeout-minutes: 20 - run: ./build.sh build - - - name: Test Pythia - timeout-minutes: 5 - run: ./build.sh test - - - name: Test compat - timeout-minutes: 10 - env: - CODECOV_TOKEN: local - # FIXME:72: - # Re-enable compat tests. - run: true ./build.sh compat - - - name: Upload testing package - timeout-minutes: 5 - run: | - mkdir -pv ~/.ssh/ - touch priv_key - chmod 600 priv_key - echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key - echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts - ./publish_dist.sh ; rm priv_key - - - name: Tmate debug on failure - if: ${{ !cancelled() && runner.debug }} - uses: chevah/action-tmate@v3 - with: - limit-access-to-actor: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..247eea67 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,385 @@ +# +# GitHub actions for building and testing the Python version used by Pythia. +# Typically, Windows and macOS builds are on bare VMs . +# For Linux builds we use a container to allow building MUSL or ARM64. +# +# These jobs are preparing the OS, then building the Python package, +# run some tests and upload the result. +# +name: CI + +on: + push: + branches: [ main ] + pull_request: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + CI: 'true' + +jobs: + + linux: + runs-on: ubuntu-latest + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + # Alpine Linux 3.15 has musl 1.2.2, Amazon Linux 2 has glibc 2.26. + container: [ 'alpine:3.15', 'amazonlinux:2' ] + timeout-minutes: 60 + steps: + + # OpenSSL gets updated by apk, but that is the Alpine way, so it's fine. + - name: Alpine setup + if: startsWith(matrix.container, 'alpine') + run: | + apk upgrade -U + apk add git curl bash lddtree shadow openssh-client file unzip musl-locales dejagnu sudo + apk del util-linux-dev + curl --output /usr/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 + chmod +x /usr/bin/paxctl + + - name: Amazon setup + if: startsWith(matrix.container, 'amazonlinux') + run: | + yum -y upgrade + yum -y install git-core dejagnu bzip2 sudo + # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. + # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. + rm -v /usr/include/libdb/db.h + rm -v /usr/include/gdbm.h + # This is for avoiding building the NIS module with Python 3.12. + rm -v /usr/include/rpc/rpc.h + + # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. + - name: Clone repo independently + run: | + git clone https://github.com/chevah/pythia.git + cd pythia + git checkout ${GITHUB_HEAD_REF} + + - name: Chevah user setup + run: | + useradd -g adm -s /bin/bash -m chevah + echo '%adm ALL=NOPASSWD: ALL' > /etc/sudoers + + - name: Build Pythia + timeout-minutes: 30 + run: | + chown -R chevah pythia + cd pythia + su chevah -c "./astral-to-pythia.sh linux-x64" + + - name: Test Pythia + timeout-minutes: 5 + run: | + cd pythia + su chevah -c "./test.sh linux-x64" + + - name: Test compat + timeout-minutes: 10 + env: + USER: chevah + CODECOV_TOKEN: local + run: | + cd pythia + # FIXME:72: + # Re-enable compat tests. + true su chevah -c "./test_compat.sh" + + # Using `~/` is problematic under Docker, use `/root/`. + # Remove key in same step to avoid leaving it on disk if publishing fails. + - name: Upload testing package + timeout-minutes: 5 + run: | + mkdir -pv /root/.ssh/ + cd pythia + touch priv_key + chmod 600 priv_key + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > /root/.ssh/known_hosts + ./publish_dist.sh ; rm priv_key + + # If one of the above steps fails, fire up tmate for remote debugging. + # Not all platforms are supported by tmate. + # Currently not supported: Alpine, Amazon 2, CentOS 5 (no glibc or too old). + - name: Tmate debug on failure + if: ${{ !cancelled() && runner.debug }} + uses: chevah/action-tmate@v3 + with: + sudo: false + limit-access-to-actor: true + + + linux-arm64: + runs-on: ubuntu-24.04-arm + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + # Amazon Linux 2 has glibc 2.26. + container: [ 'amazonlinux:2' ] + timeout-minutes: 60 + steps: + + - name: Amazon setup + run: | + yum -y upgrade + yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu bzip2 sudo + # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. + # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. + rm -v /usr/include/libdb/db.h + rm -v /usr/include/gdbm.h + # This is for avoiding building the NIS module with Python 3.12. + rm -v /usr/include/rpc/rpc.h + + # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. + - name: Clone repo independently + run: | + git clone https://github.com/chevah/pythia.git + cd pythia + git checkout ${GITHUB_HEAD_REF} + + - name: Chevah user setup + run: | + useradd -g adm -s /bin/bash -m chevah + echo '%adm ALL=NOPASSWD: ALL' > /etc/sudoers + + - name: Build Pythia + timeout-minutes: 30 + run: | + chown -R chevah pythia + cd pythia + su chevah -c "./astral-to-pythia.sh linux-arm64" + + - name: Test Pythia + timeout-minutes: 5 + run: | + cd pythia + su chevah -c "./test.sh linux-arm64" + + - name: Test compat + timeout-minutes: 10 + env: + USER: chevah + CODECOV_TOKEN: local + run: | + cd pythia + # FIXME:72: + # Re-enable compat tests. + true su chevah -c "./test_compat.sh" + + # Using `~/` is problematic under Docker, use `/root/`. + # Remove key in same step to avoid leaving it on disk if publishing fails. + - name: Upload testing package + timeout-minutes: 5 + run: | + mkdir -pv /root/.ssh/ + cd pythia + touch priv_key + chmod 600 priv_key + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > /root/.ssh/known_hosts + ./publish_dist.sh ; rm priv_key + + # If one of the above steps fails, fire up tmate for remote debugging. + # Not all platforms are supported by tmate. + # Currently not supported: Alpine, Amazon 2, CentOS 5 (no glibc or too old). + - name: Tmate debug on failure + if: ${{ !cancelled() && runner.debug }} + uses: chevah/action-tmate@v3 + with: + sudo: false + limit-access-to-actor: true + + + windows: + # The type of runner that the job will run on + runs-on: windows-latest + timeout-minutes: 45 + + permissions: + # Give the default GITHUB_TOKEN write permission to commit requirements.txt + contents: write + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # Explicitly run our scripts with Bash, not PowerShell (GitHub's default). + - name: Build Pythia + timeout-minutes: 10 + run: bash ./astral-to-pythia.sh windows-x64 + + - name: Test Pythia + timeout-minutes: 5 + run: bash ./test.sh windows-x64 + + - name: Test compat + timeout-minutes: 10 + env: + USER: runneradmin + CODECOV_TOKEN: local + # FIXME:72: + # Re-enable compat tests. + run: true bash ./test_compat.sh + + # Commit changed requirements.txt back to the repository + - uses: chevah/git-auto-commit-action@HEAD + with: + commit_message: Automated update of requirements.txt from Windows build. + file_pattern: 'requirements.txt' + + # To use an RSA key with SFTPPlus, install upstream OpenSSH package, + # which is more finicky in regards to file permissions. + # Beware the commands in this step run under PowerShell. + - name: Prepare SFTP upload + run: | + mkdir -p ~/.ssh/ + touch priv_key + icacls .\priv_key /inheritance:r + icacls .\priv_key /grant:r runneradmin:"(F)" + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts + choco install --yes --no-progress openssh + + # Upload using a (per-OS selected) sftp command, then show final links. + # Remove key in same step to avoid leaving it on disk if publishing fails. + - name: Upload testing package + timeout-minutes: 5 + run: bash -c './publish_dist.sh ; rm priv_key' + + - name: Tmate debug on failure + if: ${{ !cancelled() && runner.debug }} + uses: chevah/action-tmate@v3 + with: + limit-access-to-actor: true + + # To access the Windows desktop for advanced debugging, as per + # https://github.com/nelsonjchen/reverse-rdp-windows-github-actions, + # but using the ngrok token as password for the runnneradmin user. + # Basically use the ngrok token and the ngrok URL (from ngrok's dashboard). + - name: Ngrok debugging on failure + if: ${{ failure() && !cancelled() && runner.debug }} + env: + NGROK_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} + run: | + choco install --yes --no-progress ngrok + ngrok.exe authtoken $env:NGROK_TOKEN + Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server'-name "fDenyTSConnections" -Value 0 + Enable-NetFirewallRule -DisplayGroup "Remote Desktop" + Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 1 + Set-LocalUser -Name "runneradmin" -Password (ConvertTo-SecureString -AsPlainText "$env:NGROK_TOKEN" -Force) + ngrok.exe tcp 3389 + + + macos: + # For macOS on Intel at GitHub. + runs-on: macos-13 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # Some Homebrew libs pollute the build. + # Also make sure binaries depending on these libs are out of the way. + - name: Hack Homebrew + run: | + sudo mv -v /usr/local/bin/git{,.saved} + sudo chmod -v a-r /usr/local/include/libintl.h + sudo chmod -v a-r /usr/local/include/gdbm.h + sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* + sudo chmod -v a-r /usr/local/opt/gdbm/lib/libgdbm* + sudo chmod -v a-r /usr/local/opt/mpdecimal/lib/libmpdec* + + - name: Build Pythia + timeout-minutes: 30 + run: ./astral-to-pythia.sh macos-x64 + + # Fix back Homebrew, for working Shellcheck tests and tmate debugging. + - name: Unhack Homebrew + run: | + sudo chmod -v a+r /usr/local/include/libintl.h + sudo chmod -v a-r /usr/local/include/gdbm.h + sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl* + sudo chmod -v a+r /usr/local/opt/gdbm/lib/libgdbm* + sudo chmod -v a+r /usr/local/opt/mpdecimal/lib/libmpdec* + sudo mv -v /usr/local/bin/git{.saved,} + + - name: Test Pythia + timeout-minutes: 5 + run: ./test.sh macos-x64 + + - name: Test compat + timeout-minutes: 10 + env: + CODECOV_TOKEN: local + # FIXME:72: + # Re-enable compat tests. + run: true ./test_compat.sh + + - name: Upload testing package + timeout-minutes: 5 + run: | + mkdir -pv ~/.ssh/ + touch priv_key + chmod 600 priv_key + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts + ./publish_dist.sh ; rm priv_key + + - name: Tmate debug on failure + if: ${{ !cancelled() && runner.debug }} + uses: chevah/action-tmate@v3 + with: + limit-access-to-actor: true + + + macos-m1: + # For macOS on Apple Silicon at GitHub. + runs-on: macos-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Build Pythia + timeout-minutes: 20 + run: ./astral-to-pythia.sh macos-arm64 + + - name: Test Pythia + timeout-minutes: 5 + run: ./test.sh macos-arm64 + + - name: Test compat + timeout-minutes: 10 + env: + CODECOV_TOKEN: local + # FIXME:72: + # Re-enable compat tests. + run: true ./test_compat.sh + + - name: Upload testing package + timeout-minutes: 5 + run: | + mkdir -pv ~/.ssh/ + touch priv_key + chmod 600 priv_key + echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key + echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts + ./publish_dist.sh ; rm priv_key + + - name: Tmate debug on failure + if: ${{ !cancelled() && runner.debug }} + uses: chevah/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml deleted file mode 100644 index d5a27a21..00000000 --- a/.github/workflows/docker.yaml +++ /dev/null @@ -1,203 +0,0 @@ -# -# GitHub actions for building and testing. -# -# For best support, use `-latest` for runners spinning up containers. More at -# https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners. - -name: Docker - -on: - push: - branches: [ master ] - pull_request: - -concurrency: - group: docker-${{ github.ref }} - cancel-in-progress: true - -env: - CI: 'true' - -# Using a job name that doesn't contain the OS name, to minimize the risk of -# confusion with the OS names of the containers, which are the relevant ones. -jobs: - linux: - runs-on: ubuntu-latest - container: ${{ matrix.container }} - strategy: - fail-fast: false - matrix: - # Alpine Linux 3.15 has musl 1.2.2, Amazon Linux 2 has glibc 2.26. - container: [ 'alpine:3.15', 'amazonlinux:2' ] - timeout-minutes: 60 - steps: - - # OpenSSL gets updated by apk, but that is the Alpine way, so it's fine. - - name: Alpine setup - if: startsWith(matrix.container, 'alpine') - run: | - apk upgrade -U - apk add git curl bash gcc make m4 patch musl-dev linux-headers lddtree shadow openssh-client file unzip perl g++ musl-locales dejagnu sudo - apk del util-linux-dev - curl --output /usr/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 - chmod +x /usr/bin/paxctl - - - name: Amazon setup - if: startsWith(matrix.container, 'amazonlinux') - run: | - yum -y upgrade - yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu bzip2 sudo - # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. - # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. - rm -v /usr/include/libdb/db.h - rm -v /usr/include/gdbm.h - # This is for avoiding building the NIS module with Python 3.12. - rm -v /usr/include/rpc/rpc.h - - - name: Ubuntu setup - if: startsWith(matrix.container, 'ubuntu') - run: | - apt update - apt --yes dist-upgrade - apt --yes install wget curl gcc make m4 patch openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu sudo - - # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - - name: Clone repo independently - run: | - git clone https://github.com/chevah/pythia.git - cd pythia - git checkout ${GITHUB_HEAD_REF} - - - name: Chevah user setup - run: | - useradd -g adm -s /bin/bash -m chevah - echo '%adm ALL=NOPASSWD: ALL' > /etc/sudoers - - - name: Build Pythia - timeout-minutes: 30 - run: | - chown -R chevah pythia - cd pythia - su chevah -c "./build.sh build" - - - name: Test Pythia - timeout-minutes: 5 - run: | - cd pythia - su chevah -c "./build.sh test" - - - name: Test compat - timeout-minutes: 10 - env: - USER: chevah - CODECOV_TOKEN: local - run: | - cd pythia - # FIXME:72: - # Re-enable compat tests. - true su chevah -c "./build.sh compat" - - # Using `~/` is problematic under Docker, use `/root/`. - # Remove key in same step to avoid leaving it on disk if publishing fails. - - name: Upload testing package - timeout-minutes: 5 - run: | - mkdir -pv /root/.ssh/ - cd pythia - touch priv_key - chmod 600 priv_key - echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key - echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > /root/.ssh/known_hosts - ./publish_dist.sh ; rm priv_key - - # If one of the above steps fails, fire up tmate for remote debugging. - # Not all platforms are supported by tmate. - # Currently not supported: Alpine, Amazon 2, CentOS 5 (no glibc or too old). - - name: Tmate debug on failure - if: ${{ !cancelled() && runner.debug }} - uses: chevah/action-tmate@v3 - with: - sudo: false - limit-access-to-actor: true - - - linux-arm64: - runs-on: ubuntu-24.04-arm - container: ${{ matrix.container }} - strategy: - fail-fast: false - matrix: - # Amazon Linux 2 has glibc 2.26. - container: [ 'amazonlinux:2' ] - timeout-minutes: 60 - steps: - - - name: Amazon setup - run: | - yum -y upgrade - yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu bzip2 sudo - # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. - # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. - rm -v /usr/include/libdb/db.h - rm -v /usr/include/gdbm.h - # This is for avoiding building the NIS module with Python 3.12. - rm -v /usr/include/rpc/rpc.h - - # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - - name: Clone repo independently - run: | - git clone https://github.com/chevah/pythia.git - cd pythia - git checkout ${GITHUB_HEAD_REF} - - - name: Chevah user setup - run: | - useradd -g adm -s /bin/bash -m chevah - echo '%adm ALL=NOPASSWD: ALL' > /etc/sudoers - - - name: Build Pythia - timeout-minutes: 30 - run: | - chown -R chevah pythia - cd pythia - su chevah -c "./build.sh build" - - - name: Test Pythia - timeout-minutes: 5 - run: | - cd pythia - su chevah -c "./build.sh test" - - - name: Test compat - timeout-minutes: 10 - env: - USER: chevah - CODECOV_TOKEN: local - run: | - cd pythia - # FIXME:72: - # Re-enable compat tests. - true su chevah -c "./build.sh compat" - - # Using `~/` is problematic under Docker, use `/root/`. - # Remove key in same step to avoid leaving it on disk if publishing fails. - - name: Upload testing package - timeout-minutes: 5 - run: | - mkdir -pv /root/.ssh/ - cd pythia - touch priv_key - chmod 600 priv_key - echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key - echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > /root/.ssh/known_hosts - ./publish_dist.sh ; rm priv_key - - # If one of the above steps fails, fire up tmate for remote debugging. - # Not all platforms are supported by tmate. - # Currently not supported: Alpine, Amazon 2, CentOS 5 (no glibc or too old). - - name: Tmate debug on failure - if: ${{ !cancelled() && runner.debug }} - uses: chevah/action-tmate@v3 - with: - sudo: false - limit-access-to-actor: true diff --git a/README.rst b/README.rst index 469165c0..43bd6bf6 100644 --- a/README.rst +++ b/README.rst @@ -1,21 +1,33 @@ Pythia - a portable Python package ================================== +.. image:: https://img.shields.io/badge/License-MIT-yellow.svg + :target: https://opensource.org/licenses/MIT + +.. image:: https://github.com/chevah/pythia/workflows/ci/badge.svg + :target: https://github.com/chevah/pythia/actions + +.. image:: https://img.shields.io/github/issues/chevah/pythia.svg + :target: https://github.com/chevah/pythia/issues + + Build system for a portable Python package. -A derivative of https://github.com/chevah/python-package/. -Building: +It is based on `astral-sh/python-build-standalone `_ -* ``./build.sh build`` +It builds on top of `python-build-standalone` by allowing you to create a local Python virtual environment without the need to use `uv` -Testing: +The `pythia.sh` script can automatically download and create the Python venv. -* ``./build.sh test`` -* ``./build.sh compat`` +It is designed to be used for self-contained applications that are distributed together with Python. -Use ``./build.sh help`` to discover all available commands. +It is also designed for applications that are packaged only Linux GLIBC, +with Linux MUSL, macOS and Windows installation packages also generated on Linux. -Note that compat tests are currently only working on the ``python2.7`` branch. +It comes with a few pre-installed packages, depending on the OS. +This is done to simplify building the final distributable package on Linux, +regardless of the target OS. +Example on Windows, pywin32 is preinstaleld. Supported platforms @@ -33,46 +45,34 @@ Platforms that should work, but are not regularly tested: * all glibc-based Linux distributions (glibc 2.26+) * all musl-based Linux distributions (musl 1.1.24+). -Platforms built in the past, but not any more: - -* FreeBSD 12 and newer -* OpenBSD 6.7 and newer -* Solaris 11.4 (x86 only). - Where not noted, supported architecture is x64 (also known as X86-64 or AMD64). -Note that https://github.com/chevah/python-package/ supported more platforms, -but only for Python 2.7. +Usage +----- -Patching upstream code ----------------------- +Copy `pythia.sh` script into your repo. +Copy `pythia.conf` file into your repo and configure your mirrors and targeted Python version. -This repository contains some patches for upstream code, e.g. Python and bzip2. +It is designed to create the Python virtual environment and automatically call paver inside the newly created environment. +`pythia.conf` contains a set of Python packages that needs to be intallled to boostrap the `paver` usage. -These patches are applied at build time when added as: -* ``src/$PROJECT/*.patch`` +Development +----------- -An example for creating a patch for pristine Python 3.11.3 sources:: +It is designed to download the Python .tar.gz from `python-build-standalone` GitHub release page, convert it into a format usable by `pythia.sh` +and upload the resulting tar.gz into a download/mirror server. - # Make a copy of the file to be patched from Python sources: - cp setup.py setup.py.orig - # Modify it as needed, then check the diff: - diff -ru setup.py.orig setup.py - # Save the diff into a file such as: - src/Python/disabled_modules.patch +It has support for uploading into testing vs production. -Finally, edit the corresponding ``chevahbs`` script in ``/src`` to apply -the new patch on platforms that require it before building from sources. -When applying a patch on top of another patch, make sure you get the order -right, then save the diff to the sources patched with the preceding patch. +Building: -.. image:: https://img.shields.io/badge/License-MIT-yellow.svg - :target: https://opensource.org/licenses/MIT +* `./astral-to-pytia.sh TARGET_ARCH` -.. image:: https://github.com/chevah/pythia/workflows/GitHub-CI/badge.svg - :target: https://github.com/chevah/pythia/actions +TARGET_ARCH needs to match the local arch. -.. image:: https://img.shields.io/github/issues/chevah/pythia.svg - :target: https://github.com/chevah/pythia/issues +Testing: + +* `./test_pythia.sh` +* `./test_compat.sh` diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh new file mode 100755 index 00000000..dca5fa92 --- /dev/null +++ b/astral-to-pythia.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Script to generate Python virtual environments compatible with pythia +# based on the astra-sh/python-build-standalone Python builds. + +set -o errexit + +# linux-x64 | linux-arm64 | linux_musl-x64 | macos-x64 | macos-arm64 +# windows-x64 +TARGET="$1" + +source build.conf + +# x86_64_v2-* +# Targets 64-bit Intel/AMD CPUs approximately newer than Nehalem (released in 2008). +# Binaries will have SSE3, SSE4, and other CPU instructions added after the ~initial x86-64 CPUs were launched in 2003. + +LINUX_X64="x86_64_v2-unknown-linux-gnu" +LINUX_ARM64="aarch64-unknown-linux-gnu" +LINUX_MUSL_X64="x86_64_v2-unknown-linux-musl" +MACOS_ARM64="aarch64-apple-darwin" +MACOS_X64="x86_64-apple-darwin" +WINDOWS_X64="x86_64-pc-windows-msvc" + +# Start with a clean build +#rm -rf "$BUILD_DIR" +#mkdir "$BUILD_DIR" + +rm -rf "$DIST_DIR" +mkdir "$DIST_DIR" + +cd $BUILD_DIR + +case "$TARGET" in + linux-x64) + pbs_arch=${LINUX_X64} + ;; + linux-arm64) + pbs_arch=${LINUX_ARM64} + ;; + linux_musl-x64) + pbs_arch="${LINUX_MUSL_X64}" + ;; + macos-x64) + pbs_arch="${MACOS_X64}" + ;; + macos-arm64) + pbs_arch="${MACOS_ARM64}" + ;; + windows-x64) + pbs_arch="${WINDOWS_X64}" + ;; + *) + # + echo "Unknown target ${file_name}" + exit 1 + + ;; +esac + + +file_name="${pbs_arch}-install_only_stripped.tar.gz" +source_url="https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-${PY_RELEASE}+${PBS_RELEASE}-${file_name}" +echo "> Downloading $file_name" +curl -L -o $file_name $source_url + +echo "> Extracting $file_name" + +# Delete any leftover from a previous failed extraction. +# Used during dev when the whole build folder is not cleaned. +rm -rf python + + +tar xzf $file_name + + +pythia_version="${PY_RELEASE}.${PBS_RELEASE}-${pythia_arch}" + +case "$TARGET" in + *windows*) + # Do stuff + mv python lib + mkdir python + mv lib python + rm -rf python/lib/include + python_bin="python/lib/python" + ;; + *) + # Linux and MacOS + mv python/include python/lib + rm -rf python/share + rm -rf python/lib/itcl* + rm -rf python/lib/tcl* + rm -rf python/lib/tk* + rm -rf python/lib/thread* + rm -f python/lib/libtcl* + rm -rf python/lib/python${PY_VERSION}/tkinter + rm -rf python/lib/python${PY_VERSION}/turtledemo + python_bin="python/bin/python" + ;; +esac + +# Compile all source files to speed up load time and write Pythia version +"${python_bin}" -m compileall -r 100 python +echo -n "${pythia_version}" > python/lib/PYTHIA_VERSION + +pythia_base="python${PY_VERSION}-${TARGET}" +mv python ${pythia_base} +echo "> Creating ${pythia_base} to ../dist/" + +tar czf ../dist/python-${PYTHIA_RELEASE}-${TARGET}.tar.gz ${pythia_base} diff --git a/build.conf b/build.conf index 861ef967..e93ce12f 100644 --- a/build.conf +++ b/build.conf @@ -6,68 +6,23 @@ BUILD_DIR="build" # This is also defined independently in "publish_dist.sh". DIST_DIR="dist" -# Setting this as 0 disables tests during building (skips chevahbs_test phase). -PYTHIA_BUILD_TESTS="${PYTHIA_BUILD_TESTS:-1}" +UPLOAD_SERVER="bin.chevah.com" +UPLOAD_USER="github-upload" +# testing | production +TARGET_DEPLOY='testing' -# Set versions for Python and its libraries. -# For major Python updates, e.g. 3.11->3.12, also update pythia.conf and -# src/Python-Windows/chevahbs (e.g. "python311._pth"->"python312._pth"). -PYTHON_BUILD_VERSION="3.13.2" -LIBFFI_VERSION="3.4.6" -ZLIB_VERSION="1.3.1" -BZIP2_VERSION="1.0.8" -# To check the signature of the XZ download: -# "gpg --keyserver-options auto-key-retrieve --verify xz-*.sig xz-*.gz". -XZ_VERSION="5.6.4" -# Statically build the BSD libedit on selected platforms to get the -# readline module available without linking to the GPL-only readline libraries. -# If there's a need to reenable this, our libedit patch for Python 3.9 was -# https://github.com/chevah/pythia/pull/5/commits/09c128154d23feb6b1a7cb5a8d79. -# A newer patch is available at https://github.com/python/cpython/issues/57710. -LIBEDIT_VERSION="20170329-3.1" -# Our OpenSSL libraries are only used for Python's "ssl" module lately. -OPENSSL_VERSION="3.0.16" -# Use the version of the "sqlite-autoconf-VERSION.tar.gz" upstream download. -# To get its SHA3-256 signature: "openssl dgst -sha3-256 sqlite-autoconf-*". -# When updating this, also UPDATE THE YEAR in src/sqlite/chevahbs, if needed. -SQLITE_VERSION="3460000" - -# Where ensurepip is not avaiable, this is used to bootstrap pip. -# If latest get-pip.py stops working on older platforms, use something like: -# https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/ -BOOTSTRAP_GET_PIP="https://bootstrap.pypa.io/get-pip.py" - -# Python modules installed after bootstraping pip. -PIP_VERSION="25.0" -SETUPTOOLS_VERSION="75.8.0" -# pycparser is explicitly installed to work around setuptools auto dependencies. -PYCPARSER_VERSION="2.22" +PY_VERSION="3.12" +PY_RELEASE="3.12.13" +PBS_RELEASE="20260408" +PYTHIA_RELEASE="${PY_RELEASE}.${PBS_RELEASE}" # Python modules that have to be built and/or installed in Pythia. +# WARGNING: Also update requirement.txt when this is updated. PYWIN32_VERSION="308" -# To be removed when upstream builds a musl wheel for psutil. -# More at https://github.com/giampaolo/psutil/pull/2126. -PSUTIL_VERSION="6.1.1" - -# Global flags for building required libraries. -BUILD_LIBFFI="no" -BUILD_ZLIB="no" -BUILD_BZIP2="yes" -BUILD_XZ="no" -BUILD_LIBEDIT="no" -BUILD_OPENSSL="yes" -BUILD_SQLITE="yes" # Our custom PyPI server overwrites the default pypi.org option from "build.sh". PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple" -# Array of *default* Python modules installed using pip. -# This can be extended for some OS'es (check "os_quirks.sh" for details). -# Binary-only wheels should be installed when needed, not through Pythia. -# Still building psutil on musl Linux, that's why it's present here. -PIP_LIBRARIES=(\ - psutil=="$PSUTIL_VERSION" \ - ) # Array of default pip options. PIP_ARGS=(\ --index-url="$PIP_INDEX_URL" \ diff --git a/build.sh b/build.sh deleted file mode 100755 index 0f152f08..00000000 --- a/build.sh +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env bash -# -# Pythia's script for building Python. - - -# Bash checks -set -o nounset # always check if variables exist -set -o errexit # always exit on error -set -o errtrace # trap errors in functions as well -set -o pipefail # don't ignore exit codes when piping output -set -o functrace # inherit DEBUG and RETURN traps - -# Default PyPI server to use. Can be overwritten in build.conf. -PIP_INDEX_URL="https://pypi.org/simple" - -# Set versions for the software to be built and other defaults. -source build.conf - -# Import shared and specific code. -source ./functions.sh -source ./functions_build.sh - -# Git revision to inject into Python's sys.version string through chevahbs -# on non-Windows platforms. Also used for compat tests and archived in the dist. -PYTHIA_VERSION="$(git log -n 1 --no-merges --pretty=format:%h)" -exit_on_error $? 250 - -# Export the variables needed by the chevahbs scripts and the test phase. -export PYTHON_BUILD_VERSION PYTHIA_VERSION PYTHIA_BUILD_TESTS -export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL - -# OS detection is done by the common pythia.sh. The values are saved in a file. -if [ ! -s BUILD_ENV_VARS ]; then - execute ./pythia.sh detect_os -fi -source BUILD_ENV_VARS - -# On Unix, use $ARCH to choose between 32bit or 64bit packages. It's possible -# to force a 32bit build on a 64bit machine, e.g. by setting ARCH in pythia.sh -# as "x86" instead of "x64" for a certain platform. -# $ARCH is also used when "building" Python on Windows and for testing. -# $OS is used when patching/configuring/building/testing. -export ARCH OS - -# Local variables for the build process. -PYTHON_BUILD_DIR="$PYTHON_VERSION-$OS-$ARCH" -INSTALL_DIR="$PWD/$BUILD_DIR/$PYTHON_BUILD_DIR" -PYTHON_BIN="$INSTALL_DIR/bin/$PYTHON_VERSION" - -# Explicitly choose the C compiler in order to make it possible to switch -# between native compilers and GCC on platforms such as the BSDs and Solaris. -export CC="${CC:-gcc}" -# Used for testing Python C++ extensions (test_cppext). -export CXX="${CXX:-g++}" -# Other needed tools (GNU flavours preferred). -# For proper quoting, _CMD vars are Bash arrays of commands and optional flags. -MAKE_CMD=(make) -SHA_CMD=(sha512sum --check --status --warn) -TAR_CMD=(tar xfz) -ZIP_CMD=(unzip -q) -# $GET_CMD must save to custom filename, which must be appended before the link. -# E.g., to use wget, GET_CMD should be (wget --quiet -O). -GET_CMD=(curl --silent --location --output) - -# OS quirks. Sourced last to allow overwriting the above variables. -source os_quirks.sh - - -# shellcheck disable=SC2034 # Only used through compgen. -help_text_clean="Clean build dir. Add -a to remove downloads and saved values." -command_clean() { - echo "#### Removing previous build sub-directory, if existing... ####" - execute rm -rf "$BUILD_DIR" - - if [ $# -ne 0 ]; then - if [ "$1" = "-a" ]; then - echo "## Removing all downloads from src/... ##" - execute rm -fv src/*/*.{tar.gz,tgz,zip} - echo "## Removing all local files with saved values... ##" - execute rm -fv BUILD_ENV_VARS BUILD_ENV_ARRAYS - fi - fi -} - - -# shellcheck disable=SC2034 # Only used through compgen. -help_text_build="Build Python binaries for current platform." -command_build() { - echo "::group::Package/command checks" - # Check for packages required to build on current OS. - echo "#### Checking for required packages... ####" - source pkg_checks.sh - echo "::endgroup::" - - # Clean build dir to avoid contamination from previous builds, - # but without removing the download archives, to speed up next builds. - command_clean "$@" - - # Build stuff statically on most platforms, install headers and libs in the - # following locations, making sure they are picked up when building Python. - execute mkdir -p "$INSTALL_DIR"/{include,lib} - export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS:-}" - export PKG_CONFIG_PATH="$INSTALL_DIR/lib/pkgconfig/:${PKG_CONFIG_PATH:-}" - # On certain OS'es, some modules require this (zlib, bz2, lzma, sqlite3). - export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include" - - build_dep "$BUILD_LIBFFI" libffi "$LIBFFI_VERSION" - build_dep "$BUILD_ZLIB" zlib "$ZLIB_VERSION" - build_dep "$BUILD_BZIP2" bzip2 "$BZIP2_VERSION" - build_dep "$BUILD_XZ" xz "$XZ_VERSION" - build_dep "$BUILD_LIBEDIT" libedit "$LIBEDIT_VERSION" - build_dep "$BUILD_SQLITE" sqlite-autoconf "$SQLITE_VERSION" - build_dep "$BUILD_OPENSSL" openssl "$OPENSSL_VERSION" - - build_python - - # Python modules installed w/ pip. Some are built locally (not on Windows). - command_install_python_modules - - # Cleanups the dir to be packaged, also moves include/ from the root dir. - cleanup_install_dir - - # Build the new package. - make_dist "$PYTHON_BUILD_DIR" - - # Generate a SFTP batch for uploading the package. - build_publish_dist_sftp_batch - - # Put include/ back where it belongs, for building testing modules. - bring_back_include -} - - -# This builds Python's dependencies: libffi, bzip2, openssl, etc. -build_dep() { - local dep_boolean="$1" - local dep_name="$2" - local dep_version="$3" - - if [ "$dep_boolean" = "yes" ]; then - # This is where building happens. - build "$dep_name" "$dep_version" - # If there's something to be done post-build, here's the place. - elif [ "$dep_boolean" = "no" ]; then - (>&2 echo -e "\tSkip building $dep_name") - else - (>&2 echo "Unknown env var for building $dep_name. Exiting!") - exit 248 - fi -} - - -# This builds Python itself. -build_python() { - if [ "$OS" = "windows" ]; then - # Python "build" is a very special case under Windows. - execute pushd src/Python-Windows - execute ./chevahbs Python "$PYTHON_BUILD_VERSION" "$INSTALL_DIR" - execute popd - else - build Python "$PYTHON_BUILD_VERSION" - fi -} - -bootstrap_pip(){ - echo "### Bootstrapping pip... ###" - if [ "$OS" = "windows" ]; then - # The embeddable Windows package doesn't include "ensurepip". - echo "## Downloading get-pip.py... ##" - if [ ! -e "$BUILD_DIR"/get-pip.py ]; then - execute "${GET_CMD[@]}" "$BUILD_DIR"/get-pip.py "$BOOTSTRAP_GET_PIP" - fi - execute "$PYTHON_BIN" "$BUILD_DIR"/get-pip.py "${PIP_ARGS[@]}" \ - pip=="$PIP_VERSION" --no-setuptools \ - setuptools=="$SETUPTOOLS_VERSION" - else - echo "## Installing pip from included ensurepip module... ##" - execute "$PYTHON_BIN" -m ensurepip --upgrade - fi -} -# Compile and install all Python extra libraries. -command_install_python_modules() { - echo "::group::Install Python modules with pip $PIP_VERSION" - echo "#### Installing Python modules... ####" - - # Install latest PIP, then instruct it to get exact version of setuptools. - bootstrap_pip - echo "# Installing latest pip with preferred setuptools version... #" - execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" \ - pip=="$PIP_VERSION" setuptools=="$SETUPTOOLS_VERSION" - - # pycparser is installed first as setup_requires is ugly. - # https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires - echo "# Installing pycparser with preferred setuptools version... #" - execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" \ - -U pycparser=="$PYCPARSER_VERSION" - - if [ "$OS" = "windows" ]; then - echo -e "\tSkip makefile updating on Windows" - else - echo "# Updating Python config Makefile for newly-built Python... #" - makefile="$(ls "$INSTALL_DIR"/lib/"$PYTHON_VERSION"/config*/Makefile)" - makefile_orig="$makefile".orig - - execute cp "$makefile" "$makefile_orig" - execute sed "s#^prefix=.*#prefix= $INSTALL_DIR#" "$makefile_orig" \ - > "$makefile" - fi - - for library in "${PIP_LIBRARIES[@]}" ; do - execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" "$library" - done - - echo "# Uninstalling wheel... #" - execute "$PYTHON_BIN" -m pip uninstall --yes wheel - - echo "# Regenerating requirements.txt file... #" - execute "$PYTHON_BIN" -m pip freeze --all > requirements.txt - - echo "::endgroup::" -} - - -# shellcheck disable=SC2034 # Only used through compgen. -help_text_test="Run own tests for the newly-build Python distribution." -command_test() { - local test_file="test_python_binary_dist.py" - local python_binary="$PYTHON_BIN" - - echo "::group::Chevah tests" - if [ ! -d "$BUILD_DIR" ]; then - (>&2 echo "No $BUILD_DIR sub-directory present, try 'build' first!") - exit 220 - fi - - echo "#### Executing Chevah Python tests... ####" - if [ "$OS" != "windows" ]; then - # Post-cleanup, the binary in /bin is named "python", not "python3.x". - local python_binary="$INSTALL_DIR/bin/python" - fi - execute cp src/chevah-python-tests/"$test_file" "$BUILD_DIR" - execute cp src/chevah-python-tests/get_binaries_deps.sh "$BUILD_DIR" - execute pushd "$BUILD_DIR" - execute "$python_binary" "$test_file" - execute popd - echo "::endgroup::" - - echo "::group::Security tests" - echo "## Testing for outdated packages... ##" - execute "$python_binary" -m pip list --outdated --format=columns - echo "::endgroup::" - - echo "::group::Shell tests" - echo "#### Executing Chevah shell tests... ####" - execute ./src/chevah-bash-tests/shellcheck_tests.sh - echo "::endgroup::" -} - - -# shellcheck disable=SC2034 # Only used through compgen. -help_text_compat="Run the test suite from chevah/compat master." -command_compat() { - local new_python_ver="$PYTHON_BUILD_VERSION.$PYTHIA_VERSION" - execute pushd "$BUILD_DIR" - - # This is quite hackish, as compat is arm-twisted to use the local version. - echo "::group::Compat tests" - echo "#### Running chevah's compat tests... ####" - echo "## Removing any pre-existing compat code... ##" - execute rm -rf compat/ - echo "## Cloning compat's master branch... ##" - execute git clone https://github.com/chevah/compat.git --depth=1 -b master - execute pushd compat - # Make sure everything is done from scratch in the current dir. - echo "## Unsetting CHEVAH_CACHE and CHEVAH_BUILD... ##" - unset CHEVAH_CACHE CHEVAH_BUILD - # Copy over current pythia stuff, as some changes might require it. - echo "## Patching compat code to use current pythia version... ##" - execute cp ../../pythia.{conf,sh} ./ - # Patch compat to use the current's branch version. - echo -e "\nPYTHON_CONFIGURATION=default@${new_python_ver}" >>pythia.conf - execute mkdir cache - # Copy dist file to local cache, if existing. If not, maybe it's online. - cp ../../"$DIST_DIR"/"$PYTHON_BUILD_VERSION.$PYTHIA_VERSION"/* cache/ - # Some tests could fail due to causes not related to the new Python. - echo "## Getting compat deps... ##" - execute ./pythia.sh deps - echo "## Running normal compat tests... ##" - # Why not test_normal? See https://github.com/chevah/compat/issues/691. - execute ./pythia.sh test -vs normal - if [ "${CI:-}" = "true" ]; then - echo "## Running ci2 compat tests... ##" - execute ./pythia.sh test_ci2 - fi - execute popd - echo "::endgroup::" -} - - - -# -# Launch the whole thing. -# - -# Bash arrays are not exported to child processes. More details at -# https://www.mail-archive.com/bug-bash@gnu.org/msg01774.html. -# Therefore, pass _CMD stuff to the "chevahbs" scripts in a file to be sourced. -# The unusual quoting avoids mixing strings and arrays. -# Indentation helps when showing the arrays during debugging. -( - echo -e "\tMAKE_CMD=(" "${MAKE_CMD[@]}" ")" - echo -e "\tGET_CMD=(" "${GET_CMD[@]}" ")" - echo -e "\tSHA_CMD=(" "${SHA_CMD[@]}" ")" - echo -e "\tTAR_CMD=(" "${TAR_CMD[@]}" ")" - echo -e "\tZIP_CMD=(" "${ZIP_CMD[@]}" ")" -) > BUILD_ENV_ARRAYS - -if [ "$DEBUG" -ne 0 ]; then - build_flags=(OS ARCH CC CFLAGS BUILD_LIBFFI BUILD_ZLIB BUILD_BZIP2 \ - BUILD_XZ BUILD_LIBEDIT BUILD_OPENSSL BUILD_SQLITE) - echo -e "Build variables:" - for build_var in "${build_flags[@]}"; do - # This uses Bash's indirect expansion. - echo -e "\t$build_var: ${!build_var}" - done - echo "Bash arrays to import in chevahbs scripts:" - cat BUILD_ENV_ARRAYS -fi - -select_command "$@" diff --git a/functions.sh b/functions.sh deleted file mode 100644 index a467f01d..00000000 --- a/functions.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# -# Shared code for all binary-dist scripts. -# - -# Check if debugging environment variable is set and initialize with 0 if not. -DEBUG="${DEBUG-0}" - -# shellcheck disable=SC2034 # Only used through compgen. -help_text_help="Show help for a command." -command_help() { - local command="${1:-}" - local help_command="help_$command" - # Test for a valid help method, otherwise call general help. - set +o errexit - if type "$help_command" &> /dev/null; then - "$help_command" - else - echo "Available commands are:" - for help_text in $(compgen -A variable help_text_); do - command_name="${help_text#help_text_}" - echo -e " $command_name\t${!help_text}" - done - fi - set -o errexit -} - -# -# Main command selection. -# -# Select functions which are made public. -# -select_command() { - local command="${1:-}" - case "$command" in - "") - command_help - exit 99 - ;; - *) - shift - # Test for a valid command, otherwise call general help. - call_command="command_$command" - set +o errexit - if type "$call_command" &> /dev/null; then - "$call_command" "$@" - else - command_help - (>&2 echo -e "\nUnknown command: $command.") - exit 98 - fi - set -o errexit - ;; - esac -} - - -exit_on_error() { - error_code="$1" - exit_code="$2" - if [ "$error_code" -ne 0 ]; then - exit "$exit_code" - fi -} - - -execute() { - if [ "$DEBUG" -ne 0 ]; then - (>&2 echo -e "\tExecuting:" "$@") - fi - - #Make sure $@ is called in quotes as otherwise it will not work. - "$@" - exit_code="$?" - if [ "$DEBUG" -ne 0 ]; then - (>&2 echo -e "\tExit code was: $exit_code") - fi - if [ "$exit_code" -ne 0 ]; then - (>&2 echo "PWD :" "$(pwd)") - (>&2 echo "Fail:" "$@") - exit 97 - fi -} diff --git a/functions_build.sh b/functions_build.sh deleted file mode 100644 index f3c34fb8..00000000 --- a/functions_build.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env bash -# -# Pythia-specific build functions. - -# Global variables. -COMMAND="" -OS="" -INSTALL_DIR="" - -# -# Chevah Build Script command selection. -# -select_chevahbs_command() { - if [ "$DEBUG" -ne 0 ]; then - echo "select_chevahbs_command:" "$@" - fi - COMMAND="$1" - OS="$2" - INSTALL_DIR="$3" - # Shift first 3 arguments, remaining ones are passed along as $@. - shift 3 - - chevahbs_command="chevahbs_$COMMAND" - if type "$chevahbs_command" &> /dev/null; then - "$chevahbs_command" "$@" - else - (>&2 echo "Don't know what to do with command: $COMMAND.") - exit 90 - fi -} - - -# -# Internal function for downloading sources on-the-fly as needed. -# -download_sources(){ - local project_name="$1" - local project_ver="$2" - local link="$3" - local archive_ext="$4" - # Sometimes the archive is not named $name-$version.$ext, let's force it. - local archive_filename="$project_name-$project_ver.$archive_ext" - # Only download the sources if they are not present. - if [ ! -e "$archive_filename" ]; then - echo "## Downloading $project_name version $project_ver... ##" - execute "${GET_CMD[@]}" "$archive_filename" "$link" - else - echo -e "\t$archive_filename already present, not downloading again." - fi - - echo "## Verifying checksums for $archive_filename... ##" - execute "${SHA_CMD[@]}" sha512.sum - - # Current dir is src/$project_name, and sources have full hierarchy. - # So far, all upstream archives respected the $name-$version convention - # for the name of the root directory in the archive. - case "$archive_ext" in - tar.gz|tgz) - echo "## Unpacking archive $archive_filename... ##" - execute "${TAR_CMD[@]}" "$archive_filename" -C ../../build/ - ;; - zip) - echo "## Unpacking archive $archive_filename... ##" - execute "${ZIP_CMD[@]}" "$archive_filename" -d ../../build/ - ;; - exe|amd64*) - # No need to use ../../build/"$project_name"-"$project_ver"/ here. - echo -e "\tNothing to unpack in build/ for $archive_filename." - ;; - *) - (>&2 echo "Unknown archive type for $archive_filename, exiting!") - exit 89 - ;; - esac -} - - -# -# Internal function for calling build script on each source. -# -chevahbs_build() { - if [ -n "$(type -t chevahbs_patch)" ]; then - # Looks like the chevahbs script has patches to apply. - echo "## Patching... ##" - chevahbs_patch "$@" - fi - echo "## Configuring... ##" - chevahbs_configure "$@" - echo "## Compiling... ##" - chevahbs_compile "$@" -} - -chevahbs_test() { - if [ "${PYTHIA_BUILD_TESTS:-1}" -ne 0 ]; then - chevahbs_try "$@" - else - (>&2 echo "PYTHIA_BUILD_TESTS is set to 0. Skipping build tests!") - fi -} - -chevahbs_install() { - chevahbs_cp "$@" -} - -# -# Build-related stuff. -# -build() { - # First parameter can be "libffi", "zlib", "bzip", "Python", etc. - # It's a sub-dir in src/ containing chevahbs scripts / checksums / patches. - # Also used when downloading the gzipped tarball and unpacking it. - project_name="$1" - # Second parameter has the form: "3.2.1", "1.1.1t", "3410200", etc. - project_ver="$2" - echo "::group::" "$1 $2" "build" - echo "#### Building $1 version $2... ####" - - # This is where sources are unpacked, patched, and built. - version_dir="$1-$2" - own_build_dir="$BUILD_DIR/$version_dir" - - # This is were the builds are installed. - install_dir="$PWD/$BUILD_DIR/$PYTHON_BUILD_DIR" - # Whole build/ sub-dir should be cleared already, let's make sure. - echo "## Removing $own_build_dir if existing... ##" - execute rm -rf "$own_build_dir" - - # Downloads happen in src/ to not get lost when wiping the build/ dir. - echo "## Downloading in src/$project_name... ##" - execute pushd "src/$project_name" - # Go through local project's chevahbs to pick up the link and come back - # in download_sources() to get the sources, check them, and unpack. - execute ./chevahbs getsources "$OS" "$install_dir" \ - "$project_name" "$project_ver" - execute popd - - # The build script is then copied alongide patches to the current build dir. - execute cp src/"$project_name"/chevahbs "$own_build_dir"/ - if [ "$(find src/"$project_name" -name '*.patch' | wc -l)" -gt 0 ]; then - echo "The following patches are to be copied:" - execute ls -1 src/"$project_name"/*.patch - execute cp src/"$project_name"/*.patch "$own_build_dir"/ - fi - - # The actual build happens here. - execute pushd "$own_build_dir" - execute ./chevahbs build "$OS" "$install_dir" "$project_ver" - echo "::endgroup::" - - echo "::group::" "$1 $2" "test" - echo "#### Testing $1 version $2... ####" - execute ./chevahbs test "$OS" - echo "::endgroup::" - - echo "::group::" "$1 $2" "install" - echo "#### Installing $1 version $2... ####" - execute ./chevahbs install "$OS" "$install_dir" - if [ -e "Makefile" ]; then - lib_config_dir="$install_dir/lib/config" - makefile_name="Makefile.$OS.$version_dir" - execute mkdir -p "$lib_config_dir" - execute cp Makefile "$lib_config_dir/$makefile_name" - fi - execute popd - echo "::endgroup::" -} - - -# -# Put stuff where it's expected and remove some of the cruft. -# -cleanup_install_dir() { - local python_lib_file="lib${PYTHON_VERSION}.a" - - echo "::group::Clean up Python install dir" - execute pushd "$BUILD_DIR/$PYTHON_BUILD_DIR" - echo "Cleaning up Python's caches and compiled files..." - find lib/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf - - # Move include/ to lib/include/. - echo "Moving the include/ sub-dir out of the way..." - execute mv include/ lib/ - - case $OS in - windows) - echo -e "\tSkipping further cleaning of install dir" - ;; - *) - execute rm -rf tmp - # Move all binaries to lib/config - execute mkdir -p lib/config - execute mv bin/ lib/config/ - execute mkdir bin - execute pushd lib/config/bin/ - # Move Python binary back as bin/python, then link to it. - execute mv "$PYTHON_VERSION" ../../../bin/python - execute ln -s ../../../bin/python "$PYTHON_VERSION" - # OS-related fixed for the Python binaries. - case "$OS" in - macos) - # The binary is already stripped on macOS. - execute rm python3 - execute ln -s "$PYTHON_VERSION" python3 - ;; - *) - execute strip "$PYTHON_VERSION" - ;; - esac - # Remove the sizable sqlite3 binary. - execute rm sqlite3 - execute popd - # OS-related stripping for libs. - case "$OS" in - macos) - # Darwin's strip command is different. - execute strip -r lib/lib*.a - ;; - *) - execute strip lib/lib*.a - # On CentOS 5, libffi and OpenSSL install to lib64/ - # by default. To have all libs under lib/, required - # files are copied by chevahbs scripts during build. - # Here, make sure there's nothing installed to lib64/. - if [ -d lib64 ]; then - echo "lib64/ sub-dir found!" - exit 88 - fi - ;; - esac - # Symlink the copy of libpython3.*.a too. - execute pushd lib/"$PYTHON_VERSION"/config-* - execute rm "$python_lib_file" - execute ln -s ../../"$python_lib_file" - execute popd - # Remove the big test/ sub-dir. - execute rm -rf lib/"$PYTHON_VERSION"/test/ - # Remove OpenSSL files if present. - execute rm -rf ssl/ - # Remove (mostly OpenSSL) docs and manuals. - execute rm -rf share/ - # Move stray pkgconfig/* to lib/pkgconfig/. - if [ -d pkgconfig ]; then - execute mv pkgconfig/* lib/pkgconfig/ - execute rmdir pkgconfig - fi - # Compress packaged Makefiles to save some space when unpacked. - execute bzip2 lib/config/Makefile* - ;; - esac - # Test that only bin/ and lib/ sub-dirs are left. - for element in *; do - case "$element" in - bin|lib) - true - ;; - *) - echo "Unwanted element in root dir: $element" - exit 97 - ;; - esac - done - execute popd - - # Output Pythia's own version to a dedicated file in the archive. - echo "$PYTHON_BUILD_VERSION.$PYTHIA_VERSION-$OS-$ARCH" \ - > "$BUILD_DIR/$PYTHON_BUILD_DIR/lib/PYTHIA_VERSION" - - echo "::endgroup::" -} - - -# -# Create the distributable archive. -# -# Args: -# * target_dir = name of the dir to be archived. -# -make_dist(){ - local target_dir="$1" - local full_ver="$PYTHON_BUILD_VERSION.$PYTHIA_VERSION" - local target_path="../$DIST_DIR/$full_ver" - local target_tar="$target_path/python-$full_ver-$OS-$ARCH.tar" - - # Clean dist dir and only create a sub-dir for current version. - execute rm -rf "$DIST_DIR" - execute mkdir -p "$DIST_DIR/$full_ver" - - execute pushd "$BUILD_DIR" - echo "#### Creating $target_tar.gz from $target_dir. ####" - execute tar -cf "$target_tar" "$target_dir" - execute gzip "$target_tar" - execute popd -} - - -# -# Move lib/include/ back to include/ in Python's build dir, -# otherwise building modules for testing the package is going to fail. -# -bring_back_include(){ - execute pushd "$BUILD_DIR/$PYTHON_BUILD_DIR" - echo "Moving back the include/ sub-dir for building testing modules..." - execute mv lib/include/ ./ - execute popd -} - -# -# Construct a SFTP batch file for uploading testing packages. -# Files are uploaded with a temporary name and then renamed to final name. -# -build_publish_dist_sftp_batch() { - local full_ver="$PYTHON_BUILD_VERSION.$PYTHIA_VERSION" - local local_dir="$DIST_DIR/$full_ver" - local upload_dir="testing/$full_ver" - local pkg_file="python-$full_ver-$OS-$ARCH.tar.gz" - local local_file="$local_dir/$pkg_file" - local dest_file="$upload_dir/$pkg_file" - - # The mkdir command is prefixed with '-' to allow it to fail because - # $upload_dir exists if this is not the first upload for this version. - echo "-mkdir $upload_dir" > build/publish_dist_sftp_batch - echo "put $local_file $dest_file.part" >> build/publish_dist_sftp_batch - echo "rename $dest_file.part $dest_file" >> build/publish_dist_sftp_batch -} diff --git a/os_quirks.sh b/os_quirks.sh deleted file mode 100644 index 2e2fe6fe..00000000 --- a/os_quirks.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -# -# OS quirks for the Pythia build system. - -case $OS in - windows) - # On Windows, the python executable is installed in a different path. - PYTHON_BIN="$INSTALL_DIR/lib/python.exe" - # There are no actual dependency builds, only binary wheels are used. - BUILD_BZIP2="no" - BUILD_SQLITE="no" - BUILD_OPENSSL="no" - PIP_LIBRARIES+=(pywin32=="$PYWIN32_VERSION") - ;; - linux*) - # Build as portable as possible, only glibc/musl should be needed. - export CFLAGS="${CFLAGS:-} -mtune=generic" - BUILD_LIBFFI="yes" - BUILD_ZLIB="yes" - BUILD_XZ="yes" - if [ -f /etc/alpine-release ]; then - # The busybox ersatz binary on Alpine Linux is different. - SHA_CMD=(sha512sum -csw) - fi - ;; - macos) - export CC="clang" - export CXX="clang++" - if [ "$ARCH" = "x64" ]; then - export CFLAGS="${CFLAGS:-} -mmacosx-version-min=10.13" - # setup.py skips building readline by default, as it sets this to - # "10.4", and then tries to avoid the broken readline in OS X 10.4. - export MACOSX_DEPLOYMENT_TARGET=10.13 - else - export CFLAGS="${CFLAGS:-} -mmacosx-version-min=11.0" - # setup.py skips building readline by default, as it sets this to - # "10.4", and then tries to avoid the broken readline in OS X 10.4. - export MACOSX_DEPLOYMENT_TARGET=11.0 - fi - # System includes bzip2 libs by default. - BUILD_BZIP2="no" - BUILD_XZ="yes" - SHA_CMD=(shasum --algorithm 512 --check --status --warn) - ;; - fbsd*) - export CC="clang" - export CXX="clang++" - # libffi not available in the base system, only as port/package. - BUILD_LIBFFI="yes" - # System includes bzip2 libs by default. - BUILD_BZIP2="no" - BUILD_XZ="yes" - # Install package "p5-Digest-SHA" to get shasum binary. - SHA_CMD=(shasum --algorithm 512 --check --status --warn) - ;; - obsd*) - export CC="clang" - export CXX="clang++" - # libffi not available in the base system, only as port/package. - BUILD_LIBFFI="yes" - BUILD_XZ="yes" - SHA_CMD=(sha512 -q -c) - ;; - sol*) - # By default, Sun's Studio compiler is used. - export CC="cc" - export CXX="CC" - export MAKE_CMD=(gmake) - # Needed for the subprocess32 module. - # More at https://github.com/google/python-subprocess32/issues/40. - export CFLAGS="${CFLAGS:-} -DHAVE_DIRFD" - # Arch-specific bits and paths. - if [ "${ARCH%64}" = "$ARCH" ]; then - # Some libs (e.g. GMP) need to be informed of a 32bit build. - export ABI="32" - else - export CFLAGS="$CFLAGS -m64" - export LDFLAGS="${LDFLAGS:-} -m64 -L/usr/lib/64 -R/usr/lib/64" - fi - # System includes bzip2 libs by default. - BUILD_BZIP2="no" - # Solaris 11 is much more modern, but still has some quirks. - # Multiple system libffi libs are present, this is a problem in 11.4. - BUILD_LIBFFI="yes" - BUILD_XZ="yes" - # Native tar is not that compatible, but the GNU tar should be present. - TAR_CMD=(gtar xfz) - ;; -esac - -# Compiler-dependent flags. At this moment, the compiler is known. -case "$OS" in - sol*) - # Not all packages enable PIC, force it to avoid relocation issues. - export CFLAGS="$CFLAGS -Kpic" - ;; - fbsd*|obsd*) - # Use PIC (Position Independent Code) on FreeBSD and OpenBSD with Clang. - export CFLAGS="${CFLAGS:-} -fPIC" - ;; -esac - -# Use PIC (Position Independent Code) with GCC on 64-bit arches (currently all). -case "$CC" in - gcc*) - export CFLAGS="${CFLAGS:-} -fPIC" - ;; -esac - -# Get number of useful CPUs, to enable parallel builds where applicable. -case "$OS" in - windows) - # Logical CPUs (including hyper-threading) in Windows 2000 or newer. - CPUS="$NUMBER_OF_PROCESSORS" - ;; - macos|fbsd*|obsd*) - # Logical CPUs. - CPUS="$(sysctl -n hw.ncpu)" - ;; - sol*) - # Physical CPUs. - CPUS="$(/usr/sbin/psrinfo -p)" - ;; - *) - # Only Linux distros should be left, look for logical CPUS. - # Don't use lscpu/nproc or other stuff not present on older distros. - CPUS="$(getconf _NPROCESSORS_ONLN)" - ;; -esac -MAKE_CMD=("${MAKE_CMD[@]}" -j"$CPUS") diff --git a/publish_dist.sh b/publish_dist.sh index 64a44d44..a0039641 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -7,16 +7,20 @@ set -o nounset set -o errexit set -o pipefail -# This is also defined in build.conf. -DIST_DIR="dist" +source build.conf -dest_server="bin.chevah.com" -dest_user="github-upload" -root_link="https://$dest_server:20443/testing" +root_link="https://$UPLOAD_SERVER:20443/testing" # The build/publish_dist_sftp_batch file is generated by the build process. # The private key comes from GitHub Secrets through the configured workflow. -sftp_opts=(-b build/publish_dist_sftp_batch -o IdentityFile=priv_key \ + +target_dir="/${TARGET_DEPLOY}/${PYTHIA_RELEASE}/" +batch_file="${BUILD_DIR}/sftp.batch" +echo "> Uploading to ${target_dir}" +echo "mkdir ${target_dir}" > "${batch_file}" +echo "mput -v ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" + +sftp_opts=(-b ${batch_file} -o IdentityFile=priv_key \ -o StrictHostKeyChecking=yes) OS="$(uname -s)" @@ -30,7 +34,7 @@ case "$OS" in ;; esac -"$sftp_cmd" "${sftp_opts[@]}" "$dest_user"@"$dest_server" +"$sftp_cmd" "${sftp_opts[@]}" "$UPLOAD_USER"@"$UPLOAD_SERVER" # As dist/ is rebuilt on every build, it should only have 1 sub-dir with 1 pkg. upload_dir="$(cd "$DIST_DIR" && ls -1)" diff --git a/pythia.conf b/pythia.conf index ec487ed4..1a7c364c 100644 --- a/pythia.conf +++ b/pythia.conf @@ -3,11 +3,11 @@ # and remove BUILD_ENV_* files (e.g. with `./build.sh clean -a`). PYTHON_CONFIGURATION="default@3.13.2.deadbeef" # This is defined as a Bash array of options to be passed to commands. -BASE_REQUIREMENTS=("chevah-brink==1.0.15" "paver==1.3.4") +BASE_REQUIREMENTS=("chevah-brink==1.0.15" "paver==1.3.4" "setuptools==78.1.1") # Use our private PyPi server instead of the default one set in pythia.sh. PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple" # Use our production server instead of the GitHub releases set by default. -BINARY_DIST_URI="https://bin.chevah.com:20443/production" +BINARY_DIST_URI="https://bin.chevah.com:20443/testing" # For testing packages, make sure this one is the last uncommented instance: #BINARY_DIST_URI="https://bin.chevah.com:20443/testing" # This directory is used by the Python runtime by default. diff --git a/requirements.txt b/requirements.txt index beb36caa..a399889e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -pip==25.0 -psutil==6.1.1 -pycparser==2.22 +# Here to help with dependbot. +# Should be manually updated once a new version is used. pywin32==308 -setuptools==75.8.0 + diff --git a/test_compat.sh b/test_compat.sh new file mode 100644 index 00000000..df459337 --- /dev/null +++ b/test_compat.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Test the Pythia distribtution availabe in DIST agains compat. +# + +# Bash checks +set -o nounset # always check if variables exist +set -o errexit # always exit on error +set -o errtrace # trap errors in functions as well +set -o pipefail # don't ignore exit codes when piping output +set -o functrace # inherit DEBUG and RETURN traps + +# Set versions for the software to be built and other defaults. +source build.conf + +# Run the test suite from chevah/compat master. + +execute pushd "$BUILD_DIR" + +# This is quite hackish, as compat is arm-twisted to use the local version. +echo "::group::Compat tests" +echo "#### Running chevah's compat tests... ####" +echo "## Removing any pre-existing compat code... ##" +execute rm -rf compat/ +echo "## Cloning compat's master branch... ##" +execute git clone https://github.com/chevah/compat.git --depth=1 -b master +execute pushd compat + # Make sure everything is done from scratch in the current dir. + echo "## Unsetting CHEVAH_CACHE and CHEVAH_BUILD... ##" + unset CHEVAH_CACHE CHEVAH_BUILD + # Copy over current pythia stuff, as some changes might require it. + echo "## Patching compat code to use current pythia version... ##" + execute cp ../../pythia.{conf,sh} ./ + # Patch compat to use the current's branch version. + echo -e "\nPYTHON_CONFIGURATION=default@${PYTHIA_RELEASE}" >>pythia.conf + execute mkdir cache + # Copy dist file to local cache, if existing. If not, maybe it's online. + cp ../../"$DIST_DIR"/"$PYTHIA_RELEASE"/* cache/ + # Some tests could fail due to causes not related to the new Python. + echo "## Getting compat deps... ##" + execute ./pythia.sh deps + echo "## Running normal compat tests... ##" + # Why not test_normal? See https://github.com/chevah/compat/issues/691. + execute ./pythia.sh test -vs normal + if [ "${CI:-}" = "true" ]; then + echo "## Running ci2 compat tests... ##" + execute ./pythia.sh test_ci2 + fi +execute popd +echo "::endgroup::" From f6d9fbeb712c1e10810a03b5538c768523c12a0e Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 12:53:12 +0100 Subject: [PATCH 02/11] Update tests. --- README.rst | 2 + astral-to-pythia.sh | 13 +- build.conf | 20 + src/Python-Windows/chevahbs | 40 - src/Python-Windows/sha512.sum | 1 - src/Python/chevahbs | 118 - src/Python/disabled_modules.patch | 20 - src/Python/sha512.sum | 1 - src/chevah-bash-tests/README.rst | 4 - src/chevah-bash-tests/get-shellcheck.sh | 63 - src/chevah-bash-tests/shellcheck_tests.sh | 80 - .../test_python_binary_dist.py | 75 +- src/libedit/chevahbs | 59 - src/libedit/sha512.sum | 1 - src/libedit/terminal_restore.patch | 15 - src/libffi/chevahbs | 64 - src/libffi/sha512.sum | 1 - src/openssl/chevahbs | 56 - src/openssl/sha512.sum | 1 - src/sqlite-autoconf/chevahbs | 48 - src/sqlite-autoconf/sha512.sum | 1 - src/xz/chevahbs | 46 - src/xz/sha512.sum | 1 - src/zipfile_init.py | 2345 +++++++++++++++++ src/zlib/chevahbs | 42 - src/zlib/sha512.sum | 1 - test_compat.sh | 3 +- test_pythia.sh | 43 + 28 files changed, 2431 insertions(+), 733 deletions(-) delete mode 100755 src/Python-Windows/chevahbs delete mode 100644 src/Python-Windows/sha512.sum delete mode 100755 src/Python/chevahbs delete mode 100644 src/Python/disabled_modules.patch delete mode 100644 src/Python/sha512.sum delete mode 100644 src/chevah-bash-tests/README.rst delete mode 100755 src/chevah-bash-tests/get-shellcheck.sh delete mode 100755 src/chevah-bash-tests/shellcheck_tests.sh delete mode 100755 src/libedit/chevahbs delete mode 100644 src/libedit/sha512.sum delete mode 100644 src/libedit/terminal_restore.patch delete mode 100755 src/libffi/chevahbs delete mode 100644 src/libffi/sha512.sum delete mode 100755 src/openssl/chevahbs delete mode 100644 src/openssl/sha512.sum delete mode 100755 src/sqlite-autoconf/chevahbs delete mode 100644 src/sqlite-autoconf/sha512.sum delete mode 100755 src/xz/chevahbs delete mode 100644 src/xz/sha512.sum create mode 100644 src/zipfile_init.py delete mode 100755 src/zlib/chevahbs delete mode 100644 src/zlib/sha512.sum mode change 100644 => 100755 test_compat.sh create mode 100755 test_pythia.sh diff --git a/README.rst b/README.rst index 43bd6bf6..48b326ba 100644 --- a/README.rst +++ b/README.rst @@ -57,6 +57,8 @@ Copy `pythia.conf` file into your repo and configure your mirrors and targeted P It is designed to create the Python virtual environment and automatically call paver inside the newly created environment. `pythia.conf` contains a set of Python packages that needs to be intallled to boostrap the `paver` usage. +TK is removed to save space and since this is designed for server-side apps. + Development ----------- diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index dca5fa92..a9cc0fc7 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -62,7 +62,7 @@ esac file_name="${pbs_arch}-install_only_stripped.tar.gz" source_url="https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-${PY_RELEASE}+${PBS_RELEASE}-${file_name}" echo "> Downloading $file_name" -curl -L -o $file_name $source_url +#curl -L -o $file_name $source_url echo "> Extracting $file_name" @@ -70,7 +70,6 @@ echo "> Extracting $file_name" # Used during dev when the whole build folder is not cleaned. rm -rf python - tar xzf $file_name @@ -83,7 +82,15 @@ case "$TARGET" in mkdir python mv lib python rm -rf python/lib/include + rm -rf python/lib/tcl + rm -f python/lib/libs/_tkinter.lib + rm -f python/lib/DLLs/tcl*.dll + rm -f python/lib/DLLs/tk*.dll + cp ../src/zipfile_init.py python/lib/Lib/zipfile/__init__.py python_bin="python/lib/python" + ${python_bin} -m pip install \ + --index-url="$PIP_INDEX_URL" \ + pywin32==${PYWIN32_VERSION} ;; *) # Linux and MacOS @@ -96,6 +103,8 @@ case "$TARGET" in rm -f python/lib/libtcl* rm -rf python/lib/python${PY_VERSION}/tkinter rm -rf python/lib/python${PY_VERSION}/turtledemo + rm -f python/lib/python${PY_VERSION}/lib-dynload/_tkinter* + cp ../src/zipfile_init.py python/lib/python${PY_VERSION}/zipfile/__init__.py python_bin="python/bin/python" ;; esac diff --git a/build.conf b/build.conf index e93ce12f..ab39e721 100644 --- a/build.conf +++ b/build.conf @@ -28,3 +28,23 @@ PIP_ARGS=(\ --index-url="$PIP_INDEX_URL" \ --no-warn-script-location \ ) + +DEBUG="${DEBUG-0}" + +execute() { + if [ "$DEBUG" -ne 0 ]; then + (>&2 echo -e "\tExecuting:" "$@") + fi + + #Make sure $@ is called in quotes as otherwise it will not work. + "$@" + exit_code="$?" + if [ "$DEBUG" -ne 0 ]; then + (>&2 echo -e "\tExit code was: $exit_code") + fi + if [ "$exit_code" -ne 0 ]; then + (>&2 echo "PWD :" "$(pwd)") + (>&2 echo "Fail:" "$@") + exit 97 + fi +} diff --git a/src/Python-Windows/chevahbs b/src/Python-Windows/chevahbs deleted file mode 100755 index 7ee60324..00000000 --- a/src/Python-Windows/chevahbs +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for Python under Windows. -# There is no actual build, just getting binaries and installing them as needed. - - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - -# The usual two arguments, e.g. "Python" and "3.9.0", -# but the installation path is also needed, because this script is special. -NAME="$1" -VER="$2" -# There's no intrinsic need for this, but Chevah's server likes it this way. -INSTALL_DIR="$3"/lib - -# Construct the relevant download link for the Windows version. -EXT="amd64.zip" -ARCHIVE="python-$VER.$EXT" - -# For beta versions, remove everything after the b for the dir name. -DIR=${VER%%b*} -# For rc versions, remove everything after rc for the dir name -DIR=${VER%%rc*} - -LINK="https://www.python.org/ftp/python/$DIR/python-$VER-embed-$EXT" - -# Download without unpacking (amd64.zip extension is treated specially). -download_sources "$NAME" "$VER" "$LINK" "$EXT" - -# Installation consists of unpacking the ZIP'ed embeddable distribution. -echo "## Extracting ZIP archive to $INSTALL_DIR... ##" -execute unzip -q "$ARCHIVE" -d "$INSTALL_DIR" - -echo "## Adding site-packages to python*._pth file... ##" -echo "import site" >> "$INSTALL_DIR"/python313._pth -echo "site.main()" >> "$INSTALL_DIR"/python313._pth diff --git a/src/Python-Windows/sha512.sum b/src/Python-Windows/sha512.sum deleted file mode 100644 index 92dc4d89..00000000 --- a/src/Python-Windows/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -a87d1d8e74efd79de42459ab18761406d398a246cddba4832f748bc3bfe3d9b8945ff4d0a29e95c770ca9a13d120c924b7ea514af6db5905e5821ace2ec53723 python-3.13.2.amd64.zip diff --git a/src/Python/chevahbs b/src/Python/chevahbs deleted file mode 100755 index b73c7551..00000000 --- a/src/Python/chevahbs +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for Python. -# - - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../BUILD_ENV_ARRAYS - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tgz" - local dir - local link - - # For beta versions, remove everything after the b for the dir name. - dir=${ver%%b*} - # For rc versions, remove everything after rc for the dir name. - dir=${ver%%rc*} - link=https://www.python.org/ftp/python/"$dir"/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_patch() { - if [ "$BUILD_LIBEDIT" = "yes" ]; then - # Adapted from https://bugs.python.org/issue13501. - echo "# Applying readline_libedit.patch... #" - execute patch -p 1 < readline_libedit.patch - fi -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONFIG_OPTS=(--disable-shared) - - # This requires the patch referenced in chevahbs_patch(). - if [ "$BUILD_LIBEDIT" = "yes" ]; then - CONFIG_OPTS=("${CONFIG_OPTS[@]}" --with-readline=editline) - fi - - echo "# Copying collected header files among Python's ones... #" - execute cp -r "$INSTALL_DIR"/include/* ./Include/ - - # Copying headers to Include/ is not enough to find the OpenSSL ones. - if [ "$BUILD_OPENSSL" = "yes" ]; then - CONFIG_OPTS=("${CONFIG_OPTS[@]}" --with-openssl="$INSTALL_DIR") - fi - - case "$OS" in - linux) - # Needed for building Python 3.11's sqlite3 module on Linux. - # Note that this is not needed on Alpine. - LDFLAGS="$LDFLAGS -lm" - ;; - linux_musl) - # Needed for Alpine Linux 3.20. - LDFLAGS="$LDFLAGS -lffi" - ;; - fbsd*|sol11*) - LDFLAGS="$LDFLAGS -lncurses" - ;; - obsd*) - # In OpenBSD 6.1 and newer, Python binary has to marked as - # "wxneeded" because it breaks the mandatory W^X protection. - LDFLAGS="$LDFLAGS -Wl,-z,wxneeded" - ;; - esac - - echo "# Patching the git rev id into Python's version string... #" - execute cp Modules/getbuildinfo.c Modules/getbuildinfo.c.orig - # Don't use 'sed -i' here, it's not supported on macOS yet (tested on 13.3). - execute sed -e \ - s/gitid\ =\ \"main\"/gitid\ =\ \""$PYTHIA_VERSION"\"/g \ - Modules/getbuildinfo.c.orig > Modules/getbuildinfo.c - execute ./configure --prefix="$INSTALL_DIR" "${CONFIG_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - -chevahbs_try() { - if [ "$OS" = "macos" ]; then - # Extra CA certs needed to pass test_cppext/test_distutils, and included - # Mac/BuildScript/resources/install_certificates.command fix is broken. - (>&2 echo -e "\tNot running Python upstream tests on macOS.") - elif [ "$OS" = "linux_musl" ]; then - # Locales not supported on Alpine 3.12, failing locale-related tests. - (>&2 echo -e "\tNot running Python upstream tests on musl-based Linux.") - else - execute "${MAKE_CMD[@]}" test - fi -} - - -chevahbs_cp() { - if [ "$OS" = "linux_musl" ]; then - # EMUTRAMP required for full functionality under a grsec kernel. - # Don't use "paxmark", file attributes will be lost when tar'ed. - # Needed with non-grsec kernels as well, otherwise PAM-related - # compat tests crash with signal 11 (last tested on Alpine 3.12). - execute paxctl -cE python - fi - - execute "${MAKE_CMD[@]}" install -} - - -select_chevahbs_command "$@" diff --git a/src/Python/disabled_modules.patch b/src/Python/disabled_modules.patch deleted file mode 100644 index 13f4a743..00000000 --- a/src/Python/disabled_modules.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- setup.py.orig 2023-04-05 01:22:17.000000000 +0300 -+++ setup.py 2023-05-02 13:21:57.151724870 +0300 -@@ -56,7 +56,16 @@ - - - # This global variable is used to hold the list of modules to be disabled. --DISABLED_MODULE_LIST = [] -+DISABLED_MODULE_LIST = [ -+ '_curses', -+ '_curses_panel', -+ '_dbm', -+ '_gdbm', -+ '_locale', -+ '_tkinter', -+ 'nis', -+ 'readline', -+ ] - - # --list-module-names option used by Tools/scripts/generate_module_names.py - LIST_MODULE_NAMES = False diff --git a/src/Python/sha512.sum b/src/Python/sha512.sum deleted file mode 100644 index a094cdb3..00000000 --- a/src/Python/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -b388ff54d31725186420c26f47165c76155d4fcbd71c23ebc304ed01b5b246c84db622c7d625674964c387a9f7527eb00db7bddfb08fdeb363b6a69585fbdc47 Python-3.13.2.tgz diff --git a/src/chevah-bash-tests/README.rst b/src/chevah-bash-tests/README.rst deleted file mode 100644 index 4df0ceba..00000000 --- a/src/chevah-bash-tests/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -chevah-shell-tests -================== - -Scripts for checking that included shell scripts pass the Shellcheck tests. diff --git a/src/chevah-bash-tests/get-shellcheck.sh b/src/chevah-bash-tests/get-shellcheck.sh deleted file mode 100755 index 23b2c19d..00000000 --- a/src/chevah-bash-tests/get-shellcheck.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# -# Get the latest Shellcheck version into our build folder. -# -# Should be called with the build folder as the first argument. -# Snatched from chevah/server repo, and improved to also work on macOS x64. -# Another change: this uses $BUILD_DIR, not $BUILD_DIR/bin. - -# Bash checks -set -o nounset # always check if variables exist -set -o errexit # always exit on error -set -o errtrace # trap errors in functions as well -set -o pipefail # don't ignore exit codes when piping output - -BUILD_DIR="$1" -OS_STRING="$(uname | tr '[:upper:]' '[:lower:]')" -ARCH="$(uname -m)" -if [ "$ARCH" = "arm64" ]; then - ARCH="aarch64" -fi - -# Upstream Shellcheck stuff. -SHELLCHECK_LNK="https://github.com/koalaman/shellcheck/releases/download/latest" -SHELLCHECK_DIR="shellcheck-latest" -SHELLCHECK_XZ="$SHELLCHECK_DIR.$OS_STRING.$ARCH.tar.xz" - - -# Using Bash arrays for commands, to make them quotable. -# Be non-verbose by default. -if [ "${DEBUG-0}" -ne 0 ]; then - ECHO_CMD=(echo) - CURL_CMD=(curl) - TAR_CMD=(tar xvf) - MV_CMD=(mv -v) - RM_CMD=(rm -rfv) -else - ECHO_CMD=(true) - CURL_CMD=(curl --silent) - TAR_CMD=(tar xf) - MV_CMD=(mv) - RM_CMD=(rm -rf) -fi - -# -# Install latest Shellcheck binary. -# -install_latest_shellcheck() { - "${ECHO_CMD[@]}" "Installing latest Shellcheck..." - "${CURL_CMD[@]}" --location "$SHELLCHECK_LNK"/"$SHELLCHECK_XZ" \ - --output /tmp/"$SHELLCHECK_XZ" - "${TAR_CMD[@]}" /tmp/"$SHELLCHECK_XZ" --directory /tmp/ - # The end slash assures that the destination is a dir, not a new file. - "${MV_CMD[@]}" -v /tmp/"$SHELLCHECK_DIR"/shellcheck "$BUILD_DIR"/ - "${RM_CMD[@]}" /tmp/"$SHELLCHECK_DIR" -} - -if [ ! -x "$BUILD_DIR"/shellcheck ]; then - # Only install Shellcheck if it's not already present. - install_latest_shellcheck - exit 0 -fi - -"${ECHO_CMD[@]}" "Shellcheck already installed." diff --git a/src/chevah-bash-tests/shellcheck_tests.sh b/src/chevah-bash-tests/shellcheck_tests.sh deleted file mode 100755 index 77dfeab5..00000000 --- a/src/chevah-bash-tests/shellcheck_tests.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# -# Script for checking all shell scripts in this repo with Shellcheck. -# Available independently of all other scripts, except the common functions.sh. -# To be executed from the root of the repository. - -# Bash checks -set -o nounset # always check if variables exist -set -o errexit # always exit on error -set -o errtrace # trap errors in functions as well -set -o pipefail # don't ignore exit codes when piping output - -source ./functions.sh - -# BUILD_DIR is also defined in build.conf. -BUILD_DIR="build" -OS="$(uname)" -ARCH="$(uname -m)" - -case "$OS" in - MINGW*|MSYS*) - echo "Shellcheck not supported on Windows, skipping!" - exit - ;; - Darwin) - if [ "$ARCH" = "arm64" ]; then - echo "Shellcheck not supported on Apple Silicon, skipping!" - exit - fi - ;; -esac - -echo "## Getting shellcheck binary in $BUILD_DIR/ if missing... ##" -execute ./src/chevah-bash-tests/get-shellcheck.sh "$BUILD_DIR" - -echo "## Checking executable .sh files in the root dir of the repo... ##" -exec_sh_files=() -other_sh_files=() -for sh_file in ./*.sh; do - if [ -x "$sh_file" ]; then - exec_sh_files+=("$sh_file") - else - other_sh_files+=("$sh_file") - fi -done -echo "Executable shell scripts to be checked (including their sources):" -for exec_sh_file in "${exec_sh_files[@]}"; do - echo -e "\t$exec_sh_file" -done -execute "$BUILD_DIR"/shellcheck -ax "${exec_sh_files[@]}" -echo "Non-executable scripts to be skipped (should be sourced by the above):" -for other_sh_file in "${other_sh_files[@]}"; do - echo -e "\t$other_sh_file" -done - -echo "## Checking extra .sh files in \"./src/\"... ##" -extra_scripts=() -# Do not use mapfile, needs Bash 4. See https://www.shellcheck.net/wiki/SC2207. -while IFS="" read -r line; do - extra_scripts+=("$line") -done < <(find ./src -name '*.sh') -echo "Extra shell scripts to be checked under ./src (including sources):" -for extra_script in "${extra_scripts[@]}"; do - echo -e "\t$extra_script" -done -execute "$BUILD_DIR"/shellcheck -ax "${extra_scripts[@]}" - -echo "## Checking the chevahbs scripts in ./src/*/ sub-dirs in their dirs... ##" -echo "chevahbs files to be checked were found in the following sub-dirs:" -for src_dir in ./src/*; do - if [ -d "$src_dir" ]; then - if [ -e "$src_dir"/chevahbs ]; then - echo -e "\t$src_dir" - # chevahbs uses relative paths, must be checked from the same dir. - execute cd "$src_dir" - execute ../../"$BUILD_DIR"/shellcheck -x chevahbs - execute cd ../../ - fi - fi -done diff --git a/src/chevah-python-tests/test_python_binary_dist.py b/src/chevah-python-tests/test_python_binary_dist.py index 9f53e951..0e747d08 100644 --- a/src/chevah-python-tests/test_python_binary_dist.py +++ b/src/chevah-python-tests/test_python_binary_dist.py @@ -15,8 +15,6 @@ print('Could not get $OS/$ARCH Chevah env vars.') sys.exit(120) -BUILD_LIBEDIT = os.environ.get('BUILD_LIBEDIT', 'no').lower() == 'yes' - def get_allowed_deps(): """ @@ -238,10 +236,15 @@ def get_actual_deps(script_helper): else: if any(string in line for string in linux_ignored_strings): continue + if len(line.split()) < 3: + continue dep = line.split()[2] else: # For other OS'es, the third field in each line is needed. dep = line.split()[2] + if dep == 'not': + sys.stderr.write(f'Broken deps {line}.\n') + sys.exit(121) libs_deps.append(dep) return list(set(libs_deps)) @@ -302,25 +305,6 @@ def test_dependencies(): return 0 -def egg_check(module): - """ - Check that the tested module is in the current path. - If not, it may be pulled from ~/.python-eggs and that's not good. - - Return 0 on success, non zero on error. - """ - if not os.getcwd() in module.__file__: - sys.stderr.write( - "{0} module not in current path, ".format(module.__name__) + - "is zip_safe set to True for it?\n" - "\tcurrent path: {0}".format(os.getcwd()) + "\n" - "\tmodule file: {0}".format(module.__file__) + "\n" - ) - return 125 - - return 0 - - def main(): """ Launch tests to check required modules and OS-specific dependencies. @@ -343,10 +327,7 @@ def main(): try: from ssl import OPENSSL_VERSION as current_openssl_version import _hashlib - exit_code = egg_check(_hashlib) | exit_code - expecting_openssl_version = u'OpenSSL 3.0.16 11 Feb 2025' - if CHEVAH_OS == "windows": - expecting_openssl_version = u'OpenSSL 3.0.15 3 Sep 2024' + expecting_openssl_version = 'OpenSSL 3.5.5 27 Jan 2026' if current_openssl_version != expecting_openssl_version: sys.stderr.write('Expecting %s, got %s.\n' % ( expecting_openssl_version, current_openssl_version)) @@ -413,17 +394,7 @@ def main(): sys.stderr.write('"sqlite3" is missing or broken.\n') exit_code = 153 else: - print('sqlite3 %s - sqlite %s' % ( - sqlite.version, sqlite.sqlite_version)) - - try: - import psutil - cpu_percent = psutil.cpu_percent() - except: - sys.stderr.write('"psutil" is missing or broken.\n') - exit_code = 160 - else: - print('psutil %s' % (psutil.__version__,)) + print('sqlite {}'.format(sqlite.sqlite_version)) try: import uuid @@ -461,38 +432,8 @@ def main(): pywin32_version = pywin32_version.strip() print('pywin32 %s' % (pywin32_version)) - else: - # Linux / Unix stuff. - # Check for the git revision in Python's sys.version on Linux and Unix. - try: - git_rev_cmd = ['git', 'log', '-1', '--no-merges', '--format=%h'] - git_rev = subprocess.check_output(git_rev_cmd).strip().decode() - except: - sys.stderr.write("Couldn't get the git rev for the current tree.\n") - exit_code = 157 - else: - bin_ver = sys.version.split('(')[1].split(',')[0] - if bin_ver != git_rev: - sys.stderr.write("Python version doesn't match git revision!\n" - "\tBin ver: {0}".format(bin_ver) + "\n" - "\tGit rev: {0}".format(git_rev) + "\n") - exit_code = 158 - - # The readline module is built using libedit only on selected platforms. - if BUILD_LIBEDIT: - try: - import readline - readline.get_history_length() - except: - sys.stderr.write('"readline" is missing or broken.\n') - exit_code = 162 - else: - print('"readline" module is present.') - - - exit_code = test_dependencies() | exit_code - + exit_code = test_dependencies() or exit_code sys.exit(exit_code) diff --git a/src/libedit/chevahbs b/src/libedit/chevahbs deleted file mode 100755 index b5d136e3..00000000 --- a/src/libedit/chevahbs +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for libedit. - -# Import shared code. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://thrysoee.dk/editline/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_patch() { - # With libedit, terminal state is not properly restored on some OS'es, eg. - # after exiting the repl on systems with ncurses 6.0, such as Ubuntu 16.04. - # Patch from https://trac.macports.org/ticket/48807#comment:141. - # Upstream bugfix (not recommended): https://gnats.netbsd.org/48957. - echo "# Applying terminal_restore.patch... #" - execute patch -p0 < terminal_restore.patch -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(--disable-shared --enable-static) - case "$OS" in - sol*) - # Required when linking to libedit, which has a simpler configure. - CFLAGS="$CFLAGS -xcode=pic32" - esac - execute ./configure --prefix="" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - case "$OS" in - sol*) - # Path to newer awk. - PATH="/usr/xpg4/bin:$PATH" - ;; - esac - execute "${MAKE_CMD[@]}" -} - - -chevahbs_cp() { - execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR" -} - - -select_chevahbs_command "$@" diff --git a/src/libedit/sha512.sum b/src/libedit/sha512.sum deleted file mode 100644 index fe121e7b..00000000 --- a/src/libedit/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -597ed2282240be6dc3c4a7bb8b4ade31fe16ebfeb2229e3aec58643a65882653cc91deb300343d1b1cc588959dbf95698122c4005e47dfb947efa3e6b134bb82 libedit-20170329-3.1.tar.gz diff --git a/src/libedit/terminal_restore.patch b/src/libedit/terminal_restore.patch deleted file mode 100644 index 0c5e7dd9..00000000 --- a/src/libedit/terminal_restore.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff -ur src/readline.c src/readline.c ---- src/readline.c -+++ src/readline.c -@@ -2035,7 +2035,10 @@ - } else - wbuf = NULL; - (*(void (*)(const char *))rl_linefunc)(wbuf); -- el_set(e, EL_UNBUFFERED, 1); -+ /* rl_linefunc might have called rl_callback_handler_remove */ -+ if (rl_linefunc != NULL) { -+ el_set(e, EL_UNBUFFERED, 1); -+ } - } - } - diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs deleted file mode 100755 index cfd9e5bf..00000000 --- a/src/libffi/chevahbs +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for libffi. - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://github.com/libffi/libffi/releases/download/v"$ver"/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(--disable-shared --enable-static) - execute ./configure --prefix="" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - -# This requires DejaGnu to actually do the tests. -chevahbs_try() { - # Tests fail on Amazon 2: https://github.com/libffi/libffi/issues/785. - if [ "$OS" = "linux" ]; then - if [ -f /etc/os-release ]; then - if grep -q ^'PRETTY_NAME="Amazon Linux 2"'$ /etc/os-release; then - (>&2 echo -e "\tSkipping libffi tests on Amazon Linux 2.") - return - fi - fi - fi - execute "${MAKE_CMD[@]}" check -} - - -chevahbs_cp() { - case "$OS" in - linux*) - echo "Installing manually to avoid messing with a lib64/ sub-dir:" - execute cp -v ./*-linux-*/.libs/libffi.a "$INSTALL_DIR"/lib/ - execute cp -v ./*-linux-*/libffi.la "$INSTALL_DIR"/lib/ - execute cp -v ./*-linux-*/include/*.h "$INSTALL_DIR"/include/ - execute mkdir "$INSTALL_DIR"/lib/pkgconfig/ - execute cp -v ./*-linux-*/libffi.pc "$INSTALL_DIR"/lib/pkgconfig/ - ;; - *) - execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR" - ;; - esac -} - -select_chevahbs_command "$@" diff --git a/src/libffi/sha512.sum b/src/libffi/sha512.sum deleted file mode 100644 index 7a3f08cf..00000000 --- a/src/libffi/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -033d2600e879b83c6bce0eb80f69c5f32aa775bf2e962c9d39fbd21226fa19d1e79173d8eaa0d0157014d54509ea73315ad86842356fc3a303c0831c94c6ab39 libffi-3.4.6.tar.gz diff --git a/src/openssl/chevahbs b/src/openssl/chevahbs deleted file mode 100755 index 66a7074c..00000000 --- a/src/openssl/chevahbs +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for OpenSSL. - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://www.openssl.org/source/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(no-shared) - # OpenSSL's build requires an absolute path for --prefix, - # can't use --prefix="" as in the other chevahbs scripts. - execute ./config --prefix="$INSTALL_DIR" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - - -chevahbs_try() { - execute "${MAKE_CMD[@]}" test -} - - -chevahbs_cp() { - case "$OS" in - linux*) - echo "Installing manually to avoid messing with a lib64/ sub-dir:" - execute cp -v libcrypto.a libssl.a "$INSTALL_DIR"/lib/ - execute cp -rv include/openssl/ "$INSTALL_DIR"/include/ - execute cp -v ./*.pc "$INSTALL_DIR"/lib/pkgconfig/ - ;; - *) - execute "${MAKE_CMD[@]}" install - ;; - esac -} - - -select_chevahbs_command "$@" diff --git a/src/openssl/sha512.sum b/src/openssl/sha512.sum deleted file mode 100644 index 62825ac3..00000000 --- a/src/openssl/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -5eea2b0c60d870549fc2b8755f1220a57f870d95fbc8d5cc5abb9589f212d10945f355c3e88ff48540a7ee1c4db774b936023ca33d7c799ea82d91eef9c1c16d openssl-3.0.16.tar.gz diff --git a/src/sqlite-autoconf/chevahbs b/src/sqlite-autoconf/chevahbs deleted file mode 100755 index aae540ac..00000000 --- a/src/sqlite-autoconf/chevahbs +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Script for copying required SQLite source and headers. - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - # SQLite has a peculiar naming scheme for downloads. - # The year probably needs to be updated when updating the version. - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://sqlite.org/2024/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(--disable-shared --enable-static) - execute ./configure --prefix="" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - - -# FIXME:48: -# Test SQLite when building it from source. -chevahbs_try() { - execute "${MAKE_CMD[@]}" check -} - - -chevahbs_cp() { - execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR" -} - - -select_chevahbs_command "$@" diff --git a/src/sqlite-autoconf/sha512.sum b/src/sqlite-autoconf/sha512.sum deleted file mode 100644 index 592bfd12..00000000 --- a/src/sqlite-autoconf/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -631ffe4b39dffbafdcb8ac09a6a84cd7959505ecc588d8ad9278d0ff0c3ea467f87c11167e1b1a3f56d62178e679780e2be313ae3badae8ea056709d71bd4817 sqlite-autoconf-3460000.tar.gz diff --git a/src/xz/chevahbs b/src/xz/chevahbs deleted file mode 100755 index 4748d0db..00000000 --- a/src/xz/chevahbs +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for xz. - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://downloads.sourceforge.net/project/lzmautils/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(--enable-static --disable-shared) - execute ./configure --prefix="" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - - -chevahbs_try() { - execute "${MAKE_CMD[@]}" check -} - - -chevahbs_cp() { - # xz's installation copies binaries too, but there's no need for them. - execute cp -R src/liblzma/api/lzma* "$INSTALL_DIR"/include/ - execute cp src/liblzma/.libs/liblzma.a "$INSTALL_DIR"/lib/ -} - - -select_chevahbs_command "$@" diff --git a/src/xz/sha512.sum b/src/xz/sha512.sum deleted file mode 100644 index e51e3672..00000000 --- a/src/xz/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -b966950eb9206e31f284c9dc3bb0a79f2fabbaf515e88c89da53d3da41ddfeebd2fc6d3d3f8fcd150c70caaaefa43dec0bce84aa183e08bf339d1aebbe041751 xz-5.6.4.tar.gz diff --git a/src/zipfile_init.py b/src/zipfile_init.py new file mode 100644 index 00000000..549466f0 --- /dev/null +++ b/src/zipfile_init.py @@ -0,0 +1,2345 @@ +""" +The patched version for Python 3.12 + +Read and write ZIP files. + +XXX references to utf-8 need further investigation. +""" +import binascii +import importlib.util +import io +import os +import shutil +import stat +import struct +import sys +import threading +import time + +try: + import zlib # We may need its compression method + crc32 = zlib.crc32 +except ImportError: + zlib = None + crc32 = binascii.crc32 + +try: + import bz2 # We may need its compression method +except ImportError: + bz2 = None + +try: + import lzma # We may need its compression method +except ImportError: + lzma = None + +__all__ = ["BadZipFile", "BadZipfile", "error", + "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", + "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", + "Path"] + +class BadZipFile(Exception): + pass + + +class LargeZipFile(Exception): + """ + Raised when writing a zipfile, the zipfile requires ZIP64 extensions + and those extensions are disabled. + """ + +error = BadZipfile = BadZipFile # Pre-3.2 compatibility names + + +ZIP64_LIMIT = (1 << 31) - 1 +ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 +ZIP_MAX_COMMENT = (1 << 16) - 1 + +# constants for Zip file compression methods +ZIP_STORED = 0 +ZIP_DEFLATED = 8 +ZIP_BZIP2 = 12 +ZIP_LZMA = 14 +# Other ZIP compression methods not supported + +DEFAULT_VERSION = 20 +ZIP64_VERSION = 45 +BZIP2_VERSION = 46 +LZMA_VERSION = 63 +# we recognize (but not necessarily support) all features up to that version +MAX_EXTRACT_VERSION = 63 + +# Below are some formats and associated data for reading/writing headers using +# the struct module. The names and structures of headers/records are those used +# in the PKWARE description of the ZIP file format: +# http://www.pkware.com/documents/casestudies/APPNOTE.TXT +# (URL valid as of January 2008) + +# The "end of central directory" structure, magic number, size, and indices +# (section V.I in the format document) +structEndArchive = b"<4s4H2LH" +stringEndArchive = b"PK\005\006" +sizeEndCentDir = struct.calcsize(structEndArchive) + +_ECD_SIGNATURE = 0 +_ECD_DISK_NUMBER = 1 +_ECD_DISK_START = 2 +_ECD_ENTRIES_THIS_DISK = 3 +_ECD_ENTRIES_TOTAL = 4 +_ECD_SIZE = 5 +_ECD_OFFSET = 6 +_ECD_COMMENT_SIZE = 7 +# These last two indices are not part of the structure as defined in the +# spec, but they are used internally by this module as a convenience +_ECD_COMMENT = 8 +_ECD_LOCATION = 9 + +# The "central directory" structure, magic number, size, and indices +# of entries in the structure (section V.F in the format document) +structCentralDir = "<4s4B4HL2L5H2L" +stringCentralDir = b"PK\001\002" +sizeCentralDir = struct.calcsize(structCentralDir) + +# indexes of entries in the central directory structure +_CD_SIGNATURE = 0 +_CD_CREATE_VERSION = 1 +_CD_CREATE_SYSTEM = 2 +_CD_EXTRACT_VERSION = 3 +_CD_EXTRACT_SYSTEM = 4 +_CD_FLAG_BITS = 5 +_CD_COMPRESS_TYPE = 6 +_CD_TIME = 7 +_CD_DATE = 8 +_CD_CRC = 9 +_CD_COMPRESSED_SIZE = 10 +_CD_UNCOMPRESSED_SIZE = 11 +_CD_FILENAME_LENGTH = 12 +_CD_EXTRA_FIELD_LENGTH = 13 +_CD_COMMENT_LENGTH = 14 +_CD_DISK_NUMBER_START = 15 +_CD_INTERNAL_FILE_ATTRIBUTES = 16 +_CD_EXTERNAL_FILE_ATTRIBUTES = 17 +_CD_LOCAL_HEADER_OFFSET = 18 + +# General purpose bit flags +# Zip Appnote: 4.4.4 general purpose bit flag: (2 bytes) +_MASK_ENCRYPTED = 1 << 0 +# Bits 1 and 2 have different meanings depending on the compression used. +_MASK_COMPRESS_OPTION_1 = 1 << 1 +# _MASK_COMPRESS_OPTION_2 = 1 << 2 +# _MASK_USE_DATA_DESCRIPTOR: If set, crc-32, compressed size and uncompressed +# size are zero in the local header and the real values are written in the data +# descriptor immediately following the compressed data. +_MASK_USE_DATA_DESCRIPTOR = 1 << 3 +# Bit 4: Reserved for use with compression method 8, for enhanced deflating. +# _MASK_RESERVED_BIT_4 = 1 << 4 +_MASK_COMPRESSED_PATCH = 1 << 5 +_MASK_STRONG_ENCRYPTION = 1 << 6 +# _MASK_UNUSED_BIT_7 = 1 << 7 +# _MASK_UNUSED_BIT_8 = 1 << 8 +# _MASK_UNUSED_BIT_9 = 1 << 9 +# _MASK_UNUSED_BIT_10 = 1 << 10 +_MASK_UTF_FILENAME = 1 << 11 +# Bit 12: Reserved by PKWARE for enhanced compression. +# _MASK_RESERVED_BIT_12 = 1 << 12 +# _MASK_ENCRYPTED_CENTRAL_DIR = 1 << 13 +# Bit 14, 15: Reserved by PKWARE +# _MASK_RESERVED_BIT_14 = 1 << 14 +# _MASK_RESERVED_BIT_15 = 1 << 15 + +# The "local file header" structure, magic number, size, and indices +# (section V.A in the format document) +structFileHeader = "<4s2B4HL2L2H" +stringFileHeader = b"PK\003\004" +sizeFileHeader = struct.calcsize(structFileHeader) + +_FH_SIGNATURE = 0 +_FH_EXTRACT_VERSION = 1 +_FH_EXTRACT_SYSTEM = 2 +_FH_GENERAL_PURPOSE_FLAG_BITS = 3 +_FH_COMPRESSION_METHOD = 4 +_FH_LAST_MOD_TIME = 5 +_FH_LAST_MOD_DATE = 6 +_FH_CRC = 7 +_FH_COMPRESSED_SIZE = 8 +_FH_UNCOMPRESSED_SIZE = 9 +_FH_FILENAME_LENGTH = 10 +_FH_EXTRA_FIELD_LENGTH = 11 + +# The "Zip64 end of central directory locator" structure, magic number, and size +structEndArchive64Locator = "<4sLQL" +stringEndArchive64Locator = b"PK\x06\x07" +sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) + +# The "Zip64 end of central directory" record, magic number, size, and indices +# (section V.G in the format document) +structEndArchive64 = "<4sQ2H2L4Q" +stringEndArchive64 = b"PK\x06\x06" +sizeEndCentDir64 = struct.calcsize(structEndArchive64) + +_CD64_SIGNATURE = 0 +_CD64_DIRECTORY_RECSIZE = 1 +_CD64_CREATE_VERSION = 2 +_CD64_EXTRACT_VERSION = 3 +_CD64_DISK_NUMBER = 4 +_CD64_DISK_NUMBER_START = 5 +_CD64_NUMBER_ENTRIES_THIS_DISK = 6 +_CD64_NUMBER_ENTRIES_TOTAL = 7 +_CD64_DIRECTORY_SIZE = 8 +_CD64_OFFSET_START_CENTDIR = 9 + +_DD_SIGNATURE = 0x08074b50 + +_EXTRA_FIELD_STRUCT = struct.Struct(' 1: + raise BadZipFile("zipfiles that span multiple disks are not supported") + + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + + sig, sz, create_version, read_version, disk_num, disk_dir, \ + dircount, dircount2, dirsize, diroffset = \ + struct.unpack(structEndArchive64, data) + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") + + # Update the original endrec using data from the ZIP64 record + endrec[_ECD_SIGNATURE] = sig + endrec[_ECD_DISK_NUMBER] = disk_num + endrec[_ECD_DISK_START] = disk_dir + endrec[_ECD_ENTRIES_THIS_DISK] = dircount + endrec[_ECD_ENTRIES_TOTAL] = dircount2 + endrec[_ECD_SIZE] = dirsize + endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz + return endrec + + +def _EndRecData(fpin): + """Return data from the "End of Central Directory" record, or None. + + The data is a list of the nine items in the ZIP "End of central dir" + record followed by a tenth item, the file seek offset of this record.""" + + # Determine file size + fpin.seek(0, 2) + filesize = fpin.tell() + + # Check to see if this is ZIP file with no archive comment (the + # "end of central directory" structure should be the last item in the + # file if this is the case). + try: + fpin.seek(-sizeEndCentDir, 2) + except OSError: + return None + data = fpin.read(sizeEndCentDir) + if (len(data) == sizeEndCentDir and + data[0:4] == stringEndArchive and + data[-2:] == b"\000\000"): + # the signature is correct and there's no comment, unpack structure + endrec = struct.unpack(structEndArchive, data) + endrec=list(endrec) + + # Append a blank comment and record start offset + endrec.append(b"") + endrec.append(filesize - sizeEndCentDir) + + # Try to read the "Zip64 end of central directory" structure + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) + + # Either this is not a ZIP file, or it is a ZIP file with an archive + # comment. Search the end of the file for the "end of central directory" + # record signature. The comment is the last item in the ZIP file and may be + # up to 64K long. It is assumed that the "end of central directory" magic + # number does not appear in the comment. + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) + fpin.seek(maxCommentStart, 0) + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) + start = data.rfind(stringEndArchive) + if start >= 0: + # found the magic number; attempt to unpack and interpret + recData = data[start:start+sizeEndCentDir] + if len(recData) != sizeEndCentDir: + # Zip file is corrupted. + return None + endrec = list(struct.unpack(structEndArchive, recData)) + commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file + comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] + endrec.append(comment) + endrec.append(maxCommentStart + start) + + # Try to read the "Zip64 end of central directory" structure + return _EndRecData64(fpin, maxCommentStart + start, endrec) + + # Unable to find a valid end of central directory structure + return None + +def _sanitize_filename(filename): + """Terminate the file name at the first null byte and + ensure paths always use forward slashes as the directory separator.""" + + # Terminate the file name at the first null byte. Null bytes in file + # names are used as tricks by viruses in archives. + null_byte = filename.find(chr(0)) + if null_byte >= 0: + filename = filename[0:null_byte] + # This is used to ensure paths in generated ZIP files always use + # forward slashes as the directory separator, as required by the + # ZIP format specification. + if os.sep != "/" and os.sep in filename: + filename = filename.replace(os.sep, "/") + if os.altsep and os.altsep != "/" and os.altsep in filename: + filename = filename.replace(os.altsep, "/") + return filename + + +class ZipInfo (object): + """Class with attributes describing each file in the ZIP archive.""" + + __slots__ = ( + 'orig_filename', + 'filename', + 'date_time', + 'compress_type', + '_compresslevel', + 'comment', + 'extra', + 'create_system', + 'create_version', + 'extract_version', + 'reserved', + 'flag_bits', + 'volume', + 'internal_attr', + 'external_attr', + 'header_offset', + 'CRC', + 'compress_size', + 'file_size', + '_raw_time', + '_end_offset', + ) + + def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): + self.orig_filename = filename # Original file name in archive + + # Terminate the file name at the first null byte and + # ensure paths always use forward slashes as the directory separator. + filename = _sanitize_filename(filename) + + self.filename = filename # Normalized file name + self.date_time = date_time # year, month, day, hour, min, sec + + if date_time[0] < 1980: + raise ValueError('ZIP does not support timestamps before 1980') + + # Standard values: + self.compress_type = ZIP_STORED # Type of compression for the file + self._compresslevel = None # Level for the compressor + self.comment = b"" # Comment for each file + self.extra = b"" # ZIP extra data + if sys.platform == 'win32': + self.create_system = 0 # System which created ZIP archive + else: + # Assume everything else is unix-y + self.create_system = 3 # System which created ZIP archive + self.create_version = DEFAULT_VERSION # Version which created ZIP archive + self.extract_version = DEFAULT_VERSION # Version needed to extract archive + self.reserved = 0 # Must be zero + self.flag_bits = 0 # ZIP flag bits + self.volume = 0 # Volume number of file header + self.internal_attr = 0 # Internal attributes + self.external_attr = 0 # External file attributes + self.compress_size = 0 # Size of the compressed file + self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory + # Other attributes are set by class ZipFile: + # header_offset Byte offset to the file header + # CRC CRC-32 of the uncompressed file + + def __repr__(self): + result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] + if self.compress_type != ZIP_STORED: + result.append(' compress_type=%s' % + compressor_names.get(self.compress_type, + self.compress_type)) + hi = self.external_attr >> 16 + lo = self.external_attr & 0xFFFF + if hi: + result.append(' filemode=%r' % stat.filemode(hi)) + if lo: + result.append(' external_attr=%#x' % lo) + isdir = self.is_dir() + if not isdir or self.file_size: + result.append(' file_size=%r' % self.file_size) + if ((not isdir or self.compress_size) and + (self.compress_type != ZIP_STORED or + self.file_size != self.compress_size)): + result.append(' compress_size=%r' % self.compress_size) + result.append('>') + return ''.join(result) + + def FileHeader(self, zip64=None): + """Return the per-file header as a bytes object. + + When the optional zip64 arg is None rather than a bool, we will + decide based upon the file_size and compress_size, if known, + False otherwise. + """ + dt = self.date_time + dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + if self.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # Set these to zero because we write them after the file data + CRC = compress_size = file_size = 0 + else: + CRC = self.CRC + compress_size = self.compress_size + file_size = self.file_size + + extra = self.extra + + min_version = 0 + if zip64 is None: + # We always explicitly pass zip64 within this module.... This + # remains for anyone using ZipInfo.FileHeader as a public API. + zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT + if zip64: + fmt = '= 4: + tp, ln = unpack(' len(extra): + raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) + if tp == 0x0001: + data = extra[4:ln+4] + # ZIP64 extension (large files and/or large archives) + try: + if self.file_size in (0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF): + field = "File size" + self.file_size, = unpack(' 2107: + date_time = (2107, 12, 31, 23, 59, 59) + # Create ZipInfo instance to store file information + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += '/' + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.file_size = st.st_size + + return zinfo + + def is_dir(self): + """Return True if this archive member is a directory.""" + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False + + +# ZIP encryption uses the CRC32 one-byte primitive for scrambling some +# internal keys. We noticed that a direct implementation is faster than +# relying on binascii.crc32(). + +_crctable = None +def _gen_crc(crc): + for j in range(8): + if crc & 1: + crc = (crc >> 1) ^ 0xEDB88320 + else: + crc >>= 1 + return crc + +# ZIP supports a password-based form of encryption. Even though known +# plaintext attacks have been found against it, it is still useful +# to be able to get data out of such a file. +# +# Usage: +# zd = _ZipDecrypter(mypwd) +# plain_bytes = zd(cypher_bytes) + +def _ZipDecrypter(pwd): + key0 = 305419896 + key1 = 591751049 + key2 = 878082192 + + global _crctable + if _crctable is None: + _crctable = list(map(_gen_crc, range(256))) + crctable = _crctable + + def crc32(ch, crc): + """Compute the CRC32 primitive on one byte.""" + return (crc >> 8) ^ crctable[(crc ^ ch) & 0xFF] + + def update_keys(c): + nonlocal key0, key1, key2 + key0 = crc32(c, key0) + key1 = (key1 + (key0 & 0xFF)) & 0xFFFFFFFF + key1 = (key1 * 134775813 + 1) & 0xFFFFFFFF + key2 = crc32(key1 >> 24, key2) + + for p in pwd: + update_keys(p) + + def decrypter(data): + """Decrypt a bytes object.""" + result = bytearray() + append = result.append + for c in data: + k = key2 | 2 + c ^= ((k * (k^1)) >> 8) & 0xFF + update_keys(c) + append(c) + return bytes(result) + + return decrypter + + +class LZMACompressor: + + def __init__(self): + self._comp = None + + def _init(self): + props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) + self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ + lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) + ]) + return struct.pack('> 8) & 0xff + else: + # compare against the CRC otherwise + check_byte = (zipinfo.CRC >> 24) & 0xff + h = self._init_decrypter() + if h != check_byte: + raise RuntimeError("Bad password for file %r" % zipinfo.orig_filename) + + + def _init_decrypter(self): + self._decrypter = _ZipDecrypter(self._pwd) + # The first 12 bytes in the cypher stream is an encryption header + # used to strengthen the algorithm. The first 11 bytes are + # completely random, while the 12th contains the MSB of the CRC, + # or the MSB of the file time depending on the header type + # and is used to check the correctness of the password. + header = self._fileobj.read(12) + self._compress_left -= 12 + return self._decrypter(header)[11] + + def __repr__(self): + result = ['<%s.%s' % (self.__class__.__module__, + self.__class__.__qualname__)] + if not self.closed: + result.append(' name=%r mode=%r' % (self.name, self.mode)) + if self._compress_type != ZIP_STORED: + result.append(' compress_type=%s' % + compressor_names.get(self._compress_type, + self._compress_type)) + else: + result.append(' [closed]') + result.append('>') + return ''.join(result) + + def readline(self, limit=-1): + """Read and return a line from the stream. + + If limit is specified, at most limit bytes will be read. + """ + + if limit < 0: + # Shortcut common case - newline found in buffer. + i = self._readbuffer.find(b'\n', self._offset) + 1 + if i > 0: + line = self._readbuffer[self._offset: i] + self._offset = i + return line + + return io.BufferedIOBase.readline(self, limit) + + def peek(self, n=1): + """Returns buffered bytes without advancing the position.""" + if n > len(self._readbuffer) - self._offset: + chunk = self.read(n) + if len(chunk) > self._offset: + self._readbuffer = chunk + self._readbuffer[self._offset:] + self._offset = 0 + else: + self._offset -= len(chunk) + + # Return up to 512 bytes to reduce allocation overhead for tight loops. + return self._readbuffer[self._offset: self._offset + 512] + + def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + return True + + def read(self, n=-1): + """Read and return up to n bytes. + If the argument is omitted, None, or negative, data is read and returned until EOF is reached. + """ + if self.closed: + raise ValueError("read from closed file.") + if n is None or n < 0: + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while not self._eof: + buf += self._read1(self.MAX_N) + return buf + + end = n + self._offset + if end < len(self._readbuffer): + buf = self._readbuffer[self._offset:end] + self._offset = end + return buf + + n = end - len(self._readbuffer) + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while n > 0 and not self._eof: + data = self._read1(n) + if n < len(data): + self._readbuffer = data + self._offset = n + buf += data[:n] + break + buf += data + n -= len(data) + return buf + + def _update_crc(self, newdata): + # Update the CRC using the given data. + if self._expected_crc is None: + # No need to compute the CRC if we don't have a reference value + return + self._running_crc = crc32(newdata, self._running_crc) + # Check the CRC if we're at the end of the file + if self._eof and self._running_crc != self._expected_crc: + raise BadZipFile("Bad CRC-32 for file %r" % self.name) + + def read1(self, n): + """Read up to n bytes with at most one read() system call.""" + + if n is None or n < 0: + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while not self._eof: + data = self._read1(self.MAX_N) + if data: + buf += data + break + return buf + + end = n + self._offset + if end < len(self._readbuffer): + buf = self._readbuffer[self._offset:end] + self._offset = end + return buf + + n = end - len(self._readbuffer) + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + if n > 0: + while not self._eof: + data = self._read1(n) + if n < len(data): + self._readbuffer = data + self._offset = n + buf += data[:n] + break + if data: + buf += data + break + return buf + + def _read1(self, n): + # Read up to n compressed bytes with at most one read() system call, + # decrypt and decompress them. + if self._eof or n <= 0: + return b'' + + # Read from file. + if self._compress_type == ZIP_DEFLATED: + ## Handle unconsumed data. + data = self._decompressor.unconsumed_tail + if n > len(data): + data += self._read2(n - len(data)) + else: + data = self._read2(n) + + if self._compress_type == ZIP_STORED: + self._eof = self._compress_left <= 0 + elif self._compress_type == ZIP_DEFLATED: + n = max(n, self.MIN_READ_SIZE) + data = self._decompressor.decompress(data, n) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + not self._decompressor.unconsumed_tail) + if self._eof: + data += self._decompressor.flush() + else: + data = self._decompressor.decompress(data) + self._eof = self._decompressor.eof or self._compress_left <= 0 + + data = data[:self._left] + self._left -= len(data) + if self._left <= 0: + self._eof = True + self._update_crc(data) + return data + + def _read2(self, n): + if self._compress_left <= 0: + return b'' + + n = max(n, self.MIN_READ_SIZE) + n = min(n, self._compress_left) + + data = self._fileobj.read(n) + self._compress_left -= len(data) + if not data: + raise EOFError + + if self._decrypter is not None: + data = self._decrypter(data) + return data + + def close(self): + try: + if self._close_fileobj: + self._fileobj.close() + finally: + super().close() + + def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + return self._seekable + + def seek(self, offset, whence=os.SEEK_SET): + if self.closed: + raise ValueError("seek on closed file.") + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + curr_pos = self.tell() + if whence == os.SEEK_SET: + new_pos = offset + elif whence == os.SEEK_CUR: + new_pos = curr_pos + offset + elif whence == os.SEEK_END: + new_pos = self._orig_file_size + offset + else: + raise ValueError("whence must be os.SEEK_SET (0), " + "os.SEEK_CUR (1), or os.SEEK_END (2)") + + if new_pos > self._orig_file_size: + new_pos = self._orig_file_size + + if new_pos < 0: + new_pos = 0 + + read_offset = new_pos - curr_pos + buff_offset = read_offset + self._offset + + if buff_offset >= 0 and buff_offset < len(self._readbuffer): + # Just move the _offset index if the new position is in the _readbuffer + self._offset = buff_offset + read_offset = 0 + # Fast seek uncompressed unencrypted file + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset != 0: + # disable CRC checking after first seeking - it would be invalid + self._expected_crc = None + # seek actual file taking already buffered data into account + read_offset -= len(self._readbuffer) - self._offset + self._fileobj.seek(read_offset, os.SEEK_CUR) + self._left -= read_offset + self._compress_left -= read_offset + self._eof = self._left <= 0 + read_offset = 0 + # flush read buffer + self._readbuffer = b'' + self._offset = 0 + elif read_offset < 0: + # Position is before the current position. Reset the ZipExtFile + self._fileobj.seek(self._orig_compress_start) + self._running_crc = self._orig_start_crc + self._expected_crc = self._orig_crc + self._compress_left = self._orig_compress_size + self._left = self._orig_file_size + self._readbuffer = b'' + self._offset = 0 + self._decompressor = _get_decompressor(self._compress_type) + self._eof = False + read_offset = new_pos + if self._decrypter is not None: + self._init_decrypter() + + while read_offset > 0: + read_len = min(self.MAX_SEEK_READ, read_offset) + self.read(read_len) + read_offset -= read_len + + return self.tell() + + def tell(self): + if self.closed: + raise ValueError("tell on closed file.") + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset + return filepos + + +class _ZipWriteFile(io.BufferedIOBase): + def __init__(self, zf, zinfo, zip64): + self._zinfo = zinfo + self._zip64 = zip64 + self._zipfile = zf + self._compressor = _get_compressor(zinfo.compress_type, + zinfo._compresslevel) + self._file_size = 0 + self._compress_size = 0 + self._crc = 0 + + @property + def _fileobj(self): + return self._zipfile.fp + + def writable(self): + return True + + def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') + + # Accept any data that supports the buffer protocol + if isinstance(data, (bytes, bytearray)): + nbytes = len(data) + else: + data = memoryview(data) + nbytes = data.nbytes + self._file_size += nbytes + + self._crc = crc32(data, self._crc) + if self._compressor: + data = self._compressor.compress(data) + self._compress_size += len(data) + self._fileobj.write(data) + return nbytes + + def close(self): + if self.closed: + return + try: + super().close() + # Flush any data from the compressor, and update header info + if self._compressor: + buf = self._compressor.flush() + self._compress_size += len(buf) + self._fileobj.write(buf) + self._zinfo.compress_size = self._compress_size + else: + self._zinfo.compress_size = self._file_size + self._zinfo.CRC = self._crc + self._zinfo.file_size = self._file_size + + if not self._zip64: + if self._file_size > ZIP64_LIMIT: + raise RuntimeError("File size too large, try using force_zip64") + if self._compress_size > ZIP64_LIMIT: + raise RuntimeError("Compressed size too large, try using force_zip64") + + # Write updated header info + if self._zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # Write CRC and file sizes after the file data + fmt = '') + return ''.join(result) + + def _RealGetContents(self): + """Read in the table of contents for the ZIP file.""" + fp = self.fp + try: + endrec = _EndRecData(fp) + except OSError: + raise BadZipFile("File is not a zip file") + if not endrec: + raise BadZipFile("File is not a zip file") + if self.debug > 1: + print(endrec) + size_cd = endrec[_ECD_SIZE] # bytes in central directory + offset_cd = endrec[_ECD_OFFSET] # offset of central directory + self._comment = endrec[_ECD_COMMENT] # archive comment + + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd + + if self.debug > 2: + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) + # self.start_dir: Position of start of central directory + self.start_dir = offset_cd + concat + if self.start_dir < 0: + raise BadZipFile("Bad offset for central directory") + fp.seek(self.start_dir, 0) + data = fp.read(size_cd) + fp = io.BytesIO(data) + total = 0 + while total < size_cd: + centdir = fp.read(sizeCentralDir) + if len(centdir) != sizeCentralDir: + raise BadZipFile("Truncated central directory") + centdir = struct.unpack(structCentralDir, centdir) + if centdir[_CD_SIGNATURE] != stringCentralDir: + raise BadZipFile("Bad magic number for central directory") + if self.debug > 2: + print(centdir) + filename = fp.read(centdir[_CD_FILENAME_LENGTH]) + orig_filename_crc = crc32(filename) + flags = centdir[_CD_FLAG_BITS] + if flags & _MASK_UTF_FILENAME: + # UTF-8 file names extension + filename = filename.decode('utf-8') + else: + # Historical ZIP filename encoding + filename = filename.decode(self.metadata_encoding or 'cp437') + # Create ZipInfo instance to store file information + x = self._ZipInfo(filename) + x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) + x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) + x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] + (x.create_version, x.create_system, x.extract_version, x.reserved, + x.flag_bits, x.compress_type, t, d, + x.CRC, x.compress_size, x.file_size) = centdir[1:12] + if x.extract_version > MAX_EXTRACT_VERSION: + raise NotImplementedError("zip file version %.1f" % + (x.extract_version / 10)) + x.volume, x.internal_attr, x.external_attr = centdir[15:18] + # Convert date/time code to (year, month, day, hour, min, sec) + x._raw_time = t + x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, + t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) + x._decodeExtra(orig_filename_crc) + x.header_offset = x.header_offset + concat + self.filelist.append(x) + self.NameToInfo[x.filename] = x + + # update total bytes read from central directory + total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] + + centdir[_CD_EXTRA_FIELD_LENGTH] + + centdir[_CD_COMMENT_LENGTH]) + + if self.debug > 2: + print("total", total) + + end_offset = self.start_dir + for zinfo in reversed(sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset)): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset + + def namelist(self): + """Return a list of file names in the archive.""" + return [data.filename for data in self.filelist] + + def infolist(self): + """Return a list of class ZipInfo instances for files in the + archive.""" + return self.filelist + + def printdir(self, file=None): + """Print a table of contents for the zip file.""" + print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), + file=file) + for zinfo in self.filelist: + date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] + print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), + file=file) + + def testzip(self): + """Read all the files and check the CRC. + + Return None if all files could be read successfully, or the name + of the offending file otherwise.""" + chunk_size = 2 ** 20 + for zinfo in self.filelist: + try: + # Read by chunks, to avoid an OverflowError or a + # MemoryError with very large embedded files. + with self.open(zinfo.filename, "r") as f: + while f.read(chunk_size): # Check CRC-32 + pass + except BadZipFile: + return zinfo.filename + + def getinfo(self, name): + """Return the instance of ZipInfo given 'name'.""" + info = self.NameToInfo.get(name) + if info is None: + raise KeyError( + 'There is no item named %r in the archive' % name) + + return info + + def setpassword(self, pwd): + """Set default password for encrypted files.""" + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) + if pwd: + self.pwd = pwd + else: + self.pwd = None + + @property + def comment(self): + """The comment text associated with the ZIP file.""" + return self._comment + + @comment.setter + def comment(self, comment): + if not isinstance(comment, bytes): + raise TypeError("comment: expected bytes, got %s" % type(comment).__name__) + # check for valid comment length + if len(comment) > ZIP_MAX_COMMENT: + import warnings + warnings.warn('Archive comment is too long; truncating to %d bytes' + % ZIP_MAX_COMMENT, stacklevel=2) + comment = comment[:ZIP_MAX_COMMENT] + self._comment = comment + self._didModify = True + + def read(self, name, pwd=None): + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" + with self.open(name, "r", pwd) as fp: + return fp.read() + + def open(self, name, mode="r", pwd=None, *, force_zip64=False): + """Return file-like object for 'name'. + + name is a string for the file name within the ZIP file, or a ZipInfo + object. + + mode should be 'r' to read a file already in the ZIP file, or 'w' to + write to a file newly added to the archive. + + pwd is the password to decrypt files (only used for reading). + + When writing, if the file size is not known in advance but may exceed + 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large + files. If the size is known in advance, it is best to pass a ZipInfo + instance for name, with zinfo.file_size set. + """ + if mode not in {"r", "w"}: + raise ValueError('open() requires mode "r" or "w"') + if pwd and (mode == "w"): + raise ValueError("pwd is only supported for reading files") + if not self.fp: + raise ValueError( + "Attempt to use ZIP archive that was already closed") + + # Make sure we have an info object + if isinstance(name, self._ZipInfo): + # 'name' is already an info object + zinfo = name + elif mode == 'w': + zinfo = self._ZipInfo(name) + zinfo.compress_type = self.compression + zinfo._compresslevel = self.compresslevel + else: + # Get info object for name + zinfo = self.getinfo(name) + + if mode == 'w': + return self._open_to_write(zinfo, force_zip64=force_zip64) + + if self._writing: + raise ValueError("Can't read from the ZIP file while there " + "is an open writing handle on it. " + "Close the writing handle before trying to read.") + + # Open for reading: + self._fileRefCnt += 1 + zef_file = _SharedFile(self.fp, zinfo.header_offset, + self._fpclose, self._lock, lambda: self._writing) + try: + # Skip the file header: + fheader = zef_file.read(sizeFileHeader) + if len(fheader) != sizeFileHeader: + raise BadZipFile("Truncated file header") + fheader = struct.unpack(structFileHeader, fheader) + if fheader[_FH_SIGNATURE] != stringFileHeader: + raise BadZipFile("Bad magic number for file header") + + fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) + if fheader[_FH_EXTRA_FIELD_LENGTH]: + zef_file.seek(fheader[_FH_EXTRA_FIELD_LENGTH], whence=1) + + if zinfo.flag_bits & _MASK_COMPRESSED_PATCH: + # Zip 2.7: compressed patched data + raise NotImplementedError("compressed patched data (flag bit 5)") + + if zinfo.flag_bits & _MASK_STRONG_ENCRYPTION: + # strong encryption + raise NotImplementedError("strong encryption (flag bit 6)") + + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & _MASK_UTF_FILENAME: + # UTF-8 filename + fname_str = fname.decode("utf-8") + else: + fname_str = fname.decode(self.metadata_encoding or "cp437") + + if fname_str != zinfo.orig_filename: + raise BadZipFile( + 'File name in directory %r and header %r differ.' + % (zinfo.orig_filename, fname)) + + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + if zinfo._end_offset == zinfo.header_offset: + import warnings + warnings.warn( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)", + skip_file_prefixes=(os.path.dirname(__file__),)) + else: + raise BadZipFile( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)") + + # check for encrypted flag & handle password + is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED + if is_encrypted: + if not pwd: + pwd = self.pwd + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) + if not pwd: + raise RuntimeError("File %r is encrypted, password " + "required for extraction" % name) + else: + pwd = None + + return self._ZipExtFile(zef_file, mode, zinfo, pwd, True) + except: + zef_file.close() + raise + + def _open_to_write(self, zinfo, force_zip64=False): + if force_zip64 and not self._allowZip64: + raise ValueError( + "force_zip64 is True, but allowZip64 was False when opening " + "the ZIP file." + ) + if self._writing: + raise ValueError("Can't write to the ZIP file while there is " + "another write handle open on it. " + "Close the first handle before opening another.") + + # Size and CRC are overwritten with correct data after processing the file + zinfo.compress_size = 0 + zinfo.CRC = 0 + + zinfo.flag_bits = 0x00 + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + if not self._seekable: + zinfo.flag_bits |= _MASK_USE_DATA_DESCRIPTOR + + if not zinfo.external_attr: + zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- + + # Compressed size can be larger than uncompressed size + zip64 = force_zip64 or (zinfo.file_size * 1.05 > ZIP64_LIMIT) + if not self._allowZip64 and zip64: + raise LargeZipFile("Filesize would require ZIP64 extensions") + + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() + + self._writecheck(zinfo) + self._didModify = True + + self.fp.write(zinfo.FileHeader(zip64)) + + self._writing = True + return _ZipWriteFile(self, zinfo, zip64) + + def extract(self, member, path=None, pwd=None): + """Extract a member from the archive to the current working directory, + using its full name. Its file information is extracted as accurately + as possible. `member' may be a filename or a ZipInfo object. You can + specify a different directory using `path'. You can specify the + password to decrypt the file using 'pwd'. + """ + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + + return self._extract_member(member, path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all members from the archive to the current working + directory. `path' specifies a different directory to extract to. + `members' is optional and must be a subset of the list returned + by namelist(). You can specify the password to decrypt all files + using 'pwd'. + """ + if members is None: + members = self.namelist() + + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + + for zipinfo in members: + self._extract_member(zipinfo, path, pwd) + + @classmethod + def _sanitize_windows_name(cls, arcname, pathsep): + """Replace bad characters and remove trailing dots from parts.""" + table = cls._windows_illegal_name_trans_table + if not table: + illegal = ':<>|"?*' + table = str.maketrans(illegal, '_' * len(illegal)) + cls._windows_illegal_name_trans_table = table + arcname = arcname.translate(table) + # remove trailing dots and spaces + arcname = (x.rstrip(' .') for x in arcname.split(pathsep)) + # rejoin, removing empty parts. + arcname = pathsep.join(x for x in arcname if x) + return arcname + + def _extract_member(self, member, targetpath, pwd): + """Extract the ZipInfo object 'member' to a physical + file on the path targetpath. + """ + if not isinstance(member, self._ZipInfo): + member = self.getinfo(member) + + # build the destination pathname, replacing + # forward slashes to platform specific separators. + arcname = member.filename.replace('/', os.path.sep) + + if os.path.altsep: + arcname = arcname.replace(os.path.altsep, os.path.sep) + # interpret absolute pathname as relative, remove drive letter or + # UNC path, redundant separators, "." and ".." components. + arcname = os.path.splitdrive(arcname)[1] + invalid_path_parts = ('', os.path.curdir, os.path.pardir) + arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) + if x not in invalid_path_parts) + if os.path.sep == '\\': + # filter illegal characters on Windows + arcname = self._sanitize_windows_name(arcname, os.path.sep) + + if not arcname and not member.is_dir(): + raise ValueError("Empty filename.") + + targetpath = os.path.join(targetpath, arcname) + targetpath = os.path.normpath(targetpath) + + # Create all upper directories if necessary. + upperdirs = os.path.dirname(targetpath) + if upperdirs and not os.path.exists(upperdirs): + os.makedirs(upperdirs) + + if member.is_dir(): + if not os.path.isdir(targetpath): + os.mkdir(targetpath) + return targetpath + + with self.open(member, pwd=pwd) as source, \ + open(targetpath, "wb") as target: + shutil.copyfileobj(source, target) + + return targetpath + + def _writecheck(self, zinfo): + """Check for errors before writing a file to the archive.""" + if zinfo.filename in self.NameToInfo: + import warnings + warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) + if self.mode not in ('w', 'x', 'a'): + raise ValueError("write() requires mode 'w', 'x', or 'a'") + if not self.fp: + raise ValueError( + "Attempt to write ZIP archive that was already closed") + _check_compression(zinfo.compress_type) + if not self._allowZip64: + requires_zip64 = None + if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: + requires_zip64 = "Files count" + elif zinfo.file_size > ZIP64_LIMIT: + requires_zip64 = "Filesize" + elif zinfo.header_offset > ZIP64_LIMIT: + requires_zip64 = "Zipfile size" + if requires_zip64: + raise LargeZipFile(requires_zip64 + + " would require ZIP64 extensions") + + def write(self, filename, arcname=None, + compress_type=None, compresslevel=None): + """Put the bytes from filename into the archive under the name + arcname.""" + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists" + ) + + zinfo = self._ZipInfo.from_file(filename, arcname, + strict_timestamps=self._strict_timestamps) + + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 + self.mkdir(zinfo) + else: + if compress_type is not None: + zinfo.compress_type = compress_type + else: + zinfo.compress_type = self.compression + + if compresslevel is not None: + zinfo._compresslevel = compresslevel + else: + zinfo._compresslevel = self.compresslevel + + with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: + shutil.copyfileobj(src, dest, 1024*8) + + def writestr(self, zinfo_or_arcname, data, + compress_type=None, compresslevel=None): + """Write a file into the archive. The contents is 'data', which + may be either a 'str' or a 'bytes' instance; if it is a 'str', + it is encoded as UTF-8 first. + 'zinfo_or_arcname' is either a ZipInfo instance or + the name of the file in the archive.""" + if isinstance(data, str): + data = data.encode("utf-8") + if not isinstance(zinfo_or_arcname, self._ZipInfo): + zinfo = self._ZipInfo(filename=zinfo_or_arcname, + date_time=time.localtime(time.time())[:6]) + zinfo.compress_type = self.compression + zinfo._compresslevel = self.compresslevel + if zinfo.filename.endswith('/'): + zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.external_attr = 0o600 << 16 # ?rw------- + else: + zinfo = zinfo_or_arcname + + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists." + ) + + if compress_type is not None: + zinfo.compress_type = compress_type + + if compresslevel is not None: + zinfo._compresslevel = compresslevel + + zinfo.file_size = len(data) # Uncompressed size + with self._lock: + with self.open(zinfo, mode='w') as dest: + dest.write(data) + + def mkdir(self, zinfo_or_directory_name, mode=511): + """Creates a directory inside the zip archive.""" + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + if not zinfo.is_dir(): + raise ValueError("The given ZipInfo does not describe a directory") + elif isinstance(zinfo_or_directory_name, str): + directory_name = zinfo_or_directory_name + if not directory_name.endswith("/"): + directory_name += "/" + zinfo = self._ZipInfo(directory_name) + zinfo.compress_size = 0 + zinfo.CRC = 0 + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + raise TypeError("Expected type str or ZipInfo") + + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + + self._writecheck(zinfo) + self._didModify = True + + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self.fp.write(zinfo.FileHeader(False)) + self.start_dir = self.fp.tell() + + def __del__(self): + """Call the "close()" method in case the user forgot.""" + self.close() + + def close(self): + """Close the file, and for mode 'w', 'x' and 'a' write the ending + records.""" + if self.fp is None: + return + + if self._writing: + raise ValueError("Can't close the ZIP file while there is " + "an open writing handle on it. " + "Close the writing handle before closing the zip.") + + try: + if self.mode in ('w', 'x', 'a') and self._didModify: # write ending records + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + self._write_end_record() + finally: + fp = self.fp + self.fp = None + self._fpclose(fp) + + def _write_end_record(self): + for zinfo in self.filelist: # write central directory + dt = zinfo.date_time + dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + extra = [] + if zinfo.file_size > ZIP64_LIMIT \ + or zinfo.compress_size > ZIP64_LIMIT: + extra.append(zinfo.file_size) + extra.append(zinfo.compress_size) + file_size = 0xffffffff + compress_size = 0xffffffff + else: + file_size = zinfo.file_size + compress_size = zinfo.compress_size + + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) + header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset + + extra_data = zinfo.extra + min_version = 0 + if extra: + # Append a ZIP64 field to the extra's + extra_data = _strip_extra(extra_data, (1,)) + extra_data = struct.pack( + ' ZIP_FILECOUNT_LIMIT: + requires_zip64 = "Files count" + elif centDirOffset > ZIP64_LIMIT: + requires_zip64 = "Central directory offset" + elif centDirSize > ZIP64_LIMIT: + requires_zip64 = "Central directory size" + if requires_zip64: + # Need to write the ZIP64 end-of-archive records + if not self._allowZip64: + raise LargeZipFile(requires_zip64 + + " would require ZIP64 extensions") + zip64endrec = struct.pack( + structEndArchive64, stringEndArchive64, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, + centDirSize, centDirOffset) + self.fp.write(zip64endrec) + + zip64locrec = struct.pack( + structEndArchive64Locator, + stringEndArchive64Locator, 0, pos2, 1) + self.fp.write(zip64locrec) + centDirCount = min(centDirCount, 0xFFFF) + centDirSize = min(centDirSize, 0xFFFFFFFF) + centDirOffset = min(centDirOffset, 0xFFFFFFFF) + + endrec = struct.pack(structEndArchive, stringEndArchive, + 0, 0, centDirCount, centDirCount, + centDirSize, centDirOffset, len(self._comment)) + self.fp.write(endrec) + self.fp.write(self._comment) + if self.mode == "a": + self.fp.truncate() + self.fp.flush() + + def _fpclose(self, fp): + assert self._fileRefCnt > 0 + self._fileRefCnt -= 1 + if not self._fileRefCnt and not self._filePassed: + fp.close() + + +class PyZipFile(ZipFile): + """Class to create ZIP archives with Python library files and packages.""" + + def __init__(self, file, mode="r", compression=ZIP_STORED, + allowZip64=True, optimize=-1): + ZipFile.__init__(self, file, mode=mode, compression=compression, + allowZip64=allowZip64) + self._optimize = optimize + + def writepy(self, pathname, basename="", filterfunc=None): + """Add all files from "pathname" to the ZIP archive. + + If pathname is a package directory, search the directory and + all package subdirectories recursively for all *.py and enter + the modules into the archive. If pathname is a plain + directory, listdir *.py and enter all modules. Else, pathname + must be a Python *.py file and the module will be put into the + archive. Added modules are always module.pyc. + This method will compile the module.py into module.pyc if + necessary. + If filterfunc(pathname) is given, it is called with every argument. + When it is False, the file or directory is skipped. + """ + pathname = os.fspath(pathname) + if filterfunc and not filterfunc(pathname): + if self.debug: + label = 'path' if os.path.isdir(pathname) else 'file' + print('%s %r skipped by filterfunc' % (label, pathname)) + return + dir, name = os.path.split(pathname) + if os.path.isdir(pathname): + initname = os.path.join(pathname, "__init__.py") + if os.path.isfile(initname): + # This is a package directory, add it + if basename: + basename = "%s/%s" % (basename, name) + else: + basename = name + if self.debug: + print("Adding package in", pathname, "as", basename) + fname, arcname = self._get_codename(initname[0:-3], basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + dirlist = sorted(os.listdir(pathname)) + dirlist.remove("__init__.py") + # Add all *.py files and package subdirectories + for filename in dirlist: + path = os.path.join(pathname, filename) + root, ext = os.path.splitext(filename) + if os.path.isdir(path): + if os.path.isfile(os.path.join(path, "__init__.py")): + # This is a package directory, add it + self.writepy(path, basename, + filterfunc=filterfunc) # Recursive call + elif ext == ".py": + if filterfunc and not filterfunc(path): + if self.debug: + print('file %r skipped by filterfunc' % path) + continue + fname, arcname = self._get_codename(path[0:-3], + basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + else: + # This is NOT a package directory, add its files at top level + if self.debug: + print("Adding files from directory", pathname) + for filename in sorted(os.listdir(pathname)): + path = os.path.join(pathname, filename) + root, ext = os.path.splitext(filename) + if ext == ".py": + if filterfunc and not filterfunc(path): + if self.debug: + print('file %r skipped by filterfunc' % path) + continue + fname, arcname = self._get_codename(path[0:-3], + basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + else: + if pathname[-3:] != ".py": + raise RuntimeError( + 'Files added with writepy() must end with ".py"') + fname, arcname = self._get_codename(pathname[0:-3], basename) + if self.debug: + print("Adding file", arcname) + self.write(fname, arcname) + + def _get_codename(self, pathname, basename): + """Return (filename, archivename) for the path. + + Given a module name path, return the correct file path and + archive name, compiling if necessary. For example, given + /python/lib/string, return (/python/lib/string.pyc, string). + """ + def _compile(file, optimize=-1): + import py_compile + if self.debug: + print("Compiling", file) + try: + py_compile.compile(file, doraise=True, optimize=optimize) + except py_compile.PyCompileError as err: + print(err.msg) + return False + return True + + file_py = pathname + ".py" + file_pyc = pathname + ".pyc" + pycache_opt0 = importlib.util.cache_from_source(file_py, optimization='') + pycache_opt1 = importlib.util.cache_from_source(file_py, optimization=1) + pycache_opt2 = importlib.util.cache_from_source(file_py, optimization=2) + if self._optimize == -1: + # legacy mode: use whatever file is present + if (os.path.isfile(file_pyc) and + os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyc file. + arcname = fname = file_pyc + elif (os.path.isfile(pycache_opt0) and + os.stat(pycache_opt0).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt0 + arcname = file_pyc + elif (os.path.isfile(pycache_opt1) and + os.stat(pycache_opt1).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt1 + arcname = file_pyc + elif (os.path.isfile(pycache_opt2) and + os.stat(pycache_opt2).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt2 + arcname = file_pyc + else: + # Compile py into PEP 3147 pyc file. + if _compile(file_py): + if sys.flags.optimize == 0: + fname = pycache_opt0 + elif sys.flags.optimize == 1: + fname = pycache_opt1 + else: + fname = pycache_opt2 + arcname = file_pyc + else: + fname = arcname = file_py + else: + # new mode: use given optimization level + if self._optimize == 0: + fname = pycache_opt0 + arcname = file_pyc + else: + arcname = file_pyc + if self._optimize == 1: + fname = pycache_opt1 + elif self._optimize == 2: + fname = pycache_opt2 + else: + msg = "invalid value for 'optimize': {!r}".format(self._optimize) + raise ValueError(msg) + if not (os.path.isfile(fname) and + os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): + if not _compile(file_py, optimize=self._optimize): + fname = arcname = file_py + archivename = os.path.split(arcname)[1] + if basename: + archivename = "%s/%s" % (basename, archivename) + return (fname, archivename) + + +def main(args=None): + import argparse + + description = 'A simple command-line interface for zipfile module.' + parser = argparse.ArgumentParser(description=description) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-l', '--list', metavar='', + help='Show listing of a zipfile') + group.add_argument('-e', '--extract', nargs=2, + metavar=('', ''), + help='Extract zipfile into target dir') + group.add_argument('-c', '--create', nargs='+', + metavar=('', ''), + help='Create zipfile from sources') + group.add_argument('-t', '--test', metavar='', + help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='', + help='Specify encoding of member names for -l, -e and -t') + args = parser.parse_args(args) + + encoding = args.metadata_encoding + + if args.test is not None: + src = args.test + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + badfile = zf.testzip() + if badfile: + print("The following enclosed file is corrupted: {!r}".format(badfile)) + print("Done testing") + + elif args.list is not None: + src = args.list + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.printdir() + + elif args.extract is not None: + src, curdir = args.extract + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.extractall(curdir) + + elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + + zip_name = args.create.pop(0) + files = args.create + + def addToZip(zf, path, zippath): + if os.path.isfile(path): + zf.write(path, zippath, ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in sorted(os.listdir(path)): + addToZip(zf, + os.path.join(path, nm), os.path.join(zippath, nm)) + # else: ignore + + with ZipFile(zip_name, 'w') as zf: + for path in files: + zippath = os.path.basename(path) + if not zippath: + zippath = os.path.basename(os.path.dirname(path)) + if zippath in ('', os.curdir, os.pardir): + zippath = '' + addToZip(zf, path, zippath) + + +from ._path import ( # noqa: E402 + Path, + + # used privately for tests + CompleteDirs, # noqa: F401 +) diff --git a/src/zlib/chevahbs b/src/zlib/chevahbs deleted file mode 100755 index edb7d501..00000000 --- a/src/zlib/chevahbs +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Chevah Build Script for zlib. - -# Import shared code. -# The relative paths work in both src/blabla and build/blabla. -source ../../functions.sh -source ../../functions_build.sh -source ../../"BUILD_ENV_ARRAYS" - - -chevahbs_getsources() { - local name="$1" - local ver="$2" - local ext="tar.gz" - local link=https://www.zlib.net/"$name"-"$ver"."$ext" - - download_sources "$name" "$ver" "$link" "$ext" -} - - -chevahbs_configure() { - # To properly quote multiple options, this is a Bash array. - CONF_OPTS=(--static) - execute ./configure --prefix="" "${CONF_OPTS[@]}" -} - - -chevahbs_compile() { - execute "${MAKE_CMD[@]}" -} - -chevahbs_try() { - execute "${MAKE_CMD[@]}" test -} - -chevahbs_cp() { - execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR" -} - - -select_chevahbs_command "$@" diff --git a/src/zlib/sha512.sum b/src/zlib/sha512.sum deleted file mode 100644 index f383544f..00000000 --- a/src/zlib/sha512.sum +++ /dev/null @@ -1 +0,0 @@ -580677aad97093829090d4b605ac81c50327e74a6c2de0b85dd2e8525553f3ddde17556ea46f8f007f89e435493c9a20bc997d1ef1c1c2c23274528e3c46b94f zlib-1.3.1.tar.gz diff --git a/test_compat.sh b/test_compat.sh old mode 100644 new mode 100755 index df459337..720656fa --- a/test_compat.sh +++ b/test_compat.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # Test the Pythia distribtution availabe in DIST agains compat. +# Test are done agains the fully packed version from dist. # # Bash checks @@ -35,7 +36,7 @@ execute pushd compat echo -e "\nPYTHON_CONFIGURATION=default@${PYTHIA_RELEASE}" >>pythia.conf execute mkdir cache # Copy dist file to local cache, if existing. If not, maybe it's online. - cp ../../"$DIST_DIR"/"$PYTHIA_RELEASE"/* cache/ + cp ../../"$DIST_DIR"/* cache/ # Some tests could fail due to causes not related to the new Python. echo "## Getting compat deps... ##" execute ./pythia.sh deps diff --git a/test_pythia.sh b/test_pythia.sh new file mode 100755 index 00000000..e05039f0 --- /dev/null +++ b/test_pythia.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Run own tests for the newly-build Python distribution. +# Tests are done with the version from build folder. +# +# Bash checks +set -o nounset # always check if variables exist +set -o errexit # always exit on error +set -o errtrace # trap errors in functions as well +set -o pipefail # don't ignore exit codes when piping output +set -o functrace # inherit DEBUG and RETURN traps + +TARGET=$1 + +# Set versions for the software to be built and other defaults. +source build.conf + +INSTALL_DIR="$PWD/$BUILD_DIR/python$PY_VERSION-$TARGET" + +OS="${TARGET%-*}" # Retain everything before the last hyphen +ARCH="${TARGET#*-}" # Retain everything after the first hyphen +export OS +export ARCH + +echo "::group::Chevah tests for $OS $ARCH" +if [ ! -d "$BUILD_DIR" ]; then + (>&2 echo "No $BUILD_DIR sub-directory present, try 'build' first!") + exit 220 +fi + +echo "#### Executing Chevah Python tests... ####" +python_binary="$INSTALL_DIR/bin/python" +if [ "$TARGET" == "windows-x64" ]; then + # Post-cleanup, the binary in /bin is named "python", not "python3.x". + python_binary="$INSTALL_DIR/lib/python" +fi +test_file="test_python_binary_dist.py" +cp src/chevah-python-tests/"$test_file" "$BUILD_DIR" +cp src/chevah-python-tests/get_binaries_deps.sh "$BUILD_DIR" +pushd "$BUILD_DIR" +execute "$python_binary" "$test_file" +popd +echo "::endgroup::" From ccdd49b150ae66da0f2bf55eda95429b0ed7ba9f Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 13:09:33 +0100 Subject: [PATCH 03/11] Fix ci. --- .github/workflows/ci.yaml | 21 ++++++--------------- astral-to-pythia.sh | 6 +++--- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 247eea67..07d35561 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,9 @@ # These jobs are preparing the OS, then building the Python package, # run some tests and upload the result. # +# Available list of runners: +# https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job#choosing-github-hosted-runners +# name: CI on: @@ -48,12 +51,6 @@ jobs: run: | yum -y upgrade yum -y install git-core dejagnu bzip2 sudo - # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. - # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. - rm -v /usr/include/libdb/db.h - rm -v /usr/include/gdbm.h - # This is for avoiding building the NIS module with Python 3.12. - rm -v /usr/include/rpc/rpc.h # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - name: Clone repo independently @@ -129,13 +126,7 @@ jobs: - name: Amazon setup run: | yum -y upgrade - yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu bzip2 sudo - # To avoid linking against libdb and gdmb libraries on Amazon Linux 2. - # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd. - rm -v /usr/include/libdb/db.h - rm -v /usr/include/gdbm.h - # This is for avoiding building the NIS module with Python 3.12. - rm -v /usr/include/rpc/rpc.h + yum -y install git-core dejagnu bzip2 sudo # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - name: Clone repo independently @@ -282,7 +273,7 @@ jobs: macos: # For macOS on Intel at GitHub. - runs-on: macos-13 + runs-on: macos-15-intel timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -345,7 +336,7 @@ jobs: macos-m1: # For macOS on Apple Silicon at GitHub. - runs-on: macos-latest + runs-on: macos-14 timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index a9cc0fc7..66ab1d20 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -24,7 +24,7 @@ WINDOWS_X64="x86_64-pc-windows-msvc" # Start with a clean build #rm -rf "$BUILD_DIR" -#mkdir "$BUILD_DIR" +mkdir "$BUILD_DIR" rm -rf "$DIST_DIR" mkdir "$DIST_DIR" @@ -52,7 +52,7 @@ case "$TARGET" in ;; *) # - echo "Unknown target ${file_name}" + echo "Unknown target ${TARGET}" exit 1 ;; @@ -62,7 +62,7 @@ esac file_name="${pbs_arch}-install_only_stripped.tar.gz" source_url="https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-${PY_RELEASE}+${PBS_RELEASE}-${file_name}" echo "> Downloading $file_name" -#curl -L -o $file_name $source_url +curl -L -o $file_name $source_url echo "> Extracting $file_name" From c4d1a56ebd66f62780c4dc9ff9a622bcc8bc491d Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 13:25:40 +0100 Subject: [PATCH 04/11] More fixes. --- .github/workflows/ci.yaml | 10 +++++----- README.rst | 8 ++++---- astral-to-pythia.sh | 7 +++---- build.conf | 4 ++-- publish_dist.sh | 2 +- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 07d35561..79506fac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: timeout-minutes: 5 run: | cd pythia - su chevah -c "./test.sh linux-x64" + su chevah -c "./test_pythia.sh linux-x64" - name: Test compat timeout-minutes: 10 @@ -151,7 +151,7 @@ jobs: timeout-minutes: 5 run: | cd pythia - su chevah -c "./test.sh linux-arm64" + su chevah -c "./test_pythia.sh linux-arm64" - name: Test compat timeout-minutes: 10 @@ -211,7 +211,7 @@ jobs: - name: Test Pythia timeout-minutes: 5 - run: bash ./test.sh windows-x64 + run: bash ./test_pythia.sh windows-x64 - name: Test compat timeout-minutes: 10 @@ -307,7 +307,7 @@ jobs: - name: Test Pythia timeout-minutes: 5 - run: ./test.sh macos-x64 + run: ./test_pythia.sh macos-x64 - name: Test compat timeout-minutes: 10 @@ -349,7 +349,7 @@ jobs: - name: Test Pythia timeout-minutes: 5 - run: ./test.sh macos-arm64 + run: ./test_pythia.sh macos-arm64 - name: Test compat timeout-minutes: 10 diff --git a/README.rst b/README.rst index 48b326ba..cbe26513 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ with Linux MUSL, macOS and Windows installation packages also generated on Linux It comes with a few pre-installed packages, depending on the OS. This is done to simplify building the final distributable package on Linux, regardless of the target OS. -Example on Windows, pywin32 is preinstaleld. +Example on Windows, pywin32 is preinstalled. Supported platforms @@ -55,7 +55,7 @@ Copy `pythia.sh` script into your repo. Copy `pythia.conf` file into your repo and configure your mirrors and targeted Python version. It is designed to create the Python virtual environment and automatically call paver inside the newly created environment. -`pythia.conf` contains a set of Python packages that needs to be intallled to boostrap the `paver` usage. +`pythia.conf` contains a set of Python packages that needs to be installed to bootstrap the `paver` usage. TK is removed to save space and since this is designed for server-side apps. @@ -70,11 +70,11 @@ It has support for uploading into testing vs production. Building: -* `./astral-to-pytia.sh TARGET_ARCH` +* `./astral-to-pythia.sh TARGET_ARCH` TARGET_ARCH needs to match the local arch. Testing: -* `./test_pythia.sh` +* `./test_pythia.sh TARGET_ARCH` * `./test_compat.sh` diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index 66ab1d20..3fe8f2f4 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -24,10 +24,10 @@ WINDOWS_X64="x86_64-pc-windows-msvc" # Start with a clean build #rm -rf "$BUILD_DIR" -mkdir "$BUILD_DIR" +mkdir -p "$BUILD_DIR" rm -rf "$DIST_DIR" -mkdir "$DIST_DIR" +mkdir -p "$DIST_DIR" cd $BUILD_DIR @@ -73,8 +73,6 @@ rm -rf python tar xzf $file_name -pythia_version="${PY_RELEASE}.${PBS_RELEASE}-${pythia_arch}" - case "$TARGET" in *windows*) # Do stuff @@ -111,6 +109,7 @@ esac # Compile all source files to speed up load time and write Pythia version "${python_bin}" -m compileall -r 100 python +pythia_version="${PY_RELEASE}.${PBS_RELEASE}-${TARGET}" echo -n "${pythia_version}" > python/lib/PYTHIA_VERSION pythia_base="python${PY_VERSION}-${TARGET}" diff --git a/build.conf b/build.conf index ab39e721..b7cef9ef 100644 --- a/build.conf +++ b/build.conf @@ -17,8 +17,8 @@ PBS_RELEASE="20260408" PYTHIA_RELEASE="${PY_RELEASE}.${PBS_RELEASE}" # Python modules that have to be built and/or installed in Pythia. -# WARGNING: Also update requirement.txt when this is updated. -PYWIN32_VERSION="308" +# WARNING: Also update requirement.txt when this is updated. +PYWIN32_VERSION="306" # Our custom PyPI server overwrites the default pypi.org option from "build.sh". PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple" diff --git a/publish_dist.sh b/publish_dist.sh index a0039641..b62466cc 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -17,7 +17,7 @@ root_link="https://$UPLOAD_SERVER:20443/testing" target_dir="/${TARGET_DEPLOY}/${PYTHIA_RELEASE}/" batch_file="${BUILD_DIR}/sftp.batch" echo "> Uploading to ${target_dir}" -echo "mkdir ${target_dir}" > "${batch_file}" +echo "-mkdir ${target_dir}" > "${batch_file}" echo "mput -v ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" sftp_opts=(-b ${batch_file} -o IdentityFile=priv_key \ From c2d6f0993bbdf4ec1f480e0479e28674ffd6d700 Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 13:38:07 +0100 Subject: [PATCH 05/11] Fix port for upload. install tar. debug linux x64. --- .github/workflows/ci.yaml | 2 +- astral-to-pythia.sh | 4 +++- publish_dist.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 79506fac..5d2ebd1a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -126,7 +126,7 @@ jobs: - name: Amazon setup run: | yum -y upgrade - yum -y install git-core dejagnu bzip2 sudo + yum -y install git-core dejagnu bzip2 sudo tar unzip # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - name: Clone repo independently diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index 3fe8f2f4..6b0209d8 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -107,6 +107,8 @@ case "$TARGET" in ;; esac +ls -al + # Compile all source files to speed up load time and write Pythia version "${python_bin}" -m compileall -r 100 python pythia_version="${PY_RELEASE}.${PBS_RELEASE}-${TARGET}" @@ -114,6 +116,6 @@ echo -n "${pythia_version}" > python/lib/PYTHIA_VERSION pythia_base="python${PY_VERSION}-${TARGET}" mv python ${pythia_base} -echo "> Creating ${pythia_base} to ../dist/" +echo "> Creating ${pythia_base} for ${pythia_version} to ../dist/" tar czf ../dist/python-${PYTHIA_RELEASE}-${TARGET}.tar.gz ${pythia_base} diff --git a/publish_dist.sh b/publish_dist.sh index b62466cc..b1b46b7e 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -20,7 +20,7 @@ echo "> Uploading to ${target_dir}" echo "-mkdir ${target_dir}" > "${batch_file}" echo "mput -v ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" -sftp_opts=(-b ${batch_file} -o IdentityFile=priv_key \ +sftp_opts=(-P 10022 -b ${batch_file} -o IdentityFile=priv_key \ -o StrictHostKeyChecking=yes) OS="$(uname -s)" From acba4a0127e68c73a5f76a7c62320ef40f99b03e Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 13:50:36 +0100 Subject: [PATCH 06/11] Fix upload. --- astral-to-pythia.sh | 2 +- publish_dist.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index 6b0209d8..2a23e79f 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -107,7 +107,7 @@ case "$TARGET" in ;; esac -ls -al +ls -al python/* # Compile all source files to speed up load time and write Pythia version "${python_bin}" -m compileall -r 100 python diff --git a/publish_dist.sh b/publish_dist.sh index b1b46b7e..fda86a6f 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -18,7 +18,7 @@ target_dir="/${TARGET_DEPLOY}/${PYTHIA_RELEASE}/" batch_file="${BUILD_DIR}/sftp.batch" echo "> Uploading to ${target_dir}" echo "-mkdir ${target_dir}" > "${batch_file}" -echo "mput -v ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" +echo "mput ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" sftp_opts=(-P 10022 -b ${batch_file} -o IdentityFile=priv_key \ -o StrictHostKeyChecking=yes) From fcb67953fc4bd694165aede85d323b857fc3a52f Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 14:03:06 +0100 Subject: [PATCH 07/11] Fix python binary. --- astral-to-pythia.sh | 9 ++++++++- src/chevah-python-tests/test_python_binary_dist.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index 2a23e79f..01543194 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -93,6 +93,13 @@ case "$TARGET" in *) # Linux and MacOS mv python/include python/lib + + # Cleanup bin folder with simple bin/python + mv python/bin/python${PY_VERSION} python/python + rm -rf python/bin + mkdir python/bin + mv python/python python/bin/ + rm -rf python/share rm -rf python/lib/itcl* rm -rf python/lib/tcl* @@ -103,7 +110,7 @@ case "$TARGET" in rm -rf python/lib/python${PY_VERSION}/turtledemo rm -f python/lib/python${PY_VERSION}/lib-dynload/_tkinter* cp ../src/zipfile_init.py python/lib/python${PY_VERSION}/zipfile/__init__.py - python_bin="python/bin/python" + python_bin="./python/bin/python" ;; esac diff --git a/src/chevah-python-tests/test_python_binary_dist.py b/src/chevah-python-tests/test_python_binary_dist.py index 0e747d08..f7011ff5 100644 --- a/src/chevah-python-tests/test_python_binary_dist.py +++ b/src/chevah-python-tests/test_python_binary_dist.py @@ -322,6 +322,14 @@ def main(): else: print('zlib %s' % (zlib.ZLIB_VERSION,)) + try: + import zipfile + zipfile.ZipFile._ZipExtFile + zipfile.ZipFile._ZipInfo + except: + sys.stderr.write('"zipfile" not patched.\n') + exit_code = 132 + # Check OpenSSL version to prevent linking to OS libs. # On Windows, this version is what upstream embedded with Python. try: From ae98f16398da07556274fd5e97021ecb0050dfbe Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 14:12:45 +0100 Subject: [PATCH 08/11] Debug publish. --- .github/workflows/ci.yaml | 2 +- publish_dist.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d2ebd1a..0c41642e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,7 +50,7 @@ jobs: if: startsWith(matrix.container, 'amazonlinux') run: | yum -y upgrade - yum -y install git-core dejagnu bzip2 sudo + yum -y install git-core dejagnu bzip2 sudo tar # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - name: Clone repo independently diff --git a/publish_dist.sh b/publish_dist.sh index fda86a6f..35aef975 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -16,6 +16,7 @@ root_link="https://$UPLOAD_SERVER:20443/testing" target_dir="/${TARGET_DEPLOY}/${PYTHIA_RELEASE}/" batch_file="${BUILD_DIR}/sftp.batch" +ls -al ${DIST_DIR} echo "> Uploading to ${target_dir}" echo "-mkdir ${target_dir}" > "${batch_file}" echo "mput ${DIST_DIR}/python-*.tar.gz ${target_dir}" >> "${batch_file}" From e65086d0adb958c60e4afe6a4721a8ce8adcae4d Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 14:30:27 +0100 Subject: [PATCH 09/11] Fix alpine and macos build. --- .github/workflows/ci.yaml | 6 ++++-- astral-to-pythia.sh | 2 -- src/chevah-python-tests/test_python_binary_dist.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0c41642e..623aec1f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,12 +45,14 @@ jobs: apk del util-linux-dev curl --output /usr/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 chmod +x /usr/bin/paxctl + echo "TARGET_ARCH=linux-arm64" >> $GITHUB_ENV - name: Amazon setup if: startsWith(matrix.container, 'amazonlinux') run: | yum -y upgrade yum -y install git-core dejagnu bzip2 sudo tar + echo "TARGET_ARCH=linux-x64" >> $GITHUB_ENV # actions/checkout doesn't work on Amazon Linux 2, requires glibc 2.27. - name: Clone repo independently @@ -69,13 +71,13 @@ jobs: run: | chown -R chevah pythia cd pythia - su chevah -c "./astral-to-pythia.sh linux-x64" + su chevah -c "./astral-to-pythia.sh $TARGET_ARCH" - name: Test Pythia timeout-minutes: 5 run: | cd pythia - su chevah -c "./test_pythia.sh linux-x64" + su chevah -c "./test_pythia.sh $TARGET_ARCH" - name: Test compat timeout-minutes: 10 diff --git a/astral-to-pythia.sh b/astral-to-pythia.sh index 01543194..9fa6fc89 100755 --- a/astral-to-pythia.sh +++ b/astral-to-pythia.sh @@ -116,8 +116,6 @@ esac ls -al python/* -# Compile all source files to speed up load time and write Pythia version -"${python_bin}" -m compileall -r 100 python pythia_version="${PY_RELEASE}.${PBS_RELEASE}-${TARGET}" echo -n "${pythia_version}" > python/lib/PYTHIA_VERSION diff --git a/src/chevah-python-tests/test_python_binary_dist.py b/src/chevah-python-tests/test_python_binary_dist.py index f7011ff5..ab0b7724 100644 --- a/src/chevah-python-tests/test_python_binary_dist.py +++ b/src/chevah-python-tests/test_python_binary_dist.py @@ -147,6 +147,7 @@ def get_allowed_deps(): '/usr/lib/libz.1.dylib', '@rpath/_rust.abi3.so', '@rpath/_bcrypt.abi3.s', + '@rpath/libpython3.12.dylib', ] elif platform_system == 'freebsd': # Deps for FreeBSD 12, with full path. From 5d10e518dc9e134d52a87728ffdb60552eff249c Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 14:40:46 +0100 Subject: [PATCH 10/11] Simplify upload. --- publish_dist.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/publish_dist.sh b/publish_dist.sh index 35aef975..4b4afb19 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -36,11 +36,3 @@ case "$OS" in esac "$sftp_cmd" "${sftp_opts[@]}" "$UPLOAD_USER"@"$UPLOAD_SERVER" - -# As dist/ is rebuilt on every build, it should only have 1 sub-dir with 1 pkg. -upload_dir="$(cd "$DIST_DIR" && ls -1)" -pkg_name="$(cd "$DIST_DIR"/"$upload_dir" && ls -1)" - -# Local hierarchy matches the remote one. -echo "Package $pkg_name uploaded to: $root_link/$upload_dir/" -echo "Direct link: $root_link/$upload_dir/$pkg_name" From 5da3e8e7dfd17c4e139645e72845eff07c4fec44 Mon Sep 17 00:00:00 2001 From: Adi Roiban Date: Mon, 13 Apr 2026 14:47:19 +0100 Subject: [PATCH 11/11] Fix Musl build. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 623aec1f..132c0218 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,7 +45,7 @@ jobs: apk del util-linux-dev curl --output /usr/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 chmod +x /usr/bin/paxctl - echo "TARGET_ARCH=linux-arm64" >> $GITHUB_ENV + echo "TARGET_ARCH=linux_musl-x64" >> $GITHUB_ENV - name: Amazon setup if: startsWith(matrix.container, 'amazonlinux')