From 73a95157d82051aa9d47093a3463d981c707da5a Mon Sep 17 00:00:00 2001 From: nicobo Date: Tue, 9 Feb 2021 22:55:09 +0100 Subject: [PATCH] + adding version constraints to prevent some automatic upgrades (that BREAK build !) ~ debian.Dockerfile : rust is copied from an existing image because it's just too complex to maintain --- .github/workflows/dockerhub.yml | 6 ++--- .github/workflows/pypi.yml | 2 +- .travis.yml | 2 +- Develop.md | 4 +-- Makefile | 2 +- README.md | 2 +- alpine.Dockerfile | 9 ++++--- constraints.lock | 46 +++++++++++++++++++++++++++++++++ constraints.txt | 10 +++++++ debian.Dockerfile | 34 +++++++++++++++++------- docker/docker-images.puml | 2 ++ requirements-build.txt | 2 ++ requirements-runtime.txt | 2 ++ 13 files changed, 102 insertions(+), 21 deletions(-) create mode 100644 constraints.lock create mode 100644 constraints.txt diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 440e89d..a983efc 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -35,7 +35,7 @@ jobs: - name: Generate tags run: | python -m pip install --upgrade pip - pip install -r requirements-build.txt + pip install -c constraints.txt -r requirements-build.txt # Stores a variable for each variant echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV @@ -118,7 +118,7 @@ jobs: - name: Generate tags run: | python -m pip install --upgrade pip - pip install -r requirements-build.txt + pip install -c constraints.txt -r requirements-build.txt # Stores a variable for each variant echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV @@ -203,7 +203,7 @@ jobs: - name: Generate tags run: | python -m pip install --upgrade pip - pip install -r requirements-build.txt + pip install -c constraints.txt -r requirements-build.txt # Stores a variable for each variant echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index aab2258..5053109 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements-build.txt + pip install -c constraints.txt -r requirements-build.txt - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} diff --git a/.travis.yml b/.travis.yml index 1b1a44c..d77e333 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ addons: before_install: - curl https://sh.rustup.rs -sSf | sh -s -- -y && source $HOME/.cargo/env install: - - pip3 install -r requirements-build.txt -r requirements-runtime.txt + - pip3 install -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt # Generates nicobot/version.py # 'python' points to Python 2.7 on macOS but points to Python 3.8 on Linux and Windows # 'python3' is a 'command not found' error on Windows but 'py' works on Windows only diff --git a/Develop.md b/Develop.md index 1646794..1b20dd9 100644 --- a/Develop.md +++ b/Develop.md @@ -10,7 +10,7 @@ Install Python dependencies (for both building and running) and generate `nicobot/version.py` with : - pip3 install -r requirements-build.txt -r requirements-runtime.txt + pip3 install -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt python3 setup.py build To run unit tests : @@ -37,7 +37,7 @@ To install the test package from test.pypi.org and check that it works : python3 -m venv venv/pypi_test && source venv/pypi_test/bin/activate # Then install dependencies using the regular pypi repo - pip3 install -r requirements-runtime.txt + pip3 install -c constraints.txt -r requirements-runtime.txt # Finally install this package from the test repo pip3 install -i https://test.pypi.org/simple/ --no-deps nicobot diff --git a/Makefile b/Makefile index 0720f3c..cfcc9ed 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ clean: rm -rf build build: - pip3 install --upgrade -r requirements-build.txt -r requirements-runtime.txt + pip3 install --upgrade -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt python3 setup.py build sdist bdist_wheel docker-build-alpine: diff --git a/README.md b/README.md index 191317d..2b7c80c 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ To install from source you need to fulfill the requirements for a package instal git clone https://github.com/nicolabs/nicobot.git cd nicobot python3 setup.py build - pip3 install -r requirements-runtime.txt . + pip3 install -c constraints.txt -r requirements-runtime.txt . > **NOTE** > Depending on your platform, `pip install` may trigger a compilation for some or all of the dependencies (i.e. when *Python wheels* are not available). diff --git a/alpine.Dockerfile b/alpine.Dockerfile index 407e90b..d2c5d32 100644 --- a/alpine.Dockerfile +++ b/alpine.Dockerfile @@ -12,6 +12,8 @@ # STAGE 1 : Builder image # +# This builder must have a Python version comptabile with the final image +# So built artifacts will work FROM python:3-alpine as builder # python:3-alpine misses gcc, ffi.h, ... @@ -48,12 +50,13 @@ RUN apk add --no-cache build-base gcc abuild binutils cmake \ WORKDIR /usr/src/app # Builds & installs requirements (shoduld not change often) -COPY requirements-*.txt \ +COPY constraints.txt \ + requirements-*.txt \ setup.py \ . # This step WILL trigger a compilation on platforms without matching Python wheels RUN python3 -m pip install --no-cache-dir --user --upgrade pip && \ - python3 -m pip install --no-cache-dir --user -r requirements-build.txt -r requirements-runtime.txt + python3 -m pip install --no-cache-dir --user -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt # Builds & installs nicobot (should change often, especially the .git directory) COPY LICENSE \ @@ -70,7 +73,7 @@ RUN python3 -m pip install --no-cache-dir --user . # # The base image must provide : -# - Python version > 3.4.2 +# - Python matching setup.py's python_requires # - bash # - glibc FROM python:3-alpine diff --git a/constraints.lock b/constraints.lock new file mode 100644 index 0000000..ef6be72 --- /dev/null +++ b/constraints.lock @@ -0,0 +1,46 @@ +aiodns==2.0.0 +bleach==3.3.0 +certifi==2020.12.5 +cffi==1.14.4 +chardet==4.0.0 +colorama==0.4.4 +cryptography==3.4.3 +docutils==0.16 +DoubleRatchet==0.7.0 +emoji-country-flag==1.2.3 +idna==2.10 +jeepney==0.6.0 +keyring==22.0.1 +OMEMO==0.12.0 +omemo-backend-signal==0.2.6 +packaging==20.9 +pip==20.0.2 +pkg-resources==0.0.0 +pkginfo==1.7.0 +protobuf==3.14.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycares==3.1.1 +pycparser==2.20 +Pygments==2.7.4 +PyNaCl==1.4.0 +pyparsing==2.4.7 +python-i18n==0.3.9 +PyYAML==5.4.1 +readme-renderer==28.0 +requests==2.25.1 +requests-toolbelt==0.9.1 +rfc3986==1.4.0 +SecretStorage==3.3.1 +setuptools==44.0.0 +setuptools-scm==5.0.1 +six==1.15.0 +slixmpp==1.7.0 +slixmpp-omemo==0.4.0 +tqdm==4.56.0 +twine==3.3.0 +urllib3==1.26.3 +webencodings==0.5.1 +wheel==0.36.2 +X3DH==0.5.9 +XEdDSA==0.4.7 diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..36475ed --- /dev/null +++ b/constraints.txt @@ -0,0 +1,10 @@ +# Syntax : https://pip.pypa.io/en/stable/user_guide/#constraints-files + +# Since 3.4.3 cryptography requires rust 1.45+, which is not available on all platforms +# See https://cryptography.io/en/latest/changelog.html#v3-4-3 +# For now there is a patch (see debian.Dockerfile) with CRYPTOGRAPHY_DONT_BUILD_RUST=1 +# but next versions of cryptography will probably break the build so we lock it +cryptography<=3.4.3 + +# Since 1.7.0 slixmpp does not compile on Python 3.5 so it should be upgraded by hand +slixmpp<=1.7.0 diff --git a/debian.Dockerfile b/debian.Dockerfile index 284cc42..bccae40 100644 --- a/debian.Dockerfile +++ b/debian.Dockerfile @@ -2,6 +2,8 @@ # STAGE 1 : Builder image # +# This builder must have a Python version comptabile with the final image +# So built artifacts will work FROM python:3 as builder RUN apt-get update && \ @@ -19,25 +21,39 @@ RUN apt-get update && \ git # Rust is a requirement to build the 'cryptography' Python module -# The recommended procedure is to use 'rustup but the both Debian & -# Alpine ship with more CPU architectures so we use the OS' packages. -# At the time of writing rustup only provides installers for x86_64 and -# aarch64 (arm64). +# but it's sooo complicated to install it on many platforms... +# The recommended procedure is to use 'rustup but Alpine ships with more CPU +# architectures so we use the OS' packages. (At the time of writing rustup only +# provides installers for x86_64 and aarch64 (arm64).) # https://forge.rust-lang.org/infra/other-installation-methods.html # Alpine packages : https://pkgs.alpinelinux.org/packages?name=rust # Debian packages : https://packages.debian.org/buster/rustc -# Here we try the official method first and fall back to the package -RUN (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y) || apt-get install -y rustc +# FIXME The rustup script does not work for linux/386 : it seems it installs x86_64 instead +#RUN (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && . $HOME/.cargo/env) || apt-get install -y rustc +# As of writing, copying from the rust image is supported for the following archs : 386,amd64,armv7,arm64 +COPY --from=rust:slim /usr/local/cargo /usr/local/cargo +COPY --from=rust:slim /usr/local/rustup /usr/local/rustup +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +RUN rustc --version WORKDIR /usr/src/app # Builds & installs requirements (shoduld not change often) -COPY requirements-*.txt \ +COPY constraints.txt \ + requirements-*.txt \ setup.py \ . +# # FIXME Either with rustup or rustc package, rust version for linux/386 on debian is only 1.41 as of buster +# # => Since 3.4.3 cryptography requires rust 1.45+, which is not available on all platforms +# # https://cryptography.io/en/latest/changelog.html#v3-4-3 +# # => For now we use the patch below to disable rust but the next version of cryptography +# # will probably force us to use packages from debian testing or to use an older cryptography version +# ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 # This step WILL trigger a compilation on platforms without matching Python wheels RUN python3 -m pip install --no-cache-dir --user --upgrade pip && \ - python3 -m pip install --no-cache-dir --user -r requirements-build.txt -r requirements-runtime.txt + python3 -m pip install --no-cache-dir --user -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt # Builds & installs nicobot (should change often, especially the .git directory) COPY LICENSE \ @@ -54,7 +70,7 @@ RUN python3 -m pip install --no-cache-dir --user . # # The base image must provide : -# - Python version > 3.4.2 +# - Python matching setup.py's python_requires # - bash # - glibc FROM python:3-slim diff --git a/docker/docker-images.puml b/docker/docker-images.puml index 49fe1b1..e695eb4 100644 --- a/docker/docker-images.puml +++ b/docker/docker-images.puml @@ -16,6 +16,7 @@ left to right direction object "Docker context" as docker_context $dockercontext object "python:3" as python3 $baseimage object "python:3-slim" as python3_slim $baseimage +object "rust:slim" as rust_slim $baseimage object "debian builder" as nicobot_debian_builder <> $tempimage { Builds nicobot & dependencies } @@ -24,6 +25,7 @@ object "nicolabs/nicobot:debian" as nicobot_debian $finalimage { } python3 --> nicobot_debian_builder : FROM +rust_slim --> nicobot_debian_builder : COPY docker_context ...> nicobot_debian_builder : COPY python3_slim --> nicobot_debian : FROM diff --git a/requirements-build.txt b/requirements-build.txt index 665ecb0..aad8bba 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1,4 +1,6 @@ ##### Requirements for building only ##### +# Syntax : https://pip.pypa.io/en/stable/user_guide/#requirements-files + setuptools wheel twine diff --git a/requirements-runtime.txt b/requirements-runtime.txt index 583384d..782933b 100644 --- a/requirements-runtime.txt +++ b/requirements-runtime.txt @@ -1,3 +1,5 @@ +# Syntax : https://pip.pypa.io/en/stable/user_guide/#requirements-files + ##### Requirements for signalcli ##### python-i18n