+ adding version constraints to prevent some automatic upgrades (that BREAK build !)

~ debian.Dockerfile : rust is copied from an existing image because it's 
just too complex to maintain

.github/workflows/dockerhub.yml

@@ -35,7 +35,7 @@ jobs:
     - name: Generate tags
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-build.txt
+        pip install -c constraints.txt -r requirements-build.txt
         # Stores a variable for each variant
         echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV
         echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV
@@ -118,7 +118,7 @@ jobs:
     - name: Generate tags
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-build.txt
+        pip install -c constraints.txt -r requirements-build.txt
         # Stores a variable for each variant
         echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV
         echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV
@@ -203,7 +203,7 @@ jobs:
     - name: Generate tags
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-build.txt
+        pip install -c constraints.txt -r requirements-build.txt
         # Stores a variable for each variant
         echo DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV
         echo SIGNAL_DEBIAN_TAGS=$(python3 setup.py -q docker_tags --image nicolabs/nicobot --variant signal-debian --ref ${{ github.ref }} --sep ,) >> $GITHUB_ENV

.github/workflows/pypi.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-build.txt
+        pip install -c constraints.txt -r requirements-build.txt
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}

.travis.yml

@@ -30,7 +30,7 @@ addons:
 before_install:
   - curl https://sh.rustup.rs -sSf | sh -s -- -y && source $HOME/.cargo/env
 install:
-  - pip3 install -r requirements-build.txt -r requirements-runtime.txt
+  - pip3 install -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt
   # Generates nicobot/version.py
   # 'python' points to Python 2.7 on macOS but points to Python 3.8 on Linux and Windows
   # 'python3' is a 'command not found' error on Windows but 'py' works on Windows only

Develop.md

@@ -10,7 +10,7 @@
 
 Install Python dependencies (for both building and running) and generate `nicobot/version.py` with :
 
-    pip3 install -r requirements-build.txt -r requirements-runtime.txt
+    pip3 install -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt
     python3 setup.py build
 
 To run unit tests :
@@ -37,7 +37,7 @@ To install the test package from test.pypi.org and check that it works :
     python3 -m venv venv/pypi_test && source venv/pypi_test/bin/activate
 
     # Then install dependencies using the regular pypi repo
-    pip3 install -r requirements-runtime.txt
+    pip3 install -c constraints.txt -r requirements-runtime.txt
 
     # Finally install this package from the test repo
     pip3 install -i https://test.pypi.org/simple/ --no-deps nicobot

Makefile

@@ -2,7 +2,7 @@ clean:
 	rm -rf build
 
 build:
-	pip3 install --upgrade -r requirements-build.txt -r requirements-runtime.txt
+	pip3 install --upgrade -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt
 	python3 setup.py build sdist bdist_wheel
 
 docker-build-alpine:

README.md

@@ -70,7 +70,7 @@ To install from source you need to fulfill the requirements for a package instal
     git clone https://github.com/nicolabs/nicobot.git
     cd nicobot
     python3 setup.py build
-    pip3 install -r requirements-runtime.txt .
+    pip3 install -c constraints.txt -r requirements-runtime.txt .
 
 > **NOTE**
 > Depending on your platform, `pip install` may trigger a compilation for some or all of the dependencies (i.e. when *Python wheels* are not available).

alpine.Dockerfile

@@ -12,6 +12,8 @@
 # STAGE 1 : Builder image
 #
 
+# This builder must have a Python version comptabile with the final image
+# So built artifacts will work
 FROM python:3-alpine as builder
 
 # python:3-alpine misses gcc, ffi.h, ...
@@ -48,12 +50,13 @@ RUN apk add --no-cache build-base gcc abuild binutils cmake \
 WORKDIR /usr/src/app
 
 # Builds & installs requirements (shoduld not change often)
-COPY requirements-*.txt \
+COPY constraints.txt \
+     requirements-*.txt \
      setup.py \
      .
 # This step WILL trigger a compilation on platforms without matching Python wheels
 RUN python3 -m pip install --no-cache-dir --user --upgrade pip && \
-    python3 -m pip install --no-cache-dir --user -r requirements-build.txt -r requirements-runtime.txt
+    python3 -m pip install --no-cache-dir --user -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt
 
 # Builds & installs nicobot (should change often, especially the .git directory)
 COPY LICENSE \
@@ -70,7 +73,7 @@ RUN python3 -m pip install --no-cache-dir --user .
 #
 
 # The base image must provide :
-# - Python version > 3.4.2
+# - Python matching setup.py's python_requires
 # - bash
 # - glibc
 FROM python:3-alpine

constraints.lock

@@ -0,0 +1,46 @@
+aiodns==2.0.0
+bleach==3.3.0
+certifi==2020.12.5
+cffi==1.14.4
+chardet==4.0.0
+colorama==0.4.4
+cryptography==3.4.3
+docutils==0.16
+DoubleRatchet==0.7.0
+emoji-country-flag==1.2.3
+idna==2.10
+jeepney==0.6.0
+keyring==22.0.1
+OMEMO==0.12.0
+omemo-backend-signal==0.2.6
+packaging==20.9
+pip==20.0.2
+pkg-resources==0.0.0
+pkginfo==1.7.0
+protobuf==3.14.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycares==3.1.1
+pycparser==2.20
+Pygments==2.7.4
+PyNaCl==1.4.0
+pyparsing==2.4.7
+python-i18n==0.3.9
+PyYAML==5.4.1
+readme-renderer==28.0
+requests==2.25.1
+requests-toolbelt==0.9.1
+rfc3986==1.4.0
+SecretStorage==3.3.1
+setuptools==44.0.0
+setuptools-scm==5.0.1
+six==1.15.0
+slixmpp==1.7.0
+slixmpp-omemo==0.4.0
+tqdm==4.56.0
+twine==3.3.0
+urllib3==1.26.3
+webencodings==0.5.1
+wheel==0.36.2
+X3DH==0.5.9
+XEdDSA==0.4.7

constraints.txt

@@ -0,0 +1,10 @@
+# Syntax : https://pip.pypa.io/en/stable/user_guide/#constraints-files
+
+# Since 3.4.3 cryptography requires rust 1.45+, which is not available on all platforms
+# See https://cryptography.io/en/latest/changelog.html#v3-4-3
+# For now there is a patch (see debian.Dockerfile) with CRYPTOGRAPHY_DONT_BUILD_RUST=1
+# but next versions of cryptography will probably break the build so we lock it
+cryptography<=3.4.3
+
+# Since 1.7.0 slixmpp does not compile on Python 3.5 so it should be upgraded by hand
+slixmpp<=1.7.0

debian.Dockerfile

@@ -2,6 +2,8 @@
 # STAGE 1 : Builder image
 #
 
+# This builder must have a Python version comptabile with the final image
+# So built artifacts will work
 FROM python:3 as builder
 
 RUN apt-get update && \
@@ -19,25 +21,39 @@ RUN apt-get update && \
         git
 
 # Rust is a requirement to build the 'cryptography' Python module
-# The recommended procedure is to use 'rustup but the both Debian &
+# but it's sooo complicated to install it on many platforms...
-# Alpine ship with more CPU architectures so we use the OS' packages.
+# The recommended procedure is to use 'rustup but Alpine ships with more CPU
-# At the time of writing rustup only provides installers for x86_64 and
+# architectures so we use the OS' packages. (At the time of writing rustup only
-# aarch64 (arm64).
+# provides installers for x86_64 and aarch64 (arm64).)
 # https://forge.rust-lang.org/infra/other-installation-methods.html
 # Alpine packages : https://pkgs.alpinelinux.org/packages?name=rust
 # Debian packages : https://packages.debian.org/buster/rustc
-# Here we try the official method first and fall back to the package
+# FIXME The rustup script does not work for linux/386 : it seems it installs x86_64 instead
-RUN (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y) || apt-get install -y rustc
+#RUN (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && . $HOME/.cargo/env) || apt-get install -y rustc
+# As of writing, copying from the rust image is supported for the following archs : 386,amd64,armv7,arm64
+COPY --from=rust:slim /usr/local/cargo /usr/local/cargo
+COPY --from=rust:slim /usr/local/rustup /usr/local/rustup
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+RUN rustc --version
 
 WORKDIR /usr/src/app
 
 # Builds & installs requirements (shoduld not change often)
-COPY requirements-*.txt \
+COPY constraints.txt \
+     requirements-*.txt \
      setup.py \
      .
+# # FIXME Either with rustup or rustc package, rust version for linux/386 on debian is only 1.41 as of buster
+# # => Since 3.4.3 cryptography requires rust 1.45+, which is not available on all platforms
+# # https://cryptography.io/en/latest/changelog.html#v3-4-3
+# # => For now we use the patch below to disable rust but the next version of cryptography
+# # will probably force us to use packages from debian testing or to use an older cryptography version
+# ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 # This step WILL trigger a compilation on platforms without matching Python wheels
 RUN python3 -m pip install --no-cache-dir --user --upgrade pip && \
-    python3 -m pip install --no-cache-dir --user -r requirements-build.txt -r requirements-runtime.txt
+    python3 -m pip install --no-cache-dir --user -c constraints.txt -r requirements-build.txt -r requirements-runtime.txt
 
 # Builds & installs nicobot (should change often, especially the .git directory)
 COPY LICENSE \
@@ -54,7 +70,7 @@ RUN python3 -m pip install --no-cache-dir --user .
 #
 
 # The base image must provide :
-# - Python version > 3.4.2
+# - Python matching setup.py's python_requires
 # - bash
 # - glibc
 FROM python:3-slim

docker/docker-images.puml

@@ -16,6 +16,7 @@ left to right direction
 object "Docker context" as docker_context $dockercontext
 object "<i>python:3</i>" as python3 $baseimage
 object "<i>python:3-slim</i>" as python3_slim $baseimage
+object "<i>rust:slim</i>" as rust_slim $baseimage
 object "debian builder" as nicobot_debian_builder <<temp>> $tempimage {
     Builds nicobot & dependencies
 }
@@ -24,6 +25,7 @@ object "nicolabs/nicobot:debian" as nicobot_debian $finalimage {
 }
 
 python3 --> nicobot_debian_builder : FROM
+rust_slim --> nicobot_debian_builder : COPY
 docker_context ...> nicobot_debian_builder : COPY
 
 python3_slim --> nicobot_debian : FROM

requirements-build.txt

@@ -1,4 +1,6 @@
 ##### Requirements for building only #####
+# Syntax : https://pip.pypa.io/en/stable/user_guide/#requirements-files
+
 setuptools
 wheel
 twine

requirements-runtime.txt

@@ -1,3 +1,5 @@
+# Syntax : https://pip.pypa.io/en/stable/user_guide/#requirements-files
+
 ##### Requirements for signalcli #####
 python-i18n