From fdbbbc3e3d5f3d52ef89b2bd86b4da1a08f364c0 Mon Sep 17 00:00:00 2001 From: KIMB-technologies Date: Fri, 4 Oct 2024 17:47:12 +0200 Subject: [PATCH] Fixes for Actions --- build-images.sh | 6 +++--- docker/Dockerfile | 12 ++++++++++-- docker/Dockerfile.gpu | 4 ++-- docker/requirements.txt | 1 - push-images.sh | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/build-images.sh b/build-images.sh index 315650a..c28ef6a 100755 --- a/build-images.sh +++ b/build-images.sh @@ -17,8 +17,8 @@ fi; for platform in $PLATFORMS; do if [ "$platform" == "gpu" ]; then platform="amd64" - from_image="tmp_local/gpu_baseimage:amd64" - tag_image="$IMAGE_OWNER/$IMAGE_NAME:gpu-amd64" + from_image="$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME:tmp-local-only" + tag_image="$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME:gpu-amd64" docker build \ --pull \ @@ -28,7 +28,7 @@ for platform in $PLATFORMS; do "$SCRIPTPATH" else from_image="$CPU_BASEIMAGE" - tag_image="$IMAGE_OWNER/$IMAGE_NAME:cpu-$platform" + tag_image="$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME:cpu-$platform" docker pull --platform "linux/$platform" "$BASEIMAGE" fi; diff --git a/docker/Dockerfile b/docker/Dockerfile index 393c13e..2101820 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,8 +29,16 @@ RUN apt-get update \ curl wget \ htop screen -RUN python3 -m pip install --no-cache-dir --break-system-packages \ - torch --index-url https://download.pytorch.org/whl/cpu +# if cpu container (install cpu only versions; for gpu, gpu versions) +RUN if [ ! -d /usr/local/cuda/ ]; then \ + python3 -m pip install --no-cache-dir --break-system-packages \ + torch --index-url https://download.pytorch.org/whl/cpu; \ + python3 -m pip install --no-cache-dir --break-system-packages \ + faiss-cpu; \ + else \ + python3 -m pip install --no-cache-dir --break-system-packages \ + faiss-gpu; \ + fi; RUN mkdir /ums-agenten/ diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 44ce4ed..3b25a2c 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -20,12 +20,12 @@ RUN apt update && \ libopenmpi-dev && \ rm -rf /var/lib/apt/lists -RUN python3 -m pip install --no-cache-dir \ +RUN python3 -m pip install --no-cache-dir --break-system-packages \ torch --index-url https://download.pytorch.org/whl/cu124 RUN git clone https://github.com/NVIDIA/apex RUN cd apex && \ - pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ + pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --break-system-packages --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ # fix library path for tensorflow and link a file ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/lib64" diff --git a/docker/requirements.txt b/docker/requirements.txt index b358c38..3af9526 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -17,7 +17,6 @@ matplotlib transformers huggingface_hub -faiss-cpu datasets accelerate peft \ No newline at end of file diff --git a/push-images.sh b/push-images.sh index a38c038..7ec0d5b 100755 --- a/push-images.sh +++ b/push-images.sh @@ -7,7 +7,7 @@ source "$SCRIPTPATH/vars.sh" day_tag=$(date '+%Y-%m-%d') -docker image ls "$IMAGE_OWNER/$IMAGE_NAME" --format '{{.Tag}}' | while read image_tag ; +docker image ls "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME" --format '{{.Tag}}' | while read image_tag ; do if [[ "$image_tag" =~ ^((gpu)|(cpu))-((arm64)|(amd64))$ ]];