Begin Management in Docker

2024-10-04 21:34:09 +02:00 · 2024-10-04 21:34:09 +02:00 · 80d9d90aaa
commit 80d9d90aaa
parent 622f8bc009
24 changed files with 341 additions and 336 deletions
--- a/.gitea/workflows/docker-build.yml
+++ b/.gitea/workflows/docker-build.yml
@ -0,0 +1,38 @@
+name: Build and push Docker image at git tag
+on:
+  push:
+    tags:        
+      - '*'
+
+env:
+  IMAGE_REGISTRY: git.chai.uni-hamburg.de
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Get repository code
+      uses: actions/checkout@v4
+
+    - name: ARM64 add QEMU 
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: arm64
+    - name: ARM64 setup Docker buildx 
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build the Management
+      run: bash ./build-mgmt.sh -no-updates
+    #- name: Build the Agent
+    #  run: bash ./build-agent.sh -no-updates
+
+    - name: Docker login
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.IMAGE_REGISTRY }}
+        username: ${{ secrets.REGISTRY_USERNAME }}
+        password: ${{ secrets.REGISTRY_TOKEN }}
+    - name: Push the images
+      run: bash ./push-images.sh
+      
--- a/3
+++ b/3
@ -1,3 +0,0 @@
-FROM git.chai.uni-hamburg.de/ums-agenten/base-process:cpu
-
-RUN apt install my-pkg
--- a/Requests.md
+++ b/Requests.md
@ -1,132 +0,0 @@
-### Ablauf
-1. Daten und Rästel eingeben
-    - an Management von Benutzer (oder als Subrätsel)
-    ```json
-    {
-        "id" : "xyz",
-        "riddle" : {
-            "context" : "The values of the variables of the following math problem are given in the text document as written numbers.",
-            "question" : "What is x+y?",
-            "solution_before" : "", # if trial > 0
-            "review_before": "" # if trial > 0
-        },
-        "trial" : 0, # did we try this before
-        "steps" : {
-            "extract" : true,
-            "solve" : true,
-            "validate": true,
-        },
-        "data" : [
-            {
-                "type" : "text",
-                "file" : "./my-test.txt"
-            },
-            ...
-        ]
-    }
-    ```
-2. Daten verteilen (`if "extract" : true`)
-    - an alle "🤖 Daten einlesen" vom Management
-    ```json 
-    {
-        "id" : "xyz",
-        "data" : [
-            {
-                "type" : "text",
-                "file" : "./my-test.txt"
-            },
-            ...
-        ]    
-    }
-    ```
-3. Verarbeitete Daten annehmen (`if "extract" : true|false`)
-    - von allen "🤖 Daten einlesen" an Management 
-    ```json 
-    {
-        "id" : "xyz",
-        "data" : [
-            {
-                "type" : "text",
-                "file" : "./my-test.txt",
-                "extraction" : "./my-test.json" # schema für alle gleich
-            },
-            ...
-        ]    
-    }
-    ```
-4. Rätsel lösen (`if "solve" : true|false`)
-    - an (alle) "🤖 Lösung bestimmen" von Management
-    ```json
-    {
-        "id" : "xyz",
-        "riddle" : { ... },
-        "data" : [
-            {
-                "type" : "text",
-                "file" : "./my-test.txt",
-                "extraction" : "./my-test.json"
-            },
-            ...
-        ]
-    }
-    ```
-    - Muss ein "Subrätsel" erzeugen (können), wenn weitere Datenanalyse notwenig ist
-    - Mit einer Submenge der Daten bzw. konkreten Fragen etc.
-5. Lösung eines Rätsel annehmen (`if "solve" : true`)
-    - von (alle) "🤖 Lösung bestimmen" an Management
-     ```json
-    {
-        "id" : "xyz",
-        "solution" : "26",
-        "explanation" : "x = 12, y = 14, ...",
-        "sub_riddles" : [    # optional
-            "xyz-1",
-            "xyz-2"
-        ],
-        "used_data" : [ # optional
-            {
-                "type" : "text",
-                "file" : "./my-test.txt",
-                "extraction" : "./my-test.json"
-            },
-            ...
-        ]
-    }
-    ``` 
-6. Lösung validieren lassen (`if "validate": true`)
-    - an (alle) "🤖 Lösung validieren" durch Management
-     ```json
-    {
-        "id" : "xyz",
-        "riddle" : { ... }
-        "solution" : "26",
-        "explanation" : "x = 12, y = 14, ...",
-        "sub_riddles" : [    # optional
-            "xyz-1",
-            "xyz-2"
-        ],
-        "used_data" : [ # optional
-            {
-                "type" : "text",
-                "file" : "./my-test.txt",
-                "extraction" : "./my-test.json"
-            },
-            ...
-        ]
-    }
-    ``` 
-7. Validierung annehmen (`if "validate": true`)
-    - von (alle) "🤖 Lösung validieren" an Management
-    ```json
-     {
-        "id" : "xyz",
-        "solution" : "26",
-        "explanation" : "x = 12, y = 14, ...",
-        "accept" : false # or true
-        "review" : "y does not match Y"
-     }
-    ```
-8. Ausgabe oder Wiedervorlage
-    - von Management an User oder als neues Riddle
-    - `if "accept" : false`: Neues Riddle mit `trial+1`, `solution_before`, `review_before`
-    - `if "accept" : true`: User bekommt `solution, explanation, review`
--- a/build-mgmt.sh
+++ b/build-mgmt.sh
@ -0,0 +1,36 @@
+#/bin/bash
+
+# https://stackoverflow.com/a/4774063
+SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
+
+source "$SCRIPTPATH/vars.sh"
+
+requirements="requirements-frozen.txt"
+if [ "$1" != "-no-updates" ]; then
+	echo "Update the depencendies in requirements.txt? [may break app] (y/n)"
+	read newlockfile
+	if [ "$newlockfile" == "y" ]; then
+		requirements="requirements.txt"
+	fi;
+fi;
+
+for platform in $PLATFORMS; do
+	if [ "$platform" != "gpu" ]; then 
+		docker build \
+			--pull \
+			--platform "linux/$platform" \
+			--file "$SCRIPTPATH/docker-mgmt/Dockerfile" \
+			--build-arg H_UID=1050 \
+			--build-arg PIP_REQ_FILE="$requirements" \
+			--build-arg H_GID=1050 \
+			--tag "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT:$platform" \
+			"$SCRIPTPATH"
+	fi;
+done;
+
+if [ "$requirements" == "requirements.txt" ]; then
+	# extract requirements-frozen.txt
+	cid=$(docker create "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT:arm64")
+	docker cp "$cid:/ums-agenten/requirements.txt" "$SCRIPTPATH/docker-mgmt/requirements-frozen.txt"
+	docker rm "$cid"
+fi;
--- a/docker-agent/Dockerfile
+++ b/docker-agent/Dockerfile
@ -0,0 +1,5 @@
+ARG FROM_IMAGE=
+
+FROM $FROM_IMAGE
+
+
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,21 +0,0 @@
-
-services:
-  management:
-    image: git.chai.uni-hamburg.de/ums-agenten/management:latest
-    ports:
-      - 
-    environment:
-      - AGENTS_PROCESS=http://agent_process_1:3001,http://agent_process_2:3001
-      - AGENTS_SOLVE=http://agent_solve_1:3001
-      - AGENTS_GATEKEEPER=http://agent_gatekeeper_1:3001
-    volumes:
-      - /xxx/:
-
-  agent_process_1:
-    image: git.chai.uni-hamburg.de/ums-agenten/base-process:cpu
-    #image: git.chai.uni-hamburg.de/ums-agenten/base-process:gpu
-    build: .
-    environment:
-      - MANAGEMENT=http://management:3001
-    volumes:
-      - /xxx/:/ums-agenten/shared/
--- a/docker-mgmt/Dockerfile
+++ b/docker-mgmt/Dockerfile
@ -0,0 +1,50 @@
+FROM ubuntu:24.04
+
+ARG H_GID
+ARG H_UID
+ARG PIP_REQ_FILE
+
+RUN apt update && \
+	apt install -y bash \
+		build-essential \
+		git \
+		curl \
+		ca-certificates \
+		python3-dev \
+		python3-pip 
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Berlin apt-get install -y tzdata \
+	&& cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime \
+	&& echo "Europe/Berlin" > /etc/timezone 
+
+RUN apt-get install -y vim htop \
+		nginx supervisor \ 
+	&& rm -rf /var/lib/apt/lists
+
+# sytem and user setup 
+RUN ln -s /usr/bin/python3 /usr/local/bin/python \
+	&& addgroup --gid $H_GID user \
+	&& adduser user --uid $H_UID --ingroup user --gecos "" --home /home/user/ --disabled-password
+
+RUN mkdir -p /ums-agenten/plattform/
+
+COPY ./docker-mgmt/$PIP_REQ_FILE /ums-agenten/requirements.txt
+RUN pip3 install --break-system-packages --no-cache-dir -r /ums-agenten/requirements.txt \
+	&& pip3 freeze > /ums-agenten/requirements.txt
+
+# nginx settings and startup
+COPY ./docker-mgmt/supervisor.conf /etc/supervisor/supervisord.conf 
+COPY ./docker-mgmt/nginx.conf /etc/nginx/nginx.conf
+COPY ./docker-mgmt/app.conf /etc/nginx/sites-enabled/default
+
+# install the code of the repo
+COPY ./docker-mgmt/setup.py /ums-agenten/plattform/
+RUN pip3 install --break-system-packages -e /ums-agenten/plattform/
+
+WORKDIR /ums-agenten/plattform/ums/
+
+COPY --chown=user:user ./ums/ /ums-agenten/plattform/ums/
+COPY --chown=user:user ./web/ /ums-agenten/plattform/web/
+
+# run nginx and uvicorn
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
--- a/docker-mgmt/app.conf
+++ b/docker-mgmt/app.conf
@ -0,0 +1,27 @@
+server {
+
+	listen 80 default_server;
+	
+	server_name _;
+
+	root /ums-agenten/plattform/web/public/;
+	index index.html;
+
+	location / {
+		try_files $uri $uri/ @dynamic;
+	}
+
+	location @dynamic {
+		proxy_set_header Host $http_host;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "upgrade";
+
+		proxy_redirect off;
+		proxy_buffering off;
+
+		proxy_pass http://unix:/tmp/uvicorn.sock;
+	}
+
+}
--- a/docker-mgmt/nginx.conf
+++ b/docker-mgmt/nginx.conf
@ -0,0 +1,47 @@
+user user;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+	worker_connections 768;
+}
+
+http {
+
+	##
+	# Basic Settings
+	##
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	server_tokens off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# SSL Settings
+	##
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 
+	ssl_prefer_server_ciphers on;
+
+	##
+	# Logging Settings
+	##
+	access_log off;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+	gzip on;
+	
+	##
+	# Virtual Host Configs
+	##
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}
--- a/docker-mgmt/requirements-frozen.txt
+++ b/docker-mgmt/requirements-frozen.txt
@ -0,0 +1,31 @@
+annotated-types==0.7.0
+anyio==4.6.0
+certifi==2024.8.30
+charset-normalizer==3.3.2
+click==8.1.7
+fastapi==0.115.0
+h11==0.14.0
+httptools==0.6.1
+idna==3.10
+Jinja2==3.1.4
+MarkupSafe==2.1.5
+pdoc==14.7.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.12
+PyYAML==6.0.2
+requests==2.32.3
+setuptools==68.1.2
+sniffio==1.3.1
+starlette==0.38.6
+supervisor==4.2.5
+tqdm==4.66.5
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.31.0
+uvloop==0.20.0
+watchfiles==0.24.0
+websockets==13.1
+wheel==0.42.0
--- a/docker-mgmt/requirements.txt
+++ b/docker-mgmt/requirements.txt
@ -0,0 +1,7 @@
+# non frozen dependecies, to use latest
+requests
+tqdm
+pdoc
+fastapi
+uvicorn[standard]
+python-multipart
--- a/docker-mgmt/setup.py
+++ b/docker-mgmt/setup.py
@ -0,0 +1,9 @@
+from setuptools import find_packages, setup
+
+setup(
+    name='ums',
+    packages=find_packages(),
+    version='0.0.0',
+    description='UMS-Agenten',
+    author='Magnus Bender',
+)
--- a/docker-mgmt/supervisor.conf
+++ b/docker-mgmt/supervisor.conf
@ -0,0 +1,25 @@
+[supervisord]
+nodaemon=true
+user=root
+
+[program:setup]
+command=/bin/sh -c "chown user:user -R /ums-agenten"
+user=root
+autostart=true
+autorestart=false
+
+[fcgi-program:uvicorn]
+socket=unix:///tmp/uvicorn.sock
+command=/usr/local/bin/uvicorn ums.management.main:app --uds /tmp/uvicorn.sock --proxy-headers
+numprocs=4
+process_name=uvicorn-%(process_num)d
+user=user
+autostart=true
+autorestart=true
+priority=10
+
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+autostart=true
+autorestart=true
+priority=20
--- a/push-images.sh
+++ b/push-images.sh
@ -0,0 +1,34 @@
+#/bin/bash
+
+# https://stackoverflow.com/a/4774063
+SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
+
+source "$SCRIPTPATH/vars.sh"
+
+day_tag=$(date '+%Y-%m-%d')
+
+images_a=$(docker image ls "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_AGENT" --format '{{.Repository}}:{{.Tag}}')
+images_b=$(docker image ls "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT" --format '{{.Repository}}:{{.Tag}}')
+
+echo "$images_a\n$images_b" | while read image_url ;
+do
+
+	image_name="${image_url##*/}"
+	image_name="${image_name%%:*}"
+	image_tag="${image_url##*:}"
+
+	if [[ "$image_tag" =~ ^((gpu)|(cpu)-)?((arm64)|(amd64))$ ]]; 
+	then
+
+		echo "Push:"
+		echo "	$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag"
+		echo "	$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag"
+
+		#docker push "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag"
+
+		#docker tag "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag" \
+		#		"$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag"
+
+		#docker push "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag"
+	fi;
+done
--- a/src/agent/agent.py
+++ b/src/agent/agent.py
@ -1,57 +0,0 @@
-
-from abc import abstractmethod
-from typing import List, Dict
-
-from o_types import *
-
-class Agent():
-
-	def process_result():
-		pass
-
-	# hier die Kommunikation mit dem Management Server
-
-""" ===== """
-
-class ProcessAgent(Agent):
-	TYPE:str = None
-
-	@abstractmethod
-	def on_process_data(self, data:List[Data]):
-		pass
-
-class TextAgentExample(ProcessAgent):
-	TYPE = Data.TYPE_TEXT
-	def on_process_data(self, data:List[Data]):
-		# Studi Code
-		self.process_result(Data(...))
-		
-class ImageAgentExample(ProcessAgent):
-	TYPE = Data.TYPE_IMAGE
-
-class AudioAgentExample(ProcessAgent):
-	TYPE = Data.TYPE_AUDIO
-	
-""" ===== """
-
-class SolveAgent(Agent):
-	
-	@abstractmethod
-	def on_riddle_data(self, riddle:Riddle, data:List[Data]):
-		pass
-
-		# Studi Code
-
-		self.new_riddle(Riddle('x', 'y', 'z'))
-		self.send_riddle_result(...)
-
-""" ===== """
-
-class GatekeeperAgent(Agent):
-	
-	@abstractmethod
-	def on_result_check(self, xyz):
-
-		# Studi Code
-		
-		self.send_check_result(accept=True|False, ...)
--- a/src/agent/basic_agent.py
+++ b/src/agent/basic_agent.py
@ -1,80 +0,0 @@
-
-from agent import Agent
-
-class ProcessTextAgent(Agent):
-
-	TYPE = "text" or "image"
-
-
-	def on_process_data(self, data:List[Dict]):
-		return filter(lambda x: x["type"] == self.TYPE, data)
-
-		# {type: "text", "file" : ""}
-
-class ExampleAgent(ProcessTextAgent):
-
-
-	def on_process_data(self, data:List[Dict]):
-		# {type: "text", "file" : ""}
-
-		self.after_process_data(result)
-
-
-
-# disk -> 123
-#	agent 1 disk -> 123
-
-class Agent:
-	def request_management(self, request):
-		request.get(os.gentenv('MANAGEMENT'), data)
-		# https ...
-	
-	def write_result(...):
-		self.request_management.write(...)
-
-	def read_result(...):
-		self.request_management.read(...)
-
-
-class ProcessAgent(Agent):
-	TYPE : str = None
-
-	def on_process_data(self, data:List[Dict]):
-		pass
-
-class TextAgent(ProcessAgent):
-	TYPE = 'text'
-	text_data = self.on_process_data()
-
-	def on_process_data(self, data:List[Dict]):
-		text_data = filter_text_data()
-		processed_data = text_func1(text_data)
-
-		self.write_result(processed_data)
-	
-	def text_func1():
-		...
-	
-	def text_func2():
-		...
-
-	def text_func3():
-		...	
-
-class ImageAgent(ProcessAgent):
-	TYPE = 'image'
-
-	image_data = self.on_process_data()
-
-
-	def on_process_data(self, data:list[Dict])
-		image_data = filter_image_data()
-
-	def image_func1():
-		...
-
-	...
-
-
-
-	
--- a/src/agent/o_types.py
+++ b/src/agent/o_types.py
@ -1,43 +0,0 @@
-
-from typing import List
-
-class Riddle():
-
-	def __init__(self):
-		self.id = "" # -> management.get_riddle_by_id(id)
-		self.parent_id = "" # ?!
-
-		self.context = "1234"
-		self.question = "12" 
-
-		self.data:List[Data] = []
-
-	def from_json(json):
-		return Riddle(json)
-
-	def to_json(self):
-		return {
-			"context" : self.context,
-       		"question" : self.question,
-        		"solution_before" : "", # if trial > 0
-        		"review_before": "" # if trial > 0
-		}
-	
-class Data():
-
-	TYPE_TEXT = "text"
-	TYPE_AUDIO = "audio"
-	TYPE_IMAGE = "image"
-
-	def __init__(self):
-		self.type = 1
-		self.file = 2
-		self.extraction = 3
-
-	def to_json(self):
-		return {
-			"type" : self.type,
-			"file" : self.file,
-			"extraction" : self.extraction
-		}
-		
--- a/ums/cli.py
+++ b/ums/cli.py
--- a/ums/init.py
+++ b/ums/init.py
--- a/ums/agent/init.py
+++ b/ums/agent/init.py
--- a/ums/management/init.py
+++ b/ums/management/init.py
--- a/ums/management/main.py
+++ b/ums/management/main.py
@ -0,0 +1,17 @@
+
+# TEST ONLY
+
+from typing import Union
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+@app.get("/")
+def read_root():
+	return {"Hello": "World"}
+
+
+@app.get("/items/{item_id}")
+def read_item(item_id: int, q: Union[str, None] = None):
+	return {"item_id": item_id, "q": q}
--- a/vars.sh
+++ b/vars.sh
@ -0,0 +1,7 @@
+#/bin/bash
+
+IMAGE_REGISTRY="git.chai.uni-hamburg.de"
+IMAGE_OWNER="ums-agenten"
+IMAGE_NAME_AGENT="base-agent"
+IMAGE_NAME_MGMT="management"
+PLATFORMS="amd64 arm64 gpu"
--- a/web/public/index.html
+++ b/web/public/index.html
@ -0,0 +1,8 @@
+<html>
+	<head>
+		<title>UMS-Agenten Management</title>
+	</head>
+	<body>
+		<h1>Empty</h1>
+	</body>
+</html>