diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml new file mode 100644 index 0000000..d5c4c17 --- /dev/null +++ b/.gitea/workflows/docker-build.yml @@ -0,0 +1,38 @@ +name: Build and push Docker image at git tag +on: + push: + tags: + - '*' + +env: + IMAGE_REGISTRY: git.chai.uni-hamburg.de + +jobs: + build: + runs-on: ubuntu-latest + steps: + + - name: Get repository code + uses: actions/checkout@v4 + + - name: ARM64 add QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: ARM64 setup Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Build the Management + run: bash ./build-mgmt.sh -no-updates + #- name: Build the Agent + # run: bash ./build-agent.sh -no-updates + + - name: Docker login + uses: docker/login-action@v3 + with: + registry: ${{ env.IMAGE_REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Push the images + run: bash ./push-images.sh + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c0ab7de..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM git.chai.uni-hamburg.de/ums-agenten/base-process:cpu - -RUN apt install my-pkg \ No newline at end of file diff --git a/Requests.md b/Requests.md deleted file mode 100644 index f402b9f..0000000 --- a/Requests.md +++ /dev/null @@ -1,132 +0,0 @@ -### Ablauf -1. Daten und Rästel eingeben - - an Management von Benutzer (oder als Subrätsel) - ```json - { - "id" : "xyz", - "riddle" : { - "context" : "The values of the variables of the following math problem are given in the text document as written numbers.", - "question" : "What is x+y?", - "solution_before" : "", # if trial > 0 - "review_before": "" # if trial > 0 - }, - "trial" : 0, # did we try this before - "steps" : { - "extract" : true, - "solve" : true, - "validate": true, - }, - "data" : [ - { - "type" : "text", - "file" : "./my-test.txt" - }, - ... - ] - } - ``` -2. Daten verteilen (`if "extract" : true`) - - an alle "🤖 Daten einlesen" vom Management - ```json - { - "id" : "xyz", - "data" : [ - { - "type" : "text", - "file" : "./my-test.txt" - }, - ... - ] - } - ``` -3. Verarbeitete Daten annehmen (`if "extract" : true|false`) - - von allen "🤖 Daten einlesen" an Management - ```json - { - "id" : "xyz", - "data" : [ - { - "type" : "text", - "file" : "./my-test.txt", - "extraction" : "./my-test.json" # schema für alle gleich - }, - ... - ] - } - ``` -4. Rätsel lösen (`if "solve" : true|false`) - - an (alle) "🤖 Lösung bestimmen" von Management - ```json - { - "id" : "xyz", - "riddle" : { ... }, - "data" : [ - { - "type" : "text", - "file" : "./my-test.txt", - "extraction" : "./my-test.json" - }, - ... - ] - } - ``` - - Muss ein "Subrätsel" erzeugen (können), wenn weitere Datenanalyse notwenig ist - - Mit einer Submenge der Daten bzw. konkreten Fragen etc. -5. Lösung eines Rätsel annehmen (`if "solve" : true`) - - von (alle) "🤖 Lösung bestimmen" an Management - ```json - { - "id" : "xyz", - "solution" : "26", - "explanation" : "x = 12, y = 14, ...", - "sub_riddles" : [ # optional - "xyz-1", - "xyz-2" - ], - "used_data" : [ # optional - { - "type" : "text", - "file" : "./my-test.txt", - "extraction" : "./my-test.json" - }, - ... - ] - } - ``` -6. Lösung validieren lassen (`if "validate": true`) - - an (alle) "🤖 Lösung validieren" durch Management - ```json - { - "id" : "xyz", - "riddle" : { ... } - "solution" : "26", - "explanation" : "x = 12, y = 14, ...", - "sub_riddles" : [ # optional - "xyz-1", - "xyz-2" - ], - "used_data" : [ # optional - { - "type" : "text", - "file" : "./my-test.txt", - "extraction" : "./my-test.json" - }, - ... - ] - } - ``` -7. Validierung annehmen (`if "validate": true`) - - von (alle) "🤖 Lösung validieren" an Management - ```json - { - "id" : "xyz", - "solution" : "26", - "explanation" : "x = 12, y = 14, ...", - "accept" : false # or true - "review" : "y does not match Y" - } - ``` -8. Ausgabe oder Wiedervorlage - - von Management an User oder als neues Riddle - - `if "accept" : false`: Neues Riddle mit `trial+1`, `solution_before`, `review_before` - - `if "accept" : true`: User bekommt `solution, explanation, review` \ No newline at end of file diff --git a/build-mgmt.sh b/build-mgmt.sh new file mode 100755 index 0000000..9340f2a --- /dev/null +++ b/build-mgmt.sh @@ -0,0 +1,36 @@ +#/bin/bash + +# https://stackoverflow.com/a/4774063 +SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" + +source "$SCRIPTPATH/vars.sh" + +requirements="requirements-frozen.txt" +if [ "$1" != "-no-updates" ]; then + echo "Update the depencendies in requirements.txt? [may break app] (y/n)" + read newlockfile + if [ "$newlockfile" == "y" ]; then + requirements="requirements.txt" + fi; +fi; + +for platform in $PLATFORMS; do + if [ "$platform" != "gpu" ]; then + docker build \ + --pull \ + --platform "linux/$platform" \ + --file "$SCRIPTPATH/docker-mgmt/Dockerfile" \ + --build-arg H_UID=1050 \ + --build-arg PIP_REQ_FILE="$requirements" \ + --build-arg H_GID=1050 \ + --tag "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT:$platform" \ + "$SCRIPTPATH" + fi; +done; + +if [ "$requirements" == "requirements.txt" ]; then + # extract requirements-frozen.txt + cid=$(docker create "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT:arm64") + docker cp "$cid:/ums-agenten/requirements.txt" "$SCRIPTPATH/docker-mgmt/requirements-frozen.txt" + docker rm "$cid" +fi; diff --git a/docker-agent/Dockerfile b/docker-agent/Dockerfile new file mode 100644 index 0000000..530d90e --- /dev/null +++ b/docker-agent/Dockerfile @@ -0,0 +1,5 @@ +ARG FROM_IMAGE= + +FROM $FROM_IMAGE + + diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 765be11..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ - -services: - management: - image: git.chai.uni-hamburg.de/ums-agenten/management:latest - ports: - - - environment: - - AGENTS_PROCESS=http://agent_process_1:3001,http://agent_process_2:3001 - - AGENTS_SOLVE=http://agent_solve_1:3001 - - AGENTS_GATEKEEPER=http://agent_gatekeeper_1:3001 - volumes: - - /xxx/: - - agent_process_1: - image: git.chai.uni-hamburg.de/ums-agenten/base-process:cpu - #image: git.chai.uni-hamburg.de/ums-agenten/base-process:gpu - build: . - environment: - - MANAGEMENT=http://management:3001 - volumes: - - /xxx/:/ums-agenten/shared/ \ No newline at end of file diff --git a/docker-mgmt/Dockerfile b/docker-mgmt/Dockerfile new file mode 100644 index 0000000..01b9e33 --- /dev/null +++ b/docker-mgmt/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:24.04 + +ARG H_GID +ARG H_UID +ARG PIP_REQ_FILE + +RUN apt update && \ + apt install -y bash \ + build-essential \ + git \ + curl \ + ca-certificates \ + python3-dev \ + python3-pip + +RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Berlin apt-get install -y tzdata \ + && cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime \ + && echo "Europe/Berlin" > /etc/timezone + +RUN apt-get install -y vim htop \ + nginx supervisor \ + && rm -rf /var/lib/apt/lists + +# sytem and user setup +RUN ln -s /usr/bin/python3 /usr/local/bin/python \ + && addgroup --gid $H_GID user \ + && adduser user --uid $H_UID --ingroup user --gecos "" --home /home/user/ --disabled-password + +RUN mkdir -p /ums-agenten/plattform/ + +COPY ./docker-mgmt/$PIP_REQ_FILE /ums-agenten/requirements.txt +RUN pip3 install --break-system-packages --no-cache-dir -r /ums-agenten/requirements.txt \ + && pip3 freeze > /ums-agenten/requirements.txt + +# nginx settings and startup +COPY ./docker-mgmt/supervisor.conf /etc/supervisor/supervisord.conf +COPY ./docker-mgmt/nginx.conf /etc/nginx/nginx.conf +COPY ./docker-mgmt/app.conf /etc/nginx/sites-enabled/default + +# install the code of the repo +COPY ./docker-mgmt/setup.py /ums-agenten/plattform/ +RUN pip3 install --break-system-packages -e /ums-agenten/plattform/ + +WORKDIR /ums-agenten/plattform/ums/ + +COPY --chown=user:user ./ums/ /ums-agenten/plattform/ums/ +COPY --chown=user:user ./web/ /ums-agenten/plattform/web/ + +# run nginx and uvicorn +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] \ No newline at end of file diff --git a/docker-mgmt/app.conf b/docker-mgmt/app.conf new file mode 100644 index 0000000..d624fa4 --- /dev/null +++ b/docker-mgmt/app.conf @@ -0,0 +1,27 @@ +server { + + listen 80 default_server; + + server_name _; + + root /ums-agenten/plattform/web/public/; + index index.html; + + location / { + try_files $uri $uri/ @dynamic; + } + + location @dynamic { + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_redirect off; + proxy_buffering off; + + proxy_pass http://unix:/tmp/uvicorn.sock; + } + +} \ No newline at end of file diff --git a/docker-mgmt/nginx.conf b/docker-mgmt/nginx.conf new file mode 100644 index 0000000..5f4540a --- /dev/null +++ b/docker-mgmt/nginx.conf @@ -0,0 +1,47 @@ +user user; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; +} + +http { + + ## + # Basic Settings + ## + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # SSL Settings + ## + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + + ## + # Logging Settings + ## + access_log off; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + gzip on; + + ## + # Virtual Host Configs + ## + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} \ No newline at end of file diff --git a/docker-mgmt/requirements-frozen.txt b/docker-mgmt/requirements-frozen.txt new file mode 100644 index 0000000..4c0dd65 --- /dev/null +++ b/docker-mgmt/requirements-frozen.txt @@ -0,0 +1,31 @@ +annotated-types==0.7.0 +anyio==4.6.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +fastapi==0.115.0 +h11==0.14.0 +httptools==0.6.1 +idna==3.10 +Jinja2==3.1.4 +MarkupSafe==2.1.5 +pdoc==14.7.0 +pydantic==2.9.2 +pydantic_core==2.23.4 +Pygments==2.18.0 +python-dotenv==1.0.1 +python-multipart==0.0.12 +PyYAML==6.0.2 +requests==2.32.3 +setuptools==68.1.2 +sniffio==1.3.1 +starlette==0.38.6 +supervisor==4.2.5 +tqdm==4.66.5 +typing_extensions==4.12.2 +urllib3==2.2.3 +uvicorn==0.31.0 +uvloop==0.20.0 +watchfiles==0.24.0 +websockets==13.1 +wheel==0.42.0 diff --git a/docker-mgmt/requirements.txt b/docker-mgmt/requirements.txt new file mode 100644 index 0000000..841427d --- /dev/null +++ b/docker-mgmt/requirements.txt @@ -0,0 +1,7 @@ +# non frozen dependecies, to use latest +requests +tqdm +pdoc +fastapi +uvicorn[standard] +python-multipart \ No newline at end of file diff --git a/docker-mgmt/setup.py b/docker-mgmt/setup.py new file mode 100644 index 0000000..c0c8677 --- /dev/null +++ b/docker-mgmt/setup.py @@ -0,0 +1,9 @@ +from setuptools import find_packages, setup + +setup( + name='ums', + packages=find_packages(), + version='0.0.0', + description='UMS-Agenten', + author='Magnus Bender', +) \ No newline at end of file diff --git a/docker-mgmt/supervisor.conf b/docker-mgmt/supervisor.conf new file mode 100644 index 0000000..b5229bb --- /dev/null +++ b/docker-mgmt/supervisor.conf @@ -0,0 +1,25 @@ +[supervisord] +nodaemon=true +user=root + +[program:setup] +command=/bin/sh -c "chown user:user -R /ums-agenten" +user=root +autostart=true +autorestart=false + +[fcgi-program:uvicorn] +socket=unix:///tmp/uvicorn.sock +command=/usr/local/bin/uvicorn ums.management.main:app --uds /tmp/uvicorn.sock --proxy-headers +numprocs=4 +process_name=uvicorn-%(process_num)d +user=user +autostart=true +autorestart=true +priority=10 + +[program:nginx] +command=/usr/sbin/nginx -g 'daemon off;' +autostart=true +autorestart=true +priority=20 \ No newline at end of file diff --git a/push-images.sh b/push-images.sh new file mode 100755 index 0000000..a98be1b --- /dev/null +++ b/push-images.sh @@ -0,0 +1,34 @@ +#/bin/bash + +# https://stackoverflow.com/a/4774063 +SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" + +source "$SCRIPTPATH/vars.sh" + +day_tag=$(date '+%Y-%m-%d') + +images_a=$(docker image ls "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_AGENT" --format '{{.Repository}}:{{.Tag}}') +images_b=$(docker image ls "$IMAGE_REGISTRY/$IMAGE_OWNER/$IMAGE_NAME_MGMT" --format '{{.Repository}}:{{.Tag}}') + +echo "$images_a\n$images_b" | while read image_url ; +do + + image_name="${image_url##*/}" + image_name="${image_name%%:*}" + image_tag="${image_url##*:}" + + if [[ "$image_tag" =~ ^((gpu)|(cpu)-)?((arm64)|(amd64))$ ]]; + then + + echo "Push:" + echo " $IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag" + echo " $IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag" + + #docker push "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag" + + #docker tag "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag" \ + # "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag" + + #docker push "$IMAGE_REGISTRY/$IMAGE_OWNER/$image_name:$image_tag-$day_tag" + fi; +done \ No newline at end of file diff --git a/src/agent/agent.py b/src/agent/agent.py deleted file mode 100644 index d6ba862..0000000 --- a/src/agent/agent.py +++ /dev/null @@ -1,57 +0,0 @@ - -from abc import abstractmethod -from typing import List, Dict - -from o_types import * - -class Agent(): - - def process_result(): - pass - - # hier die Kommunikation mit dem Management Server - -""" ===== """ - -class ProcessAgent(Agent): - TYPE:str = None - - @abstractmethod - def on_process_data(self, data:List[Data]): - pass - -class TextAgentExample(ProcessAgent): - TYPE = Data.TYPE_TEXT - def on_process_data(self, data:List[Data]): - # Studi Code - self.process_result(Data(...)) - -class ImageAgentExample(ProcessAgent): - TYPE = Data.TYPE_IMAGE - -class AudioAgentExample(ProcessAgent): - TYPE = Data.TYPE_AUDIO - -""" ===== """ - -class SolveAgent(Agent): - - @abstractmethod - def on_riddle_data(self, riddle:Riddle, data:List[Data]): - pass - - # Studi Code - - self.new_riddle(Riddle('x', 'y', 'z')) - self.send_riddle_result(...) - -""" ===== """ - -class GatekeeperAgent(Agent): - - @abstractmethod - def on_result_check(self, xyz): - - # Studi Code - - self.send_check_result(accept=True|False, ...) \ No newline at end of file diff --git a/src/agent/basic_agent.py b/src/agent/basic_agent.py deleted file mode 100644 index a0608ad..0000000 --- a/src/agent/basic_agent.py +++ /dev/null @@ -1,80 +0,0 @@ - -from agent import Agent - -class ProcessTextAgent(Agent): - - TYPE = "text" or "image" - - - def on_process_data(self, data:List[Dict]): - return filter(lambda x: x["type"] == self.TYPE, data) - - # {type: "text", "file" : ""} - -class ExampleAgent(ProcessTextAgent): - - - def on_process_data(self, data:List[Dict]): - # {type: "text", "file" : ""} - - self.after_process_data(result) - - - -# disk -> 123 -# agent 1 disk -> 123 - -class Agent: - def request_management(self, request): - request.get(os.gentenv('MANAGEMENT'), data) - # https ... - - def write_result(...): - self.request_management.write(...) - - def read_result(...): - self.request_management.read(...) - - -class ProcessAgent(Agent): - TYPE : str = None - - def on_process_data(self, data:List[Dict]): - pass - -class TextAgent(ProcessAgent): - TYPE = 'text' - text_data = self.on_process_data() - - def on_process_data(self, data:List[Dict]): - text_data = filter_text_data() - processed_data = text_func1(text_data) - - self.write_result(processed_data) - - def text_func1(): - ... - - def text_func2(): - ... - - def text_func3(): - ... - -class ImageAgent(ProcessAgent): - TYPE = 'image' - - image_data = self.on_process_data() - - - def on_process_data(self, data:list[Dict]) - image_data = filter_image_data() - - def image_func1(): - ... - - ... - - - - diff --git a/src/agent/o_types.py b/src/agent/o_types.py deleted file mode 100644 index 7eae870..0000000 --- a/src/agent/o_types.py +++ /dev/null @@ -1,43 +0,0 @@ - -from typing import List - -class Riddle(): - - def __init__(self): - self.id = "" # -> management.get_riddle_by_id(id) - self.parent_id = "" # ?! - - self.context = "1234" - self.question = "12" - - self.data:List[Data] = [] - - def from_json(json): - return Riddle(json) - - def to_json(self): - return { - "context" : self.context, - "question" : self.question, - "solution_before" : "", # if trial > 0 - "review_before": "" # if trial > 0 - } - -class Data(): - - TYPE_TEXT = "text" - TYPE_AUDIO = "audio" - TYPE_IMAGE = "image" - - def __init__(self): - self.type = 1 - self.file = 2 - self.extraction = 3 - - def to_json(self): - return { - "type" : self.type, - "file" : self.file, - "extraction" : self.extraction - } - \ No newline at end of file diff --git a/ums/__cli__.py b/ums/__cli__.py new file mode 100644 index 0000000..e69de29 diff --git a/ums/__init__.py b/ums/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ums/agent/__init__.py b/ums/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ums/management/__init__.py b/ums/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ums/management/main.py b/ums/management/main.py new file mode 100644 index 0000000..eff3227 --- /dev/null +++ b/ums/management/main.py @@ -0,0 +1,17 @@ + +# TEST ONLY + +from typing import Union + +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/") +def read_root(): + return {"Hello": "World"} + + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: Union[str, None] = None): + return {"item_id": item_id, "q": q} \ No newline at end of file diff --git a/vars.sh b/vars.sh new file mode 100755 index 0000000..8271083 --- /dev/null +++ b/vars.sh @@ -0,0 +1,7 @@ +#/bin/bash + +IMAGE_REGISTRY="git.chai.uni-hamburg.de" +IMAGE_OWNER="ums-agenten" +IMAGE_NAME_AGENT="base-agent" +IMAGE_NAME_MGMT="management" +PLATFORMS="amd64 arm64 gpu" diff --git a/web/public/index.html b/web/public/index.html new file mode 100644 index 0000000..bdd923b --- /dev/null +++ b/web/public/index.html @@ -0,0 +1,8 @@ + + + UMS-Agenten Management + + +

Empty

+ + \ No newline at end of file