UMS-Agenten/Agenten-Plattform
3
0
Fork 0
Code Issues Actions Packages Releases Activity

Fix #1
All checks were successful
Build and push Docker image at git tag / build (push) Successful in 55s
Details

Browse Source
This commit is contained in:
Magnus Bender 2024-10-31 16:25:37 +01:00
parent e7910021dd
commit cc4bb9a7e8
Signed by: bender
GPG Key ID: 5149A211831F2BD7
13 changed files with 1157 additions and 989 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker-compose.yml
View File

@ -21,6 +21,7 @@ services:
environment: environment:
- SOLUTION_MAX_TRIALS=5 - SOLUTION_MAX_TRIALS=5
- MESSAGE_MAX_CONTACTS=100 - MESSAGE_MAX_CONTACTS=100
- REQUIRE_FULL_EXTRACT=false
- MANAGEMENT_URL=http://management - MANAGEMENT_URL=http://management
- AGENTS_PROCESS=http://agent_all:8000 - AGENTS_PROCESS=http://agent_all:8000
- AGENTS_SOLVE=http://agent_all:8000 - AGENTS_SOLVE=http://agent_all:8000

6
ums/agent/agent.py
View File

@ -8,7 +8,7 @@
# source code released under the terms of GNU Public License Version 3 # source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt # https://www.gnu.org/licenses/gpl-3.0.txt
import random, os, json import random, os, json, time
from abc import abstractmethod, ABC from abc import abstractmethod, ABC
from enum import Enum from enum import Enum
@ -64,6 +64,10 @@ class BasicAgent(ABC):
pass pass
def _respond(self): def _respond(self):
# do a very short sleep
time.sleep(random.random())
# sending
send_it = lambda: self._send_message(self._response) send_it = lambda: self._send_message(self._response)
if self.before_response(self._response, send_it) and self._do_response: if self.before_response(self._response, send_it) and self._do_response:
send_it() send_it()

2
ums/agent/main.py
View File

@ -60,6 +60,6 @@ class WebMain():
def message(request: Request, message:AgentMessage, background_tasks: BackgroundTasks) -> AgentResponse: def message(request: Request, message:AgentMessage, background_tasks: BackgroundTasks) -> AgentResponse:
return self.msg_process.new_message(message, background_tasks) return self.msg_process.new_message(message, background_tasks)
if __name__ == "ums.agent.main" and os.environ.get('SERVE', 'false') == 'true': if __name__ == "ums.agent.main" and os.environ.get('SERVE', 'false').lower() == 'true':
main = WebMain() main = WebMain()
app = main.app app = main.app

2
ums/management/main.py
View File

@ -120,6 +120,6 @@ class WebMain():
return msg return msg
if __name__ == "ums.management.main" and os.environ.get('SERVE', 'false') == 'true': if __name__ == "ums.management.main" and os.environ.get('SERVE', 'false').lower() == 'true':
main = WebMain() main = WebMain()
app = main.app app = main.app

52
ums/management/process.py
View File

@ -15,13 +15,15 @@ import requests
from fastapi import BackgroundTasks from fastapi import BackgroundTasks
from ums.management.db import DB from ums.management.db import DB
from ums.utils import AgentMessage, AgentResponse, logger from ums.utils import AgentMessage, AgentResponse, logger, RiddleData
class MessageProcessor(): class MessageProcessor():
SOLUTION_MAX_TRIALS = int(os.environ.get('SOLUTION_MAX_TRIALS', 5)) SOLUTION_MAX_TRIALS = int(os.environ.get('SOLUTION_MAX_TRIALS', 5))
MESSAGE_MAX_CONTACTS = int(os.environ.get('MESSAGE_MAX_CONTACTS', 100)) MESSAGE_MAX_CONTACTS = int(os.environ.get('MESSAGE_MAX_CONTACTS', 100))
REQUIRE_FULL_EXTRACT = os.environ.get('REQUIRE_FULL_EXTRACT', 'false').lower() == 'true'
MANAGEMENT_URL = os.environ.get('MANAGEMENT_URL', 'http://127.0.0.1:80').strip().strip('/') MANAGEMENT_URL = os.environ.get('MANAGEMENT_URL', 'http://127.0.0.1:80').strip().strip('/')
AGENTS_PROCESS = tuple(map( AGENTS_PROCESS = tuple(map(
@ -79,12 +81,23 @@ class MessageProcessor():
# do not process processed messages again # do not process processed messages again
return return
# now message processed!
self.db.set_processed(count=count, processed=True)
# increment contacts counter # increment contacts counter
db_message.message.contacts += 1 db_message.message.contacts += 1
if db_message.message.contacts > self.MESSAGE_MAX_CONTACTS: if db_message.message.contacts > self.MESSAGE_MAX_CONTACTS:
logger.warning(f"Message reached max number of contacts! {db_message.message.id}, {count}") logger.warning(f"Message reached max number of contacts! {db_message.message.id}, {count}")
return return
# combine different extractions in data items
# will update items in `db_message.message.data`
fully_extracted = self._add_extractions(db_message.message.id, db_message.message.data)
if (self.REQUIRE_FULL_EXTRACT and not fully_extracted) \
and not (db_message.message.status.extract.required and not db_message.message.status.extract.finished):
logger.warning(f"Postpone message, wait for full extract of items! {db_message.message.id}, {count}")
return
# check which step/ state the message requires the management to do # check which step/ state the message requires the management to do
if db_message.message.status.extract.required and not db_message.message.status.extract.finished: if db_message.message.status.extract.required and not db_message.message.status.extract.finished:
# send to extract agents # send to extract agents
@ -114,8 +127,41 @@ class MessageProcessor():
# try again # try again
self._do_again(db_message.message) self._do_again(db_message.message)
# now message processed! def _hash_data(self, d:RiddleData) -> int:
self.db.set_processed(count=count, processed=True) return hash((d.file_plain, d.type, d.prompt))
def _add_extractions(self, riddle_id:str, data:List[RiddleData]) -> bool:
# get all the data items without extraction
empty_data = {}
for i, d in enumerate(data):
if d.file_extracted is None:
empty_data[self._hash_data(d)] = i
# search db for extractions already available
for row in self.db.iterate(
id=riddle_id,
limit=min(self.db.len(id=riddle_id), 250)
):
# check for required extraction
for d in row.message.data:
# already extracted ?
# extraction file exists ?
# one of the items, we do not have extractions for ?
# the same data item ?
if not d.file_extracted is None \
and not d.file_extracted.startswith("missing:") \
and self._hash_data(d) in empty_data:
# copy the reference to the extracted data
data[empty_data[self._hash_data(d)]].file_extracted = d.file_extracted
# remove from items we need extracted data for
del empty_data[self._hash_data(d)]
# break if all extractions found
if len(empty_data) == 0:
break
return len(empty_data) == 0 # fully extracted
def _do_again(self, message:AgentMessage): def _do_again(self, message:AgentMessage):
if message.status.trial < self.SOLUTION_MAX_TRIALS: if message.status.trial < self.SOLUTION_MAX_TRIALS:

5
ums/utils/__init__.py
View File

@ -11,7 +11,7 @@
from ums.utils.const import * from ums.utils.const import *
import logging, os import logging, os
if os.environ.get('SERVE', 'false') == 'true': if os.environ.get('SERVE', 'false').lower() == 'true':
logging.basicConfig( logging.basicConfig(
handlers=[ handlers=[
logging.FileHandler(LOG_FILE), logging.FileHandler(LOG_FILE),
@ -41,5 +41,6 @@ from ums.utils.functions import list_shared_data, list_shared_schema
from ums.utils.schema import ( from ums.utils.schema import (
ExtractionSchema, ExtractionSchema,
ExtractedData ExtractedData,
ExtractedContent, ExtractedPositions
) )

2
web/public/docs/search.js
View File

File diff suppressed because one or more lines are too long

1278
web/public/docs/ums/agent/agent.html
View File

File diff suppressed because it is too large Load Diff

2
web/public/docs/ums/agent/main.html
View File

@ -124,7 +124,7 @@
</span><span id="L-60"><a href="#L-60"><span class="linenos">60</span></a> <span class="k">def</span> <span class="nf">message</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span> </span><span id="L-60"><a href="#L-60"><span class="linenos">60</span></a> <span class="k">def</span> <span class="nf">message</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span>
</span><span id="L-61"><a href="#L-61"><span class="linenos">61</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg_process</span><span class="o">.</span><span class="n">new_message</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">background_tasks</span><span class="p">)</span> </span><span id="L-61"><a href="#L-61"><span class="linenos">61</span></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg_process</span><span class="o">.</span><span class="n">new_message</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">background_tasks</span><span class="p">)</span>
</span><span id="L-62"><a href="#L-62"><span class="linenos">62</span></a> </span><span id="L-62"><a href="#L-62"><span class="linenos">62</span></a>
</span><span id="L-63"><a href="#L-63"><span class="linenos">63</span></a><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;ums.agent.main&quot;</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span> </span><span id="L-63"><a href="#L-63"><span class="linenos">63</span></a><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;ums.agent.main&quot;</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span>
</span><span id="L-64"><a href="#L-64"><span class="linenos">64</span></a> <span class="n">main</span> <span class="o">=</span> <span class="n">WebMain</span><span class="p">()</span> </span><span id="L-64"><a href="#L-64"><span class="linenos">64</span></a> <span class="n">main</span> <span class="o">=</span> <span class="n">WebMain</span><span class="p">()</span>
</span><span id="L-65"><a href="#L-65"><span class="linenos">65</span></a> <span class="n">app</span> <span class="o">=</span> <span class="n">main</span><span class="o">.</span><span class="n">app</span> </span><span id="L-65"><a href="#L-65"><span class="linenos">65</span></a> <span class="n">app</span> <span class="o">=</span> <span class="n">main</span><span class="o">.</span><span class="n">app</span>
</span></pre></div> </span></pre></div>

2
web/public/docs/ums/management/main.html
View File

@ -187,7 +187,7 @@
</span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a> </span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a>
</span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="k">return</span> <span class="n">msg</span> </span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="k">return</span> <span class="n">msg</span>
</span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a> </span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a>
</span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;ums.management.main&quot;</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span> </span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;ums.management.main&quot;</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span>
</span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="n">main</span> <span class="o">=</span> <span class="n">WebMain</span><span class="p">()</span> </span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="n">main</span> <span class="o">=</span> <span class="n">WebMain</span><span class="p">()</span>
</span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> <span class="n">app</span> <span class="o">=</span> <span class="n">main</span><span class="o">.</span><span class="n">app</span> </span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> <span class="n">app</span> <span class="o">=</span> <span class="n">main</span><span class="o">.</span><span class="n">app</span>
</span></pre></div> </span></pre></div>

745
web/public/docs/ums/management/process.html
View File

@ -44,6 +44,9 @@
<li> <li>
<a class="variable" href="#MessageProcessor.MESSAGE_MAX_CONTACTS">MESSAGE_MAX_CONTACTS</a> <a class="variable" href="#MessageProcessor.MESSAGE_MAX_CONTACTS">MESSAGE_MAX_CONTACTS</a>
</li> </li>
<li>
<a class="variable" href="#MessageProcessor.REQUIRE_FULL_EXTRACT">REQUIRE_FULL_EXTRACT</a>
</li>
<li> <li>
<a class="variable" href="#MessageProcessor.MANAGEMENT_URL">MANAGEMENT_URL</a> <a class="variable" href="#MessageProcessor.MANAGEMENT_URL">MANAGEMENT_URL</a>
</li> </li>
@ -106,164 +109,210 @@
</span><span id="L-15"><a href="#L-15"><span class="linenos"> 15</span></a><span class="kn">from</span> <span class="nn">fastapi</span> <span class="kn">import</span> <span class="n">BackgroundTasks</span> </span><span id="L-15"><a href="#L-15"><span class="linenos"> 15</span></a><span class="kn">from</span> <span class="nn">fastapi</span> <span class="kn">import</span> <span class="n">BackgroundTasks</span>
</span><span id="L-16"><a href="#L-16"><span class="linenos"> 16</span></a> </span><span id="L-16"><a href="#L-16"><span class="linenos"> 16</span></a>
</span><span id="L-17"><a href="#L-17"><span class="linenos"> 17</span></a><span class="kn">from</span> <span class="nn">ums.management.db</span> <span class="kn">import</span> <span class="n">DB</span> </span><span id="L-17"><a href="#L-17"><span class="linenos"> 17</span></a><span class="kn">from</span> <span class="nn">ums.management.db</span> <span class="kn">import</span> <span class="n">DB</span>
</span><span id="L-18"><a href="#L-18"><span class="linenos"> 18</span></a><span class="kn">from</span> <span class="nn">ums.utils</span> <span class="kn">import</span> <span class="n">AgentMessage</span><span class="p">,</span> <span class="n">AgentResponse</span><span class="p">,</span> <span class="n">logger</span> </span><span id="L-18"><a href="#L-18"><span class="linenos"> 18</span></a><span class="kn">from</span> <span class="nn">ums.utils</span> <span class="kn">import</span> <span class="n">AgentMessage</span><span class="p">,</span> <span class="n">AgentResponse</span><span class="p">,</span> <span class="n">logger</span><span class="p">,</span> <span class="n">RiddleData</span>
</span><span id="L-19"><a href="#L-19"><span class="linenos"> 19</span></a> </span><span id="L-19"><a href="#L-19"><span class="linenos"> 19</span></a>
</span><span id="L-20"><a href="#L-20"><span class="linenos"> 20</span></a><span class="k">class</span> <span class="nc">MessageProcessor</span><span class="p">():</span> </span><span id="L-20"><a href="#L-20"><span class="linenos"> 20</span></a><span class="k">class</span> <span class="nc">MessageProcessor</span><span class="p">():</span>
</span><span id="L-21"><a href="#L-21"><span class="linenos"> 21</span></a> </span><span id="L-21"><a href="#L-21"><span class="linenos"> 21</span></a>
</span><span id="L-22"><a href="#L-22"><span class="linenos"> 22</span></a> <span class="n">SOLUTION_MAX_TRIALS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SOLUTION_MAX_TRIALS&#39;</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> </span><span id="L-22"><a href="#L-22"><span class="linenos"> 22</span></a> <span class="n">SOLUTION_MAX_TRIALS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SOLUTION_MAX_TRIALS&#39;</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
</span><span id="L-23"><a href="#L-23"><span class="linenos"> 23</span></a> <span class="n">MESSAGE_MAX_CONTACTS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MESSAGE_MAX_CONTACTS&#39;</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span> </span><span id="L-23"><a href="#L-23"><span class="linenos"> 23</span></a> <span class="n">MESSAGE_MAX_CONTACTS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MESSAGE_MAX_CONTACTS&#39;</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span>
</span><span id="L-24"><a href="#L-24"><span class="linenos"> 24</span></a> </span><span id="L-24"><a href="#L-24"><span class="linenos"> 24</span></a>
</span><span id="L-25"><a href="#L-25"><span class="linenos"> 25</span></a> <span class="n">MANAGEMENT_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MANAGEMENT_URL&#39;</span><span class="p">,</span> <span class="s1">&#39;http://127.0.0.1:80&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> </span><span id="L-25"><a href="#L-25"><span class="linenos"> 25</span></a> <span class="n">REQUIRE_FULL_EXTRACT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;REQUIRE_FULL_EXTRACT&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span>
</span><span id="L-26"><a href="#L-26"><span class="linenos"> 26</span></a> </span><span id="L-26"><a href="#L-26"><span class="linenos"> 26</span></a>
</span><span id="L-27"><a href="#L-27"><span class="linenos"> 27</span></a> <span class="n">AGENTS_PROCESS</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="L-27"><a href="#L-27"><span class="linenos"> 27</span></a> <span class="n">MANAGEMENT_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MANAGEMENT_URL&#39;</span><span class="p">,</span> <span class="s1">&#39;http://127.0.0.1:80&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
</span><span id="L-28"><a href="#L-28"><span class="linenos"> 28</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="L-28"><a href="#L-28"><span class="linenos"> 28</span></a>
</span><span id="L-29"><a href="#L-29"><span class="linenos"> 29</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_PROCESS&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="L-29"><a href="#L-29"><span class="linenos"> 29</span></a> <span class="n">AGENTS_PROCESS</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="L-30"><a href="#L-30"><span class="linenos"> 30</span></a> <span class="p">))</span> </span><span id="L-30"><a href="#L-30"><span class="linenos"> 30</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="L-31"><a href="#L-31"><span class="linenos"> 31</span></a> <span class="n">AGENTS_SOLVE</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="L-31"><a href="#L-31"><span class="linenos"> 31</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_PROCESS&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="L-32"><a href="#L-32"><span class="linenos"> 32</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="L-32"><a href="#L-32"><span class="linenos"> 32</span></a> <span class="p">))</span>
</span><span id="L-33"><a href="#L-33"><span class="linenos"> 33</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_SOLVE&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="L-33"><a href="#L-33"><span class="linenos"> 33</span></a> <span class="n">AGENTS_SOLVE</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="L-34"><a href="#L-34"><span class="linenos"> 34</span></a> <span class="p">))</span> </span><span id="L-34"><a href="#L-34"><span class="linenos"> 34</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="L-35"><a href="#L-35"><span class="linenos"> 35</span></a> <span class="n">AGENTS_GATEKEEPER</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="L-35"><a href="#L-35"><span class="linenos"> 35</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_SOLVE&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="L-36"><a href="#L-36"><span class="linenos"> 36</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="L-36"><a href="#L-36"><span class="linenos"> 36</span></a> <span class="p">))</span>
</span><span id="L-37"><a href="#L-37"><span class="linenos"> 37</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_GATEKEEPER&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="L-37"><a href="#L-37"><span class="linenos"> 37</span></a> <span class="n">AGENTS_GATEKEEPER</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="L-38"><a href="#L-38"><span class="linenos"> 38</span></a> <span class="p">))</span> </span><span id="L-38"><a href="#L-38"><span class="linenos"> 38</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="L-39"><a href="#L-39"><span class="linenos"> 39</span></a> </span><span id="L-39"><a href="#L-39"><span class="linenos"> 39</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_GATEKEEPER&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="L-40"><a href="#L-40"><span class="linenos"> 40</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span> </span><span id="L-40"><a href="#L-40"><span class="linenos"> 40</span></a> <span class="p">))</span>
</span><span id="L-41"><a href="#L-41"><span class="linenos"> 41</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span> </span><span id="L-41"><a href="#L-41"><span class="linenos"> 41</span></a>
</span><span id="L-42"><a href="#L-42"><span class="linenos"> 42</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span> </span><span id="L-42"><a href="#L-42"><span class="linenos"> 42</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span>
</span><span id="L-43"><a href="#L-43"><span class="linenos"> 43</span></a> </span><span id="L-43"><a href="#L-43"><span class="linenos"> 43</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span>
</span><span id="L-44"><a href="#L-44"><span class="linenos"> 44</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="L-44"><a href="#L-44"><span class="linenos"> 44</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span>
</span><span id="L-45"><a href="#L-45"><span class="linenos"> 45</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="L-45"><a href="#L-45"><span class="linenos"> 45</span></a>
</span><span id="L-46"><a href="#L-46"><span class="linenos"> 46</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="L-46"><a href="#L-46"><span class="linenos"> 46</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="L-47"><a href="#L-47"><span class="linenos"> 47</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="L-47"><a href="#L-47"><span class="linenos"> 47</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="L-48"><a href="#L-48"><span class="linenos"> 48</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="L-48"><a href="#L-48"><span class="linenos"> 48</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="L-49"><a href="#L-49"><span class="linenos"> 49</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="L-49"><a href="#L-49"><span class="linenos"> 49</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="L-50"><a href="#L-50"><span class="linenos"> 50</span></a> </span><span id="L-50"><a href="#L-50"><span class="linenos"> 50</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="L-51"><a href="#L-51"><span class="linenos"> 51</span></a> <span class="k">def</span> <span class="nf">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span><span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span> </span><span id="L-51"><a href="#L-51"><span class="linenos"> 51</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="L-52"><a href="#L-52"><span class="linenos"> 52</span></a> <span class="n">m</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;^https?://([^:]*)(?::(\d+))?$&#39;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> </span><span id="L-52"><a href="#L-52"><span class="linenos"> 52</span></a>
</span><span id="L-53"><a href="#L-53"><span class="linenos"> 53</span></a> <span class="k">return</span> <span class="s2">&quot;unknown&quot;</span> <span class="k">if</span> <span class="n">m</span> <span class="o">==</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> </span><span id="L-53"><a href="#L-53"><span class="linenos"> 53</span></a> <span class="k">def</span> <span class="nf">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span><span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
</span><span id="L-54"><a href="#L-54"><span class="linenos"> 54</span></a> </span><span id="L-54"><a href="#L-54"><span class="linenos"> 54</span></a> <span class="n">m</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;^https?://([^:]*)(?::(\d+))?$&#39;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
</span><span id="L-55"><a href="#L-55"><span class="linenos"> 55</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> </span><span id="L-55"><a href="#L-55"><span class="linenos"> 55</span></a> <span class="k">return</span> <span class="s2">&quot;unknown&quot;</span> <span class="k">if</span> <span class="n">m</span> <span class="o">==</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
</span><span id="L-56"><a href="#L-56"><span class="linenos"> 56</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span> </span><span id="L-56"><a href="#L-56"><span class="linenos"> 56</span></a>
</span><span id="L-57"><a href="#L-57"><span class="linenos"> 57</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span> </span><span id="L-57"><a href="#L-57"><span class="linenos"> 57</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
</span><span id="L-58"><a href="#L-58"><span class="linenos"> 58</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span> </span><span id="L-58"><a href="#L-58"><span class="linenos"> 58</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span>
</span><span id="L-59"><a href="#L-59"><span class="linenos"> 59</span></a> </span><span id="L-59"><a href="#L-59"><span class="linenos"> 59</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span>
</span><span id="L-60"><a href="#L-60"><span class="linenos"> 60</span></a> <span class="k">try</span><span class="p">:</span> </span><span id="L-60"><a href="#L-60"><span class="linenos"> 60</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span>
</span><span id="L-61"><a href="#L-61"><span class="linenos"> 61</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="L-61"><a href="#L-61"><span class="linenos"> 61</span></a>
</span><span id="L-62"><a href="#L-62"><span class="linenos"> 62</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span> </span><span id="L-62"><a href="#L-62"><span class="linenos"> 62</span></a> <span class="k">try</span><span class="p">:</span>
</span><span id="L-63"><a href="#L-63"><span class="linenos"> 63</span></a> </span><span id="L-63"><a href="#L-63"><span class="linenos"> 63</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="L-64"><a href="#L-64"><span class="linenos"> 64</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="L-64"><a href="#L-64"><span class="linenos"> 64</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span>
</span><span id="L-65"><a href="#L-65"><span class="linenos"> 65</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span> </span><span id="L-65"><a href="#L-65"><span class="linenos"> 65</span></a>
</span><span id="L-66"><a href="#L-66"><span class="linenos"> 66</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span> </span><span id="L-66"><a href="#L-66"><span class="linenos"> 66</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="L-67"><a href="#L-67"><span class="linenos"> 67</span></a> <span class="p">)</span> </span><span id="L-67"><a href="#L-67"><span class="linenos"> 67</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span>
</span><span id="L-68"><a href="#L-68"><span class="linenos"> 68</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> </span><span id="L-68"><a href="#L-68"><span class="linenos"> 68</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span>
</span><span id="L-69"><a href="#L-69"><span class="linenos"> 69</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="L-69"><a href="#L-69"><span class="linenos"> 69</span></a> <span class="p">)</span>
</span><span id="L-70"><a href="#L-70"><span class="linenos"> 70</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> </span><span id="L-70"><a href="#L-70"><span class="linenos"> 70</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
</span><span id="L-71"><a href="#L-71"><span class="linenos"> 71</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> </span><span id="L-71"><a href="#L-71"><span class="linenos"> 71</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="L-72"><a href="#L-72"><span class="linenos"> 72</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> </span><span id="L-72"><a href="#L-72"><span class="linenos"> 72</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
</span><span id="L-73"><a href="#L-73"><span class="linenos"> 73</span></a> <span class="p">)</span> </span><span id="L-73"><a href="#L-73"><span class="linenos"> 73</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
</span><span id="L-74"><a href="#L-74"><span class="linenos"> 74</span></a> </span><span id="L-74"><a href="#L-74"><span class="linenos"> 74</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
</span><span id="L-75"><a href="#L-75"><span class="linenos"> 75</span></a> <span class="k">def</span> <span class="nf">_process_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">count</span><span class="p">:</span><span class="nb">int</span><span class="p">,</span> <span class="n">ignore_processed</span><span class="p">:</span><span class="nb">bool</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> </span><span id="L-75"><a href="#L-75"><span class="linenos"> 75</span></a> <span class="p">)</span>
</span><span id="L-76"><a href="#L-76"><span class="linenos"> 76</span></a> <span class="n">db_message</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">by_count</span><span class="p">(</span><span class="n">count</span><span class="p">)</span> </span><span id="L-76"><a href="#L-76"><span class="linenos"> 76</span></a>
</span><span id="L-77"><a href="#L-77"><span class="linenos"> 77</span></a> </span><span id="L-77"><a href="#L-77"><span class="linenos"> 77</span></a> <span class="k">def</span> <span class="nf">_process_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">count</span><span class="p">:</span><span class="nb">int</span><span class="p">,</span> <span class="n">ignore_processed</span><span class="p">:</span><span class="nb">bool</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
</span><span id="L-78"><a href="#L-78"><span class="linenos"> 78</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">processed</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">ignore_processed</span><span class="p">:</span> </span><span id="L-78"><a href="#L-78"><span class="linenos"> 78</span></a> <span class="n">db_message</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">by_count</span><span class="p">(</span><span class="n">count</span><span class="p">)</span>
</span><span id="L-79"><a href="#L-79"><span class="linenos"> 79</span></a> <span class="c1"># do not process processed messages again</span> </span><span id="L-79"><a href="#L-79"><span class="linenos"> 79</span></a>
</span><span id="L-80"><a href="#L-80"><span class="linenos"> 80</span></a> <span class="k">return</span> </span><span id="L-80"><a href="#L-80"><span class="linenos"> 80</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">processed</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">ignore_processed</span><span class="p">:</span>
</span><span id="L-81"><a href="#L-81"><span class="linenos"> 81</span></a> </span><span id="L-81"><a href="#L-81"><span class="linenos"> 81</span></a> <span class="c1"># do not process processed messages again</span>
</span><span id="L-82"><a href="#L-82"><span class="linenos"> 82</span></a> <span class="c1"># increment contacts counter</span> </span><span id="L-82"><a href="#L-82"><span class="linenos"> 82</span></a> <span class="k">return</span>
</span><span id="L-83"><a href="#L-83"><span class="linenos"> 83</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">+=</span> <span class="mi">1</span> </span><span id="L-83"><a href="#L-83"><span class="linenos"> 83</span></a>
</span><span id="L-84"><a href="#L-84"><span class="linenos"> 84</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">MESSAGE_MAX_CONTACTS</span><span class="p">:</span> </span><span id="L-84"><a href="#L-84"><span class="linenos"> 84</span></a> <span class="c1"># now message processed!</span>
</span><span id="L-85"><a href="#L-85"><span class="linenos"> 85</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Message reached max number of contacts! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="L-85"><a href="#L-85"><span class="linenos"> 85</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</span><span id="L-86"><a href="#L-86"><span class="linenos"> 86</span></a> <span class="k">return</span> </span><span id="L-86"><a href="#L-86"><span class="linenos"> 86</span></a>
</span><span id="L-87"><a href="#L-87"><span class="linenos"> 87</span></a> </span><span id="L-87"><a href="#L-87"><span class="linenos"> 87</span></a> <span class="c1"># increment contacts counter</span>
</span><span id="L-88"><a href="#L-88"><span class="linenos"> 88</span></a> <span class="c1"># check which step/ state the message requires the management to do</span> </span><span id="L-88"><a href="#L-88"><span class="linenos"> 88</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">+=</span> <span class="mi">1</span>
</span><span id="L-89"><a href="#L-89"><span class="linenos"> 89</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="L-89"><a href="#L-89"><span class="linenos"> 89</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">MESSAGE_MAX_CONTACTS</span><span class="p">:</span>
</span><span id="L-90"><a href="#L-90"><span class="linenos"> 90</span></a> <span class="c1"># send to extract agents</span> </span><span id="L-90"><a href="#L-90"><span class="linenos"> 90</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Message reached max number of contacts! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="L-91"><a href="#L-91"><span class="linenos"> 91</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="L-91"><a href="#L-91"><span class="linenos"> 91</span></a> <span class="k">return</span>
</span><span id="L-92"><a href="#L-92"><span class="linenos"> 92</span></a> </span><span id="L-92"><a href="#L-92"><span class="linenos"> 92</span></a>
</span><span id="L-93"><a href="#L-93"><span class="linenos"> 93</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="L-93"><a href="#L-93"><span class="linenos"> 93</span></a> <span class="c1"># combine different extractions in data items</span>
</span><span id="L-94"><a href="#L-94"><span class="linenos"> 94</span></a> <span class="c1"># send to solve agents</span> </span><span id="L-94"><a href="#L-94"><span class="linenos"> 94</span></a> <span class="c1"># will update items in `db_message.message.data`</span>
</span><span id="L-95"><a href="#L-95"><span class="linenos"> 95</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="L-95"><a href="#L-95"><span class="linenos"> 95</span></a> <span class="n">fully_extracted</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add_extractions</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
</span><span id="L-96"><a href="#L-96"><span class="linenos"> 96</span></a> </span><span id="L-96"><a href="#L-96"><span class="linenos"> 96</span></a> <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">REQUIRE_FULL_EXTRACT</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">fully_extracted</span><span class="p">)</span> \
</span><span id="L-97"><a href="#L-97"><span class="linenos"> 97</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="L-97"><a href="#L-97"><span class="linenos"> 97</span></a> <span class="ow">and</span> <span class="ow">not</span> <span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">):</span>
</span><span id="L-98"><a href="#L-98"><span class="linenos"> 98</span></a> <span class="c1"># send to solve agents</span> </span><span id="L-98"><a href="#L-98"><span class="linenos"> 98</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Postpone message, wait for full extract of items! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="L-99"><a href="#L-99"><span class="linenos"> 99</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="L-99"><a href="#L-99"><span class="linenos"> 99</span></a> <span class="k">return</span>
</span><span id="L-100"><a href="#L-100"><span class="linenos">100</span></a> </span><span id="L-100"><a href="#L-100"><span class="linenos">100</span></a>
</span><span id="L-101"><a href="#L-101"><span class="linenos">101</span></a> <span class="k">else</span><span class="p">:</span> <span class="c1"># all steps &quot;done&quot;</span> </span><span id="L-101"><a href="#L-101"><span class="linenos">101</span></a> <span class="c1"># check which step/ state the message requires the management to do</span>
</span><span id="L-102"><a href="#L-102"><span class="linenos">102</span></a> </span><span id="L-102"><a href="#L-102"><span class="linenos">102</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="L-103"><a href="#L-103"><span class="linenos">103</span></a> <span class="c1"># validate not required? (then solved will never be set to true, thus set it here)</span> </span><span id="L-103"><a href="#L-103"><span class="linenos">103</span></a> <span class="c1"># send to extract agents</span>
</span><span id="L-104"><a href="#L-104"><span class="linenos">104</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="L-104"><a href="#L-104"><span class="linenos">104</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="L-105"><a href="#L-105"><span class="linenos">105</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span> <span class="o">=</span> <span class="kc">True</span> </span><span id="L-105"><a href="#L-105"><span class="linenos">105</span></a>
</span><span id="L-106"><a href="#L-106"><span class="linenos">106</span></a> </span><span id="L-106"><a href="#L-106"><span class="linenos">106</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="L-107"><a href="#L-107"><span class="linenos">107</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span><span class="p">:</span> </span><span id="L-107"><a href="#L-107"><span class="linenos">107</span></a> <span class="c1"># send to solve agents</span>
</span><span id="L-108"><a href="#L-108"><span class="linenos">108</span></a> <span class="c1"># yay, message is solved</span> </span><span id="L-108"><a href="#L-108"><span class="linenos">108</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="L-109"><a href="#L-109"><span class="linenos">109</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">True</span><span class="p">);</span> </span><span id="L-109"><a href="#L-109"><span class="linenos">109</span></a>
</span><span id="L-110"><a href="#L-110"><span class="linenos">110</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="L-110"><a href="#L-110"><span class="linenos">110</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="L-111"><a href="#L-111"><span class="linenos">111</span></a> <span class="c1"># not solved, but all steps done</span> </span><span id="L-111"><a href="#L-111"><span class="linenos">111</span></a> <span class="c1"># send to solve agents</span>
</span><span id="L-112"><a href="#L-112"><span class="linenos">112</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">False</span><span class="p">);</span> </span><span id="L-112"><a href="#L-112"><span class="linenos">112</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="L-113"><a href="#L-113"><span class="linenos">113</span></a> </span><span id="L-113"><a href="#L-113"><span class="linenos">113</span></a>
</span><span id="L-114"><a href="#L-114"><span class="linenos">114</span></a> <span class="c1"># try again</span> </span><span id="L-114"><a href="#L-114"><span class="linenos">114</span></a> <span class="k">else</span><span class="p">:</span> <span class="c1"># all steps &quot;done&quot;</span>
</span><span id="L-115"><a href="#L-115"><span class="linenos">115</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_do_again</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="L-115"><a href="#L-115"><span class="linenos">115</span></a>
</span><span id="L-116"><a href="#L-116"><span class="linenos">116</span></a> </span><span id="L-116"><a href="#L-116"><span class="linenos">116</span></a> <span class="c1"># validate not required? (then solved will never be set to true, thus set it here)</span>
</span><span id="L-117"><a href="#L-117"><span class="linenos">117</span></a> <span class="c1"># now message processed!</span> </span><span id="L-117"><a href="#L-117"><span class="linenos">117</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="L-118"><a href="#L-118"><span class="linenos">118</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> </span><span id="L-118"><a href="#L-118"><span class="linenos">118</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span> <span class="o">=</span> <span class="kc">True</span>
</span><span id="L-119"><a href="#L-119"><span class="linenos">119</span></a> </span><span id="L-119"><a href="#L-119"><span class="linenos">119</span></a>
</span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a> <span class="k">def</span> <span class="nf">_do_again</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">):</span> </span><span id="L-120"><a href="#L-120"><span class="linenos">120</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span><span class="p">:</span>
</span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">SOLUTION_MAX_TRIALS</span><span class="p">:</span> </span><span id="L-121"><a href="#L-121"><span class="linenos">121</span></a> <span class="c1"># yay, message is solved</span>
</span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a> <span class="c1"># try again, recycle message</span> </span><span id="L-122"><a href="#L-122"><span class="linenos">122</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">True</span><span class="p">);</span>
</span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a> </span><span id="L-123"><a href="#L-123"><span class="linenos">123</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="c1"># require steps again</span> </span><span id="L-124"><a href="#L-124"><span class="linenos">124</span></a> <span class="c1"># not solved, but all steps done</span>
</span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="L-125"><a href="#L-125"><span class="linenos">125</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">False</span><span class="p">);</span>
</span><span id="L-126"><a href="#L-126"><span class="linenos">126</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="L-126"><a href="#L-126"><span class="linenos">126</span></a>
</span><span id="L-127"><a href="#L-127"><span class="linenos">127</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="L-127"><a href="#L-127"><span class="linenos">127</span></a> <span class="c1"># try again</span>
</span><span id="L-128"><a href="#L-128"><span class="linenos">128</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="L-128"><a href="#L-128"><span class="linenos">128</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_do_again</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="L-129"><a href="#L-129"><span class="linenos">129</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="L-129"><a href="#L-129"><span class="linenos">129</span></a>
</span><span id="L-130"><a href="#L-130"><span class="linenos">130</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="L-130"><a href="#L-130"><span class="linenos">130</span></a> <span class="k">def</span> <span class="nf">_hash_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span><span class="n">RiddleData</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
</span><span id="L-131"><a href="#L-131"><span class="linenos">131</span></a> </span><span id="L-131"><a href="#L-131"><span class="linenos">131</span></a> <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="n">d</span><span class="o">.</span><span class="n">file_plain</span><span class="p">,</span> <span class="n">d</span><span class="o">.</span><span class="n">type</span><span class="p">,</span> <span class="n">d</span><span class="o">.</span><span class="n">prompt</span><span class="p">))</span>
</span><span id="L-132"><a href="#L-132"><span class="linenos">132</span></a> <span class="c1"># increment trial</span> </span><span id="L-132"><a href="#L-132"><span class="linenos">132</span></a>
</span><span id="L-133"><a href="#L-133"><span class="linenos">133</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">+=</span> <span class="mi">1</span> </span><span id="L-133"><a href="#L-133"><span class="linenos">133</span></a> <span class="k">def</span> <span class="nf">_add_extractions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">riddle_id</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="n">RiddleData</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="L-134"><a href="#L-134"><span class="linenos">134</span></a> </span><span id="L-134"><a href="#L-134"><span class="linenos">134</span></a>
</span><span id="L-135"><a href="#L-135"><span class="linenos">135</span></a> <span class="c1"># append current solution als old one</span> </span><span id="L-135"><a href="#L-135"><span class="linenos">135</span></a> <span class="c1"># get all the data items without extraction</span>
</span><span id="L-136"><a href="#L-136"><span class="linenos">136</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> </span><span id="L-136"><a href="#L-136"><span class="linenos">136</span></a> <span class="n">empty_data</span> <span class="o">=</span> <span class="p">{}</span>
</span><span id="L-137"><a href="#L-137"><span class="linenos">137</span></a> <span class="n">message</span><span class="o">.</span><span class="n">riddle</span><span class="o">.</span><span class="n">solutions_before</span><span class="o">.</span><span class="n">append</span><span class="p">(</span> </span><span id="L-137"><a href="#L-137"><span class="linenos">137</span></a> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
</span><span id="L-138"><a href="#L-138"><span class="linenos">138</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> </span><span id="L-138"><a href="#L-138"><span class="linenos">138</span></a> <span class="k">if</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
</span><span id="L-139"><a href="#L-139"><span class="linenos">139</span></a> <span class="p">)</span> </span><span id="L-139"><a href="#L-139"><span class="linenos">139</span></a> <span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]</span> <span class="o">=</span> <span class="n">i</span>
</span><span id="L-140"><a href="#L-140"><span class="linenos">140</span></a> <span class="c1"># reset current solution</span> </span><span id="L-140"><a href="#L-140"><span class="linenos">140</span></a>
</span><span id="L-141"><a href="#L-141"><span class="linenos">141</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="o">=</span> <span class="kc">None</span> </span><span id="L-141"><a href="#L-141"><span class="linenos">141</span></a> <span class="c1"># search db for extractions already available</span>
</span><span id="L-142"><a href="#L-142"><span class="linenos">142</span></a> </span><span id="L-142"><a href="#L-142"><span class="linenos">142</span></a> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">iterate</span><span class="p">(</span>
</span><span id="L-143"><a href="#L-143"><span class="linenos">143</span></a> <span class="c1"># add the riddle as new to management</span> </span><span id="L-143"><a href="#L-143"><span class="linenos">143</span></a> <span class="nb">id</span><span class="o">=</span><span class="n">riddle_id</span><span class="p">,</span>
</span><span id="L-144"><a href="#L-144"><span class="linenos">144</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="L-144"><a href="#L-144"><span class="linenos">144</span></a> <span class="n">limit</span><span class="o">=</span><span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">len</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="n">riddle_id</span><span class="p">),</span> <span class="mi">250</span><span class="p">)</span>
</span><span id="L-145"><a href="#L-145"><span class="linenos">145</span></a> </span><span id="L-145"><a href="#L-145"><span class="linenos">145</span></a> <span class="p">):</span>
</span><span id="L-146"><a href="#L-146"><span class="linenos">146</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="L-146"><a href="#L-146"><span class="linenos">146</span></a> <span class="c1"># check for required extraction</span>
</span><span id="L-147"><a href="#L-147"><span class="linenos">147</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsolved riddle after max number of trials: </span><span class="si">{</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="L-147"><a href="#L-147"><span class="linenos">147</span></a> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">row</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">data</span><span class="p">:</span>
</span><span id="L-148"><a href="#L-148"><span class="linenos">148</span></a> </span><span id="L-148"><a href="#L-148"><span class="linenos">148</span></a> <span class="c1"># already extracted ? </span>
</span><span id="L-149"><a href="#L-149"><span class="linenos">149</span></a> <span class="k">def</span> <span class="nf">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipients</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> </span><span id="L-149"><a href="#L-149"><span class="linenos">149</span></a> <span class="c1"># extraction file exists ?</span>
</span><span id="L-150"><a href="#L-150"><span class="linenos">150</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="kc">True</span> </span><span id="L-150"><a href="#L-150"><span class="linenos">150</span></a> <span class="c1"># one of the items, we do not have extractions for ?</span>
</span><span id="L-151"><a href="#L-151"><span class="linenos">151</span></a> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">recipients</span><span class="p">:</span> </span><span id="L-151"><a href="#L-151"><span class="linenos">151</span></a> <span class="c1"># the same data item ?</span>
</span><span id="L-152"><a href="#L-152"><span class="linenos">152</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="n">ok</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="L-152"><a href="#L-152"><span class="linenos">152</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span> <span class="ow">is</span> <span class="kc">None</span> \
</span><span id="L-153"><a href="#L-153"><span class="linenos">153</span></a> <span class="k">return</span> <span class="n">ok</span> </span><span id="L-153"><a href="#L-153"><span class="linenos">153</span></a> <span class="ow">and</span> <span class="ow">not</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;missing:&quot;</span><span class="p">)</span> \
</span><span id="L-154"><a href="#L-154"><span class="linenos">154</span></a> </span><span id="L-154"><a href="#L-154"><span class="linenos">154</span></a> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)</span> <span class="ow">in</span> <span class="n">empty_data</span><span class="p">:</span>
</span><span id="L-155"><a href="#L-155"><span class="linenos">155</span></a> <span class="k">def</span> <span class="nf">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipient</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> </span><span id="L-155"><a href="#L-155"><span class="linenos">155</span></a> <span class="c1"># copy the reference to the extracted data</span>
</span><span id="L-156"><a href="#L-156"><span class="linenos">156</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span> </span><span id="L-156"><a href="#L-156"><span class="linenos">156</span></a> <span class="n">data</span><span class="p">[</span><span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]]</span><span class="o">.</span><span class="n">file_extracted</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span>
</span><span id="L-157"><a href="#L-157"><span class="linenos">157</span></a> <span class="n">sender</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">management_name</span><span class="p">,</span> </span><span id="L-157"><a href="#L-157"><span class="linenos">157</span></a> <span class="c1"># remove from items we need extracted data for</span>
</span><span id="L-158"><a href="#L-158"><span class="linenos">158</span></a> <span class="n">recipient</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span> </span><span id="L-158"><a href="#L-158"><span class="linenos">158</span></a> <span class="k">del</span> <span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]</span>
</span><span id="L-159"><a href="#L-159"><span class="linenos">159</span></a> <span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> </span><span id="L-159"><a href="#L-159"><span class="linenos">159</span></a>
</span><span id="L-160"><a href="#L-160"><span class="linenos">160</span></a> <span class="n">processed</span><span class="o">=</span><span class="kc">False</span> </span><span id="L-160"><a href="#L-160"><span class="linenos">160</span></a> <span class="c1"># break if all extractions found</span>
</span><span id="L-161"><a href="#L-161"><span class="linenos">161</span></a> <span class="p">)</span> </span><span id="L-161"><a href="#L-161"><span class="linenos">161</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">empty_data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="L-162"><a href="#L-162"><span class="linenos">162</span></a> </span><span id="L-162"><a href="#L-162"><span class="linenos">162</span></a> <span class="k">break</span>
</span><span id="L-163"><a href="#L-163"><span class="linenos">163</span></a> <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span> </span><span id="L-163"><a href="#L-163"><span class="linenos">163</span></a>
</span><span id="L-164"><a href="#L-164"><span class="linenos">164</span></a> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">/message&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span> </span><span id="L-164"><a href="#L-164"><span class="linenos">164</span></a> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">empty_data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="c1"># fully extracted</span>
</span><span id="L-165"><a href="#L-165"><span class="linenos">165</span></a> <span class="n">data</span><span class="o">=</span><span class="n">message</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">(),</span> </span><span id="L-165"><a href="#L-165"><span class="linenos">165</span></a>
</span><span id="L-166"><a href="#L-166"><span class="linenos">166</span></a> <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;accept&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">,</span> <span class="s2">&quot;content-type&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">}</span> </span><span id="L-166"><a href="#L-166"><span class="linenos">166</span></a> <span class="k">def</span> <span class="nf">_do_again</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">):</span>
</span><span id="L-167"><a href="#L-167"><span class="linenos">167</span></a> <span class="p">)</span> </span><span id="L-167"><a href="#L-167"><span class="linenos">167</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">SOLUTION_MAX_TRIALS</span><span class="p">:</span>
</span><span id="L-168"><a href="#L-168"><span class="linenos">168</span></a> </span><span id="L-168"><a href="#L-168"><span class="linenos">168</span></a> <span class="c1"># try again, recycle message</span>
</span><span id="L-169"><a href="#L-169"><span class="linenos">169</span></a> <span class="k">if</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> </span><span id="L-169"><a href="#L-169"><span class="linenos">169</span></a>
</span><span id="L-170"><a href="#L-170"><span class="linenos">170</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">db_count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> </span><span id="L-170"><a href="#L-170"><span class="linenos">170</span></a> <span class="c1"># require steps again</span>
</span><span id="L-171"><a href="#L-171"><span class="linenos">171</span></a> <span class="k">return</span> <span class="kc">True</span> </span><span id="L-171"><a href="#L-171"><span class="linenos">171</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="L-172"><a href="#L-172"><span class="linenos">172</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="L-172"><a href="#L-172"><span class="linenos">172</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="L-173"><a href="#L-173"><span class="linenos">173</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error sending message to: </span><span class="si">{</span><span class="n">recipient</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="L-173"><a href="#L-173"><span class="linenos">173</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="L-174"><a href="#L-174"><span class="linenos">174</span></a> <span class="k">return</span> <span class="kc">False</span> </span><span id="L-174"><a href="#L-174"><span class="linenos">174</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="L-175"><a href="#L-175"><span class="linenos">175</span></a> </span><span id="L-175"><a href="#L-175"><span class="linenos">175</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="L-176"><a href="#L-176"><span class="linenos">176</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="L-177"><a href="#L-177"><span class="linenos">177</span></a>
</span><span id="L-178"><a href="#L-178"><span class="linenos">178</span></a> <span class="c1"># increment trial</span>
</span><span id="L-179"><a href="#L-179"><span class="linenos">179</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">+=</span> <span class="mi">1</span>
</span><span id="L-180"><a href="#L-180"><span class="linenos">180</span></a>
</span><span id="L-181"><a href="#L-181"><span class="linenos">181</span></a> <span class="c1"># append current solution als old one</span>
</span><span id="L-182"><a href="#L-182"><span class="linenos">182</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
</span><span id="L-183"><a href="#L-183"><span class="linenos">183</span></a> <span class="n">message</span><span class="o">.</span><span class="n">riddle</span><span class="o">.</span><span class="n">solutions_before</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
</span><span id="L-184"><a href="#L-184"><span class="linenos">184</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span>
</span><span id="L-185"><a href="#L-185"><span class="linenos">185</span></a> <span class="p">)</span>
</span><span id="L-186"><a href="#L-186"><span class="linenos">186</span></a> <span class="c1"># reset current solution</span>
</span><span id="L-187"><a href="#L-187"><span class="linenos">187</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="o">=</span> <span class="kc">None</span>
</span><span id="L-188"><a href="#L-188"><span class="linenos">188</span></a>
</span><span id="L-189"><a href="#L-189"><span class="linenos">189</span></a> <span class="c1"># add the riddle as new to management</span>
</span><span id="L-190"><a href="#L-190"><span class="linenos">190</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="L-191"><a href="#L-191"><span class="linenos">191</span></a>
</span><span id="L-192"><a href="#L-192"><span class="linenos">192</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="L-193"><a href="#L-193"><span class="linenos">193</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsolved riddle after max number of trials: </span><span class="si">{</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="L-194"><a href="#L-194"><span class="linenos">194</span></a>
</span><span id="L-195"><a href="#L-195"><span class="linenos">195</span></a> <span class="k">def</span> <span class="nf">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipients</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="L-196"><a href="#L-196"><span class="linenos">196</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="kc">True</span>
</span><span id="L-197"><a href="#L-197"><span class="linenos">197</span></a> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">recipients</span><span class="p">:</span>
</span><span id="L-198"><a href="#L-198"><span class="linenos">198</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="n">ok</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="L-199"><a href="#L-199"><span class="linenos">199</span></a> <span class="k">return</span> <span class="n">ok</span>
</span><span id="L-200"><a href="#L-200"><span class="linenos">200</span></a>
</span><span id="L-201"><a href="#L-201"><span class="linenos">201</span></a> <span class="k">def</span> <span class="nf">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipient</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="L-202"><a href="#L-202"><span class="linenos">202</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span>
</span><span id="L-203"><a href="#L-203"><span class="linenos">203</span></a> <span class="n">sender</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">management_name</span><span class="p">,</span>
</span><span id="L-204"><a href="#L-204"><span class="linenos">204</span></a> <span class="n">recipient</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span>
</span><span id="L-205"><a href="#L-205"><span class="linenos">205</span></a> <span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
</span><span id="L-206"><a href="#L-206"><span class="linenos">206</span></a> <span class="n">processed</span><span class="o">=</span><span class="kc">False</span>
</span><span id="L-207"><a href="#L-207"><span class="linenos">207</span></a> <span class="p">)</span>
</span><span id="L-208"><a href="#L-208"><span class="linenos">208</span></a>
</span><span id="L-209"><a href="#L-209"><span class="linenos">209</span></a> <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
</span><span id="L-210"><a href="#L-210"><span class="linenos">210</span></a> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">/message&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span>
</span><span id="L-211"><a href="#L-211"><span class="linenos">211</span></a> <span class="n">data</span><span class="o">=</span><span class="n">message</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">(),</span>
</span><span id="L-212"><a href="#L-212"><span class="linenos">212</span></a> <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;accept&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">,</span> <span class="s2">&quot;content-type&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">}</span>
</span><span id="L-213"><a href="#L-213"><span class="linenos">213</span></a> <span class="p">)</span>
</span><span id="L-214"><a href="#L-214"><span class="linenos">214</span></a>
</span><span id="L-215"><a href="#L-215"><span class="linenos">215</span></a> <span class="k">if</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
</span><span id="L-216"><a href="#L-216"><span class="linenos">216</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">db_count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</span><span id="L-217"><a href="#L-217"><span class="linenos">217</span></a> <span class="k">return</span> <span class="kc">True</span>
</span><span id="L-218"><a href="#L-218"><span class="linenos">218</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="L-219"><a href="#L-219"><span class="linenos">219</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error sending message to: </span><span class="si">{</span><span class="n">recipient</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="L-220"><a href="#L-220"><span class="linenos">220</span></a> <span class="k">return</span> <span class="kc">False</span>
</span><span id="L-221"><a href="#L-221"><span class="linenos">221</span></a>
</span></pre></div> </span></pre></div>
@ -284,156 +333,202 @@
</span><span id="MessageProcessor-23"><a href="#MessageProcessor-23"><span class="linenos"> 23</span></a> <span class="n">SOLUTION_MAX_TRIALS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SOLUTION_MAX_TRIALS&#39;</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> </span><span id="MessageProcessor-23"><a href="#MessageProcessor-23"><span class="linenos"> 23</span></a> <span class="n">SOLUTION_MAX_TRIALS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SOLUTION_MAX_TRIALS&#39;</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
</span><span id="MessageProcessor-24"><a href="#MessageProcessor-24"><span class="linenos"> 24</span></a> <span class="n">MESSAGE_MAX_CONTACTS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MESSAGE_MAX_CONTACTS&#39;</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span> </span><span id="MessageProcessor-24"><a href="#MessageProcessor-24"><span class="linenos"> 24</span></a> <span class="n">MESSAGE_MAX_CONTACTS</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MESSAGE_MAX_CONTACTS&#39;</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span>
</span><span id="MessageProcessor-25"><a href="#MessageProcessor-25"><span class="linenos"> 25</span></a> </span><span id="MessageProcessor-25"><a href="#MessageProcessor-25"><span class="linenos"> 25</span></a>
</span><span id="MessageProcessor-26"><a href="#MessageProcessor-26"><span class="linenos"> 26</span></a> <span class="n">MANAGEMENT_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MANAGEMENT_URL&#39;</span><span class="p">,</span> <span class="s1">&#39;http://127.0.0.1:80&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> </span><span id="MessageProcessor-26"><a href="#MessageProcessor-26"><span class="linenos"> 26</span></a> <span class="n">REQUIRE_FULL_EXTRACT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;REQUIRE_FULL_EXTRACT&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span>
</span><span id="MessageProcessor-27"><a href="#MessageProcessor-27"><span class="linenos"> 27</span></a> </span><span id="MessageProcessor-27"><a href="#MessageProcessor-27"><span class="linenos"> 27</span></a>
</span><span id="MessageProcessor-28"><a href="#MessageProcessor-28"><span class="linenos"> 28</span></a> <span class="n">AGENTS_PROCESS</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="MessageProcessor-28"><a href="#MessageProcessor-28"><span class="linenos"> 28</span></a> <span class="n">MANAGEMENT_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;MANAGEMENT_URL&#39;</span><span class="p">,</span> <span class="s1">&#39;http://127.0.0.1:80&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
</span><span id="MessageProcessor-29"><a href="#MessageProcessor-29"><span class="linenos"> 29</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="MessageProcessor-29"><a href="#MessageProcessor-29"><span class="linenos"> 29</span></a>
</span><span id="MessageProcessor-30"><a href="#MessageProcessor-30"><span class="linenos"> 30</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_PROCESS&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="MessageProcessor-30"><a href="#MessageProcessor-30"><span class="linenos"> 30</span></a> <span class="n">AGENTS_PROCESS</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="MessageProcessor-31"><a href="#MessageProcessor-31"><span class="linenos"> 31</span></a> <span class="p">))</span> </span><span id="MessageProcessor-31"><a href="#MessageProcessor-31"><span class="linenos"> 31</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="MessageProcessor-32"><a href="#MessageProcessor-32"><span class="linenos"> 32</span></a> <span class="n">AGENTS_SOLVE</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="MessageProcessor-32"><a href="#MessageProcessor-32"><span class="linenos"> 32</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_PROCESS&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="MessageProcessor-33"><a href="#MessageProcessor-33"><span class="linenos"> 33</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="MessageProcessor-33"><a href="#MessageProcessor-33"><span class="linenos"> 33</span></a> <span class="p">))</span>
</span><span id="MessageProcessor-34"><a href="#MessageProcessor-34"><span class="linenos"> 34</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_SOLVE&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="MessageProcessor-34"><a href="#MessageProcessor-34"><span class="linenos"> 34</span></a> <span class="n">AGENTS_SOLVE</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="MessageProcessor-35"><a href="#MessageProcessor-35"><span class="linenos"> 35</span></a> <span class="p">))</span> </span><span id="MessageProcessor-35"><a href="#MessageProcessor-35"><span class="linenos"> 35</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="MessageProcessor-36"><a href="#MessageProcessor-36"><span class="linenos"> 36</span></a> <span class="n">AGENTS_GATEKEEPER</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span> </span><span id="MessageProcessor-36"><a href="#MessageProcessor-36"><span class="linenos"> 36</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_SOLVE&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="MessageProcessor-37"><a href="#MessageProcessor-37"><span class="linenos"> 37</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span> </span><span id="MessageProcessor-37"><a href="#MessageProcessor-37"><span class="linenos"> 37</span></a> <span class="p">))</span>
</span><span id="MessageProcessor-38"><a href="#MessageProcessor-38"><span class="linenos"> 38</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_GATEKEEPER&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span> </span><span id="MessageProcessor-38"><a href="#MessageProcessor-38"><span class="linenos"> 38</span></a> <span class="n">AGENTS_GATEKEEPER</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
</span><span id="MessageProcessor-39"><a href="#MessageProcessor-39"><span class="linenos"> 39</span></a> <span class="p">))</span> </span><span id="MessageProcessor-39"><a href="#MessageProcessor-39"><span class="linenos"> 39</span></a> <span class="k">lambda</span> <span class="n">s</span><span class="p">:</span><span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">),</span>
</span><span id="MessageProcessor-40"><a href="#MessageProcessor-40"><span class="linenos"> 40</span></a> </span><span id="MessageProcessor-40"><a href="#MessageProcessor-40"><span class="linenos"> 40</span></a> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AGENTS_GATEKEEPER&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
</span><span id="MessageProcessor-41"><a href="#MessageProcessor-41"><span class="linenos"> 41</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span> </span><span id="MessageProcessor-41"><a href="#MessageProcessor-41"><span class="linenos"> 41</span></a> <span class="p">))</span>
</span><span id="MessageProcessor-42"><a href="#MessageProcessor-42"><span class="linenos"> 42</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span> </span><span id="MessageProcessor-42"><a href="#MessageProcessor-42"><span class="linenos"> 42</span></a>
</span><span id="MessageProcessor-43"><a href="#MessageProcessor-43"><span class="linenos"> 43</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span> </span><span id="MessageProcessor-43"><a href="#MessageProcessor-43"><span class="linenos"> 43</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span>
</span><span id="MessageProcessor-44"><a href="#MessageProcessor-44"><span class="linenos"> 44</span></a> </span><span id="MessageProcessor-44"><a href="#MessageProcessor-44"><span class="linenos"> 44</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span>
</span><span id="MessageProcessor-45"><a href="#MessageProcessor-45"><span class="linenos"> 45</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor-45"><a href="#MessageProcessor-45"><span class="linenos"> 45</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span>
</span><span id="MessageProcessor-46"><a href="#MessageProcessor-46"><span class="linenos"> 46</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-46"><a href="#MessageProcessor-46"><span class="linenos"> 46</span></a>
</span><span id="MessageProcessor-47"><a href="#MessageProcessor-47"><span class="linenos"> 47</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor-47"><a href="#MessageProcessor-47"><span class="linenos"> 47</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor-48"><a href="#MessageProcessor-48"><span class="linenos"> 48</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-48"><a href="#MessageProcessor-48"><span class="linenos"> 48</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-49"><a href="#MessageProcessor-49"><span class="linenos"> 49</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor-49"><a href="#MessageProcessor-49"><span class="linenos"> 49</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor-50"><a href="#MessageProcessor-50"><span class="linenos"> 50</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-50"><a href="#MessageProcessor-50"><span class="linenos"> 50</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-51"><a href="#MessageProcessor-51"><span class="linenos"> 51</span></a> </span><span id="MessageProcessor-51"><a href="#MessageProcessor-51"><span class="linenos"> 51</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor-52"><a href="#MessageProcessor-52"><span class="linenos"> 52</span></a> <span class="k">def</span> <span class="nf">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span><span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span> </span><span id="MessageProcessor-52"><a href="#MessageProcessor-52"><span class="linenos"> 52</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-53"><a href="#MessageProcessor-53"><span class="linenos"> 53</span></a> <span class="n">m</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;^https?://([^:]*)(?::(\d+))?$&#39;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> </span><span id="MessageProcessor-53"><a href="#MessageProcessor-53"><span class="linenos"> 53</span></a>
</span><span id="MessageProcessor-54"><a href="#MessageProcessor-54"><span class="linenos"> 54</span></a> <span class="k">return</span> <span class="s2">&quot;unknown&quot;</span> <span class="k">if</span> <span class="n">m</span> <span class="o">==</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> </span><span id="MessageProcessor-54"><a href="#MessageProcessor-54"><span class="linenos"> 54</span></a> <span class="k">def</span> <span class="nf">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span><span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
</span><span id="MessageProcessor-55"><a href="#MessageProcessor-55"><span class="linenos"> 55</span></a> </span><span id="MessageProcessor-55"><a href="#MessageProcessor-55"><span class="linenos"> 55</span></a> <span class="n">m</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;^https?://([^:]*)(?::(\d+))?$&#39;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
</span><span id="MessageProcessor-56"><a href="#MessageProcessor-56"><span class="linenos"> 56</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> </span><span id="MessageProcessor-56"><a href="#MessageProcessor-56"><span class="linenos"> 56</span></a> <span class="k">return</span> <span class="s2">&quot;unknown&quot;</span> <span class="k">if</span> <span class="n">m</span> <span class="o">==</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
</span><span id="MessageProcessor-57"><a href="#MessageProcessor-57"><span class="linenos"> 57</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span> </span><span id="MessageProcessor-57"><a href="#MessageProcessor-57"><span class="linenos"> 57</span></a>
</span><span id="MessageProcessor-58"><a href="#MessageProcessor-58"><span class="linenos"> 58</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span> </span><span id="MessageProcessor-58"><a href="#MessageProcessor-58"><span class="linenos"> 58</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
</span><span id="MessageProcessor-59"><a href="#MessageProcessor-59"><span class="linenos"> 59</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span> </span><span id="MessageProcessor-59"><a href="#MessageProcessor-59"><span class="linenos"> 59</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span>
</span><span id="MessageProcessor-60"><a href="#MessageProcessor-60"><span class="linenos"> 60</span></a> </span><span id="MessageProcessor-60"><a href="#MessageProcessor-60"><span class="linenos"> 60</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span>
</span><span id="MessageProcessor-61"><a href="#MessageProcessor-61"><span class="linenos"> 61</span></a> <span class="k">try</span><span class="p">:</span> </span><span id="MessageProcessor-61"><a href="#MessageProcessor-61"><span class="linenos"> 61</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span>
</span><span id="MessageProcessor-62"><a href="#MessageProcessor-62"><span class="linenos"> 62</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-62"><a href="#MessageProcessor-62"><span class="linenos"> 62</span></a>
</span><span id="MessageProcessor-63"><a href="#MessageProcessor-63"><span class="linenos"> 63</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span> </span><span id="MessageProcessor-63"><a href="#MessageProcessor-63"><span class="linenos"> 63</span></a> <span class="k">try</span><span class="p">:</span>
</span><span id="MessageProcessor-64"><a href="#MessageProcessor-64"><span class="linenos"> 64</span></a> </span><span id="MessageProcessor-64"><a href="#MessageProcessor-64"><span class="linenos"> 64</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-65"><a href="#MessageProcessor-65"><span class="linenos"> 65</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="MessageProcessor-65"><a href="#MessageProcessor-65"><span class="linenos"> 65</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span>
</span><span id="MessageProcessor-66"><a href="#MessageProcessor-66"><span class="linenos"> 66</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span> </span><span id="MessageProcessor-66"><a href="#MessageProcessor-66"><span class="linenos"> 66</span></a>
</span><span id="MessageProcessor-67"><a href="#MessageProcessor-67"><span class="linenos"> 67</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span> </span><span id="MessageProcessor-67"><a href="#MessageProcessor-67"><span class="linenos"> 67</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="MessageProcessor-68"><a href="#MessageProcessor-68"><span class="linenos"> 68</span></a> <span class="p">)</span> </span><span id="MessageProcessor-68"><a href="#MessageProcessor-68"><span class="linenos"> 68</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span>
</span><span id="MessageProcessor-69"><a href="#MessageProcessor-69"><span class="linenos"> 69</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> </span><span id="MessageProcessor-69"><a href="#MessageProcessor-69"><span class="linenos"> 69</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span>
</span><span id="MessageProcessor-70"><a href="#MessageProcessor-70"><span class="linenos"> 70</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="MessageProcessor-70"><a href="#MessageProcessor-70"><span class="linenos"> 70</span></a> <span class="p">)</span>
</span><span id="MessageProcessor-71"><a href="#MessageProcessor-71"><span class="linenos"> 71</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> </span><span id="MessageProcessor-71"><a href="#MessageProcessor-71"><span class="linenos"> 71</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
</span><span id="MessageProcessor-72"><a href="#MessageProcessor-72"><span class="linenos"> 72</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> </span><span id="MessageProcessor-72"><a href="#MessageProcessor-72"><span class="linenos"> 72</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="MessageProcessor-73"><a href="#MessageProcessor-73"><span class="linenos"> 73</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> </span><span id="MessageProcessor-73"><a href="#MessageProcessor-73"><span class="linenos"> 73</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
</span><span id="MessageProcessor-74"><a href="#MessageProcessor-74"><span class="linenos"> 74</span></a> <span class="p">)</span> </span><span id="MessageProcessor-74"><a href="#MessageProcessor-74"><span class="linenos"> 74</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
</span><span id="MessageProcessor-75"><a href="#MessageProcessor-75"><span class="linenos"> 75</span></a> </span><span id="MessageProcessor-75"><a href="#MessageProcessor-75"><span class="linenos"> 75</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
</span><span id="MessageProcessor-76"><a href="#MessageProcessor-76"><span class="linenos"> 76</span></a> <span class="k">def</span> <span class="nf">_process_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">count</span><span class="p">:</span><span class="nb">int</span><span class="p">,</span> <span class="n">ignore_processed</span><span class="p">:</span><span class="nb">bool</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> </span><span id="MessageProcessor-76"><a href="#MessageProcessor-76"><span class="linenos"> 76</span></a> <span class="p">)</span>
</span><span id="MessageProcessor-77"><a href="#MessageProcessor-77"><span class="linenos"> 77</span></a> <span class="n">db_message</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">by_count</span><span class="p">(</span><span class="n">count</span><span class="p">)</span> </span><span id="MessageProcessor-77"><a href="#MessageProcessor-77"><span class="linenos"> 77</span></a>
</span><span id="MessageProcessor-78"><a href="#MessageProcessor-78"><span class="linenos"> 78</span></a> </span><span id="MessageProcessor-78"><a href="#MessageProcessor-78"><span class="linenos"> 78</span></a> <span class="k">def</span> <span class="nf">_process_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">count</span><span class="p">:</span><span class="nb">int</span><span class="p">,</span> <span class="n">ignore_processed</span><span class="p">:</span><span class="nb">bool</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
</span><span id="MessageProcessor-79"><a href="#MessageProcessor-79"><span class="linenos"> 79</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">processed</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">ignore_processed</span><span class="p">:</span> </span><span id="MessageProcessor-79"><a href="#MessageProcessor-79"><span class="linenos"> 79</span></a> <span class="n">db_message</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">by_count</span><span class="p">(</span><span class="n">count</span><span class="p">)</span>
</span><span id="MessageProcessor-80"><a href="#MessageProcessor-80"><span class="linenos"> 80</span></a> <span class="c1"># do not process processed messages again</span> </span><span id="MessageProcessor-80"><a href="#MessageProcessor-80"><span class="linenos"> 80</span></a>
</span><span id="MessageProcessor-81"><a href="#MessageProcessor-81"><span class="linenos"> 81</span></a> <span class="k">return</span> </span><span id="MessageProcessor-81"><a href="#MessageProcessor-81"><span class="linenos"> 81</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">processed</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">ignore_processed</span><span class="p">:</span>
</span><span id="MessageProcessor-82"><a href="#MessageProcessor-82"><span class="linenos"> 82</span></a> </span><span id="MessageProcessor-82"><a href="#MessageProcessor-82"><span class="linenos"> 82</span></a> <span class="c1"># do not process processed messages again</span>
</span><span id="MessageProcessor-83"><a href="#MessageProcessor-83"><span class="linenos"> 83</span></a> <span class="c1"># increment contacts counter</span> </span><span id="MessageProcessor-83"><a href="#MessageProcessor-83"><span class="linenos"> 83</span></a> <span class="k">return</span>
</span><span id="MessageProcessor-84"><a href="#MessageProcessor-84"><span class="linenos"> 84</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">+=</span> <span class="mi">1</span> </span><span id="MessageProcessor-84"><a href="#MessageProcessor-84"><span class="linenos"> 84</span></a>
</span><span id="MessageProcessor-85"><a href="#MessageProcessor-85"><span class="linenos"> 85</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">MESSAGE_MAX_CONTACTS</span><span class="p">:</span> </span><span id="MessageProcessor-85"><a href="#MessageProcessor-85"><span class="linenos"> 85</span></a> <span class="c1"># now message processed!</span>
</span><span id="MessageProcessor-86"><a href="#MessageProcessor-86"><span class="linenos"> 86</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Message reached max number of contacts! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-86"><a href="#MessageProcessor-86"><span class="linenos"> 86</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</span><span id="MessageProcessor-87"><a href="#MessageProcessor-87"><span class="linenos"> 87</span></a> <span class="k">return</span> </span><span id="MessageProcessor-87"><a href="#MessageProcessor-87"><span class="linenos"> 87</span></a>
</span><span id="MessageProcessor-88"><a href="#MessageProcessor-88"><span class="linenos"> 88</span></a> </span><span id="MessageProcessor-88"><a href="#MessageProcessor-88"><span class="linenos"> 88</span></a> <span class="c1"># increment contacts counter</span>
</span><span id="MessageProcessor-89"><a href="#MessageProcessor-89"><span class="linenos"> 89</span></a> <span class="c1"># check which step/ state the message requires the management to do</span> </span><span id="MessageProcessor-89"><a href="#MessageProcessor-89"><span class="linenos"> 89</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">+=</span> <span class="mi">1</span>
</span><span id="MessageProcessor-90"><a href="#MessageProcessor-90"><span class="linenos"> 90</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="MessageProcessor-90"><a href="#MessageProcessor-90"><span class="linenos"> 90</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">contacts</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">MESSAGE_MAX_CONTACTS</span><span class="p">:</span>
</span><span id="MessageProcessor-91"><a href="#MessageProcessor-91"><span class="linenos"> 91</span></a> <span class="c1"># send to extract agents</span> </span><span id="MessageProcessor-91"><a href="#MessageProcessor-91"><span class="linenos"> 91</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Message reached max number of contacts! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-92"><a href="#MessageProcessor-92"><span class="linenos"> 92</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-92"><a href="#MessageProcessor-92"><span class="linenos"> 92</span></a> <span class="k">return</span>
</span><span id="MessageProcessor-93"><a href="#MessageProcessor-93"><span class="linenos"> 93</span></a> </span><span id="MessageProcessor-93"><a href="#MessageProcessor-93"><span class="linenos"> 93</span></a>
</span><span id="MessageProcessor-94"><a href="#MessageProcessor-94"><span class="linenos"> 94</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="MessageProcessor-94"><a href="#MessageProcessor-94"><span class="linenos"> 94</span></a> <span class="c1"># combine different extractions in data items</span>
</span><span id="MessageProcessor-95"><a href="#MessageProcessor-95"><span class="linenos"> 95</span></a> <span class="c1"># send to solve agents</span> </span><span id="MessageProcessor-95"><a href="#MessageProcessor-95"><span class="linenos"> 95</span></a> <span class="c1"># will update items in `db_message.message.data`</span>
</span><span id="MessageProcessor-96"><a href="#MessageProcessor-96"><span class="linenos"> 96</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-96"><a href="#MessageProcessor-96"><span class="linenos"> 96</span></a> <span class="n">fully_extracted</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_add_extractions</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
</span><span id="MessageProcessor-97"><a href="#MessageProcessor-97"><span class="linenos"> 97</span></a> </span><span id="MessageProcessor-97"><a href="#MessageProcessor-97"><span class="linenos"> 97</span></a> <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">REQUIRE_FULL_EXTRACT</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">fully_extracted</span><span class="p">)</span> \
</span><span id="MessageProcessor-98"><a href="#MessageProcessor-98"><span class="linenos"> 98</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span> </span><span id="MessageProcessor-98"><a href="#MessageProcessor-98"><span class="linenos"> 98</span></a> <span class="ow">and</span> <span class="ow">not</span> <span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">):</span>
</span><span id="MessageProcessor-99"><a href="#MessageProcessor-99"><span class="linenos"> 99</span></a> <span class="c1"># send to solve agents</span> </span><span id="MessageProcessor-99"><a href="#MessageProcessor-99"><span class="linenos"> 99</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Postpone message, wait for full extract of items! </span><span class="si">{</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-100"><a href="#MessageProcessor-100"><span class="linenos">100</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-100"><a href="#MessageProcessor-100"><span class="linenos">100</span></a> <span class="k">return</span>
</span><span id="MessageProcessor-101"><a href="#MessageProcessor-101"><span class="linenos">101</span></a> </span><span id="MessageProcessor-101"><a href="#MessageProcessor-101"><span class="linenos">101</span></a>
</span><span id="MessageProcessor-102"><a href="#MessageProcessor-102"><span class="linenos">102</span></a> <span class="k">else</span><span class="p">:</span> <span class="c1"># all steps &quot;done&quot;</span> </span><span id="MessageProcessor-102"><a href="#MessageProcessor-102"><span class="linenos">102</span></a> <span class="c1"># check which step/ state the message requires the management to do</span>
</span><span id="MessageProcessor-103"><a href="#MessageProcessor-103"><span class="linenos">103</span></a> </span><span id="MessageProcessor-103"><a href="#MessageProcessor-103"><span class="linenos">103</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="MessageProcessor-104"><a href="#MessageProcessor-104"><span class="linenos">104</span></a> <span class="c1"># validate not required? (then solved will never be set to true, thus set it here)</span> </span><span id="MessageProcessor-104"><a href="#MessageProcessor-104"><span class="linenos">104</span></a> <span class="c1"># send to extract agents</span>
</span><span id="MessageProcessor-105"><a href="#MessageProcessor-105"><span class="linenos">105</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="MessageProcessor-105"><a href="#MessageProcessor-105"><span class="linenos">105</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-106"><a href="#MessageProcessor-106"><span class="linenos">106</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span> <span class="o">=</span> <span class="kc">True</span> </span><span id="MessageProcessor-106"><a href="#MessageProcessor-106"><span class="linenos">106</span></a>
</span><span id="MessageProcessor-107"><a href="#MessageProcessor-107"><span class="linenos">107</span></a> </span><span id="MessageProcessor-107"><a href="#MessageProcessor-107"><span class="linenos">107</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="MessageProcessor-108"><a href="#MessageProcessor-108"><span class="linenos">108</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span><span class="p">:</span> </span><span id="MessageProcessor-108"><a href="#MessageProcessor-108"><span class="linenos">108</span></a> <span class="c1"># send to solve agents</span>
</span><span id="MessageProcessor-109"><a href="#MessageProcessor-109"><span class="linenos">109</span></a> <span class="c1"># yay, message is solved</span> </span><span id="MessageProcessor-109"><a href="#MessageProcessor-109"><span class="linenos">109</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-110"><a href="#MessageProcessor-110"><span class="linenos">110</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">True</span><span class="p">);</span> </span><span id="MessageProcessor-110"><a href="#MessageProcessor-110"><span class="linenos">110</span></a>
</span><span id="MessageProcessor-111"><a href="#MessageProcessor-111"><span class="linenos">111</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="MessageProcessor-111"><a href="#MessageProcessor-111"><span class="linenos">111</span></a> <span class="k">elif</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span><span class="p">:</span>
</span><span id="MessageProcessor-112"><a href="#MessageProcessor-112"><span class="linenos">112</span></a> <span class="c1"># not solved, but all steps done</span> </span><span id="MessageProcessor-112"><a href="#MessageProcessor-112"><span class="linenos">112</span></a> <span class="c1"># send to solve agents</span>
</span><span id="MessageProcessor-113"><a href="#MessageProcessor-113"><span class="linenos">113</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">False</span><span class="p">);</span> </span><span id="MessageProcessor-113"><a href="#MessageProcessor-113"><span class="linenos">113</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">,</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-114"><a href="#MessageProcessor-114"><span class="linenos">114</span></a> </span><span id="MessageProcessor-114"><a href="#MessageProcessor-114"><span class="linenos">114</span></a>
</span><span id="MessageProcessor-115"><a href="#MessageProcessor-115"><span class="linenos">115</span></a> <span class="c1"># try again</span> </span><span id="MessageProcessor-115"><a href="#MessageProcessor-115"><span class="linenos">115</span></a> <span class="k">else</span><span class="p">:</span> <span class="c1"># all steps &quot;done&quot;</span>
</span><span id="MessageProcessor-116"><a href="#MessageProcessor-116"><span class="linenos">116</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_do_again</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-116"><a href="#MessageProcessor-116"><span class="linenos">116</span></a>
</span><span id="MessageProcessor-117"><a href="#MessageProcessor-117"><span class="linenos">117</span></a> </span><span id="MessageProcessor-117"><a href="#MessageProcessor-117"><span class="linenos">117</span></a> <span class="c1"># validate not required? (then solved will never be set to true, thus set it here)</span>
</span><span id="MessageProcessor-118"><a href="#MessageProcessor-118"><span class="linenos">118</span></a> <span class="c1"># now message processed!</span> </span><span id="MessageProcessor-118"><a href="#MessageProcessor-118"><span class="linenos">118</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="MessageProcessor-119"><a href="#MessageProcessor-119"><span class="linenos">119</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> </span><span id="MessageProcessor-119"><a href="#MessageProcessor-119"><span class="linenos">119</span></a> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span> <span class="o">=</span> <span class="kc">True</span>
</span><span id="MessageProcessor-120"><a href="#MessageProcessor-120"><span class="linenos">120</span></a> </span><span id="MessageProcessor-120"><a href="#MessageProcessor-120"><span class="linenos">120</span></a>
</span><span id="MessageProcessor-121"><a href="#MessageProcessor-121"><span class="linenos">121</span></a> <span class="k">def</span> <span class="nf">_do_again</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">):</span> </span><span id="MessageProcessor-121"><a href="#MessageProcessor-121"><span class="linenos">121</span></a> <span class="k">if</span> <span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solved</span><span class="p">:</span>
</span><span id="MessageProcessor-122"><a href="#MessageProcessor-122"><span class="linenos">122</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">SOLUTION_MAX_TRIALS</span><span class="p">:</span> </span><span id="MessageProcessor-122"><a href="#MessageProcessor-122"><span class="linenos">122</span></a> <span class="c1"># yay, message is solved</span>
</span><span id="MessageProcessor-123"><a href="#MessageProcessor-123"><span class="linenos">123</span></a> <span class="c1"># try again, recycle message</span> </span><span id="MessageProcessor-123"><a href="#MessageProcessor-123"><span class="linenos">123</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">True</span><span class="p">);</span>
</span><span id="MessageProcessor-124"><a href="#MessageProcessor-124"><span class="linenos">124</span></a> </span><span id="MessageProcessor-124"><a href="#MessageProcessor-124"><span class="linenos">124</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="MessageProcessor-125"><a href="#MessageProcessor-125"><span class="linenos">125</span></a> <span class="c1"># require steps again</span> </span><span id="MessageProcessor-125"><a href="#MessageProcessor-125"><span class="linenos">125</span></a> <span class="c1"># not solved, but all steps done</span>
</span><span id="MessageProcessor-126"><a href="#MessageProcessor-126"><span class="linenos">126</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="MessageProcessor-126"><a href="#MessageProcessor-126"><span class="linenos">126</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_solution</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">solution</span><span class="o">=</span><span class="kc">False</span><span class="p">);</span>
</span><span id="MessageProcessor-127"><a href="#MessageProcessor-127"><span class="linenos">127</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="MessageProcessor-127"><a href="#MessageProcessor-127"><span class="linenos">127</span></a>
</span><span id="MessageProcessor-128"><a href="#MessageProcessor-128"><span class="linenos">128</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="MessageProcessor-128"><a href="#MessageProcessor-128"><span class="linenos">128</span></a> <span class="c1"># try again</span>
</span><span id="MessageProcessor-129"><a href="#MessageProcessor-129"><span class="linenos">129</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="MessageProcessor-129"><a href="#MessageProcessor-129"><span class="linenos">129</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_do_again</span><span class="p">(</span><span class="n">db_message</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-130"><a href="#MessageProcessor-130"><span class="linenos">130</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span> </span><span id="MessageProcessor-130"><a href="#MessageProcessor-130"><span class="linenos">130</span></a>
</span><span id="MessageProcessor-131"><a href="#MessageProcessor-131"><span class="linenos">131</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span> </span><span id="MessageProcessor-131"><a href="#MessageProcessor-131"><span class="linenos">131</span></a> <span class="k">def</span> <span class="nf">_hash_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span><span class="n">RiddleData</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
</span><span id="MessageProcessor-132"><a href="#MessageProcessor-132"><span class="linenos">132</span></a> </span><span id="MessageProcessor-132"><a href="#MessageProcessor-132"><span class="linenos">132</span></a> <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="n">d</span><span class="o">.</span><span class="n">file_plain</span><span class="p">,</span> <span class="n">d</span><span class="o">.</span><span class="n">type</span><span class="p">,</span> <span class="n">d</span><span class="o">.</span><span class="n">prompt</span><span class="p">))</span>
</span><span id="MessageProcessor-133"><a href="#MessageProcessor-133"><span class="linenos">133</span></a> <span class="c1"># increment trial</span> </span><span id="MessageProcessor-133"><a href="#MessageProcessor-133"><span class="linenos">133</span></a>
</span><span id="MessageProcessor-134"><a href="#MessageProcessor-134"><span class="linenos">134</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">+=</span> <span class="mi">1</span> </span><span id="MessageProcessor-134"><a href="#MessageProcessor-134"><span class="linenos">134</span></a> <span class="k">def</span> <span class="nf">_add_extractions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">riddle_id</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="n">RiddleData</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="MessageProcessor-135"><a href="#MessageProcessor-135"><span class="linenos">135</span></a> </span><span id="MessageProcessor-135"><a href="#MessageProcessor-135"><span class="linenos">135</span></a>
</span><span id="MessageProcessor-136"><a href="#MessageProcessor-136"><span class="linenos">136</span></a> <span class="c1"># append current solution als old one</span> </span><span id="MessageProcessor-136"><a href="#MessageProcessor-136"><span class="linenos">136</span></a> <span class="c1"># get all the data items without extraction</span>
</span><span id="MessageProcessor-137"><a href="#MessageProcessor-137"><span class="linenos">137</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> </span><span id="MessageProcessor-137"><a href="#MessageProcessor-137"><span class="linenos">137</span></a> <span class="n">empty_data</span> <span class="o">=</span> <span class="p">{}</span>
</span><span id="MessageProcessor-138"><a href="#MessageProcessor-138"><span class="linenos">138</span></a> <span class="n">message</span><span class="o">.</span><span class="n">riddle</span><span class="o">.</span><span class="n">solutions_before</span><span class="o">.</span><span class="n">append</span><span class="p">(</span> </span><span id="MessageProcessor-138"><a href="#MessageProcessor-138"><span class="linenos">138</span></a> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">d</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
</span><span id="MessageProcessor-139"><a href="#MessageProcessor-139"><span class="linenos">139</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> </span><span id="MessageProcessor-139"><a href="#MessageProcessor-139"><span class="linenos">139</span></a> <span class="k">if</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
</span><span id="MessageProcessor-140"><a href="#MessageProcessor-140"><span class="linenos">140</span></a> <span class="p">)</span> </span><span id="MessageProcessor-140"><a href="#MessageProcessor-140"><span class="linenos">140</span></a> <span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]</span> <span class="o">=</span> <span class="n">i</span>
</span><span id="MessageProcessor-141"><a href="#MessageProcessor-141"><span class="linenos">141</span></a> <span class="c1"># reset current solution</span> </span><span id="MessageProcessor-141"><a href="#MessageProcessor-141"><span class="linenos">141</span></a>
</span><span id="MessageProcessor-142"><a href="#MessageProcessor-142"><span class="linenos">142</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="o">=</span> <span class="kc">None</span> </span><span id="MessageProcessor-142"><a href="#MessageProcessor-142"><span class="linenos">142</span></a> <span class="c1"># search db for extractions already available</span>
</span><span id="MessageProcessor-143"><a href="#MessageProcessor-143"><span class="linenos">143</span></a> </span><span id="MessageProcessor-143"><a href="#MessageProcessor-143"><span class="linenos">143</span></a> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">iterate</span><span class="p">(</span>
</span><span id="MessageProcessor-144"><a href="#MessageProcessor-144"><span class="linenos">144</span></a> <span class="c1"># add the riddle as new to management</span> </span><span id="MessageProcessor-144"><a href="#MessageProcessor-144"><span class="linenos">144</span></a> <span class="nb">id</span><span class="o">=</span><span class="n">riddle_id</span><span class="p">,</span>
</span><span id="MessageProcessor-145"><a href="#MessageProcessor-145"><span class="linenos">145</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-145"><a href="#MessageProcessor-145"><span class="linenos">145</span></a> <span class="n">limit</span><span class="o">=</span><span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">len</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="n">riddle_id</span><span class="p">),</span> <span class="mi">250</span><span class="p">)</span>
</span><span id="MessageProcessor-146"><a href="#MessageProcessor-146"><span class="linenos">146</span></a> </span><span id="MessageProcessor-146"><a href="#MessageProcessor-146"><span class="linenos">146</span></a> <span class="p">):</span>
</span><span id="MessageProcessor-147"><a href="#MessageProcessor-147"><span class="linenos">147</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="MessageProcessor-147"><a href="#MessageProcessor-147"><span class="linenos">147</span></a> <span class="c1"># check for required extraction</span>
</span><span id="MessageProcessor-148"><a href="#MessageProcessor-148"><span class="linenos">148</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsolved riddle after max number of trials: </span><span class="si">{</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-148"><a href="#MessageProcessor-148"><span class="linenos">148</span></a> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">row</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">data</span><span class="p">:</span>
</span><span id="MessageProcessor-149"><a href="#MessageProcessor-149"><span class="linenos">149</span></a> </span><span id="MessageProcessor-149"><a href="#MessageProcessor-149"><span class="linenos">149</span></a> <span class="c1"># already extracted ? </span>
</span><span id="MessageProcessor-150"><a href="#MessageProcessor-150"><span class="linenos">150</span></a> <span class="k">def</span> <span class="nf">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipients</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> </span><span id="MessageProcessor-150"><a href="#MessageProcessor-150"><span class="linenos">150</span></a> <span class="c1"># extraction file exists ?</span>
</span><span id="MessageProcessor-151"><a href="#MessageProcessor-151"><span class="linenos">151</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="kc">True</span> </span><span id="MessageProcessor-151"><a href="#MessageProcessor-151"><span class="linenos">151</span></a> <span class="c1"># one of the items, we do not have extractions for ?</span>
</span><span id="MessageProcessor-152"><a href="#MessageProcessor-152"><span class="linenos">152</span></a> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">recipients</span><span class="p">:</span> </span><span id="MessageProcessor-152"><a href="#MessageProcessor-152"><span class="linenos">152</span></a> <span class="c1"># the same data item ?</span>
</span><span id="MessageProcessor-153"><a href="#MessageProcessor-153"><span class="linenos">153</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="n">ok</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor-153"><a href="#MessageProcessor-153"><span class="linenos">153</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span> <span class="ow">is</span> <span class="kc">None</span> \
</span><span id="MessageProcessor-154"><a href="#MessageProcessor-154"><span class="linenos">154</span></a> <span class="k">return</span> <span class="n">ok</span> </span><span id="MessageProcessor-154"><a href="#MessageProcessor-154"><span class="linenos">154</span></a> <span class="ow">and</span> <span class="ow">not</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;missing:&quot;</span><span class="p">)</span> \
</span><span id="MessageProcessor-155"><a href="#MessageProcessor-155"><span class="linenos">155</span></a> </span><span id="MessageProcessor-155"><a href="#MessageProcessor-155"><span class="linenos">155</span></a> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)</span> <span class="ow">in</span> <span class="n">empty_data</span><span class="p">:</span>
</span><span id="MessageProcessor-156"><a href="#MessageProcessor-156"><span class="linenos">156</span></a> <span class="k">def</span> <span class="nf">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipient</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> </span><span id="MessageProcessor-156"><a href="#MessageProcessor-156"><span class="linenos">156</span></a> <span class="c1"># copy the reference to the extracted data</span>
</span><span id="MessageProcessor-157"><a href="#MessageProcessor-157"><span class="linenos">157</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span> </span><span id="MessageProcessor-157"><a href="#MessageProcessor-157"><span class="linenos">157</span></a> <span class="n">data</span><span class="p">[</span><span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]]</span><span class="o">.</span><span class="n">file_extracted</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">file_extracted</span>
</span><span id="MessageProcessor-158"><a href="#MessageProcessor-158"><span class="linenos">158</span></a> <span class="n">sender</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">management_name</span><span class="p">,</span> </span><span id="MessageProcessor-158"><a href="#MessageProcessor-158"><span class="linenos">158</span></a> <span class="c1"># remove from items we need extracted data for</span>
</span><span id="MessageProcessor-159"><a href="#MessageProcessor-159"><span class="linenos">159</span></a> <span class="n">recipient</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span> </span><span id="MessageProcessor-159"><a href="#MessageProcessor-159"><span class="linenos">159</span></a> <span class="k">del</span> <span class="n">empty_data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_hash_data</span><span class="p">(</span><span class="n">d</span><span class="p">)]</span>
</span><span id="MessageProcessor-160"><a href="#MessageProcessor-160"><span class="linenos">160</span></a> <span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> </span><span id="MessageProcessor-160"><a href="#MessageProcessor-160"><span class="linenos">160</span></a>
</span><span id="MessageProcessor-161"><a href="#MessageProcessor-161"><span class="linenos">161</span></a> <span class="n">processed</span><span class="o">=</span><span class="kc">False</span> </span><span id="MessageProcessor-161"><a href="#MessageProcessor-161"><span class="linenos">161</span></a> <span class="c1"># break if all extractions found</span>
</span><span id="MessageProcessor-162"><a href="#MessageProcessor-162"><span class="linenos">162</span></a> <span class="p">)</span> </span><span id="MessageProcessor-162"><a href="#MessageProcessor-162"><span class="linenos">162</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">empty_data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor-163"><a href="#MessageProcessor-163"><span class="linenos">163</span></a> </span><span id="MessageProcessor-163"><a href="#MessageProcessor-163"><span class="linenos">163</span></a> <span class="k">break</span>
</span><span id="MessageProcessor-164"><a href="#MessageProcessor-164"><span class="linenos">164</span></a> <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span> </span><span id="MessageProcessor-164"><a href="#MessageProcessor-164"><span class="linenos">164</span></a>
</span><span id="MessageProcessor-165"><a href="#MessageProcessor-165"><span class="linenos">165</span></a> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">/message&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span> </span><span id="MessageProcessor-165"><a href="#MessageProcessor-165"><span class="linenos">165</span></a> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">empty_data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="c1"># fully extracted</span>
</span><span id="MessageProcessor-166"><a href="#MessageProcessor-166"><span class="linenos">166</span></a> <span class="n">data</span><span class="o">=</span><span class="n">message</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">(),</span> </span><span id="MessageProcessor-166"><a href="#MessageProcessor-166"><span class="linenos">166</span></a>
</span><span id="MessageProcessor-167"><a href="#MessageProcessor-167"><span class="linenos">167</span></a> <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;accept&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">,</span> <span class="s2">&quot;content-type&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">}</span> </span><span id="MessageProcessor-167"><a href="#MessageProcessor-167"><span class="linenos">167</span></a> <span class="k">def</span> <span class="nf">_do_again</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">):</span>
</span><span id="MessageProcessor-168"><a href="#MessageProcessor-168"><span class="linenos">168</span></a> <span class="p">)</span> </span><span id="MessageProcessor-168"><a href="#MessageProcessor-168"><span class="linenos">168</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">SOLUTION_MAX_TRIALS</span><span class="p">:</span>
</span><span id="MessageProcessor-169"><a href="#MessageProcessor-169"><span class="linenos">169</span></a> </span><span id="MessageProcessor-169"><a href="#MessageProcessor-169"><span class="linenos">169</span></a> <span class="c1"># try again, recycle message</span>
</span><span id="MessageProcessor-170"><a href="#MessageProcessor-170"><span class="linenos">170</span></a> <span class="k">if</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> </span><span id="MessageProcessor-170"><a href="#MessageProcessor-170"><span class="linenos">170</span></a>
</span><span id="MessageProcessor-171"><a href="#MessageProcessor-171"><span class="linenos">171</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">db_count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> </span><span id="MessageProcessor-171"><a href="#MessageProcessor-171"><span class="linenos">171</span></a> <span class="c1"># require steps again</span>
</span><span id="MessageProcessor-172"><a href="#MessageProcessor-172"><span class="linenos">172</span></a> <span class="k">return</span> <span class="kc">True</span> </span><span id="MessageProcessor-172"><a href="#MessageProcessor-172"><span class="linenos">172</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="MessageProcessor-173"><a href="#MessageProcessor-173"><span class="linenos">173</span></a> <span class="k">else</span><span class="p">:</span> </span><span id="MessageProcessor-173"><a href="#MessageProcessor-173"><span class="linenos">173</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">extract</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="MessageProcessor-174"><a href="#MessageProcessor-174"><span class="linenos">174</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error sending message to: </span><span class="si">{</span><span class="n">recipient</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span> </span><span id="MessageProcessor-174"><a href="#MessageProcessor-174"><span class="linenos">174</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="MessageProcessor-175"><a href="#MessageProcessor-175"><span class="linenos">175</span></a> <span class="k">return</span> <span class="kc">False</span> </span><span id="MessageProcessor-175"><a href="#MessageProcessor-175"><span class="linenos">175</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">solve</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="MessageProcessor-176"><a href="#MessageProcessor-176"><span class="linenos">176</span></a> <span class="k">if</span> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">required</span><span class="p">:</span>
</span><span id="MessageProcessor-177"><a href="#MessageProcessor-177"><span class="linenos">177</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">validate</span><span class="o">.</span><span class="n">finished</span> <span class="o">=</span> <span class="kc">False</span>
</span><span id="MessageProcessor-178"><a href="#MessageProcessor-178"><span class="linenos">178</span></a>
</span><span id="MessageProcessor-179"><a href="#MessageProcessor-179"><span class="linenos">179</span></a> <span class="c1"># increment trial</span>
</span><span id="MessageProcessor-180"><a href="#MessageProcessor-180"><span class="linenos">180</span></a> <span class="n">message</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">trial</span> <span class="o">+=</span> <span class="mi">1</span>
</span><span id="MessageProcessor-181"><a href="#MessageProcessor-181"><span class="linenos">181</span></a>
</span><span id="MessageProcessor-182"><a href="#MessageProcessor-182"><span class="linenos">182</span></a> <span class="c1"># append current solution als old one</span>
</span><span id="MessageProcessor-183"><a href="#MessageProcessor-183"><span class="linenos">183</span></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
</span><span id="MessageProcessor-184"><a href="#MessageProcessor-184"><span class="linenos">184</span></a> <span class="n">message</span><span class="o">.</span><span class="n">riddle</span><span class="o">.</span><span class="n">solutions_before</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
</span><span id="MessageProcessor-185"><a href="#MessageProcessor-185"><span class="linenos">185</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span>
</span><span id="MessageProcessor-186"><a href="#MessageProcessor-186"><span class="linenos">186</span></a> <span class="p">)</span>
</span><span id="MessageProcessor-187"><a href="#MessageProcessor-187"><span class="linenos">187</span></a> <span class="c1"># reset current solution</span>
</span><span id="MessageProcessor-188"><a href="#MessageProcessor-188"><span class="linenos">188</span></a> <span class="n">message</span><span class="o">.</span><span class="n">solution</span> <span class="o">=</span> <span class="kc">None</span>
</span><span id="MessageProcessor-189"><a href="#MessageProcessor-189"><span class="linenos">189</span></a>
</span><span id="MessageProcessor-190"><a href="#MessageProcessor-190"><span class="linenos">190</span></a> <span class="c1"># add the riddle as new to management</span>
</span><span id="MessageProcessor-191"><a href="#MessageProcessor-191"><span class="linenos">191</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-192"><a href="#MessageProcessor-192"><span class="linenos">192</span></a>
</span><span id="MessageProcessor-193"><a href="#MessageProcessor-193"><span class="linenos">193</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="MessageProcessor-194"><a href="#MessageProcessor-194"><span class="linenos">194</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsolved riddle after max number of trials: </span><span class="si">{</span><span class="n">message</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-195"><a href="#MessageProcessor-195"><span class="linenos">195</span></a>
</span><span id="MessageProcessor-196"><a href="#MessageProcessor-196"><span class="linenos">196</span></a> <span class="k">def</span> <span class="nf">_send_messages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipients</span><span class="p">:</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="MessageProcessor-197"><a href="#MessageProcessor-197"><span class="linenos">197</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="kc">True</span>
</span><span id="MessageProcessor-198"><a href="#MessageProcessor-198"><span class="linenos">198</span></a> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">recipients</span><span class="p">:</span>
</span><span id="MessageProcessor-199"><a href="#MessageProcessor-199"><span class="linenos">199</span></a> <span class="n">ok</span> <span class="o">=</span> <span class="n">ok</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_send_message</span><span class="p">(</span><span class="n">r</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor-200"><a href="#MessageProcessor-200"><span class="linenos">200</span></a> <span class="k">return</span> <span class="n">ok</span>
</span><span id="MessageProcessor-201"><a href="#MessageProcessor-201"><span class="linenos">201</span></a>
</span><span id="MessageProcessor-202"><a href="#MessageProcessor-202"><span class="linenos">202</span></a> <span class="k">def</span> <span class="nf">_send_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recipient</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
</span><span id="MessageProcessor-203"><a href="#MessageProcessor-203"><span class="linenos">203</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span>
</span><span id="MessageProcessor-204"><a href="#MessageProcessor-204"><span class="linenos">204</span></a> <span class="n">sender</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">management_name</span><span class="p">,</span>
</span><span id="MessageProcessor-205"><a href="#MessageProcessor-205"><span class="linenos">205</span></a> <span class="n">recipient</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span>
</span><span id="MessageProcessor-206"><a href="#MessageProcessor-206"><span class="linenos">206</span></a> <span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
</span><span id="MessageProcessor-207"><a href="#MessageProcessor-207"><span class="linenos">207</span></a> <span class="n">processed</span><span class="o">=</span><span class="kc">False</span>
</span><span id="MessageProcessor-208"><a href="#MessageProcessor-208"><span class="linenos">208</span></a> <span class="p">)</span>
</span><span id="MessageProcessor-209"><a href="#MessageProcessor-209"><span class="linenos">209</span></a>
</span><span id="MessageProcessor-210"><a href="#MessageProcessor-210"><span class="linenos">210</span></a> <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
</span><span id="MessageProcessor-211"><a href="#MessageProcessor-211"><span class="linenos">211</span></a> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">/message&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">recipient</span><span class="p">),</span>
</span><span id="MessageProcessor-212"><a href="#MessageProcessor-212"><span class="linenos">212</span></a> <span class="n">data</span><span class="o">=</span><span class="n">message</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">(),</span>
</span><span id="MessageProcessor-213"><a href="#MessageProcessor-213"><span class="linenos">213</span></a> <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;accept&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">,</span> <span class="s2">&quot;content-type&quot;</span> <span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">}</span>
</span><span id="MessageProcessor-214"><a href="#MessageProcessor-214"><span class="linenos">214</span></a> <span class="p">)</span>
</span><span id="MessageProcessor-215"><a href="#MessageProcessor-215"><span class="linenos">215</span></a>
</span><span id="MessageProcessor-216"><a href="#MessageProcessor-216"><span class="linenos">216</span></a> <span class="k">if</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
</span><span id="MessageProcessor-217"><a href="#MessageProcessor-217"><span class="linenos">217</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">set_processed</span><span class="p">(</span><span class="n">db_count</span><span class="p">,</span> <span class="n">processed</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</span><span id="MessageProcessor-218"><a href="#MessageProcessor-218"><span class="linenos">218</span></a> <span class="k">return</span> <span class="kc">True</span>
</span><span id="MessageProcessor-219"><a href="#MessageProcessor-219"><span class="linenos">219</span></a> <span class="k">else</span><span class="p">:</span>
</span><span id="MessageProcessor-220"><a href="#MessageProcessor-220"><span class="linenos">220</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error sending message to: </span><span class="si">{</span><span class="n">recipient</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">text</span><span class="p">,</span><span class="w"> </span><span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor-221"><a href="#MessageProcessor-221"><span class="linenos">221</span></a> <span class="k">return</span> <span class="kc">False</span>
</span></pre></div> </span></pre></div>
@ -449,16 +544,16 @@
</div> </div>
<a class="headerlink" href="#MessageProcessor.__init__"></a> <a class="headerlink" href="#MessageProcessor.__init__"></a>
<div class="pdoc-code codehilite"><pre><span></span><span id="MessageProcessor.__init__-41"><a href="#MessageProcessor.__init__-41"><span class="linenos">41</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span> <div class="pdoc-code codehilite"><pre><span></span><span id="MessageProcessor.__init__-43"><a href="#MessageProcessor.__init__-43"><span class="linenos">43</span></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">db</span><span class="p">:</span><span class="n">DB</span><span class="p">):</span>
</span><span id="MessageProcessor.__init__-42"><a href="#MessageProcessor.__init__-42"><span class="linenos">42</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span> </span><span id="MessageProcessor.__init__-44"><a href="#MessageProcessor.__init__-44"><span class="linenos">44</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">db</span> <span class="o">=</span> <span class="n">db</span>
</span><span id="MessageProcessor.__init__-43"><a href="#MessageProcessor.__init__-43"><span class="linenos">43</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span> </span><span id="MessageProcessor.__init__-45"><a href="#MessageProcessor.__init__-45"><span class="linenos">45</span></a> <span class="bp">self</span><span class="o">.</span><span class="n">management_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">MANAGEMENT_URL</span><span class="p">)</span>
</span><span id="MessageProcessor.__init__-44"><a href="#MessageProcessor.__init__-44"><span class="linenos">44</span></a> </span><span id="MessageProcessor.__init__-46"><a href="#MessageProcessor.__init__-46"><span class="linenos">46</span></a>
</span><span id="MessageProcessor.__init__-45"><a href="#MessageProcessor.__init__-45"><span class="linenos">45</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor.__init__-47"><a href="#MessageProcessor.__init__-47"><span class="linenos">47</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_PROCESS</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor.__init__-46"><a href="#MessageProcessor.__init__-46"><span class="linenos">46</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor.__init__-48"><a href="#MessageProcessor.__init__-48"><span class="linenos">48</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Process Agent (AGENTS_PROCESS) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor.__init__-47"><a href="#MessageProcessor.__init__-47"><span class="linenos">47</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor.__init__-49"><a href="#MessageProcessor.__init__-49"><span class="linenos">49</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_SOLVE</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor.__init__-48"><a href="#MessageProcessor.__init__-48"><span class="linenos">48</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor.__init__-50"><a href="#MessageProcessor.__init__-50"><span class="linenos">50</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Solve Agent (AGENTS_SOLVE) found, this may be a problem!&quot;</span><span class="p">)</span>
</span><span id="MessageProcessor.__init__-49"><a href="#MessageProcessor.__init__-49"><span class="linenos">49</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> </span><span id="MessageProcessor.__init__-51"><a href="#MessageProcessor.__init__-51"><span class="linenos">51</span></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">AGENTS_GATEKEEPER</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
</span><span id="MessageProcessor.__init__-50"><a href="#MessageProcessor.__init__-50"><span class="linenos">50</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span> </span><span id="MessageProcessor.__init__-52"><a href="#MessageProcessor.__init__-52"><span class="linenos">52</span></a> <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Not Gatekeeper Agent (AGENTS_GATEKEEPER) found, this may be a problem!&quot;</span><span class="p">)</span>
</span></pre></div> </span></pre></div>
@ -488,6 +583,18 @@
</div>
<div id="MessageProcessor.REQUIRE_FULL_EXTRACT" class="classattr">
<div class="attr variable">
<span class="name">REQUIRE_FULL_EXTRACT</span> =
<span class="default_value">False</span>
</div>
<a class="headerlink" href="#MessageProcessor.REQUIRE_FULL_EXTRACT"></a>
</div> </div>
<div id="MessageProcessor.MANAGEMENT_URL" class="classattr"> <div id="MessageProcessor.MANAGEMENT_URL" class="classattr">
<div class="attr variable"> <div class="attr variable">
@ -570,25 +677,25 @@
</div> </div>
<a class="headerlink" href="#MessageProcessor.new_message"></a> <a class="headerlink" href="#MessageProcessor.new_message"></a>
<div class="pdoc-code codehilite"><pre><span></span><span id="MessageProcessor.new_message-56"><a href="#MessageProcessor.new_message-56"><span class="linenos">56</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <div class="pdoc-code codehilite"><pre><span></span><span id="MessageProcessor.new_message-58"><a href="#MessageProcessor.new_message-58"><span class="linenos">58</span></a> <span class="k">def</span> <span class="nf">new_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
</span><span id="MessageProcessor.new_message-57"><a href="#MessageProcessor.new_message-57"><span class="linenos">57</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span> </span><span id="MessageProcessor.new_message-59"><a href="#MessageProcessor.new_message-59"><span class="linenos">59</span></a> <span class="n">sender</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">receiver</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span><span class="n">AgentMessage</span><span class="p">,</span>
</span><span id="MessageProcessor.new_message-58"><a href="#MessageProcessor.new_message-58"><span class="linenos">58</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span> </span><span id="MessageProcessor.new_message-60"><a href="#MessageProcessor.new_message-60"><span class="linenos">60</span></a> <span class="n">background_tasks</span><span class="p">:</span> <span class="n">BackgroundTasks</span>
</span><span id="MessageProcessor.new_message-59"><a href="#MessageProcessor.new_message-59"><span class="linenos">59</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span> </span><span id="MessageProcessor.new_message-61"><a href="#MessageProcessor.new_message-61"><span class="linenos">61</span></a> <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AgentResponse</span><span class="p">:</span>
</span><span id="MessageProcessor.new_message-60"><a href="#MessageProcessor.new_message-60"><span class="linenos">60</span></a> </span><span id="MessageProcessor.new_message-62"><a href="#MessageProcessor.new_message-62"><span class="linenos">62</span></a>
</span><span id="MessageProcessor.new_message-61"><a href="#MessageProcessor.new_message-61"><span class="linenos">61</span></a> <span class="k">try</span><span class="p">:</span> </span><span id="MessageProcessor.new_message-63"><a href="#MessageProcessor.new_message-63"><span class="linenos">63</span></a> <span class="k">try</span><span class="p">:</span>
</span><span id="MessageProcessor.new_message-62"><a href="#MessageProcessor.new_message-62"><span class="linenos">62</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span> </span><span id="MessageProcessor.new_message-64"><a href="#MessageProcessor.new_message-64"><span class="linenos">64</span></a> <span class="n">db_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">add_message</span><span class="p">(</span><span class="n">sender</span><span class="p">,</span> <span class="n">receiver</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
</span><span id="MessageProcessor.new_message-63"><a href="#MessageProcessor.new_message-63"><span class="linenos">63</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span> </span><span id="MessageProcessor.new_message-65"><a href="#MessageProcessor.new_message-65"><span class="linenos">65</span></a> <span class="n">background_tasks</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_message</span><span class="p">,</span> <span class="n">db_count</span><span class="p">)</span>
</span><span id="MessageProcessor.new_message-64"><a href="#MessageProcessor.new_message-64"><span class="linenos">64</span></a> </span><span id="MessageProcessor.new_message-66"><a href="#MessageProcessor.new_message-66"><span class="linenos">66</span></a>
</span><span id="MessageProcessor.new_message-65"><a href="#MessageProcessor.new_message-65"><span class="linenos">65</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="MessageProcessor.new_message-67"><a href="#MessageProcessor.new_message-67"><span class="linenos">67</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="MessageProcessor.new_message-66"><a href="#MessageProcessor.new_message-66"><span class="linenos">66</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span> </span><span id="MessageProcessor.new_message-68"><a href="#MessageProcessor.new_message-68"><span class="linenos">68</span></a> <span class="n">count</span><span class="o">=</span><span class="n">db_count</span><span class="p">,</span>
</span><span id="MessageProcessor.new_message-67"><a href="#MessageProcessor.new_message-67"><span class="linenos">67</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span> </span><span id="MessageProcessor.new_message-69"><a href="#MessageProcessor.new_message-69"><span class="linenos">69</span></a> <span class="n">msg</span><span class="o">=</span><span class="s2">&quot;Added message to queue&quot;</span>
</span><span id="MessageProcessor.new_message-68"><a href="#MessageProcessor.new_message-68"><span class="linenos">68</span></a> <span class="p">)</span> </span><span id="MessageProcessor.new_message-70"><a href="#MessageProcessor.new_message-70"><span class="linenos">70</span></a> <span class="p">)</span>
</span><span id="MessageProcessor.new_message-69"><a href="#MessageProcessor.new_message-69"><span class="linenos">69</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> </span><span id="MessageProcessor.new_message-71"><a href="#MessageProcessor.new_message-71"><span class="linenos">71</span></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
</span><span id="MessageProcessor.new_message-70"><a href="#MessageProcessor.new_message-70"><span class="linenos">70</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span> </span><span id="MessageProcessor.new_message-72"><a href="#MessageProcessor.new_message-72"><span class="linenos">72</span></a> <span class="k">return</span> <span class="n">AgentResponse</span><span class="p">(</span>
</span><span id="MessageProcessor.new_message-71"><a href="#MessageProcessor.new_message-71"><span class="linenos">71</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> </span><span id="MessageProcessor.new_message-73"><a href="#MessageProcessor.new_message-73"><span class="linenos">73</span></a> <span class="n">count</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
</span><span id="MessageProcessor.new_message-72"><a href="#MessageProcessor.new_message-72"><span class="linenos">72</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> </span><span id="MessageProcessor.new_message-74"><a href="#MessageProcessor.new_message-74"><span class="linenos">74</span></a> <span class="n">error</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
</span><span id="MessageProcessor.new_message-73"><a href="#MessageProcessor.new_message-73"><span class="linenos">73</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> </span><span id="MessageProcessor.new_message-75"><a href="#MessageProcessor.new_message-75"><span class="linenos">75</span></a> <span class="n">error_msg</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
</span><span id="MessageProcessor.new_message-74"><a href="#MessageProcessor.new_message-74"><span class="linenos">74</span></a> <span class="p">)</span> </span><span id="MessageProcessor.new_message-76"><a href="#MessageProcessor.new_message-76"><span class="linenos">76</span></a> <span class="p">)</span>
</span></pre></div> </span></pre></div>

7
web/public/docs/ums/utils.html
View File

@ -77,7 +77,7 @@
</span><span id="L-11"><a href="#L-11"><span class="linenos">11</span></a><span class="kn">from</span> <span class="nn">ums.utils.const</span> <span class="kn">import</span> <span class="o">*</span> </span><span id="L-11"><a href="#L-11"><span class="linenos">11</span></a><span class="kn">from</span> <span class="nn">ums.utils.const</span> <span class="kn">import</span> <span class="o">*</span>
</span><span id="L-12"><a href="#L-12"><span class="linenos">12</span></a> </span><span id="L-12"><a href="#L-12"><span class="linenos">12</span></a>
</span><span id="L-13"><a href="#L-13"><span class="linenos">13</span></a><span class="kn">import</span> <span class="nn">logging</span><span class="o">,</span> <span class="nn">os</span> </span><span id="L-13"><a href="#L-13"><span class="linenos">13</span></a><span class="kn">import</span> <span class="nn">logging</span><span class="o">,</span> <span class="nn">os</span>
</span><span id="L-14"><a href="#L-14"><span class="linenos">14</span></a><span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span> </span><span id="L-14"><a href="#L-14"><span class="linenos">14</span></a><span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;SERVE&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span><span class="p">:</span>
</span><span id="L-15"><a href="#L-15"><span class="linenos">15</span></a> <span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span> </span><span id="L-15"><a href="#L-15"><span class="linenos">15</span></a> <span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span>
</span><span id="L-16"><a href="#L-16"><span class="linenos">16</span></a> <span class="n">handlers</span><span class="o">=</span><span class="p">[</span> </span><span id="L-16"><a href="#L-16"><span class="linenos">16</span></a> <span class="n">handlers</span><span class="o">=</span><span class="p">[</span>
</span><span id="L-17"><a href="#L-17"><span class="linenos">17</span></a> <span class="n">logging</span><span class="o">.</span><span class="n">FileHandler</span><span class="p">(</span><span class="n">LOG_FILE</span><span class="p">),</span> </span><span id="L-17"><a href="#L-17"><span class="linenos">17</span></a> <span class="n">logging</span><span class="o">.</span><span class="n">FileHandler</span><span class="p">(</span><span class="n">LOG_FILE</span><span class="p">),</span>
@ -107,8 +107,9 @@
</span><span id="L-41"><a href="#L-41"><span class="linenos">41</span></a> </span><span id="L-41"><a href="#L-41"><span class="linenos">41</span></a>
</span><span id="L-42"><a href="#L-42"><span class="linenos">42</span></a><span class="kn">from</span> <span class="nn">ums.utils.schema</span> <span class="kn">import</span> <span class="p">(</span> </span><span id="L-42"><a href="#L-42"><span class="linenos">42</span></a><span class="kn">from</span> <span class="nn">ums.utils.schema</span> <span class="kn">import</span> <span class="p">(</span>
</span><span id="L-43"><a href="#L-43"><span class="linenos">43</span></a> <span class="n">ExtractionSchema</span><span class="p">,</span> </span><span id="L-43"><a href="#L-43"><span class="linenos">43</span></a> <span class="n">ExtractionSchema</span><span class="p">,</span>
</span><span id="L-44"><a href="#L-44"><span class="linenos">44</span></a> <span class="n">ExtractedData</span> </span><span id="L-44"><a href="#L-44"><span class="linenos">44</span></a> <span class="n">ExtractedData</span><span class="p">,</span>
</span><span id="L-45"><a href="#L-45"><span class="linenos">45</span></a><span class="p">)</span> </span><span id="L-45"><a href="#L-45"><span class="linenos">45</span></a> <span class="n">ExtractedContent</span><span class="p">,</span> <span class="n">ExtractedPositions</span>
</span><span id="L-46"><a href="#L-46"><span class="linenos">46</span></a><span class="p">)</span>
</span></pre></div> </span></pre></div>

4
web/public/docs/ums/utils/types.html
View File

@ -949,7 +949,7 @@ is set to <code>'allow'</code>.</li>
</div> </div>
<div id="RiddleData.file_plain" class="classattr"> <div id="RiddleData.file_plain" class="classattr">
<div class="attr variable"> <div class="attr variable">
<span class="name">file_plain</span><span class="annotation">: Annotated[str, AfterValidator(func=&lt;function _check_data_file at 0x104aaa3e0&gt;), WrapValidator(func=&lt;function _ignore_file_missing at 0x104b5b100&gt;, json_schema_input_type=PydanticUndefined)]</span> <span class="name">file_plain</span><span class="annotation">: Annotated[str, AfterValidator(func=&lt;function _check_data_file at 0x102682480&gt;), WrapValidator(func=&lt;function _ignore_file_missing at 0x1027331a0&gt;, json_schema_input_type=PydanticUndefined)]</span>
</div> </div>
@ -965,7 +965,7 @@ The file must exist.</p>
</div> </div>
<div id="RiddleData.file_extracted" class="classattr"> <div id="RiddleData.file_extracted" class="classattr">
<div class="attr variable"> <div class="attr variable">
<span class="name">file_extracted</span><span class="annotation">: Optional[Annotated[str, AfterValidator(func=&lt;function _check_data_file at 0x104aaa3e0&gt;), WrapValidator(func=&lt;function _ignore_file_missing at 0x104b5b100&gt;, json_schema_input_type=PydanticUndefined)]]</span> <span class="name">file_extracted</span><span class="annotation">: Optional[Annotated[str, AfterValidator(func=&lt;function _check_data_file at 0x102682480&gt;), WrapValidator(func=&lt;function _ignore_file_missing at 0x1027331a0&gt;, json_schema_input_type=PydanticUndefined)]]</span>
</div> </div>