Skip to content

diligence

dandy.llm.diligence.stop_word_removal.diligence

StopWordRemovalDiligence

Bases: BaseDiligence

Source code in dandy/llm/diligence/diligence.py 
13
14
def __init__(self):
    self.is_activated = False

trigger_level = 0.2 class-attribute instance-attribute

trigger_operator = operator.le class-attribute instance-attribute

requires_new_llm_request = False class-attribute instance-attribute

apply classmethod

Source code in dandy/llm/diligence/stop_word_removal/diligence.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
@classmethod
def apply(cls, llm_connector: LlmConnector) -> None:
    for i in range(len(llm_connector.request_body.messages)):
        if isinstance(llm_connector.request_body.messages[i], list):
            for k in range(len(llm_connector.request_body.messages[i])):
                for j in range(len(llm_connector.request_body.messages[i][k].content)):
                    stripped_text = cls.remove_stop_words(llm_connector.request_body.messages[i][k].content[j].text)
                    llm_connector.request_body.messages[i][k].content[j].text = stripped_text
        else:
            for j in range(len(llm_connector.request_body.messages[i].content)):
                stripped_text = cls.remove_stop_words(llm_connector.request_body.messages[i].content[j].text)
                llm_connector.request_body.messages[i].content[j].text = stripped_text

    llm_connector.request_body.messages.add_message(
        role='system',
        text='Assume Stop Words',
        prepend=True,
    )

remove_stop_words staticmethod

Source code in dandy/llm/diligence/stop_word_removal/diligence.py 
38
39
40
41
42
43
44
45
@staticmethod
def remove_stop_words(text: str) -> str:
    stop_words = [re.escape(word) for word in STOP_WORDS]
    pattern = r'\b(' + '|'.join(stop_words) + r')\b'

    cleaned_text = re.sub(pattern, '', text.lower())

    return re.sub(r'\s+', ' ', cleaned_text).strip()