utils

some utility functions

get_webapp_dir

def get_webapp_dir():

Get the webapp directory path

source

get_models_dir

def get_models_dir():

Get the models directory path

source

get_datadir

def get_datadir(
    subfolder:NoneType=None
):

Get the data directory path, optionally with a subfolder. Creates the main data dir and any requested subfolder if they don’t exist.

Args: subfolder: Optional subfolder name to append to the data directory path

Returns: Path to the data directory or subfolder

source

download

def download(
    url, filename, verify:bool=False
):

Call self as a function.

source

df_to_md

def df_to_md(
    df, caption:NoneType=None
):

Converts pd.Dataframe to markdown

source

html_to_df

def html_to_df(
    html_str:str
)->Any:

Convert HTML to dataframe.

source

md_to_df

def md_to_df(
    md_str:str
)->Any:

Convert Markdown to dataframe.

source

extract_noun_phrases

def extract_noun_phrases(
    text:str
):

Extracts noun phrases from text, including coordinated phrases like “generative AI and live fire testing”, and removes subphrases like “AI” if “generative AI” is also found. Example: text = “Natural language processing (NLP) is a field of computer science, artificial intelligence,” “and computational linguistics concerned with the interactions between computers and human” “(natural) languages.” extract_noun_phrases(text) [‘Natural language processing’, ‘NLP’, ‘field’, ‘computer science’, ‘artificial intelligence’, ‘computational linguistics’, ‘interactions’, ‘computers’, ‘languages’, ‘human’]

source

contains_sentence

def contains_sentence(
    sentence, text
):

Returns True if sentence is contained in text ignoring whether tokens are delmited by spaces or newlines or tabs.

source

remove_sentence

def remove_sentence(
    sentence, text, remove_follow:bool=False, flags:RegexFlag=re.IGNORECASE
):

Removes a sentence or phrase from text ignoring whether tokens are delimited by spaces or newlines or tabs.

If remove_follow=True, then subsequent text until the first newline is also removed.

source

segment

def segment(
    text:str, unit:str='paragraph', maxchars:int=2048
):

Segments text into a list of paragraphs or sentences depending on value of unit (one of {'paragraph', 'sentence'}. The maxchars parameter is the maximum size of any unit of text.

source

filtered_generator

def filtered_generator(
    generator, criteria:list=[]
):

Filters a generator based on a given predicate function.

Args: generator: The generator to filter. criteria: List of functions that take an element from the generator and return True if the element should be included, False otherwise.

Yields: Elements from the original generator that satisfy the predicate.

source

batch_generator

def batch_generator(
    iterable, batch_size
):

Batched results from generator

source

batch_list

def batch_list(
    input_list, batch_size
):

Split list into chunks

source

get_template_vars

def get_template_vars(
    template_str:str
)->List:

Get template variables from a template string.

source

format_string

def format_string(
    string_to_format:str, **kwargs:str
)->str:

Format a string with kwargs

source

SafeFormatter

def SafeFormatter(
    format_dict:Optional=None
):

Safe string formatter that does not raise KeyError if key is missing. Adapted from llama_index.