import math

import tiktoken

import configs.config as config

encoding = tiktoken.get_encoding("cl100k_base")
encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")


def check_token_limit(page_text, token_limit=config.OPENAI_TOKEN_LIMIT):
    """
    Checks the token limit compared to the config file
    """
    token_count = len(encoding.encode(page_text))
    token_counter = math.ceil(token_count / token_limit)
    token_counter = max(token_counter, 1)
    # split page text into token_counter number of pieces
    new_page_text = []
    page_text_split = int(len(page_text) / token_counter)
    if token_counter == 1:
        new_page_text.append(page_text)
    else:
        for i in range(0, token_counter):
            # array_end = (i + 1) * page_text_split
            # if (i + 1) * page_text_split > len(page_text) - 1:
            #     array_end = len(page_text) - 1

            new_page_text.append(
                page_text[i * page_text_split : (i + 1) * page_text_split]
            )

    return new_page_text