Skip to content

model_calculations

dandy.calculator.calculations.llm.model_calculations

QuantizationBitSizes = Literal[64, 32, 16, 8, 6, 5, 4, 3, 2] module-attribute

hidden_dimension_and_number_of_layers_from_parameter_count

Source code in dandy/calculator/calculations/llm/model_calculations.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
def hidden_dimension_and_number_of_layers_from_parameter_count(
        parameter_count: int | float,
) -> tuple[int, int]:
    parameter_count_billions = parameter_count / 1e9

    HIDDEN_DIMENSION = (
        (671, 20480, 160),
        (405, 16384, 120),
        (120, 12288, 96),
        (65, 8192, 80),
        (30, 7168, 60),
        (13, 5120, 40),
        (7, 4096, 32),
        (3, 3072, 26),
        (1, 2048, 22),
        (0, 1024, 18),
    )

    for value in HIDDEN_DIMENSION:
        if parameter_count_billions >= value[0]:
            return value[1], value[2]

    raise ValueError("Invalid parameter count")

key_value_cache_per_token_size_bytes_calculation

Source code in dandy/calculator/calculations/llm/model_calculations.py 
33
34
35
36
37
38
def key_value_cache_per_token_size_bytes_calculation(
        hidden_dimension: int,
        quantization_size_bits: int,
        number_of_layers: int,
) -> float:
    return hidden_dimension * 2 * bits_to_bytes(quantization_size_bits) * number_of_layers

key_value_cache_size_bytes_calculation

Source code in dandy/calculator/calculations/llm/model_calculations.py 
41
42
43
44
45
46
def key_value_cache_size_bytes_calculation(
        batch_size: int,
        sequence_length: int,
        key_value_cache_per_token_size_bytes: int | float,
) -> float:
    return batch_size * sequence_length * key_value_cache_per_token_size_bytes

model_inference_activation_size_bytes_calculation

Source code in dandy/calculator/calculations/llm/model_calculations.py 
49
50
51
52
def model_inference_activation_size_bytes_calculation(
        model_size_bytes: int | float,
) -> float:
    return model_size_bytes * 0.2

model_size_bytes_calculation

Source code in dandy/calculator/calculations/llm/model_calculations.py 
55
56
57
58
59
def model_size_bytes_calculation(
        parameter_count: int | float,
        quantization_size_bits: QuantizationBitSizes
) -> int | float:
    return parameter_count * bits_to_bytes(quantization_size_bits)

model_training_activation_size_bytes_calculation

Source code in dandy/calculator/calculations/llm/model_calculations.py 
62
63
64
65
def model_training_activation_size_bytes_calculation(
        model_size_bytes: int | float,
) -> float:
    return model_size_bytes * 2.0