Skip to content

vllm.benchmarks.lib.utils

InfEncoder

Bases: JSONEncoder

Source code in vllm/benchmarks/lib/utils.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
class InfEncoder(json.JSONEncoder):
    def clear_inf(self, o: Any):
        if isinstance(o, dict):
            return {
                str(k)
                if not isinstance(k, (str, int, float, bool, type(None)))
                else k: self.clear_inf(v)
                for k, v in o.items()
            }
        elif isinstance(o, list):
            return [self.clear_inf(v) for v in o]
        elif isinstance(o, float) and math.isinf(o):
            return "inf"
        return o

    def iterencode(self, o: Any, *args, **kwargs) -> Any:
        return super().iterencode(self.clear_inf(o), *args, **kwargs)

clear_inf 

clear_inf(o: Any)
Source code in vllm/benchmarks/lib/utils.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def clear_inf(self, o: Any):
    if isinstance(o, dict):
        return {
            str(k)
            if not isinstance(k, (str, int, float, bool, type(None)))
            else k: self.clear_inf(v)
            for k, v in o.items()
        }
    elif isinstance(o, list):
        return [self.clear_inf(v) for v in o]
    elif isinstance(o, float) and math.isinf(o):
        return "inf"
    return o

iterencode 

iterencode(o: Any, *args, **kwargs) -> Any
Source code in vllm/benchmarks/lib/utils.py 
102
103
def iterencode(self, o: Any, *args, **kwargs) -> Any:
    return super().iterencode(self.clear_inf(o), *args, **kwargs)

convert_to_pytorch_benchmark_format 

convert_to_pytorch_benchmark_format(
    args: Namespace,
    metrics: dict[str, list],
    extra_info: dict[str, Any],
) -> list

Save the benchmark results in the format used by PyTorch OSS benchmark with on metric per record https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database

Source code in vllm/benchmarks/lib/utils.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def convert_to_pytorch_benchmark_format(
    args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
) -> list:
    """
    Save the benchmark results in the format used by PyTorch OSS benchmark with
    on metric per record
    https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
    """
    records = []
    if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
        return records

    for name, benchmark_values in metrics.items():
        record = {
            "benchmark": {
                "name": "vLLM benchmark",
                "extra_info": {
                    "args": vars(args),
                    "compilation_config.mode": extract_field(
                        args, extra_info, "compilation_config.mode"
                    ),
                    "optimization_level": extract_field(
                        args, extra_info, "optimization_level"
                    ),
                    # A boolean field used by vLLM benchmark HUD dashboard
                    "use_compile": use_compile(args, extra_info),
                },
            },
            "model": {
                "name": args.model,
            },
            "metric": {
                "name": name,
                "benchmark_values": benchmark_values,
                "extra_info": extra_info,
            },
        }

        tp = record["benchmark"]["extra_info"]["args"].get("tensor_parallel_size")
        # Save tensor_parallel_size parameter if it's part of the metadata
        if not tp and "tensor_parallel_size" in extra_info:
            record["benchmark"]["extra_info"]["args"]["tensor_parallel_size"] = (
                extra_info["tensor_parallel_size"]
            )

        records.append(record)

    return records

extract_field 

extract_field(
    args: Namespace,
    extra_info: dict[str, Any],
    field_name: str,
) -> str
Source code in vllm/benchmarks/lib/utils.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
def extract_field(
    args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
) -> str:
    if field_name in extra_info:
        return extra_info[field_name]

    v = args
    # For example, args.compilation_config.mode
    for nested_field in field_name.split("."):
        if not hasattr(v, nested_field):
            return ""
        v = getattr(v, nested_field)
    return v

use_compile 

use_compile(
    args: Namespace, extra_info: dict[str, Any]
) -> bool

Check if the benchmark is run with torch.compile

Source code in vllm/benchmarks/lib/utils.py 
26
27
28
29
30
31
32
33
34
def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
    """
    Check if the benchmark is run with torch.compile
    """
    return not (
        extract_field(args, extra_info, "compilation_config.mode") == "0"
        or "eager" in getattr(args, "output_json", "")
        or "eager" in getattr(args, "result_filename", "")
    )

write_to_json 

write_to_json(filename: str, records: list) -> None
Source code in vllm/benchmarks/lib/utils.py 
106
107
108
109
110
111
112
113
def write_to_json(filename: str, records: list) -> None:
    with open(filename, "w") as f:
        json.dump(
            records,
            f,
            cls=InfEncoder,
            default=lambda o: f"<{type(o).__name__} is not JSON serializable>",
        )