@dataclass
class ProcessorInputs:
"""Represents the keyword arguments to
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
"""
prompt: list[int]
mm_data_items: MultiModalDataItems
mm_uuid_items: MultiModalUUIDItems | None = None
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
cache: BaseMultiModalProcessorCache | None = None
def fork_media_refs(self) -> "ProcessorInputs":
"""Give processing releasable wrappers around borrowed media refs."""
owned_items = MultiModalDataItems()
for modality, items in self.mm_data_items.items():
if isinstance(items, ProcessorBatchItems):
items = items.map_raw(
lambda item: item.fork() if isinstance(item, MediaRef) else item
)
owned_items[modality] = items
return replace(self, mm_data_items=owned_items)
@property
def can_use_cache(self) -> bool:
"""Whether these inputs can use the processor cache."""
return self.cache is not None and not any(
items.get_passthrough_data() for items in self.mm_data_items.values()
)
def get_mm_hashes(
self,
model_id: str,
hash_algorithm: MMHasherAlgorithm,
) -> MultiModalHashes:
mm_data_items = self.mm_data_items
mm_uuid_items = self.mm_uuid_items or {}
mm_hashes = dict[str, list[str]]()
hasher = MultiModalHasher
for modality, data_items in mm_data_items.items():
# Each cache entry represents one item from one modality. Keep
# modality-scoped options out of unrelated entries, while
# retaining flat processor kwargs because they are shared by HF.
if modality in _HF_MODALITY_PROCESSOR_KWARGS:
modality_media_io_kwargs = self.media_io_kwargs.get(modality, {})
media_io_kwargs = (
{modality: modality_media_io_kwargs}
if modality_media_io_kwargs
else {}
)
scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality]
mm_processor_kwargs = {
key: value
for key, value in self.hf_processor_mm_kwargs.items()
if key not in _HF_MODALITY_PROCESSOR_KWARGS.values()
or key == scoped_key
}
else:
# Unified modalities such as ``vision_chunk`` can contain
# items originating from multiple media modalities. There is
# no per-item origin metadata here, so preserve all factors.
media_io_kwargs = dict(self.media_io_kwargs)
mm_processor_kwargs = dict(self.hf_processor_mm_kwargs)
hash_factors = {
"media_io_kwargs": media_io_kwargs,
"mm_processor_kwargs": mm_processor_kwargs,
}
hash_factors = {key: value for key, value in hash_factors.items() if value}
uuid_items = (
mm_uuid_items[modality]
if modality in mm_uuid_items
else ([None] * len(data_items))
)
# For None entries, compute a hash; otherwise, use provided ID.
hashes: list[str] = []
for i, item in enumerate(data_items.get_all_raw()):
uuid_item = uuid_items[i]
# A MediaRef's decode spec is already folded into its key, so
# media_io_kwargs must not be counted twice for it. When the
# client supplies a UUID the key is not consulted at all, so
# every factor still has to be hashed in.
item_factors = hash_factors
if uuid_item is None and isinstance(item, MediaRef):
item_factors = {
key: value
for key, value in hash_factors.items()
if key != "media_io_kwargs"
}
# NOTE: Even if a uuid_item is provided, model output depends
# on the current modality's hash factors, so they are taken
# into account.
if uuid_item is None or item_factors:
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
item = uuid_item if uuid_item is not None else item
hashes.append(
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**item_factors,
)
)
else:
# If there are no extra kwargs, use the client-provided UUID.
hashes.append(uuid_item)
mm_hashes[modality] = hashes
return mm_hashes