vllm.multimodal.processing

Modules:

• context –
• dummy_inputs –
• inputs –
• processor –

Classes:

• BaseDummyInputsBuilder –

  Abstract base class that constructs the dummy data to profile

• BaseMultiModalProcessor –

  Abstract base class to process multi-modal inputs to be used in vLLM.

• BaseProcessingInfo –

  Base class to provide the information necessary for data processing.

• EncDecMultiModalProcessor –
• InputProcessingContext –

  Contains information about the model which may be used to

• MultiModalApplyState –

  Intermediate state between apply_phase1 and apply_phase2.

• ProcessorInputs –

  Represents the keyword arguments to

• PromptIndexTargets –
• PromptInsertion –

  Defines how to insert placeholder tokens into a prompt.

• PromptReplacement –

  Defines how to replace portions of an input prompt with placeholder tokens.

• PromptUpdate –

  Defines how to update a prompt with placeholder tokens.

• PromptUpdateDetails –

  Details about the token sequence that is part of the update.

• TimingContext –

  Helper class to record execution times during multi-modal processing.

Functions:

• cached_encode –

  Encode text while caching repeated tokenizer calls.

BaseDummyInputsBuilder

Bases: ABC, Generic[_I]

Abstract base class that constructs the dummy data to profile multi-modal models.

Methods:

• get_dummy_mm_data –

  Build the multimodal input which, after processing, results in

• get_dummy_text –

  Build the text input corresponding to mm_counts.

Source code in vllm/multimodal/processing/dummy_inputs.py

class BaseDummyInputsBuilder(ABC, Generic[_I]):
    """Abstract base class that constructs the dummy data to profile
    multi-modal models.
    """

    def __init__(self, info: _I) -> None:
        super().__init__()

        self.info = info

    @abstractmethod
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        """Build the text input corresponding to `mm_counts`."""
        raise NotImplementedError

    @abstractmethod
    def get_dummy_mm_data(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
        mm_options: MultiModalDummyOptions,
    ) -> MultiModalDataDict:
        """Build the multimodal input which, after processing, results in
        the maximum possible number of placeholder tokens.

        Args:
            seq_len: Sequence length
            mm_counts: Count of items per modality
            mm_options: Configurable options per modality (optional).
                       If None, use model defaults for backward compatibility.
                       If provided, models can use these to customize dummy
                       data generation.

        """
        raise NotImplementedError

    def _get_dummy_audios(
        self,
        *,
        length: int,
        num_audios: int,
        overrides: AudioDummyOptions | None = None,
    ) -> list[npt.NDArray]:
        if num_audios == 0:
            return []
        if overrides and overrides.length:
            if overrides.length > length:
                logger.warning(
                    "audio.length override (%d) exceeds model's "
                    "maximum length (%d), will be ignored",
                    overrides.length,
                    length,
                )
            length = min(length, overrides.length)
        audio = np.zeros((length,))
        return [audio] * num_audios

    def _get_dummy_images(
        self,
        *,
        width: int,
        height: int,
        num_images: int,
        overrides: ImageDummyOptions | None = None,
    ) -> list[Image.Image]:
        if num_images == 0:
            return []
        if overrides:
            if overrides.width:
                if overrides.width > width:
                    logger.warning(
                        "image.width override (%d) exceeds model's "
                        "maximum width (%d), will be ignored",
                        overrides.width,
                        width,
                    )
                width = min(width, overrides.width)
            if overrides.height:
                if overrides.height > height:
                    logger.warning(
                        "image.height override (%d) exceeds model's "
                        "maximum height (%d), will be ignored",
                        overrides.height,
                        height,
                    )
                height = min(height, overrides.height)
        image = Image.new("RGB", (width, height), color=255)
        return [image] * num_images

    def _get_dummy_videos(
        self,
        *,
        width: int,
        height: int,
        num_frames: int,
        num_videos: int,
        overrides: VideoDummyOptions | None = None,
    ) -> list[npt.NDArray]:
        if num_videos == 0:
            return []
        if overrides:
            if overrides.num_frames:
                if overrides.num_frames > num_frames:
                    logger.warning(
                        "video.num_frames override (%d) exceeds model's "
                        "maximum number of frames (%d), will be ignored",
                        overrides.num_frames,
                        num_frames,
                    )
                num_frames = min(num_frames, overrides.num_frames)
            if overrides.width:
                if overrides.width > width:
                    logger.warning(
                        "video.width override (%d) exceeds model's "
                        "maximum width (%d), will be ignored",
                        overrides.width,
                        width,
                    )
                width = min(width, overrides.width)
            if overrides.height:
                if overrides.height > height:
                    logger.warning(
                        "video.height override (%d) exceeds model's "
                        "maximum height (%d), will be ignored",
                        overrides.height,
                        height,
                    )
                height = min(height, overrides.height)
        video = np.full((num_frames, height, width, 3), 255, dtype=np.uint8)
        return [video] * num_videos

get_dummy_mm_data(seq_len, mm_counts, mm_options) abstractmethod

Build the multimodal input which, after processing, results in the maximum possible number of placeholder tokens.

Parameters:

• seq_len

  (int) –

  Sequence length

• mm_counts

  (Mapping[str, int]) –

  Count of items per modality

• mm_options

  (MultiModalDummyOptions) –

  Configurable options per modality (optional). If None, use model defaults for backward compatibility. If provided, models can use these to customize dummy data generation.

Source code in vllm/multimodal/processing/dummy_inputs.py

@abstractmethod
def get_dummy_mm_data(
    self,
    seq_len: int,
    mm_counts: Mapping[str, int],
    mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
    """Build the multimodal input which, after processing, results in
    the maximum possible number of placeholder tokens.

    Args:
        seq_len: Sequence length
        mm_counts: Count of items per modality
        mm_options: Configurable options per modality (optional).
                   If None, use model defaults for backward compatibility.
                   If provided, models can use these to customize dummy
                   data generation.

    """
    raise NotImplementedError

get_dummy_text(mm_counts) abstractmethod

Build the text input corresponding to mm_counts.

Source code in vllm/multimodal/processing/dummy_inputs.py

@abstractmethod
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
    """Build the text input corresponding to `mm_counts`."""
    raise NotImplementedError

BaseMultiModalProcessor

Bases: ABC, Generic[_I]

Abstract base class to process multi-modal inputs to be used in vLLM.

Not to be confused with transformers.ProcessorMixin.

Methods:

• apply –

  Process multi-modal inputs to be used in vLLM.

• apply_phase1 –

  Phase 1 of apply(): hashing, cache lookup, and submission of

• apply_phase2 –

  Phase 2 of apply(): HF processing, cache merge, and prompt

• get_dummy_inputs –

  Build the input which, after processing, results in

• get_dummy_mm_inputs –

  Create dummy data for profiling the memory usage of a model.

Attributes:

• supports_two_phase_apply (bool) –

  Whether apply_phase1/apply_phase2 replicate apply().

Source code in vllm/multimodal/processing/processor.py

class BaseMultiModalProcessor(ABC, Generic[_I]):
    """Abstract base class to process multi-modal inputs to be used in vLLM.

    Not to be confused with `transformers.ProcessorMixin`.
    """

    requires_tokenizer: ClassVar[bool] = True

    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
    ) -> None:
        super().__init__()

        if self.requires_tokenizer and info.ctx.tokenizer is None:
            raise ValueError(
                f"{type(self).__name__} requires a tokenizer and cannot be "
                "initialized when `skip_tokenizer_init=True`. Disable "
                "`--skip-tokenizer-init`."
            )

        self.info = info
        self.dummy_inputs = dummy_inputs

        self.data_parser = self.info.get_data_parser()

    def get_dummy_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
        mm_options: MultiModalDummyOptions,
    ) -> ProcessorInputs:
        """Build the input which, after processing, results in
        the maximum possible number of placeholder tokens.

        Args:
            seq_len: Sequence length
            mm_counts: Count of items per modality
            mm_options: Configurable options per modality (optional)

        """
        builder = self.dummy_inputs
        dummy_text = builder.get_dummy_text(mm_counts)
        dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
        dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)

        tokenizer = self.info.ctx.tokenizer
        dummy_prompt: list[int]
        if tokenizer is None:
            # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
            # accept embeddings and have an empty dummy text, so there are no
            # prompt tokens.
            dummy_prompt = []
        else:
            dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)

        return ProcessorInputs(
            prompt=dummy_prompt,
            mm_data_items=dummy_mm_items,
        )

    def get_dummy_mm_inputs(
        self,
        mm_counts: Mapping[str, int],
        *,
        cache: BaseMultiModalProcessorCache | None = None,
        scheduler_config: "SchedulerConfig | None" = None,
    ) -> MultiModalInput:
        """Create dummy data for profiling the memory usage of a model."""
        model_config = self.info.ctx.model_config
        seq_len = model_config.max_model_len
        if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
            seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)

        mm_config = model_config.get_multimodal_config()

        processor_inputs = self.get_dummy_inputs(
            seq_len=seq_len,
            mm_counts=mm_counts,
            mm_options=mm_config.limit_per_prompt,
        )
        processor_inputs.cache = cache

        mm_inputs = self.apply(
            processor_inputs,
            timing_ctx=TimingContext(enabled=False),
        )

        prompt_token_ids = mm_inputs["prompt_token_ids"]
        total_len = len(prompt_token_ids)
        if total_len < seq_len:
            prompt_token_ids.extend([0] * (seq_len - total_len))

        return mm_inputs

    def __call__(
        self,
        prompt: str | list[int],
        mm_items: MultiModalDataItems,
        mm_uuid_items: MultiModalUUIDItems | None = None,
        hf_processor_mm_kwargs: Mapping[str, object] | None = None,
        cache: BaseMultiModalProcessorCache | None = None,
    ) -> MultiModalInput:
        if isinstance(prompt, str):
            tokenizer = self.info.get_tokenizer()
            prompt = tokenizer.encode(
                prompt,
                **self.info.default_tok_params.get_encode_kwargs(),
            )

        processor_inputs = ProcessorInputs(
            prompt,
            mm_items,
            mm_uuid_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
            cache=cache,
        )

        return self.apply(processor_inputs, TimingContext(enabled=False))

    @abstractmethod
    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

    @abstractmethod
    def _get_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> Sequence[PromptUpdate]:
        """Given the original multi-modal items for this modality
        and HF-processed data, output the updates to perform.

        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
        for each multi-modal item.
        """
        raise NotImplementedError

    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        return mm_prompt_updates

    def _find_mm_placeholders(
        self,
        new_token_ids: list[int],
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
        return find_mm_placeholders(new_token_ids, mm_prompt_updates)

    def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
        """Get the text to pass to the HF processor alongside the multi-modal data.

        By default, no text is passed. If the HF processor requires that
        text and multi-modal items correspond to each other, you should
        override this method to return dummy text generated by
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
        """
        return None

    def _get_hf_mm_inputs(
        self,
        mm_items: MultiModalDataItems,
        hf_kwargs: Mapping[str, object],
    ) -> HFMultiModalInputs:
        """Extract the multi-modal data and corresponding keyword arguments
        for HF processing.

        If the HF processor expects the multi-modal data under different keys
        than those provided by the multi-modal items, or requires additional
        keyword arguments, you should override this method.
        """
        hf_data = dict[str, object]()
        passthrough_data = dict[str, object]()

        for items in mm_items.values():
            if not items:
                continue

            hf_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())

        if hf_data:
            prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
            if prompt_text is not None:
                hf_data = dict(text=prompt_text, **hf_data)

                # vLLM needs the untruncated sequence to keep placeholder
                # tokens aligned. Note that the text inputs are just dummy
                # text, not the original prompt. The original prompt is
                # already tokenized by the renderer.
                hf_kwargs = {"truncation": False, **hf_kwargs}

        # HF processors accept (images, text, videos, audio)
        # so we need to remap vLLM's "audios" -> HF "audio"
        if "audios" in hf_data:
            hf_data["audio"] = hf_data.pop("audios")

        return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)

    def _postprocess_hf_mm_data(
        self,
        mm_inputs: Mapping[str, object],
        hf_processor_mm_kwargs: Mapping[str, object],
        processed_data: BatchFeature,
    ) -> BatchFeature:
        """Post-process the combined processor and passthrough data.

        By default, the output is returned as-is. If you need to modify the
        output of the HF processor before it is converted into multi-modal
        keyword arguments, you should override this method.
        """
        return processed_data

    def _call_hf_processor(
        self,
        hf_data: Mapping[str, object],
        hf_kwargs: Mapping[str, object],
    ) -> BatchFeature:
        """Call the HF processor on the normalized multi-modal inputs."""
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**hf_kwargs),
            hf_data,
            hf_kwargs,
        )

    def _finalize_hf_mm_data(
        self,
        hf_data: Mapping[str, object],
        hf_kwargs: Mapping[str, object],
        passthrough_data: Mapping[str, object],
        processed_data: BatchFeature | None = None,
    ) -> BatchFeature:
        """Merge passthrough fields and apply model-specific postprocessing."""
        if processed_data is None:
            from transformers.feature_extraction_utils import BatchFeature

            processed_data = BatchFeature()

        processed_data.update(passthrough_data)
        return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)

    def _apply_hf_processor_main(
        self,
        mm_items: MultiModalDataItems,
        hf_kwargs: Mapping[str, object],
    ) -> BatchFeature:
        """Apply the HF processor on the multi-modal data."""
        hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
            mm_items, hf_kwargs
        )

        if hf_data:
            processed_data = self._call_hf_processor(hf_data, hf_kwargs)
        else:
            processed_data = None

        return self._finalize_hf_mm_data(
            hf_data, hf_kwargs, passthrough_data, processed_data
        )

    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
        """Post-process the prompt token IDs before locating or applying
        multi-modal prompt updates.

        By default, the prompt is returned as-is. If the HF processor (or
        chat template) applies additional transformations to the prompt
        that are not reflected in its multi-modal outputs, you should
        override this method to replicate them.
        """
        return prompt

    def _get_cache_missing_items(
        self,
        cache: BaseMultiModalProcessorCache,
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
        mm_is_cached = {
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
        }

        mm_missing_idxs = {
            modality: [
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }

        mm_missing_items = MultiModalDataItems()
        for modality, idxs in mm_missing_idxs.items():
            if not idxs:
                continue

            items = mm_data_items[modality]
            for idx in idxs:
                data = items.get_raw(idx)
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
                        f"but data is not provided."
                    )
            mm_missing_items[modality] = items.select(idxs)

        return mm_is_cached, mm_missing_items

    def _decode_ref_items(
        self,
        mm_data_items: MultiModalDataItems,
        decodes: list[MediaDecodeJob],
    ) -> None:
        """Decode all undecoded media refs in parallel on the media thread
        pool.

        Uses no instance state; wraps the first decode failure as
        VLLMUnprocessableEntityError so corrupt media surfaces as a client
        error (422) instead of a server error.
        """
        start = len(decodes)
        _submit_ref_decodes(mm_data_items, decodes)
        collect_media_decodes(decodes[start:], _decode_error)

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

    def _merge_mm_kwargs(
        self,
        cache: BaseMultiModalProcessorCache,
        mm_hashes: MultiModalHashes,
        mm_is_cached: MultiModalIsCached,
        mm_missing_kwargs: MultiModalKwargsItems,
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)

        mm_missing_next_idx = defaultdict[str, int](lambda: 0)

        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]

                    mm_missing_next_idx[modality] += 1

                    item = missing_kwargs_item, missing_updates_item
                else:
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )

        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)

        return mm_kwargs, mm_prompt_updates

    def _apply_hf_processor(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalProcessingResult:
        """The no-cache/passthrough path: decode everything, then process."""
        state = self._apply_hf_processor_phase1(inputs, timing_ctx, use_cache=False)
        with state:
            state.wait_decodes()
            return self._apply_hf_processor_phase2(state)

    def _apply_hf_processor_phase1(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
        *,
        use_cache: bool,
    ) -> MultiModalApplyState:
        """Phase 1: hashing, cache lookup, and submission of media decodes.

        Runs on the single mm worker and never blocks on decoding, so the
        worker stays free for other requests while the media thread pool
        decodes. The caller must drain `state.decodes` (off the mm worker)
        before calling `_apply_hf_processor_phase2`.
        """
        inputs = inputs.fork_media_refs()
        state = MultiModalApplyState(inputs, timing_ctx, None, [])
        try:
            if not use_cache:
                with timing_ctx.record("decode_mm_items"):
                    _submit_ref_decodes(inputs.mm_data_items, state.decodes)
                return state

            cache = inputs.cache
            assert cache is not None

            with timing_ctx.record("get_mm_hashes"):
                mm_hashes = inputs.get_mm_hashes(
                    self.info.model_id,
                    self.info.ctx.get_mm_config().mm_hasher_algorithm,
                )

            with timing_ctx.record("get_cache_missing_items"):
                _, mm_missing_data_items = self._get_cache_missing_items(
                    cache=cache,
                    mm_data_items=inputs.mm_data_items,
                    mm_hashes=mm_hashes,
                )

            with timing_ctx.record("decode_mm_items"):
                # Submit miss-item decodes without joining; the join happens off
                # the mm worker (see MultiModalApplyState.wait_decodes*). The
                # lookup is re-done in phase 2, so a stale miss set here only
                # wastes a redundant decode. Nothing is released here: a hit can
                # still be evicted before phase 2, which then has to reprocess it
                # from the very bytes a release would drop.
                _submit_ref_decodes(mm_missing_data_items, state.decodes)

            state.mm_hashes = mm_hashes
            return state
        except BaseException:
            state.close()
            raise

    def _apply_hf_processor_phase2(
        self,
        state: MultiModalApplyState,
    ) -> MultiModalProcessingResult:
        """Phase 2: HF processing and cache merge; runs on the mm worker
        after the state's decode futures have completed."""
        if state.mm_hashes is None:
            return self._apply_hf_processor_no_cache(state.inputs, state.timing_ctx)
        return self._cached_apply_hf_processor_phase2(state)

    def _apply_hf_processor_no_cache(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalProcessingResult:
        # Hashes are computed after the HF processor ran on this path; that is
        # safe because a ref's key was derived when it was built.
        with timing_ctx.record("apply_hf_processor"):
            mm_processed_data = self._apply_hf_processor_main(
                mm_items=inputs.mm_data_items,
                hf_kwargs=inputs.hf_processor_mm_kwargs,
            )

        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
            mm_processed_data,
            self._get_mm_fields_config(
                mm_processed_data, inputs.hf_processor_mm_kwargs
            ),
        )

        # Use overrides if provided; fallback to data-dependent hashing.
        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(
                self.info.model_id,
                self.info.ctx.get_mm_config().mm_hasher_algorithm,
            )

        mm_prompt_updates = self._get_mm_prompt_updates(
            inputs.mm_data_items,
            inputs.hf_processor_mm_kwargs,
            mm_kwargs,
        )

        return MultiModalProcessingResult(
            prompt_ids=self._postprocess_prompt(inputs.prompt),
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

    def _cached_apply_hf_processor(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalProcessingResult:
        """Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.

        Synchronous composition of phase 1 + decode join + phase 2.
        """
        if not inputs.can_use_cache:
            return self._apply_hf_processor(inputs, timing_ctx)
        state = self._apply_hf_processor_phase1(
            inputs, timing_ctx, use_cache=inputs.can_use_cache
        )
        with state:
            state.wait_decodes()
            return self._apply_hf_processor_phase2(state)

    def _cached_apply_hf_processor_phase2(
        self,
        state: MultiModalApplyState,
    ) -> MultiModalProcessingResult:
        inputs = state.inputs
        timing_ctx = state.timing_ctx
        mm_hashes = state.mm_hashes
        assert mm_hashes is not None

        cache = inputs.cache
        assert cache is not None

        # Re-derive the cache state instead of reusing phase 1's: between
        # the phases, other requests' phases may have inserted this
        # request's miss items (miss->hit flip, saving the HF call below)
        # or evicted its hit items (hit->miss flip, handled by the blocking
        # decode fallback and the normal miss handling below).
        with timing_ctx.record("get_cache_missing_items"):
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=inputs.mm_data_items,
                mm_hashes=mm_hashes,
            )

        with timing_ctx.record("decode_mm_items"):
            # Normally a no-op: phase-1 decodes were drained before phase 2.
            # Only items that flipped hit->miss (evicted between the phases)
            # decode here, from the bytes phase 1 deliberately kept.
            self._decode_ref_items(mm_missing_data_items, state.decodes)

        # NOTE: The prompt does not correspond to `mm_missing_data_items`,
        # so we can't apply prompt updates until the new multimodal
        # items are combined with the cached multimodal items
        with timing_ctx.record("apply_hf_processor"):
            mm_missing_processed_data = self._apply_hf_processor_main(
                mm_items=mm_missing_data_items,
                hf_kwargs=inputs.hf_processor_mm_kwargs,
            )

        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
            mm_missing_processed_data,
            self._get_mm_fields_config(
                mm_missing_processed_data, inputs.hf_processor_mm_kwargs
            ),
        )

        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            inputs.hf_processor_mm_kwargs,
            mm_missing_kwargs,
        )

        with timing_ctx.record("merge_mm_kwargs"):
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )

        return MultiModalProcessingResult(
            prompt_ids=self._postprocess_prompt(inputs.prompt),
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

    def _apply_token_matches(
        self,
        prompt: list[int],
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        return apply_token_matches(prompt, mm_prompt_updates)

    def _apply_token_matches_with_placeholders(
        self,
        token_ids: list[int],
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[
        list[int],
        MultiModalPromptUpdatesApplyResult,
        Mapping[str, list[PlaceholderFeaturesInfo]],
    ]:
        return _apply_token_matches_with_placeholders(
            token_ids,
            mm_prompt_updates,
        )

    def _iter_text_matches(
        self,
        update: ResolvedPromptUpdate,
        prompt_text: str,
        tokenizer: TokenizerLike,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch, None, None]:
        """Yield each instance of `update.target` found in `prompt_text`."""
        target = update.target

        if isinstance(target, PromptIndex):
            match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)

            return

        target_text = tokenizer.decode(target)
        if not target_text:
            return

        idx = prompt_text.find(target_text, start_idx)
        while idx != -1:
            end_idx = idx + len(target_text)
            yield PromptTargetMatch(idx, end_idx)
            idx = prompt_text.find(target_text, end_idx)

    def _apply_prompt_updates_via_text(
        self,
        token_ids: list[int],
        mm_prompt_updates: MultiModalPromptUpdates,
        *,
        encode_segments_separately: bool = False,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        """Apply multi-modal prompt updates in text space, then encode the
        result.

        A non-special-token target may be tokenized differently inside the
        prompt because the tokens may go across the boundaries of the target
        (e.g. when searching for "foo" in "food", if "food" itself makes up
        a token, then the token ID of "foo" will not appear at all). Since
        it is inefficient to search for all possible tokenizations of the
        target in the prompt, we instead perform text-based updates on the
        decoded token IDs, then encode them back.
        """
        tokenizer = self.info.get_tokenizer()
        prompt_text = tokenizer.decode(token_ids)

        matched_updates, result = _plan_prompt_updates_with(
            mm_prompt_updates,
            lambda update, start_idx: self._iter_text_matches(
                update, prompt_text, tokenizer, start_idx=start_idx
            ),
        )

        out_texts = list[str]()
        prev_end_idx = 0
        for matched_update in matched_updates:
            update = matched_update.update
            match = matched_update.match

            if update.mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif update.mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(update.mode)

            out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
            out_texts.append(tokenizer.decode(update.content.full))
            prev_end_idx = match.end_idx

        out_texts.append(prompt_text[prev_end_idx:])

        if encode_segments_separately:
            # Encode each segment separately instead of joining into one
            # text and encoding it in a single pass. Joining first would
            # let BPE merge tokens across a segment boundary, silently
            # changing how a non-special-token placeholder is tokenized.
            new_token_ids = flatten_2d_lists(
                [tokenizer.encode(text, add_special_tokens=False) for text in out_texts]
            )
        else:
            new_token_ids = tokenizer.encode(
                "".join(out_texts), add_special_tokens=False
            )

        return new_token_ids, result

    def _matched_updates_from_result(
        self,
        mm_prompt_updates: MultiModalPromptUpdates,
        match_result: MultiModalPromptUpdatesApplyResult,
    ) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
                    f"mm_items[{modality!r}][{item_idx}]"
                )

                matched_updates[modality].append(
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )

        return dict(matched_updates)

    def _apply_prompt_updates(
        self,
        token_ids: list[int],
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
        """Apply multi-modal prompt updates to token IDs."""
        new_token_ids, match_result, placeholders = (
            self._apply_token_matches_with_placeholders(
                token_ids,
                mm_prompt_updates,
            )
        )

        if all(
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
            placeholders = {
                modality: modality_placeholders
                for modality, modality_placeholders in placeholders.items()
                if modality_placeholders
            }
            return new_token_ids, placeholders

        # A non-special-token target may be tokenized differently inside
        # the prompt, so fall back to performing the updates on the
        # decoded text, then encoding the result back.
        new_token_ids, match_result = self._apply_prompt_updates_via_text(
            token_ids,
            mm_prompt_updates,
        )

        placeholders = self._find_mm_placeholders(
            new_token_ids,
            self._matched_updates_from_result(mm_prompt_updates, match_result),
        )

        return new_token_ids, placeholders

    def _validate_mm_kwargs(
        self,
        mm_kwargs: MultiModalKwargsOptionalItems,
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            items = mm_kwargs.get(modality, [])

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_apply_hf_processor_main` and `_get_mm_fields_config`)."
                )

    def _validate_mm_updates(
        self,
        mm_updates: MultiModalPromptUpdates,
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_updates.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt updates "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt updates! "
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
                    "sure you have applied it before calling `LLM.generate`."
                )

    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_apply_hf_processor_main` "
                    "and `_get_mm_fields_config` are consistent with each other."
                )

    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        mm_res: MultiModalProcessingResult,
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
        mm_item_counts = mm_items.get_all_counts()
        self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
        self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)

        prompt_ids, mm_placeholders = self._apply_prompt_updates(
            mm_res.prompt_ids,
            mm_res.prompt_updates,
        )

        self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

        return prompt_ids, mm_placeholders

    @property
    def supports_two_phase_apply(self) -> bool:
        """Whether `apply_phase1`/`apply_phase2` replicate `apply()`.

        Only stock implementations are phase-safe: model-specific overrides
        of `apply()` / `_cached_apply_hf_processor()` /
        `_apply_hf_processor()` (e.g. paligemma's trailing-newline fixup or
        dithering cache bypasses) wrap the blocking composition, so the
        renderer keeps those processors on the blocking path.
        """
        cls = type(self)
        overrides = [
            name
            for name in ("apply", "_cached_apply_hf_processor", "_apply_hf_processor")
            if getattr(cls, name) is not getattr(BaseMultiModalProcessor, name)
        ]
        if overrides:
            logger.debug_once(
                "%s uses blocking multimodal processing because it overrides %s",
                cls.__qualname__,
                ", ".join(overrides),
            )
        return not overrides

    def apply_phase1(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalApplyState:
        """Phase 1 of `apply()`: hashing, cache lookup, and submission of
        lazy decodes without joining them.

        Runs on the single mm worker; the returned state's decode futures
        must be drained (off the mm worker, via `wait_decodes` /
        `wait_decodes_async`) before `apply_phase2`. Use the state as a
        context manager to drain and release it on completion or failure.
        """
        return self._apply_hf_processor_phase1(
            inputs, timing_ctx, use_cache=inputs.can_use_cache
        )

    def apply_phase2(self, state: MultiModalApplyState) -> MultiModalInput:
        """Phase 2 of `apply()`: HF processing, cache merge, and prompt
        updates; runs on the mm worker after the state's decodes completed.
        """
        mm_res = self._apply_hf_processor_phase2(state)
        return self._build_mm_input(state.inputs, state.timing_ctx, mm_res)

    def _build_mm_input(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
        mm_res: MultiModalProcessingResult,
    ) -> MultiModalInput:
        with timing_ctx.record("apply_prompt_updates"):
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
                mm_items=inputs.mm_data_items,
                mm_res=mm_res,
            )

        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }

        return mm_input(
            prompt_token_ids=prompt_ids,
            mm_kwargs=mm_res.kwargs,
            mm_hashes=mm_res.hashes,
            mm_placeholders=mm_placeholder_ranges,
        )

    def apply(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalInput:
        """Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Perform MM-specific post-processing to the token inputs.
        2. Apply HF Processor on multi-modal data together with optional
           dummy text, outputting processed multi-modal data tensors.
        3. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        4. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
        return self._build_mm_input(inputs, timing_ctx, mm_res)

supports_two_phase_apply property

Whether apply_phase1/apply_phase2 replicate apply().

Only stock implementations are phase-safe: model-specific overrides of apply() / _cached_apply_hf_processor() / _apply_hf_processor() (e.g. paligemma's trailing-newline fixup or dithering cache bypasses) wrap the blocking composition, so the renderer keeps those processors on the blocking path.

_apply_hf_processor(inputs, timing_ctx)

The no-cache/passthrough path: decode everything, then process.

Source code in vllm/multimodal/processing/processor.py

def _apply_hf_processor(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
    """The no-cache/passthrough path: decode everything, then process."""
    state = self._apply_hf_processor_phase1(inputs, timing_ctx, use_cache=False)
    with state:
        state.wait_decodes()
        return self._apply_hf_processor_phase2(state)

_apply_hf_processor_main(mm_items, hf_kwargs)

Apply the HF processor on the multi-modal data.

Source code in vllm/multimodal/processing/processor.py

def _apply_hf_processor_main(
    self,
    mm_items: MultiModalDataItems,
    hf_kwargs: Mapping[str, object],
) -> BatchFeature:
    """Apply the HF processor on the multi-modal data."""
    hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
        mm_items, hf_kwargs
    )

    if hf_data:
        processed_data = self._call_hf_processor(hf_data, hf_kwargs)
    else:
        processed_data = None

    return self._finalize_hf_mm_data(
        hf_data, hf_kwargs, passthrough_data, processed_data
    )

_apply_hf_processor_phase1(inputs, timing_ctx, *, use_cache)

Phase 1: hashing, cache lookup, and submission of media decodes.

Runs on the single mm worker and never blocks on decoding, so the worker stays free for other requests while the media thread pool decodes. The caller must drain state.decodes (off the mm worker) before calling _apply_hf_processor_phase2.

Source code in vllm/multimodal/processing/processor.py

def _apply_hf_processor_phase1(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
    *,
    use_cache: bool,
) -> MultiModalApplyState:
    """Phase 1: hashing, cache lookup, and submission of media decodes.

    Runs on the single mm worker and never blocks on decoding, so the
    worker stays free for other requests while the media thread pool
    decodes. The caller must drain `state.decodes` (off the mm worker)
    before calling `_apply_hf_processor_phase2`.
    """
    inputs = inputs.fork_media_refs()
    state = MultiModalApplyState(inputs, timing_ctx, None, [])
    try:
        if not use_cache:
            with timing_ctx.record("decode_mm_items"):
                _submit_ref_decodes(inputs.mm_data_items, state.decodes)
            return state

        cache = inputs.cache
        assert cache is not None

        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(
                self.info.model_id,
                self.info.ctx.get_mm_config().mm_hasher_algorithm,
            )

        with timing_ctx.record("get_cache_missing_items"):
            _, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=inputs.mm_data_items,
                mm_hashes=mm_hashes,
            )

        with timing_ctx.record("decode_mm_items"):
            # Submit miss-item decodes without joining; the join happens off
            # the mm worker (see MultiModalApplyState.wait_decodes*). The
            # lookup is re-done in phase 2, so a stale miss set here only
            # wastes a redundant decode. Nothing is released here: a hit can
            # still be evicted before phase 2, which then has to reprocess it
            # from the very bytes a release would drop.
            _submit_ref_decodes(mm_missing_data_items, state.decodes)

        state.mm_hashes = mm_hashes
        return state
    except BaseException:
        state.close()
        raise

_apply_hf_processor_phase2(state)

Phase 2: HF processing and cache merge; runs on the mm worker after the state's decode futures have completed.

Source code in vllm/multimodal/processing/processor.py

def _apply_hf_processor_phase2(
    self,
    state: MultiModalApplyState,
) -> MultiModalProcessingResult:
    """Phase 2: HF processing and cache merge; runs on the mm worker
    after the state's decode futures have completed."""
    if state.mm_hashes is None:
        return self._apply_hf_processor_no_cache(state.inputs, state.timing_ctx)
    return self._cached_apply_hf_processor_phase2(state)

_apply_prompt_updates(token_ids, mm_prompt_updates)

Apply multi-modal prompt updates to token IDs.

Source code in vllm/multimodal/processing/processor.py

def _apply_prompt_updates(
    self,
    token_ids: list[int],
    mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
    """Apply multi-modal prompt updates to token IDs."""
    new_token_ids, match_result, placeholders = (
        self._apply_token_matches_with_placeholders(
            token_ids,
            mm_prompt_updates,
        )
    )

    if all(
        all(update_idx is not None for update_idx in update_idxs)
        for update_idxs in match_result.values()
    ):
        placeholders = {
            modality: modality_placeholders
            for modality, modality_placeholders in placeholders.items()
            if modality_placeholders
        }
        return new_token_ids, placeholders

    # A non-special-token target may be tokenized differently inside
    # the prompt, so fall back to performing the updates on the
    # decoded text, then encoding the result back.
    new_token_ids, match_result = self._apply_prompt_updates_via_text(
        token_ids,
        mm_prompt_updates,
    )

    placeholders = self._find_mm_placeholders(
        new_token_ids,
        self._matched_updates_from_result(mm_prompt_updates, match_result),
    )

    return new_token_ids, placeholders

_apply_prompt_updates_via_text(token_ids, mm_prompt_updates, *, encode_segments_separately=False)

Apply multi-modal prompt updates in text space, then encode the result.

A non-special-token target may be tokenized differently inside the prompt because the tokens may go across the boundaries of the target (e.g. when searching for "foo" in "food", if "food" itself makes up a token, then the token ID of "foo" will not appear at all). Since it is inefficient to search for all possible tokenizations of the target in the prompt, we instead perform text-based updates on the decoded token IDs, then encode them back.

Source code in vllm/multimodal/processing/processor.py

def _apply_prompt_updates_via_text(
    self,
    token_ids: list[int],
    mm_prompt_updates: MultiModalPromptUpdates,
    *,
    encode_segments_separately: bool = False,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
    """Apply multi-modal prompt updates in text space, then encode the
    result.

    A non-special-token target may be tokenized differently inside the
    prompt because the tokens may go across the boundaries of the target
    (e.g. when searching for "foo" in "food", if "food" itself makes up
    a token, then the token ID of "foo" will not appear at all). Since
    it is inefficient to search for all possible tokenizations of the
    target in the prompt, we instead perform text-based updates on the
    decoded token IDs, then encode them back.
    """
    tokenizer = self.info.get_tokenizer()
    prompt_text = tokenizer.decode(token_ids)

    matched_updates, result = _plan_prompt_updates_with(
        mm_prompt_updates,
        lambda update, start_idx: self._iter_text_matches(
            update, prompt_text, tokenizer, start_idx=start_idx
        ),
    )

    out_texts = list[str]()
    prev_end_idx = 0
    for matched_update in matched_updates:
        update = matched_update.update
        match = matched_update.match

        if update.mode == UpdateMode.INSERT:
            end_idx_to_insert = match.end_idx
        elif update.mode == UpdateMode.REPLACE:
            end_idx_to_insert = match.start_idx
        else:
            assert_never(update.mode)

        out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
        out_texts.append(tokenizer.decode(update.content.full))
        prev_end_idx = match.end_idx

    out_texts.append(prompt_text[prev_end_idx:])

    if encode_segments_separately:
        # Encode each segment separately instead of joining into one
        # text and encoding it in a single pass. Joining first would
        # let BPE merge tokens across a segment boundary, silently
        # changing how a non-special-token placeholder is tokenized.
        new_token_ids = flatten_2d_lists(
            [tokenizer.encode(text, add_special_tokens=False) for text in out_texts]
        )
    else:
        new_token_ids = tokenizer.encode(
            "".join(out_texts), add_special_tokens=False
        )

    return new_token_ids, result

_cached_apply_hf_processor(inputs, timing_ctx)

Apply the HF processor on the full prompt text, caching the results and reusing cached results.

Synchronous composition of phase 1 + decode join + phase 2.

Source code in vllm/multimodal/processing/processor.py

def _cached_apply_hf_processor(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
) -> MultiModalProcessingResult:
    """Apply the HF processor on the full prompt text,
    caching the results and reusing cached results.

    Synchronous composition of phase 1 + decode join + phase 2.
    """
    if not inputs.can_use_cache:
        return self._apply_hf_processor(inputs, timing_ctx)
    state = self._apply_hf_processor_phase1(
        inputs, timing_ctx, use_cache=inputs.can_use_cache
    )
    with state:
        state.wait_decodes()
        return self._apply_hf_processor_phase2(state)

_call_hf_processor(hf_data, hf_kwargs)

Call the HF processor on the normalized multi-modal inputs.

Source code in vllm/multimodal/processing/processor.py

def _call_hf_processor(
    self,
    hf_data: Mapping[str, object],
    hf_kwargs: Mapping[str, object],
) -> BatchFeature:
    """Call the HF processor on the normalized multi-modal inputs."""
    return self.info.ctx.call_hf_processor(
        self.info.get_hf_processor(**hf_kwargs),
        hf_data,
        hf_kwargs,
    )

_decode_ref_items(mm_data_items, decodes)

Decode all undecoded media refs in parallel on the media thread pool.

Uses no instance state; wraps the first decode failure as VLLMUnprocessableEntityError so corrupt media surfaces as a client error (422) instead of a server error.

Source code in vllm/multimodal/processing/processor.py

def _decode_ref_items(
    self,
    mm_data_items: MultiModalDataItems,
    decodes: list[MediaDecodeJob],
) -> None:
    """Decode all undecoded media refs in parallel on the media thread
    pool.

    Uses no instance state; wraps the first decode failure as
    VLLMUnprocessableEntityError so corrupt media surfaces as a client
    error (422) instead of a server error.
    """
    start = len(decodes)
    _submit_ref_decodes(mm_data_items, decodes)
    collect_media_decodes(decodes[start:], _decode_error)

_finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data, processed_data=None)

Merge passthrough fields and apply model-specific postprocessing.

Source code in vllm/multimodal/processing/processor.py

def _finalize_hf_mm_data(
    self,
    hf_data: Mapping[str, object],
    hf_kwargs: Mapping[str, object],
    passthrough_data: Mapping[str, object],
    processed_data: BatchFeature | None = None,
) -> BatchFeature:
    """Merge passthrough fields and apply model-specific postprocessing."""
    if processed_data is None:
        from transformers.feature_extraction_utils import BatchFeature

        processed_data = BatchFeature()

    processed_data.update(passthrough_data)
    return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)

_get_hf_mm_inputs(mm_items, hf_kwargs)

Extract the multi-modal data and corresponding keyword arguments for HF processing.

If the HF processor expects the multi-modal data under different keys than those provided by the multi-modal items, or requires additional keyword arguments, you should override this method.

Source code in vllm/multimodal/processing/processor.py

def _get_hf_mm_inputs(
    self,
    mm_items: MultiModalDataItems,
    hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
    """Extract the multi-modal data and corresponding keyword arguments
    for HF processing.

    If the HF processor expects the multi-modal data under different keys
    than those provided by the multi-modal items, or requires additional
    keyword arguments, you should override this method.
    """
    hf_data = dict[str, object]()
    passthrough_data = dict[str, object]()

    for items in mm_items.values():
        if not items:
            continue

        hf_data.update(items.get_processor_data())
        passthrough_data.update(items.get_passthrough_data())

    if hf_data:
        prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
        if prompt_text is not None:
            hf_data = dict(text=prompt_text, **hf_data)

            # vLLM needs the untruncated sequence to keep placeholder
            # tokens aligned. Note that the text inputs are just dummy
            # text, not the original prompt. The original prompt is
            # already tokenized by the renderer.
            hf_kwargs = {"truncation": False, **hf_kwargs}

    # HF processors accept (images, text, videos, audio)
    # so we need to remap vLLM's "audios" -> HF "audio"
    if "audios" in hf_data:
        hf_data["audio"] = hf_data.pop("audios")

    return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)

_get_hf_mm_text(mm_counts)

Get the text to pass to the HF processor alongside the multi-modal data.

By default, no text is passed. If the HF processor requires that text and multi-modal items correspond to each other, you should override this method to return dummy text generated by DummyInputsBuilder.

Source code in vllm/multimodal/processing/processor.py

def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
    """Get the text to pass to the HF processor alongside the multi-modal data.

    By default, no text is passed. If the HF processor requires that
    text and multi-modal items correspond to each other, you should
    override this method to return dummy text generated by
    [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
    """
    return None

_get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs) abstractmethod

Given the HF-processed data, output the metadata of each field.

Source code in vllm/multimodal/processing/processor.py

@abstractmethod
def _get_mm_fields_config(
    self,
    hf_inputs: BatchFeature,
    hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
    """Given the HF-processed data, output the metadata of each field."""
    raise NotImplementedError

_get_prompt_updates(mm_items, hf_processor_mm_kwargs, out_mm_kwargs) abstractmethod

Given the original multi-modal items for this modality and HF-processed data, output the updates to perform.

The information returned by this method is used to update token inputs which bypass the HF processor. It is also used to update the output of HF processor if the HF process does not apply prompt updates to text inputs.

Moreover, this information is critical to determine the token positions in order to construct PlaceholderRange for each multi-modal item.

Source code in vllm/multimodal/processing/processor.py

@abstractmethod
def _get_prompt_updates(
    self,
    mm_items: MultiModalDataItems,
    hf_processor_mm_kwargs: Mapping[str, object],
    out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
    """Given the original multi-modal items for this modality
    and HF-processed data, output the updates to perform.

    The information returned by this method is used to update token inputs
    which bypass the HF processor. It is also used to update the output of
    HF processor if the HF process does not apply prompt updates to text
    inputs.

    Moreover, this information is critical to determine the token positions
    in order to construct
    [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
    for each multi-modal item.
    """
    raise NotImplementedError

_iter_text_matches(update, prompt_text, tokenizer, *, start_idx=0)

Yield each instance of update.target found in prompt_text.

Source code in vllm/multimodal/processing/processor.py

def _iter_text_matches(
    self,
    update: ResolvedPromptUpdate,
    prompt_text: str,
    tokenizer: TokenizerLike,
    *,
    start_idx: int = 0,
) -> Generator[PromptTargetMatch, None, None]:
    """Yield each instance of `update.target` found in `prompt_text`."""
    target = update.target

    if isinstance(target, PromptIndex):
        match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
        if match_idx is not None:
            yield PromptTargetMatch(match_idx, match_idx)

        return

    target_text = tokenizer.decode(target)
    if not target_text:
        return

    idx = prompt_text.find(target_text, start_idx)
    while idx != -1:
        end_idx = idx + len(target_text)
        yield PromptTargetMatch(idx, end_idx)
        idx = prompt_text.find(target_text, end_idx)

_postprocess_hf_mm_data(mm_inputs, hf_processor_mm_kwargs, processed_data)

Post-process the combined processor and passthrough data.

By default, the output is returned as-is. If you need to modify the output of the HF processor before it is converted into multi-modal keyword arguments, you should override this method.

Source code in vllm/multimodal/processing/processor.py

def _postprocess_hf_mm_data(
    self,
    mm_inputs: Mapping[str, object],
    hf_processor_mm_kwargs: Mapping[str, object],
    processed_data: BatchFeature,
) -> BatchFeature:
    """Post-process the combined processor and passthrough data.

    By default, the output is returned as-is. If you need to modify the
    output of the HF processor before it is converted into multi-modal
    keyword arguments, you should override this method.
    """
    return processed_data

_postprocess_prompt(prompt)

Post-process the prompt token IDs before locating or applying multi-modal prompt updates.

By default, the prompt is returned as-is. If the HF processor (or chat template) applies additional transformations to the prompt that are not reflected in its multi-modal outputs, you should override this method to replicate them.

Source code in vllm/multimodal/processing/processor.py

def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
    """Post-process the prompt token IDs before locating or applying
    multi-modal prompt updates.

    By default, the prompt is returned as-is. If the HF processor (or
    chat template) applies additional transformations to the prompt
    that are not reflected in its multi-modal outputs, you should
    override this method to replicate them.
    """
    return prompt

_recompute_cached_prompt_update(cached_update, new_item_idx)

Override this if other attributes of ResolvedPromptUpdate also need to be recomputed after retrieving from the cache.

Source code in vllm/multimodal/processing/processor.py

def _recompute_cached_prompt_update(
    self,
    cached_update: ResolvedPromptUpdate,
    new_item_idx: int,
) -> ResolvedPromptUpdate:
    """Override this if other attributes of `ResolvedPromptUpdate`
    also need to be recomputed after retrieving from the cache.
    """
    return replace(cached_update, item_idx=new_item_idx)

apply(inputs, timing_ctx)

Process multi-modal inputs to be used in vLLM.

The main steps are:

1. Perform MM-specific post-processing to the token inputs.
2. Apply HF Processor on multi-modal data together with optional dummy text, outputting processed multi-modal data tensors.
3. Find and update sequences in the token IDs with placeholder tokens. The number of placeholder tokens equals the feature size of the multi-modal data outputted by the multi-modal encoder.
4. Extract information about the placeholder tokens from the processed token IDs.

Source code in vllm/multimodal/processing/processor.py

def apply(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
) -> MultiModalInput:
    """Process multi-modal inputs to be used in vLLM.

    The main steps are:

    1. Perform MM-specific post-processing to the token inputs.
    2. Apply HF Processor on multi-modal data together with optional
       dummy text, outputting processed multi-modal data tensors.
    3. Find and update sequences in the token IDs with placeholder tokens.
       The number of placeholder tokens equals the feature size of the
       multi-modal data outputted by the multi-modal encoder.
    4. Extract information about the placeholder tokens from the
       processed token IDs.
    """
    mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
    return self._build_mm_input(inputs, timing_ctx, mm_res)

apply_phase1(inputs, timing_ctx)

Phase 1 of apply(): hashing, cache lookup, and submission of lazy decodes without joining them.

Runs on the single mm worker; the returned state's decode futures must be drained (off the mm worker, via wait_decodes / wait_decodes_async) before apply_phase2. Use the state as a context manager to drain and release it on completion or failure.

Source code in vllm/multimodal/processing/processor.py

def apply_phase1(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
) -> MultiModalApplyState:
    """Phase 1 of `apply()`: hashing, cache lookup, and submission of
    lazy decodes without joining them.

    Runs on the single mm worker; the returned state's decode futures
    must be drained (off the mm worker, via `wait_decodes` /
    `wait_decodes_async`) before `apply_phase2`. Use the state as a
    context manager to drain and release it on completion or failure.
    """
    return self._apply_hf_processor_phase1(
        inputs, timing_ctx, use_cache=inputs.can_use_cache
    )

apply_phase2(state)

Phase 2 of apply(): HF processing, cache merge, and prompt updates; runs on the mm worker after the state's decodes completed.

Source code in vllm/multimodal/processing/processor.py

def apply_phase2(self, state: MultiModalApplyState) -> MultiModalInput:
    """Phase 2 of `apply()`: HF processing, cache merge, and prompt
    updates; runs on the mm worker after the state's decodes completed.
    """
    mm_res = self._apply_hf_processor_phase2(state)
    return self._build_mm_input(state.inputs, state.timing_ctx, mm_res)

get_dummy_inputs(seq_len, mm_counts, mm_options)

Build the input which, after processing, results in the maximum possible number of placeholder tokens.

Parameters:

• seq_len

  (int) –

  Sequence length

• mm_counts

  (Mapping[str, int]) –

  Count of items per modality

• mm_options

  (MultiModalDummyOptions) –

  Configurable options per modality (optional)

Source code in vllm/multimodal/processing/processor.py

def get_dummy_inputs(
    self,
    seq_len: int,
    mm_counts: Mapping[str, int],
    mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
    """Build the input which, after processing, results in
    the maximum possible number of placeholder tokens.

    Args:
        seq_len: Sequence length
        mm_counts: Count of items per modality
        mm_options: Configurable options per modality (optional)

    """
    builder = self.dummy_inputs
    dummy_text = builder.get_dummy_text(mm_counts)
    dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
    dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)

    tokenizer = self.info.ctx.tokenizer
    dummy_prompt: list[int]
    if tokenizer is None:
        # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
        # accept embeddings and have an empty dummy text, so there are no
        # prompt tokens.
        dummy_prompt = []
    else:
        dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)

    return ProcessorInputs(
        prompt=dummy_prompt,
        mm_data_items=dummy_mm_items,
    )

get_dummy_mm_inputs(mm_counts, *, cache=None, scheduler_config=None)

Create dummy data for profiling the memory usage of a model.

Source code in vllm/multimodal/processing/processor.py

def get_dummy_mm_inputs(
    self,
    mm_counts: Mapping[str, int],
    *,
    cache: BaseMultiModalProcessorCache | None = None,
    scheduler_config: "SchedulerConfig | None" = None,
) -> MultiModalInput:
    """Create dummy data for profiling the memory usage of a model."""
    model_config = self.info.ctx.model_config
    seq_len = model_config.max_model_len
    if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
        seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)

    mm_config = model_config.get_multimodal_config()

    processor_inputs = self.get_dummy_inputs(
        seq_len=seq_len,
        mm_counts=mm_counts,
        mm_options=mm_config.limit_per_prompt,
    )
    processor_inputs.cache = cache

    mm_inputs = self.apply(
        processor_inputs,
        timing_ctx=TimingContext(enabled=False),
    )

    prompt_token_ids = mm_inputs["prompt_token_ids"]
    total_len = len(prompt_token_ids)
    if total_len < seq_len:
        prompt_token_ids.extend([0] * (seq_len - total_len))

    return mm_inputs

BaseProcessingInfo

Base class to provide the information necessary for data processing.

Methods:

• get_data_parser –

  Constructs a parser to preprocess multi-modal data items

• get_default_tok_params –

  Construct the default parameters for tokenization.

• get_hf_processor –

  Subclasses can override this method to handle

• get_media_io –

  Resolve the decoder for one modality's fetched media.

• get_mm_max_tokens_per_item –

  Return the maximum number of tokens per item of for each modality.

• get_supported_mm_limits –

  Return the maximum supported number of items for each modality.

• get_supported_mm_processor_kwargs –

  Return supported kwarg names for each HF processor kwargs scope.

• parse_mm_data –

  Normalize MultiModalDataDict

• validate_num_items –

  Raise ValueError if the number of input items for the given modality

Attributes:

• allow_missing_mm_embeddings (bool) –

  Whether pre-computed embedding tensors may be omitted.

• allowed_mm_limits (Mapping[str, int]) –

  The maximum allowed number of items for each modality.

• supported_mm_limits (Mapping[str, int | None]) –

  The maximum supported number of items for each modality.

• supported_mm_processor_kwargs (dict[str, set[str]]) –

  Supported kwarg names for each HF processor kwargs scope.

Source code in vllm/multimodal/processing/context.py

class BaseProcessingInfo:
    """Base class to provide the information necessary for data processing."""

    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()

        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> TokenizerLike:
        return self.ctx.get_tokenizer()

    def get_hf_config(self) -> PreTrainedConfig:
        return self.ctx.get_hf_config()

    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
        """Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

    def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
        """Return supported kwarg names for each HF processor kwargs scope."""
        processor = self.get_hf_processor()
        processor_kwargs = get_type_hints(processor.valid_processor_kwargs)

        supported = {
            "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__),
        }
        if "image" in self.supported_mm_limits:
            supported["images_kwargs"] = set(
                processor.image_processor.valid_kwargs.__annotations__
            )
        if "video" in self.supported_mm_limits:
            supported["videos_kwargs"] = set(
                processor.video_processor.valid_kwargs.__annotations__
            )
        if "audio" in self.supported_mm_limits:
            supported["audio_kwargs"] = set(
                processor_kwargs["audio_kwargs"].__annotations__
            )

        return supported

    @cached_property
    def supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
        """Supported kwarg names for each HF processor kwargs scope."""
        return self.get_supported_mm_processor_kwargs()

    def _merge_and_resolve_mm_processor_kwargs(
        self,
        mm_kwargs: Mapping[str, object],
    ) -> dict[str, Any]:
        """Merge configured and request ``mm_processor_kwargs``.

        Flat kwargs are routed into the HuggingFace processor kwarg scopes that
        support them after the configured/request merge. When a routed flat value
        conflicts with an existing scoped value, the scoped value takes precedence.
        """
        return self.ctx.get_merged_mm_kwargs(
            mm_kwargs,
            supported_mm_processor_kwargs=self.supported_mm_processor_kwargs,
        )

    def get_default_tok_params(self) -> TokenizeParams:
        """Construct the default parameters for tokenization."""
        from vllm.renderers import TokenizeParams

        model_config = self.ctx.model_config
        encoder_config = model_config.encoder_config or {}

        return TokenizeParams(
            max_total_tokens=model_config.max_model_len,
            do_lower_case=encoder_config.get("do_lower_case", False),
            add_special_tokens=True,
        )

    @cached_property
    def default_tok_params(self) -> TokenizeParams:
        return self.get_default_tok_params()

    def _get_expected_hidden_size(self) -> int | None:
        """Get expected hidden size for embedding validation if `mm_embeds` are enabled.

        This validates hidden dimensions to prevent a vulnerability where embeddings
        with correct `ndim` but wrong `shape` could cause crashes at inference time.
        """
        model_config = self.ctx.model_config
        mm_config = model_config.get_multimodal_config()

        if mm_config.enable_mm_embeds:
            return model_config.get_inputs_embeds_size()

        return None

    @property
    def allow_missing_mm_embeddings(self) -> bool:
        """Whether pre-computed embedding tensors may be omitted."""
        mm_config = self.ctx.model_config.multimodal_config
        return mm_config is not None and mm_config.allow_missing_mm_embeddings

    def get_data_parser(self) -> MultiModalDataParser:
        """Constructs a parser to preprocess multi-modal data items
        before passing them to
        [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].

        You can support additional modalities by creating a subclass
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
        """
        return MultiModalDataParser(
            expected_hidden_size=self._get_expected_hidden_size(),
            allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
        )

    @cached_property
    def data_parser(self) -> MultiModalDataParser:
        return self.get_data_parser()

    def get_media_io(
        self,
        modality: str,
        media_io_kwargs: Mapping[str, Mapping[str, Any]] | None = None,
    ) -> MediaIO[Any]:
        """Resolve the decoder for one modality's fetched media.

        Media fetching is transport-only: the connector hands back the encoded
        bytes and pairs them with whatever decoder the processor supplies.
        Resolution lives here because this is the layer that has the model
        config, so a decode choice that depends on the model stays a
        model-side decision instead of being threaded up through the request
        parser.

        Args:
            modality: The modality to decode (`"image"`, `"video"`, `"audio"`).
            media_io_kwargs: Per-modality media I/O options, already merged
                from `--media-io-kwargs` and the request's `media_io_kwargs`.

        Returns:
            The [`MediaIO`][vllm.multimodal.media.base.MediaIO] that decodes
            this modality's bytes. Its `get_decode_spec()` is the spec folded
            into the fetched item's cache key.

        """
        all_kwargs = media_io_kwargs or {}

        if modality == "image":
            return ImageMediaIO(**dict(all_kwargs.get("image") or {}))
        if modality == "audio":
            return AudioMediaIO(**dict(all_kwargs.get("audio") or {}))
        if modality == "video":
            kwargs = dict(all_kwargs.get("video") or {})
            # Backends that reproduce a specific HF video processor's frame
            # sampling are bound to it in `VIDEO_LOADER_REGISTRY`; an explicit
            # `video_backend` from `--media-io-kwargs` or the request wins.
            if "video_backend" not in kwargs:
                backend = get_video_loader_backend_for_processor(
                    get_video_processor_cls_name(self.ctx.model_config)
                )
                if backend is not None:
                    kwargs["video_backend"] = backend
            image_io = ImageMediaIO(**dict(all_kwargs.get("image") or {}))
            return VideoMediaIO(image_io, **kwargs)

        raise ValueError(f"Unsupported media modality: {modality!r}")

    @property
    def skip_prompt_length_check(self) -> bool:
        return False

    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
        """Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

    @cached_property
    def supported_mm_limits(self) -> Mapping[str, int | None]:
        """The maximum supported number of items for each modality."""
        return self.get_supported_mm_limits()

    @cached_property
    def allowed_mm_limits(self) -> Mapping[str, int]:
        """The maximum allowed number of items for each modality."""
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in self.supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

            allowed_limits[modality] = (
                user_limit
                if supported_limit is None
                else min(user_limit, supported_limit)
            )

        return allowed_limits

    def validate_num_items(self, modality: str, num_items: int) -> None:
        """Raise `ValueError` if the number of input items for the given modality
        is invalid.
        """
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise VLLMValidationError(msg, parameter=modality)

    def parse_mm_data(
        self,
        mm_data: MultiModalDataDict,
        *,
        validate: bool = True,
    ) -> MultiModalDataItems:
        """Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
        """
        mm_items = self.data_parser.parse_mm_data(mm_data)

        if validate:
            mm_config = self.ctx.get_mm_config()

            for modality, items in mm_items.items():
                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                    if not mm_config.enable_mm_embeds:
                        raise ValueError(
                            f"You must set `--enable-mm-embeds` to input "
                            f"`{modality}_embeds`"
                        )
                    if mm_config.get_limit_per_prompt(modality) == 0:
                        logger.debug(
                            "Skipping count validation for modality "
                            "'%s' (embeddings with limit=0)",
                            modality,
                        )
                        continue
                self.validate_num_items(modality, len(items))

        return mm_items

    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int] | None:
        """Return the maximum number of tokens per item of for each modality.

        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

        Note:
            The maximum number of tokens per item of each modality returned
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.

        """
        return None

allow_missing_mm_embeddings property

Whether pre-computed embedding tensors may be omitted.

allowed_mm_limits cached property

The maximum allowed number of items for each modality.

supported_mm_limits cached property

The maximum supported number of items for each modality.

supported_mm_processor_kwargs cached property

Supported kwarg names for each HF processor kwargs scope.

_get_expected_hidden_size()

Get expected hidden size for embedding validation if mm_embeds are enabled.

This validates hidden dimensions to prevent a vulnerability where embeddings with correct ndim but wrong shape could cause crashes at inference time.

Source code in vllm/multimodal/processing/context.py

def _get_expected_hidden_size(self) -> int | None:
    """Get expected hidden size for embedding validation if `mm_embeds` are enabled.

    This validates hidden dimensions to prevent a vulnerability where embeddings
    with correct `ndim` but wrong `shape` could cause crashes at inference time.
    """
    model_config = self.ctx.model_config
    mm_config = model_config.get_multimodal_config()

    if mm_config.enable_mm_embeds:
        return model_config.get_inputs_embeds_size()

    return None

_merge_and_resolve_mm_processor_kwargs(mm_kwargs)

Merge configured and request mm_processor_kwargs.

Flat kwargs are routed into the HuggingFace processor kwarg scopes that support them after the configured/request merge. When a routed flat value conflicts with an existing scoped value, the scoped value takes precedence.

Source code in vllm/multimodal/processing/context.py

def _merge_and_resolve_mm_processor_kwargs(
    self,
    mm_kwargs: Mapping[str, object],
) -> dict[str, Any]:
    """Merge configured and request ``mm_processor_kwargs``.

    Flat kwargs are routed into the HuggingFace processor kwarg scopes that
    support them after the configured/request merge. When a routed flat value
    conflicts with an existing scoped value, the scoped value takes precedence.
    """
    return self.ctx.get_merged_mm_kwargs(
        mm_kwargs,
        supported_mm_processor_kwargs=self.supported_mm_processor_kwargs,
    )

get_data_parser()

Constructs a parser to preprocess multi-modal data items before passing them to _get_hf_mm_inputs.

You can support additional modalities by creating a subclass of MultiModalDataParser that has additional subparsers.

Source code in vllm/multimodal/processing/context.py

def get_data_parser(self) -> MultiModalDataParser:
    """Constructs a parser to preprocess multi-modal data items
    before passing them to
    [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].

    You can support additional modalities by creating a subclass
    of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
    that has additional subparsers.
    """
    return MultiModalDataParser(
        expected_hidden_size=self._get_expected_hidden_size(),
        allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
    )

get_default_tok_params()

Construct the default parameters for tokenization.

Source code in vllm/multimodal/processing/context.py

def get_default_tok_params(self) -> TokenizeParams:
    """Construct the default parameters for tokenization."""
    from vllm.renderers import TokenizeParams

    model_config = self.ctx.model_config
    encoder_config = model_config.encoder_config or {}

    return TokenizeParams(
        max_total_tokens=model_config.max_model_len,
        do_lower_case=encoder_config.get("do_lower_case", False),
        add_special_tokens=True,
    )

get_hf_processor(**kwargs)

Subclasses can override this method to handle specific kwargs from model config or user inputs.

Source code in vllm/multimodal/processing/context.py

def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
    """Subclasses can override this method to handle
    specific kwargs from model config or user inputs.
    """
    return self.ctx.get_hf_processor(**kwargs)

get_media_io(modality, media_io_kwargs=None)

Resolve the decoder for one modality's fetched media.

Media fetching is transport-only: the connector hands back the encoded bytes and pairs them with whatever decoder the processor supplies. Resolution lives here because this is the layer that has the model config, so a decode choice that depends on the model stays a model-side decision instead of being threaded up through the request parser.

Parameters:

• modality

  (str) –

  The modality to decode ("image", "video", "audio").

• media_io_kwargs

  (Mapping[str, Mapping[str, Any]] | None, default: None ) –

  Per-modality media I/O options, already merged from --media-io-kwargs and the request's media_io_kwargs.

Returns:

• MediaIO[Any] –

  The MediaIO that decodes

• MediaIO[Any] –

  this modality's bytes. Its get_decode_spec() is the spec folded

• MediaIO[Any] –

  into the fetched item's cache key.

Source code in vllm/multimodal/processing/context.py

def get_media_io(
    self,
    modality: str,
    media_io_kwargs: Mapping[str, Mapping[str, Any]] | None = None,
) -> MediaIO[Any]:
    """Resolve the decoder for one modality's fetched media.

    Media fetching is transport-only: the connector hands back the encoded
    bytes and pairs them with whatever decoder the processor supplies.
    Resolution lives here because this is the layer that has the model
    config, so a decode choice that depends on the model stays a
    model-side decision instead of being threaded up through the request
    parser.

    Args:
        modality: The modality to decode (`"image"`, `"video"`, `"audio"`).
        media_io_kwargs: Per-modality media I/O options, already merged
            from `--media-io-kwargs` and the request's `media_io_kwargs`.

    Returns:
        The [`MediaIO`][vllm.multimodal.media.base.MediaIO] that decodes
        this modality's bytes. Its `get_decode_spec()` is the spec folded
        into the fetched item's cache key.

    """
    all_kwargs = media_io_kwargs or {}

    if modality == "image":
        return ImageMediaIO(**dict(all_kwargs.get("image") or {}))
    if modality == "audio":
        return AudioMediaIO(**dict(all_kwargs.get("audio") or {}))
    if modality == "video":
        kwargs = dict(all_kwargs.get("video") or {})
        # Backends that reproduce a specific HF video processor's frame
        # sampling are bound to it in `VIDEO_LOADER_REGISTRY`; an explicit
        # `video_backend` from `--media-io-kwargs` or the request wins.
        if "video_backend" not in kwargs:
            backend = get_video_loader_backend_for_processor(
                get_video_processor_cls_name(self.ctx.model_config)
            )
            if backend is not None:
                kwargs["video_backend"] = backend
        image_io = ImageMediaIO(**dict(all_kwargs.get("image") or {}))
        return VideoMediaIO(image_io, **kwargs)

    raise ValueError(f"Unsupported media modality: {modality!r}")

get_mm_max_tokens_per_item(seq_len, mm_counts)

Return the maximum number of tokens per item of for each modality.

When None (the default) is returned, vLLM will generate dummy inputs (images/videos) at maximum possible sizes and process them to determine the maximum token count per modality.

This approach works but can be very slow for certain models (e.g., Qwen2.5-VL), leading to very long startup time. For better performance, each model can override this method to return pre-computed maximum token counts, avoiding the need for dummy input generation and processing.

Note

The maximum number of tokens per item of each modality returned from this function should respect the model's maximum sequence length and the maximum number of items of each modality allowed, and agree with dummy inputs (images/videos) at maximum possible sizes.

Source code in vllm/multimodal/processing/context.py

def get_mm_max_tokens_per_item(
    self,
    seq_len: int,
    mm_counts: Mapping[str, int],
) -> Mapping[str, int] | None:
    """Return the maximum number of tokens per item of for each modality.

    When `None` (the default) is returned, vLLM will generate dummy inputs
    (images/videos) at maximum possible sizes and process them to determine
    the maximum token count per modality.

    This approach works but can be very slow for certain models (e.g.,
    Qwen2.5-VL), leading to very long startup time. For better performance,
    each model can override this method to return pre-computed maximum token
    counts, avoiding the need for dummy input generation and processing.

    Note:
        The maximum number of tokens per item of each modality returned
        from this function should respect the model's maximum sequence
        length and the maximum number of items of each modality allowed,
        and agree with dummy inputs (images/videos) at maximum possible
        sizes.

    """
    return None

get_supported_mm_limits()

Return the maximum supported number of items for each modality.

A value of None means unlimited number of items.

Omitting a modality from the returned dictionary means that it is not supported at all.

Source code in vllm/multimodal/processing/context.py

def get_supported_mm_limits(self) -> Mapping[str, int | None]:
    """Return the maximum supported number of items for each modality.

    A value of `None` means unlimited number of items.

    Omitting a modality from the returned dictionary means that
    it is not supported at all.
    """
    raise NotImplementedError

get_supported_mm_processor_kwargs()

Return supported kwarg names for each HF processor kwargs scope.

Source code in vllm/multimodal/processing/context.py

def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
    """Return supported kwarg names for each HF processor kwargs scope."""
    processor = self.get_hf_processor()
    processor_kwargs = get_type_hints(processor.valid_processor_kwargs)

    supported = {
        "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__),
    }
    if "image" in self.supported_mm_limits:
        supported["images_kwargs"] = set(
            processor.image_processor.valid_kwargs.__annotations__
        )
    if "video" in self.supported_mm_limits:
        supported["videos_kwargs"] = set(
            processor.video_processor.valid_kwargs.__annotations__
        )
    if "audio" in self.supported_mm_limits:
        supported["audio_kwargs"] = set(
            processor_kwargs["audio_kwargs"].__annotations__
        )

    return supported

parse_mm_data(mm_data, *, validate=True)

Normalize MultiModalDataDict to MultiModalDataItems before passing them to _get_hf_mm_inputs.

Source code in vllm/multimodal/processing/context.py

def parse_mm_data(
    self,
    mm_data: MultiModalDataDict,
    *,
    validate: bool = True,
) -> MultiModalDataItems:
    """Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
    to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
    before passing them to
    [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
    """
    mm_items = self.data_parser.parse_mm_data(mm_data)

    if validate:
        mm_config = self.ctx.get_mm_config()

        for modality, items in mm_items.items():
            if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                if not mm_config.enable_mm_embeds:
                    raise ValueError(
                        f"You must set `--enable-mm-embeds` to input "
                        f"`{modality}_embeds`"
                    )
                if mm_config.get_limit_per_prompt(modality) == 0:
                    logger.debug(
                        "Skipping count validation for modality "
                        "'%s' (embeddings with limit=0)",
                        modality,
                    )
                    continue
            self.validate_num_items(modality, len(items))

    return mm_items

validate_num_items(modality, num_items)

Raise ValueError if the number of input items for the given modality is invalid.

Source code in vllm/multimodal/processing/context.py

def validate_num_items(self, modality: str, num_items: int) -> None:
    """Raise `ValueError` if the number of input items for the given modality
    is invalid.
    """
    supported_limit = self.supported_mm_limits.get(modality, 0)
    allowed_limit = self.allowed_mm_limits.get(modality, 0)

    if supported_limit is None:
        supported_limit = allowed_limit

    limit = min(supported_limit, allowed_limit)

    if num_items > limit:
        msg = f"At most {limit} {modality}(s) may be provided in one prompt."

        if num_items <= supported_limit:
            msg += " Set `--limit-mm-per-prompt` to increase this limit."

        raise VLLMValidationError(msg, parameter=modality)

EncDecMultiModalProcessor

Bases: BaseMultiModalProcessor[_I]

Methods:

• apply –

  Process multi-modal inputs to be used in vLLM.

• create_decoder_prompt –

  Create input prompt for the decoder.

• create_encoder_prompt –

  Create input prompt for the encoder. HF processor will be applied on

Source code in vllm/multimodal/processing/processor.py

class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    skip_decoder_start_token: bool = False

    @abstractmethod
    def create_encoder_prompt(
        self,
        prompt: list[int],
        mm_items: MultiModalDataItems,
    ) -> list[int]:
        """Create input prompt for the encoder. HF processor will be applied on
        this prompt during profiling and generation.
        """
        raise NotImplementedError

    def create_decoder_prompt(
        self,
        prompt: str | list[int],
        mm_items: MultiModalDataItems,
    ) -> str | list[int]:
        """Create input prompt for the decoder."""
        return prompt

    def _get_enc_dec_inputs(
        self,
        prompt: list[int],
        mm_items: MultiModalDataItems,
        encoder_inputs: MultiModalInput,
    ):
        tokenizer = self.info.get_tokenizer()
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
        if isinstance(decoder_prompt_raw, str):
            decoder_prompt_text = decoder_prompt_raw
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
            )
        else:
            decoder_prompt_text = None
            decoder_prompt_ids = decoder_prompt_raw

        return mm_enc_dec_input(
            encoder_inputs,
            decoder_prompt_ids,
            decoder_prompt=decoder_prompt_text,
        )

    def apply(
        self,
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
    ) -> MultiModalEncDecInput:
        """Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(
            inputs.prompt,
            inputs.mm_data_items,
        )
        encoder_processor_inputs = ProcessorInputs(
            encoder_prompt,
            inputs.mm_data_items,
            inputs.mm_uuid_items,
            hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
            media_io_kwargs=inputs.media_io_kwargs,
            cache=inputs.cache,
        )

        encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)

        return self._get_enc_dec_inputs(
            prompt=inputs.prompt,
            mm_items=inputs.mm_data_items,
            encoder_inputs=encoder_inputs,
        )

apply(inputs, timing_ctx)

Process multi-modal inputs to be used in vLLM. The main processing steps are modified to fit encoder-decoder model: 1. Create encoder prompt from input prompt text. 2. Apply the HF processor on encoder prompt. 3. Copy the input prompt text as decoder prompt inputs.

Source code in vllm/multimodal/processing/processor.py

def apply(
    self,
    inputs: ProcessorInputs,
    timing_ctx: TimingContext,
) -> MultiModalEncDecInput:
    """Process multi-modal inputs to be used in vLLM.
    The main processing steps are modified to fit encoder-decoder model:
    1. Create encoder prompt from input prompt text.
    2. Apply the HF processor on encoder prompt.
    3. Copy the input prompt text as decoder prompt inputs.
    """
    encoder_prompt = self.create_encoder_prompt(
        inputs.prompt,
        inputs.mm_data_items,
    )
    encoder_processor_inputs = ProcessorInputs(
        encoder_prompt,
        inputs.mm_data_items,
        inputs.mm_uuid_items,
        hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
        media_io_kwargs=inputs.media_io_kwargs,
        cache=inputs.cache,
    )

    encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)

    return self._get_enc_dec_inputs(
        prompt=inputs.prompt,
        mm_items=inputs.mm_data_items,
        encoder_inputs=encoder_inputs,
    )

create_decoder_prompt(prompt, mm_items)

Create input prompt for the decoder.

Source code in vllm/multimodal/processing/processor.py

def create_decoder_prompt(
    self,
    prompt: str | list[int],
    mm_items: MultiModalDataItems,
) -> str | list[int]:
    """Create input prompt for the decoder."""
    return prompt

create_encoder_prompt(prompt, mm_items) abstractmethod

Create input prompt for the encoder. HF processor will be applied on this prompt during profiling and generation.

Source code in vllm/multimodal/processing/processor.py

@abstractmethod
def create_encoder_prompt(
    self,
    prompt: list[int],
    mm_items: MultiModalDataItems,
) -> list[int]:
    """Create input prompt for the encoder. HF processor will be applied on
    this prompt during profiling and generation.
    """
    raise NotImplementedError

InputProcessingContext dataclass

Contains information about the model which may be used to modify the inputs.

Methods:

• call_hf_processor –

  Call hf_processor on the prompt data

• get_hf_config –

  Get the HuggingFace configuration

• get_hf_image_processor_config –

  Get the HuggingFace image processor configuration of the model.

• get_hf_processor –

  Get the HuggingFace processor

• get_merged_mm_kwargs –

  Merge configured and request mm_processor_kwargs.

• get_mm_config –

  Get the multimodal config of the model.

• init_processor –

  Initialize a HuggingFace-like processor class, merging the

Attributes:

• model_config (ModelConfig) –

  The configuration of the model.

• tokenizer (TokenizerLike | None) –

  The tokenizer used to tokenize the inputs.

Source code in vllm/multimodal/processing/context.py

@dataclass(frozen=True)
class InputProcessingContext:
    """Contains information about the model which may be used to
    modify the inputs.
    """

    model_config: ModelConfig
    """The configuration of the model."""

    tokenizer: TokenizerLike | None
    """The tokenizer used to tokenize the inputs."""

    def get_tokenizer(self) -> TokenizerLike:
        if self.tokenizer is None:
            raise ValueError(
                "You cannot pass text prompts when `skip_tokenizer_init=True`"
            )

        return self.tokenizer

    @overload
    def get_hf_config(self, /) -> PreTrainedConfig: ...

    @overload
    def get_hf_config(
        self,
        typ: type[_C] | tuple[type[_C], ...],
        /,
    ) -> _C: ...

    def get_hf_config(
        self,
        typ: type[Any] | tuple[type[Any], ...] | None = None,
        /,
    ) -> Any:
        """Get the HuggingFace configuration
        (`transformers.PreTrainedConfig`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the configuration is not of the specified type.

        """
        if typ is None:
            from transformers.configuration_utils import PreTrainedConfig

            typ = PreTrainedConfig

        hf_config = self.model_config.hf_config
        if not isinstance(hf_config, typ):
            raise TypeError(
                "Invalid type of HuggingFace config. "
                f"Expected type: {typ}, but "
                f"found type: {type(hf_config)}"
            )

        return hf_config

    def get_hf_image_processor_config(self) -> dict[str, Any]:
        """Get the HuggingFace image processor configuration of the model."""
        return self.model_config.hf_image_processor_config

    def get_mm_config(self):
        """Get the multimodal config of the model.

        Raises:
            RuntimeError: If the model is not a multimodal model.

        """
        mm_config = self.model_config.multimodal_config
        if mm_config is None:
            raise RuntimeError("Not a multimodal model")

        return mm_config

    @overload
    def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...

    @overload
    def get_hf_processor(
        self,
        typ: type[_P] | tuple[type[_P], ...],
        /,
        **kwargs: object,
    ) -> _P: ...

    def get_hf_processor(
        self,
        typ: type[Any] | tuple[type[Any], ...] | None = None,
        /,
        **kwargs: object,
    ) -> Any:
        """Get the HuggingFace processor
        (`transformers.ProcessorMixin`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the processor is not of the specified type.

        """
        if typ is None:
            from transformers.processing_utils import ProcessorMixin

            typ = ProcessorMixin

        tokenizer = self.tokenizer
        if is_mistral_tokenizer(tokenizer):
            tokenizer = tokenizer.transformers_tokenizer  # type: ignore[union-attr]

        merged_kwargs = self.get_merged_mm_kwargs(kwargs)
        merged_kwargs.pop("tokenizer", None)

        return cached_processor_from_config(
            self.model_config,
            processor_cls=typ,
            tokenizer=tokenizer,
            **merged_kwargs,
        )

    def init_processor(
        self,
        typ: type[_T],
        /,
        **kwargs: object,
    ) -> _T:
        """Initialize a HuggingFace-like processor class, merging the
        keyword arguments with those in the model's configuration.
        """
        merged_kwargs = self.get_merged_mm_kwargs(kwargs)

        return typ(**merged_kwargs)

    def _postprocess_output(
        self,
        output: JSONTree,
    ) -> JSONTree:
        # "torch_shm" puts tensors on a torch.multiprocessing queue, which
        # shares device tensors by CUDA IPC handle, so a device-side processor
        # can hand `pixel_values` straight to the worker. Every other transport
        # serializes host bytes, so the result has to be copied back first.
        keep_on_device = (
            self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm"
        )

        def _postprocess_one(x: object):
            if not isinstance(x, torch.Tensor):
                return x

            # Bind to a Tensor-typed local: reassigning the `object`-typed
            # parameter would discard the isinstance narrowing.
            tensor = x

            # This mimics the behavior of transformers.BatchFeature
            if tensor.is_floating_point():
                tensor = tensor.to(dtype=self.model_config.dtype)

            if not tensor.is_cpu and not keep_on_device:
                tensor = tensor.cpu()

            return tensor

        return json_map_leaves(_postprocess_one, output)

    def get_merged_mm_kwargs(
        self,
        kwargs: Mapping[str, object],
        *,
        supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
    ) -> dict[str, Any]:
        """Merge configured and request ``mm_processor_kwargs``.

        When ``supported_mm_processor_kwargs`` is provided, flat keys are matched
        against the keys supported by each HuggingFace processor kwarg scope.
        Matching flat keys are routed to every scope that supports them and
        removed from the shared flat namespace; keys not supported by any scope
        remain flat.

        Without ``supported_mm_processor_kwargs``, flat keys already represented
        in at least one existing mapping-valued scope are removed from the shared
        flat namespace; all other flat keys are left unchanged because there is
        no information to determine which scopes should receive them.
        """
        mm_config = self.model_config.get_multimodal_config()
        merged = mm_config.merge_mm_processor_kwargs(kwargs)
        return _resolve_mm_processor_kwargs(
            merged,
            supported_mm_processor_kwargs=supported_mm_processor_kwargs,
        )

    def call_hf_processor(
        self,
        hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
        data: Mapping[str, object],
        kwargs: Mapping[str, object] = {},
    ) -> BatchFeature:
        """Call `hf_processor` on the prompt `data`
        (text, image, audio...) with configurable options `kwargs`.
        """
        assert callable(hf_processor)

        merged_kwargs = self.get_merged_mm_kwargs(kwargs)

        allowed_kwargs = get_allowed_kwarg_only_overrides(
            hf_processor,
            merged_kwargs,
            requires_kw_only=False,
            allow_var_kwargs=True,
        )
        allowed_kwargs.setdefault("return_tensors", "pt")

        try:
            output = hf_processor(**data, **allowed_kwargs)
        except Exception as exc:
            msg = (
                f"Failed to apply {type(hf_processor).__name__} "
                f"on data={data} with kwargs={allowed_kwargs}"
            )

            raise ValueError(msg) from exc

        # this emulates output.to(dtype=self.model_config.dtype)
        from transformers.feature_extraction_utils import BatchFeature

        if isinstance(output, BatchFeature):
            output_ = self._postprocess_output(output.data)
            return BatchFeature(output_)  # type: ignore

        logger.warning_once(
            "%s did not return `BatchFeature`. "
            "Make sure to match the behaviour of `ProcessorMixin` when "
            "implementing custom processors.",
            type(hf_processor).__name__,
        )

        return self._postprocess_output(output)  # type: ignore

model_config instance-attribute

The configuration of the model.

tokenizer instance-attribute

The tokenizer used to tokenize the inputs.

call_hf_processor(hf_processor, data, kwargs={})

Call hf_processor on the prompt data (text, image, audio...) with configurable options kwargs.

Source code in vllm/multimodal/processing/context.py

def call_hf_processor(
    self,
    hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
    data: Mapping[str, object],
    kwargs: Mapping[str, object] = {},
) -> BatchFeature:
    """Call `hf_processor` on the prompt `data`
    (text, image, audio...) with configurable options `kwargs`.
    """
    assert callable(hf_processor)

    merged_kwargs = self.get_merged_mm_kwargs(kwargs)

    allowed_kwargs = get_allowed_kwarg_only_overrides(
        hf_processor,
        merged_kwargs,
        requires_kw_only=False,
        allow_var_kwargs=True,
    )
    allowed_kwargs.setdefault("return_tensors", "pt")

    try:
        output = hf_processor(**data, **allowed_kwargs)
    except Exception as exc:
        msg = (
            f"Failed to apply {type(hf_processor).__name__} "
            f"on data={data} with kwargs={allowed_kwargs}"
        )

        raise ValueError(msg) from exc

    # this emulates output.to(dtype=self.model_config.dtype)
    from transformers.feature_extraction_utils import BatchFeature

    if isinstance(output, BatchFeature):
        output_ = self._postprocess_output(output.data)
        return BatchFeature(output_)  # type: ignore

    logger.warning_once(
        "%s did not return `BatchFeature`. "
        "Make sure to match the behaviour of `ProcessorMixin` when "
        "implementing custom processors.",
        type(hf_processor).__name__,
    )

    return self._postprocess_output(output)  # type: ignore

get_hf_config(typ=None)

get_hf_config() -> PreTrainedConfig

get_hf_config(typ: type[_C] | tuple[type[_C], ...]) -> _C

Get the HuggingFace configuration (transformers.PreTrainedConfig) of the model, additionally checking its type.

Raises:

• TypeError –

  If the configuration is not of the specified type.

Source code in vllm/multimodal/processing/context.py

def get_hf_config(
    self,
    typ: type[Any] | tuple[type[Any], ...] | None = None,
    /,
) -> Any:
    """Get the HuggingFace configuration
    (`transformers.PreTrainedConfig`) of the model,
    additionally checking its type.

    Raises:
        TypeError: If the configuration is not of the specified type.

    """
    if typ is None:
        from transformers.configuration_utils import PreTrainedConfig

        typ = PreTrainedConfig

    hf_config = self.model_config.hf_config
    if not isinstance(hf_config, typ):
        raise TypeError(
            "Invalid type of HuggingFace config. "
            f"Expected type: {typ}, but "
            f"found type: {type(hf_config)}"
        )

    return hf_config

get_hf_image_processor_config()

Get the HuggingFace image processor configuration of the model.

Source code in vllm/multimodal/processing/context.py

def get_hf_image_processor_config(self) -> dict[str, Any]:
    """Get the HuggingFace image processor configuration of the model."""
    return self.model_config.hf_image_processor_config

get_hf_processor(typ=None, /, **kwargs)

get_hf_processor(**kwargs: object) -> ProcessorMixin

get_hf_processor(
    typ: type[_P] | tuple[type[_P], ...],
    /,
    **kwargs: object,
) -> _P

Get the HuggingFace processor (transformers.ProcessorMixin) of the model, additionally checking its type.

Raises:

• TypeError –

  If the processor is not of the specified type.

Source code in vllm/multimodal/processing/context.py

def get_hf_processor(
    self,
    typ: type[Any] | tuple[type[Any], ...] | None = None,
    /,
    **kwargs: object,
) -> Any:
    """Get the HuggingFace processor
    (`transformers.ProcessorMixin`) of the model,
    additionally checking its type.

    Raises:
        TypeError: If the processor is not of the specified type.

    """
    if typ is None:
        from transformers.processing_utils import ProcessorMixin

        typ = ProcessorMixin

    tokenizer = self.tokenizer
    if is_mistral_tokenizer(tokenizer):
        tokenizer = tokenizer.transformers_tokenizer  # type: ignore[union-attr]

    merged_kwargs = self.get_merged_mm_kwargs(kwargs)
    merged_kwargs.pop("tokenizer", None)

    return cached_processor_from_config(
        self.model_config,
        processor_cls=typ,
        tokenizer=tokenizer,
        **merged_kwargs,
    )

get_merged_mm_kwargs(kwargs, *, supported_mm_processor_kwargs=None)

Merge configured and request mm_processor_kwargs.

When supported_mm_processor_kwargs is provided, flat keys are matched against the keys supported by each HuggingFace processor kwarg scope. Matching flat keys are routed to every scope that supports them and removed from the shared flat namespace; keys not supported by any scope remain flat.

Without supported_mm_processor_kwargs, flat keys already represented in at least one existing mapping-valued scope are removed from the shared flat namespace; all other flat keys are left unchanged because there is no information to determine which scopes should receive them.

Source code in vllm/multimodal/processing/context.py

def get_merged_mm_kwargs(
    self,
    kwargs: Mapping[str, object],
    *,
    supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
) -> dict[str, Any]:
    """Merge configured and request ``mm_processor_kwargs``.

    When ``supported_mm_processor_kwargs`` is provided, flat keys are matched
    against the keys supported by each HuggingFace processor kwarg scope.
    Matching flat keys are routed to every scope that supports them and
    removed from the shared flat namespace; keys not supported by any scope
    remain flat.

    Without ``supported_mm_processor_kwargs``, flat keys already represented
    in at least one existing mapping-valued scope are removed from the shared
    flat namespace; all other flat keys are left unchanged because there is
    no information to determine which scopes should receive them.
    """
    mm_config = self.model_config.get_multimodal_config()
    merged = mm_config.merge_mm_processor_kwargs(kwargs)
    return _resolve_mm_processor_kwargs(
        merged,
        supported_mm_processor_kwargs=supported_mm_processor_kwargs,
    )

get_mm_config()

Get the multimodal config of the model.

Raises:

• RuntimeError –

  If the model is not a multimodal model.

Source code in vllm/multimodal/processing/context.py

def get_mm_config(self):
    """Get the multimodal config of the model.

    Raises:
        RuntimeError: If the model is not a multimodal model.

    """
    mm_config = self.model_config.multimodal_config
    if mm_config is None:
        raise RuntimeError("Not a multimodal model")

    return mm_config

init_processor(typ, /, **kwargs)

Initialize a HuggingFace-like processor class, merging the keyword arguments with those in the model's configuration.

Source code in vllm/multimodal/processing/context.py

def init_processor(
    self,
    typ: type[_T],
    /,
    **kwargs: object,
) -> _T:
    """Initialize a HuggingFace-like processor class, merging the
    keyword arguments with those in the model's configuration.
    """
    merged_kwargs = self.get_merged_mm_kwargs(kwargs)

    return typ(**merged_kwargs)

MultiModalApplyState dataclass

Intermediate state between apply_phase1 and apply_phase2.

Phase 1 (hashing, cache lookup, decode submission) runs on the single mm worker; the submitted decode futures are awaited off the mm worker (on the event loop for async callers) so the worker stays free for other requests while the media thread pool decodes; phase 2 (HF processing, cache merge) runs on the mm worker again.

Methods:

• close –

  Drain submitted work before releasing owned refs.

• wait_decodes –

  Join all submitted decodes (blocking; for the sync apply path).

• wait_decodes_async –

  Await all submitted decodes without blocking the event loop.

Attributes:

• decodes (list[MediaDecodeJob]) –

  Submitted decodes, with their original request locations.

• mm_hashes (MultiModalHashes | None) –

  None on the no-cache path, where hashes are computed in phase 2.

Source code in vllm/multimodal/processing/processor.py

@dataclass
class MultiModalApplyState:
    """Intermediate state between `apply_phase1` and `apply_phase2`.

    Phase 1 (hashing, cache lookup, decode submission) runs on the single
    mm worker; the submitted decode futures are awaited off the mm worker
    (on the event loop for async callers) so the worker stays free for
    other requests while the media thread pool decodes; phase 2 (HF
    processing, cache merge) runs on the mm worker again.
    """

    inputs: ProcessorInputs
    timing_ctx: TimingContext
    mm_hashes: MultiModalHashes | None
    """None on the no-cache path, where hashes are computed in phase 2."""

    decodes: list[MediaDecodeJob]
    """Submitted decodes, with their original request locations."""

    _released: bool = field(default=False, init=False, repr=False)

    def __enter__(self) -> Self:
        return self

    def __exit__(self, exc_type, exc_value, traceback) -> None:
        self.close()

    def close(self) -> None:
        """Drain submitted work before releasing owned refs."""
        try:
            for job in self.decodes:
                # Wait methods report failures; cleanup preserves the
                # exception already propagating from processing.
                with suppress(BaseException):
                    job.future.result()
        finally:
            self._release_refs()

    def _release_refs(self) -> None:
        """Release owned refs once, after decoding and processing have finished.

        Selected cache misses share the state's wrappers. Borrowed input refs
        and their mapped parents remain caller-owned and reusable.
        """
        if self._released:
            return
        self._released = True
        for items in self.inputs.mm_data_items.values():
            if isinstance(items, ProcessorBatchItems):
                for item in items.get_all_raw():
                    if isinstance(item, MediaRef):
                        item.release()

    def wait_decodes(self) -> None:
        """Join all submitted decodes (blocking; for the sync apply path).

        In-flight decodes are always awaited, never abandoned; the first
        failure is raised as VLLMUnprocessableEntityError.
        """
        if not self.decodes:
            return
        with self.timing_ctx.record("decode_mm_items"):
            collect_media_decodes(self.decodes, _decode_error)

    async def wait_decodes_async(self) -> None:
        """Await all submitted decodes without blocking the event loop.

        Same completion/error semantics as `wait_decodes`.
        """
        if not self.decodes:
            return
        with self.timing_ctx.record("decode_mm_items"):
            await collect_media_decodes_async(self.decodes, _decode_error)

decodes instance-attribute

Submitted decodes, with their original request locations.

mm_hashes instance-attribute

None on the no-cache path, where hashes are computed in phase 2.

_release_refs()

Release owned refs once, after decoding and processing have finished.

Selected cache misses share the state's wrappers. Borrowed input refs and their mapped parents remain caller-owned and reusable.

Source code in vllm/multimodal/processing/processor.py

def _release_refs(self) -> None:
    """Release owned refs once, after decoding and processing have finished.

    Selected cache misses share the state's wrappers. Borrowed input refs
    and their mapped parents remain caller-owned and reusable.
    """
    if self._released:
        return
    self._released = True
    for items in self.inputs.mm_data_items.values():
        if isinstance(items, ProcessorBatchItems):
            for item in items.get_all_raw():
                if isinstance(item, MediaRef):
                    item.release()

close()

Drain submitted work before releasing owned refs.

Source code in vllm/multimodal/processing/processor.py

def close(self) -> None:
    """Drain submitted work before releasing owned refs."""
    try:
        for job in self.decodes:
            # Wait methods report failures; cleanup preserves the
            # exception already propagating from processing.
            with suppress(BaseException):
                job.future.result()
    finally:
        self._release_refs()

wait_decodes()

Join all submitted decodes (blocking; for the sync apply path).

In-flight decodes are always awaited, never abandoned; the first failure is raised as VLLMUnprocessableEntityError.

Source code in vllm/multimodal/processing/processor.py

def wait_decodes(self) -> None:
    """Join all submitted decodes (blocking; for the sync apply path).

    In-flight decodes are always awaited, never abandoned; the first
    failure is raised as VLLMUnprocessableEntityError.
    """
    if not self.decodes:
        return
    with self.timing_ctx.record("decode_mm_items"):
        collect_media_decodes(self.decodes, _decode_error)

wait_decodes_async() async

Await all submitted decodes without blocking the event loop.

Same completion/error semantics as wait_decodes.

Source code in vllm/multimodal/processing/processor.py

async def wait_decodes_async(self) -> None:
    """Await all submitted decodes without blocking the event loop.

    Same completion/error semantics as `wait_decodes`.
    """
    if not self.decodes:
        return
    with self.timing_ctx.record("decode_mm_items"):
        await collect_media_decodes_async(self.decodes, _decode_error)

ProcessorInputs dataclass

Represents the keyword arguments to vllm.multimodal.processing.BaseMultiModalProcessor.apply.

Methods:

• fork_media_refs –

  Give processing releasable wrappers around borrowed media refs.

Attributes:

• can_use_cache (bool) –

  Whether these inputs can use the processor cache.

Source code in vllm/multimodal/processing/inputs.py

@dataclass
class ProcessorInputs:
    """Represents the keyword arguments to
    [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
    """

    prompt: list[int]
    mm_data_items: MultiModalDataItems
    mm_uuid_items: MultiModalUUIDItems | None = None
    hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
    media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)

    cache: BaseMultiModalProcessorCache | None = None

    def fork_media_refs(self) -> "ProcessorInputs":
        """Give processing releasable wrappers around borrowed media refs."""
        owned_items = MultiModalDataItems()
        for modality, items in self.mm_data_items.items():
            if isinstance(items, ProcessorBatchItems):
                items = items.map_raw(
                    lambda item: item.fork() if isinstance(item, MediaRef) else item
                )
            owned_items[modality] = items
        return replace(self, mm_data_items=owned_items)

    @property
    def can_use_cache(self) -> bool:
        """Whether these inputs can use the processor cache."""
        return self.cache is not None and not any(
            items.get_passthrough_data() for items in self.mm_data_items.values()
        )

    def get_mm_hashes(
        self,
        model_id: str,
        hash_algorithm: MMHasherAlgorithm,
    ) -> MultiModalHashes:
        mm_data_items = self.mm_data_items
        mm_uuid_items = self.mm_uuid_items or {}

        mm_hashes = dict[str, list[str]]()
        hasher = MultiModalHasher

        for modality, data_items in mm_data_items.items():
            # Each cache entry represents one item from one modality. Keep
            # modality-scoped options out of unrelated entries, while
            # retaining flat processor kwargs because they are shared by HF.
            if modality in _HF_MODALITY_PROCESSOR_KWARGS:
                modality_media_io_kwargs = self.media_io_kwargs.get(modality, {})
                media_io_kwargs = (
                    {modality: modality_media_io_kwargs}
                    if modality_media_io_kwargs
                    else {}
                )
                scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality]
                mm_processor_kwargs = {
                    key: value
                    for key, value in self.hf_processor_mm_kwargs.items()
                    if key not in _HF_MODALITY_PROCESSOR_KWARGS.values()
                    or key == scoped_key
                }
            else:
                # Unified modalities such as ``vision_chunk`` can contain
                # items originating from multiple media modalities. There is
                # no per-item origin metadata here, so preserve all factors.
                media_io_kwargs = dict(self.media_io_kwargs)
                mm_processor_kwargs = dict(self.hf_processor_mm_kwargs)

            hash_factors = {
                "media_io_kwargs": media_io_kwargs,
                "mm_processor_kwargs": mm_processor_kwargs,
            }
            hash_factors = {key: value for key, value in hash_factors.items() if value}

            uuid_items = (
                mm_uuid_items[modality]
                if modality in mm_uuid_items
                else ([None] * len(data_items))
            )

            # For None entries, compute a hash; otherwise, use provided ID.
            hashes: list[str] = []
            for i, item in enumerate(data_items.get_all_raw()):
                uuid_item = uuid_items[i]

                # A MediaRef's decode spec is already folded into its key, so
                # media_io_kwargs must not be counted twice for it. When the
                # client supplies a UUID the key is not consulted at all, so
                # every factor still has to be hashed in.
                item_factors = hash_factors
                if uuid_item is None and isinstance(item, MediaRef):
                    item_factors = {
                        key: value
                        for key, value in hash_factors.items()
                        if key != "media_io_kwargs"
                    }

                # NOTE: Even if a uuid_item is provided, model output depends
                # on the current modality's hash factors, so they are taken
                # into account.
                if uuid_item is None or item_factors:
                    # NOTE: use provided hash string to hash with kwargs
                    # if available for better performance.
                    item = uuid_item if uuid_item is not None else item
                    hashes.append(
                        hasher.hash_kwargs(
                            hash_algorithm,
                            model_id=model_id,
                            **{modality: item},
                            **item_factors,
                        )
                    )
                else:
                    # If there are no extra kwargs, use the client-provided UUID.
                    hashes.append(uuid_item)

            mm_hashes[modality] = hashes

        return mm_hashes

can_use_cache property

Whether these inputs can use the processor cache.

fork_media_refs()

Give processing releasable wrappers around borrowed media refs.

Source code in vllm/multimodal/processing/inputs.py

def fork_media_refs(self) -> "ProcessorInputs":
    """Give processing releasable wrappers around borrowed media refs."""
    owned_items = MultiModalDataItems()
    for modality, items in self.mm_data_items.items():
        if isinstance(items, ProcessorBatchItems):
            items = items.map_raw(
                lambda item: item.fork() if isinstance(item, MediaRef) else item
            )
        owned_items[modality] = items
    return replace(self, mm_data_items=owned_items)

PromptIndexTargets

Methods:

• end –

  Resolves to the end of the prompt (after the last token).

• prefix –

  Resolves to a location in the prompt after the given prefix.

• start –

  Resolves to the start of the prompt (before the first token).

Source code in vllm/multimodal/processing/processor.py

class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(
            lambda prompt, start_idx=0: 0,
            lambda tokenizer, prompt, start_idx=0: 0,
        )

    @staticmethod
    def prefix(seq: list[int]) -> PromptIndex:
        """Resolves to a location in the prompt after the given prefix."""

        def get_match_index(
            prompt: list[int],
            start_idx: int = 0,
        ) -> int | None:
            if start_idx != 0:
                return None

            match_idx = len(seq)
            return match_idx if prompt[:match_idx] == seq else None

        def get_text_match_index(
            tokenizer: TokenizerLike,
            prompt: str,
            start_idx: int = 0,
        ) -> int | None:
            if start_idx != 0:
                return None

            prefix = tokenizer.decode(seq)
            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index, get_text_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(
            lambda prompt, start_idx=0: len(prompt),
            lambda tokenizer, prompt, start_idx=0: len(prompt),
        )

end() staticmethod

Resolves to the end of the prompt (after the last token).

This results in a match even if the prompt is empty.

Source code in vllm/multimodal/processing/processor.py

@staticmethod
def end() -> PromptIndex:
    """Resolves to the end of the prompt (after the last token).

    This results in a match even if the prompt is empty.
    """
    return PromptIndex(
        lambda prompt, start_idx=0: len(prompt),
        lambda tokenizer, prompt, start_idx=0: len(prompt),
    )

prefix(seq) staticmethod

Resolves to a location in the prompt after the given prefix.

Source code in vllm/multimodal/processing/processor.py

@staticmethod
def prefix(seq: list[int]) -> PromptIndex:
    """Resolves to a location in the prompt after the given prefix."""

    def get_match_index(
        prompt: list[int],
        start_idx: int = 0,
    ) -> int | None:
        if start_idx != 0:
            return None

        match_idx = len(seq)
        return match_idx if prompt[:match_idx] == seq else None

    def get_text_match_index(
        tokenizer: TokenizerLike,
        prompt: str,
        start_idx: int = 0,
    ) -> int | None:
        if start_idx != 0:
            return None

        prefix = tokenizer.decode(seq)
        match_idx = len(prefix)
        return match_idx if prompt[:match_idx] == prefix else None

    return PromptIndex(get_match_index, get_text_match_index)

start() staticmethod

Resolves to the start of the prompt (before the first token).

This results in a match even if the prompt is empty.

Source code in vllm/multimodal/processing/processor.py

@staticmethod
def start() -> PromptIndex:
    """Resolves to the start of the prompt (before the first token).

    This results in a match even if the prompt is empty.
    """
    return PromptIndex(
        lambda prompt, start_idx=0: 0,
        lambda tokenizer, prompt, start_idx=0: 0,
    )

PromptInsertion dataclass

Bases: PromptUpdate

Defines how to insert placeholder tokens into a prompt.

Example: For each image, insert a number of <image> feature placeholders equal to the feature size of the vision encoder after the <s> token:

PromptInsertion(
    modality="image",
    target=[bos_token_id],
    insertion=[image_token_id] * image_feature_size,
)

Insert these tokens at the start of the prompt:

PromptInsertion(
    modality="image",
    target=PromptIndexTargets.start(),
    insertion=[image_token_id] * image_feature_size,
)

Insert these tokens after a prefix Images::

PromptInsertion(
    modality="image",
    target=PromptIndexTargets.prefix(images_prefix_token_ids),
    insertion=[image_token_id] * image_feature_size,
)

Insert these tokens at the end of the prompt:

PromptInsertion(
    modality="image",
    target=PromptIndexTargets.end(),
    insertion=[image_token_id] * image_feature_size,
)

Attributes:

• insertion (PromptUpdateContent) –

  Given the index of the processed item within

Source code in vllm/multimodal/processing/processor.py

@dataclass
class PromptInsertion(PromptUpdate):
    """Defines how to insert placeholder tokens into a prompt.

    Example:
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:

    ```python
    PromptInsertion(
        modality="image",
        target=[bos_token_id],
        insertion=[image_token_id] * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion=[image_token_id] * image_feature_size,
    )
    ```

    Insert these tokens after a prefix `Images:`:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix(images_prefix_token_ids),
        insertion=[image_token_id] * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion=[image_token_id] * image_feature_size,
    )
    ```

    """

    insertion: PromptUpdateContent = field(repr=False)
    """
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].

    For convenience, you can directly pass in the token sequence
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT

insertion = field(repr=False) class-attribute instance-attribute

Given the index of the processed item within modality, output the token sequence to insert right after target.

For convenience, you can directly pass in the token sequence instead of a function if it does not depend on the input.

PromptReplacement dataclass

Bases: PromptUpdate

Defines how to replace portions of an input prompt with placeholder tokens.

Example: For each image, replace one <image> input placeholder in the prompt with a number of <image> feature placeholders equal to the feature size of the vision encoder:

PromptReplacement(
    modality="image",
    target=[image_token_id],
    replacement=[image_token_id] * image_feature_size,
)

As above, but further pad the feature placeholders with <image_bos> and <image_eos>, which are not supposed to be passed to the vision encoder:

PromptReplacement(
    modality="image",
    target=[image_token_id],
    replacement=PromptUpdateDetails(
        full=(
            [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
        ),
        is_embed=lambda full: torch.tensor(full) == image_token_id,
    ),
)

Attributes:

• replacement (PromptUpdateContent) –

  Given the index of the processed item within

Source code in vllm/multimodal/processing/processor.py

@dataclass
class PromptReplacement(PromptUpdate):
    """Defines how to replace portions of an input prompt with placeholder tokens.

    Example:
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=[image_token_id] * image_feature_size,
    )
    ```

    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
            is_embed=lambda full: torch.tensor(full) == image_token_id,
        ),
    )
    ```

    """

    replacement: PromptUpdateContent = field(repr=False)
    """
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].

    For convenience, you can directly pass in the token sequence
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE

replacement = field(repr=False) class-attribute instance-attribute

Given the index of the processed item within modality, output the token sequence to replace target.

For convenience, you can directly pass in the token sequence instead of a function if it does not depend on the input.

PromptUpdate dataclass

Bases: ABC

Defines how to update a prompt with placeholder tokens.

Note

The target and content are token sequences. When converting text to token sequences, remember to encode it with add_special_tokens=False to match how the prompt itself is tokenized. Otherwise, the updated prompt may contain duplicated special tokens or fail to match the target.

Methods:

• resolve –

  Given the index of the processed item within

Attributes:

• content (PromptUpdateContent) –

  The placeholder tokens that are part of the update.

• modality (str) –

  The modality for which the update is made.

• mode (UpdateMode) –

  Defines how to update the prompt.

• target (PromptUpdateTarget) –

  The token sequence to update.

Source code in vllm/multimodal/processing/processor.py

@dataclass
class PromptUpdate(ABC):
    """Defines how to update a prompt with placeholder tokens.

    Note:
        The target and content are token sequences. When converting text
        to token sequences, remember to encode it with
        `add_special_tokens=False` to match how the prompt itself is
        tokenized. Otherwise, the updated prompt may contain duplicated
        special tokens or fail to match the target.

    """

    modality: str
    """The modality for which the update is made."""

    target: PromptUpdateTarget
    """The token sequence to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

    def _resolve_target(self, item_idx: int) -> UpdateTarget:
        target = self.target
        if callable(target):
            target = target(item_idx)

        return target

    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return content

    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
        """Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
        )

content abstractmethod property

The placeholder tokens that are part of the update.

modality instance-attribute

The modality for which the update is made.

mode abstractmethod property

Defines how to update the prompt.

target instance-attribute

The token sequence to update.

resolve(item_idx)

Given the index of the processed item within modality, output a copy of this object with its lazy attributes resolved.

Source code in vllm/multimodal/processing/processor.py

def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
    """Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output a copy of this object with its lazy attributes resolved.
    """
    return ResolvedPromptUpdate(
        modality=self.modality,
        item_idx=item_idx,
        mode=self.mode,
        target=self._resolve_target(item_idx),
        content=self._resolve_content(item_idx),
    )

PromptUpdateDetails dataclass

Details about the token sequence that is part of the update.

Attributes:

• full (list[int]) –

  The full content.

• is_embed (Callable[[list[int]], Tensor] | None) –

  Given full,

Source code in vllm/multimodal/processing/processor.py

@dataclass
class PromptUpdateDetails:
    """Details about the token sequence that is part of the update."""

    full: list[int]
    """The full content."""

    is_embed: Callable[[list[int]], torch.Tensor] | None = None
    """
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
    """

    @staticmethod
    def from_seq(seq: list[int]) -> "PromptUpdateDetails":
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_token_id(
        seq: list[int],
        embed_token_id: int,
    ) -> "PromptUpdateDetails":
        def is_embed(full: list[int]) -> torch.Tensor:
            return torch.tensor(full) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_ids(
        seq: list[int],
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails":
        def is_embed(full: list[int]) -> torch.Tensor:
            return torch.isin(
                torch.tensor(full),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

full instance-attribute

The full content.

is_embed = None class-attribute instance-attribute

Given full, return a boolean mask of shape (len(full),) indicating which positions of full to assign embeddings to.

None (default) means to assign embeddings to all positions of full.

The embeddings are obtained by calling SupportsMultiModal.embed_multimodal.

TimingContext dataclass

Helper class to record execution times during multi-modal processing.

Methods:

• record –

  Record the execution time for a processing stage.

Attributes:

• enabled (bool) –

  If disabled, TimingContext.record becomes a no-op.

• stage_secs (dict[str, float]) –

  The execution time (in seconds) for each processing stage.

Source code in vllm/multimodal/processing/context.py

@dataclass
class TimingContext:
    """Helper class to record execution times during multi-modal processing."""

    enabled: bool = True
    """If disabled, `TimingContext.record` becomes a no-op."""

    stage_secs: dict[str, float] = field(default_factory=dict)
    """The execution time (in seconds) for each processing stage."""

    @property
    def total_secs(self) -> float:
        return sum(self.stage_secs.values())

    @contextmanager
    def record(self, stage: str):
        """Record the execution time for a processing stage."""
        if not self.enabled:
            yield
            return

        start_time = time.perf_counter()
        try:
            yield
        finally:
            elapsed = time.perf_counter() - start_time
            self.stage_secs.setdefault(stage, 0.0)
            self.stage_secs[stage] += elapsed

    def get_stats_dict(self):
        stats_dict = {
            f"{stage}_secs": time_s for stage, time_s in self.stage_secs.items()
        }
        stats_dict["preprocessor_total_secs"] = self.total_secs

        return stats_dict

enabled = True class-attribute instance-attribute

If disabled, TimingContext.record becomes a no-op.

stage_secs = field(default_factory=dict) class-attribute instance-attribute

The execution time (in seconds) for each processing stage.

record(stage)

Record the execution time for a processing stage.

Source code in vllm/multimodal/processing/context.py

@contextmanager
def record(self, stage: str):
    """Record the execution time for a processing stage."""
    if not self.enabled:
        yield
        return

    start_time = time.perf_counter()
    try:
        yield
    finally:
        elapsed = time.perf_counter() - start_time
        self.stage_secs.setdefault(stage, 0.0)
        self.stage_secs[stage] += elapsed

cached_encode(tokenizer, text, *, truncation=None, add_special_tokens=True) cached

Encode text while caching repeated tokenizer calls.

Source code in vllm/multimodal/processing/processor.py

@lru_cache(maxsize=2048)
def cached_encode(
    tokenizer: TokenizerLike,
    text: str,
    *,
    truncation: bool | None = None,
    add_special_tokens: bool = True,
) -> list[int]:
    """Encode text while caching repeated tokenizer calls."""
    return tokenizer.encode(
        text,
        truncation=truncation,
        add_special_tokens=add_special_tokens,
    )