模型調優簡介

# 使用前安裝：pip install opencv-python
import math
import os
import logging
import cv2

logger = logging.getLogger(__name__)

FRAME_FACTOR = 2

# Qwen3.6、Qwen3.5、Qwen3-VL、qwen-vl-max-0813、qwen-vl-plus-0815、qwen-vl-plus-0710等模型，映像縮放因子為32
IMAGE_FACTOR = 32

#  其他模型，映像縮放因子為28
# IMAGE_FACTOR = 28

# 視訊框架的最大長寬比
MAX_RATIO = 200
# 視訊框架的像素下限
VIDEO_MIN_PIXELS = 4 * 32 * 32
# 視訊框架的像素上限，使用Qwen3-VL-Plus模型，VIDEO_MAX_PIXELS為640 * 32 * 32，其他模型為768 * 32 * 32
VIDEO_MAX_PIXELS = 640 * 32 * 32

# 使用者未傳入FPS參數，則fps使用預設值
FPS = 2.0
# 最少抽取幀數
FPS_MIN_FRAMES = 4
# 最大抽取幀數（根據模型選擇設定值）
FPS_MAX_FRAMES = 2000

# 視頻輸入的最大像素值，使用Qwen3-VL-Plus模型，請將VIDEO_TOTAL_PIXELS設定為131072 * 32 * 32，其他模型設定為65536 * 32 * 32
VIDEO_TOTAL_PIXELS = int(float(os.environ.get('VIDEO_TOTAL_PIXELS', 131072 * 32 * 32)))

def round_by_factor(number: int, factor: int) -> int:
    """返回與”number“最接近的整數，該整數可被”factor“整除。"""
    return round(number / factor) * factor

def ceil_by_factor(number: int, factor: int) -> int:
    """返回大於或等於“number”且可被“factor”整除的最小整數。"""
    return math.ceil(number / factor) * factor

def floor_by_factor(number: int, factor: int) -> int:
    """返回小於或等於“number”且可被“factor”整除的最大整數。"""
    return math.floor(number / factor) * factor

def extract_vision_info(conversations):
    vision_infos = []
    if isinstance(conversations[0], dict):
        conversations = [conversations]
    for conversation in conversations:
        for message in conversation:
            if isinstance(message["content"], list):
                for ele in message["content"]:
                    if (
                        "image" in ele
                        or "image_url" in ele
                        or "video" in ele
                        or ele.get("type","") in ("image", "image_url", "video")
                    ):
                        vision_infos.append(ele)
    return vision_infos

def smart_nframes(ele,total_frames,video_fps):
    """用於計算抽取的視訊框架數。

    Args:
        ele (dict): 包含視頻配置的字典格式
            - fps: fps用於控制提模數型輸入幀的數量。
        total_frames (int): 視頻的原始總幀數。
        video_fps (int | float): 視頻的原始幀率

    Raises:
        nframes應該在[FRAME_FACTOR，total_frames]間隔內，否則會報錯

    Returns:
        用於模型輸入的視訊框架數。
    """
    assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
    fps = ele.get("fps", FPS)
    min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
    max_frames = floor_by_factor(ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR)
    duration = total_frames / video_fps if video_fps != 0 else 0
    if duration-int(duration)>(1/fps):
        total_frames = math.ceil(duration * video_fps)
    else:
        total_frames = math.ceil(int(duration)*video_fps)
    nframes = total_frames / video_fps * fps
    if nframes > total_frames:
        logger.warning(f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]")
    nframes = int(min(min(max(nframes, min_frames), max_frames), total_frames))
    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
        raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")

    return nframes

def get_video(video_path):
    # 擷取視頻資訊
    cap = cv2.VideoCapture(video_path)

    frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
    # 擷取視頻高度
    frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    video_fps = cap.get(cv2.CAP_PROP_FPS)
    return frame_height, frame_width, total_frames, video_fps

def smart_resize(ele, path, factor=IMAGE_FACTOR):
    # 擷取原視頻的寬和高
    height, width, total_frames, video_fps = get_video(path)
    # 視訊框架的Token下限
    min_pixels = VIDEO_MIN_PIXELS
    total_pixels = VIDEO_TOTAL_PIXELS
    # 抽取的視訊框架數
    nframes = smart_nframes(ele, total_frames, video_fps)
    max_pixels = max(min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),int(min_pixels * 1.05))

    # 視頻的長寬比不應超過200:1或1:200
    if max(height, width) / min(height, width) > MAX_RATIO:
        raise ValueError(
            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
        )

    h_bar = max(factor, round_by_factor(height, factor))
    w_bar = max(factor, round_by_factor(width, factor))
    if h_bar * w_bar > max_pixels:
        beta = math.sqrt((height * width) / max_pixels)
        h_bar = floor_by_factor(height / beta, factor)
        w_bar = floor_by_factor(width / beta, factor)
    elif h_bar * w_bar < min_pixels:
        beta = math.sqrt(min_pixels / (height * width))
        h_bar = ceil_by_factor(height * beta, factor)
        w_bar = ceil_by_factor(width * beta, factor)
    return h_bar, w_bar

def token_calculate(video_path, fps):
    # 傳入的視訊路徑和fps抽幀參數
    messages = [{"content": [{"video": video_path, "fps": fps}]}]
    vision_infos = extract_vision_info(messages)[0]

    resized_height, resized_width = smart_resize(vision_infos, video_path)

    height, width, total_frames, video_fps = get_video(video_path)
    num_frames = smart_nframes(vision_infos, total_frames, video_fps)
    print(f"原視頻尺寸：{height}*{width}， 輸入模型的尺寸：{resized_height}*{resized_width}，視頻總幀數:{total_frames}，fps等於{fps}時，抽取的總幀數：{num_frames}", end="，")
    video_token = int(math.ceil(num_frames / 2) * resized_height / 32 * resized_width / 32)
    video_token += 2   # 系統會自動添加<|vision_bos|>和<|vision_eos|>視覺標記（各計1個Token）
    return video_token

video_token = token_calculate("xxx/test.mp4", 1)
print("視頻tokens:", video_token)

# 使用前安裝：pip install Pillow
import math
import os
import logging
from typing import Tuple
from PIL import Image

logger = logging.getLogger(__name__)

# ==================== 常量定義 ====================
FRAME_FACTOR = 2
# Qwen3-VL、qwen-vl-max-0813、qwen-vl-plus-0815、qwen-vl-plus-0710模型，縮放因子為32
IMAGE_FACTOR = 32

#  其他模型，縮放因子為28
# IMAGE_FACTOR = 28

# Token計算相關常量
TOKEN_DIVISOR = 32  # token計算時的除數
VISION_SPECIAL_TOKENS = 2  # <|vision_bos|>和<|vision_eos|>標記

# 視訊框架的最大長寬比
MAX_RATIO = 200
# 視訊框架的像素下限
VIDEO_MIN_PIXELS = 4 * 32 * 32
# 視訊框架的像素上限，使用Qwen3-VL-Plus模型，VIDEO_MAX_PIXELS為640 * 32 * 32，其他模型為768 * 32 * 32
VIDEO_MAX_PIXELS = 640 * 32 * 32

# 視頻輸入的最大像素值，使用Qwen3-VL-Plus模型，請將VIDEO_TOTAL_PIXELS設定為131072 * 32 * 32，其他模型設定為65536 * 32 * 32
VIDEO_TOTAL_PIXELS = int(float(os.environ.get('VIDEO_TOTAL_PIXELS', 131072 * 32 * 32)))

def round_by_factor(number: int, factor: int) -> int:
    """返回與”number“最接近的整數，該整數可被”factor“整除。"""
    return round(number / factor) * factor

def ceil_by_factor(number: int, factor: int) -> int:
    """返回大於或等於“number”且可被“factor”整除的最小整數。"""
    return math.ceil(number / factor) * factor

def floor_by_factor(number: int, factor: int) -> int:
    """返回小於或等於“number”且可被“factor”整除的最大整數。"""
    return math.floor(number / factor) * factor

def get_image_size(image_path: str) -> Tuple[int, int]:
    if not os.path.exists(image_path):
        raise FileNotFoundError(f"影像檔不存在: {image_path}")

    try:
        image = Image.open(image_path)
        height = image.height
        width = image.width
        image.close()  # 及時關閉檔案
        return height, width
    except Exception as e:
        raise ValueError(f"無法讀取影像檔 {image_path}: {str(e)}")

def smart_resize(height: int, width: int, nframes: int, factor: int = IMAGE_FACTOR) -> Tuple[int, int]:
    """
    計算映像縮放後的尺寸

    Args:
        height: 原始映像高度
        width: 原始映像寬度
        nframes: 視訊框架數
        factor: 縮放因子，預設為IMAGE_FACTOR

    Returns:
        (resized_height, resized_width) 縮放後的高度和寬度

    Raises:
        ValueError: 長寬比超過限制
    """
    # 視訊框架的Token下限
    min_pixels = VIDEO_MIN_PIXELS
    total_pixels = VIDEO_TOTAL_PIXELS
    # 抽取的視訊框架數
    max_pixels = max(min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR), int(min_pixels * 1.05))

    # 視頻的長寬比不應超過200:1或1:200
    aspect_ratio = max(height, width) / min(height, width)
    if aspect_ratio > MAX_RATIO:
        raise ValueError(
            f"映像長寬比必須小於 {MAX_RATIO}:1，當前為 {aspect_ratio:.2f}:1"
        )

    h_bar = max(factor, round_by_factor(height, factor))
    w_bar = max(factor, round_by_factor(width, factor))
    if h_bar * w_bar > max_pixels:
        beta = math.sqrt((height * width) / max_pixels)
        h_bar = floor_by_factor(height / beta, factor)
        w_bar = floor_by_factor(width / beta, factor)
    elif h_bar * w_bar < min_pixels:
        beta = math.sqrt(min_pixels / (height * width))
        h_bar = ceil_by_factor(height * beta, factor)
        w_bar = ceil_by_factor(width * beta, factor)
    return h_bar, w_bar

def calculate_video_tokens(image_path: str, nframes: int = 1, factor: int = IMAGE_FACTOR, verbose: bool = True) -> int:
    """

    Args:
        image_path: 視訊框架檔案路徑
        nframes: 視訊框架數，
        factor: 縮放因子，預設為IMAGE_FACTOR
        verbose: 是否列印詳細資料

    Returns:
        所消耗的token數量

    Raises:
        FileNotFoundError: 檔案不存在
        ValueError: 檔案格式無效或長寬比超限
    """
    # 擷取原始映像尺寸（唯讀取一次）
    height, width = get_image_size(image_path)

    # 計算縮放後的尺寸
    resized_height, resized_width = smart_resize(height, width, nframes, factor)

    # 計算token數量
    # 公式：⌈幀數/2⌉ × (高度/TOKEN_DIVISOR) × (寬度/TOKEN_DIVISOR) + VISION_SPECIAL_TOKENS
    video_token = int(
        math.ceil(nframes / 2) *
        (resized_height / TOKEN_DIVISOR) *
        (resized_width / TOKEN_DIVISOR)
    )
    # 添加視覺標記token（<|vision_bos|>和<|vision_eos|>）
    video_token += VISION_SPECIAL_TOKENS

    if verbose:
        print(f"原視訊框架尺寸：{height}×{width}，輸入模型的尺寸：{resized_height}×{resized_width}，", end="")

    return video_token

if __name__ == "__main__":
    try:
        video_token = calculate_video_tokens("xxx/test.jpg", nframes=30)
        print(f"視頻tokens: {video_token}\n")
    except Exception as e:
        print(f"錯誤: {str(e)}\n")