svm_head

Bases: head

A support vector machine (SVM)-based head for implementing multi-channel modules.

This head supports linear, Gaussian RBF, and inverse quadratic RBF kernels, along with optional parameter reconciliation, residual connections, and output processing functions.

Attributes:

Name	Type	Description
m	int	Input dimension of the head.
n	int	Output dimension of the head.
kernel	str	Type of kernel function ('linear', 'gaussian_rbf', 'inverse_quadratic_rbf').
channel_num	int	Number of channels for multi-channel processing.
parameters_init_method	str	Initialization method for parameters.
device	str	Device to host the head (e.g., 'cpu' or 'cuda').

Methods:

Name	Description
__init__	Initializes the SVM head with specified configurations.

Source code in tinybig/head/basic_heads.py

class svm_head(head):
    """
    A support vector machine (SVM)-based head for implementing multi-channel modules.

    This head supports linear, Gaussian RBF, and inverse quadratic RBF kernels, along with
    optional parameter reconciliation, residual connections, and output processing functions.

    Attributes
    ----------
    m : int
        Input dimension of the head.
    n : int
        Output dimension of the head.
    kernel : str
        Type of kernel function ('linear', 'gaussian_rbf', 'inverse_quadratic_rbf').
    channel_num : int
        Number of channels for multi-channel processing.
    parameters_init_method : str
        Initialization method for parameters.
    device : str
        Device to host the head (e.g., 'cpu' or 'cuda').

    Methods
    -------
    __init__(...)
        Initializes the SVM head with specified configurations.
    """
    def __init__(
        self, m: int, n: int,
        name: str = 'svm_head',
        kernel: str = 'linear',
        base_range: tuple = (-1, 1),
        num_interval: int = 10,
        epsilon: float = 1.0,
        enable_bias: bool = False,
        # optional parameters
        with_lorr: bool = False,
        r: int = 3,
        with_residual: bool = False,
        channel_num: int = 1,
        with_batch_norm: bool = False,
        with_softmax: bool = False,
        # other parameters
        parameters_init_method: str = 'xavier_normal',
        device: str = 'cpu', *args, **kwargs
    ):
        """
        Initializes the SVM head.

        Parameters
        ----------
        m : int
            Input dimension.
        n : int
            Output dimension.
        name : str, optional
            Name of the SVM head, default is 'svm_head'.
        kernel : str, optional
            Type of kernel function ('linear', 'gaussian_rbf', 'inverse_quadratic_rbf'), default is 'linear'.
        base_range : tuple, optional
            Range for Gaussian RBF kernels, default is (-1, 1).
        num_interval : int, optional
            Number of intervals for the kernel, default is 10.
        epsilon : float, optional
            Epsilon value for kernels, default is 1.0.
        enable_bias : bool, optional
            Whether to enable bias in reconciliation functions, default is False.
        with_lorr : bool, optional
            Whether to use LORR reconciliation, default is False.
        r : int, optional
            Parameter for reconciliation functions, default is 3.
        with_residual : bool, optional
            Whether to include a residual connection, default is False.
        channel_num : int, optional
            Number of channels, default is 1.
        with_batch_norm : bool, optional
            Whether to include batch normalization, default is False.
        with_softmax : bool, optional
            Whether to use softmax activation, default is False.
        parameters_init_method : str, optional
            Initialization method for parameters, default is 'xavier_normal'.
        device : str, optional
            Device to host the head, default is 'cpu'.

        Returns
        -------
        None
        """
        if kernel == 'linear':
            data_transformation = identity_expansion(
                device=device,
            )
        elif kernel == 'gaussian_rbf':
            data_transformation = gaussian_rbf_expansion(
                base_range=base_range,
                num_interval=num_interval,
                epsilon=epsilon,
                device=device,
            )
        elif kernel == 'inverse_quadratic_rbf':
            data_transformation = inverse_quadratic_rbf_expansion(
                base_range=base_range,
                num_interval=num_interval,
                epsilon=epsilon,
                device=device,
            )
        else:
            raise ValueError('kernel must be linear or gaussian_rbf or inverse_quadratic_rbf...')

        if with_lorr:
            parameter_fabrication = lorr_reconciliation(
                r=r,
                enable_bias=enable_bias,
                device=device,
            )
        else:
            parameter_fabrication = identity_reconciliation(
                enable_bias=enable_bias,
                device=device,
            )

        if with_residual:
            remainder = linear_remainder(
                device=device
            )
        else:
            remainder = zero_remainder(
                device=device,
            )

        output_process_functions = []
        if with_batch_norm:
            output_process_functions.append(torch.nn.BatchNorm1d(num_features=n, device=device))
        if with_softmax:
            output_process_functions.append(torch.nn.Softmax(dim=-1))

        super().__init__(
            m=m, n=n, name=name,
            data_transformation=data_transformation,
            parameter_fabrication=parameter_fabrication,
            remainder=remainder,
            output_process_functions=output_process_functions,
            channel_num=channel_num,
            parameters_init_method=parameters_init_method,
            device=device, *args, **kwargs
        )

__init__(m, n, name='svm_head', kernel='linear', base_range=(-1, 1), num_interval=10, epsilon=1.0, enable_bias=False, with_lorr=False, r=3, with_residual=False, channel_num=1, with_batch_norm=False, with_softmax=False, parameters_init_method='xavier_normal', device='cpu', *args, **kwargs)

Initializes the SVM head.

Parameters:

Name	Type	Description	Default
m	int	Input dimension.	required
n	int	Output dimension.	required
name	str	Name of the SVM head, default is 'svm_head'.	'svm_head'
kernel	str	Type of kernel function ('linear', 'gaussian_rbf', 'inverse_quadratic_rbf'), default is 'linear'.	'linear'
base_range	tuple	Range for Gaussian RBF kernels, default is (-1, 1).	(-1, 1)
num_interval	int	Number of intervals for the kernel, default is 10.	10
epsilon	float	Epsilon value for kernels, default is 1.0.	1.0
enable_bias	bool	Whether to enable bias in reconciliation functions, default is False.	False
with_lorr	bool	Whether to use LORR reconciliation, default is False.	False
r	int	Parameter for reconciliation functions, default is 3.	3
with_residual	bool	Whether to include a residual connection, default is False.	False
channel_num	int	Number of channels, default is 1.	1
with_batch_norm	bool	Whether to include batch normalization, default is False.	False
with_softmax	bool	Whether to use softmax activation, default is False.	False
parameters_init_method	str	Initialization method for parameters, default is 'xavier_normal'.	'xavier_normal'
device	str	Device to host the head, default is 'cpu'.	'cpu'

Returns:

Type	Description
None

Source code in tinybig/head/basic_heads.py

def __init__(
    self, m: int, n: int,
    name: str = 'svm_head',
    kernel: str = 'linear',
    base_range: tuple = (-1, 1),
    num_interval: int = 10,
    epsilon: float = 1.0,
    enable_bias: bool = False,
    # optional parameters
    with_lorr: bool = False,
    r: int = 3,
    with_residual: bool = False,
    channel_num: int = 1,
    with_batch_norm: bool = False,
    with_softmax: bool = False,
    # other parameters
    parameters_init_method: str = 'xavier_normal',
    device: str = 'cpu', *args, **kwargs
):
    """
    Initializes the SVM head.

    Parameters
    ----------
    m : int
        Input dimension.
    n : int
        Output dimension.
    name : str, optional
        Name of the SVM head, default is 'svm_head'.
    kernel : str, optional
        Type of kernel function ('linear', 'gaussian_rbf', 'inverse_quadratic_rbf'), default is 'linear'.
    base_range : tuple, optional
        Range for Gaussian RBF kernels, default is (-1, 1).
    num_interval : int, optional
        Number of intervals for the kernel, default is 10.
    epsilon : float, optional
        Epsilon value for kernels, default is 1.0.
    enable_bias : bool, optional
        Whether to enable bias in reconciliation functions, default is False.
    with_lorr : bool, optional
        Whether to use LORR reconciliation, default is False.
    r : int, optional
        Parameter for reconciliation functions, default is 3.
    with_residual : bool, optional
        Whether to include a residual connection, default is False.
    channel_num : int, optional
        Number of channels, default is 1.
    with_batch_norm : bool, optional
        Whether to include batch normalization, default is False.
    with_softmax : bool, optional
        Whether to use softmax activation, default is False.
    parameters_init_method : str, optional
        Initialization method for parameters, default is 'xavier_normal'.
    device : str, optional
        Device to host the head, default is 'cpu'.

    Returns
    -------
    None
    """
    if kernel == 'linear':
        data_transformation = identity_expansion(
            device=device,
        )
    elif kernel == 'gaussian_rbf':
        data_transformation = gaussian_rbf_expansion(
            base_range=base_range,
            num_interval=num_interval,
            epsilon=epsilon,
            device=device,
        )
    elif kernel == 'inverse_quadratic_rbf':
        data_transformation = inverse_quadratic_rbf_expansion(
            base_range=base_range,
            num_interval=num_interval,
            epsilon=epsilon,
            device=device,
        )
    else:
        raise ValueError('kernel must be linear or gaussian_rbf or inverse_quadratic_rbf...')

    if with_lorr:
        parameter_fabrication = lorr_reconciliation(
            r=r,
            enable_bias=enable_bias,
            device=device,
        )
    else:
        parameter_fabrication = identity_reconciliation(
            enable_bias=enable_bias,
            device=device,
        )

    if with_residual:
        remainder = linear_remainder(
            device=device
        )
    else:
        remainder = zero_remainder(
            device=device,
        )

    output_process_functions = []
    if with_batch_norm:
        output_process_functions.append(torch.nn.BatchNorm1d(num_features=n, device=device))
    if with_softmax:
        output_process_functions.append(torch.nn.Softmax(dim=-1))

    super().__init__(
        m=m, n=n, name=name,
        data_transformation=data_transformation,
        parameter_fabrication=parameter_fabrication,
        remainder=remainder,
        output_process_functions=output_process_functions,
        channel_num=channel_num,
        parameters_init_method=parameters_init_method,
        device=device, *args, **kwargs
    )