Skip to content

perceptron_head

Bases: head

A perceptron-based head for implementing multi-channel modules.

The perceptron head is designed to work with various data transformations, parameter reconciliation, and remainder functions. It also supports optional features such as dropout, batch normalization, and activation functions.

Attributes:

Name Type Description
m int

Input dimension of the head.
n int

Output dimension of the head.
channel_num int

Number of channels for multi-channel processing.
parameters_init_method str

Initialization method for parameters.
device str

Device to host the head (e.g., 'cpu' or 'cuda').

Methods:

Name Description
__init__

Initializes the perceptron head with specified configurations.
Source code in tinybig/head/basic_heads.py 
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
class perceptron_head(head):
    """
    A perceptron-based head for implementing multi-channel modules.

    The perceptron head is designed to work with various data transformations, parameter reconciliation,
    and remainder functions. It also supports optional features such as dropout, batch normalization, and activation functions.

    Attributes
    ----------
    m : int
        Input dimension of the head.
    n : int
        Output dimension of the head.
    channel_num : int
        Number of channels for multi-channel processing.
    parameters_init_method : str
        Initialization method for parameters.
    device : str
        Device to host the head (e.g., 'cpu' or 'cuda').

    Methods
    -------
    __init__(...)
        Initializes the perceptron head with specified configurations.
    """
    def __init__(
        self, m: int, n: int,
        name: str = 'perceptron_head',
        channel_num: int = 1,
        # data expansion function
        with_bspline: bool = False,
        with_taylor: bool = False, d: int = 2,
        with_hybrid_expansion: bool = False,
        # parameter reconciliation function parameters
        with_dual_lphm: bool = False,
        with_lorr: bool = False, r: int = 3,
        enable_bias: bool = True,
        # remainder function parameters
        with_residual: bool = False,
        # output processing function parameters
        with_batch_norm: bool = False,
        with_relu: bool = False,
        with_dropout: bool = True, p: float = 0.5,
        with_softmax: bool = False,
        # other parameters
        parameters_init_method: str = 'xavier_normal',
        device: str = 'cpu', *args, **kwargs
    ):
        """
        Initializes the perceptron head.

        Parameters
        ----------
        m : int
            Input dimension.
        n : int
            Output dimension.
        name : str, optional
            Name of the perceptron head, default is 'perceptron_head'.
        channel_num : int, optional
            Number of channels for processing, default is 1.
        with_bspline : bool, optional
            Whether to use B-spline expansion, default is False.
        with_taylor : bool, optional
            Whether to use Taylor expansion, default is False.
        d : int, optional
            Degree for the expansion functions, default is 2.
        with_hybrid_expansion : bool, optional
            Whether to use hybrid expansion, default is False.
        with_dual_lphm : bool, optional
            Whether to use dual LPHM reconciliation, default is False.
        with_lorr : bool, optional
            Whether to use LORR reconciliation, default is False.
        r : int, optional
            Parameter for reconciliation functions, default is 3.
        enable_bias : bool, optional
            Whether to enable bias in reconciliation functions, default is True.
        with_residual : bool, optional
            Whether to include a residual connection, default is False.
        with_batch_norm : bool, optional
            Whether to include batch normalization, default is False.
        with_relu : bool, optional
            Whether to use ReLU activation, default is False.
        with_dropout : bool, optional
            Whether to include dropout, default is True.
        p : float, optional
            Dropout probability, default is 0.5.
        with_softmax : bool, optional
            Whether to use softmax activation, default is False.
        parameters_init_method : str, optional
            Initialization method for parameters, default is 'xavier_normal'.
        device : str, optional
            Device to host the head, default is 'cpu'.

        Returns
        -------
        None
        """
        if with_taylor:
            data_transformation = taylor_expansion(
                d=d,
                device=device,
            )
        elif with_bspline:
            data_transformation = bspline_expansion(
                d=d,
                device=device,
            )
        else:
            data_transformation = identity_expansion(
                device=device,
            )
        print('** data_transformation', data_transformation)

        if with_dual_lphm:
            parameter_fabrication = dual_lphm_reconciliation(
                r=r,
                enable_bias=enable_bias,
                device=device
            )
        elif with_lorr:
            parameter_fabrication = lorr_reconciliation(
                r=r,
                enable_bias=enable_bias,
                device=device,
            )
        else:
            parameter_fabrication = identity_reconciliation(
                enable_bias=enable_bias,
                device=device,
            )
        print('** parameter_fabrication', parameter_fabrication)

        if with_residual:
            remainder = linear_remainder(
                device=device
            )
        else:
            remainder = zero_remainder(
                device=device,
            )
        print('** remainder', remainder)

        output_process_functions = []
        if with_batch_norm:
            output_process_functions.append(torch.nn.BatchNorm1d(num_features=n, device=device))
        if with_relu:
            output_process_functions.append(torch.nn.ReLU())
        if with_dropout:
            output_process_functions.append(torch.nn.Dropout(p=p))
        if with_softmax:
            output_process_functions.append(torch.nn.Softmax(dim=-1))

        print('** output_process_functions', output_process_functions)

        super().__init__(
            m=m, n=n, name=name,
            data_transformation=data_transformation,
            parameter_fabrication=parameter_fabrication,
            remainder=remainder,
            output_process_functions=output_process_functions,
            channel_num=channel_num,
            parameters_init_method=parameters_init_method,
            device=device, *args, **kwargs
        )

__init__(m, n, name='perceptron_head', channel_num=1, with_bspline=False, with_taylor=False, d=2, with_hybrid_expansion=False, with_dual_lphm=False, with_lorr=False, r=3, enable_bias=True, with_residual=False, with_batch_norm=False, with_relu=False, with_dropout=True, p=0.5, with_softmax=False, parameters_init_method='xavier_normal', device='cpu', *args, **kwargs)

Initializes the perceptron head.

Parameters:

Name Type Description Default
m int

Input dimension.

 required
n int

Output dimension.

 required
name str

Name of the perceptron head, default is 'perceptron_head'.

 'perceptron_head'
channel_num int

Number of channels for processing, default is 1.

 1
with_bspline bool

Whether to use B-spline expansion, default is False.

 False
with_taylor bool

Whether to use Taylor expansion, default is False.

 False
d int

Degree for the expansion functions, default is 2.

 2
with_hybrid_expansion bool

Whether to use hybrid expansion, default is False.

 False
with_dual_lphm bool

Whether to use dual LPHM reconciliation, default is False.

 False
with_lorr bool

Whether to use LORR reconciliation, default is False.

 False
r int

Parameter for reconciliation functions, default is 3.

 3
enable_bias bool

Whether to enable bias in reconciliation functions, default is True.

 True
with_residual bool

Whether to include a residual connection, default is False.

 False
with_batch_norm bool

Whether to include batch normalization, default is False.

 False
with_relu bool

Whether to use ReLU activation, default is False.

 False
with_dropout bool

Whether to include dropout, default is True.

 True
p float

Dropout probability, default is 0.5.

 0.5
with_softmax bool

Whether to use softmax activation, default is False.

 False
parameters_init_method str

Initialization method for parameters, default is 'xavier_normal'.

 'xavier_normal'
device str

Device to host the head, default is 'cpu'.

 'cpu'

Returns:

Type Description
None
Source code in tinybig/head/basic_heads.py 
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
def __init__(
    self, m: int, n: int,
    name: str = 'perceptron_head',
    channel_num: int = 1,
    # data expansion function
    with_bspline: bool = False,
    with_taylor: bool = False, d: int = 2,
    with_hybrid_expansion: bool = False,
    # parameter reconciliation function parameters
    with_dual_lphm: bool = False,
    with_lorr: bool = False, r: int = 3,
    enable_bias: bool = True,
    # remainder function parameters
    with_residual: bool = False,
    # output processing function parameters
    with_batch_norm: bool = False,
    with_relu: bool = False,
    with_dropout: bool = True, p: float = 0.5,
    with_softmax: bool = False,
    # other parameters
    parameters_init_method: str = 'xavier_normal',
    device: str = 'cpu', *args, **kwargs
):
    """
    Initializes the perceptron head.

    Parameters
    ----------
    m : int
        Input dimension.
    n : int
        Output dimension.
    name : str, optional
        Name of the perceptron head, default is 'perceptron_head'.
    channel_num : int, optional
        Number of channels for processing, default is 1.
    with_bspline : bool, optional
        Whether to use B-spline expansion, default is False.
    with_taylor : bool, optional
        Whether to use Taylor expansion, default is False.
    d : int, optional
        Degree for the expansion functions, default is 2.
    with_hybrid_expansion : bool, optional
        Whether to use hybrid expansion, default is False.
    with_dual_lphm : bool, optional
        Whether to use dual LPHM reconciliation, default is False.
    with_lorr : bool, optional
        Whether to use LORR reconciliation, default is False.
    r : int, optional
        Parameter for reconciliation functions, default is 3.
    enable_bias : bool, optional
        Whether to enable bias in reconciliation functions, default is True.
    with_residual : bool, optional
        Whether to include a residual connection, default is False.
    with_batch_norm : bool, optional
        Whether to include batch normalization, default is False.
    with_relu : bool, optional
        Whether to use ReLU activation, default is False.
    with_dropout : bool, optional
        Whether to include dropout, default is True.
    p : float, optional
        Dropout probability, default is 0.5.
    with_softmax : bool, optional
        Whether to use softmax activation, default is False.
    parameters_init_method : str, optional
        Initialization method for parameters, default is 'xavier_normal'.
    device : str, optional
        Device to host the head, default is 'cpu'.

    Returns
    -------
    None
    """
    if with_taylor:
        data_transformation = taylor_expansion(
            d=d,
            device=device,
        )
    elif with_bspline:
        data_transformation = bspline_expansion(
            d=d,
            device=device,
        )
    else:
        data_transformation = identity_expansion(
            device=device,
        )
    print('** data_transformation', data_transformation)

    if with_dual_lphm:
        parameter_fabrication = dual_lphm_reconciliation(
            r=r,
            enable_bias=enable_bias,
            device=device
        )
    elif with_lorr:
        parameter_fabrication = lorr_reconciliation(
            r=r,
            enable_bias=enable_bias,
            device=device,
        )
    else:
        parameter_fabrication = identity_reconciliation(
            enable_bias=enable_bias,
            device=device,
        )
    print('** parameter_fabrication', parameter_fabrication)

    if with_residual:
        remainder = linear_remainder(
            device=device
        )
    else:
        remainder = zero_remainder(
            device=device,
        )
    print('** remainder', remainder)

    output_process_functions = []
    if with_batch_norm:
        output_process_functions.append(torch.nn.BatchNorm1d(num_features=n, device=device))
    if with_relu:
        output_process_functions.append(torch.nn.ReLU())
    if with_dropout:
        output_process_functions.append(torch.nn.Dropout(p=p))
    if with_softmax:
        output_process_functions.append(torch.nn.Softmax(dim=-1))

    print('** output_process_functions', output_process_functions)

    super().__init__(
        m=m, n=n, name=name,
        data_transformation=data_transformation,
        parameter_fabrication=parameter_fabrication,
        remainder=remainder,
        output_process_functions=output_process_functions,
        channel_num=channel_num,
        parameters_init_method=parameters_init_method,
        device=device, *args, **kwargs
    )