SushiAI::NN

enum class Nonlinearity

The activation a layer feeds, which sets its initialisation gain.

LINEAR

No activation, or an identity one.

TANH

Hyperbolic tangent.

RELU

Rectified linear unit; also GELU and SiLU in practice.

LEAKY_RELU

Leaky rectified linear unit.

SIGMOID

Logistic sigmoid.

enum class InitFamily

Which variance rule to apply to weights.

KAIMING

Scale by fan-in; the right default for a rectifier stack.

XAVIER

Scale by both fans; the right default for tanh or sigmoid.

enum class ParameterRole

What a parameter is for, which is what decides how it is initialised.

WEIGHT

A weight matrix; gets a fan-scaled random initialisation.

BIAS

A bias vector; always starts at zero.

using SushiAI::NN::LayerFactory = std::unique_ptr<ILayer> (*)(const Shape& input, const Config::Value& attributes, std::string_view context)

Builds one layer from its spec.

constexpr std::array< LayerEntry, 13 > LAYER_REGISTRY = { LayerEntry{"linear", &make_linear}, LayerEntry{"gelu", &make_gelu}, LayerEntry{"relu", &make_relu}, LayerEntry{"tanh", &make_tanh}, LayerEntry{"sigmoid", &make_sigmoid}, LayerEntry{"leaky_relu", &make_leaky_relu}, LayerEntry{"unflatten", &make_unflatten}, LayerEntry{"conv2d", &make_conv2d}, LayerEntry{"maxpool2d", &make_maxpool2d}, LayerEntry{"avgpool2d", &make_avgpool2d}, LayerEntry{"global_avgpool", &make_global_avgpool}, LayerEntry{"flatten", &make_flatten}, LayerEntry{"upsample", &make_upsample}, }

Every layer type a model.json may name.

A file-scope constant table, in the shape of Ops::ALL and the fusion CATALOG.

template <typename T>
constexpr bool Module = Detail::IsModule<T>::value

Concept variable checking whether T defines fields() and forward().

template <typename T>
constexpr bool ModuleContainer = Detail::IsModuleContainer<T>::value

Concept variable checking whether T supports visit_children.

double calculate_gain(Nonlinearity nonlinearity, double negative_slope=0.01) noexcept

The variance-preserving gain for nonlinearity.

void kaiming_normal(SushiBLAS::Engine &engine, SushiBLAS::Tensor &tensor, int64_t fan_in, double gain)

Fills tensor from N(0, (gain / sqrt(fan_in))^2).

void kaiming_uniform(SushiBLAS::Engine &engine, SushiBLAS::Tensor &tensor, int64_t fan_in, double gain)

Fills tensor uniformly on [-b, b] with b = gain * sqrt(3 / fan_in).

void xavier_normal(SushiBLAS::Engine &engine, SushiBLAS::Tensor &tensor, int64_t fan_in, int64_t fan_out, double gain)

Fills tensor from N(0, (gain * sqrt(2 / (fan_in + fan_out)))^2).

void xavier_uniform(SushiBLAS::Engine &engine, SushiBLAS::Tensor &tensor, int64_t fan_in, int64_t fan_out, double gain)

Fills tensor uniformly on [-a, a] with a = gain * sqrt(6 / (fan_in + fan_out)).

void zeros(SushiBLAS::Engine &engine, SushiBLAS::Tensor &tensor)

Fills tensor with zeros.

template <typename M>
void initialize(SushiBLAS::Engine &engine, M &module, Graph::TensorPool &pool, const InitPolicy &policy={})

Initializes every parameter of module and flushes the device.

std::unique_ptr< ILayer > make_linear(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a Linear from "units" and an optional "use_bias".

std::unique_ptr< ILayer > make_gelu(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a GELU, which takes no keys at all.

std::unique_ptr< ILayer > make_relu(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a ReLU, which takes no key.

std::unique_ptr< ILayer > make_tanh(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a Tanh, which takes no key.

std::unique_ptr< ILayer > make_sigmoid(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a Sigmoid, which takes no key.

std::unique_ptr< ILayer > make_leaky_relu(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a LeakyReLU from an optional "slope", 0.01 when absent.

std::unique_ptr< ILayer > make_unflatten(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds an Unflatten from "height", "width" and "channels", all required.

std::unique_ptr< ILayer > make_conv2d(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a Conv2d from "filters" and "kernel", with optional "stride", "padding", "dilation" and "use_bias".

std::unique_ptr< ILayer > make_maxpool2d(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a MaxPool2d from "kernel", with optional "stride" and "padding".

std::unique_ptr< ILayer > make_avgpool2d(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds an AvgPool2d from "kernel", with optional "stride" and "padding".

std::unique_ptr< ILayer > make_global_avgpool(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a GlobalAvgPool, which takes no key.

std::unique_ptr< ILayer > make_flatten(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds a Flatten, which takes no key.

std::unique_ptr< ILayer > make_upsample(const Shape &input, const Config::Value &attributes, std::string_view context)

Builds an Upsample2d from an optional "scale", 2 when absent.

LayerStack build_layer_stack(const Config::ModelSpec &model)

Builds the whole layer stack, chaining each layer's per-sample shape.

template <typename... Ls>
Sequential(Ls...) -> Sequential< Ls... >

Deduces Sequential's layer pack from its constructor arguments.

template <typename M>
constexpr ParameterField< M > field(std::string_view name, Parameter M::*member) noexcept

Builds one entry of a layer's fields() list.

template <typename M, typename Fn>
void for_each_parameter(M &module, std::string_view prefix, Fn &&visitor)

Traverses declared parameters of a module in deterministic order.

template <typename M, typename Fn>
void for_each_parameter(M &module, Fn &&visitor)

Visits every declared parameter of module from the root.

template <typename M>
std::size_t parameter_count(M &module)

Counts module's declared parameters.

template <typename M>
int64_t parameter_elements(M &module)

Sums the element counts of module's declared parameters.