Building a decoder transformer

# we'll download the tiny shakespeare dataset to train on
!wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt

# read the input file
with open('input.txt', 'r', encoding='utf-8') as f:
    text = f.read()
print(len(text)) #we're working with roughly 1 million characters 

# to look at the first 100 characters
print(text[:100])
# this text is just a sequence of characters

chars = sorted(list(set(text))) # why set? to get just unique chars, why list?: for ordering purposes(sets are unordered), sorted?: ensure proper ordering
vocab_size = len(chars)
print(''.join(chars))
print(vocab_size)

# create a mapping from characters to integers
itos = {i: ch for i, ch in enumerate(chars)}
stoi = {ch: i for i, ch in enumerate(chars)}
encode = lambda s: [stoi[c] for c in s] #encoder: take a string, output a list of integers
decode = lambda l: ''.join([itos[i] for i in l]) #decoder: take a list of integers, output a string

print(encode("sup bro"))
print(decode(encode("sup bro")))

import tiktoken
enc = tiktoken.get_encoding('gpt2')
print(enc.n_vocab) #our earlier example: we had 65 tokens A-Za-zetc, but here its 50k+ tokens

# and when we encode the exact same stuff
print(enc.encode("sup bro"))
print(enc.decode(enc.encode("sup bro")))

# encode entire text dataset, and store it into a Torch.tensor
import torch
data = torch.tensor(encode(text), dtype=torch.long) #encode all text and wrap it into torch.Tensor
print(len(encode(text)))
print(data.shape, data.dtype)
print(data[:100]) #same as prev but encoded and in a tensor

# we want to separate/split our dataset into training and validation sets
n = int(0.9 * len(data)) #first 90% will be train, rest 10% will be val
train_data = data[:n]
val_data = data[n:]

# its purpose: to help us understand, to what extent our model is overfitting, we're hiding the validation data for now 
# cuz we don't want an exact memorization of the exact Shakespeare
# we want a neural network that creates a shakespeare like text

block_size = 8
train_data[:block_size+1] #why +1? lets look at the first 9 characters in the training set
# note: when we sample a chunk of data, this actually has way more example packed into it cuz all characters follow each other

# we want to illustrate that same example with code
x = train_data[:block_size] #inputs to the transformer
y = train_data[1:block_size + 1] #y: offset by 1, cuz y: targets for each position in input
for t in range(block_size):
    context = x[:t+1] #context is always all characters upto t and including t, ah yes, initially t=0, t+1 confused me
    target = y[t]
    print(f"when input is {context} the target: {target}")

torch.manual_seed(1337) #since we're gonna sample random locations in dataset to pull chunks from, setting seed in rng ensures same result for andrej and me
batch_size = 4 #how many independent sequences will we process in parallel in every forward/backward pass of transformer?
block_size = 8 #what is the maximum context length for predictions?

def get_batch(split):
    # generate a small batch of data of inputs x and targets y
    data = train_data if split == 'train' else val_data
    ix = torch.randint(len(data) - block_size, (batch_size, )) #generate batch_size number of random offsets, and since batch_size = 4, ix will be 4 randomly generated numbers betn 0 and len(data) - block_size
    x = torch.stack([data[i: i + block_size] for i in ix])
    y = torch.stack([data[i+1: i+block_size+1] for i in ix])
    return x, y

xb, yb = get_batch('train')
print('inputs:')
print(xb.shape)
print(xb)
print('targets:')
print(yb.shape)
print(yb)

print('---')

for b in range(batch_size): #batch_dimension
    for t in range(block_size): #time_dimension
        context = xb[b, :t+1]
        target = yb[b, t]
        print(f"when input is {context.tolist()} the target: {target}")

print(xb) #we now have our batch of input that we want to feed to our transformer, let's feed it to neural networks

import torch
import torch.nn as nn
from torch.nn import functional as F
torch.manual_seed(1337)

# constructing a Bigram language model which is a subass of NN module
class BigramLanguageModel(nn.Module):

    def __init__(self, vocab_size):
        super().__init__()
        # each token directly reads off the logits for the next token from a lookup table
        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)
        # nn.Embedding() is a very thin wrapper around a tensor of shape vocab_size x vocab_size

    def forward(self, idx, targets):

        # idx and targets are both (B,T) tensors of integers
        logits = self.token_embedding_table(idx) # (B,T,C) #when we pass idx here, 
        # each idx in our input will refer the embedding table xb above, and pluck a row out of that embedding table corresponding to its index 
        # eg. 24 will go to the embedding table and pluck out the 24th row, then 43 will go and pluck out 43rd row and so on
        # pytorch will arrange all of this into a Batch by Time by Channel tensor, C = vocab_size
        # simply, we pluck out rows and then arrange them in a BXTXC, they are interpreted as logits:(scores for next characters in the sequence)
        # currently the tokens are not talking to each other / not seeing any context, 
        # we're predicting what comes next just based on the individual identity of a single token
        # currently, a token knows what comes next just by knowing what token it is cuz certain characters follow other characters 

        # by running the above cell, we got the predictions/scores for every one of the 4x8 positions about what comes next
        # now we want to evaluate the loss function: use negative log likelihood loss (in makemore), in pytorch nll is under the name cross_entropy

        B,T,C = logits.shape #unpack the numbers
        logits = logits.view(B*T, C) #2-D array
        # stretch out all positions in a 1-dimensional sequence and preserve channel dimension as second dimension, so we stretched it out into 2 dimensions
        # we also have to do the same with targets, cuz currently targets are of shape BxT but we want it to be 1D i.e. B*T
        targets = targets.view(B*T) #targets.view(-1) also works as pytorch will figure out the same, but better to be as explicit as possible
        
        loss = F.cross_entropy(logits, targets) #measures quality of logits w.r.t target, i.e. how well are we predicting the next character based on logits
        # optimally, the correct dimension of logits (based on target) should be very high, and all other dimensions should be very low 
        # but obviously, this won't be so initially
        return logits, loss

# m = BigramLanguageModel(vocab_size)
# logits, loss = m(xb, yb)
# print(logits.shape, loss)

# why did we get the error? if you look at pytorch documentation
# https://docs.pytorch.org/docs/2.13/generated/torch.nn.functional.cross_entropy.html
# it expects inputs in the form of (N,C,d1,d2,...,dK), i.e. channels in middle, but we instead have BxTxC
# so we need to reshape our logits

m = BigramLanguageModel(vocab_size)
logits, loss = m(xb, yb)
print(logits.shape, loss)

# we saw how to evaluate quality of model, now we want to see how to generate from the model
import torch
import torch.nn as nn
from torch.nn import functional as F
torch.manual_seed(1337)

class BigramLanguageModel(nn.Module):

    def __init__(self, vocab_size):
        super().__init__()
        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)

    def forward(self, idx, targets=None):

        logits = self.token_embedding_table(idx) # (B,T,C) #when we pass idx here, 
        if targets is None:
            loss = None
        else:
            B,T,C = logits.shape #unpack the numbers
            logits = logits.view(B*T, C) #2-D array
            targets = targets.view(B*T) #targets.view(-1) also works as pytorch will figure out the same, but better to be as explicit as possible
            loss = F.cross_entropy(logits, targets)
        return logits, loss

    def generate(self, idx, max_new_tokens):

        # idx is (B,T) array of indices in the current context of some characters in a batch
        for _ in range(max_new_tokens):
            logits, loss = self(idx) #get the predictions
            logits = logits[:, -1, :] #focus only on the last time step # becomes (B,C)
            probs = F.softmax(logits, dim=1) #apply softmax to get probabilities # (B,C)
            idx_next = torch.multinomial(probs, num_samples=1) #sample from the distribution #(B,1)
            idx = torch.cat((idx, idx_next), dim=1) #append sampled index to the running sequence #(B, T+1)

        return idx

m = BigramLanguageModel(vocab_size)
logits, loss = m(xb, yb)
print(logits.shape, loss)
# print(decode(m.generate(torch.zeros((1,1), dtype=torch.long), max_new_tokens=100)[0].tolist()))

# idx = torch.zeros((1,1), dtype=torch.long) #creating a 1x1 tensor with batch and time 1-d, all values 0 (more like single value 0)
                  # note from above that 0 = \n probably, so its reasonable to feed in as the very first character in a sequence
# print(decode(m.generate(idx), max_new_tokens=100)[0].tolist())

print(decode(m.generate(torch.zeros((1,1), dtype=torch.long), max_new_tokens=100)[0].tolist()))
# since generate() works on the level of batches, we have to index into the 0th row to unplug the single batch dimension that exists
# then it gives us time steps: just a 1D array of all indices which we will convert to list from pytorch tensor
# then feed it into our decode function and convert those integers into text

# create a pytorch optimization object
# we're using the AdamW optimizer, in makemore we only used stochastic gradient descent: simplest possible optimizer: you can replace AdamW with SGD
# AdamW is much more advanced and popular than SGD
# a good learning rate is actually lower, but for very small networks, we can get away with much higher learning rates 
optimizer = torch.optim.AdamW(m.parameters(), lr=1e-3)
# the optimizer object will take the gradients and update the parameters using the gradients

batch_size = 32

# our training loop
for steps in range(19000):

    # sample a batch of data
    xb, yb = get_batch('train')

    # evaluate the loss
    logits, loss = m(xb, yb)
    optimizer.zero_grad(set_to_none=True) #zeroing out the gradients from the previous step
    loss.backward() #getting the gradients for all the parameters
    optimizer.step() #using those gradients to update our parameter

print(loss.item())

print(decode(m.generate(torch.zeros((1,1), dtype=torch.long), max_new_tokens=100)[0].tolist()))
# we're expecting this output to be somewhat reasonable

print(decode(m.generate(torch.zeros((1,1), dtype=torch.long), max_new_tokens=300)[0].tolist()))
# we're expecting this output to be somewhat reasonable

# consider the toy example:
torch.manual_seed(1337)
B,T,C = 4,8,2 #batch, time, channels(some info at each point in the sequence)
x = torch.randn(B,T,C)
x.shape

# we want x[b, t] = mean_{i<=t} x[b, i]
# bow: bag of words: a term used when we're averaging up things: we're averaging up words stored on every one of these 8 locations
xbow = torch.zeros((B,T,C))
for b in range(B):
    for t in range(T):
        xprev = x[b, :t+1] # (t, C) #previous tokens are at the batch dimension b and everything upto the t'th token
        xbow[b, t] = torch.mean(xprev, 0) #averaging out over the mean/0th dimension: over time, we'll get a 1d vector: C, which we'll store in bag of words

x[0]

xbow[0]

torch.manual_seed(42)
a = torch.ones(3,3)
b = torch.randint(0, 10, (3, 2)).float()
c = a @ b
print('a=')
print(a)
print('--')
print('b=')
print(b)
print('--')
print('c=') #3x3 x 3x2 => 3x2
print(c)

# because the rows in a's are all ones, in c notice that each element is the sum of columns of b

# that all ones is boring, instead look at torch.tril
torch.tril(torch.ones(3, 3))

# so, what happens if we use this instead?
torch.manual_seed(42)
a = torch.tril(torch.ones(3,3))
b = torch.randint(0, 10, (3, 2)).float()
c = a @ b
print('a=')
print(a)
print('--')
print('b=')
print(b)
print('--')
print('c=') #3x3 x 3x2 => 3x2
print(c)

# because some values inside the rows in a's are zeros, some values in b get ignored, 
# in c notice that each element is the sum of some columns of b

torch.manual_seed(42)
a = torch.tril(torch.ones(3,3))
a = a / torch.sum(a, 1, keepdim=True)
b = torch.randint(0, 10, (3, 2)).float()
c = a @ b
print('a=')
print(a)
print('--')
print('b=')
print(b)
print('--')
print('c=') #3x3 x 3x2 => 3x2
print(c)

# because some values inside the rows in a's are zeros, some values in b get ignored, 
# in c notice that each element is the sum of some columns of b

wei = torch.tril(torch.ones(T, T))
wei = wei / wei.sum(1, keepdim = True)
wei

# consider the toy example:
torch.manual_seed(1337)
B,T,C = 4,8,2 #batch, time, channels(some info at each point in the sequence)
x = torch.randn(B,T,C)
x.shape

# we want x[b, t] = mean_{i<=t} x[b, i]
# bow: bag of words: a term used when we're averaging up things: we're averaging up words stored on every one of these 8 locations
xbow = torch.zeros((B,T,C))
for b in range(B):
    for t in range(T):
        xprev = x[b, :t+1] # (t, C) #previous tokens are at the batch dimension b and everything upto the t'th token
        xbow[b, t] = torch.mean(xprev, 0) #averaging out over the mean/0th dimension: over time, we'll get a 1d vector: C, which we'll store in bag of words

wei = torch.tril(torch.ones(T, T))
wei = wei / wei.sum(1, keepdim = True)
xbow2 = wei @ x # (T,T) @ (B,T,C) ---> pytorch sees difference, and broadcasts by adding a Batch dimension as
# ---> (B, T, T) @ (B, T, C) --> so it'll apply matrix multiplication in all batch elements in parallely and individuall
# output ---> (B, T, C)

# torch.allclose(xbow, xbow2)
torch.allclose(xbow, xbow2, atol=1e-5)

xbow[0], xbow2[0] #should be identical

# version 3: use softmax
tril = torch.tril(torch.ones(T, T))
print(tril)
wei = torch.zeros((T, T)) #tells us: how much of each token from the past do we want to aggregate/average up
print(wei)
wei = wei.masked_fill(tril == 0, float('-inf')) #tokens from the future cannot communicate
print(wei)
wei = F.softmax(wei, dim=1) # a normalization operation, we exponentiate every single one of the values and divide by their sum
print(wei)
xbow3 = wei @ x
torch.allclose(xbow, xbow3, atol=1e-5)

# you can do weighted aggregations of your past elements by using matrix multiplication of a lower triangular
# fashion and then the elements here in the lower triangular part are telling you how much each element fuses
# into this position so we're going to use this trick now to develop the self attention block

# version 4: self-attention
torch.manual_seed(1337)
B, T, C = 4, 8, 32
x = torch.randn(B, T, C)

tril = torch.tril(torch.ones(T, T))
wei = torch.zeros((T, T))
wei = wei.masked_fill(tril == 0, float('-inf'))
wei = F.softmax(wei, dim = -1)
out = wei @ x

out.shape

tril

wei

# =============================================================================
# version 4: self-attention
# -----------------------------------------------------------------------------
# A single self-attention head, built from scratch, step by step.
#
# Shapes:
#   B = batch size          (4)
#   T = time / seq length   (8)   -> number of tokens in the sequence
#   C = channels / emb dim  (32)  -> size of each token's embedding vector
#   head_size = 16                -> output dim of this single attention head
#
# The whole point: tokens start out completely independent ("private" info in
# x). Self-attention lets them *communicate* in a data-dependent way, so each
# token can pull in information from the tokens it finds relevant.
# =============================================================================

torch.manual_seed(1337)
B, T, C = 4, 8, 32
x = torch.randn(B, T, C)

# -----------------------------------------------------------------------------
# Let's see a SINGLE head perform self-attention.
# -----------------------------------------------------------------------------
head_size = 16

# Three linear projections: key, query, value.
# bias=False because these are just matrix multiplies with fixed weights.
# Each one maps C -> head_size. Applied to x, every token in every batch
# element is projected INDEPENDENTLY and IN PARALLEL — no communication yet.
key   = nn.Linear(C, head_size, bias=False)
query = nn.Linear(C, head_size, bias=False)
value = nn.Linear(C, head_size, bias=False)

# Forward x through the key and query projections.
#   k, q : (B, T, 16)
# Every token at every position produces its own key and query vector.
k = key(x)     # B, T, 16
q = query(x)   # B, T, 16   <-- (fixed: was incorrectly `key(x)` before)

# -----------------------------------------------------------------------------
# "wei" (pronounced "way" in the notes) = raw affinities between queries & keys.
#
# We compute  q @ k.transpose(-2, -1) :
#   transpose only the LAST TWO dims (NOT the batch dim!), because we have a
#   batch dimension and must keep it intact.
#
#   (B, T, 16) @ (B, 16, T)  ->  (B, T, T)
#
# For every batch element we now get a T x T matrix of dot-product affinities
# between all pairs of tokens. Unlike earlier versions where `wei` was a
# constant (all zeros), it is now DATA-DEPENDENT: every batch element gets a
# different `wei` because it contains different tokens at different positions.
# -----------------------------------------------------------------------------
wei = q @ k.transpose(-2, -1)   # (B, T, T)

# -----------------------------------------------------------------------------
# Causal (upper-triangular) masking.
#
# Token i is only allowed to attend to tokens j <= i (the past and itself).
# We build a lower-triangular matrix of ones, then fill every "future"
# (upper-triangular) position with -inf so that softmax sends them to 0.
#
# Intuition from the notes: if I'm the 5th token, I must NOT aggregate
# anything from the 6th, 7th, or 8th token.
# -----------------------------------------------------------------------------
tril = torch.tril(torch.ones(T, T))
# wei = torch.zeros((T, T))   # (older, constant version — kept for reference)
wei = wei.masked_fill(tril == 0, float('-inf'))

# -----------------------------------------------------------------------------
# Softmax over the last dimension.
#
# Raw wei values can be negative (roughly -2 to +2) — raw affinities. We don't
# want to aggregate a NEGATIVE amount of information (e.g. -0.11) from a token;
# that's nonsense. So we exponentiate and normalize into a proper probability
# distribution that sums to 1 along the last dim.
#
# Result: for each token, a data-dependent distribution telling us how much
# information to aggregate from each of the PAST tokens (future tokens are
# already -inf -> 0 after softmax).
# -----------------------------------------------------------------------------
wei = F.softmax(wei, dim=-1)   # (B, T, T), rows sum to 1

# out = wei @ x   # (older version aggregated the raw input x)

# -----------------------------------------------------------------------------
# Value projection.
#
# We do NOT aggregate the raw x. Instead we aggregate a VALUE projection v.
#
# Think of it this way:
#   x  = a token's PRIVATE information (its identity + position).
#   q  = "here's what I'm looking for."
#   k  = "here's what I have."
#   v  = "here's what I will communicate to you if you find me interesting."
#
# So v is the thing that actually gets aggregated across tokens for this head.
# -----------------------------------------------------------------------------
v = value(x)      # B, T, 16

# -----------------------------------------------------------------------------
# Weighted aggregation.
#
# (B, T, T) @ (B, T, 16) -> (B, T, 16)
#
# Each token's output is a weighted sum of the values of the tokens it was
# allowed to attend to, with weights given by `wei`.
# Output is 16-dimensional because head_size = 16.
# -----------------------------------------------------------------------------
out = wei @ v     # B, T, 16

out.shape         # torch.Size([4, 8, 16])

# =============================================================================
# Recap of the single self-attention head:
#   1. Project x -> k, q, v (independent, parallel, no communication).
#   2. Affinities:  wei = q @ k^T        -> (B, T, T), data-dependent.
#   3. Mask the future (-inf) so tokens can't look ahead (causal / decoder).
#   4. Softmax(wei)                      -> nice distribution over past tokens.
#   5. Aggregate values:  out = wei @ v  -> (B, T, head_size).
#
# This is exactly what one head of self-attention does. In a full transformer,
# multiple heads (each with its own k/q/v and head_size) are run in parallel
# and their outputs concatenated back up to C, then passed through an MLP.
# =============================================================================

wei

wei[0]

# -----------------------------------------------------------------------------
# Causal masking (this line is what makes it a DECODER block, not an encoder).
#
#   wei = wei.masked_fill(tril == 0, float('-inf'))
#
#   - `tril` is a lower-triangular matrix of ones.
#   - `tril == 0` is True at the upper-triangular (FUTURE) positions.
#   - masked_fill sets those future positions to -inf.
#
# After softmax, exp(-inf) = 0, so future tokens get ZERO attention weight
# and each token can only attend to itself + the past. This is required for
# autoregressive language modeling, where a token must not "see" the answer
# (the future tokens it is trying to predict).
#
# ENCODER vs DECODER:
#   - DECODER block (this one): keep this line  -> causal / triangular.
#   - ENCODER block: DELETE this line           -> all tokens talk to each
#     other fully (bidirectional), e.g. for sentiment analysis where later
#     tokens legitimately help inform earlier ones.
# -----------------------------------------------------------------------------
tril = torch.tril(torch.ones(T, T))
wei = wei.masked_fill(tril == 0, float('-inf'))
# eg. in sleep stage classification, if static, we use bidirectional in most cases
# both are allowed and attention doesn't care, attention supports arbitrary connectivity between nodes

k = torch.randn(B, T, head_size)
q = torch.randn(B, T, head_size)
# wei = q @ k.transpose(-2, -1)
wei = q @ k.transpose(-2, -1) * head_size ** -0.5

k.var()

q.var()
# notice the change in output here in 2 different cases

wei.var()

torch.softmax(torch.tensor([0.1, -0.2, 0.3, -0.2, 0.5]), dim=-1)

torch.softmax(torch.tensor([0.1, -0.2, 0.3, -0.2, 0.5]) * 8, dim=-1)
# note: notice that the numbers move towards the max
# and we don't want these values to be too extreme, specially during initialization
# otherwise softmax will peak a lot
# so, scaling is used (divide by square root of head_size) to control variance at initialization

# version 4: self-attention

torch.manual_seed(1337)
B, T, C = 4, 8, 32
x = torch.randn(B, T, C)

head_size = 16

# Three linear projections: key, query, value.
key   = nn.Linear(C, head_size, bias=False)
query = nn.Linear(C, head_size, bias=False)
value = nn.Linear(C, head_size, bias=False)

# Forward x through the key and query projections.
#   k, q : (B, T, 16)
k = key(x)     # B, T, 16
q = query(x)   # B, T, 16

wei = q @ k.transpose(-2, -1)   # (B, T, T)

# Causal (upper-triangular) masking.
tril = torch.tril(torch.ones(T, T))
# wei = torch.zeros((T, T))   # (older, constant version — kept for reference)
wei = wei.masked_fill(tril == 0, float('-inf'))
wei = F.softmax(wei, dim=-1)   # (B, T, T), rows sum to 1
v = value(x)      # B, T, 16

# Weighted aggregation.
# (B, T, T) @ (B, T, 16) -> (B, T, 16)
out = wei @ v     # B, T, 16
out.shape         # torch.Size([4, 8, 16])

class BatchNorm1d:

    def __init__(self, dim, eps=1e-5, momentum = 0.1):
        self.eps = eps
        self.momentum = momentum
        self.training = True

        # parameters(trained with backprop)
        self.gamma = torch.ones(dim)
        self.beta = torch.zeros(dim)

        # buffers (trained with a running 'momentum update')
        self.running_mean = torch.zeros(dim)
        self.running_var = torch.ones(dim)

    def __call__(self, x):

        # calculate the forward pass
        if self.training:
            xmean = x.mean(0, keepdim=True) #batch mean
            xvar = x.var(0, keepdim=True, unbiased=True) #batch variance
        else:
            xmean = self.running_mean
            xvar = self.running_var

        xhat = (x - xmean) / torch.sqrt(xvar + self.eps) #normalize to unit variance
        self.out = self.gamma * xhat + self.beta

        # update the buffers
        if self.training:
            with torch.no_grad():
                self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean
                self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar
        return self.out

    def parameters(self):
        return [self.gamma, self.beta]

torch.manual_seed(1337)
module = BatchNorm1d(100)
x = torch.randn(32, 100)
x = module(x)
x.shape

x[:,0].mean(), x[:,0].std() #mean, std of one feature across all batch inputs
# it guarantees 0mean, 1s.d. for every single column, its normalizing every single column of this input

x[0,:].mean(), x[0,:].std() #mean, std of a single input from the batch, of its features
# the rows are not normalized by default

# layernorm: don't normalize the columns, normalize the rows
class LayerNorm1d:

    def __init__(self, dim, eps=1e-5, momentum = 0.1):
        self.eps = eps
        self.gamma = torch.ones(dim)
        self.beta = torch.zeros(dim)

    def __call__(self, x):
        # calculate the forward pass
        xmean = x.mean(1, keepdim=True) #batch mean
        xvar = x.var(1, keepdim=True, unbiased=True) #batch variance
        xhat = (x - xmean) / torch.sqrt(xvar + self.eps) #normalize to unit variance
        self.out = self.gamma * xhat + self.beta
        return self.out

    def parameters(self):
        return [self.gamma, self.beta]

torch.manual_seed(1337)
module = LayerNorm1d(100)
x = torch.randn(32, 100)
x = module(x)
x.shape

x[:,0].mean(), x[:,0].std()
# the columns are not normalized by default

x[0,:].mean(), x[0,:].std() 
# it guarantees 0mean, 1s.d. for every single row, its normalizing every single row of this input