sig[x_]:=1/(1+E^(-x))
hammardDot[a_,b_]:={MapThread[Dot,{a,b}]};

(*The input, given for that example*)

O1 = {{0}};

(*The weight matrices, given for that example*)

W12 = {{0.3},{-0.1},{0.2}};
W23 = {{0.4,0.1,-0.1},{0.1,0.1,-0.5}};
W34 = {{0.4,-0.1},{-0.4,0.6},{-0.2,-0.2}};

(*The biases, given for that example*)

B2 = {{1},{-0.2},{0.3}};
B3 = {{0.1},{-0.5}};
B4 = {{-0.3},{0.2},{0.7}};

(*calculate outputs (feed forward)*)

O2=N[sig[#]]&/@ (W12 . O1 +B2);

O3=N[sig[#]]&/@ (W23.O2 + B3);

O4=N[sig[#]]&/@ (W34.O3 + B4) ;

(*The correct outputs from feed forward for each layer*)

O2//MatrixForm

O3//MatrixForm

O4//MatrixForm

grad[x_]:=x*(1-x);
(*back propagation*)
(*learning rate*)
lr = 0.1;

(*Expected output*)

O5={{1},{1},{0}};

(*layer 4*)
E4 = O5 - O4;
(*sigmoid*)
gradient = N[grad[#]]&/@ O4;
deltaW34 = lr * (gradient* E4).Transpose[O3];
deltaB4=lr * (gradient* E4);

(*layer 3*)
E3 = Transpose[(W34 + deltaW34)] . E4;
(*sigmoid*)
gradient = N[grad[#]]&/@ O3
deltaW23 = lr * (gradient* E3).Transpose[O2];
deltaB3=lr * (gradient* E3);

(*layer 2*)
E2 = Transpose[(W23 + deltaW23)] . E3;
(*sigmoid*)
gradient = N[grad[#]]&/@ O2
deltaW12 = lr * (gradient* E2).Transpose[O1];
deltaB2=lr * (gradient* E2);

(*Results*)

deltaW34//MatrixForm

deltaW23//MatrixForm

deltaW12//MatrixForm

E4//MatrixForm

E3//MatrixForm

E2//MatrixForm

deltaB2//MatrixForm

deltaB3//MatrixForm

deltaB4//MatrixForm