Multilayer neuron network

app/Events/PerceptronTrainingIteration.php

@@ -38,8 +38,17 @@ class PerceptronTrainingIteration implements ShouldBroadcast
 
     public function broadcastWith(): array
     {
+        $lastIterationIndex = count($this->iterations) - 1;
+        $iterations = array_map(
+            fn (array $iteration, int $index): array => $index === $lastIterationIndex
+                ? $iteration
+                : [...$iteration, 'weights' => []],
+            $this->iterations,
+            array_keys($this->iterations),
+        );
+
         return [
-            'iterations' => $this->iterations,
+            'iterations' => $iterations,
             'trainingId' => $this->trainingId,
         ];
     }

app/Http/Controllers/PerceptronController.php

@@ -6,6 +6,7 @@ use App\Events\PerceptronInitialization;
 use App\Models\NetworksTraining\ADALINEPerceptronTraining;
 use App\Models\NetworksTraining\GradientDescentPerceptronTraining;
 use App\Models\NetworksTraining\MonoLayerPerceptronTraining;
+use App\Models\NetworksTraining\MultiLayerPerceptronTraining;
 use App\Models\NetworksTraining\SimpleBinaryPerceptronTraining;
 use App\Services\DatasetReader\IDataSetReader;
 use App\Services\DatasetReader\LinearOrderDataSetReader;
@@ -13,12 +14,11 @@ use App\Services\DatasetReader\RandomOrderDataSetReader;
 use App\Services\IterationEventBuffer\PerceptronIterationEventBuffer;
 use App\Services\IterationEventBuffer\PerceptronLimitedEpochEventBuffer;
 use App\Services\SynapticWeightsProvider\ISynapticWeightsProvider;
+use App\Services\SynapticWeightsProvider\RandomSynapticWeights;
 use App\Services\SynapticWeightsProvider\ZeroSynapticWeights;
-use Illuminate\Contracts\Queue\Job;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\DB;
-use Symfony\Contracts\EventDispatcher\Event;
-use Tests\Services\IterationEventBuffer\DullIterationEventBuffer;
+use Illuminate\Support\Facades\Validator;
 
 class PerceptronController extends Controller
 {
@@ -38,9 +38,16 @@ class PerceptronController extends Controller
                 $learningRate = 0.015;
                 $maxIterations = 150;
                 break;
-            case 'gradientdescent' || 'adaline':
+            case 'gradientdescent':
+            case 'adaline':
                 $learningRate = 0.00003;
                 break;
+            case 'monolayer':
+                $learningRate = 0.03;
+                break;
+            case 'multilayer':
+                $learningRate = 0.8;
+                break;
         }
 
         return inertia('PerceptronViewer', [
@@ -50,6 +57,7 @@ class PerceptronController extends Controller
             'minError' => $minError,
             'learningRate' => $learningRate,
             'maxIterations' => $maxIterations,
+            'maxDisplayedWeights' => config('perceptron.max_displayed_weights'),
         ]);
     }
 
@@ -106,7 +114,8 @@ class PerceptronController extends Controller
                             case 'simple':
                                 $dataset['defaultLearningRate'] = 0.015;
                                 break;
-                            case 'gradientdescent' || 'adaline':
+                            case 'gradientdescent':
+                            case 'adaline':
                                 $dataset['defaultLearningRate'] = 0.001;
                                 break;
                         }
@@ -114,6 +123,15 @@ class PerceptronController extends Controller
                     case 'table_2_11':
                         $dataset['defaultMinError'] = 0.02;
                         break;
+                    case 'table_4_12':
+                        switch ($perceptronType) {
+                            case 'multilayer':
+                                $dataset['defaultLearningRate'] = 0.8;
+                                $dataset['defaultMinError'] = 0.001;
+                                $dataset['defaultMaxIterations'] = 2000;
+                                break;
+                        }
+                        break;
                 }
                 $datasets[] = $dataset;
             }
@@ -133,7 +151,19 @@ class PerceptronController extends Controller
     {
         $startTime = microtime(true);
 
+        // Verifications
+        $validator = Validator::make($request->all(), config('perceptron.run_inputs_validation'));
+
+        if ($validator->fails()) {
+            return response()->json([
+                'message' => 'Invalid input parameters',
+                'errors' => $validator->errors(),
+            ], 400);
+        }
+
         $perceptronType = $request->input('type');
+        $hiddenLayers = $request->input('hidden_layers', 2);
+        $hiddenLayersNeurons = $request->input('hidden_layers_neurons', 3);
         $minError = $request->input('min_error', 0.01);
         $weightInitMethod = $request->input('weight_init_method', 'random');
         $dataSet = $request->input('dataset');
@@ -145,7 +175,11 @@ class PerceptronController extends Controller
         // Remove the jobs for the sessionId
         DB::table('jobs')->where('payload', 'like', '%s:9:\"sessionId\";s:40:\"'. $sessionId .'\";%')->delete();
 
-        if ($weightInitMethod === 'zeros') {
+        // Zero initialization prevents hidden layers from receiving a gradient.
+        if ($perceptronType === 'multilayer' && $weightInitMethod === 'zeros') {
+            $synapticWeightsProvider = new RandomSynapticWeights;
+        }
+        else if ($weightInitMethod === 'zeros') {
             $synapticWeightsProvider = new ZeroSynapticWeights;
         }
 
@@ -162,6 +196,7 @@ class PerceptronController extends Controller
             'gradientdescent' => new GradientDescentPerceptronTraining($datasetReader, $learningRate, $maxEpochs, $synapticWeightsProvider, $iterationEventBuffer, $sessionId, $trainingId, $minError),
             'adaline' => new ADALINEPerceptronTraining($datasetReader, $learningRate, $maxEpochs, $synapticWeightsProvider, $iterationEventBuffer, $sessionId, $trainingId, $minError),
             'monolayer' => new MonoLayerPerceptronTraining($datasetReader, $learningRate, $maxEpochs, $synapticWeightsProvider, $iterationEventBuffer, $sessionId, $trainingId, $minError),
+            'multilayer' => new MultiLayerPerceptronTraining($datasetReader, $learningRate, $maxEpochs, $hiddenLayers, $hiddenLayersNeurons, $synapticWeightsProvider, $iterationEventBuffer, $sessionId, $trainingId, $minError),
             default => null,
         };
 

app/Models/NetworksTraining/MonoLayerPerceptronTraining.php

@@ -7,8 +7,6 @@ use App\Models\ActivationsFunctions;
 use App\Models\Perceptrons\GradientDescentPerceptron;
 use App\Models\Perceptrons\NetworkPerceptron;
 use App\Models\Perceptrons\Perceptron;
-use App\Models\Perceptrons\SimpleBinaryPerceptron2;
-use App\Models\Perceptrons\SimpleBinaryPerceptron;
 use App\Services\DatasetReader\IDataSetReader;
 use App\Services\IterationEventBuffer\IPerceptronIterationEventBuffer;
 use App\Services\SynapticWeightsProvider\ISynapticWeightsProvider;
@@ -47,7 +45,7 @@ class MonoLayerPerceptronTraining extends NetworkTraining
             ),
             $datasetReader->getInputSize(),
             GradientDescentPerceptron::class, // No hidden layer
-            SimpleBinaryPerceptron2::class,
+            GradientDescentPerceptron::class,
         );
         $this->labels = $datasetReader->getLabels();
     }

app/Models/NetworksTraining/MultiLayerPerceptronTraining.php

@@ -0,0 +1,259 @@
+<?php
+
+namespace App\Models\NetworksTraining;
+
+use App\Events\PerceptronTrainingEnded;
+use App\Models\ActivationsFunctions;
+use App\Models\Perceptrons\GradientDescentPerceptron;
+use App\Models\Perceptrons\NetworkPerceptron;
+use App\Models\Perceptrons\Perceptron;
+use App\Models\Perceptrons\SigmoidPerceptron;
+use App\Services\DatasetReader\IDataSetReader;
+use App\Services\IterationEventBuffer\IPerceptronIterationEventBuffer;
+use App\Services\SynapticWeightsProvider\ISynapticWeightsProvider;
+use App\Services\SynapticWeightsProvider\SimpleNetworkWeightsProvider;
+use Illuminate\Support\Arr;
+
+class MultiLayerPerceptronTraining extends NetworkTraining
+{
+    private Perceptron $network;
+
+    private array $labels;
+
+    private bool $isRegression;
+
+    public ActivationsFunctions $activationFunction = ActivationsFunctions::SIGMOID;
+    public ?ActivationsFunctions $presentationLayerActivationFunction = ActivationsFunctions::STEP;
+
+    private float $epochError;
+
+    public function __construct(
+        IDataSetReader $datasetReader,
+        protected float $learningRate,
+        int $maxEpochs,
+        protected int $hiddenLayers,
+        protected int $hiddenLayersNeurons,
+        ISynapticWeightsProvider $synapticWeightsProvider,
+        IPerceptronIterationEventBuffer $iterationEventBuffer,
+        string $sessionId,
+        string $trainingId,
+        private float $minError,
+    ) {
+        parent::__construct($datasetReader, $maxEpochs, $iterationEventBuffer, $sessionId, $trainingId);
+        $this->labels = $datasetReader->getLabels();
+        $this->isRegression = $datasetReader->getOutputSize() === 1
+            || ($datasetReader->getOutputSize() > 2
+                && $datasetReader->getOutputSize() * 2 >= $datasetReader->getEpochExamplesCount());
+        if ($this->isRegression) {
+            $this->activationFunction = ActivationsFunctions::LINEAR;
+        }
+
+        $networkWeightsProvider = new SimpleNetworkWeightsProvider($synapticWeightsProvider);
+        $this->network = new NetworkPerceptron(
+            $networkWeightsProvider->generate(
+                $datasetReader->getInputSize(),
+                $this->isRegression ? 1 : $datasetReader->getOutputSize(),
+                $this->hiddenLayers,
+                $this->hiddenLayersNeurons,
+            ),
+            $datasetReader->getInputSize(),
+            SigmoidPerceptron::class,
+            GradientDescentPerceptron::class,
+        );
+    }
+
+    public function start(): void
+    {
+        $this->epoch = 0;
+        do {
+            $this->epochError = 0;
+            $this->epoch++;
+
+            $inputsForCurrentEpoch = [];
+
+            while ($nextRow = $this->datasetReader->getNextLine()) {
+                $inputsForCurrentEpoch[] = $nextRow;
+                $inputs = array_slice($nextRow, 0, -1);
+                $correctOutput = (float) end($nextRow);
+
+                $iterationError = $this->iterationFunction($inputs, $correctOutput);
+
+                // Synaptic weights correction after each example
+                $synaptic_weights = $this->network->getSynapticWeights();
+                $inputs_with_bias = array_merge([1], $inputs); // Add bias input
+
+                // Updates the weights
+                $this->network->setSynapticWeights(
+                    $this->getUpdatedSynapticWeights($synaptic_weights, $iterationError, $inputs_with_bias)
+                );
+
+                // Broadcast the training iteration event
+                $this->addIterationToBuffer(array_sum($iterationError), $this->network->getSynapticWeights());
+
+                // $this->iterationEventBuffer->flush();
+            }
+
+            // Calculte the average error for the epoch with the last synaptic weights
+            foreach ($inputsForCurrentEpoch as $inputsWithLabel) {
+                $inputs = array_slice($inputsWithLabel, 0, -1);
+                $correctOutput = (float) end($inputsWithLabel);
+                $iterationError = $this->iterationFunction($inputs, $correctOutput);
+                foreach ($iterationError as $error) {
+                    $this->epochError += ($error ** 2) / 2; // Squared error for the example
+                }
+            }
+            $this->epochError /= $this->datasetReader->getEpochExamplesCount(); // Average error for the epoch
+
+            $this->datasetReader->reset(); // Reset the dataset for the next iteration
+        } while ($this->epoch < $this->maxEpochs && ! $this->stopCondition());
+
+        $this->iterationEventBuffer->flush(); // Ensure all iterations are sent to the frontend
+
+        $this->checkPassedMaxIterations($this->epochError);
+    }
+
+    protected function stopCondition(): bool
+    {
+        $condition = $this->epochError <= $this->minError;
+        if ($condition === true) {
+            event(new PerceptronTrainingEnded('Le perceptron à atteint l\'erreur minimale', $this->sessionId, $this->trainingId));
+        }
+
+        return $condition;
+    }
+
+    private function iterationFunction(array $inputs, float $correctOutput): array
+    {
+        $outputs = $this->network->test($inputs);
+        $desiredOutput = $this->getDesiredOutputFromCorrectOutput($correctOutput);
+
+        $errors = [];
+        foreach ($outputs as $index => $output) {
+            $error = $desiredOutput[$index] - $output;
+            $errors[] = $error;
+        }
+
+        return $errors;
+    }
+
+    /**
+     * Backpropagation  of error gradients to update synaptic weights.
+     *
+     */
+    private function getUpdatedSynapticWeights(array $synaptic_weights, array $iterationError, array $inputs): array
+    {
+        $layerInputs = [$inputs];
+
+        // Reproduce NetworkPerceptron::test() using each neuron's activation function.
+        foreach ($synaptic_weights as $layerIndex => $layerWeights) {
+            $previousLayerOutputs = $layerInputs[array_key_last($layerInputs)];
+            $layerOutputs = [];
+
+            foreach ($layerWeights as $neuronIndex => $neuronWeights) {
+                $weightedSum = array_sum(array_map(
+                    fn ($input, $weight): float => $input * $weight,
+                    $previousLayerOutputs,
+                    $neuronWeights,
+                ));
+
+                $neuron = $this->network->network[$layerIndex + 1][$neuronIndex];
+                $layerOutputs[] = $neuron->activationFunction($weightedSum);
+            }
+
+            $layerInputs[] = array_merge([1], $layerOutputs);
+        }
+
+        $deltas = array_fill(0, count($synaptic_weights), []);
+        $lastLayerIndex = count($synaptic_weights) - 1;
+
+        // Output delta includes the output neuron's activation derivative.
+        foreach ($iterationError as $neuronIndex => $error) {
+            $neuron = $this->network->network[$lastLayerIndex + 1][$neuronIndex];
+            $output = $layerInputs[$lastLayerIndex + 1][$neuronIndex + 1];
+
+            $deltas[$lastLayerIndex][$neuronIndex] =
+                $error * $this->activationDerivative($neuron, $output);
+        }
+
+        // Hidden-layer deltas use the original, unchanged weights.
+        for ($layerIndex = $lastLayerIndex - 1; $layerIndex >= 0; $layerIndex--) {
+            foreach ($synaptic_weights[$layerIndex] as $neuronIndex => $unusedNeuronWeights) {
+                $nextLayerDelta = 0.0;
+
+                foreach ($synaptic_weights[$layerIndex + 1] as $nextNeuronIndex => $nextNeuronWeights) {
+                    // Index zero is the next layer's bias weight.
+                    $nextLayerDelta +=
+                        $nextNeuronWeights[$neuronIndex + 1]
+                        * $deltas[$layerIndex + 1][$nextNeuronIndex];
+                }
+
+                $neuron = $this->network->network[$layerIndex + 1][$neuronIndex];
+                $output = $layerInputs[$layerIndex + 1][$neuronIndex + 1];
+
+                $deltas[$layerIndex][$neuronIndex] =
+                    $nextLayerDelta * $this->activationDerivative($neuron, $output);
+            }
+        }
+
+        $updatedWeights = [];
+
+        foreach ($synaptic_weights as $layerIndex => $layerWeights) {
+            $updatedLayerWeights = [];
+
+            foreach ($layerWeights as $neuronIndex => $neuronWeights) {
+                $updatedLayerWeights[] = array_map(
+                    fn ($weight, $weightIndex): float => $weight
+                        + $this->learningRate
+                        * $deltas[$layerIndex][$neuronIndex]
+                        * $layerInputs[$layerIndex][$weightIndex],
+                    $neuronWeights,
+                    array_keys($neuronWeights),
+                );
+            }
+
+            $updatedWeights[] = $updatedLayerWeights;
+        }
+
+        return $updatedWeights;
+    }
+
+    private function activationDerivative(Perceptron $neuron, float $output): float
+    {
+        // Numerical derivative works for any activationFunction implementation.
+        // $epsilon = 1e-6;
+
+        // The activation function requires weighted input, which is unavailable here.
+        // For sigmoid, use its derivative directly.
+        if ($neuron instanceof SigmoidPerceptron) {
+            return $output * (1 - $output);
+        }
+
+        // Linear activation, used commonly by gradient-descent output neurons.
+        return 1.0;
+    }
+
+    private function getDesiredOutputFromCorrectOutput(float $correctOutput): array
+    {
+        if ($this->isRegression) {
+            return [$correctOutput];
+        }
+
+        $desiredOutput = array_fill(0, count($this->labels), 0);
+        $labelIndex = Arr::first(
+            array_keys($this->labels),
+            fn ($key) => $this->labels[$key] == $correctOutput
+        );
+
+        if ($labelIndex !== null) {
+            $desiredOutput[$labelIndex] = 1;
+        }
+
+        return $desiredOutput;
+    }
+
+    public function getSynapticWeights(): array
+    {
+        return [[$this->network->getSynapticWeights()]];
+    }
+}
+

app/Models/Perceptrons/NetworkPerceptron.php

@@ -25,7 +25,7 @@ class NetworkPerceptron extends Perceptron
         }
 
         // Hidden Layer
-        for ($layerIndex = 0; $layerIndex < count($synaptic_weights) - 2; $layerIndex++) {
+        for ($layerIndex = 0; $layerIndex < count($synaptic_weights) - 1; $layerIndex++) {
             $this->network[$layerIndex + 1] = [];
 
             foreach ($synaptic_weights[$layerIndex] as $neuronWeights) {

app/Models/Perceptrons/SigmoidPerceptron.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Models\Perceptrons;
+
+class SigmoidPerceptron extends Perceptron
+{
+    public function activationFunction(float $weighted_sum): float
+    {
+        $weighted_sum = max(-40, min(40, $weighted_sum));
+
+        return 1 / (1 + exp(-$weighted_sum));
+    }
+}

app/Services/IterationEventBuffer/PerceptronIterationEventBuffer.php

@@ -28,13 +28,24 @@ class PerceptronIterationEventBuffer implements IPerceptronIterationEventBuffer
 
     public function addIteration(int $epoch, int $exampleIndex, float $error, array $synaptic_weights): void
     {
-        $this->data[] = [
+        $iteration = [
             'epoch' => $epoch,
             'exampleIndex' => $exampleIndex,
             'error' => $error,
             'weights' => $synaptic_weights,
         ];
 
+        $payload = [
+            'iterations' => [...$this->data, $iteration],
+            'trainingId' => $this->trainingId,
+        ];
+
+        if ($this->data !== [] && strlen(json_encode($payload, JSON_THROW_ON_ERROR)) > config('broadcasting.broadcast_max_payload_size')) {
+            $this->flush();
+        }
+
+        $this->data[] = $iteration;
+
         if ($this->underSizeIncreaseCount <= $this->sizeIncreaseStart) { // We can still send a single date because we are under the increase start threshold
             $this->underSizeIncreaseCount++;
             $this->flush();

app/Services/SynapticWeightsProvider/SimpleNetworkWeightsProvider.php

@@ -18,9 +18,11 @@ class SimpleNetworkWeightsProvider implements INetworkSynapticWeightsProvider
 
         // Generate Hidden Layer weights
         for ($hiddenLayerNeuronIndex = 0; $hiddenLayerNeuronIndex < $hidden_layers_count; $hiddenLayerNeuronIndex++) {
+            $layer = [];
             for ($neuronIndex = 0; $neuronIndex < $hidden_layers_neurons_count; $neuronIndex++) {
-                $synaptic_weights[] = $this->synapticWeightsProvider->generate($lastLayerSize);
+                $layer[] = $this->synapticWeightsProvider->generate($lastLayerSize);
             }
+            $synaptic_weights[] = $layer;
             $lastLayerSize = $hidden_layers_neurons_count;
         }