Refactor and optimizations

app/Services/DatasetReader/LinearOrderDataSetReader.php

@@ -8,7 +8,7 @@ class LinearOrderDataSetReader implements IDataSetReader
 {
     public array $lines = [];
 
-    private array $currentLines = [];
+    private int $currentLineIndex = 0;
 
     private int $lastReadLineIndex = -1;
 
@@ -35,13 +35,13 @@ class LinearOrderDataSetReader implements IDataSetReader
 
     public function getNextLine(): ?array
     {
-        if (! isset($this->currentLines[0])) {
+        if (! isset($this->lines[$this->currentLineIndex])) {
             return null; // No more lines to read
         }
 
-        $this->lastReadLineIndex = array_search($this->currentLines[0], $this->lines, true);
+        $this->lastReadLineIndex = $this->currentLineIndex;
 
-        return array_shift($this->currentLines);
+        return $this->lines[$this->currentLineIndex++];
     }
 
     public function getInputSize(): int
@@ -53,18 +53,20 @@ class LinearOrderDataSetReader implements IDataSetReader
     {
         // Count the number of unique labels in the dataset
         $labels = array_map(fn ($line) => end($line), $this->lines);
+
         return count(array_unique($labels));
     }
 
     public function getLabels(): array
     {
         $labels = array_map(fn ($line) => end($line), $this->lines);
+
         return array_values(array_unique($labels));
     }
 
     public function reset(): void
     {
-        $this->currentLines = $this->lines;
+        $this->currentLineIndex = 0;
     }
 
     public function getLastReadLineIndex(): int

app/Services/DatasetReader/RandomOrderDataSetReader.php

@@ -8,7 +8,9 @@ class RandomOrderDataSetReader implements IDataSetReader
 {
     public array $lines = [];
 
-    private array $currentLines = [];
+    private array $currentLineIndexes = [];
+
+    private int $currentLineIndex = 0;
 
     private int $lastReadLineIndex = -1;
 
@@ -35,19 +37,14 @@ class RandomOrderDataSetReader implements IDataSetReader
 
     public function getNextLine(): ?array
     {
-        if (empty($this->currentLines)) {
+        if (! isset($this->currentLineIndexes[$this->currentLineIndex])) {
             return null; // No more lines to read
         }
-        $randomNumber = array_rand($this->currentLines);
-        $randomLine = $this->currentLines[$randomNumber];
+        $lineIndex = $this->currentLineIndexes[$this->currentLineIndex++];
 
-        // Remove the line from the current lines to avoid repetition
-        unset($this->currentLines[$randomNumber]);
+        $this->lastReadLineIndex = $lineIndex;
 
-        // Remember the index of the last read line in the full list
-        $this->lastReadLineIndex = array_search($randomLine, $this->lines, true);
-
-        return $randomLine;
+        return $this->lines[$lineIndex];
     }
 
     public function getInputSize(): int
@@ -59,18 +56,22 @@ class RandomOrderDataSetReader implements IDataSetReader
     {
         // Count the number of unique labels in the dataset
         $labels = array_map(fn ($line) => end($line), $this->lines);
+
         return count(array_unique($labels));
     }
 
     public function getLabels(): array
     {
         $labels = array_map(fn ($line) => end($line), $this->lines);
+
         return array_values(array_unique($labels));
     }
 
     public function reset(): void
     {
-        $this->currentLines = $this->lines;
+        $this->currentLineIndexes = array_keys($this->lines);
+        shuffle($this->currentLineIndexes);
+        $this->currentLineIndex = 0;
     }
 
     public function getLastReadLineIndex(): int

app/Services/IterationEventBuffer/PerceptronIterationEventBuffer.php

@@ -2,27 +2,24 @@
 
 namespace App\Services\IterationEventBuffer;
 
+use App\Events\PerceptronTrainingIteration;
+
 class PerceptronIterationEventBuffer implements IPerceptronIterationEventBuffer
 {
-    private $data;
-
-    private int $nextSizeIncreaseThreshold;
-
-    private int $underSizeIncreaseCount = 0;
+    private array $data = [];
 
     public function __construct(
         private string $sessionId,
         private string $trainingId,
-        private int $sizeIncreaseStart = 10,
-        private int $sizeIncreaseFactor = 2,
-    ) {
-        $this->data = [];
-        $this->nextSizeIncreaseThreshold = $sizeIncreaseStart;
-    }
+    ) {}
 
     public function flush(): void
     {
-        event(new \App\Events\PerceptronTrainingIteration($this->data, $this->sessionId, $this->trainingId));
+        if ($this->data === []) {
+            return;
+        }
+
+        event(new PerceptronTrainingIteration($this->data, $this->sessionId, $this->trainingId));
         $this->data = [];
     }
 
@@ -35,27 +32,24 @@ class PerceptronIterationEventBuffer implements IPerceptronIterationEventBuffer
             'weights' => $synaptic_weights,
         ];
 
-        $payload = [
-            'iterations' => [...$this->data, $iteration],
-            'trainingId' => $this->trainingId,
-        ];
-
-        if ($this->data !== [] && strlen(json_encode($payload, JSON_THROW_ON_ERROR)) > config('broadcasting.broadcast_max_payload_size')) {
-            $this->flush();
-        }
-
         $this->data[] = $iteration;
 
-        if ($this->underSizeIncreaseCount <= $this->sizeIncreaseStart) { // We can still send a single date because we are under the increase start threshold
-            $this->underSizeIncreaseCount++;
+        if ($this->data !== [] && $this->payloadExceedsLimit()) {
+            $lastIteration = array_pop($this->data);
             $this->flush();
-        } elseif (count($this->data) >= $this->nextSizeIncreaseThreshold) {
-            $this->flush();
-            $this->nextSizeIncreaseThreshold *= $this->sizeIncreaseFactor;
+            $this->data[] = $lastIteration;
+        }
 
-            if ($this->nextSizeIncreaseThreshold > config('perceptron.broadcast_iteration_size')) {
-                $this->nextSizeIncreaseThreshold = config('perceptron.broadcast_iteration_size'); // Cap the threshold to the maximum size
-            }
+        if (count($this->data) >= config('perceptron.broadcast_iteration_size')) {
+            $this->flush();
         }
     }
+
+    private function payloadExceedsLimit(): bool
+    {
+        return strlen(json_encode([
+            'iterations' => $this->data,
+            'trainingId' => $this->trainingId,
+        ], JSON_THROW_ON_ERROR)) > config('broadcasting.broadcast_max_payload_size');
+    }
 }

app/Services/IterationEventBuffer/PerceptronLimitedEpochEventBuffer.php

@@ -2,24 +2,29 @@
 
 namespace App\Services\IterationEventBuffer;
 
+use App\Events\PerceptronTrainingIteration;
+
 class PerceptronLimitedEpochEventBuffer implements IPerceptronIterationEventBuffer
 {
-    private array $data;
+    private array $data = [];
 
-    private int $underSizeIncreaseCount = 0;
+    private ?int $activeEpoch = null;
+
+    private bool $shouldBroadcastEpoch = false;
 
     public function __construct(
         private string $sessionId,
         private string $trainingId,
         private int $epochInterval,
-        private int $sizeIncreaseStart = 10,
-    ) {
-        $this->data = [];
-    }
+    ) {}
 
     public function flush(): void
     {
-        event(new \App\Events\PerceptronTrainingIteration($this->data, $this->sessionId, $this->trainingId));
+        if ($this->data === []) {
+            return;
+        }
+
+        event(new PerceptronTrainingIteration($this->data, $this->sessionId, $this->trainingId));
         $this->data = [];
     }
 
@@ -32,16 +37,28 @@ class PerceptronLimitedEpochEventBuffer implements IPerceptronIterationEventBuff
             'weights' => $synaptic_weights,
         ];
 
-        $lastEpoch = $this->data[0]['epoch'] ?? null;
-        if ($this->data && $lastEpoch !== $epoch) { // Current Epoch has changed from the last one
-            if ($lastEpoch == 1 || $lastEpoch % $this->epochInterval === 0) { // The last saved epoch need to be sent
-                $this->flush(); // Flush all data from the previous epoch
-            } else {
-                $this->data = []; // We clear the data without sending it as we are saving the next epoch data
-            }
-
-            $lastEpoch = $epoch;
+        if ($this->activeEpoch !== $epoch) {
+            $this->flush();
+            $this->activeEpoch = $epoch;
+            $this->shouldBroadcastEpoch = $epoch === 1 || $epoch % $this->epochInterval === 0;
         }
+
+        if (! $this->shouldBroadcastEpoch) {
+            return;
+        }
+
         $this->data[] = $newData;
+
+        if ($this->payloadExceedsLimit() || count($this->data) >= config('perceptron.broadcast_iteration_size')) {
+            $this->flush();
+        }
+    }
+
+    private function payloadExceedsLimit(): bool
+    {
+        return strlen(json_encode([
+            'iterations' => $this->data,
+            'trainingId' => $this->trainingId,
+        ], JSON_THROW_ON_ERROR)) > config('broadcasting.broadcast_max_payload_size');
     }
 }