Merge pull request #97 from open-runtimes/feat-use-bridge-networks

feat: executor to manage its own networks
open-runtimes · Aug 30, 2024 · c6077f4 · c6077f4
2 parents 80b4ea6 + fcedcb7
commit c6077f4
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 40 deletions.
diff --git a/.env b/.env
@@ -4,9 +4,10 @@ OPR_EXECUTOR_CONNECTION_STORAGE=
 OPR_EXECUTOR_INACTIVE_TRESHOLD=60
 OPR_EXECUTOR_MAINTENANCE_INTERVAL=60
 OPR_EXECUTOR_NETWORK=openruntimes-runtimes
+OPR_EXECUTOR_IMAGE=executor-local
 OPR_EXECUTOR_SECRET=executor-secret-key
 OPR_EXECUTOR_LOGGING_PROVIDER=
 OPR_EXECUTOR_LOGGING_CONFIG=
 OPR_EXECUTOR_DOCKER_HUB_USERNAME=
 OPR_EXECUTOR_DOCKER_HUB_PASSWORD=
-OPR_EXECUTOR_RUNTIME_VERSIONS=v4
+OPR_EXECUTOR_RUNTIME_VERSIONS=v4
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -29,4 +29,4 @@ jobs:
 
     - name: Run Tests
       run: |
-        docker run --rm -v $PWD:/app -v  /var/run/docker.sock:/var/run/docker.sock --network openruntimes-runtimes -w /app phpswoole/swoole:5.1.2-php8.3-alpine sh -c "apk update && apk add docker-cli && composer install --profile --ignore-platform-reqs && composer test"
+        docker run --rm -v $PWD:/app -v  /var/run/docker.sock:/var/run/docker.sock --network openruntimes-runtimes -w /app phpswoole/swoole:5.1.2-php8.3-alpine sh -c "apk update && apk add docker-cli && composer install --profile --ignore-platform-reqs && composer test"
diff --git a/app/http.php b/app/http.php
@@ -134,7 +134,7 @@
  * Create a Swoole table of usage stats (separate for host and containers)
  */
 $register->set('statsContainers', function () {
-    $table = new Table(1024);
+    $table = new Table(4096);
 
     $table->column('usage', Table::TYPE_FLOAT, 8);
     $table->create();
@@ -143,24 +143,24 @@
 });
 
 $register->set('statsHost', function () {
-    $table = new Table(1024);
+    $table = new Table(4096);
 
     $table->column('usage', Table::TYPE_FLOAT, 8);
     $table->create();
 
     return $table;
 });
 
+
 /** Set Resources */
+Http::setResource('log', fn () => new Log());
 Http::setResource('register', fn () => $register);
 Http::setResource('orchestration', fn (Registry $register) => $register->get('orchestration'), ['register']);
 Http::setResource('activeRuntimes', fn (Registry $register) => $register->get('activeRuntimes'), ['register']);
 Http::setResource('logger', fn (Registry $register) => $register->get('logger'), ['register']);
 Http::setResource('statsContainers', fn (Registry $register) => $register->get('statsContainers'), ['register']);
 Http::setResource('statsHost', fn (Registry $register) => $register->get('statsHost'), ['register']);
 
-Http::setResource('log', fn () => new Log());
-
 function logError(Log $log, Throwable $error, string $action, Logger $logger = null, Route $route = null): void
 {
     Console::error('[Error] Type: ' . get_class($error));
@@ -285,20 +285,79 @@ function getStorageDevice(string $root): Device
     }
 }
 
-function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration): void
+
+/**
+ * @param array<string> $networks
+ *
+ * @return array<string>
+ */
+function createNetworks(Orchestration $orchestration, array $networks): array
+{
+    $jobs = [];
+    $createdNetworks = [];
+    foreach ($networks as $network) {
+        $jobs[] = function () use ($orchestration, $network, &$createdNetworks) {
+            if (!$orchestration->networkExists($network)) {
+                try {
+                    $orchestration->createNetwork($network, false);
+                    Console::success("Created network: $network");
+                    $createdNetworks[] = $network;
+                } catch (Exception $e) {
+                    Console::error("Failed to create network $network: " . $e->getMessage());
+                }
+            } else {
+                Console::info("Network $network already exists");
+                $createdNetworks[] = $network;
+            }
+        };
+    }
+    batch($jobs);
+
+    $image = Http::getEnv('OPR_EXECUTOR_IMAGE');
+    if (empty($image)) {
+        throw new \Exception('Variable OPR_EXECUTOR_IMAGE is not set');
+    }
+
+    $containers = $orchestration->list(['label' => "com.openruntimes.executor.image=$image"]);
+
+    if (count($containers) < 1) {
+        $containerName = '';
+        Console::warning('No matching executor found. Please check the value of OPR_EXECUTOR_IMAGE. Executor will need to be connected manually.');
+    } else {
+        $containerName = $containers[0]->getName();
+        Console::success('Found matching executor. Executor will be connected automatically.');
+    }
+
+    if (!empty($containerName)) {
+        foreach ($createdNetworks as $network) {
+            try {
+                $orchestration->networkConnect($containerName, $network);
+                Console::success("Successfully connected executor '$containerName' to network '$network'");
+            } catch (Exception $e) {
+                Console::error("Failed to connect executor '$containerName' to network '$network': " . $e->getMessage());
+            }
+        }
+    }
+
+    return $createdNetworks;
+}
+
+/**
+ * @param array<string> $networks
+ */
+function cleanUp(Orchestration $orchestration, Table $activeRuntimes, array $networks = []): void
 {
-    Console::log('Cleaning up containers...');
+    Console::log('Cleaning up containers and networks...');
 
     $functionsToRemove = $orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]);
 
     if (\count($functionsToRemove) === 0) {
         Console::info('No containers found to clean up.');
     }
 
-    $callables = [];
-
+    $jobsRuntimes = [];
     foreach ($functionsToRemove as $container) {
-        $callables[] = function () use ($container, $activeRuntimes, $orchestration) {
+        $jobsRuntimes[] = function () use ($container, $activeRuntimes, $orchestration) {
             try {
                 $orchestration->remove($container->getId(), true);
 
@@ -315,8 +374,20 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
             }
         };
     }
+    batch($jobsRuntimes);
 
-    batch($callables);
+    $jobsNetworks = [];
+    foreach ($networks as $network) {
+        $jobsNetworks[] = function () use ($orchestration, $network) {
+            try {
+                $orchestration->removeNetwork($network);
+                Console::success("Removed network: $network");
+            } catch (Exception $e) {
+                Console::error("Failed to remove network $network: " . $e->getMessage());
+            }
+        };
+    }
+    batch($jobsNetworks);
 
     Console::success('Cleanup finished.');
 }
@@ -413,11 +484,12 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
     ->param('memory', 512, new Integer(), 'Comtainer RAM memory.', true)
     ->param('version', 'v4', new WhiteList(['v2', 'v4']), 'Runtime Open Runtime version.', true)
     ->param('restartPolicy', DockerAPI::RESTART_NO, new WhiteList([DockerAPI::RESTART_NO, DockerAPI::RESTART_ALWAYS, DockerAPI::RESTART_ON_FAILURE, DockerAPI::RESTART_UNLESS_STOPPED], true), 'Define restart policy for the runtime once an exit code is returned. Default value is "no". Possible values are "no", "always", "on-failure", "unless-stopped".', true)
+    ->inject('networks')
     ->inject('orchestration')
     ->inject('activeRuntimes')
     ->inject('response')
     ->inject('log')
-    ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) {
+    ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, array $networks, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) {
         $runtimeName = System::getHostname() . '-' . $runtimeId;
 
         $runtimeHostname = \uniqid();
@@ -513,8 +585,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
             $codeMountPath = $version === 'v2' ? '/usr/code' : '/mnt/code';
             $workdir = $version === 'v2' ? '/usr/code' : '';
 
-            $openruntimes_networks = explode(',', str_replace(' ', '', Http::getEnv('OPR_EXECUTOR_NETWORK') ?: 'executor_runtimes'));
-            $openruntimes_network = $openruntimes_networks[array_rand($openruntimes_networks)];
+            $network = $networks[array_rand($networks)];
 
             $volumes = [
                 \dirname($tmpSource) . ':/tmp:rw',
@@ -537,7 +608,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
                     'openruntimes-runtime-id' => $runtimeId
                 ],
                 volumes: $volumes,
-                network: \strval($openruntimes_network) ?: 'executor_runtimes',
+                network: \strval($network),
                 workdir: $workdir,
                 restart: $restartPolicy
             );
@@ -1097,8 +1168,8 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix
                 $logs = '';
                 $errors = '';
                 if (!empty($fileId)) {
-                    $logFile = '/tmp/'.$runtimeName .'/logs/' . $fileId . '_logs.log';
-                    $errorFile = '/tmp/'.$runtimeName .'/logs/' . $fileId . '_errors.log';
+                    $logFile = '/tmp/' . $runtimeName . '/logs/' . $fileId . '_logs.log';
+                    $errorFile = '/tmp/' . $runtimeName . '/logs/' . $fileId . '_errors.log';
 
                     $logDevice = new Local();
 
@@ -1348,14 +1419,21 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix
     $activeRuntimes = $register->get('activeRuntimes');
     $statsHost = $register->get('statsHost');
 
-    /**
-     * Remove residual runtimes
-     */
-    Console::info('Removing orphan runtimes...');
+    $networks = explode(',', Http::getEnv('OPR_EXECUTOR_NETWORK') ?: 'openruntimes-runtimes');
 
-    removeAllRuntimes($activeRuntimes, $orchestration);
+    /*
+     * Remove residual runtimes and networks
+     */
+    Console::info('Removing orphan runtimes and networks...');
+    cleanUp($orchestration, $activeRuntimes);
+    Console::success("Orphan runtimes and networks removal finished.");
 
-    Console::success("Orphan runtimes removal finished.");
+    /**
+     * Create and store Docker Bridge networks used for communication between executor and runtimes
+     */
+    Console::info('Creating networks...');
+    $createdNetworks = createNetworks($orchestration, $networks);
+    Http::setResource('networks', fn () => $createdNetworks);
 
     /**
      * Warmup: make sure images are ready to run fast 🚀
@@ -1500,10 +1578,10 @@ function getStats(Table $statsHost, Table $statsContainers, Orchestration $orche
 
     Console::success('Executor is ready.');
 
-    Process::signal(SIGINT, fn () => removeAllRuntimes($activeRuntimes, $orchestration));
-    Process::signal(SIGQUIT, fn () => removeAllRuntimes($activeRuntimes, $orchestration));
-    Process::signal(SIGKILL, fn () => removeAllRuntimes($activeRuntimes, $orchestration));
-    Process::signal(SIGTERM, fn () => removeAllRuntimes($activeRuntimes, $orchestration));
+    Process::signal(SIGINT, fn () => cleanUp($orchestration, $activeRuntimes, $networks));
+    Process::signal(SIGQUIT, fn () => cleanUp($orchestration, $activeRuntimes, $networks));
+    Process::signal(SIGKILL, fn () => cleanUp($orchestration, $activeRuntimes, $networks));
+    Process::signal(SIGTERM, fn () => cleanUp($orchestration, $activeRuntimes, $networks));
 
     $http->start();
 });
diff --git a/composer.json b/composer.json
@@ -35,7 +35,7 @@
         "utopia-php/registry": "0.5.*",
         "utopia-php/preloader": "0.2.*",
         "utopia-php/system": "0.8.*",
-        "utopia-php/orchestration": "0.13.*",
+        "utopia-php/orchestration": "0.14.*",
         "appwrite/php-runtimes": "0.15.*"
     },
     "require-dev": {

diff --git a/composer.lock b/composer.lock
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -12,20 +12,23 @@ x-logging: &x-logging
 
 services:
   openruntimes-executor:
-    container_name: openruntimes-executor
     hostname: executor
     <<: *x-logging
     stop_signal: SIGINT
     build:
       context: .
-    networks:
-      openruntimes-runtimes:
     ports:
       - 9900:80
+    labels:
+      com.openruntimes.executor.image: $OPR_EXECUTOR_IMAGE
+    networks:
+      - openruntimes-runtimes
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./app:/usr/local/app:rw
       - ./src:/usr/local/src:rw
+      - ./tests:/usr/local/tests:rw
+      - ./phpunit.xml:/usr/local/phpunit.xml
       - openruntimes-builds:/storage/builds:rw
       - openruntimes-functions:/storage/functions:rw
       - /tmp:/tmp:rw
@@ -38,6 +41,7 @@ services:
       - OPR_EXECUTOR_MAINTENANCE_INTERVAL
       - OPR_EXECUTOR_NETWORK
       - OPR_EXECUTOR_SECRET
+      - OPR_EXECUTOR_IMAGE
       - OPR_EXECUTOR_LOGGING_PROVIDER
       - OPR_EXECUTOR_LOGGING_CONFIG
       - OPR_EXECUTOR_DOCKER_HUB_USERNAME
@@ -50,4 +54,4 @@ networks:
 
 volumes:
   openruntimes-builds:
-  openruntimes-functions:
+  openruntimes-functions: