Add storage monitor service

README.md

@@ -36,6 +36,10 @@ It is a good idea to leave the source tree alone and copy any edits you would li
 
 As this overlaying happens after the main build, it is easy to add extra files such as ML models and datasets to your images. You will also be able to rebuild quickly if your file overrides are made here.
 
+Any directories and files that you add into `opt/storage` will be made available in the running container at `$WORKSPACE/storage`.  
+
+This directory is monitored by `inotifywait`. Any items appearing in this directory can be automatically linked to an arbitrary number of application directories as defined in `/opt/ai-dock/storage_monitor/etc/mappings.sh`.  This is particularly useful if you need to run several applications that each need to make use of the stored files.
+
 ## Run Locally
 
 A 'feature-complete' `docker-compose.yaml` file is included for your convenience. All features of the image are included - Simply edit the environment variables in `.env`, save and then type `docker compose up`.
@@ -302,6 +306,10 @@ This script follows and prints the log files for each of the above services to s
 
 If you are logged into the container you can follow the logs by running `logtail.sh`
 
+### Storage Monitor
+
+This service detects changes to files in `$WORKSPACE_STORAGE` and creates symbolic links to the application directories defined in `/opt/ai-dock/storage_monitor/etc/mappings.sh`
+
 ## Open Ports
 
 Some ports need to be open for the services to run or for certain features of the provided software to function

build/COPY_ROOT/etc/supervisor/supervisord/conf.d/storagemonitor.conf

@@ -0,0 +1,19 @@
+[program:storagemonitor]
+command=supervisor-storagemonitor.sh
+process_name=%(program_name)s
+numprocs=1
+directory=/root
+priority=1000
+autostart=true
+startsecs=5
+startretries=3
+autorestart=unexpected
+stopsignal=TERM
+stopwaitsecs=10
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/var/log/supervisor/storagemonitor.log
+stdout_logfile_maxbytes=10MB
+stdout_logfile_backups=1
+redirect_stderr=true
+environment=PROC_NAME="%(program_name)s"

build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh

@@ -26,6 +26,7 @@ $APT_INSTALL \
     git \
     git-lfs \
     gpg \
+    inotify-tools \
     jq \
     less \
     libcap2-bin \

build/COPY_ROOT/opt/ai-dock/bin/init.sh

@@ -166,6 +166,7 @@ function init_set_workspace() {
     fi
 
     mkdir -p "${WORKSPACE}"remote/.cache
+    mkdir -p "${WORKSPACE}"storage
 
     # Determine workspace mount status
     if mountpoint "$WORKSPACE" > /dev/null 2>&1; then
@@ -200,7 +201,6 @@ function init_sync_mamba_envs() {
       # Complete the copy if not serverless
       if [[ ${SERVERLESS,,} != 'true' ]]; then
           printf "Moving mamba environments to %s...\n" "${WORKSPACE}"
-          mkdir -p ${WORKSPACE}environments
           while sleep 10; do printf "Waiting for workspace mamba sync...\n"; done &
           rsync -auSHh --stats /opt/micromamba/ "${ws_mamba_target}"
           kill $!

build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh

@@ -2,8 +2,8 @@
 
 trap cleanup EXIT
 
-LISTEN_PORT=11111
-METRICS_PORT=21111
+LISTEN_PORT=${SERVICEPORTAL_PORT_LOCAL:-11111}
+METRICS_PORT=${SERVICEPORTAL_METRICS_PORT:-21111}
 PROXY_PORT=${SERVICEPORTAL_PORT_HOST:-1111}
 # Auth is true for defined paths - See /opt/caddy/share/service_config_11111_auth
 PROXY_SECURE=true

build/COPY_ROOT/opt/ai-dock/bin/supervisor-storagemonitor.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+trap cleanup EXIT
+
+function cleanup() {
+    kill $(jobs -p) > /dev/null 2>&1
+}
+
+function start() {
+    printf "Starting storage monitor..\n"
+    exec /opt/ai-dock/storage_monitor/bin/storage-monitor.sh
+}
+
+start 2>&1

build/COPY_ROOT/opt/ai-dock/storage_monitor/bin/manage-symlinks.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+storage_dir="$1"
+stored_file="$2"
+event_type="$3"
+
+absolute_stored_file=$(realpath "$stored_file")
+subfolder=$(realpath --relative-to="$storage_dir" "$(dirname "$stored_file")")
+
+# Simplify per-image settings by keeping mappings separate
+source /opt/ai-dock/storage_monitor/etc/mappings.sh
+
+# Function to create symlinks for a given file and repository directory
+manage_symlinks() {
+    for app_directory in "${!storage_map[@]}"; do
+        if [[ "$subfolder" == "$app_directory" ]]; then
+            read -ra target_dirs <<< "${storage_map["$app_directory"]}"
+            for target_directory in "${target_dirs[@]}"; do
+                symlink_target="$target_directory/$(basename "$stored_file")"
+                symlink_target_dir="$(dirname "$symlink_target")"
+                if [[ -e "$stored_file" ]]; then
+                    # Create symlinks for existing or newly created files
+                    mkdir -p "$symlink_target_dir"
+                    ln -sf "$absolute_stored_file" "$symlink_target"
+                else
+                    # Remove symlink for deleted files
+                    rm -f "$symlink_target"
+                fi
+            done
+        fi
+    done
+}
+
+# Call the function to create or remove symlinks for the stored file
+manage_symlinks

build/COPY_ROOT/opt/ai-dock/storage_monitor/bin/storage-monitor.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+storage_dir="${WORKSPACE}storage"
+image_storage_dir="/opt/storage"
+source /opt/ai-dock/storage_monitor/etc/mappings.sh
+
+# Link files bundled in the image to $storage_dir
+if [[ -d $image_storage_dir ]]; then
+    IFS=$'\n'
+    for filepath in $(find "$image_storage_dir" -type f -name "[!.]*" ); do
+        file_name=$(basename "$filepath")
+        dir_name=$(dirname "$filepath")
+        ws_file_path=${storage_dir}/$(realpath --relative-to="$image_storage_dir" "$filepath")
+        ws_dir_name=$(dirname "$ws_file_path")
+
+        mkdir -p "$ws_dir_name"
+        ln -sf "$filepath" "$ws_file_path"
+    done
+fi
+
+# Initial pass for existing files
+find "$storage_dir" -exec bash /opt/ai-dock/storage_monitor/bin/manage-symlinks.sh "$storage_dir" {} \;
+
+# Delete any broken symlinks caused by containers sharing a volume
+for app_directory in "${!storage_map[@]}"; do
+    read -ra target_dirs <<< "${storage_map["$app_directory"]}"
+    for target_directory in "${target_dirs[@]}"; do
+        if [[ -e $target_directory ]]; then
+            find "$target_directory" -xtype l -delete
+        fi
+    done
+done
+
+# Inotify loop for future changes in $storage_dir
+inotifywait -m -r -e create -e delete -e move --format '%e %w%f' "$storage_dir" |
+while read -r changed_item
+do
+    event_type=$(echo "$changed_item" | awk '{print $1}')
+    stored_file=$(echo "$changed_item" | awk '{print $2}')
+    # Call the function to create or remove symlinks for the changed item
+    bash /opt/ai-dock/storage_monitor/bin/manage-symlinks.sh "$storage_dir" "$stored_file" "$event_type"
+done

build/COPY_ROOT/opt/ai-dock/storage_monitor/etc/mappings.sh

@@ -0,0 +1,7 @@
+# Key is relative to $WORKSPACE/storage/
+
+declare -A storage_map
+#storage_map["my/models"]="/opt/app_1/models /opt/app_2/models"
+#storage_map["my/datasets"]="/opt/app_1/datasets /opt/app_2/datasets"
+
+# Add more mappings for other repository directories as needed

build/COPY_ROOT_EXTRA/opt/storage/README

@@ -0,0 +1,7 @@
+Any files present in the container image at /opt/storage will be symlinked
+to $WORKSPACE/storage at startup.
+
+Files in $WORKSPACE/storage will then be linked/unlinked as defined in
+/opt/storage_monitor/etc/mappings.sh as they are added/moved/removed.
+
+This is where you should add models and datasets if you would like to bundle them.

docker-compose.yaml

@@ -64,13 +64,14 @@ services:
 
     ports:
         # SSH available on host machine port 2222 to avoid conflict. Change to suit
-        - ${SSH_PORT_HOST:-2222}:${SSH_PORT:-22}
-        # Redirect to Cloudflare quick tunnel
-        - ${SERVICEPORTAL_PORT_HOST:-1111}:1111
+        - ${SSH_PORT_HOST:-2222}:${SSH_PORT_LOCAL:-22}
+        # Caddy port for service portal
+        - ${SERVICEPORTAL_PORT_HOST:-1111}:${SERVICEPORTAL_PORT_HOST:-1111}
         # Rclone webserver for interactive configuration
-        - ${RCLONE_PORT_HOST:-53682}:53682
+        - ${RCLONE_PORT_HOST:-53682}:${RCLONE_PORT_HOST:-53682}
 
     environment:
+        # Important: Edit values in .env - not here.
         # Don't enclose values in quotes
         - DIRECT_ADDRESS=${DIRECT_ADDRESS:-127.0.0.1}
         - DIRECT_ADDRESS_GET_WAN=${DIRECT_ADDRESS_GET_WAN:-false}
@@ -82,4 +83,9 @@ services:
         - WEB_USER=${WEB_USER:-user}
         - WEB_PASSWORD=${WEB_PASSWORD:-password}
         - SERVERLESS=${SERVERLESS:-false}
+        - SSH_PORT_HOST=${SSH_PORT_HOST:-2222}
+        - SSH_PORT_LOCAL=${SSH_PORT_LOCAL:-22}
+        - SERVICEPORTAL_PORT_HOST=${SERVICEPORTAL_PORT_HOST:-1111}
+        - SERVICEPORTAL_PORT_LOCAL=${SERVICEPORTAL_PORT_LOCAL:-11111}
+        - SERVICEPORTAL_METRICS_PORT=${SERVICEPORTAL_METRICS_PORT:-21111}
         #- PROVISIONING_SCRIPT=https://raw.githubusercontent.com/ai-dock/base-image/main/config/provisioning/default.sh