Merge pull request #53 from maffettone/fix-multi-profile-and-publisher

Fix: variable profile and keep publishers from startup

README.md

@@ -20,6 +20,9 @@ To get a bluesky terminal in this pod run
 bash launch_bluesky.sh
 ```
 
+Setting the environment variable `BLUESKY_PROFILE_DIR` to an ipython profile will allow you to use a custom profile in the `launch_bluesky.sh` script or the Queue Server container.
+In both cases, the RunEngine (RE), databroker, and Kafka subscriptions must be initialized in the startup profile.
+
 To get a default QT gui for the queue server run
 ```sh
 bash launch_bluesky.sh bluesky queue-monitor
@@ -45,6 +48,7 @@ instance running the pod which are proxied via nginx.  If the pod is running
   containers can be accessed via DNS with their names.
 
 
+# Contents
 
 ## Get podman
 
@@ -86,6 +90,9 @@ X11UseLocalhost no
 If podman is running on the machine you are sitting in front of, or if you would like
 to run in "headless" mode, no action is required.
 
+## Repository Contents
+
+
 ## Other examples
 
 - [Wright Group bluesky-in-a-box](https://github.com/wright-group/bluesky-in-a-box)

bluesky_config/ipython/profile_default/startup/00-base.py

@@ -1,32 +1,29 @@
-import logging
 import json
+import logging
 from functools import partial
 from queue import Empty
 
+import bluesky.plans as bp
+import databroker
+import happi
+import happi.loader
 import IPython
 import matplotlib.pyplot as plt
-
-import redis
 import msgpack
 import msgpack_numpy as mpn
-
+import redis
 from bluesky import RunEngine
-import bluesky.plans as bp
-
 from bluesky.callbacks.best_effort import BestEffortCallback
 from bluesky.callbacks.zmq import Publisher as zmqPublisher
+from bluesky.plans import *
 from bluesky_kafka import Publisher as kafkaPublisher
 
-from bluesky_adaptive.per_start import adaptive_plan
-
-import databroker
-import happi
-import happi.loader
+# from bluesky_adaptive.per_start import adaptive_plan # This is incompatible with the queue-server (default args)
 
 ip = IPython.get_ipython()
 
-hclient = happi.Client(path='/usr/local/share/happi/test_db.json')
-db = databroker.catalog['MAD']
+hclient = happi.Client(path="/usr/local/share/happi/test_db.json")
+db = databroker.catalog["MAD"]
 
 RE = RunEngine()
 bec = BestEffortCallback()
@@ -97,7 +94,8 @@ def get(self, timeout=0, block=True):
 
 devs = {v.name: v for v in [happi.loader.from_container(_) for _ in hclient.all_items]}
 
-ip.user_ns.update(devs)
+if ip is not None:
+    ip.user_ns.update(devs)
 
 # do from another
 # http POST 0.0.0.0:8081/add_to_queue plan:='{"plan":"scan", "args":[["det"], "motor", -1, 1, 10]}'

bluesky_config/ipython/profile_default/startup/user_group_permissions.yaml

@@ -0,0 +1,48 @@
+user_groups:
+  root:  # Defines the rules for preliminary filtering of plan/device/function names for all groups.
+    allowed_plans:
+      - null  # Allow all
+    forbidden_plans:
+      - ":^_"  # All plans with names starting with '_'
+    allowed_devices:
+      - null  # Allow all
+    forbidden_devices:
+      - ":^_:?.*"  # All devices with names starting with '_'
+    allowed_functions:
+      - null  # Allow all
+    forbidden_functions:
+      - ":^_"  # All functions with names starting with '_'
+  primary:  # Default group. The group can be renamed or other groups may be created.
+    allowed_plans:
+      - ":.*"  # Different way to allow all plans.
+    forbidden_plans:
+      - null  # Nothing is forbidden
+    allowed_devices:
+      - ":?.*:depth=5"  # Allow all device and subdevices. Maximum deepth for subdevices is 5.
+    forbidden_devices:
+      - null  # Nothing is forbidden
+    allowed_functions:
+      - "function_sleep"  # Explicitly listed name
+      - ":^func_for_test"
+  test_user:  # Another group used for unit tests.
+    allowed_plans:
+      - ":^count"  # Use regular expression patterns
+      - ":scan$"
+    forbidden_plans:
+      - ":^adaptive_scan$" # Use regular expression patterns
+      - ":^inner_product"
+    allowed_devices:
+      - ":^det:?.*"  # Use regular expression patterns
+      - ":^motor:?.*"
+      - ":^sim_bundle_A:?.*"
+    forbidden_devices:
+      - ":^det[3-5]$:?.*" # Use regular expression patterns
+      - ":^motor\\d+$:?.*"
+    allowed_functions:
+      - ":element$"
+      - ":elements$"
+      - "function_sleep"
+      - "clear_buffer"
+      - ":^unit_test.*$"
+    forbidden_functions:
+      - ":^_"  # All functions with names starting with '_'

compose/acq-pod/docker-compose.yaml

@@ -124,10 +124,19 @@ services:
       - ../../bluesky_config/databroker:/usr/local/share/intake
 
   # QS parts
+  # Loads QServer looking at an ipython profile named profile_qserver, optionally loaded from BLUESKY_PROFILE_DIR
   queue_manager:
     image: bluesky
     build: ../bluesky
-    command: start-re-manager --kafka-server=kafka:29092 --zmq-publish-console ON --redis-addr redis
+    command: start-re-manager --startup-profile qserver --keep-re --zmq-publish-console ON --redis-addr redis
+    environment:
+      - IPYTHONDIR=/usr/local/share/ipython
+      - PYTHONPATH=/usr/local/share/ipython
+    volumes:
+      - ${BLUESKY_PROFILE_DIR:-../../bluesky_config/ipython/profile_default}:/usr/local/share/ipython/profile_qserver:ro
+      - ../../bluesky_config/databroker:/usr/local/share/intake:ro
+      - ../../bluesky_config/ipython/localdevs.py:/usr/local/share/ipython/localdevs.py:ro
+      - ../../bluesky_config/happi:/usr/local/share/happi:ro
     depends_on:
       kafka:
         condition: service_started

compose/acq-pod/launch_bluesky.sh

@@ -39,6 +39,10 @@ xauth nlist $LOCAL_DISPLAY | sed -e 's/^..../ffff/' | xauth -f $XAUTH nmerge -
 # https://stackoverflow.com/questions/24112727/relative-paths-based-on-file-location-instead-of-current-working-directory
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 
+if [ -z "BLUESKY_PROFILE_DIR" ]; then
+    echo "BLUESKY_PROFILE_DIR is set to $BLUESKY_PROFILE_DIR"
+fi
+
 if [ "$1" != "" ]; then
     imagename=$1
 else
@@ -61,7 +65,8 @@ podman run --pod pod_acq-pod  \
        -v $XAUTH:/tmp/.docker.xauth \
        -e XAUTHORITY=/tmp/.docker.xauth \
        -v `pwd`:'/app' -w '/app' \
-       -v $parent_path/../../bluesky_config/ipython:/usr/local/share/ipython \
+       -v ${BLUESKY_PROFILE_DIR:-$parent_path/../../bluesky_config/ipython/profile_default}:/usr/local/share/ipython/profile_default \
+       -v $parent_path/../../bluesky_config/ipython/localdevs.py:/usr/local/share/ipython/localdevs.py \
        -v $parent_path/../../bluesky_config/databroker:/usr/local/share/intake \
        -v $parent_path/../../bluesky_config/happi:/usr/local/share/happi \
        -e XDG_RUNTIME_DIR=/tmp/runtime-$USER \