scriptcheck fixes

.gitignore

@@ -7,6 +7,7 @@ db.db
 /config/brancol.py
 /config/manudo.py
 /config/stpatrick.py
+/config/edge2.py
 secret*
 /videos/
 /ondeck/

copy_enc_copy.py

@@ -0,0 +1,100 @@
+import os
+import shutil
+from pathlib import Path
+import subprocess
+from subprocess import CompletedProcess
+import asyncio
+from asyncio import Semaphore, Lock
+import fileinput
+import io
+import sys
+import click
+
+
+
+async def foo(pipelinelock: Semaphore, inputlock: Lock, cpulock: Semaphore, outputlock: Lock, from_path: Path, tmp_path:Path, to_path: Path, rm_original: bool):
+    async with pipelinelock:
+        async with inputlock:
+            cmd = "cp " + str(from_path.absolute()) + " " + str(tmp_path.absolute())
+            print(cmd)
+            stdout = io.StringIO()
+            stderr = io.StringIO()
+            proc = await asyncio.create_subprocess_shell(cmd, asyncio.subprocess.PIPE, asyncio.subprocess.PIPE)
+            stdout, stderr = await proc.communicate()
+            if proc.returncode != 0:
+                print("error copying " +from_path.name+" to tmp folder", stdout.decode(), stderr.decode())
+                return
+
+        async with cpulock:
+            cmd = "gpg -e --batch --trust-model always -r edgedevice --output " + str(tmp_path.absolute()) + ".enc " + str(tmp_path.absolute()) 
+            print(cmd)
+            proc = await asyncio.create_subprocess_shell(cmd, asyncio.subprocess.PIPE, asyncio.subprocess.PIPE)
+            stdout, stderr = await proc.communicate()
+            if proc.returncode != 0:
+                print("error running gpg " +tmp_path.name+".enc")
+                return
+
+        async with outputlock:
+            cmd = "cp " + str(tmp_path.absolute()) + ".enc " + str(to_path.absolute()) + ".enc"
+            print(cmd)
+            proc = await asyncio.create_subprocess_shell(cmd, asyncio.subprocess.PIPE, asyncio.subprocess.PIPE)
+            stdout, stderr = await proc.communicate()
+            if proc.returncode != 0:
+                print("error copying " +tmp_path.name+".enc to tmp folder")
+                return
+
+    cmd = "rm " + str(tmp_path.absolute()) + " " + str(tmp_path.absolute()) + ".enc"
+    print(cmd)
+    proc = await asyncio.create_subprocess_shell(cmd, asyncio.subprocess.PIPE, asyncio.subprocess.PIPE)
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        print("error cleaning up " +tmp_path.name+" in tmp folder")
+        return
+
+    if rm_original:    
+        cmd = "rm " + str(from_path.absolute()) 
+        print(cmd)
+        proc = await asyncio.create_subprocess_shell(cmd, asyncio.subprocess.PIPE, asyncio.subprocess.PIPE)
+        stdout, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            print("error cleaning up " +tmp_path.name+" in tmp folder")
+            return
+
+
+
+async def setup_stdin(rm_original: bool, same_input_output_lock: bool):
+
+    pipelinelock = Semaphore(10)
+    cpulock = Semaphore(5)
+    locka = Lock()
+    lockb = Lock()
+
+    all = []
+    for line in sys.stdin.readlines():
+        line = line.strip()
+        vid = Path(line)
+        if not vid.name.endswith('.avi.done'):
+            continue
+
+        # print(str(original_path.absolute()))
+
+        tmp_path = Path("/tmp/"+vid.name)
+        usb_path = Path("./"+vid.name)
+
+        # print(str(tmp_path.absolute()))
+        all.append(asyncio.create_task(foo(pipelinelock, locka, cpulock, locka if same_input_output_lock else lockb, vid, tmp_path, usb_path, rm_original)))
+
+    for t in all:
+        await t
+
+
+@click.command()
+@click.option('--rm-original', is_flag=True, help='should the original be deleted')
+@click.option('--same-input-output-lock', is_flag=True, help='should the original be deleted')
+def main(rm_original, same_input_output_lock):
+    asyncio.run(setup_stdin(rm_original, same_input_output_lock))
+
+if __name__ == "__main__":
+    main()
+
+

migrations/versions/ba08d4e11cc7_ondeckdata_more_data_columns.py

@@ -0,0 +1,31 @@
+"""ondeckdata_more_data_columns
+
+Revision ID: ba08d4e11cc7
+Revises: 495235ece5f0
+Create Date: 2023-10-11 17:33:42.633350
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ba08d4e11cc7'
+down_revision = '495235ece5f0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column('ondeckdata', sa.Column('overallcatches', sa.Integer(), nullable=True))
+    op.add_column('ondeckdata', sa.Column('overalldiscards', sa.Integer(), nullable=True))
+    op.add_column('ondeckdata', sa.Column('detection_confidence', sa.REAL(), nullable=True))
+
+
+
+def downgrade() -> None:
+    op.drop_column('ondeckdata', 'detection_confidence')
+    op.drop_column('ondeckdata', 'overalldiscards')
+    op.drop_column('ondeckdata', 'overallcatches')
+
+

scripts/adduser_aifish.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+
+scriptdir="$(dirname -- "$( readlink -f -- "$0")")"
+
+if [ "$UID" -lt 1000 ] ; then
+  echo "This script should be run as a non-root user with 'sudo' access"
+  exit 1
+fi
+
+if ! [ -e "$scriptdir/secret_adduser_aifish.txt" ] ; then
+  echo "Cannot adduser without secrets file containing password"
+  exit 1
+fi
+
+USERHOME="/home/aifish"
+
+sudo /bin/bash <<EOF
+
+adduser aifish < "$scriptdir/secret_adduser_aifish.txt"
+
+if ! [ -d "$USERHOME/.ssh" ] ; then
+  mkdir "$USERHOME/.ssh"
+  chmod go-rwx "$USERHOME/.ssh"
+  chown aifish:aifish "$USERHOME/.ssh"
+fi
+
+if ! [ -e "$USERHOME/.ssh/authorized_keys" ] ; then
+  touch "$USERHOME/.ssh/authorized_keys"
+  chmod go-rwx "$USERHOME/.ssh/authorized_keys"
+  chown aifish:aifish "$USERHOME/.ssh/authorized_keys"
+fi
+
+while read k; do
+  if ! grep -q "\$k" "$USERHOME/.ssh/authorized_keys" ; then
+    echo "\$k" >> "$USERHOME/.ssh/authorized_keys"
+  fi
+done <"$scriptdir/aifish_authorized_keys.txt"
+
+EOF
+
+# mktemp
+
+# on dev machine, user can run anything as sudo
+# sudo usermod -aG sudo aifish
+
+# on prod machines, user can only run docker commands
+# aifish ALL=NOPASSWD: /usr/bin/docker *
+

scripts/adduser_ondeck.sh

@@ -31,9 +31,9 @@ if ! [ -e "$USERHOME/.ssh/authorized_keys" ] ; then
   chown ondeck:ondeck "$USERHOME/.ssh/authorized_keys"
 fi
 
-while read k; do
-  if ! grep -q "$k" "$USERHOME/.ssh/authorized_keys" ; then
-    echo "$k" >> "$USERHOME/.ssh/authorized_keys"
+while read -r k; do
+  if ! grep -q "\$k" "$USERHOME/.ssh/authorized_keys" ; then
+    echo "\$k" >> "$USERHOME/.ssh/authorized_keys"
   fi
 done <"$scriptdir/ondeck_authorized_keys.txt"
 
@@ -48,7 +48,7 @@ gapp_creds_config_line=$(sudo grep -E '^export GOOGLE_APPLICATION_CREDENTIALS='
 
 if [ $? -eq 0 ] && [ "x$gapp_creds_config_line" != "x" ] ; then
   # eval to make this value available in this script
-  eval $gapp_creds_config_line
+  eval "$gapp_creds_config_line"
 else
 
   # set it up in .bashrc

scripts/app-install.sh

@@ -1,14 +1,14 @@
-
+#!/bin/bash
 
 scriptdir="$(dirname -- "$( readlink -f -- "$0")")"
 
-cd "$scriptdir/.."
+cd "$scriptdir/.." || exit
 
 USERNAME="$(whoami)"
 USERHOME="/home/$USERNAME"
 WORKINGDIR="$USERHOME/tnc-edge-service"
 
-cd "$WORKINGDIR"
+cd "$WORKINGDIR" || exit
 
 
 if ! [ -e ./venv/bin/activate ] ; then

scripts/netplan-autoswitcher.sh

@@ -10,12 +10,24 @@ fi
 
 
 
-if ! [ -e /etc/netplan/01_eth0_dhcp.yaml* ] ; then
+FOUND=""
+for file in /etc/netplan/01_eth0_dhcp.yaml* ; do
+  if [ -e "$file" ] ; then
+    FOUND="y"
+  fi
+done
+if ! [ "$FOUND" ] ; then
   echo "could not find /etc/netplan/01_eth0_dhcp.yaml*"
   exit 1
 fi
 
-if ! [ -e /etc/netplan/01_eth0_static.yaml* ] ; then
+FOUND=""
+for file in /etc/netplan/01_eth0_static.yaml* ; do
+  if [ -e "$file" ] ; then
+    FOUND="y"
+  fi
+done
+if ! [ "$FOUND" ] ; then
   echo "could not find /etc/netplan/01_eth0_static.yaml*"
   exit 1
 fi
@@ -51,7 +63,7 @@ elif grep -q "method=auto" /run/NetworkManager/system-connections/netplan-eth0.n
   GATEWAYIP="$(nmcli d show eth0 | grep IP4.GATEWAY | awk '{print $2;}')"
   STATICGWIP="$(grep "via:" /etc/netplan/01_eth0_static.yaml* | awk '{print $2;}')"
 
-  if [ "x$GATEWAYIP" == "x$STATICGWIP" ] ; then
+  if [ "$GATEWAYIP" == "$STATICGWIP" ] ; then
     if ping "$STATICGWIP" -c 3 >/dev/null 2>&1 ; then
       if ping "api.oceanbox2.com" -c 3 >/dev/null 2>&1 ; then
         switch_to_static