Add Redis queue system for online mode and parameter presets

.github/workflows/build-windows-executable-app.yaml

@@ -59,12 +59,14 @@ jobs:
         repository: t0mdavid-m/OpenMS
         ref: FVdeploy
         path: 'OpenMS'
-    
-    - name: Install Qt
+
+    - name: Install Qt (Windows)
       uses: jurplel/install-qt-action@v4
       with:
-        version: '6.8.3'
+        version: '6.8.3'    ## Note this version is build with win64_msvc2022_64 and should always match what we use
         arch: 'win64_msvc2022_64'
+        cache: 'false'
+        archives: 'qtsvg qtimageformats qtbase'
 
     # https://github.com/marketplace/actions/visual-studio-shell
     - name: Set up Visual Studio shell
@@ -112,6 +114,12 @@ jobs:
         rm contrib_build-Windows.tar.gz
         ls
 
+    - name: Add contrib to PATH
+      shell: bash
+      run: |
+        # Add contrib library path for runtime DLL resolution
+        echo "${{ github.workspace }}/OpenMS/contrib/lib" >> $GITHUB_PATH
+
     - name: Setup ccache cache
       uses: actions/cache@v3
       with:
@@ -145,9 +153,9 @@ jobs:
       shell: bash
       run: |
           mkdir $GITHUB_WORKSPACE/OpenMS/bld/
+          bash OpenMS/tools/ci/capture-env.sh -v $GITHUB_WORKSPACE/OpenMS/bld/CMakeCache.txt
           ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
       env:
-          #OS_PREFIX_PATH: "${{ env.Qt5_DIR }}/lib/cmake;${{ env.Qt5_DIR }}"
           OPENMS_CONTRIB_LIBS: "${{ github.workspace }}/OpenMS/contrib"
           CI_PROVIDER: "GitHub-Actions"
           CMAKE_GENERATOR: "Ninja"
@@ -157,6 +165,7 @@ jobs:
           ENABLE_TOPP_TESTING: "ON"
           ENABLE_CLASS_TESTING: "ON"
           WITH_GUI: "OFF"
+          WITH_PARQUET: "OFF"
           ADDRESS_SANITIZER: "Off"
           BUILD_TYPE: "Release"
           OPENMP: "On"
@@ -199,7 +208,7 @@ jobs:
   build-executable:
     runs-on: windows-2022
     needs: [build-openms, build-vue-js-component]
-    
+
     steps:
     - name: Checkout
       uses: actions/checkout@v3
@@ -259,21 +268,16 @@ jobs:
         cp $PYTHON_DIR/DLLs/tcl86t.dll $EMBED_DIR/
         cp $PYTHON_DIR/DLLs/tk86t.dll $EMBED_DIR/
 
-    - name: Install pip
-      run: |
-        curl -O https://bootstrap.pypa.io/get-pip.py
-        ./python-${{ env.PYTHON_VERSION }}/python get-pip.py --no-warn-script-location
-        rm get-pip.py
-
     - name: Uncomment 'import site' in python311._pth file
       run: |
         sed -i 's/#import site/import site/' python-${{ env.PYTHON_VERSION }}/python311._pth
 
     - name: Install Required Packages
+      # Use system Python (which has pip and dev headers) to compile packages,
+      # installing into the embeddable Python's site-packages directory.
       run: |
-        .\python-${{ env.PYTHON_VERSION }}\python -m pip install --force-reinstall -r requirements.txt --no-warn-script-location
-        .\python-${{ env.PYTHON_VERSION }}\python -m pip uninstall polars --yes
-        .\python-${{ env.PYTHON_VERSION }}\python -m pip install 'polars-lts-cpu >= 1.0'
+        python -m pip install -r requirements.txt --target python-${{ env.PYTHON_VERSION }}/Lib/site-packages --upgrade --no-warn-script-location
+        python -m pip install 'polars-lts-cpu >= 1.0' --target python-${{ env.PYTHON_VERSION }}/Lib/site-packages --upgrade --no-warn-script-location
 
     - name: Set to offline deployment
       run: |
@@ -284,6 +288,15 @@ jobs:
     - name: Create .bat file
       run: |
         echo '@echo off' > ${{ env.APP_NAME }}.bat
+        echo 'setlocal EnableDelayedExpansion' > ${{ env.APP_NAME }}.bat
+        echo '' >> ${{ env.APP_NAME }}.bat
+        echo 'REM Set OpenMS data path for TOPP tools' >> ${{ env.APP_NAME }}.bat
+        echo 'set OPENMS_DATA_PATH=%~dp0share\OpenMS' >> ${{ env.APP_NAME }}.bat
+        echo '' >> ${{ env.APP_NAME }}.bat
+        echo 'REM Add each subfolder in share\OpenMS\THIRDPARTY to PATH' >> ${{ env.APP_NAME }}.bat
+        echo 'for /D %%D in ("%OPENMS_DATA_PATH%\THIRDPARTY\*") do (' >> ${{ env.APP_NAME }}.bat
+        echo '    set "PATH=!PATH!;%%D"' >> ${{ env.APP_NAME }}.bat
+        echo ')' >> ${{ env.APP_NAME }}.bat
         echo '' >> ${{ env.APP_NAME }}.bat
         echo 'REM Create .streamlit directory in user''s home if it doesn''t exist' >> ${{ env.APP_NAME }}.bat
         echo 'if not exist "%USERPROFILE%\.streamlit" mkdir "%USERPROFILE%\.streamlit"' >> ${{ env.APP_NAME }}.bat
@@ -375,7 +388,7 @@ jobs:
         <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
           <Product Id="*" Name="${{ env.APP_NAME }}" Language="1033" Version="1.0.0.0" Codepage="1252" Manufacturer="OpenMS Developer Team" UpgradeCode="${{ env.APP_UpgradeCode }}">
             <Package Id="*" InstallerVersion="300" Compressed="yes" InstallPrivileges="elevated" Platform="x64" />
-            <Media Id="1" Cabinet="streamlit.cab" EmbedCab="yes" />
+            <MediaTemplate EmbedCab="yes" MaximumUncompressedMediaSize="200" CabinetTemplate="data{0}.cab" />
 
             <!-- Folder structure -->
             <Property Id="WIXUI_INSTALLDIR" Value="INSTALLFOLDER" />

.github/workflows/test-win-exe-w-embed-py.yaml

@@ -17,29 +17,31 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python (regular distribution)
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }} # Use the same version as the embeddable version
+
       - name: Download python embeddable version
         run: |
           mkdir python-${{ env.PYTHON_VERSION }}
           curl -O https://www.python.org/ftp/python/${{ env.PYTHON_VERSION }}/python-${{ env.PYTHON_VERSION }}-embed-amd64.zip
           unzip python-${{ env.PYTHON_VERSION }}-embed-amd64.zip -d python-${{ env.PYTHON_VERSION }}
           rm python-${{ env.PYTHON_VERSION }}-embed-amd64.zip
 
-      - name: Install pip
-        run: |
-          curl -O https://bootstrap.pypa.io/get-pip.py
-          ./python-${{ env.PYTHON_VERSION }}/python get-pip.py --no-warn-script-location
-          rm get-pip.py
-
       - name: Uncomment 'import site' in python311._pth file
         run: |
           sed -i 's/#import site/import site/' python-${{ env.PYTHON_VERSION }}/python311._pth
-          
+
       - name: Print content of python311._pth file
         run: |
           cat python-${{ env.PYTHON_VERSION }}/python311._pth
 
       - name: Install Required Packages
-        run: .\python-${{ env.PYTHON_VERSION }}\python -m pip install -r requirements.txt --no-warn-script-location
+        # Use system Python (which has development headers) to compile packages,
+        # installing into the embeddable Python's site-packages directory.
+        # The embeddable Python lacks Python.h headers needed for native extensions.
+        run: python -m pip install -r requirements.txt --target python-${{ env.PYTHON_VERSION }}/Lib/site-packages --upgrade --no-warn-script-location
 
       - name: Create .bat file
         run: |

.gitignore

@@ -29,3 +29,4 @@ run_app.bat
 python*
 gdpr_consent/node_modules/
 *~
+.streamlit/secrets.toml

.streamlit/config.toml

@@ -5,8 +5,12 @@ gatherUsageStats = false
 developmentMode = false
 
 [server]
+address = "0.0.0.0"
 maxUploadSize =  2000 #MB
 port = 8501 # should be same as configured in deployment repo
+enableCORS = false
+enableXsrfProtection = false
+
 
 [theme]
 # The preset Streamlit theme that your custom theme inherits from. One of "light" or "dark".

.streamlit/secrets.toml.example

@@ -0,0 +1,8 @@
+# Streamlit Secrets Configuration
+# Copy this file to secrets.toml and fill in your values.
+# IMPORTANT: Never commit secrets.toml to version control!
+
+[admin]
+# Password required to save workspaces as demo workspaces (online mode only)
+# Set a strong, unique password here
+password = "your-secure-admin-password-here"

Dockerfile

@@ -79,8 +79,8 @@ SHELL ["/bin/bash", "--rcfile", "~/.bashrc"]
 SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"]
 
 # Install up-to-date cmake via mamba and packages for pyOpenMS build.
-RUN mamba install cmake
-RUN pip install --upgrade pip && python -m pip install -U setuptools nose 'Cython>=3.1' 'autowrap==0.23.0' pandas 'numpy>=2.0' pytest
+RUN mamba install -y cmake
+RUN pip install --upgrade pip && python -m pip install -U setuptools nose 'Cython>=3.1' 'autowrap==0.24' pandas 'numpy>=2.0' pytest
 
 # Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules.
 RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS
@@ -128,7 +128,7 @@ ENV PATH="/openms/bin/:${PATH}"
 
 # Copy TOPP tools bin directory, add to PATH.
 RUN cp -r openms-build/lib /openms/lib
-ENV LD_LIBRARY_PATH="/openms/lib/:${LD_LIBRARY_PATH}"
+ENV LD_LIBRARY_PATH="/openms/lib/:/root/miniforge3/envs/streamlit-env/lib:${LD_LIBRARY_PATH}"
 
 # Copy share folder, add to PATH, remove source directory.
 RUN cp -r OpenMS/share/OpenMS /openms/share
@@ -140,6 +140,14 @@ RUN rm -rf openms-build
 
 # Prepare and run streamlit app.
 FROM compile-openms AS run-app
+
+# Install Redis server for job queue and nginx for load balancing
+RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create Redis data directory
+RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis
+
 # Create workdir and copy over all streamlit related files/folders.
 
 # note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure
@@ -157,15 +165,79 @@ COPY src/ /app/src
 COPY app.py /app/app.py
 COPY settings.json /app/settings.json
 COPY default-parameters.json /app/default-parameters.json
+COPY presets.json /app/presets.json
 
 # add cron job to the crontab
 RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab -
 
-# create entrypoint script to start cron service and launch streamlit app
-RUN echo "#!/bin/bash" > /app/entrypoint.sh && \
-    echo "source /root/miniforge3/bin/activate streamlit-env" >> /app/entrypoint.sh && \
-    echo "service cron start" >> /app/entrypoint.sh && \
-    echo "streamlit run app.py" >> /app/entrypoint.sh
+# Set default worker count (can be overridden via environment variable)
+ENV RQ_WORKER_COUNT=1
+ENV REDIS_URL=redis://localhost:6379/0
+
+# Number of Streamlit server instances for load balancing (default: 1 = no load balancer)
+# Set to >1 to enable nginx load balancer with multiple Streamlit instances
+ENV STREAMLIT_SERVER_COUNT=1
+
+# create entrypoint script to start cron, Redis, RQ workers, and Streamlit
+RUN echo -e '#!/bin/bash\n\
+set -e\n\
+source /root/miniforge3/bin/activate streamlit-env\n\
+\n\
+# Start cron for workspace cleanup\n\
+service cron start\n\
+\n\
+# Start Redis server in background\n\
+echo "Starting Redis server..."\n\
+redis-server --daemonize yes --dir /var/lib/redis --appendonly no\n\
+\n\
+# Wait for Redis to be ready\n\
+until redis-cli ping > /dev/null 2>&1; do\n\
+    echo "Waiting for Redis..."\n\
+    sleep 1\n\
+done\n\
+echo "Redis is ready"\n\
+\n\
+# Start RQ worker(s) in background\n\
+WORKER_COUNT=${RQ_WORKER_COUNT:-1}\n\
+echo "Starting $WORKER_COUNT RQ worker(s)..."\n\
+for i in $(seq 1 $WORKER_COUNT); do\n\
+    rq worker openms-workflows --url $REDIS_URL --name worker-$i &\n\
+done\n\
+\n\
+# Load balancer setup\n\
+SERVER_COUNT=${STREAMLIT_SERVER_COUNT:-1}\n\
+\n\
+if [ "$SERVER_COUNT" -gt 1 ]; then\n\
+    echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..."\n\
+\n\
+    # Generate nginx upstream block\n\
+    UPSTREAM_SERVERS=""\n\
+    BASE_PORT=8510\n\
+    for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\
+        PORT=$((BASE_PORT + i))\n\
+        UPSTREAM_SERVERS="${UPSTREAM_SERVERS}        server 127.0.0.1:${PORT};\\n"\n\
+    done\n\
+\n\
+    # Write nginx config\n\
+    mkdir -p /etc/nginx\n\
+    echo -e "worker_processes auto;\\npid /run/nginx.pid;\\n\\nevents {\\n    worker_connections 1024;\\n}\\n\\nhttp {\\n    client_max_body_size 0;\\n\\n    map \\$cookie_stroute \\$route_key {\\n        \\x22\\x22      \\$request_id;\\n        default \\$cookie_stroute;\\n    }\\n\\n    upstream streamlit_backend {\\n        hash \\$route_key consistent;\\n${UPSTREAM_SERVERS}    }\\n\\n    map \\$http_upgrade \\$connection_upgrade {\\n        default upgrade;\\n        \\x27\\x27 close;\\n    }\\n\\n    server {\\n        listen 0.0.0.0:8501;\\n\\n        location / {\\n            proxy_pass http://streamlit_backend;\\n            proxy_http_version 1.1;\\n            proxy_set_header Upgrade \\$http_upgrade;\\n            proxy_set_header Connection \\$connection_upgrade;\\n            proxy_set_header Host \\$host;\\n            proxy_set_header X-Real-IP \\$remote_addr;\\n            proxy_set_header X-Forwarded-For \\$proxy_add_x_forwarded_for;\\n            proxy_set_header X-Forwarded-Proto \\$scheme;\\n            proxy_read_timeout 86400;\\n            proxy_send_timeout 86400;\\n            proxy_buffering off;\\n            add_header Set-Cookie \\x22stroute=\\$route_key; Path=/; HttpOnly; SameSite=Lax\\x22 always;\\n        }\\n    }\\n}" > /etc/nginx/nginx.conf\n\
+\n\
+    # Start Streamlit instances on internal ports\n\
+    for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\
+        PORT=$((BASE_PORT + i))\n\
+        echo "Starting Streamlit instance on port $PORT..."\n\
+        streamlit run app.py --server.port $PORT --server.address 0.0.0.0 &\n\
+    done\n\
+\n\
+    sleep 2\n\
+    echo "Starting nginx load balancer on port 8501..."\n\
+    exec /usr/sbin/nginx -g "daemon off;"\n\
+else\n\
+    # Single instance mode (default) - run Streamlit directly on port 8501\n\
+    echo "Starting Streamlit app..."\n\
+    exec streamlit run app.py --server.address 0.0.0.0\n\
+fi\n\
+' > /app/entrypoint.sh
 # make the script executable
 RUN chmod +x /app/entrypoint.sh
 

docker-compose.yml

@@ -12,6 +12,9 @@ services:
       - 8501:8501
     volumes:
       - workspaces-streamlit-template:/workspaces-streamlit-template
-    command: streamlit run openms-streamlit-template/app.py
+    environment:
+      # Number of Streamlit server instances (default: 1 = no load balancer).
+      # Set to >1 to enable nginx load balancing across multiple Streamlit instances.
+      - STREAMLIT_SERVER_COUNT=1
 volumes:
   workspaces-streamlit-template: