Immich: point immich-ml at the moved ml_start.sh and clear failed units

The update moves ml_start.sh into app/machine-learning, but only rewrote
immich-web's ExecStart, so older containers kept starting ML from
/opt/immich/ml_start.sh and failed with 203/EXEC. Rewrite immich-ml the same
way. A crash loop before the update can also leave both units rate-limited,
which makes the final restart fail; reset them first.

ct/immich.sh

@@ -115,7 +115,10 @@ EOF
   fi
 
   if check_for_gh_release "Immich" "immich-app/immich" "${RELEASE}" "each release is tested individually before the version is updated. Please do not open issues for this"; then
-    if [[ $(cat ~/.immich) > "2.5.1" ]]; then
+    msg_warn "The update takes 5-15 minutes, do not interrupt it"
+    PREV_IMMICH="$(cat ~/.immich)"
+    # An interrupted update leaves app/ empty, so there may be no immich-admin to call.
+    if [[ "$PREV_IMMICH" > "2.5.1" && -x /opt/immich/app/bin/immich-admin ]]; then
       msg_info "Enabling Maintenance Mode"
       cd /opt/immich/app/bin
       $STD ./immich-admin enable-maintenance-mode || true
@@ -273,12 +276,14 @@ EOF
       msg_ok "Pre-installed Python ${ML_PYTHON}"
       msg_info "Updating Intel OpenVINO machine-learning"
       for attempt in $(seq 1 3); do
-        $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
-        [[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
-        msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
+        $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
+        [[ $attempt -eq 3 ]] && { ML_FAILED=1; break; }
+        # Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
+        export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
+        msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
       done
-      patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
-      msg_ok "Updated Intel OpenVINO machine-learning"
+      [[ "${ML_FAILED:-0}" == 1 ]] || patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
+      [[ "${ML_FAILED:-0}" == 1 ]] || msg_ok "Updated Intel OpenVINO machine-learning"
     else
       ML_PYTHON="python3.13"
       msg_info "Pre-installing Python ${ML_PYTHON} for machine-learning"
@@ -289,12 +294,15 @@ EOF
       msg_ok "Pre-installed Python ${ML_PYTHON}"
       msg_info "Updating machine-learning"
       for attempt in $(seq 1 3); do
-        $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
-        [[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
-        msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
+        $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
+        [[ $attempt -eq 3 ]] && { ML_FAILED=1; break; }
+        # Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
+        export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
+        msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
       done
-      msg_ok "Updated machine-learning"
+      [[ "${ML_FAILED:-0}" == 1 ]] || msg_ok "Updated machine-learning"
     fi
+    [[ "${ML_FAILED:-0}" == 1 ]] && msg_warn "uv sync failed three times, machine learning was not updated"
     cd "$SRC_DIR"
     cp -a machine-learning/{ann,immich_ml} "$ML_DIR"
     [[ -f "$INSTALL_DIR"/ml_start.sh ]] && mv "$INSTALL_DIR"/ml_start.sh "$ML_DIR"
@@ -345,6 +353,10 @@ EOF
       sed -i "s|^ExecStart=.*|ExecStart=${APP_DIR}/bin/start.sh|" /etc/systemd/system/immich-web.service
       systemctl daemon-reload
     fi
+    if grep -q "^ExecStart=${INSTALL_DIR}/ml_start.sh" /etc/systemd/system/immich-ml.service; then
+      sed -i "s|^ExecStart=.*|ExecStart=${ML_DIR}/ml_start.sh|" /etc/systemd/system/immich-ml.service
+      systemctl daemon-reload
+    fi
 
     # MickLesk temporary patch for HEIC thumbnail gen
     MEDIA_REPO_JS="/opt/immich/app/dist/repositories/media.repository.js"
@@ -366,6 +378,15 @@ EOF
       $STD cd -
       msg_ok "Disabled Maintenance Mode"
     fi
+    # A crash loop before the update leaves the units rate-limited, and restart would refuse.
+    systemctl reset-failed immich-ml immich-web 2>/dev/null || true
+    if [[ "${ML_FAILED:-0}" == 1 ]]; then
+      systemctl restart immich-web || true
+      [[ -f /etc/systemd/system/immich-proxy.service ]] && systemctl restart immich-proxy
+      echo "$PREV_IMMICH" >~/.immich
+      msg_error "Immich runs, but without machine learning (see the uv output above). Run update again to retry."
+      exit 1
+    fi
     systemctl restart immich-ml immich-web
     [[ -f /etc/systemd/system/immich-proxy.service ]] && systemctl restart immich-proxy
     msg_ok "Updated successfully!"

install/immich-install.sh

@@ -441,9 +441,11 @@ if [[ -f ~/.openvino ]]; then
   msg_ok "Pre-installed Python ${ML_PYTHON}"
   msg_info "Installing Intel OpenVINO machine-learning"
   for attempt in $(seq 1 3); do
-    $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
+    $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
     [[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
-    msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
+    # Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
+    export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
+    msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
   done
   patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
   msg_ok "Installed Intel OpenVINO machine-learning"
@@ -457,9 +459,11 @@ else
   msg_ok "Pre-installed Python ${ML_PYTHON}"
   msg_info "Installing machine-learning"
   for attempt in $(seq 1 3); do
-    $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
+    $STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
     [[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
-    msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
+    # Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
+    export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
+    msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
   done
   msg_ok "Installed machine-learning"
 fi