diff --git a/ci/release-image/entrypoint.sh b/ci/release-image/entrypoint.sh
index 1c8c8bfff..6eddae463 100755
--- a/ci/release-image/entrypoint.sh
+++ b/ci/release-image/entrypoint.sh
@@ -5,16 +5,17 @@ set -eu
 # Otherwise the current container UID may not exist in the passwd database.
 eval "$(fixuid -q)"
 
-if [ "${DOCKER_USER-}" ] && [ "$DOCKER_USER" != "$USER" ]; then
-  echo "$DOCKER_USER ALL=(ALL) NOPASSWD:ALL" | sudo tee -a /etc/sudoers.d/nopasswd >/dev/null
-  # Unfortunately we cannot change $HOME as we cannot move any bind mounts
-  # nor can we bind mount $HOME into a new home as that requires a privileged container.
-  sudo usermod --login "$DOCKER_USER" coder
-  sudo groupmod -n "$DOCKER_USER" coder
-
+if [ "${DOCKER_USER-}" ]; then
   USER="$DOCKER_USER"
+  if [ "$DOCKER_USER" != "$(whoami)" ]; then
+    echo "$DOCKER_USER ALL=(ALL) NOPASSWD:ALL" | sudo tee -a /etc/sudoers.d/nopasswd >/dev/null
+    # Unfortunately we cannot change $HOME as we cannot move any bind mounts
+    # nor can we bind mount $HOME into a new home as that requires a privileged container.
+    sudo usermod --login "$DOCKER_USER" coder
+    sudo groupmod -n "$DOCKER_USER" coder
 
-  sudo sed -i "/coder/d" /etc/sudoers.d/nopasswd
+    sudo sed -i "/coder/d" /etc/sudoers.d/nopasswd
+  fi
 fi
 
 dumb-init /usr/bin/code-server "$@"