diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index d445e9057..2d2d91bca 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -581,8 +581,8 @@ jobs:
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
         run: SKIP_SUBMODULE_DEPS=1 yarn install
 
-      - name: Run integration tests on standalone release
-        run: yarn test:integration
+      - name: Run native module tests on standalone release
+        run: yarn test:native
 
       - name: Build packages with nfpm
         run: yarn package
diff --git a/ci/dev/test-native.sh b/ci/dev/test-native.sh
new file mode 100755
index 000000000..9362be053
--- /dev/null
+++ b/ci/dev/test-native.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+help() {
+  echo >&2 "  You can build the standalone release with 'yarn release:standalone'"
+  echo >&2 "  Or you can pass in a custom path."
+  echo >&2 "  CODE_SERVER_PATH='/var/tmp/coder/code-server/bin/code-server' yarn test:integration"
+}
+
+# Make sure a code-server release works. You can pass in the path otherwise it
+# will look for release-standalone in the current directory.
+#
+# This is to make sure we don't have Node version errors or any other
+# compilation-related errors.
+main() {
+  cd "$(dirname "$0")/../.."
+
+  source ./ci/lib.sh
+
+  local path="$RELEASE_PATH-standalone/bin/code-server"
+  if [[ ! ${CODE_SERVER_PATH-} ]]; then
+    echo "Set CODE_SERVER_PATH to test another build of code-server"
+  else
+    path="$CODE_SERVER_PATH"
+  fi
+
+  echo "Running tests with code-server binary: '$path'"
+
+  if [[ ! -f $path ]]; then
+    echo >&2 "No code-server build detected"
+    echo >&2 "Looked in $path"
+    help
+    exit 1
+  fi
+
+  CODE_SERVER_PATH="$path" ./test/node_modules/.bin/jest "$@" --coverage=false --testRegex "./test/integration/help.test.ts"
+}
+
+main "$@"
diff --git a/package.json b/package.json
index d6db46f7d..523b0b777 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
     "test:e2e:proxy": "USE_PROXY=1 ./ci/dev/test-e2e.sh",
     "test:unit": "./ci/dev/test-unit.sh --forceExit --detectOpenHandles",
     "test:integration": "./ci/dev/test-integration.sh",
+    "test:native": "./ci/dev/test-native.sh",
     "test:scripts": "./ci/dev/test-scripts.sh",
     "package": "./ci/build/build-packages.sh",
     "postinstall": "./ci/dev/postinstall.sh",