diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 834910dd2..9bd2a5341 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,186 +8,393 @@ on: branches: - main +# Note: if: success() is used in several jobs - +# this ensures that it only executes if all previous jobs succeeded. + +# if: steps.cache-yarn.outputs.cache-hit != 'true' +# will skip running `yarn install` if it successfully fetched from cache + jobs: - fmt: + prebuild: + name: Pre-build checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Run ./ci/steps/fmt.sh - uses: ./ci/images/debian10 - with: - args: ./ci/steps/fmt.sh + - name: Checkout repo + uses: actions/checkout@v2 - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run ./ci/steps/lint.sh - uses: ./ci/images/debian10 + - name: Install Node.js v12 + uses: actions/setup-node@v2 with: - args: ./ci/steps/lint.sh + node-version: "12" + + - name: Install helm + uses: azure/setup-helm@v1 + + - name: Fetch dependencies from cache + id: cache-yarn + uses: actions/cache@v2 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + + - name: Install dependencies + if: steps.cache-yarn.outputs.cache-hit != 'true' + run: yarn --frozen-lockfile - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - name: Audit for vulnerabilities - uses: ./ci/images/debian10 - with: - args: ./ci/steps/audit.sh + run: yarn _audit + if: success() - test-unit: + - name: Run yarn fmt + run: yarn fmt + if: success() + + - name: Run yarn lint + run: yarn lint + if: success() + + - name: Run code-server unit tests + run: yarn test:unit + if: success() + + build: + name: Build + needs: prebuild runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Run unit tests - uses: ./ci/images/debian10 + - uses: actions/checkout@v2 with: - args: ./ci/steps/test-unit.sh + fetch-depth: 0 + + - name: Install Node.js v12 + uses: actions/setup-node@v2 + with: + node-version: "12" + + - name: Fetch dependencies from cache + id: cache-yarn + uses: actions/cache@v2 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + + - name: Install dependencies + if: steps.cache-yarn.outputs.cache-hit != 'true' + run: yarn --frozen-lockfile + + - name: Build code-server + run: yarn build + + # Parse the hash of the latest commit inside lib/vscode + # use this to avoid rebuilding it if nothing changed + # How it works: the `git log` command fetches the hash of the last commit + # that changed a file inside `lib/vscode`. If a commit changes any file in there, + # the hash returned will change, and we rebuild vscode. If the hash did not change, + # (for example, a change to `src/` or `docs/`), we reuse the same build as last time. + # This saves a lot of time in CI, as compiling VSCode can take anywhere from 5-10 minutes. + - name: Get latest lib/vscode rev + id: vscode-rev + run: echo "::set-output name=rev::$(git log -1 --format='%H' ./lib/vscode)" + + - name: Attempt to fetch vscode build from cache + id: cache-vscode + uses: actions/cache@v2 + with: + path: | + lib/vscode/.build + lib/vscode/out-build + lib/vscode/out-vscode + lib/vscode/out-vscode-min + key: vscode-build-${{ steps.vscode-rev.outputs.rev }} + + - name: Build vscode + if: steps.cache-vscode.outputs.cache-hit != 'true' + run: yarn build:vscode + + # The release package does not contain any native modules + # and is neutral to architecture/os/libc version. + - name: Create release package + run: yarn release + if: success() + + # https://github.com/actions/upload-artifact/issues/38 + - name: Compress release package + run: tar -czf package.tar.gz release + + - name: Upload npm package artifact + uses: actions/upload-artifact@v2 + with: + name: npm-package + path: ./package.tar.gz + + # TODO: cache building yarn --production + # possibly 2m30s of savings(?) + # this requires refactoring our release scripts + package-linux-amd64: + name: x86-64 Linux build + needs: build + runs-on: ubuntu-latest + container: "centos:7" + + steps: + - uses: actions/checkout@v2 + + - name: Install Node.js v12 + uses: actions/setup-node@v2 + with: + node-version: "12" + + - name: Install development tools + run: | + yum install -y epel-release centos-release-scl + yum install -y devtoolset-9-{make,gcc,gcc-c++} jq rsync + + - name: Install nfpm and envsubst + run: | + curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b ~/.local/bin v2.3.1 + curl -L https://github.com/a8m/envsubst/releases/download/v1.1.0/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + mv envsubst ~/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install yarn + run: npm install -g yarn + + - name: Download npm package + uses: actions/download-artifact@v2 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + # NOTE: && here is deliberate - GitHub puts each line in its own `.sh` + # file when running inside a docker container. + - name: Build standalone release + run: source scl_source enable devtoolset-9 && yarn release:standalone + + - name: Sanity test standalone release + run: yarn test:standalone-release + + - name: Build packages with nfpm + run: yarn package + + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-packages + path: ./release-packages + + # NOTE@oxy: + # We use Ubuntu 16.04 here, so that our build is more compatible + # with older libc versions. We used to (Q1'20) use CentOS 7 here, + # but it has a full update EOL of Q4'20 and a 'critical security' + # update EOL of 2024. We're dropping full support a few years before + # the final EOL, but I don't believe CentOS 7 has a large arm64 userbase. + # It is not feasible to cross-compile with CentOS. + + # Cross-compile notes: To compile native dependencies for arm64, + # we install the aarch64 cross toolchain and then set it as the default + # compiler/linker/etc. with the AR/CC/CXX/LINK environment variables. + # qemu-user-static on ubuntu-16.04 currently doesn't run Node correctly, + # so we just build with "native"/x86_64 node, then download arm64 node + # and then put it in our release. We can't smoke test the arm64 build this way, + # but this means we don't need to maintain a self-hosted runner! + package-linux-arm64: + name: Linux ARM64 cross-compile build + needs: build + runs-on: ubuntu-16.04 + env: + AR: aarch64-linux-gnu-ar + CC: aarch64-linux-gnu-gcc + CXX: aarch64-linux-gnu-g++ + LINK: aarch64-linux-gnu-g++ + NPM_CONFIG_ARCH: arm64 + + steps: + - uses: actions/checkout@v2 + + - name: Install Node.js v12 + uses: actions/setup-node@v2 + with: + node-version: "12" + + - name: Install nfpm + run: | + curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b ~/.local/bin v2.3.1 + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install cross-compiler + run: sudo apt install g++-aarch64-linux-gnu + + - name: Download npm package + uses: actions/download-artifact@v2 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Replace node with arm64 equivalent + run: | + wget https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-arm64.tar.gz + tar -xzf node-v12.18.4-linux-arm64.tar.gz node-v12.18.4-linux-arm64/bin/node --strip-components=2 + mv ./node ./release-standalone/lib/node + + - name: Build packages with nfpm + run: yarn package arm64 + + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-packages + path: ./release-packages + + package-macos-amd64: + name: x86-64 macOS build + needs: build + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + + - name: Install Node.js v12 + uses: actions/setup-node@v2 + with: + node-version: "12" + + - name: Install nfpm + run: | + curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b ~/.local/bin v2.3.1 + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Download npm package + uses: actions/download-artifact@v2 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Sanity test standalone release + run: yarn test:standalone-release + + - name: Build packages with nfpm + run: yarn package + + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-packages + path: ./release-packages + test-e2e: - needs: linux-amd64 + name: End-to-end tests + needs: package-linux-amd64 runs-on: ubuntu-latest env: PASSWORD: e45432jklfdsab CODE_SERVER_ADDRESS: http://localhost:8080 steps: - uses: actions/checkout@v2 + + - name: Install Node.js v12 + uses: actions/setup-node@v2 + with: + node-version: "12" + + - name: Install playwright + uses: microsoft/playwright-github-action@v1 + + - name: Fetch dependencies from cache + id: cache-yarn + uses: actions/cache@v2 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + - name: Download release packages uses: actions/download-artifact@v2 with: name: release-packages path: ./release-packages + - name: Untar code-server file run: | cd release-packages && tar -xzf code-server*-linux-amd64.tar.gz - - uses: microsoft/playwright-github-action@v1 - - name: Install dependencies and run end-to-end tests + + - name: Install dependencies + if: steps.cache-yarn.outputs.cache-hit != 'true' + run: yarn --frozen-lockfile + + # HACK: this shouldn't need to exist, but put it here anyway + # in an attempt to solve Playwright cache failures. + - name: Reinstall playwright + if: steps.cache-yarn.outputs.cache-hit == 'true' + run: | + cd test/ + rm -r node_modules/playwright + yarn install --check-files + + - name: Run end-to-end tests run: | ./release-packages/code-server*-linux-amd64/bin/code-server --log trace & - yarn --frozen-lockfile yarn test:e2e + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v2 with: name: test-videos path: ./test/e2e/videos + - name: Remove release packages and test artifacts run: rm -rf ./release-packages ./test/e2e/videos - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run ./ci/steps/release.sh - uses: ./ci/images/debian10 - with: - args: ./ci/steps/release.sh - - name: Upload npm package artifact - uses: actions/upload-artifact@v2 - with: - name: npm-package - path: ./release-npm-package - - linux-amd64: - needs: release - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Download npm package - uses: actions/download-artifact@v2 - with: - name: npm-package - path: ./release-npm-package - - name: Run ./ci/steps/release-packages.sh - uses: ./ci/images/centos7 - with: - args: ./ci/steps/release-packages.sh - - name: Upload release artifacts - uses: actions/upload-artifact@v2 - with: - name: release-packages - path: ./release-packages - - linux-arm64: - needs: release - runs-on: ubuntu-arm64-latest - steps: - - uses: actions/checkout@v2 - - name: Download npm package - uses: actions/download-artifact@v2 - with: - name: npm-package - path: ./release-npm-package - - name: Run ./ci/steps/release-packages.sh - uses: ./ci/images/centos7 - with: - args: ./ci/steps/release-packages.sh - - name: Upload release artifacts - uses: actions/upload-artifact@v2 - with: - name: release-packages - path: ./release-packages - - name: Remove docker images - run: docker system prune -af - - macos-amd64: - needs: release - runs-on: macos-latest - # This job requires secrets, so can only run on the default branch - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v2 - - name: Download npm package - uses: actions/download-artifact@v2 - with: - name: npm-package - path: ./release-npm-package - - run: ./ci/steps/release-packages.sh - env: - # Otherwise we get rate limited when fetching the ripgrep binary. - # For whatever reason only MacOS needs it. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload release artifacts - uses: actions/upload-artifact@v2 - with: - name: release-packages - path: ./release-packages - docker-amd64: runs-on: ubuntu-latest - needs: linux-amd64 + needs: package-linux-amd64 steps: - uses: actions/checkout@v2 + - name: Download release package uses: actions/download-artifact@v2 with: name: release-packages path: ./release-packages + - name: Run ./ci/steps/build-docker-image.sh - uses: ./ci/images/debian10 - with: - args: ./ci/steps/build-docker-image.sh + run: ./ci/steps/build-docker-image.sh + - name: Upload release image uses: actions/upload-artifact@v2 with: name: release-images path: ./release-images + # TODO: this is the last place where we use our self-hosted arm64 runner. + # In the future, consider switching to docker buildx + qemu, + # thus removing the requirement for us to maintain the runner. docker-arm64: runs-on: ubuntu-arm64-latest - needs: linux-arm64 + needs: package-linux-arm64 steps: - uses: actions/checkout@v2 + - name: Download release package uses: actions/download-artifact@v2 with: name: release-packages path: ./release-packages + - name: Run ./ci/steps/build-docker-image.sh - uses: ./ci/images/centos7 - with: - args: ./ci/steps/build-docker-image.sh + run: ./ci/steps/build-docker-image.sh + - name: Upload release image uses: actions/upload-artifact@v2 with: diff --git a/ci/build/build-packages.sh b/ci/build/build-packages.sh index 06f78afff..b83ed286f 100755 --- a/ci/build/build-packages.sh +++ b/ci/build/build-packages.sh @@ -8,6 +8,12 @@ main() { cd "$(dirname "${0}")/../.." source ./ci/lib.sh + # Allow us to override architecture + # we use this for our Linux ARM64 cross compile builds + if [ "$#" -eq 1 ] && [ "$1" ]; then + ARCH=$1 + fi + mkdir -p release-packages release_archive diff --git a/ci/build/nfpm.yaml b/ci/build/nfpm.yaml index 9c3202d23..7aa51f9ef 100644 --- a/ci/build/nfpm.yaml +++ b/ci/build/nfpm.yaml @@ -10,10 +10,16 @@ description: | vendor: "Coder" homepage: "https://github.com/cdr/code-server" license: "MIT" -files: - ./ci/build/code-server-nfpm.sh: /usr/bin/code-server - ./ci/build/code-server@.service: /usr/lib/systemd/system/code-server@.service - # Only included for backwards compat with previous releases that shipped - # the user service. See #1997 - ./ci/build/code-server-user.service: /usr/lib/systemd/user/code-server.service - ./release-standalone/**/*: "/usr/lib/code-server/" + +contents: + - src: ./ci/build/code-server-nfpm.sh + dst: /usr/bin/code-server + + - src: ./ci/build/code-server@.service + dst: /usr/lib/systemd/system/code-server@.service + + - src: ./ci/build/code-server-user.service + dst: /usr/lib/systemd/user/code-server.service + + - src: ./release-standalone/* + dst: /usr/lib/code-server/ diff --git a/ci/images/centos7/Dockerfile b/ci/images/centos7/Dockerfile index cad487b16..cdcb384f3 100644 --- a/ci/images/centos7/Dockerfile +++ b/ci/images/centos7/Dockerfile @@ -26,6 +26,6 @@ ENV PATH=/usr/local/go/bin:$GOPATH/bin:$PATH # Install Go dependencies ENV GO111MODULE=on -RUN go get github.com/goreleaser/nfpm/cmd/nfpm@v1.9.0 +RUN go get github.com/goreleaser/nfpm/cmd/nfpm@v2.3.1 RUN curl -fsSL https://get.docker.com | sh diff --git a/ci/images/debian10/Dockerfile b/ci/images/debian10/Dockerfile index 3078598cc..d3c9a311b 100644 --- a/ci/images/debian10/Dockerfile +++ b/ci/images/debian10/Dockerfile @@ -39,7 +39,7 @@ ENV PATH=/usr/local/go/bin:$GOPATH/bin:$PATH # Install Go dependencies ENV GO111MODULE=on -RUN go get github.com/goreleaser/nfpm/cmd/nfpm@v1.9.0 +RUN go get github.com/goreleaser/nfpm/cmd/nfpm@v2.3.1 RUN VERSION="$(curl -fsSL https://storage.googleapis.com/kubernetes-release/release/stable.txt)" && \ curl -fsSL "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl" > /usr/local/bin/kubectl \ diff --git a/ci/steps/audit.sh b/ci/steps/audit.sh deleted file mode 100755 index fd95fcaec..000000000 --- a/ci/steps/audit.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - yarn --frozen-lockfile - - yarn _audit -} - -main "$@" diff --git a/ci/steps/fmt.sh b/ci/steps/fmt.sh deleted file mode 100755 index c1b6026d7..000000000 --- a/ci/steps/fmt.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - yarn --frozen-lockfile - - yarn fmt -} - -main "$@" diff --git a/ci/steps/lint.sh b/ci/steps/lint.sh deleted file mode 100755 index b515b24ca..000000000 --- a/ci/steps/lint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - yarn --frozen-lockfile - - yarn lint -} - -main "$@" diff --git a/ci/steps/release-packages.sh b/ci/steps/release-packages.sh deleted file mode 100755 index da39cf473..000000000 --- a/ci/steps/release-packages.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - NODE_VERSION=v12.18.4 - NODE_OS="$(uname | tr '[:upper:]' '[:lower:]')" - NODE_ARCH="$(uname -m | sed 's/86_64/64/; s/aarch64/arm64/')" - if [ "$NODE_OS" = "freebsd" ]; then - mkdir -p "$PWD/node-$NODE_VERSION-$NODE_OS-$NODE_ARCH/bin" - cp "$(which node)" "$PWD/node-$NODE_VERSION-$NODE_OS-$NODE_ARCH/bin" - else - curl -L "https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$NODE_OS-$NODE_ARCH.tar.gz" | tar -xz - fi - PATH="$PWD/node-$NODE_VERSION-$NODE_OS-$NODE_ARCH/bin:$PATH" - - # https://github.com/actions/upload-artifact/issues/38 - tar -xzf release-npm-package/package.tar.gz - - yarn release:standalone - yarn test:standalone-release - yarn package -} - -main "$@" diff --git a/ci/steps/release.sh b/ci/steps/release.sh deleted file mode 100755 index 45b837e4c..000000000 --- a/ci/steps/release.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - yarn --frozen-lockfile - yarn build - yarn build:vscode - yarn release - - # https://github.com/actions/upload-artifact/issues/38 - mkdir -p release-npm-package - tar -czf release-npm-package/package.tar.gz release -} - -main "$@" diff --git a/ci/steps/test-e2e.sh b/ci/steps/test-e2e.sh deleted file mode 100755 index 13376dc97..000000000 --- a/ci/steps/test-e2e.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - "./release-packages/code-server*-linux-amd64/bin/code-server" & - yarn --frozen-lockfile - yarn test:e2e -} - -main "$@" diff --git a/ci/steps/test-unit.sh b/ci/steps/test-unit.sh deleted file mode 100755 index 77fd547ce..000000000 --- a/ci/steps/test-unit.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - yarn --frozen-lockfile - - yarn test:unit -} - -main "$@" diff --git a/docs/install.md b/docs/install.md index 6536a69eb..4c1ead58c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -89,6 +89,9 @@ commands presented in the rest of this document. ## Debian, Ubuntu +NOTE: The standalone arm64 .deb does not support Ubuntu <16.04. +Please upgrade or [build with yarn](#yarn-npm). + ```bash curl -fOL https://github.com/cdr/code-server/releases/download/v3.9.2/code-server_3.9.2_amd64.deb sudo dpkg -i code-server_3.9.2_amd64.deb @@ -98,6 +101,9 @@ sudo systemctl enable --now code-server@$USER ## Fedora, CentOS, RHEL, SUSE +NOTE: The standalone arm64 .rpm does not support CentOS 7. +Please upgrade or [build with yarn](#yarn-npm). + ```bash curl -fOL https://github.com/cdr/code-server/releases/download/v3.9.2/code-server-3.9.2-amd64.rpm sudo rpm -i code-server-3.9.2-amd64.rpm @@ -157,8 +163,8 @@ For more context, see [comment](https://github.com/cdr/code-server/issues/1730#i We recommend installing with `yarn` or `npm` when: 1. You aren't on `amd64` or `arm64`. -2. If you're on Linux with glibc < v2.17 or glibcxx < v3.4.18 -3. You're running Alpine Linux. See [#1430](https://github.com/cdr/code-server/issues/1430#issuecomment-629883198) +2. If you're on Linux with glibc < v2.17 or glibcxx < v3.4.18 on amd64, or glibc < v2.23 or glibcxx < v3.4.21 on arm64. +3. You're running Alpine Linux, or are using a non-glibc libc. See [#1430](https://github.com/cdr/code-server/issues/1430#issuecomment-629883198) **note:** Installing via `yarn` or `npm` builds native modules on install and so requires C dependencies. See [./npm.md](./npm.md) for installing these dependencies. diff --git a/package.json b/package.json index 0d94c3aa0..11995b17e 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "parcel-bundler": "^1.12.4", "prettier": "^2.2.1", "prettier-plugin-sh": "^0.6.0", + "shellcheck": "^1.0.0", "stylelint": "^13.0.0", "stylelint-config-recommended": "^4.0.0", "ts-node": "^9.1.1", diff --git a/yarn.lock b/yarn.lock index 4dbca9829..109b0773e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7032,6 +7032,11 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +shellcheck@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shellcheck/-/shellcheck-1.0.0.tgz#263479d92c3708d63d98883f896481461cf17cd0" + integrity sha512-CdKbWXOknBwE1wNQzAnwfLf7QNOu/yqyLSGBKoq2WuChEqfg7dnZJ1pHR2P463PbVpBRz3KGkYnXJCoQrPwtYA== + signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"