From 5677ff2edfdfa3264cf4e59e964e28e02f4c808c Mon Sep 17 00:00:00 2001 From: Asher Date: Tue, 2 Jul 2019 19:10:17 -0500 Subject: [PATCH] Add build scripts --- .gitignore | 2 + .travis.yml | 12 +-- README.md | 12 ++- main.js | 10 ++ package.json | 7 +- scripts/build.sh | 51 --------- scripts/ci.bash | 64 ++++++++++++ scripts/nbin.js | 24 +++++ scripts/tasks.bash | 250 +++++++++++++++++++++++++++++++++++++++++++++ scripts/vstar.sh | 24 ----- yarn.lock | 6 +- 11 files changed, 374 insertions(+), 88 deletions(-) delete mode 100755 scripts/build.sh create mode 100755 scripts/ci.bash create mode 100644 scripts/nbin.js create mode 100755 scripts/tasks.bash delete mode 100755 scripts/vstar.sh diff --git a/.gitignore b/.gitignore index 3c3629e64..92c07dafd 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules +build +release diff --git a/.travis.yml b/.travis.yml index 828655abf..902facc95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,20 +8,20 @@ matrix: - os: linux dist: trusty env: - - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="centos" + - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="centos" - os: linux dist: trusty env: - - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="alpine" + - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="alpine" - os: osx env: - - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" + - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" before_install: -- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev - libsecret-1-dev; fi +- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y libxkbfile-dev libsecret-1-dev; fi - npm install -g yarn@1.12.3 +- npm install -g @coder/nbin script: -- scripts/build.sh +- scripts/ci.bash before_deploy: - echo "$VERSION" "$TRAVIS_COMMIT" - git config --local user.name "$USER_NAME" diff --git a/README.md b/README.md index ee1e94a70..cfcea82f7 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,19 @@ Quickstart guides for [Google Cloud](doc/admin/install/google_cloud.md), [AWS](d How to [secure your setup](/doc/security/ssl.md). ## Build -- Run `yarn build ${vscodeVersion}`in this directory (for example, `yarn build 1.35.0`). +- If you also plan on developing, set the `OUT` environment variable: ` + export OUT=/path/to/some/directory`. Otherwise it will build in this + directory which will cause issues because then `yarn watch` will try to + compile the build directory as well. +- Run `yarn build ${vscodeVersion} ${target} ${arch}`in this directory (for example: + `yarn build 1.35.0 linux x64`). ## Development - Clone VS Code. -- Clone code-server to `src/vs/server` in the VS Code source. -- Run `yarn` in this directory (only need to do this once). +- Run `yarn` in the VS Code root directory. +- Clone this repository to `src/vs/server` in the VS Code source. +- Run `yarn` in this directory. - Run `yarn watch` in this directory. - Wait for the initial compilation to complete. - Run `yarn start` in this directory. diff --git a/main.js b/main.js index 765f41812..03e616b72 100644 --- a/main.js +++ b/main.js @@ -1 +1,11 @@ +try { + const nbin = require("nbin"); + const path = require("path"); + const rootPath = path.resolve(__dirname, "../../.."); + console.log("Shimming", rootPath); + nbin.shimNativeFs(rootPath); +} catch (error) { + console.log("Not in the binary"); +} + require("../../bootstrap-amd").load("vs/server/cli"); diff --git a/package.json b/package.json index 6af0cb63a..aab6177a6 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,13 @@ { + "license": "MIT", "scripts": { + "postinstall": "rm -r node_modules/@types/node # I keep getting type conflicts", "start": "nodemon ../../../out/vs/server/main.js --watch ../../../out --verbose", "watch": "cd ../../../ && yarn watch-client", - "build": "echo TODO && exit 1" + "build": "bash ./scripts/tasks.bash build", + "package": "bash ./scripts/tasks.bash package", + "vstar": "bash ./scripts/tasks.bash vstar", + "binary": "bash ./scripts/tasks.bash binary" }, "devDependencies": { "@types/tar-stream": "^1.6.1", diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index edf534088..000000000 --- a/scripts/build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -# Build using a Docker container using the specified image and version. -function docker_build() { - local image="${1}" ; shift - local version="${1}" ; shift - - local containerId - containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}") - docker start "${containerId}" - docker exec "${containerId}" mkdir -p /src - - function docker_exec() { - docker exec "${containerId}" bash -c "$@" - } - - docker cp ./. "${containerId}":/src - docker_exec "cd /src && yarn" - docker_exec "cd /src && npm rebuild" - docker_exec "cd /src && NODE_ENV=production VERSION=${version} yarn task build:server:binary" - docker_exec "cd /src && yarn task package ${version}" - docker cp "${containerId}":/src/release/. ./release/ - - docker stop "${containerId}" -} - -function main() { - local version=${VERSION:-} - local ostype=${OSTYPE:-} - - if [[ -z "${version}" ]] ; then - >&2 echo "Must set VERSION environment variable" - exit 1 - fi - - if [[ "${ostype}" == "darwin"* ]]; then - NODE_ENV=production VERSION="${version}" yarn task build:server:binary - yarn task package "${version}" - else - local image - if [[ "$TARGET" == "alpine" ]]; then - image="codercom/nbin-alpine" - else - image="codercom/nbin-centos" - fi - docker_build "${image}" "${version}" - fi -} - -main "$@" diff --git a/scripts/ci.bash b/scripts/ci.bash new file mode 100755 index 000000000..8d9300d0b --- /dev/null +++ b/scripts/ci.bash @@ -0,0 +1,64 @@ +#!/bin/bash +set -euo pipefail + +# Build using a Docker container using the specified image and version. +function docker-build() { + local image="${1}" ; shift + local version="${1}" ; shift + local vscodeVersion="${1}" ; shift + local target="${1}" ; shift + local arch="${1}" ; shift + + local containerId + containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}") + docker start "${containerId}" + docker exec "${containerId}" mkdir -p /src + + function docker-exec() { + docker exec "${containerId}" bash -c "$@" + } + + docker cp ./. "${containerId}":/src + docker-exec "cd /src && CI=true yarn build \"${vscodeVersion}\" \"${target}\" \"${arch}\"" + docker-exec "cd /src && CI=true yarn binary \"${vscodeVersion}\" \"${target}\" \"${arch}\"" + docker-exec "cd /src && CI=true yarn package \"${vscodeVersion}\" \"${target}\" \"${arch}\" \"${version}\"" + docker cp "${containerId}":/src/release/. ./release/ + + docker stop "${containerId}" +} + +# Build code-server in the CI. +function main() { + local version="${VERSION:-}" + local vscodeVersion="${VSCODE_VERSION:-}" + local ostype="${OSTYPE:-}" + local target="${TARGET:-}" + local arch=x64 + + if [[ -z "${version}" ]] ; then + >&2 echo "Must set VERSION environment variable"; exit 1 + fi + + if [[ -z "${vscodeVersion}" ]] ; then + >&2 echo "Must set VSCODE_VERSION environment variable"; exit 1 + fi + + if [[ "${ostype}" == "darwin"* ]]; then + target=darwin + CI=true yarn build "${vscodeVersion}" "${target}" "${arch}" + CI=true yarn binary "${vscodeVersion}" "${target}" "${arch}" + CI=true yarn package "${vscodeVersion}" "${target}" "${arch}" "${version}" + else + local image + if [[ "${target}" == alpine ]]; then + image=codercom/nbin-alpine + target=musl + else + image=codercom/nbin-centos + target=linux + fi + docker-build "${image}" "${version}" "${vscodeVersion}" "${target}" "${arch}" + fi +} + +main "$@" diff --git a/scripts/nbin.js b/scripts/nbin.js new file mode 100644 index 000000000..657e88936 --- /dev/null +++ b/scripts/nbin.js @@ -0,0 +1,24 @@ +/* global require, __dirname, process */ +const { Binary } = require("@coder/nbin"); +const fs = require("fs"); +const path = require("path"); + +const target = process.argv[2]; +const arch = process.argv[3]; +const source = process.argv[4]; + +const bin = new Binary({ + mainFile: path.join(source, "out/vs/server/main.js"), + target: target, +}); + +bin.writeFiles(path.join(source, "**")); + +bin.build().then((binaryData) => { + const outputPath = path.join(source, "code-server"); + fs.writeFileSync(outputPath, binaryData); + fs.chmodSync(outputPath, "755"); +}).catch((ex) => { + console.error(ex); + process.exit(1); +}); diff --git a/scripts/tasks.bash b/scripts/tasks.bash new file mode 100755 index 000000000..6e6d143ba --- /dev/null +++ b/scripts/tasks.bash @@ -0,0 +1,250 @@ +#!/bin/bash +set -euo pipefail + +function log() { + local message="${1}" ; shift + local level="${1:-info}" + if [[ "${level}" == "error" ]] ; then + >&2 echo "${message}" + else + echo "${message}" + fi +} + +function exit-if-ci() { + if [[ -n "${ci}" ]] ; then + log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} is incorrectly built" "error" + exit 1 + fi +} + +# Copy code-server into VS Code along with its dependencies. +function copy-server() { + local serverPath="${vscodeSourcePath}/src/vs/server" + rm -rf "${serverPath}" + mkdir -p "${serverPath}" + + log "Copying server code" + + cp "${rootPath}"/*.{ts,js} "${serverPath}" + cp "${rootPath}/package.json" "${serverPath}" + cp "${rootPath}/yarn.lock" "${serverPath}" + + if [[ -d "${rootPath}/node_modules" ]] ; then + log "Copying dependencies" + cp -r "${rootPath}/node_modules" "${serverPath}" + else + log "Installing dependencies" + cd "${serverPath}" + yarn + rm -r node_modules/@types/node # I keep getting type conflicts + fi +} + +# Copy code-server into VS Code then build it. +function build-code-server() { + copy-server + + # TODO: look into making it do the full minified build for just our code + # (basically just want to skip extensions, target our server code, and get + # the same type of build you get with the vscode-linux-x64-min task). + # Something like: yarn gulp "vscode-server-${target}-${arch}-min" + cd "${vscodeSourcePath}" + yarn gulp compile-client + + rm -rf "${codeServerBuildPath}" + mkdir -p "${codeServerBuildPath}" + + cp -r "${vscodeBuildPath}/resources/app/extensions" "${codeServerBuildPath}" + cp -r "${vscodeBuildPath}/resources/app/"*.json "${codeServerBuildPath}" + cp -r "${vscodeSourcePath}/out" "${codeServerBuildPath}" + rm -rf "${codeServerBuildPath}/out/vs/server/node_modules" + cp -r "${vscodeSourcePath}/remote/node_modules" "${codeServerBuildPath}" + + log "Final build: ${codeServerBuildPath}" +} + +# Build VS Code if it hasn't already been built. If we're in the CI and it's +# not fully built, error and exit. +function build-vscode() { + if [[ ! -d "${vscodeSourcePath}" ]] ; then + exit-if-ci + log "${vscodeSourceName} does not exist, cloning" + git clone https://github.com/microsoft/vscode --quiet \ + --branch "${vscodeVersion}" --single-branch --depth=1 \ + "${vscodeSourcePath}" + else + log "${vscodeSourceName} already exists, skipping clone" + fi + + cd "${vscodeSourcePath}" + + if [[ ! -d "${vscodeSourcePath}/node_modules" ]] ; then + exit-if-ci + log "Installing VS Code dependencies" + yarn + # Not entirely sure why but there seem to be problems with native modules. + # Also vscode-ripgrep keeps complaining after the rebuild that the + # node_modules directory doesn't exist, so we're ignoring that for now. + npm rebuild || true + + # Keep just what we need to keep the pre-built archive smaller. + rm -rf "${vscodeSourcePath}/.git" + rm -rf "${vscodeSourcePath}/test" + else + log "${vscodeSourceName}/node_modules already exists, skipping install" + fi + + if [[ ! -d "${vscodeBuildPath}" ]] ; then + exit-if-ci + log "${vscodeBuildName} does not exist, building" + local builtPath="${buildPath}/VSCode-${target}-${arch}" + rm -rf "${builtPath}" + yarn gulp "vscode-${target}-${arch}-min" --max-old-space-size=32384 + mkdir -p "${vscodeBuildPath}/resources/app" + # Copy just what we need to keep the pre-built archive smaller. + mv "${builtPath}/resources/app/extensions" "${vscodeBuildPath}/resources/app" + mv "${builtPath}/resources/app/"*.json "${vscodeBuildPath}/resources/app" + rm -rf "${builtPath}" + else + log "${vscodeBuildName} already exists, skipping build" + fi +} + +# Download VS Code with either curl or wget depending on which is available. +function download-vscode() { + cd "${buildPath}" + if command -v wget &> /dev/null ; then + log "Attempting to download ${tarName} with wget" + wget "${vsSourceUrl}" --quiet + else + log "Attempting to download ${tarName} with curl" + curl "${vsSourceUrl}" --silent --fail --output "${tarName}" + fi +} + +# Download pre-built VS Code if necessary. Build if there is no available +# download but not when in the CI. The pre-built package basically just +# provides us the dependencies and extensions so we don't have to install and +# build them respectively which takes a long time. +function prepare-vscode() { + if [[ ! -d "${vscodeBuildPath}" || ! -d "${vscodeSourcePath}" ]] ; then + mkdir -p "${buildPath}" + local tarName="vstar-${vscodeVersion}-${target}-${arch}.tar.gz" + local vsSourceUrl="https://codesrv-ci.cdr.sh/${tarName}" + if download-vscode ; then + cd "${buildPath}" + rm -rf "${vscodeBuildPath}" + tar -xzf "${tarName}" + rm "${tarName}" + elif [[ -n "${ci}" ]] ; then + log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} does not exist" "error" + exit 1 + else + log "${tarName} does not exist, building" + build-vscode + return + fi + else + log "VS Code is already downloaded or built" + fi + + log "Ensuring VS Code is fully built" + build-vscode +} + +function build-task() { + prepare-vscode + build-code-server +} + +function vstar-task() { + local archivePath="${releasePath}/vstar-${vscodeVersion}-${target}-${arch}.tar.gz" + rm -f "${archivePath}" + mkdir -p "${releasePath}" + tar -C "${buildPath}" -czf "${archivePath}" "${vscodeSourceName}" "${vscodeBuildName}" + log "Archive: ${archivePath}" +} + +function package-task() { + local version="${1}" ; shift + + log " version: ${version}" + + local archiveName="code-server${version}-vsc${vscodeVersion}-${target}-${arch}" + local archivePath="${releasePath}/${archiveName}" + rm -rf "${archivePath}" + mkdir -p "${archivePath}" + + cp "${buildPath}/code-server" "${archivePath}" + cp "${rootPath}/README.md" "${archivePath}" + cp "${vscodeSourcePath}/LICENSE.txt" "${archivePath}" + cp "${vscodeSourcePath}/ThirdPartyNotices.txt" "${archivePath}" + + cd "${releasePath}" + if [[ "${target}" == "linux" ]] ; then + tar -czf "${archiveName}.tar.gz" "${archiveName}" + else + zip -r "${archiveName}.zip" "${archiveName}" + fi + + log "Archive: ${archivePath}" +} + +# Package built code into a binary. +function binary-task() { + # I had trouble getting VS Code to build with the @coder/nbin dependency due + # to the types it installs (tons of conflicts), so for now it's a global + # dependency. + cd "${rootPath}" + npm link @coder/nbin + node "${rootPath}/scripts/nbin.js" "${target}" "${arch}" "${codeServerBuildPath}" + rm node_modules/@coder/nbin + mv "${codeServerBuildPath}/code-server" "${buildPath}" + log "Binary at ${buildPath}/code-server" +} + +function main() { + local task="${1}" ; shift + local vscodeVersion="${1}" ; shift + local target="${1}" ; shift + local arch="${1}" ; shift + local ci="${CI:-}" + + local relativeRootPath + local rootPath + relativeRootPath="$(dirname "${0}")/.." + rootPath="$(realpath "${relativeRootPath}")" + + # This lets you build in a separate directory since building within this + # directory while developing makes it hard to keep developing since compiling + # will compile everything in the build directory as well. + local outPath="${OUT:-${rootPath}}" + + local releasePath="${outPath}/release" + local buildPath="${outPath}/build" + + local vscodeSourceName="vscode-${vscodeVersion}-source" + local vscodeBuildName="vscode-${vscodeVersion}-${target}-${arch}-built" + local vscodeSourcePath="${buildPath}/${vscodeSourceName}" + local vscodeBuildPath="${buildPath}/${vscodeBuildName}" + + local codeServerBuildName="code-server-${target}-${arch}-built" + local codeServerBuildPath="${buildPath}/${codeServerBuildName}" + + log "Running ${task} task" + log " rootPath: ${rootPath}" + log " outPath: ${outPath}" + log " vscodeVersion: ${vscodeVersion}" + log " target: ${target}" + log " arch: ${arch}" + if [[ -n "${ci}" ]] ; then + log " CI: yes" + else + log " CI: no" + fi + + "${task}-task" "$@" +} + +main "$@" diff --git a/scripts/vstar.sh b/scripts/vstar.sh deleted file mode 100755 index c7be0b719..000000000 --- a/scripts/vstar.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -# Builds a tarfile containing vscode sourcefiles neccessary for CI. -# Done outside the CI and uploaded to object storage to reduce CI time. - -branch=1.33.1 -dir=/tmp/vstar -outfile=/tmp/vstar-$branch.tar.gz -rm -rf $dir -mkdir -p $dir - -cd $dir -git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1 -cd vscode - -yarn - -npx gulp vscode-linux-x64 --max-old-space-size=32384 -rm -rf extensions build out* test -cd .. -mv *-x64/resources/app/extensions ./extensions -rm -rf *-x64 -tar -czvf $outfile . diff --git a/yarn.lock b/yarn.lock index c172bb317..9acdbbaa8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3,9 +3,9 @@ "@types/node@*", "@types/node@^10.12.12": - version "10.14.10" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.10.tgz#e491484c6060af8d461e12ec81c0da8a3282b8de" - integrity sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q== + version "10.14.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.12.tgz#0eec3155a46e6c4db1f27c3e588a205f767d622f" + integrity sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg== "@types/tar-stream@^1.6.1": version "1.6.1"