Add build scripts

This commit is contained in:
Asher 2019-07-02 19:10:17 -05:00
parent 61c281ec6b
commit 5677ff2edf
No known key found for this signature in database
GPG Key ID: D63C1EF81242354A
11 changed files with 374 additions and 88 deletions

2
.gitignore vendored
View File

@ -1 +1,3 @@
node_modules node_modules
build
release

View File

@ -8,20 +8,20 @@ matrix:
- os: linux - os: linux
dist: trusty dist: trusty
env: env:
- VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="centos" - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="centos"
- os: linux - os: linux
dist: trusty dist: trusty
env: env:
- VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="alpine" - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="alpine"
- os: osx - os: osx
env: env:
- VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER"
before_install: before_install:
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y libxkbfile-dev libsecret-1-dev; fi
libsecret-1-dev; fi
- npm install -g yarn@1.12.3 - npm install -g yarn@1.12.3
- npm install -g @coder/nbin
script: script:
- scripts/build.sh - scripts/ci.bash
before_deploy: before_deploy:
- echo "$VERSION" "$TRAVIS_COMMIT" - echo "$VERSION" "$TRAVIS_COMMIT"
- git config --local user.name "$USER_NAME" - git config --local user.name "$USER_NAME"

View File

@ -51,13 +51,19 @@ Quickstart guides for [Google Cloud](doc/admin/install/google_cloud.md), [AWS](d
How to [secure your setup](/doc/security/ssl.md). How to [secure your setup](/doc/security/ssl.md).
## Build ## Build
- Run `yarn build ${vscodeVersion}`in this directory (for example, `yarn build 1.35.0`). - If you also plan on developing, set the `OUT` environment variable: `
export OUT=/path/to/some/directory`. Otherwise it will build in this
directory which will cause issues because then `yarn watch` will try to
compile the build directory as well.
- Run `yarn build ${vscodeVersion} ${target} ${arch}`in this directory (for example:
`yarn build 1.35.0 linux x64`).
## Development ## Development
- Clone VS Code. - Clone VS Code.
- Clone code-server to `src/vs/server` in the VS Code source. - Run `yarn` in the VS Code root directory.
- Run `yarn` in this directory (only need to do this once). - Clone this repository to `src/vs/server` in the VS Code source.
- Run `yarn` in this directory.
- Run `yarn watch` in this directory. - Run `yarn watch` in this directory.
- Wait for the initial compilation to complete. - Wait for the initial compilation to complete.
- Run `yarn start` in this directory. - Run `yarn start` in this directory.

10
main.js
View File

@ -1 +1,11 @@
try {
const nbin = require("nbin");
const path = require("path");
const rootPath = path.resolve(__dirname, "../../..");
console.log("Shimming", rootPath);
nbin.shimNativeFs(rootPath);
} catch (error) {
console.log("Not in the binary");
}
require("../../bootstrap-amd").load("vs/server/cli"); require("../../bootstrap-amd").load("vs/server/cli");

View File

@ -1,8 +1,13 @@
{ {
"license": "MIT",
"scripts": { "scripts": {
"postinstall": "rm -r node_modules/@types/node # I keep getting type conflicts",
"start": "nodemon ../../../out/vs/server/main.js --watch ../../../out --verbose", "start": "nodemon ../../../out/vs/server/main.js --watch ../../../out --verbose",
"watch": "cd ../../../ && yarn watch-client", "watch": "cd ../../../ && yarn watch-client",
"build": "echo TODO && exit 1" "build": "bash ./scripts/tasks.bash build",
"package": "bash ./scripts/tasks.bash package",
"vstar": "bash ./scripts/tasks.bash vstar",
"binary": "bash ./scripts/tasks.bash binary"
}, },
"devDependencies": { "devDependencies": {
"@types/tar-stream": "^1.6.1", "@types/tar-stream": "^1.6.1",

View File

@ -1,51 +0,0 @@
#!/bin/bash
set -euxo pipefail
# Build using a Docker container using the specified image and version.
function docker_build() {
local image="${1}" ; shift
local version="${1}" ; shift
local containerId
containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}")
docker start "${containerId}"
docker exec "${containerId}" mkdir -p /src
function docker_exec() {
docker exec "${containerId}" bash -c "$@"
}
docker cp ./. "${containerId}":/src
docker_exec "cd /src && yarn"
docker_exec "cd /src && npm rebuild"
docker_exec "cd /src && NODE_ENV=production VERSION=${version} yarn task build:server:binary"
docker_exec "cd /src && yarn task package ${version}"
docker cp "${containerId}":/src/release/. ./release/
docker stop "${containerId}"
}
function main() {
local version=${VERSION:-}
local ostype=${OSTYPE:-}
if [[ -z "${version}" ]] ; then
>&2 echo "Must set VERSION environment variable"
exit 1
fi
if [[ "${ostype}" == "darwin"* ]]; then
NODE_ENV=production VERSION="${version}" yarn task build:server:binary
yarn task package "${version}"
else
local image
if [[ "$TARGET" == "alpine" ]]; then
image="codercom/nbin-alpine"
else
image="codercom/nbin-centos"
fi
docker_build "${image}" "${version}"
fi
}
main "$@"

64
scripts/ci.bash Executable file
View File

@ -0,0 +1,64 @@
#!/bin/bash
set -euo pipefail
# Build using a Docker container using the specified image and version.
function docker-build() {
local image="${1}" ; shift
local version="${1}" ; shift
local vscodeVersion="${1}" ; shift
local target="${1}" ; shift
local arch="${1}" ; shift
local containerId
containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}")
docker start "${containerId}"
docker exec "${containerId}" mkdir -p /src
function docker-exec() {
docker exec "${containerId}" bash -c "$@"
}
docker cp ./. "${containerId}":/src
docker-exec "cd /src && CI=true yarn build \"${vscodeVersion}\" \"${target}\" \"${arch}\""
docker-exec "cd /src && CI=true yarn binary \"${vscodeVersion}\" \"${target}\" \"${arch}\""
docker-exec "cd /src && CI=true yarn package \"${vscodeVersion}\" \"${target}\" \"${arch}\" \"${version}\""
docker cp "${containerId}":/src/release/. ./release/
docker stop "${containerId}"
}
# Build code-server in the CI.
function main() {
local version="${VERSION:-}"
local vscodeVersion="${VSCODE_VERSION:-}"
local ostype="${OSTYPE:-}"
local target="${TARGET:-}"
local arch=x64
if [[ -z "${version}" ]] ; then
>&2 echo "Must set VERSION environment variable"; exit 1
fi
if [[ -z "${vscodeVersion}" ]] ; then
>&2 echo "Must set VSCODE_VERSION environment variable"; exit 1
fi
if [[ "${ostype}" == "darwin"* ]]; then
target=darwin
CI=true yarn build "${vscodeVersion}" "${target}" "${arch}"
CI=true yarn binary "${vscodeVersion}" "${target}" "${arch}"
CI=true yarn package "${vscodeVersion}" "${target}" "${arch}" "${version}"
else
local image
if [[ "${target}" == alpine ]]; then
image=codercom/nbin-alpine
target=musl
else
image=codercom/nbin-centos
target=linux
fi
docker-build "${image}" "${version}" "${vscodeVersion}" "${target}" "${arch}"
fi
}
main "$@"

24
scripts/nbin.js Normal file
View File

@ -0,0 +1,24 @@
/* global require, __dirname, process */
const { Binary } = require("@coder/nbin");
const fs = require("fs");
const path = require("path");
const target = process.argv[2];
const arch = process.argv[3];
const source = process.argv[4];
const bin = new Binary({
mainFile: path.join(source, "out/vs/server/main.js"),
target: target,
});
bin.writeFiles(path.join(source, "**"));
bin.build().then((binaryData) => {
const outputPath = path.join(source, "code-server");
fs.writeFileSync(outputPath, binaryData);
fs.chmodSync(outputPath, "755");
}).catch((ex) => {
console.error(ex);
process.exit(1);
});

250
scripts/tasks.bash Executable file
View File

@ -0,0 +1,250 @@
#!/bin/bash
set -euo pipefail
function log() {
local message="${1}" ; shift
local level="${1:-info}"
if [[ "${level}" == "error" ]] ; then
>&2 echo "${message}"
else
echo "${message}"
fi
}
function exit-if-ci() {
if [[ -n "${ci}" ]] ; then
log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} is incorrectly built" "error"
exit 1
fi
}
# Copy code-server into VS Code along with its dependencies.
function copy-server() {
local serverPath="${vscodeSourcePath}/src/vs/server"
rm -rf "${serverPath}"
mkdir -p "${serverPath}"
log "Copying server code"
cp "${rootPath}"/*.{ts,js} "${serverPath}"
cp "${rootPath}/package.json" "${serverPath}"
cp "${rootPath}/yarn.lock" "${serverPath}"
if [[ -d "${rootPath}/node_modules" ]] ; then
log "Copying dependencies"
cp -r "${rootPath}/node_modules" "${serverPath}"
else
log "Installing dependencies"
cd "${serverPath}"
yarn
rm -r node_modules/@types/node # I keep getting type conflicts
fi
}
# Copy code-server into VS Code then build it.
function build-code-server() {
copy-server
# TODO: look into making it do the full minified build for just our code
# (basically just want to skip extensions, target our server code, and get
# the same type of build you get with the vscode-linux-x64-min task).
# Something like: yarn gulp "vscode-server-${target}-${arch}-min"
cd "${vscodeSourcePath}"
yarn gulp compile-client
rm -rf "${codeServerBuildPath}"
mkdir -p "${codeServerBuildPath}"
cp -r "${vscodeBuildPath}/resources/app/extensions" "${codeServerBuildPath}"
cp -r "${vscodeBuildPath}/resources/app/"*.json "${codeServerBuildPath}"
cp -r "${vscodeSourcePath}/out" "${codeServerBuildPath}"
rm -rf "${codeServerBuildPath}/out/vs/server/node_modules"
cp -r "${vscodeSourcePath}/remote/node_modules" "${codeServerBuildPath}"
log "Final build: ${codeServerBuildPath}"
}
# Build VS Code if it hasn't already been built. If we're in the CI and it's
# not fully built, error and exit.
function build-vscode() {
if [[ ! -d "${vscodeSourcePath}" ]] ; then
exit-if-ci
log "${vscodeSourceName} does not exist, cloning"
git clone https://github.com/microsoft/vscode --quiet \
--branch "${vscodeVersion}" --single-branch --depth=1 \
"${vscodeSourcePath}"
else
log "${vscodeSourceName} already exists, skipping clone"
fi
cd "${vscodeSourcePath}"
if [[ ! -d "${vscodeSourcePath}/node_modules" ]] ; then
exit-if-ci
log "Installing VS Code dependencies"
yarn
# Not entirely sure why but there seem to be problems with native modules.
# Also vscode-ripgrep keeps complaining after the rebuild that the
# node_modules directory doesn't exist, so we're ignoring that for now.
npm rebuild || true
# Keep just what we need to keep the pre-built archive smaller.
rm -rf "${vscodeSourcePath}/.git"
rm -rf "${vscodeSourcePath}/test"
else
log "${vscodeSourceName}/node_modules already exists, skipping install"
fi
if [[ ! -d "${vscodeBuildPath}" ]] ; then
exit-if-ci
log "${vscodeBuildName} does not exist, building"
local builtPath="${buildPath}/VSCode-${target}-${arch}"
rm -rf "${builtPath}"
yarn gulp "vscode-${target}-${arch}-min" --max-old-space-size=32384
mkdir -p "${vscodeBuildPath}/resources/app"
# Copy just what we need to keep the pre-built archive smaller.
mv "${builtPath}/resources/app/extensions" "${vscodeBuildPath}/resources/app"
mv "${builtPath}/resources/app/"*.json "${vscodeBuildPath}/resources/app"
rm -rf "${builtPath}"
else
log "${vscodeBuildName} already exists, skipping build"
fi
}
# Download VS Code with either curl or wget depending on which is available.
function download-vscode() {
cd "${buildPath}"
if command -v wget &> /dev/null ; then
log "Attempting to download ${tarName} with wget"
wget "${vsSourceUrl}" --quiet
else
log "Attempting to download ${tarName} with curl"
curl "${vsSourceUrl}" --silent --fail --output "${tarName}"
fi
}
# Download pre-built VS Code if necessary. Build if there is no available
# download but not when in the CI. The pre-built package basically just
# provides us the dependencies and extensions so we don't have to install and
# build them respectively which takes a long time.
function prepare-vscode() {
if [[ ! -d "${vscodeBuildPath}" || ! -d "${vscodeSourcePath}" ]] ; then
mkdir -p "${buildPath}"
local tarName="vstar-${vscodeVersion}-${target}-${arch}.tar.gz"
local vsSourceUrl="https://codesrv-ci.cdr.sh/${tarName}"
if download-vscode ; then
cd "${buildPath}"
rm -rf "${vscodeBuildPath}"
tar -xzf "${tarName}"
rm "${tarName}"
elif [[ -n "${ci}" ]] ; then
log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} does not exist" "error"
exit 1
else
log "${tarName} does not exist, building"
build-vscode
return
fi
else
log "VS Code is already downloaded or built"
fi
log "Ensuring VS Code is fully built"
build-vscode
}
function build-task() {
prepare-vscode
build-code-server
}
function vstar-task() {
local archivePath="${releasePath}/vstar-${vscodeVersion}-${target}-${arch}.tar.gz"
rm -f "${archivePath}"
mkdir -p "${releasePath}"
tar -C "${buildPath}" -czf "${archivePath}" "${vscodeSourceName}" "${vscodeBuildName}"
log "Archive: ${archivePath}"
}
function package-task() {
local version="${1}" ; shift
log " version: ${version}"
local archiveName="code-server${version}-vsc${vscodeVersion}-${target}-${arch}"
local archivePath="${releasePath}/${archiveName}"
rm -rf "${archivePath}"
mkdir -p "${archivePath}"
cp "${buildPath}/code-server" "${archivePath}"
cp "${rootPath}/README.md" "${archivePath}"
cp "${vscodeSourcePath}/LICENSE.txt" "${archivePath}"
cp "${vscodeSourcePath}/ThirdPartyNotices.txt" "${archivePath}"
cd "${releasePath}"
if [[ "${target}" == "linux" ]] ; then
tar -czf "${archiveName}.tar.gz" "${archiveName}"
else
zip -r "${archiveName}.zip" "${archiveName}"
fi
log "Archive: ${archivePath}"
}
# Package built code into a binary.
function binary-task() {
# I had trouble getting VS Code to build with the @coder/nbin dependency due
# to the types it installs (tons of conflicts), so for now it's a global
# dependency.
cd "${rootPath}"
npm link @coder/nbin
node "${rootPath}/scripts/nbin.js" "${target}" "${arch}" "${codeServerBuildPath}"
rm node_modules/@coder/nbin
mv "${codeServerBuildPath}/code-server" "${buildPath}"
log "Binary at ${buildPath}/code-server"
}
function main() {
local task="${1}" ; shift
local vscodeVersion="${1}" ; shift
local target="${1}" ; shift
local arch="${1}" ; shift
local ci="${CI:-}"
local relativeRootPath
local rootPath
relativeRootPath="$(dirname "${0}")/.."
rootPath="$(realpath "${relativeRootPath}")"
# This lets you build in a separate directory since building within this
# directory while developing makes it hard to keep developing since compiling
# will compile everything in the build directory as well.
local outPath="${OUT:-${rootPath}}"
local releasePath="${outPath}/release"
local buildPath="${outPath}/build"
local vscodeSourceName="vscode-${vscodeVersion}-source"
local vscodeBuildName="vscode-${vscodeVersion}-${target}-${arch}-built"
local vscodeSourcePath="${buildPath}/${vscodeSourceName}"
local vscodeBuildPath="${buildPath}/${vscodeBuildName}"
local codeServerBuildName="code-server-${target}-${arch}-built"
local codeServerBuildPath="${buildPath}/${codeServerBuildName}"
log "Running ${task} task"
log " rootPath: ${rootPath}"
log " outPath: ${outPath}"
log " vscodeVersion: ${vscodeVersion}"
log " target: ${target}"
log " arch: ${arch}"
if [[ -n "${ci}" ]] ; then
log " CI: yes"
else
log " CI: no"
fi
"${task}-task" "$@"
}
main "$@"

View File

@ -1,24 +0,0 @@
#!/bin/bash
set -euxo pipefail
# Builds a tarfile containing vscode sourcefiles neccessary for CI.
# Done outside the CI and uploaded to object storage to reduce CI time.
branch=1.33.1
dir=/tmp/vstar
outfile=/tmp/vstar-$branch.tar.gz
rm -rf $dir
mkdir -p $dir
cd $dir
git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
cd vscode
yarn
npx gulp vscode-linux-x64 --max-old-space-size=32384
rm -rf extensions build out* test
cd ..
mv *-x64/resources/app/extensions ./extensions
rm -rf *-x64
tar -czvf $outfile .

View File

@ -3,9 +3,9 @@
"@types/node@*", "@types/node@^10.12.12": "@types/node@*", "@types/node@^10.12.12":
version "10.14.10" version "10.14.12"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.10.tgz#e491484c6060af8d461e12ec81c0da8a3282b8de" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.12.tgz#0eec3155a46e6c4db1f27c3e588a205f767d622f"
integrity sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q== integrity sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==
"@types/tar-stream@^1.6.1": "@types/tar-stream@^1.6.1":
version "1.6.1" version "1.6.1"