From be032cf735707680e333eeb87ebdcb2837658030 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Thu, 30 Apr 2020 07:52:54 -0400 Subject: [PATCH 1/2] Add NPM package, debs, rpms and refactor CI/build process Closes many issues that I'll prune after adding more docs for users. --- .dockerignore | 3 +- .gitignore | 8 +- .travis.yml | 48 ++- ci/README.md | 82 ++++ ci/build.ts | 372 ------------------ ci/build/archive-static-release.sh | 41 ++ ci/build/build-code-server.sh | 29 ++ ci/build/build-release.sh | 127 ++++++ ci/build/build-static-pkgs.sh | 24 ++ ci/build/build-vscode.sh | 21 + ci/{ => build}/clean.sh | 3 +- ci/build/code-server-nfpm.sh | 3 + ci/{ => build}/code-server.sh | 14 +- ci/build/nfpm.yaml | 16 + .../test-static-release.sh} | 17 +- ci/container/Dockerfile | 30 ++ ci/container/exec.sh | 24 ++ ci/dev-image/exec.sh | 49 --- ci/{ => dev}/ci.sh | 2 +- ci/{dev-image => dev/container}/Dockerfile | 0 ci/dev/container/exec.sh | 48 +++ ci/{ => dev}/fmt.sh | 3 +- ci/{ => dev}/lint.sh | 4 +- ci/dev/test.sh | 10 + ci/{ => dev}/vscode.patch | 4 +- ci/{ => dev}/vscode.sh | 2 +- ci/dev/watch.ts | 163 ++++++++ ci/image/Dockerfile | 23 -- ci/image/run.sh | 26 -- ci/lib.sh | 47 ++- .../Dockerfile | 15 +- .../push.sh | 13 +- ci/release.sh | 76 ---- ci/steps/linux-release.sh | 26 ++ ci/steps/publish-npm.sh | 11 + ci/steps/static-release.sh | 16 + ci/steps/test.sh | 17 + ci/tsconfig.json | 4 - doc/CONTRIBUTING.md | 9 +- package.json | 53 ++- src/node/app/README.md | 2 + yarn.lock | 13 +- 42 files changed, 867 insertions(+), 631 deletions(-) create mode 100644 ci/README.md delete mode 100644 ci/build.ts create mode 100755 ci/build/archive-static-release.sh create mode 100755 ci/build/build-code-server.sh create mode 100755 ci/build/build-release.sh create mode 100755 ci/build/build-static-pkgs.sh create mode 100755 ci/build/build-vscode.sh rename ci/{ => build}/clean.sh (85%) create mode 100755 ci/build/code-server-nfpm.sh rename ci/{ => build}/code-server.sh (64%) create mode 100644 ci/build/nfpm.yaml rename ci/{build-test.sh => build/test-static-release.sh} (57%) create mode 100644 ci/container/Dockerfile create mode 100755 ci/container/exec.sh delete mode 100755 ci/dev-image/exec.sh rename ci/{ => dev}/ci.sh (76%) rename ci/{dev-image => dev/container}/Dockerfile (100%) create mode 100755 ci/dev/container/exec.sh rename ci/{ => dev}/fmt.sh (95%) rename ci/{ => dev}/lint.sh (69%) create mode 100755 ci/dev/test.sh rename ci/{ => dev}/vscode.patch (99%) rename ci/{ => dev}/vscode.sh (92%) create mode 100644 ci/dev/watch.ts delete mode 100644 ci/image/Dockerfile delete mode 100755 ci/image/run.sh mode change 100644 => 100755 ci/lib.sh rename ci/{release-image => release-container}/Dockerfile (61%) rename ci/{release-image => release-container}/push.sh (57%) delete mode 100755 ci/release.sh create mode 100755 ci/steps/linux-release.sh create mode 100755 ci/steps/publish-npm.sh create mode 100755 ci/steps/static-release.sh create mode 100755 ci/steps/test.sh delete mode 100644 ci/tsconfig.json diff --git a/.dockerignore b/.dockerignore index 75cbb64ea..b8940df74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ ** -!release +!release-github +!ci diff --git a/.gitignore b/.gitignore index fdb006ed8..86557de45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ -*.tsbuildinfo +.tsbuildinfo .cache -build dist* out* release/ -release-upload/ +release-static/ +release-github/ +release-gcp/ node_modules -binaries diff --git a/.travis.yml b/.travis.yml index 9e7add708..7b2f778de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,33 @@ -language: minimal +language: node_js +node_js: node jobs: include: - name: Test if: tag IS blank - script: ./ci/image/run.sh "yarn && git submodule update --init && yarn vscode:patch && ./ci/ci.sh" + script: ./ci/container/exec.sh ./ci/steps/test.sh deploy: null + install: null - name: Linux Release if: tag IS present - script: - - travis_wait 60 ./ci/image/run.sh "yarn && yarn vscode && ci/release.sh && ./ci/build-test.sh" - - ./ci/release-image/push.sh - - name: Linux ARM64 Release + script: ./ci/steps/linux-release.sh + install: null + - name: Linux Release if: tag IS present - script: - - ./ci/image/run.sh "yarn && yarn vscode && ci/release.sh && ./ci/build-test.sh" - - ./ci/release-image/push.sh arch: arm64 + script: | + sudo apt-get update && sudo apt-get install -y jq || exit 1 + ./ci/steps/linux-release.sh + install: null - name: MacOS Release if: tag IS present os: osx - language: node_js + # node 13/14 crashes in the build process for some reason. node_js: 12 - script: yarn && yarn vscode && travis_wait 60 ci/release.sh && ./ci/build-test.sh + script: | + HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install jq || exit 1 + travis_wait 60 ./ci/steps/static-release.sh || exit 1 + install: null before_deploy: - echo "$JSON_KEY" | base64 --decode > ./ci/key.json @@ -36,8 +41,10 @@ deploy: target_commitish: $TRAVIS_COMMIT name: $TRAVIS_TAG file: - - release/*.tar.gz - - release/*.zip + - release-github/*.tar.gz + - release-github/*.zip + - release-github/*.deb + - release-github/*.rpm on: tags: true - provider: gcs @@ -45,14 +52,25 @@ deploy: bucket: "codesrv-ci.cdr.sh" upload_dir: "releases" key_file: ./ci/key.json - local_dir: release-upload + local_dir: ./release-gcp on: tags: true # TODO: The gcs provider fails to install on arm64. - condition: $TRAVIS_CPU_ARCH = amd64 + condition: $TRAVIS_CPU_ARCH == amd64 + - provider: script + edge: true + # We do not use the travis npm deploy integration as it does not allow us to + # deploy a subpath and and v2 which should, just errors out that the src does not exist + script: ./ci/steps/publish-npm.sh + on: + tags: true + condition: $TRAVIS_CPU_ARCH == amd64 && $TRAVIS_OS_NAME == linux cache: timeout: 600 yarn: true directories: + - .cache + - out + - dist - lib/vscode/.build/extensions diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000..5c1c1c2bb --- /dev/null +++ b/ci/README.md @@ -0,0 +1,82 @@ +# ci + +This directory contains scripts used for code-server's continuous integration infrastructure. + +Many of these scripts contain more detailed documentation and options in comments at the top. + +Any file and directory added into this tree should be documented here. + +## dev + +This directory contains scripts used for the development of code-server. + +- [./dev/container](./dev/container) + - See [CONTRIBUTING.md](../doc/CONTRIBUTING.md) for docs on the development container +- [./dev/ci.sh](./dev/ci.sh) (`yarn ci`) + - Runs formatters, linters and tests +- [./dev/fmt.sh](./dev/fmt.sh) (`yarn fmt`) + - Runs formatters +- [./dev/lint.sh](./dev/lint.sh) (`yarn lint`) + - Runs linters +- [./dev/test.sh](./dev/test.sh) (`yarn test`) + - Runs tests +- [./dev/vscode.sh](./dev/vscode.sh) (`yarn vscode`) + - Ensures `lib/vscode` is cloned, patched and dependencies are installed +- [./dev/vscode.patch](./dev/vscode.patch) + - Our patch of VS Code to enable remote browser access + - Generate it with `yarn vscode:diff` and apply with `yarn vscode:patch` +- [./dev/watch.ts](./dev/watch.ts) (`yarn watch`) + - Starts a process to build and launch code-server and restart on any code changes + - Example usage in [CONTRIBUTING.md](../doc/CONTRIBUTING.md) + +## build + +This directory contains the scripts used to build code-server. + +- [./build/build-code-server.sh](./build/build-code-server.sh) (`yarn build`) + - Builds code-server into ./out and bundles the frontend into ./dist. +- [./build/build-vscode.sh](./build/build-vscode.sh) (`yarn build:vscode`) + - Builds vscode into ./lib/vscode/out-vscode. +- [./build/build-release.sh](./build/build-release.sh) (`yarn release`) + - Bundles the output of the above two scripts into a single node module at ./release. + - Will build a static release with node/node_modules into `./release-static` + if `STATIC=1` is set. +- [./build/clean.sh](./build/clean.sh) (`yarn clean`) + - Removes all git ignored files like build artifacts. + - Will also `git reset --hard lib/vscode` + - Useful to do a clean build. +- [./build/code-server.sh](./build/code-server.sh) + - Copied into static releases to run code-server with the bundled node binary. +- [./build/archive-static-release.sh](./build/archive-static-release.sh) + - Archives `./release-static` into a tar/zip for CI with the proper directory name scheme +- [./build/test-release.sh](./build/test-static-release.sh) + - Ensures code-server in the `./release-static` directory runs +- [./build/build-static-pkgs.sh](./build/build-static-pkgs.sh) (`yarn pkg`) + - Uses [nfpm](https://github.com/goreleaser/nfpm) to generate .deb and .rpm from a static release +- [./build/nfpm.yaml](./build/nfpm.yaml) + - Used to configure [nfpm](https://github.com/goreleaser/nfpm) to generate .deb and .rpm +- [./build/code-server-nfpm.sh](./build/code-server-nfpm.sh) + - Entrypoint script for code-server for .deb and .rpm + +## release-container + +This directory contains the release docker container. + +## container + +This directory contains the container for CI. + +## steps + +This directory contains a few scripts used in CI. Just helps avoid clobbering .travis.yml. + +- [./steps/test.sh](./steps/test.sh) + - Runs `yarn ci` after ensuring VS Code is patched +- [./steps/static-release.sh](./steps/static-release.sh) + - Runs the full static build process for CI +- [./steps/linux-release.sh](./steps/linux-release.sh) + - Runs the full static build process for CI + - Packages the release into a .deb and .rpm + - Builds and pushes a docker release +- [./steps/publish-npm.sh](./steps/publish-npm.sh) + - Authenticates yarn and publishes the built package from `./release` diff --git a/ci/build.ts b/ci/build.ts deleted file mode 100644 index 9cf0b5bd7..000000000 --- a/ci/build.ts +++ /dev/null @@ -1,372 +0,0 @@ -import * as cp from "child_process" -import * as fs from "fs-extra" -import Bundler from "parcel-bundler" -import * as path from "path" -import * as util from "util" - -enum Task { - Build = "build", - Watch = "watch", -} - -class Builder { - private readonly rootPath = path.resolve(__dirname, "..") - private readonly vscodeSourcePath = path.join(this.rootPath, "lib/vscode") - private readonly buildPath = path.join(this.rootPath, "build") - private readonly codeServerVersion: string - private currentTask?: Task - - public constructor() { - this.ensureArgument("rootPath", this.rootPath) - this.codeServerVersion = this.ensureArgument( - "codeServerVersion", - process.env.VERSION || require(path.join(this.rootPath, "package.json")).version, - ) - } - - public run(task: Task | undefined): void { - this.currentTask = task - this.doRun(task).catch((error) => { - console.error(error.message) - process.exit(1) - }) - } - - private async task(message: string, fn: () => Promise): Promise { - const time = Date.now() - this.log(`${message}...`, !process.env.CI) - try { - const t = await fn() - process.stdout.write(`took ${Date.now() - time}ms\n`) - return t - } catch (error) { - process.stdout.write("failed\n") - throw error - } - } - - /** - * Writes to stdout with an optional newline. - */ - private log(message: string, skipNewline = false): void { - process.stdout.write(`[${this.currentTask || "default"}] ${message}`) - if (!skipNewline) { - process.stdout.write("\n") - } - } - - private async doRun(task: Task | undefined): Promise { - if (!task) { - throw new Error("No task provided") - } - - switch (task) { - case Task.Watch: - return this.watch() - case Task.Build: - return this.build() - default: - throw new Error(`No task matching "${task}"`) - } - } - - /** - * Make sure the argument is set. Display the value if it is. - */ - private ensureArgument(name: string, arg?: string): string { - if (!arg) { - throw new Error(`${name} is missing`) - } - this.log(`${name} is "${arg}"`) - return arg - } - - /** - * Build VS Code and code-server. - */ - private async build(): Promise { - process.env.NODE_OPTIONS = "--max-old-space-size=32384 " + (process.env.NODE_OPTIONS || "") - process.env.NODE_ENV = "production" - - await this.task("cleaning up old build", async () => { - if (!process.env.SKIP_VSCODE) { - return fs.remove(this.buildPath) - } - // If skipping VS Code, keep the existing build if any. - try { - const files = await fs.readdir(this.buildPath) - return Promise.all(files.filter((f) => f !== "lib").map((f) => fs.remove(path.join(this.buildPath, f)))) - } catch (error) { - if (error.code !== "ENOENT") { - throw error - } - } - }) - - const commit = require(path.join(this.vscodeSourcePath, "build/lib/util")).getVersion(this.rootPath) as string - if (!process.env.SKIP_VSCODE) { - await this.buildVscode(commit) - } else { - this.log("skipping vs code build") - } - await this.buildCodeServer(commit) - - this.log(`final build: ${this.buildPath}`) - } - - private async buildCodeServer(commit: string): Promise { - await this.task("building code-server", async () => { - return util.promisify(cp.exec)("tsc --outDir ./out-build --tsBuildInfoFile ./.prod.tsbuildinfo", { - cwd: this.rootPath, - }) - }) - - await this.task("bundling code-server", async () => { - return this.createBundler("dist-build", commit).bundle() - }) - - await this.task("copying code-server into build directory", async () => { - await fs.mkdirp(this.buildPath) - await Promise.all([ - fs.copy(path.join(this.rootPath, "out-build"), path.join(this.buildPath, "out")), - fs.copy(path.join(this.rootPath, "dist-build"), path.join(this.buildPath, "dist")), - // For source maps and images. - fs.copy(path.join(this.rootPath, "src"), path.join(this.buildPath, "src")), - ]) - }) - - await this.copyDependencies("code-server", this.rootPath, this.buildPath, false, { - commit, - version: this.codeServerVersion, - }) - } - - private async buildVscode(commit: string): Promise { - await this.task("building vs code", () => { - return util.promisify(cp.exec)("yarn gulp compile-build", { cwd: this.vscodeSourcePath }) - }) - - await this.task("building builtin extensions", async () => { - const exists = await fs.pathExists(path.join(this.vscodeSourcePath, ".build/extensions")) - if (exists && !process.env.CI) { - process.stdout.write("already built, skipping...") - } else { - await util.promisify(cp.exec)("yarn gulp compile-extensions-build", { cwd: this.vscodeSourcePath }) - } - }) - - await this.task("optimizing vs code", async () => { - return util.promisify(cp.exec)("yarn gulp optimize --gulpfile ./coder.js", { cwd: this.vscodeSourcePath }) - }) - - if (process.env.MINIFY) { - await this.task("minifying vs code", () => { - return util.promisify(cp.exec)("yarn gulp minify --gulpfile ./coder.js", { cwd: this.vscodeSourcePath }) - }) - } - - const vscodeBuildPath = path.join(this.buildPath, "lib/vscode") - await this.task("copying vs code into build directory", async () => { - await fs.mkdirp(path.join(vscodeBuildPath, "resources/linux")) - await Promise.all([ - fs.move( - path.join(this.vscodeSourcePath, `out-vscode${process.env.MINIFY ? "-min" : ""}`), - path.join(vscodeBuildPath, "out"), - ), - fs.copy(path.join(this.vscodeSourcePath, ".build/extensions"), path.join(vscodeBuildPath, "extensions")), - fs.copy( - path.join(this.vscodeSourcePath, "resources/linux/code.png"), - path.join(vscodeBuildPath, "resources/linux/code.png"), - ), - ]) - }) - - await this.copyDependencies("vs code", this.vscodeSourcePath, vscodeBuildPath, true, { - commit, - date: new Date().toISOString(), - }) - } - - private async copyDependencies( - name: string, - sourcePath: string, - buildPath: string, - ignoreScripts: boolean, - merge: object, - ): Promise { - await this.task(`copying ${name} dependencies`, async () => { - return Promise.all( - ["node_modules", "package.json", "yarn.lock"].map((fileName) => { - return fs.copy(path.join(sourcePath, fileName), path.join(buildPath, fileName)) - }), - ) - }) - - const fileName = name === "code-server" ? "package" : "product" - await this.task(`writing final ${name} ${fileName}.json`, async () => { - const json = JSON.parse(await fs.readFile(path.join(sourcePath, `${fileName}.json`), "utf8")) - return fs.writeFile( - path.join(buildPath, `${fileName}.json`), - JSON.stringify( - { - ...json, - ...merge, - }, - null, - 2, - ), - ) - }) - - if (process.env.MINIFY) { - await this.task(`restricting ${name} to production dependencies`, async () => { - await util.promisify(cp.exec)(`yarn --production ${ignoreScripts ? "--ignore-scripts" : ""}`, { - cwd: buildPath, - }) - }) - } - } - - private async watch(): Promise { - let server: cp.ChildProcess | undefined - const restartServer = (): void => { - if (server) { - server.kill() - } - const s = cp.fork(path.join(this.rootPath, "out/node/entry.js"), process.argv.slice(3)) - console.log(`[server] spawned process ${s.pid}`) - s.on("exit", () => console.log(`[server] process ${s.pid} exited`)) - server = s - } - - const vscode = cp.spawn("yarn", ["watch"], { cwd: this.vscodeSourcePath }) - const tsc = cp.spawn("tsc", ["--watch", "--pretty", "--preserveWatchOutput"], { cwd: this.rootPath }) - const bundler = this.createBundler() - - const cleanup = (code?: number | null): void => { - this.log("killing vs code watcher") - vscode.removeAllListeners() - vscode.kill() - - this.log("killing tsc") - tsc.removeAllListeners() - tsc.kill() - - if (server) { - this.log("killing server") - server.removeAllListeners() - server.kill() - } - - this.log("killing bundler") - process.exit(code || 0) - } - - process.on("SIGINT", () => cleanup()) - process.on("SIGTERM", () => cleanup()) - - vscode.on("exit", (code) => { - this.log("vs code watcher terminated unexpectedly") - cleanup(code) - }) - tsc.on("exit", (code) => { - this.log("tsc terminated unexpectedly") - cleanup(code) - }) - const bundle = bundler.bundle().catch(() => { - this.log("parcel watcher terminated unexpectedly") - cleanup(1) - }) - bundler.on("buildEnd", () => { - console.log("[parcel] bundled") - }) - bundler.on("buildError", (error) => { - console.error("[parcel]", error) - }) - - vscode.stderr.on("data", (d) => process.stderr.write(d)) - tsc.stderr.on("data", (d) => process.stderr.write(d)) - - // From https://github.com/chalk/ansi-regex - const pattern = [ - "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)", - "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))", - ].join("|") - const re = new RegExp(pattern, "g") - - /** - * Split stdout on newlines and strip ANSI codes. - */ - const onLine = (proc: cp.ChildProcess, callback: (strippedLine: string, originalLine: string) => void): void => { - let buffer = "" - if (!proc.stdout) { - throw new Error("no stdout") - } - proc.stdout.setEncoding("utf8") - proc.stdout.on("data", (d) => { - const data = buffer + d - const split = data.split("\n") - const last = split.length - 1 - - for (let i = 0; i < last; ++i) { - callback(split[i].replace(re, ""), split[i]) - } - - // The last item will either be an empty string (the data ended with a - // newline) or a partial line (did not end with a newline) and we must - // wait to parse it until we get a full line. - buffer = split[last] - }) - } - - let startingVscode = false - let startedVscode = false - onLine(vscode, (line, original) => { - console.log("[vscode]", original) - // Wait for watch-client since "Finished compilation" will appear multiple - // times before the client starts building. - if (!startingVscode && line.includes("Starting watch-client")) { - startingVscode = true - } else if (startingVscode && line.includes("Finished compilation")) { - if (startedVscode) { - bundle.then(restartServer) - } - startedVscode = true - } - }) - - onLine(tsc, (line, original) => { - // tsc outputs blank lines; skip them. - if (line !== "") { - console.log("[tsc]", original) - } - if (line.includes("Watching for file changes")) { - bundle.then(restartServer) - } - }) - } - - private createBundler(out = "dist", commit?: string): Bundler { - return new Bundler( - [ - path.join(this.rootPath, "src/browser/pages/app.ts"), - path.join(this.rootPath, "src/browser/register.ts"), - path.join(this.rootPath, "src/browser/serviceWorker.ts"), - ], - { - cache: true, - cacheDir: path.join(this.rootPath, ".cache"), - detailedReport: true, - minify: !!process.env.MINIFY, - hmr: false, - logLevel: 1, - outDir: path.join(this.rootPath, out), - publicUrl: `/static/${commit || "development"}/dist`, - target: "browser", - }, - ) - } -} - -const builder = new Builder() -builder.run(process.argv[2] as Task) diff --git a/ci/build/archive-static-release.sh b/ci/build/archive-static-release.sh new file mode 100755 index 000000000..37a856172 --- /dev/null +++ b/ci/build/archive-static-release.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generates static code-server releases for CI. +# This script assumes that a static release is built already. + +main() { + cd "$(dirname "${0}")/../.." + source ./ci/lib.sh + + VERSION="$(pkg_json_version)" + + local OS + OS="$(os)" + + local ARCH + ARCH="$(arch)" + + local archive_name="code-server-$VERSION-$OS-$ARCH" + mkdir -p release-github + + local ext + if [[ $OS == "linux" ]]; then + ext=".tar.gz" + tar -czf "release-github/$archive_name$ext" --transform "s/^\.\/release-static/$archive_name/" ./release-static + else + mv ./release-static "./$archive_name" + ext=".zip" + zip -r "release-github/$archive_name$ext" "./$archive_name" + mv "./$archive_name" ./release-static + fi + + echo "done (release-github/$archive_name)" + + mkdir -p "release-gcp/$VERSION" + cp "release-github/$archive_name$ext" "./release-gcp/$VERSION/$OS-$ARCH$ext" + mkdir -p "release-gcp/latest" + cp "./release-github/$archive_name$ext" "./release-gcp/latest/$OS-$ARCH$ext" +} + +main "$@" diff --git a/ci/build/build-code-server.sh b/ci/build/build-code-server.sh new file mode 100755 index 000000000..82668f4d9 --- /dev/null +++ b/ci/build/build-code-server.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Builds code-server into out and the frontend into dist. + +# MINIFY controls whether parcel minifies dist. +MINIFY=${MINIFY-true} + +main() { + cd "$(dirname "${0}")/../.." + + npx tsc --outDir out --tsBuildInfoFile .cache/out.tsbuildinfo + # If out/node/entry.js does not already have the shebang, + # we make sure to add it and make it executable. + if ! grep -q -m1 "^#!/usr/bin/env node" out/node/entry.js; then + sed -i.bak "1s;^;#!/usr/bin/env node\n;" out/node/entry.js && rm out/node/entry.js.bak + chmod +x out/node/entry.js + fi + + npx parcel build \ + --public-url "/static/$(git rev-parse HEAD)/dist" \ + --out-dir dist \ + $([[ $MINIFY ]] || echo --no-minify) \ + src/browser/pages/app.ts \ + src/browser/register.ts \ + src/browser/serviceWorker.ts +} + +main "$@" diff --git a/ci/build/build-release.sh b/ci/build/build-release.sh new file mode 100755 index 000000000..4a86874c8 --- /dev/null +++ b/ci/build/build-release.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script requires code-server and vscode to be built with +# matching MINIFY. + +# RELEASE_PATH is the destination directory for the release from the root. +# Defaults to release +RELEASE_PATH="${RELEASE_PATH-release}" + +# STATIC controls whether node and node_modules are packaged into the release. +# Disabled by default. +STATIC="${STATIC-}" + +# MINIFY controls whether minified vscode is bundled and whether +# any included node_modules are pruned for production. +MINIFY="${MINIFY-true}" + +VSCODE_SRC_PATH="lib/vscode" + +VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode" + +main() { + cd "$(dirname "${0}")/../.." + source ./ci/lib.sh + + mkdir -p "$RELEASE_PATH" + + bundle_code_server + bundle_vscode + + rsync README.md "$RELEASE_PATH" + rsync LICENSE.txt "$RELEASE_PATH" + rsync ./lib/vscode/ThirdPartyNotices.txt "$RELEASE_PATH" + + if [[ $STATIC ]]; then + rsync "$RELEASE_PATH/" "$RELEASE_PATH-static" + RELEASE_PATH+=-static + VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode" + + bundle_node + else + rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules" + fi +} + +rsync() { + command rsync -a --del "$@" +} + +bundle_code_server() { + rsync out dist "$RELEASE_PATH" + + # For source maps and images. + mkdir -p "$RELEASE_PATH/src/browser" + rsync src/browser/media/ "$RELEASE_PATH/src/browser/media" + mkdir -p "$RELEASE_PATH/src/browser/pages" + rsync src/browser/pages/*.html "$RELEASE_PATH/src/browser/pages" + + rsync yarn.lock "$RELEASE_PATH" + + # Adds the commit to package.json + jq --slurp '.[0] * .[1]' package.json <( + cat << EOF + { + "commit": "$(git rev-parse HEAD)", + "scripts": { + "postinstall": "cd lib/vscode && yarn --production && cd extensions && yarn --production" + } + } +EOF + ) > "$RELEASE_PATH/package.json" +} + +bundle_vscode() { + mkdir -p "$VSCODE_OUT_PATH" + rsync "$VSCODE_SRC_PATH/out-vscode${MINIFY+-min}/" "$VSCODE_OUT_PATH/out" + rsync "$VSCODE_SRC_PATH/.build/extensions/" "$VSCODE_OUT_PATH/extensions" + rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions" + rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions" + rsync "$VSCODE_SRC_PATH/extensions/postinstall.js" "$VSCODE_OUT_PATH/extensions" + + mkdir -p "$VSCODE_OUT_PATH/resources/linux" + rsync "$VSCODE_SRC_PATH/resources/linux/code.png" "$VSCODE_OUT_PATH/resources/linux/code.png" + + rsync "$VSCODE_SRC_PATH/yarn.lock" "$VSCODE_OUT_PATH" + + # Adds the commit and date to product.json + jq --slurp '.[0] * .[1]' "$VSCODE_SRC_PATH/product.json" <( + cat << EOF + { + "commit": "$(git rev-parse HEAD)", + "date": $(jq -n 'now | todate') + } +EOF + ) > "$VSCODE_OUT_PATH/product.json" + + # We remove the scripts field so that later on we can run + # yarn to fetch node_modules if necessary without build scripts + # being ran. + # We cannot use --no-scripts because we still want dependant package scripts to run + # for native modules to be rebuilt. + jq 'del(.scripts)' < "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json" +} + +bundle_node() { + # We cannot find the path to node from $PATH because yarn shims a script to ensure + # we use the same version it's using so we instead run a script with yarn that + # will print the path to node. + local node_path + node_path="$(yarn -s node <<< 'console.info(process.execPath)')" + + mkdir -p "$RELEASE_PATH/bin" + rsync ./ci/build/code-server.sh "$RELEASE_PATH/bin/code-server" + rsync "$node_path" "$RELEASE_PATH/lib/node" + + rsync node_modules "$RELEASE_PATH" + rsync "$VSCODE_SRC_PATH/node_modules" "$VSCODE_OUT_PATH" + + if [[ $MINIFY ]]; then + pushd "$RELEASE_PATH" + yarn --production + popd + fi +} + +main "$@" diff --git a/ci/build/build-static-pkgs.sh b/ci/build/build-static-pkgs.sh new file mode 100755 index 000000000..659039aea --- /dev/null +++ b/ci/build/build-static-pkgs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generates deb and rpm packages for CI. +# Assumes a static release has already been built. + +main() { + cd "$(dirname "${0}")/../.." + source ./ci/lib.sh + + VERSION="$(pkg_json_version)" + export VERSION + + ARCH="$(arch)" + export ARCH + + local nfpm_config + nfpm_config=$(envsubst < ./ci/build/nfpm.yaml) + + nfpm pkg -f <(echo "$nfpm_config") --target release-github/code-server-"$VERSION-$ARCH.deb" + nfpm pkg -f <(echo "$nfpm_config") --target release-github/code-server-"$VERSION-$ARCH.rpm" +} + +main "$@" diff --git a/ci/build/build-vscode.sh b/ci/build/build-vscode.sh new file mode 100755 index 000000000..a7368db3a --- /dev/null +++ b/ci/build/build-vscode.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Builds vscode into lib/vscode/out-vscode. + +# MINIFY controls whether a minified version of vscode is built. +MINIFY=${MINIFY-true} + +main() { + cd "$(dirname "${0}")/../.." + cd lib/vscode + + yarn gulp compile-build + yarn gulp compile-extensions-build + yarn gulp optimize --gulpfile ./coder.js + if [[ $MINIFY ]]; then + yarn gulp minify --gulpfile ./coder.js + fi +} + +main "$@" diff --git a/ci/clean.sh b/ci/build/clean.sh similarity index 85% rename from ci/clean.sh rename to ci/build/clean.sh index df4a6bebf..df8da6de0 100755 --- a/ci/clean.sh +++ b/ci/build/clean.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash - set -euo pipefail main() { + cd "$(dirname "${0}")/../.." + git clean -Xffd git submodule foreach --recursive git clean -xffd git submodule foreach --recursive git reset --hard diff --git a/ci/build/code-server-nfpm.sh b/ci/build/code-server-nfpm.sh new file mode 100755 index 000000000..e12f493ba --- /dev/null +++ b/ci/build/code-server-nfpm.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +exec /usr/lib/code-server/bin/code-server "$@" diff --git a/ci/code-server.sh b/ci/build/code-server.sh similarity index 64% rename from ci/code-server.sh rename to ci/build/code-server.sh index 0d28736a8..8811558d5 100755 --- a/ci/code-server.sh +++ b/ci/build/code-server.sh @@ -1,18 +1,20 @@ #!/usr/bin/env sh + +# This script is intended to be bundled into the static releases. # Runs code-server with the bundled Node binary. # More complicated than readlink -f or realpath to support macOS. # See https://github.com/cdr/code-server/issues/1537 -get_installation_dir() { +bin_dir() { # We read the symlink, which may be relative from $0. dst="$(readlink "$0")" # We cd into the $0 directory. - cd "$(dirname "$0")" + cd "$(dirname "$0")" || exit 1 # Now we can cd into the dst directory. - cd "$(dirname "$dst")" + cd "$(dirname "$dst")" || exit 1 # Finally we use pwd -P to print the absolute path of the directory of $dst. - pwd -P + pwd -P || exit 1 } -dir=$(get_installation_dir) -exec "$dir/node" "$dir/out/node/entry.js" "$@" +BIN_DIR=$(bin_dir) +exec "$BIN_DIR/../lib/node" "$BIN_DIR/.." "$@" diff --git a/ci/build/nfpm.yaml b/ci/build/nfpm.yaml new file mode 100644 index 000000000..efea4ece0 --- /dev/null +++ b/ci/build/nfpm.yaml @@ -0,0 +1,16 @@ +name: "code-server" +arch: "${ARCH}" +platform: "linux" +version: "v${VERSION}" +section: "devel" +priority: "optional" +maintainer: "Anmol Sethi " +description: | + Run VS Code in the browser. +vendor: "Coder" +homepage: "https://github.com/cdr/code-server" +license: "MIT" +bindir: "/usr/bin" +files: + ./ci/build/code-server-nfpm.sh: /usr/bin/code-server + ./release-static/**/*: "/usr/lib/code-server/" diff --git a/ci/build-test.sh b/ci/build/test-static-release.sh similarity index 57% rename from ci/build-test.sh rename to ci/build/test-static-release.sh index 804e22ea6..02177fe31 100755 --- a/ci/build-test.sh +++ b/ci/build/test-static-release.sh @@ -1,21 +1,20 @@ #!/usr/bin/env bash -# build-test.bash -- Make sure the build worked. -# This is to make sure we don't have Node version errors or any other -# compilation-related errors. - set -euo pipefail -function main() { - cd "$(dirname "${0}")/.." || exit 1 +# Makes sure the release works. +# This is to make sure we don't have Node version errors or any other +# compilation-related errors. +main() { + cd "$(dirname "${0}")/../.." local output - output=$(node ./build/out/node/entry.js --list-extensions 2>&1) + output=$(./release-static/bin/code-server --list-extensions 2>&1) if echo "$output" | grep 'was compiled against a different Node.js version'; then echo "$output" exit 1 - else - echo "Build ran successfully" fi + + echo "Build ran successfully" } main "$@" diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile new file mode 100644 index 000000000..dcbdd6aa1 --- /dev/null +++ b/ci/container/Dockerfile @@ -0,0 +1,30 @@ +FROM centos:7 + +RUN yum update -y && yum install -y \ + devtoolset-6 \ + gcc-c++ \ + xz \ + ccache \ + git \ + wget \ + openssl \ + libxkbfile-devel \ + libsecret-devel \ + libx11-devel \ + gettext + +RUN yum install -y epel-release && \ + yum install -y ShellCheck jq golang + +RUN go get github.com/goreleaser/nfpm/cmd/nfpm +ENV PATH=$PATH:/root/go/bin + +RUN mkdir /usr/share/node && cd /usr/share/node \ + && curl "https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-$(uname -m | sed 's/86_//; s/aarch/arm/').tar.xz" | tar xJ --strip-components=1 -- +ENV PATH "$PATH:/usr/share/node/bin" +RUN npm install -g yarn@1.22.4 + +RUN curl -L "https://github.com/mvdan/sh/releases/download/v3.0.1/shfmt_v3.0.1_linux_$(uname -m | sed 's/x86_/amd/; s/aarch64/arm/')" > /usr/local/bin/shfmt \ + && chmod +x /usr/local/bin/shfmt + +ENTRYPOINT ["/bin/bash", "-c"] diff --git a/ci/container/exec.sh b/ci/container/exec.sh new file mode 100755 index 000000000..57627b064 --- /dev/null +++ b/ci/container/exec.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + docker build ci/container + imageTag="$(docker build -q ci/container)" + docker run \ + --rm \ + -e CI \ + -e GITHUB_TOKEN \ + -e TRAVIS_TAG \ + -e NPM_TOKEN \ + -v "$(yarn cache dir):/usr/local/share/.cache/yarn/v6" \ + $(if [[ -f ~/.npmrc ]]; then echo -v "$HOME/.npmrc:/root/.npmrc"; fi) \ + -v "$PWD:/repo" \ + -w /repo \ + $(if [[ -t 0 ]]; then echo -it; fi) \ + "$imageTag" \ + "$*" +} + +main "$@" diff --git a/ci/dev-image/exec.sh b/ci/dev-image/exec.sh deleted file mode 100755 index 8b589f4b5..000000000 --- a/ci/dev-image/exec.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# exec.sh opens an interactive bash session inside of a docker container -# for improved isolation during development -# if the container exists it is restarted if necessary, then reused - -set -euo pipefail -cd "$(dirname "$0")" - -# Ensure submodules are cloned and up to date. -git submodule update --init - -container_name=code-server-dev - -enter() { - echo "--- Entering $container_name" - docker exec -it $container_id /bin/bash -} - -run() { - echo "--- Spawning $container_name" - container_id=$(docker run \ - -it \ - --name $container_name \ - "-v=$PWD:/code-server" \ - "-w=/code-server" \ - "-p=127.0.0.1:8080:8080" \ - $([[ -t 0 ]] && echo -it || true) \ - $container_name) -} - -build() { - echo "--- Building $container_name" - cd ../../ - docker build -t $container_name -f ./ci/dev-image/Dockerfile . > /dev/null -} - -container_id=$(docker container inspect --format="{{.Id}}" $container_name 2> /dev/null) || true - -if [ "$container_id" != "" ]; then - echo "-- Starting container" - docker start $container_id > /dev/null - - enter - exit 0 -fi - -build -run -enter diff --git a/ci/ci.sh b/ci/dev/ci.sh similarity index 76% rename from ci/ci.sh rename to ci/dev/ci.sh index 79d489f02..e92682864 100755 --- a/ci/ci.sh +++ b/ci/dev/ci.sh @@ -2,7 +2,7 @@ set -euo pipefail main() { - cd "$(dirname "$0")/.." + cd "$(dirname "$0")/../.." yarn fmt yarn lint diff --git a/ci/dev-image/Dockerfile b/ci/dev/container/Dockerfile similarity index 100% rename from ci/dev-image/Dockerfile rename to ci/dev/container/Dockerfile diff --git a/ci/dev/container/exec.sh b/ci/dev/container/exec.sh new file mode 100755 index 000000000..98231c737 --- /dev/null +++ b/ci/dev/container/exec.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Opens an interactive bash session inside of a docker container +# for improved isolation during development. +# If the container exists it is restarted if necessary, then reused. + +main() { + cd "$(dirname "${0}")/../../.." + + local container_name=code-server-dev + + if docker inspect $container_name &> /dev/null; then + echo "-- Starting container" + docker start "$container_name" > /dev/null + + enter + exit 0 + fi + + build + run + enter +} + +enter() { + echo "--- Entering $container_name" + docker exec -it "$container_name" /bin/bash +} + +run() { + echo "--- Spawning $container_name" + docker run \ + -it \ + --name $container_name \ + "-v=$PWD:/code-server" \ + "-w=/code-server" \ + "-p=127.0.0.1:8080:8080" \ + $(if [[ -t 0 ]]; then echo -it; fi) \ + "$container_name" +} + +build() { + echo "--- Building $container_name" + docker build -t $container_name ./ci/dev/container > /dev/null +} + +main "$@" diff --git a/ci/fmt.sh b/ci/dev/fmt.sh similarity index 95% rename from ci/fmt.sh rename to ci/dev/fmt.sh index 722395bcc..f7c48c41c 100755 --- a/ci/fmt.sh +++ b/ci/dev/fmt.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash - set -euo pipefail main() { + cd "$(dirname "$0")/../.." + shfmt -i 2 -w -s -sr $(git ls-files "*.sh") local prettierExts diff --git a/ci/lint.sh b/ci/dev/lint.sh similarity index 69% rename from ci/lint.sh rename to ci/dev/lint.sh index d6665edc3..c8553e470 100755 --- a/ci/lint.sh +++ b/ci/dev/lint.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash - set -euo pipefail main() { + cd "$(dirname "$0")/../.." + eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js") stylelint $(git ls-files "*.css") tsc --noEmit + shellcheck -e SC2046,SC2164 $(git ls-files "*.sh") } main "$@" diff --git a/ci/dev/test.sh b/ci/dev/test.sh new file mode 100755 index 000000000..031bacf99 --- /dev/null +++ b/ci/dev/test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + mocha -r ts-node/register ./test/*.test.ts +} + +main "$@" diff --git a/ci/vscode.patch b/ci/dev/vscode.patch similarity index 99% rename from ci/vscode.patch rename to ci/dev/vscode.patch index d61ed6244..0f1302ffb 100644 --- a/ci/vscode.patch +++ b/ci/dev/vscode.patch @@ -50,7 +50,7 @@ index 7a2320d828..5768890636 100644 yarnInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron diff --git a/coder.js b/coder.js new file mode 100644 -index 0000000000..d0a8f37714 +index 0000000000..0170b47241 --- /dev/null +++ b/coder.js @@ -0,0 +1,69 @@ @@ -83,7 +83,7 @@ index 0000000000..d0a8f37714 + "out-build/bootstrap-fork.js", + "out-build/bootstrap-amd.js", + "out-build/paths.js", -+ 'out-build/vs/**/*.{svg,png,html}', ++ 'out-build/vs/**/*.{svg,png,html,ttf}', + "!out-build/vs/code/browser/workbench/*.html", + '!out-build/vs/code/electron-browser/**', + "out-build/vs/base/common/performance.js", diff --git a/ci/vscode.sh b/ci/dev/vscode.sh similarity index 92% rename from ci/vscode.sh rename to ci/dev/vscode.sh index de5db91e6..d9a677952 100755 --- a/ci/vscode.sh +++ b/ci/dev/vscode.sh @@ -5,7 +5,7 @@ set -euo pipefail # 2. Patches it. # 3. Installs it. main() { - cd "$(dirname "$0")/.." + cd "$(dirname "$0")/../.." git submodule update --init diff --git a/ci/dev/watch.ts b/ci/dev/watch.ts new file mode 100644 index 000000000..03ce2e425 --- /dev/null +++ b/ci/dev/watch.ts @@ -0,0 +1,163 @@ +import * as cp from "child_process" +import Bundler from "parcel-bundler" +import * as path from "path" + +async function main(): Promise { + try { + const watcher = new Watcher() + await watcher.watch() + } catch (error) { + console.error(error.message) + process.exit(1) + } +} + +class Watcher { + private readonly rootPath = path.resolve(__dirname, "../..") + private readonly vscodeSourcePath = path.join(this.rootPath, "lib/vscode") + + private static log(message: string, skipNewline = false): void { + process.stdout.write(message) + if (!skipNewline) { + process.stdout.write("\n") + } + } + + public async watch(): Promise { + let server: cp.ChildProcess | undefined + const restartServer = (): void => { + if (server) { + server.kill() + } + const s = cp.fork(path.join(this.rootPath, "out/node/entry.js"), process.argv.slice(2)) + console.log(`[server] spawned process ${s.pid}`) + s.on("exit", () => console.log(`[server] process ${s.pid} exited`)) + server = s + } + + const vscode = cp.spawn("yarn", ["watch"], { cwd: this.vscodeSourcePath }) + const tsc = cp.spawn("tsc", ["--watch", "--pretty", "--preserveWatchOutput"], { cwd: this.rootPath }) + const bundler = this.createBundler() + + const cleanup = (code?: number | null): void => { + Watcher.log("killing vs code watcher") + vscode.removeAllListeners() + vscode.kill() + + Watcher.log("killing tsc") + tsc.removeAllListeners() + tsc.kill() + + if (server) { + Watcher.log("killing server") + server.removeAllListeners() + server.kill() + } + + Watcher.log("killing bundler") + process.exit(code || 0) + } + + process.on("SIGINT", () => cleanup()) + process.on("SIGTERM", () => cleanup()) + + vscode.on("exit", (code) => { + Watcher.log("vs code watcher terminated unexpectedly") + cleanup(code) + }) + tsc.on("exit", (code) => { + Watcher.log("tsc terminated unexpectedly") + cleanup(code) + }) + const bundle = bundler.bundle().catch(() => { + Watcher.log("parcel watcher terminated unexpectedly") + cleanup(1) + }) + bundler.on("buildEnd", () => { + console.log("[parcel] bundled") + }) + bundler.on("buildError", (error) => { + console.error("[parcel]", error) + }) + + vscode.stderr.on("data", (d) => process.stderr.write(d)) + tsc.stderr.on("data", (d) => process.stderr.write(d)) + + // From https://github.com/chalk/ansi-regex + const pattern = [ + "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)", + "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))", + ].join("|") + const re = new RegExp(pattern, "g") + + /** + * Split stdout on newlines and strip ANSI codes. + */ + const onLine = (proc: cp.ChildProcess, callback: (strippedLine: string, originalLine: string) => void): void => { + let buffer = "" + if (!proc.stdout) { + throw new Error("no stdout") + } + proc.stdout.setEncoding("utf8") + proc.stdout.on("data", (d) => { + const data = buffer + d + const split = data.split("\n") + const last = split.length - 1 + + for (let i = 0; i < last; ++i) { + callback(split[i].replace(re, ""), split[i]) + } + + // The last item will either be an empty string (the data ended with a + // newline) or a partial line (did not end with a newline) and we must + // wait to parse it until we get a full line. + buffer = split[last] + }) + } + + let startingVscode = false + let startedVscode = false + onLine(vscode, (line, original) => { + console.log("[vscode]", original) + // Wait for watch-client since "Finished compilation" will appear multiple + // times before the client starts building. + if (!startingVscode && line.includes("Starting watch-client")) { + startingVscode = true + } else if (startingVscode && line.includes("Finished compilation")) { + if (startedVscode) { + bundle.then(restartServer) + } + startedVscode = true + } + }) + + onLine(tsc, (line, original) => { + // tsc outputs blank lines; skip them. + if (line !== "") { + console.log("[tsc]", original) + } + if (line.includes("Watching for file changes")) { + bundle.then(restartServer) + } + }) + } + + private createBundler(out = "dist"): Bundler { + return new Bundler( + [ + path.join(this.rootPath, "src/browser/pages/app.ts"), + path.join(this.rootPath, "src/browser/register.ts"), + path.join(this.rootPath, "src/browser/serviceWorker.ts"), + ], + { + outDir: path.join(this.rootPath, out), + cacheDir: path.join(this.rootPath, ".cache"), + minify: !!process.env.MINIFY, + logLevel: 1, + publicUrl: "/static/development/dist", + }, + ) + } +} + +main() diff --git a/ci/image/Dockerfile b/ci/image/Dockerfile deleted file mode 100644 index 3fec3e874..000000000 --- a/ci/image/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM centos:7 - -RUN yum update -y && yum install -y \ - devtoolset-6 \ - gcc-c++ \ - xz \ - ccache \ - git \ - wget \ - openssl \ - libxkbfile-devel \ - libsecret-devel \ - libx11-devel - -RUN mkdir /usr/share/node && cd /usr/share/node \ - && curl "https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-$(uname -m | sed 's/86_//; s/aarch/arm/').tar.xz" | tar xJ --strip-components=1 -- -ENV PATH "$PATH:/usr/share/node/bin" -RUN npm install -g yarn@1.22.4 - -RUN curl -L "https://github.com/mvdan/sh/releases/download/v3.0.1/shfmt_v3.0.1_linux_$(uname -m | sed 's/x86_/amd/; s/aarch64/arm/')" > /usr/local/bin/shfmt \ - && chmod +x /usr/local/bin/shfmt - -ENTRYPOINT ["/bin/bash", "-c"] diff --git a/ci/image/run.sh b/ci/image/run.sh deleted file mode 100755 index f9810d413..000000000 --- a/ci/image/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - # This, strangely enough, fixes the arm build being terminated for not having - # output on Travis. It's as if output is buffered and only displayed once a - # certain amount is collected. Five seconds didn't work but one second seems - # to generate enough output to make it work. - local pid - while true; do - echo 'Still running...' - sleep 1 - done & - pid=$! - - docker build ci/image - imageTag="$(docker build -q ci/image)" - docker run -t --rm -e CI -e GITHUB_TOKEN -e TRAVIS_TAG -v "$(yarn cache dir):/usr/local/share/.cache/yarn/v6" -v "$PWD:/repo" -w /repo "$imageTag" "$*" - - kill $pid -} - -main "$@" diff --git a/ci/lib.sh b/ci/lib.sh old mode 100644 new mode 100755 index 7118ddd09..d172976e1 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -1,10 +1,43 @@ #!/usr/bin/env bash -set -euo pipefail -set_version() { - local code_server_version=${VERSION:-${TRAVIS_TAG:-}} - if [[ -z $code_server_version ]]; then - code_server_version=$(grep version ./package.json | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[:space:]') - fi - export VERSION=$code_server_version +pushd() { + builtin pushd "$@" > /dev/null +} + +popd() { + builtin popd > /dev/null +} + +pkg_json_version() { + jq -r .version package.json +} + +os() { + local os + os=$(uname | tr '[:upper:]' '[:lower:]') + if [[ $os == "linux" ]]; then + # Alpine's ldd doesn't have a version flag but if you use an invalid flag + # (like --version) it outputs the version to stderr and exits with 1. + local ldd_output + ldd_output=$(ldd --version 2>&1 || true) + if echo "$ldd_output" | grep -iq musl; then + os="alpine" + fi + fi + echo "$os" +} + +arch() { + case "$(uname -m)" in + aarch64) + echo arm64 + ;; + x86_64) + echo amd64 + ;; + *) + echo "unknown architecture $(uname -a)" + exit 1 + ;; + esac } diff --git a/ci/release-image/Dockerfile b/ci/release-container/Dockerfile similarity index 61% rename from ci/release-image/Dockerfile rename to ci/release-container/Dockerfile index 9e30d30d0..0a99242f2 100644 --- a/ci/release-image/Dockerfile +++ b/ci/release-container/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update \ ssh \ sudo \ vim \ + lsb-release \ && rm -rf /var/lib/apt/lists/* # https://wiki.debian.org/Locale#Manually @@ -26,18 +27,20 @@ ENV SHELL=/bin/bash RUN adduser --gecos '' --disabled-password coder && \ echo "coder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/nopasswd -RUN curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.4/fixuid-0.4-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \ +SHELL ["/bin/bash", "-c"] + +COPY ci/lib.sh /tmp/lib.sh +RUN source /tmp/lib.sh && rm /tmp/lib.sh && \ + curl -L "https://github.com/boxboat/fixuid/releases/download/v0.4.1/fixuid-0.4.1-linux-$(arch).tar.gz" | tar -C /usr/local/bin -xzf - && \ chown root:root /usr/local/bin/fixuid && \ chmod 4755 /usr/local/bin/fixuid && \ mkdir -p /etc/fixuid && \ printf "user: coder\ngroup: coder\n" > /etc/fixuid/config.yml -COPY release/code-server*.tar.gz /tmp/ -RUN cd /tmp && tar -xzf code-server*.tar.gz && rm code-server*.tar.gz && \ - mv code-server* /usr/local/lib/code-server && \ - ln -s /usr/local/lib/code-server/code-server /usr/local/bin/code-server +COPY release-github/code-server*.deb /tmp/ +RUN dpkg -i /tmp/code-server*.deb && rm /tmp/code-server*.deb EXPOSE 8080 USER coder WORKDIR /home/coder -ENTRYPOINT ["dumb-init", "fixuid", "-q", "/usr/local/bin/code-server", "--bind-addr", "0.0.0.0:8080", "."] +ENTRYPOINT ["dumb-init", "fixuid", "-q", "/usr/bin/code-server", "--bind-addr", "0.0.0.0:8080", "."] diff --git a/ci/release-image/push.sh b/ci/release-container/push.sh similarity index 57% rename from ci/release-image/push.sh rename to ci/release-container/push.sh index 6baf73320..99faa67c2 100755 --- a/ci/release-image/push.sh +++ b/ci/release-container/push.sh @@ -5,18 +5,21 @@ set -euo pipefail main() { cd "$(dirname "$0")/../.." source ./ci/lib.sh - set_version + VERSION="$(pkg_json_version)" - if [[ ${CI:-} ]]; then + if [[ ${CI-} ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin fi imageTag="codercom/code-server:$VERSION" - if [[ ${TRAVIS_CPU_ARCH:-} == "arm64" ]]; then + if [[ $(arch) == "arm64" ]]; then imageTag+="-arm64" fi - docker build -t "$imageTag" -f ./ci/release-image/Dockerfile . - docker push codercom/code-server + + docker build \ + -t "$imageTag" \ + -f ./ci/release-container/Dockerfile . + docker push "$imageTag" } main "$@" diff --git a/ci/release.sh b/ci/release.sh deleted file mode 100755 index 6b7b47544..000000000 --- a/ci/release.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# ci.bash -- Build code-server in the CI. - -set -euo pipefail - -function package() { - local target - target=$(uname | tr '[:upper:]' '[:lower:]') - if [[ $target == "linux" ]]; then - # Alpine's ldd doesn't have a version flag but if you use an invalid flag - # (like --version) it outputs the version to stderr and exits with 1. - local ldd_output - ldd_output=$(ldd --version 2>&1 || true) - if echo "$ldd_output" | grep -iq musl; then - target="alpine" - fi - fi - - local arch - arch=$(uname -m | sed 's/aarch/arm/') - - echo -n "Creating release..." - - cp "$(command -v node)" ./build - cp README.md ./build - cp LICENSE.txt ./build - cp ./lib/vscode/ThirdPartyNotices.txt ./build - cp ./ci/code-server.sh ./build/code-server - - local archive_name="code-server-$VERSION-$target-$arch" - mkdir -p ./release - - local ext - if [[ $target == "linux" ]]; then - ext=".tar.gz" - tar -czf "release/$archive_name$ext" --transform "s/^\.\/build/$archive_name/" ./build - else - mv ./build "./$archive_name" - ext=".zip" - zip -r "release/$archive_name$ext" "./$archive_name" - mv "./$archive_name" ./build - fi - - echo "done (release/$archive_name)" - - # release-upload is for uploading to the GCP bucket whereas release is used for GitHub. - mkdir -p "./release-upload/$VERSION" - cp "./release/$archive_name$ext" "./release-upload/$VERSION/$target-$arch$ext" - mkdir -p "./release-upload/latest" - cp "./release/$archive_name$ext" "./release-upload/latest/$target-$arch$ext" -} - -# This script assumes that yarn has already ran. -function build() { - # Always minify and package on CI. - if [[ ${CI:-} ]]; then - export MINIFY="true" - fi - - yarn build -} - -function main() { - cd "$(dirname "${0}")/.." - source ./ci/lib.sh - - set_version - - build - - if [[ ${CI:-} ]]; then - package - fi -} - -main "$@" diff --git a/ci/steps/linux-release.sh b/ci/steps/linux-release.sh new file mode 100755 index 000000000..382098aa4 --- /dev/null +++ b/ci/steps/linux-release.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + source ./ci/lib.sh + + if [[ $(arch) == arm64 ]]; then + # This, strangely enough, fixes the arm build being terminated for not having + # output on Travis. It's as if output is buffered and only displayed once a + # certain amount is collected. Five seconds didn't work but one second seems + # to generate enough output to make it work. + while true; do + echo 'Still running...' + sleep 1 + done & + trap "exit" INT TERM + trap "kill 0" EXIT + fi + + ./ci/container/exec.sh ./ci/steps/static-release.sh + ./ci/container/exec.sh yarn pkg + ./ci/release-container/push.sh +} + +main "$@" diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh new file mode 100755 index 000000000..c4bcb9674 --- /dev/null +++ b/ci/steps/publish-npm.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc + ./ci/container/exec.sh yarn publish --non-interactive release +} + +main "$@" diff --git a/ci/steps/static-release.sh b/ci/steps/static-release.sh new file mode 100755 index 000000000..c0a704314 --- /dev/null +++ b/ci/steps/static-release.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + yarn + yarn vscode + yarn build + yarn build:vscode + STATIC=1 yarn release + ./ci/build/test-static-release.sh + ./ci/build/archive-static-release.sh +} + +main "$@" diff --git a/ci/steps/test.sh b/ci/steps/test.sh new file mode 100755 index 000000000..9430d1078 --- /dev/null +++ b/ci/steps/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + yarn + + git submodule update --init + # We do not `yarn vscode` to make test.sh faster. + # If the patch fails to apply, then it's likely already applied + yarn vscode:patch &> /dev/null || true + + yarn ci +} + +main "$@" diff --git a/ci/tsconfig.json b/ci/tsconfig.json deleted file mode 100644 index 5197ce276..000000000 --- a/ci/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../tsconfig.json", - "include": ["./**/*.ts"] -} diff --git a/doc/CONTRIBUTING.md b/doc/CONTRIBUTING.md index 35bcc720b..824eaad21 100644 --- a/doc/CONTRIBUTING.md +++ b/doc/CONTRIBUTING.md @@ -1,5 +1,8 @@ # Contributing +- [Detailed CI and build process docs](../ci) +- [Our VS Code Web docs](../src/node/app) + ## Development Workflow - [VS Code prerequisites](https://github.com/Microsoft/vscode/wiki/How-to-Contribute#prerequisites) @@ -13,7 +16,7 @@ yarn watch # Visit http://localhost:8080 once completed. To develop inside of an isolated docker container: ```shell -./ci/dev-image/exec.sh +./ci/dev/container/exec.sh root@12345:/code-server# yarn root@12345:/code-server# yarn vscode @@ -36,5 +39,7 @@ works internally. yarn yarn vscode yarn build -node ./build/out/node/entry.js # Run the built JavaScript with Node. +yarn build:vscode +yarn release +node ./release # Run the built JavaScript with Node. ``` diff --git a/package.json b/package.json index 5b598e317..975f3be2d 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,30 @@ { "name": "code-server", "license": "MIT", - "version": "3.3.0", - "scripts": { - "clean": "ci/clean.sh", - "vscode": "ci/vscode.sh", - "vscode:patch": "cd ./lib/vscode && git apply ../../ci/vscode.patch", - "vscode:diff": "cd ./lib/vscode && git diff HEAD > ../../ci/vscode.patch", - "test": "mocha -r ts-node/register ./test/*.test.ts", - "lint": "ci/lint.sh", - "fmt": "ci/fmt.sh", - "runner": "cd ./ci && NODE_OPTIONS=--max_old_space_size=32384 ts-node ./build.ts", - "build": "yarn runner build", - "watch": "yarn runner watch" + "version": "3.3.0-rc.7", + "description": "Run VS Code on a remote server.", + "homepage": "https://github.com/cdr/code-server", + "bugs": { + "url": "https://github.com/cdr/code-server/issues" }, + "repository": "https://github.com/cdr/code-server", + "scripts": { + "clean": "./ci/build/clean.sh", + "vscode": "./ci/dev/vscode.sh", + "vscode:patch": "cd ./lib/vscode && git apply ../../ci/dev/vscode.patch", + "vscode:diff": "cd ./lib/vscode && git diff HEAD > ../../ci/dev/vscode.patch", + "build": "./ci/build/build-code-server.sh", + "build:vscode": "./ci/build/build-vscode.sh", + "release": "./ci/build/build-release.sh", + "pkg": "./ci/build/build-static-pkgs.sh", + "_____": "", + "fmt": "./ci/dev/fmt.sh", + "lint": "./ci/dev/lint.sh", + "test": "./ci/dev/test.sh", + "ci": "./ci/dev/ci.sh", + "watch": "NODE_OPTIONS=--max_old_space_size=32384 ts-node ./ci/dev/watch.ts" + }, + "main": "out/node/entry.js", "devDependencies": { "@types/adm-zip": "^0.4.32", "@types/fs-extra": "^8.0.1", @@ -40,7 +51,8 @@ "stylelint": "^13.0.0", "stylelint-config-recommended": "^3.0.0", "ts-node": "^8.4.1", - "typescript": "3.7.2" + "typescript": "3.7.2", + "yarn": "^1.22.4" }, "resolutions": { "@types/node": "^12.12.7", @@ -48,7 +60,7 @@ "vfile-message": "^2.0.2" }, "dependencies": { - "@coder/logger": "1.1.11", + "@coder/logger": "1.1.14", "adm-zip": "^0.4.14", "fs-extra": "^8.1.0", "http-proxy": "^1.18.0", @@ -60,5 +72,16 @@ "tar": "^6.0.1", "tar-fs": "^2.0.0", "ws": "^7.2.0" - } + }, + "bin": { + "code-server": "out/node/entry.js" + }, + "keywords": [ + "vscode", + "development", + "ide", + "coder", + "vscode-remote", + "browser-ide" + ] } diff --git a/src/node/app/README.md b/src/node/app/README.md index 9e03298c6..02d2c4385 100644 --- a/src/node/app/README.md +++ b/src/node/app/README.md @@ -1,3 +1,5 @@ +# app + Implementation of [VS Code](https://code.visualstudio.com/) remote/web for use in `code-server`. diff --git a/yarn.lock b/yarn.lock index f0624c88b..8a5195a6d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -792,10 +792,10 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@coder/logger@1.1.11": - version "1.1.11" - resolved "https://registry.yarnpkg.com/@coder/logger/-/logger-1.1.11.tgz#e6f36dba9436ae61e66e3f66787d75c768617605" - integrity sha512-EEh1dqSU0AaqjjjMsVqumgZGbrZimKFKIb4t5E6o3FLfVUxJCReSME78Yj2N1xWUVAHMnqafDCxLostpuIotzw== +"@coder/logger@1.1.14": + version "1.1.14" + resolved "https://registry.yarnpkg.com/@coder/logger/-/logger-1.1.14.tgz#0242da33e0245834361dd078e31280fc1c976b7e" + integrity sha512-NuTvsOH3dqrXn/8Pbs5zy7l0gLqOSC/TPRl3nexdP/897lgG/vtHNQHrUwTBTzTzihH1ON4lklDxJjY0hD4UPg== "@iarna/toml@^2.2.0": version "2.2.5" @@ -7471,6 +7471,11 @@ yargs@^14.0.0: y18n "^4.0.0" yargs-parser "^15.0.1" +yarn@^1.22.4: + version "1.22.4" + resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.22.4.tgz#01c1197ca5b27f21edc8bc472cd4c8ce0e5a470e" + integrity sha512-oYM7hi/lIWm9bCoDMEWgffW8aiNZXCWeZ1/tGy0DWrN6vmzjCXIKu2Y21o8DYVBUtiktwKcNoxyGl/2iKLUNGA== + yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" From 1a375a44e05861055b66fa61f6970785e4a073dd Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 6 May 2020 20:26:48 -0400 Subject: [PATCH 2/2] Disable ARM64 releases as ARM on Travis is very unreliable --- .travis.yml | 18 +++++++++++------- ci/steps/linux-release.sh | 13 ------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b2f778de..652ef9535 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,17 @@ jobs: if: tag IS present script: ./ci/steps/linux-release.sh install: null - - name: Linux Release - if: tag IS present - arch: arm64 - script: | - sudo apt-get update && sudo apt-get install -y jq || exit 1 - ./ci/steps/linux-release.sh - install: null + # Unfortunately ARM on travis is very unreliable. + # We see random build failures, logging output being truncated, build being killed + # due to no output even though we use travis_wait etc. + # So we've disabled it for now. + # - name: Linux Release + # if: tag IS present + # arch: arm64 + # script: | + # sudo apt-get update && sudo apt-get install -y jq || exit 1 + # travis_wait 60 ./ci/steps/linux-release.sh + # install: null - name: MacOS Release if: tag IS present os: osx diff --git a/ci/steps/linux-release.sh b/ci/steps/linux-release.sh index 382098aa4..2d5b34d09 100755 --- a/ci/steps/linux-release.sh +++ b/ci/steps/linux-release.sh @@ -5,19 +5,6 @@ main() { cd "$(dirname "$0")/../.." source ./ci/lib.sh - if [[ $(arch) == arm64 ]]; then - # This, strangely enough, fixes the arm build being terminated for not having - # output on Travis. It's as if output is buffered and only displayed once a - # certain amount is collected. Five seconds didn't work but one second seems - # to generate enough output to make it work. - while true; do - echo 'Still running...' - sleep 1 - done & - trap "exit" INT TERM - trap "kill 0" EXIT - fi - ./ci/container/exec.sh ./ci/steps/static-release.sh ./ci/container/exec.sh yarn pkg ./ci/release-container/push.sh