From 0c2c9573128dfa79166a6e342862df5f318474a9 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 25 Feb 2019 16:29:34 -0600 Subject: [PATCH] Fix readdir for root path (#35) * Fix readdir for root path * Fix merge of webpack conf * Fix travis.yml to deploy on master * Remove windows platform from travis.yml * Enable caching * mkdirpSync * Fix build script --- .travis.yml | 7 +++- build/tasks.ts | 2 +- packages/server/package.json | 4 +- packages/server/scripts/nexe.js | 12 +++++- packages/server/src/cli.ts | 5 --- packages/server/src/fill.ts | 24 ++++++++--- packages/server/webpack.config.js | 2 +- packages/server/yarn.lock | 70 ++++++++++++++++++++----------- 8 files changed, 84 insertions(+), 42 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ce5e70ea..af1aceb5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,8 @@ language: node_js +cache: + yarn: true + directories: + - lib node_js: - 8.9.3 matrix: @@ -6,7 +10,6 @@ matrix: - os: linux dist: ubuntu - os: osx - - os: windows before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev libsecret-1-dev; fi @@ -20,4 +23,4 @@ deploy: file: packages/server/cli-$TRAVIS_OS_NAME on: repo: codercom/vscode-online - all_branches: true + branch: master diff --git a/build/tasks.ts b/build/tasks.ts index 919dd7e4f..0765d3143 100644 --- a/build/tasks.ts +++ b/build/tasks.ts @@ -190,7 +190,7 @@ const ensureCloned = register("vscode:clone", async (runner) => { if (fs.existsSync(vscodePath)) { await ensureClean(); } else { - fs.mkdirSync(libPath); + fse.mkdirpSync(libPath); runner.cwd = libPath; const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode"]); if (clone.exitCode !== 0) { diff --git a/packages/server/package.json b/packages/server/package.json index b068a7dcc..dbe8484bc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -5,7 +5,7 @@ "files": [], "scripts": { "start": "node --max-old-space-size=32384 --require ts-node/register --require tsconfig-paths/register src/cli.ts", - "build:webpack": "rm -rf ./out && export CLI=true && ../../node_modules/.bin/webpack --config ./webpack.config.js", + "build:webpack": "rm -rf ./out && export CLI=true && UV_THREADPOOL_SIZE=100 node --max-old-space-size=32384 ../../node_modules/webpack/bin/webpack.js --config ./webpack.config.js", "build:nexe": "node scripts/nexe.js", "build:bootstrap-fork": "cd ../vscode && npm run build:bootstrap-fork; mkdir -p ./packages/server/resources; cp ./bin/bootstrap-fork.js ../server/resources/bootstrap-fork.js", "build:default-extensions": "cd ../../lib/vscode && npx gulp vscode-linux-arm && cd ../.. && mkdir -p ./packages/server/resources/extensions; cp -r ./lib/VSCode-linux-arm/resources/app/extensions/* ./packages/server/resources/extensions/", @@ -20,7 +20,6 @@ "express-static-gzip": "^1.1.3", "httpolyglot": "^0.1.2", "mime-types": "^2.1.21", - "nexe": "^2.0.0-rc.34", "node-netstat": "^1.6.0", "pem": "^1.14.1", "promise.prototype.finally": "^3.1.0", @@ -34,6 +33,7 @@ "@types/pem": "^1.9.4", "@types/ws": "^6.0.1", "fs-extra": "^7.0.1", + "nexe": "^2.0.0-rc.34", "string-replace-webpack-plugin": "^0.1.3", "ts-node": "^7.0.1", "tsconfig-paths": "^3.7.0", diff --git a/packages/server/scripts/nexe.js b/packages/server/scripts/nexe.js index ad960a34f..17130c677 100644 --- a/packages/server/scripts/nexe.js +++ b/packages/server/scripts/nexe.js @@ -2,6 +2,16 @@ const fs = require("fs"); const fse = require("fs-extra"); const os = require("os"); const path = require("path"); + +const nexePath = require.resolve("nexe"); +const shimPath = path.join(path.dirname(nexePath), "lib/steps/shim.js"); +let shimContent = fs.readFileSync(shimPath).toString(); +const replaceString = `global.nativeFs = { readdir: originalReaddir, readdirSync: originalReaddirSync };`; +shimContent = shimContent.replace(/compiler\.options\.resources\.length[\s\S]*wrap\("(.*\\n)"/g, (om, a) => { + return om.replace(a, `${a}${replaceString}`); +}); +fs.writeFileSync(shimPath, shimContent); + const nexe = require("nexe"); nexe.compile({ @@ -13,7 +23,7 @@ nexe.compile({ * To include native extensions, do NOT install node_modules for each one. They * are not required as each extension is built using webpack. */ - resources: [ +resources: [ path.join(__dirname, "../package.json"), path.join(__dirname, "../build/**/*"), ], diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts index 492af59ab..02d12db40 100644 --- a/packages/server/src/cli.ts +++ b/packages/server/src/cli.ts @@ -72,11 +72,6 @@ export class Entry extends Command { return requireFork(modulePath, JSON.parse(flags.args!), builtInExtensionsDir); } - if (buildDir && buildDir.startsWith(workingDir)) { - logger.error("Cannot run binary inside of BUILD_DIR", field("build_dir", buildDir), field("cwd", process.cwd())); - process.exit(1); - } - if (!fs.existsSync(dataDir)) { fs.mkdirSync(dataDir); } diff --git a/packages/server/src/fill.ts b/packages/server/src/fill.ts index d353d8a74..7e2ccdb34 100644 --- a/packages/server/src/fill.ts +++ b/packages/server/src/fill.ts @@ -1,7 +1,10 @@ import * as fs from "fs"; +import * as path from "path"; import * as util from "util"; -import { isCli } from "./constants"; +import { isCli, buildDir } from "./constants"; +// tslint:disable:no-any +const nativeFs = (global).nativeFs as typeof fs || {}; const oldAccess = fs.access; const existsWithinBinary = (path: fs.PathLike): Promise => { return new Promise((resolve): void => { @@ -39,7 +42,7 @@ export const fillFs = (): void => { const replaceNative = (propertyName: T, func: (callOld: () => void, ...args: any[]) => any, customPromisify?: (...args: any[]) => Promise): void => { const oldFunc = (fs)[propertyName]; - fs[propertyName] = (...args: any[]) => { + fs[propertyName] = (...args: any[]): any => { try { return func(() => { return oldFunc(...args); @@ -49,7 +52,7 @@ export const fillFs = (): void => { } }; if (customPromisify) { - (fs[propertyName])[util.promisify.custom] = (...args: any[]) => { + (fs[propertyName])[util.promisify.custom] = (...args: any[]): any => { return customPromisify(...args).catch((ex) => { throw ex; }); @@ -75,7 +78,7 @@ export const fillFs = (): void => { return callOld(); }); - }, (path) => new Promise((res) => fs.exists(path, res))); + }, (path) => new Promise((res): void => fs.exists(path, res))); replaceNative("open", (callOld, path: fs.PathLike, flags: string | Number, mode: any, callback: any) => { existsWithinBinary(path).then((exists) => { @@ -110,7 +113,7 @@ export const fillFs = (): void => { callback(); }); - replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void, ) => { + replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void) => { if (!fds.has(fd)) { return callOld(); } @@ -136,7 +139,7 @@ export const fillFs = (): void => { bytesRead: number; buffer: Buffer; }> => { - return new Promise((res, rej) => { + return new Promise((res, rej): void => { fs.read(fd, buffer, offset, length, position, (err, bytesRead, buffer) => { if (err) { return rej(err); @@ -149,4 +152,13 @@ export const fillFs = (): void => { }); }); }); + + replaceNative("readdir", (callOld, directory: string, callback: (err: NodeJS.ErrnoException, paths: string[]) => void) => { + const relative = path.relative(directory, buildDir!); + if (relative.startsWith("..")) { + return callOld(); + } + + return nativeFs.readdir(directory, callback); + }); }; diff --git a/packages/server/webpack.config.js b/packages/server/webpack.config.js index 60ab2dd7a..8fd8de7f5 100644 --- a/packages/server/webpack.config.js +++ b/packages/server/webpack.config.js @@ -4,7 +4,7 @@ const merge = require("webpack-merge"); module.exports = merge({ devtool: "none", - mode: "development", + mode: "production", output: { filename: "cli.js", path: path.join(__dirname, "./out"), diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 1351aa388..25c6ce84b 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -191,9 +191,9 @@ ajax-request@^1.2.0: utils-extend "^1.0.7" ajv@^6.5.5: - version "6.7.0" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.7.0.tgz#e3ce7bb372d6577bb1839f1dfdfcbf5ad2948d96" - integrity sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg== + version "6.9.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.2.tgz#4927adb83e7f48e5a32b45729744c71ec39c9c7b" + integrity sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg== dependencies: fast-deep-equal "^2.0.1" fast-json-stable-stringify "^2.0.0" @@ -433,9 +433,9 @@ big.js@^3.1.3: integrity sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q== binary-extensions@^1.0.0: - version "1.12.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.12.0.tgz#c2d780f53d45bba8317a8902d4ceeaf3a6385b14" - integrity sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg== + version "1.13.0" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.0.tgz#9523e001306a32444b907423f1de2164222f6ab1" + integrity sha512-EgmjVLMn22z7eGGv3kcnHwSnJXmFHjISTY9E/S5lIcTD3Oxw05QTcBLNkJFzcb3cNueUdF/IN4U+d78V0zO8Hw== bl@^1.0.0: version "1.2.2" @@ -1068,9 +1068,9 @@ escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= escodegen@^1.8.1: - version "1.11.0" - resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.11.0.tgz#b27a9389481d5bfd5bec76f7bb1eb3f8f4556589" - integrity sha512-IeMV45ReixHS53K/OmfKAIztN/igDHzTJUhZM3k1jMhIZWjk45SMwAtBsEXiJp3vSPmTcu6CXn7mDvFHRN66fw== + version "1.11.1" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.11.1.tgz#c485ff8d6b4cdb89e27f4a856e91f118401ca510" + integrity sha512-JwiqFD9KdGVVpeuRa68yU3zZnBEOcPs0nKW7wZzXky8Z7tffdYUHbe11bPCV5jYlK6DVdKLWLm0f5I/QlL0Kmw== dependencies: esprima "^3.1.3" estraverse "^4.2.0" @@ -1468,9 +1468,9 @@ function-bind@^1.1.1: integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== fuse-box@^3.1.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/fuse-box/-/fuse-box-3.6.0.tgz#a4adf41a60855c7b0a0775b3095b6b0c2d35b011" - integrity sha512-j/lIXZHIiHv2w9gxOBj9k8vAORXt0GrPQt/zc4ttVG3WT5pQ0uaqpxu6m4Npwk4WgOjN4mieD3taK4S56Ol3Zw== + version "3.7.1" + resolved "https://registry.yarnpkg.com/fuse-box/-/fuse-box-3.7.1.tgz#d32879ceee4c8bcec9bbd8fcfe5b29e7142371cd" + integrity sha512-aM7t9bUcRpNNQu9M+YjXXzx9JSJQVPWeY+8iTyv7OhvJNWHrqqEWPzbn9OfcyFa2AfPwAUyC/uzWexBbjtTvsA== dependencies: acorn "^5.7.3" acorn-jsx "^4.0.1" @@ -1500,6 +1500,8 @@ fuse-box@^3.1.0: request "^2.79.0" shorthash "0.0.2" source-map "^0.7.1" + sourcemap-blender "1.0.5" + stream-browserify "^2.0.1" tslib "^1.8.0" watch "^1.0.1" ws "^1.1.1" @@ -1551,9 +1553,9 @@ get-value@^2.0.3, get-value@^2.0.6: integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= getopts@^2.1.1: - version "2.2.3" - resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.3.tgz#11d229775e2ec2067ed8be6fcc39d9b4bf39cf7d" - integrity sha512-viEcb8TpgeG05+Nqo5EzZ8QR0hxdyrYDp6ZSTZqe2M/h53Bk036NmqG38Vhf5RGirC/Of9Xql+v66B2gp256SQ== + version "2.2.4" + resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.4.tgz#3137fe8a5fddf304904059a851bdc1c22f0f54fb" + integrity sha512-Rz7DGyomZjrenu9Jx4qmzdlvJgvrEFHXHvjK0FcZtcTC1U5FmES7OdZHUwMuSnEE6QvBvwse1JODKj7TgbSEjQ== getpass@^0.1.1: version "0.1.7" @@ -1806,7 +1808,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= @@ -2353,7 +2355,19 @@ mime-db@^1.28.0, mime-db@~1.37.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== -mime-types@^2.1.12, mime-types@^2.1.21, mime-types@~2.1.18, mime-types@~2.1.19: +mime-db@~1.38.0: + version "1.38.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.38.0.tgz#1a2aab16da9eb167b49c6e4df2d9c68d63d8e2ad" + integrity sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.22" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.22.tgz#fe6b355a190926ab7698c9a0556a11199b2199bd" + integrity sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog== + dependencies: + mime-db "~1.38.0" + +mime-types@^2.1.21, mime-types@~2.1.18: version "2.1.21" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== @@ -2552,9 +2566,9 @@ normalize-url@2.0.1: sort-keys "^2.0.0" npm-bundled@^1.0.1: - version "1.0.5" - resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979" - integrity sha512-m/e6jgWu8/v5niCUKQi9qQl8QdeEduFA96xHDDzFGqly0OOjI7c+60KM/2sppfnUU9JJagf+zs+yGhqSOFj71g== + version "1.0.6" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.6.tgz#e7ba9aadcef962bb61248f91721cd932b3fe6bdd" + integrity sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g== npm-conf@^1.1.0: version "1.1.3" @@ -2565,9 +2579,9 @@ npm-conf@^1.1.0: pify "^3.0.0" npm-packlist@^1.1.6: - version "1.2.0" - resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.2.0.tgz#55a60e793e272f00862c7089274439a4cc31fc7f" - integrity sha512-7Mni4Z8Xkx0/oegoqlcao/JpPCPEMtUvsmB0q7mgvlMinykJLSRTYuFqoQLYgGY8biuxIeiHO+QNJKbCfljewQ== + version "1.4.1" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.4.1.tgz#19064cdf988da80ea3cee45533879d90192bbfbc" + integrity sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw== dependencies: ignore-walk "^3.0.1" npm-bundled "^1.0.1" @@ -3310,7 +3324,7 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -source-map@^0.7.1: +source-map@^0.7.1, source-map@^0.7.3: version "0.7.3" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== @@ -3362,6 +3376,14 @@ statuses@~1.4.0: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew== +stream-browserify@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" + integrity sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg== + dependencies: + inherits "~2.0.1" + readable-stream "^2.0.2" + strict-uri-encode@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"