tools/tool-env/isopackets.js
var assert = require('assert');
var _ = require('underscore');
var bundler = require('../isobuild/bundler.js');
import Builder from '../isobuild/builder.js';
var compiler = require('../isobuild/compiler.js');
var isopackCacheModule = require('../isobuild/isopack-cache.js');
var buildmessage = require('../utils/buildmessage.js');
var files = require('../fs/files');
var config = require('../meteor-services/config.js');
var watch = require('../fs/watch');
var Console = require('../console/console.js').Console;
var packageMapModule = require('../packaging/package-map.js');
var archinfo = require('../utils/archinfo');
var Profile = require('./profile').Profile;
// TL;DR: Isopacket is a set of isopacks. Isopackets are used only inside
// meteor-tool.
// An isopacket is a predefined set of isopackages which the meteor command-line
// tool can load into its process. This is how we use the DDP client and many
// other packages inside the tool. The isopackets are listed below in the
// ISOPACKETS constant.
//
// All packages that are in isopackets and all of their transitive dependencies
// must be part of the core Meteor git checkout (not loaded from troposphere).
//
// The requested packages will be loaded together with all of their
// dependencies. If you request to load the same isopacket more than once, you
// will efficiently get the same pre-loaded isopacket. On the other hand, two
// different loaded isopackets contain distinct copies of all of their packages
// copy of all of the packages. The return value is an object that maps package
// name to package exports (that is, it is the Package object from inside the
// sandbox created for the newly loaded packages).
//
// For built releases, all of the isopackets are pre-compiled (as JsImages,
// similar to a plugin or a server program) into the tool.
//
// When run from a checkout, all isopackets are re-compiled early in the startup
// process if any of their sources have changed.
//
// Example usage:
// var DDP = require('./isopackets.js').loadIsopackage('ddp-client').DDP;
// var reverse = DDP.connect('reverse.meteor.com');
// Console.info(reverse.call('reverse', 'hello world'));
// All of the defined isopackets. Whenever they are being built, they will be
// built in the order listed here.
export const ISOPACKETS = {
// These packages used to be divided up into distinct isopackets, but
// that resulted in extremely wasteful duplication of transitive
// dependencies, so now we have only one isopacket that combines all the
// dependencies of every former isopacket.
combined: [
// ddp
'ddp-client',
// mongo
'npm-mongo',
// ejson
'ejson',
// constraint-solver
'constraint-solver',
// cordova-support
'boilerplate-generator',
'webapp-hashing',
// cordova-support, logging
'logging',
// support for childProcess.sendMessage(topic, payload)
'inter-process-messaging',
]
};
// Caches isopackets in memory (each isopacket only needs to be loaded
// once). This is a map from isopacket name to either:
//
// - The 'Package' dictionary, if the isopacket has already been loaded
// into memory
// - null, if the isopacket hasn't been loaded into memory but its on-disk
// instance is known to be ready
//
// The subtlety here is that when running from a checkout, we don't want to
// accidentally load an isopacket before ensuring that it doesn't need to be
// rebuilt. We used to need to load a "js-analyze" isopacket as part
// of building other isopackets in ensureIsopacketsLoadable which made this
// more important, though we've simplified it now by moving that code into
// the tool itself.
var loadedIsopackets = {};
// The main entry point: loads the specified isopacket ("combined" by
// default) from cache or from disk, and returns the requested package
// dependency, complaining if the package does not exist. Note that
// ensureIsopacketsLoadable must be called first, as this function does
// not trigger any building.
export function loadIsopackage(packageName, isopacketName = "combined") {
// Small but necessary hack: because archinfo.host() calls execFileSync,
// it yields the first time we call it, which is a problem for the
// fiberHelpers.noYieldsAllowed block below. Calling it here ensures the
// result is cached, so no yielding occurs later.
assert.strictEqual(archinfo.host().split(".", 1)[0], "os");
const isopacket = function () {
if (_.has(loadedIsopackets, isopacketName)) {
if (loadedIsopackets[isopacketName]) {
return loadedIsopackets[isopacketName];
}
// This is the case where the isopacket is up to date on disk but not
// loaded.
return loadedIsopackets[isopacketName] =
loadIsopacketFromDisk(isopacketName);
}
if (_.has(ISOPACKETS, isopacketName)) {
throw Error("Can't load isopacket before it has been verified: "
+ isopacketName);
}
throw Error("Unknown isopacket: " + isopacketName);
}();
if (!_.has(isopacket, packageName)) {
throw new Error("Unknown isopacket dependency: " + packageName);
}
return isopacket[packageName];
}
var isopacketPath = function (isopacketName) {
return files.pathJoin(config.getIsopacketRoot(), isopacketName);
};
// ensureIsopacketsLoadable is called at startup and ensures that all isopackets
// exist on disk as up-to-date loadable programs.
var calledEnsure = false;
export function ensureIsopacketsLoadable() {
if (calledEnsure) {
throw Error("can't ensureIsopacketsLoadable twice!");
}
calledEnsure = true;
// If we're not running from checkout, then there's nothing to build and we
// can declare that all isopackets are loadable.
if (!files.inCheckout()) {
_.each(ISOPACKETS, function (packages, name) {
loadedIsopackets[name] = null;
});
return;
}
// We make this object lazily later.
var isopacketBuildContext = null;
var failedPackageBuild = false;
// Look at each isopacket. Check to see if it's on disk and up to date. If
// not, build it. We rebuild them in the order listed in ISOPACKETS.
var messages = Console.withProgressDisplayVisible(function () {
return buildmessage.capture(function () {
_.each(ISOPACKETS, function (packages, isopacketName) {
if (failedPackageBuild) {
return;
}
var isopacketRoot = isopacketPath(isopacketName);
var existingBuildinfo = files.readJSONOrNull(
files.pathJoin(isopacketRoot, 'isopacket-buildinfo.json'));
var needRebuild = !existingBuildinfo;
if (!needRebuild && existingBuildinfo.builtBy !== compiler.BUILT_BY) {
needRebuild = true;
}
if (!needRebuild) {
var watchSet = watch.WatchSet.fromJSON(existingBuildinfo.watchSet);
if (!watch.isUpToDate(watchSet)) {
needRebuild = true;
}
}
if (!needRebuild) {
// Great, it's loadable without a rebuild.
loadedIsopackets[isopacketName] = null;
return;
}
// We're going to need to build! Make a catalog and loader if we haven't
// yet.
if (!isopacketBuildContext) {
isopacketBuildContext = makeIsopacketBuildContext();
}
buildmessage.enterJob({
title: "bundling " + isopacketName + " packages for the tool"
}, function () {
// Build the packages into the in-memory IsopackCache.
isopacketBuildContext.isopackCache.buildLocalPackages(packages);
if (buildmessage.jobHasMessages()) {
return;
}
// Now bundle them into a program.
var built = bundler.buildJsImage({
name: "isopacket-" + isopacketName,
packageMap: isopacketBuildContext.packageMap,
isopackCache: isopacketBuildContext.isopackCache,
use: packages
});
if (buildmessage.jobHasMessages()) {
return;
}
var builder = new Builder({ outputPath: isopacketRoot });
builder.writeJson('isopacket-buildinfo.json', {
builtBy: compiler.BUILT_BY,
watchSet: built.watchSet.toJSON()
});
built.image.write(builder);
builder.complete();
// It's loadable now.
loadedIsopackets[isopacketName] = null;
});
});
});
});
// This is a build step ... but it's one that only happens in development, so
// it can just crash the app instead of being handled nicely.
if (messages.hasMessages()) {
Console.error("Errors prevented isopacket build:");
Console.printMessages(messages);
throw new Error("isopacket build failed?");
}
}
// Returns a new all-local-packages catalog to be used for building isopackets.
var newIsopacketBuildingCatalog = function () {
if (!files.inCheckout()) {
throw Error("No need to build isopackets unless in checkout!");
}
var catalogLocal = require('../packaging/catalog/catalog-local.js');
var isopacketCatalog = new catalogLocal.LocalCatalog;
var messages = buildmessage.capture(
{ title: "scanning local core packages" },
function () {
const packagesDir =
files.pathJoin(files.getCurrentToolsDir(), 'packages');
// When running from a checkout, isopacket building does use local
// packages, but *ONLY THOSE FROM THE CHECKOUT*: not app packages or
// $PACKAGE_DIRS packages. One side effect of this: we really really
// expect them to all build, and we're fine with dying if they don't
// (there's no worries about needing to springboard).
isopacketCatalog.initialize({
localPackageSearchDirs: [
packagesDir,
files.pathJoin(packagesDir, "non-core", "*", "packages"),
],
buildingIsopackets: true
});
});
if (messages.hasMessages()) {
Console.arrowError("Errors while scanning core packages:");
Console.printMessages(messages);
throw new Error("isopacket scan failed?");
}
return isopacketCatalog;
};
export function makeIsopacketBuildContext() {
var context = {};
var catalog = newIsopacketBuildingCatalog();
var versions = {};
_.each(catalog.getAllPackageNames(), function (packageName) {
versions[packageName] = catalog.getLatestVersion(packageName).version;
});
context.packageMap = new packageMapModule.PackageMap(versions, {
localCatalog: catalog
});
// Make an isopack cache that doesn't save isopacks to disk and has no
// access to versioned packages.
context.isopackCache = new isopackCacheModule.IsopackCache({
packageMap: context.packageMap,
includeCordovaUnibuild: false,
// When linking JS files, don't include the padding spaces and line number
// comments. Since isopackets are loaded as part of possibly very short
// 'meteor' tool command invocations, we care more about startup time than
// legibility, and the difference is actually observable (eg 25% speedup
// loading constraint-solver).
noLineNumbers: true
});
return context;
}
// Loads a built isopacket from disk. Always loads (the cache is in 'load', not
// this function). Does not run a build process; it must already be built.
var loadIsopacketFromDisk = function (isopacketName) {
var image = bundler.readJsImage(
files.pathJoin(isopacketPath(isopacketName), 'program.json'));
// An incredibly minimalist version of the environment from
// tools/server/boot.js. Kind of a hack.
var env = {
__meteor_bootstrap__: { startupHooks: [] },
__meteor_runtime_config__: { meteorRelease: "ISOPACKET" }
};
env.Profile = Profile;
var ret;
var messages = buildmessage.capture({
title: "loading isopacket `" + isopacketName + "`"
}, function () {
ret = image.load(env);
});
// This is a build step ... but it's one that only happens in development, so
// it can just crash the app instead of being handled nicely.
if (messages.hasMessages()) {
Console.error("Errors prevented isopacket load:");
Console.printMessages(messages);
throw new Error("isopacket load failed?");
}
// Run any user startup hooks.
while (env.__meteor_bootstrap__.startupHooks.length) {
var hook = env.__meteor_bootstrap__.startupHooks.shift();
hook();
}
// Setting this to null tells Meteor.startup to call hooks immediately.
env.__meteor_bootstrap__.startupHooks = null;
return ret;
};