From 122867c45c7f41b82a550a9665d34b7dda1c3ffa Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Sep 2018 16:01:51 +0100 Subject: WebAssembly support! Quite a lot going on in this PR: * Updates to support recent emscripten, switching to WASM which is now the default * Use emscripten's MODULARIZE option rather than wrapping it ourself, since doing so in pre-post js doesn't work anymore. * Most changes are moving the emscripten runtime functions to top-level calls rather than in the Module object. * Get rid of duplicated NULL_BYTE_PADDING_LENGTH * Fix ciphertext_length used without being declared * Fix things that caused the closure compiler to error, eg. using OLM_OPTIONS without a declaration. * Wait until module is inited to do OLM_ERROR = olm_error() The main BREAKING CHANGE here is that the module now needs to initialise asyncronously (because it has to load the wasm file). require()ing olm now gives a function which needs to be called to create an instance. The resulting object has a promise-like then() method that can be used to detect when the module is ready. (We could use MODULARIZE_INSTANCE to return the module directly as before, rather than the function, but then we don't get the .then() method). --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 154954c..f6c2ab4 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.so.$(VERSION) JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json +JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h @@ -60,7 +61,7 @@ CFLAGS += -Wall -Werror -std=c99 -fPIC CXXFLAGS += -Wall -Werror -std=c++11 -fPIC LDFLAGS += -Wall -Werror -EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 +EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 -s MODULARIZE=1 # NO_BROWSER is kept for compatibility with emscripten 1.35.24, but is no # longer needed. EMCCFLAGS += -s NO_BROWSER=1 @@ -150,6 +151,7 @@ $(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ + -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ $(JS_OBJECTS) -o $@ build_tests: $(TEST_BINARIES) -- cgit v1.2.3-70-g09d2 From 5e87db615a5e430627b17da5dfbd52f0ef7f4db9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Sep 2018 16:35:17 +0100 Subject: Make OLM_OPTIONS work again The closure compiler was just renaming the variable so it never would have picked them up. Make it an extern so it knows what to do. --- Makefile | 3 ++- javascript/externs.js | 1 + javascript/olm_pre.js | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 javascript/externs.js (limited to 'Makefile') diff --git a/Makefile b/Makefile index f6c2ab4..dcd5cc1 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK +JS_EXTERNS := javascript/externs.js PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h @@ -147,7 +148,7 @@ js: $(JS_TARGET) .PHONY: js $(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) - $(EMCC_LINK) \ + EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ diff --git a/javascript/externs.js b/javascript/externs.js new file mode 100644 index 0000000..8ec5b02 --- /dev/null +++ b/javascript/externs.js @@ -0,0 +1 @@ +var OLM_OPTIONS; diff --git a/javascript/olm_pre.js b/javascript/olm_pre.js index 5e8ed12..673b868 100644 --- a/javascript/olm_pre.js +++ b/javascript/olm_pre.js @@ -21,10 +21,8 @@ if (typeof(window) !== 'undefined') { } /* applications should define OLM_OPTIONS in the environment to override - * emscripten module settings (we still need to (re) declare the variable - * otherwise the closure compiler becomes sad). + * emscripten module settings */ -var OLM_OPTIONS; if (typeof(OLM_OPTIONS) !== 'undefined') { for (var olm_option_key in OLM_OPTIONS) { if (OLM_OPTIONS.hasOwnProperty(olm_option_key)) { -- cgit v1.2.3-70-g09d2 From 263b94428a24caaa5b899ed7f73b896620e6cdf4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 25 Sep 2018 17:13:29 +0100 Subject: Another day, another interface Change the interface again, hopefully this time a bit more normal. Now we wrap the emscripten module completely and just expose the high level objects. The olm library export is now imported as normal (ie. returns a module rather than a function returning a module) but has an `init` method which *must* be called. This returns a promise which resolves when the module is ready. It also rejects if the module failed to set up, unlike before (and unlike the promise-not-a-promise that emscripten returns). Generally catch failures to init the module. --- Makefile | 28 +++++++++++++++++++++++++++- javascript/externs.js | 3 +++ javascript/olm_post.js | 30 +++++++----------------------- javascript/olm_pre.js | 1 - javascript/olm_prefix.js | 3 +++ javascript/olm_suffix.js | 23 +++++++++++++++++++++++ javascript/test/megolm.spec.js | 10 ++++++---- javascript/test/olm.spec.js | 16 +++++++++------- javascript/test/pk.spec.js | 10 +++++----- 9 files changed, 83 insertions(+), 41 deletions(-) create mode 100644 javascript/olm_prefix.js create mode 100644 javascript/olm_suffix.js (limited to 'Makefile') diff --git a/Makefile b/Makefile index dcd5cc1..d99c8fc 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,22 @@ FUZZER_BINARIES := $(addprefix $(BUILD_DIR)/,$(basename $(FUZZER_SOURCES))) FUZZER_DEBUG_BINARIES := $(patsubst $(BUILD_DIR)/fuzzers/fuzz_%,$(BUILD_DIR)/fuzzers/debug_%,$(FUZZER_BINARIES)) TEST_BINARIES := $(patsubst tests/%,$(BUILD_DIR)/tests/%,$(basename $(TEST_SOURCES))) JS_OBJECTS := $(addprefix $(BUILD_DIR)/javascript/,$(OBJECTS)) + +# pre & post are the js-pre/js-post options to emcc. +# They are injected inside the modularised code and +# processed by the optimiser. JS_PRE := $(wildcard javascript/*pre.js) JS_POST := javascript/olm_outbound_group_session.js \ javascript/olm_inbound_group_session.js \ javascript/olm_pk.js \ javascript/olm_post.js + +# The prefix & suffix are just added onto the start & end +# of what comes out emcc, so are outside of the modularised +# code and not seen by the opimiser. +JS_PREFIX := javascript/olm_prefix.js +JS_SUFFIX := javascript/olm_suffix.js + DOCS := tracing/README.html \ docs/megolm.html \ docs/olm.html \ @@ -67,6 +78,15 @@ EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 # longer needed. EMCCFLAGS += -s NO_BROWSER=1 +# Olm generally doesn't need a lot of memory to encrypt / decrypt its usual +# payloads (ie. Matrix messages), but we do need about 128K of heap to encrypt +# a 64K event (enough to store the ciphertext and the plaintext, bearing in +# mind that the plaintext can only be 48K because base64). We also have about +# 36K of statics. So let's have 256K of memory. +# (This can't be changed by the app with wasm since it's baked into the wasm). +EMCCFLAGS += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 + + EMCC.c = $(EMCC) $(CFLAGS) $(CPPFLAGS) -c EMCC.cc = $(EMCC) $(CXXFLAGS) $(CPPFLAGS) -c EMCC_LINK = $(EMCC) $(LDFLAGS) $(EMCCFLAGS) @@ -147,13 +167,19 @@ $(STATIC_RELEASE_TARGET): $(RELEASE_OBJECTS) js: $(JS_TARGET) .PHONY: js -$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) +# Note that the output file we give to emcc determines the name of the +# wasm file baked into the js, hence messing around outputting to olm.js +# and then renaming it. +$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ $(JS_OBJECTS) -o $@ + mv $@ javascript/olmtmp.js + cat $(JS_PREFIX) javascript/olmtmp.js $(JS_SUFFIX) > $@ + rm javascript/olmtmp.js build_tests: $(TEST_BINARIES) diff --git a/javascript/externs.js b/javascript/externs.js index 8ec5b02..752e937 100644 --- a/javascript/externs.js +++ b/javascript/externs.js @@ -1 +1,4 @@ var OLM_OPTIONS; +var olm_exports; +var onInitSuccess; +var onInitFail; diff --git a/javascript/olm_post.js b/javascript/olm_post.js index 071021c..9e0294a 100644 --- a/javascript/olm_post.js +++ b/javascript/olm_post.js @@ -464,27 +464,11 @@ olm_exports["get_library_version"] = restore_stack(function() { ]; }); -// export the olm functions into the environment. -// -// make sure that we do this *after* populating olm_exports, so that we don't -// get a half-built window.Olm if there is an exception. - -if (typeof module !== 'undefined' && module.exports) { - // node / browserify - for (var olm_export in olm_exports) { - if (olm_exports.hasOwnProperty(olm_export)) { - Module[olm_export] = olm_exports[olm_export]; - } - } -} - -if (typeof(window) !== 'undefined') { - // We've been imported directly into a browser. Define the global 'Olm' object. - // (we do this even if module.exports was defined, because it's useful to have - // Olm in the global scope for browserified and webpacked apps.) - window["Olm"] = olm_exports; -} - -Module.then(function() { +Module['onRuntimeInitialized'] = function() { OLM_ERROR = Module['_olm_error'](); -}); + if (onInitSuccess) onInitSuccess(); +}; + +Module['onAbort'] = function(err) { + if (onInitFail) onInitFail(err); +}; diff --git a/javascript/olm_pre.js b/javascript/olm_pre.js index 673b868..4feff97 100644 --- a/javascript/olm_pre.js +++ b/javascript/olm_pre.js @@ -1,4 +1,3 @@ -var olm_exports = {}; var get_random_values; if (typeof(window) !== 'undefined') { diff --git a/javascript/olm_prefix.js b/javascript/olm_prefix.js new file mode 100644 index 0000000..b33dfe9 --- /dev/null +++ b/javascript/olm_prefix.js @@ -0,0 +1,3 @@ +var olm_exports = {}; +var onInitSuccess; +var onInitFail; diff --git a/javascript/olm_suffix.js b/javascript/olm_suffix.js new file mode 100644 index 0000000..023c0a5 --- /dev/null +++ b/javascript/olm_suffix.js @@ -0,0 +1,23 @@ +olm_exports['init'] = function() { + return new Promise(function(resolve, reject) { + onInitSuccess = function() { + resolve(); + }; + onInitFail = function(err) { + reject(err); + }; + Module(); + }); +}; + +if (typeof(window) !== 'undefined') { + // We've been imported directly into a browser. Define the global 'Olm' object. + // (we do this even if module.exports was defined, because it's useful to have + // Olm in the global scope for browserified and webpacked apps.) + window["Olm"] = olm_exports; +} + +// Emscripten sets the module exports to be its module +// with wrapped c functions. Clobber it with our higher +// level wrapper class. +module.exports = olm_exports; diff --git a/javascript/test/megolm.spec.js b/javascript/test/megolm.spec.js index 9d5eb72..241d4bd 100644 --- a/javascript/test/megolm.spec.js +++ b/javascript/test/megolm.spec.js @@ -1,5 +1,6 @@ /* Copyright 2016 OpenMarket Ltd +Copyright 2018 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,17 +17,18 @@ limitations under the License. "use strict"; -var Olm = require('../olm')(); +var Olm = require('../olm'); describe("megolm", function() { var aliceSession, bobSession; beforeEach(function(done) { - Olm.then(function() { + Olm.init().then(function() { + aliceSession = new Olm.OutboundGroupSession(); + bobSession = new Olm.InboundGroupSession(); + done(); }); - aliceSession = new Olm.OutboundGroupSession(); - bobSession = new Olm.InboundGroupSession(); }); afterEach(function() { diff --git a/javascript/test/olm.spec.js b/javascript/test/olm.spec.js index 94fa87b..77dd712 100644 --- a/javascript/test/olm.spec.js +++ b/javascript/test/olm.spec.js @@ -1,5 +1,6 @@ /* Copyright 2016 OpenMarket Ltd +Copyright 2018 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,7 +17,7 @@ limitations under the License. "use strict"; -var Olm = require('../olm')(); +var Olm = require('../olm'); if (!Object.keys) { Object.keys = function(o) { @@ -33,14 +34,15 @@ describe("olm", function() { beforeEach(function(done) { // This should really be in a beforeAll, but jasmine-node // doesn't support that - Olm.then(function() { + debugger; + Olm.init().then(function() { + aliceAccount = new Olm.Account(); + bobAccount = new Olm.Account(); + aliceSession = new Olm.Session(); + bobSession = new Olm.Session(); + done(); }); - - aliceAccount = new Olm.Account(); - bobAccount = new Olm.Account(); - aliceSession = new Olm.Session(); - bobSession = new Olm.Session(); }); afterEach(function() { diff --git a/javascript/test/pk.spec.js b/javascript/test/pk.spec.js index 9f7dbfd..007882f 100644 --- a/javascript/test/pk.spec.js +++ b/javascript/test/pk.spec.js @@ -16,7 +16,7 @@ limitations under the License. "use strict"; -var Olm = require('../olm')(); +var Olm = require('../olm'); if (!Object.keys) { Object.keys = function(o) { @@ -30,12 +30,12 @@ describe("pk", function() { var encryption, decryption; beforeEach(function(done) { - Olm.then(function() { + Olm.init().then(function() { + encryption = new Olm.PkEncryption(); + decryption = new Olm.PkDecryption(); + done(); }); - - encryption = new Olm.PkEncryption(); - decryption = new Olm.PkDecryption(); }); afterEach(function () { -- cgit v1.2.3-70-g09d2