From 122867c45c7f41b82a550a9665d34b7dda1c3ffa Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Sep 2018 16:01:51 +0100 Subject: WebAssembly support! Quite a lot going on in this PR: * Updates to support recent emscripten, switching to WASM which is now the default * Use emscripten's MODULARIZE option rather than wrapping it ourself, since doing so in pre-post js doesn't work anymore. * Most changes are moving the emscripten runtime functions to top-level calls rather than in the Module object. * Get rid of duplicated NULL_BYTE_PADDING_LENGTH * Fix ciphertext_length used without being declared * Fix things that caused the closure compiler to error, eg. using OLM_OPTIONS without a declaration. * Wait until module is inited to do OLM_ERROR = olm_error() The main BREAKING CHANGE here is that the module now needs to initialise asyncronously (because it has to load the wasm file). require()ing olm now gives a function which needs to be called to create an instance. The resulting object has a promise-like then() method that can be used to detect when the module is ready. (We could use MODULARIZE_INSTANCE to return the module directly as before, rather than the function, but then we don't get the .then() method). --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 154954c..f6c2ab4 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.so.$(VERSION) JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json +JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h @@ -60,7 +61,7 @@ CFLAGS += -Wall -Werror -std=c99 -fPIC CXXFLAGS += -Wall -Werror -std=c++11 -fPIC LDFLAGS += -Wall -Werror -EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 +EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 -s MODULARIZE=1 # NO_BROWSER is kept for compatibility with emscripten 1.35.24, but is no # longer needed. EMCCFLAGS += -s NO_BROWSER=1 @@ -150,6 +151,7 @@ $(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ + -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ $(JS_OBJECTS) -o $@ build_tests: $(TEST_BINARIES) -- cgit v1.2.3 From 5e87db615a5e430627b17da5dfbd52f0ef7f4db9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Sep 2018 16:35:17 +0100 Subject: Make OLM_OPTIONS work again The closure compiler was just renaming the variable so it never would have picked them up. Make it an extern so it knows what to do. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index f6c2ab4..dcd5cc1 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK +JS_EXTERNS := javascript/externs.js PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h @@ -147,7 +148,7 @@ js: $(JS_TARGET) .PHONY: js $(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) - $(EMCC_LINK) \ + EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ -- cgit v1.2.3 From 263b94428a24caaa5b899ed7f73b896620e6cdf4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 25 Sep 2018 17:13:29 +0100 Subject: Another day, another interface Change the interface again, hopefully this time a bit more normal. Now we wrap the emscripten module completely and just expose the high level objects. The olm library export is now imported as normal (ie. returns a module rather than a function returning a module) but has an `init` method which *must* be called. This returns a promise which resolves when the module is ready. It also rejects if the module failed to set up, unlike before (and unlike the promise-not-a-promise that emscripten returns). Generally catch failures to init the module. --- Makefile | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index dcd5cc1..d99c8fc 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,22 @@ FUZZER_BINARIES := $(addprefix $(BUILD_DIR)/,$(basename $(FUZZER_SOURCES))) FUZZER_DEBUG_BINARIES := $(patsubst $(BUILD_DIR)/fuzzers/fuzz_%,$(BUILD_DIR)/fuzzers/debug_%,$(FUZZER_BINARIES)) TEST_BINARIES := $(patsubst tests/%,$(BUILD_DIR)/tests/%,$(basename $(TEST_SOURCES))) JS_OBJECTS := $(addprefix $(BUILD_DIR)/javascript/,$(OBJECTS)) + +# pre & post are the js-pre/js-post options to emcc. +# They are injected inside the modularised code and +# processed by the optimiser. JS_PRE := $(wildcard javascript/*pre.js) JS_POST := javascript/olm_outbound_group_session.js \ javascript/olm_inbound_group_session.js \ javascript/olm_pk.js \ javascript/olm_post.js + +# The prefix & suffix are just added onto the start & end +# of what comes out emcc, so are outside of the modularised +# code and not seen by the opimiser. +JS_PREFIX := javascript/olm_prefix.js +JS_SUFFIX := javascript/olm_suffix.js + DOCS := tracing/README.html \ docs/megolm.html \ docs/olm.html \ @@ -67,6 +78,15 @@ EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 # longer needed. EMCCFLAGS += -s NO_BROWSER=1 +# Olm generally doesn't need a lot of memory to encrypt / decrypt its usual +# payloads (ie. Matrix messages), but we do need about 128K of heap to encrypt +# a 64K event (enough to store the ciphertext and the plaintext, bearing in +# mind that the plaintext can only be 48K because base64). We also have about +# 36K of statics. So let's have 256K of memory. +# (This can't be changed by the app with wasm since it's baked into the wasm). +EMCCFLAGS += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 + + EMCC.c = $(EMCC) $(CFLAGS) $(CPPFLAGS) -c EMCC.cc = $(EMCC) $(CXXFLAGS) $(CPPFLAGS) -c EMCC_LINK = $(EMCC) $(LDFLAGS) $(EMCCFLAGS) @@ -147,13 +167,19 @@ $(STATIC_RELEASE_TARGET): $(RELEASE_OBJECTS) js: $(JS_TARGET) .PHONY: js -$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) +# Note that the output file we give to emcc determines the name of the +# wasm file baked into the js, hence messing around outputting to olm.js +# and then renaming it. +$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ $(JS_OBJECTS) -o $@ + mv $@ javascript/olmtmp.js + cat $(JS_PREFIX) javascript/olmtmp.js $(JS_SUFFIX) > $@ + rm javascript/olmtmp.js build_tests: $(TEST_BINARIES) -- cgit v1.2.3 From 3e775938e50bffdfcb20241d598fea74ddaaf7e0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Oct 2018 16:06:15 +0100 Subject: Replace the impenetrable line of perl with python Mostly because the standard emscripten docker image does not have libjson-perl, but python always comes with json. But also because it was impenetrable. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 154954c..901e78f 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ fuzzers: $(FUZZER_BINARIES) $(FUZZER_DEBUG_BINARIES) .PHONY: fuzzers $(JS_EXPORTED_FUNCTIONS): $(PUBLIC_HEADERS) - perl -MJSON -ne '$$f{"_$$1"}=1 if /(olm_[^( ]*)\(/; END { @f=sort keys %f; print encode_json \@f }' $^ > $@.tmp + ./exports.py $^ > $@.tmp mv $@.tmp $@ all: test js lib debug doc -- cgit v1.2.3 From 602c00a8d658e8510e37e841dd06c70f276d0f00 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 4 Oct 2018 20:09:54 +0100 Subject: Dual-build wasm and asm.js olm --- Makefile | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index d99c8fc..f45762d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ AR = ar RELEASE_TARGET := $(BUILD_DIR)/libolm.so.$(VERSION) STATIC_RELEASE_TARGET := $(BUILD_DIR)/libolm.a DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.so.$(VERSION) -JS_TARGET := javascript/olm.js +JS_WASM_TARGET := javascript/olm.js +JS_ASMJS_TARGET := javascript/olm_legacy.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK @@ -84,8 +85,11 @@ EMCCFLAGS += -s NO_BROWSER=1 # mind that the plaintext can only be 48K because base64). We also have about # 36K of statics. So let's have 256K of memory. # (This can't be changed by the app with wasm since it's baked into the wasm). -EMCCFLAGS += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 +# (emscripten also mandates at least 16MB of memory for asm.js now, so +# we don't use this for the legacy build.) +EMCCFLAGS_WASM += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 +EMCCFLAGS_ASMJS += -s WASM=0 EMCC.c = $(EMCC) $(CFLAGS) $(CPPFLAGS) -c EMCC.cc = $(EMCC) $(CXXFLAGS) $(CPPFLAGS) -c @@ -121,7 +125,8 @@ $(FUZZER_DEBUG_BINARIES): LDFLAGS += $(DEBUG_OPTIMIZE_FLAGS) $(JS_OBJECTS): CFLAGS += $(JS_OPTIMIZE_FLAGS) $(JS_OBJECTS): CXXFLAGS += $(JS_OPTIMIZE_FLAGS) -$(JS_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) +$(JS_WASM_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) +$(JS_ASMJS_TARGET): LDFLAGS += $(JS_OPTIMIZE_FLAGS) ### Fix to make mkdir work on windows and linux ifeq ($(shell echo "check_quotes"),"check_quotes") @@ -164,14 +169,27 @@ static: $(STATIC_RELEASE_TARGET) $(STATIC_RELEASE_TARGET): $(RELEASE_OBJECTS) $(AR) rcs $@ $^ -js: $(JS_TARGET) +js: $(JS_WASM_TARGET) $(JS_ASMJS_TARGET) .PHONY: js # Note that the output file we give to emcc determines the name of the # wasm file baked into the js, hence messing around outputting to olm.js # and then renaming it. -$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) +$(JS_WASM_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ + $(EMCCFLAGS_WASM) \ + $(foreach f,$(JS_PRE),--pre-js $(f)) \ + $(foreach f,$(JS_POST),--post-js $(f)) \ + -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ + -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ + $(JS_OBJECTS) -o $@ + mv $@ javascript/olmtmp.js + cat $(JS_PREFIX) javascript/olmtmp.js $(JS_SUFFIX) > $@ + rm javascript/olmtmp.js + +$(JS_ASMJS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) + EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ + $(EMCCFLAGS_ASMJS) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ -- cgit v1.2.3 From 713e9aeb4d63732f7671f9c7ac3d9c4897449583 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Sep 2018 11:31:09 +0100 Subject: Build on mac --- Makefile | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 154954c..4dd1f4b 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,19 @@ AFL_CC = afl-gcc AFL_CXX = afl-g++ AR = ar -RELEASE_TARGET := $(BUILD_DIR)/libolm.so.$(VERSION) +UNAME := $(shell uname) +ifeq ($(UNAME),Darwin) + SO := dylib + OLM_LDFLAGS := +else + SO := so + OLM_LDFLAGS := -Wl,-soname,libolm.so.$(MAJOR) \ + -Wl,--version-script,version_script.ver +endif + +RELEASE_TARGET := $(BUILD_DIR)/libolm.$(SO).$(VERSION) STATIC_RELEASE_TARGET := $(BUILD_DIR)/libolm.a -DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.so.$(VERSION) +DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.$(SO).$(VERSION) JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json @@ -121,20 +131,18 @@ lib: $(RELEASE_TARGET) $(RELEASE_TARGET): $(RELEASE_OBJECTS) $(CXX) $(LDFLAGS) --shared -fPIC \ - -Wl,-soname,libolm.so.$(MAJOR) \ - -Wl,--version-script,version_script.ver \ + $(OLM_LDFLAGS) \ $(OUTPUT_OPTION) $(RELEASE_OBJECTS) - ln -sf libolm.so.$(VERSION) $(BUILD_DIR)/libolm.so.$(MAJOR) + ln -sf libolm.$(SO).$(VERSION) $(BUILD_DIR)/libolm.$(SO).$(MAJOR) debug: $(DEBUG_TARGET) .PHONY: debug $(DEBUG_TARGET): $(DEBUG_OBJECTS) $(CXX) $(LDFLAGS) --shared -fPIC \ - -Wl,-soname,libolm_debug.so.$(MAJOR) \ - -Wl,--version-script,version_script.ver \ + $(OLM_LDFLAGS) \ $(OUTPUT_OPTION) $(DEBUG_OBJECTS) - ln -sf libolm_debug.so.$(VERSION) $(BUILD_DIR)/libolm_debug.so.$(MAJOR) + ln -sf libolm_debug.$(SO).$(VERSION) $(BUILD_DIR)/libolm_debug.$(SO).$(MAJOR) static: $(STATIC_RELEASE_TARGET) .PHONY: static @@ -177,16 +185,16 @@ install-headers: $(PUBLIC_HEADERS) install-debug: debug install-headers test -d $(DESTDIR)$(PREFIX)/lib || $(call mkdir,$(DESTDIR)$(PREFIX)/lib) - install -Dm755 $(DEBUG_TARGET) $(DESTDIR)$(PREFIX)/lib/libolm_debug.so.$(VERSION) - ln -s libolm_debug.so.$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm_debug.so.$(MAJOR) - ln -s libolm_debug.so.$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm_debug.so + install -Dm755 $(DEBUG_TARGET) $(DESTDIR)$(PREFIX)/lib/libolm_debug.$(SO).$(VERSION) + ln -s libolm_debug.$(SO).$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm_debug.$(SO).$(MAJOR) + ln -s libolm_debug.$(SO).$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm_debug.$(SO) .PHONY: install-debug install: lib install-headers test -d $(DESTDIR)$(PREFIX)/lib || $(call mkdir,$(DESTDIR)$(PREFIX)/lib) - install -Dm755 $(RELEASE_TARGET) $(DESTDIR)$(PREFIX)/lib/libolm.so.$(VERSION) - ln -s libolm.so.$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm.so.$(MAJOR) - ln -s libolm.so.$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm.so + install -Dm755 $(RELEASE_TARGET) $(DESTDIR)$(PREFIX)/lib/libolm.$(SO).$(VERSION) + ln -s libolm.$(SO).$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm.$(SO).$(MAJOR) + ln -s libolm.$(SO).$(VERSION) $(DESTDIR)$(PREFIX)/lib/libolm.$(SO) .PHONY: install clean:; -- cgit v1.2.3