From e32b5a314a98a0a52c0406b5893ece9b2ccb0eb1 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 21 Mar 2023 18:53:26 +0000 Subject: [PATCH 01/49] wamr: barebones mpi implmenetation to run lammps --- src/wasm/WasmModule.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index 71daafb00..29b01fcda 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -827,7 +827,9 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) SPDLOG_TRACE("MEM - munmapping top of memory by {}", pageAligned); shrinkMemory(pageAligned); } else { - SPDLOG_WARN("MEM - unable to reclaim unmapped memory {} at {}", + // TODO: why are we hitting this warning so much now? Is this something + // we need to worry about? Move to debug temporarily + SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", pageAligned, offset); } From 0aafc270d28a5ad29c18c0792a1ae906a4785d16 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 11:22:08 +0000 Subject: [PATCH 02/49] wamr: more mpi functions working --- src/wamr/mpi.cpp | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 46f274701..a5d25c1c0 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -239,6 +239,41 @@ static int32_t MPI_Alltoall_wrapper(wasm_exec_env_t execEnv, int32_t recvCount, int32_t* recvType, int32_t* comm) +{ + ctx->checkMpiComm(comm); + faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); + faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); + + ctx->module->validateNativePointer(sendBuf, sendCount * hostSendDtype->size); + ctx->module->validateNativePointer(recvBuf, recvCount * hostRecvDtype->size); + + ctx->world.allToAll(ctx->rank, + (uint8_t*)sendBuf, + hostSendDtype, + sendCount, + (uint8_t*)recvBuf, + hostRecvDtype, + recvCount); + + return MPI_SUCCESS; +} + +static int32_t MPI_Alltoallv_wrapper(wasm_exec_env_t execEnv, + int32_t* sendBuf, + int32_t sendCount, + int32_t sdispls, + int32_t* sendType, + int32_t* recvBuf, + int32_t recvCount, + int32_t rdispls, + int32_t* recvType, + int32_t* comm) +{ + throw std::runtime_error("MPI_Alltoallv not implemented!"); +} + +static int32_t MPI_Barrier_wrapper(wasm_exec_env_t execEnv, + int32_t* comm) { MPI_FUNC_ARGS("S - MPI_Alltoall {} {} {} {} {} {} {}", (uintptr_t)sendBuf, @@ -882,6 +917,64 @@ static int32_t MPI_Scatter_wrapper(wasm_exec_env_t execEnv, return MPI_SUCCESS; } +static int32_t MPI_Reduce_wrapper(wasm_exec_env_t execEnv, + int32_t* sendBuf, + int32_t* recvBuf, + int32_t count, + int32_t* datatype, + int32_t* op, + int32_t root, + int32_t* comm) +{ + ctx->checkMpiComm(comm); + faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); + faabric_op_t* hostOp = ctx->getFaasmOp(op); + + ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); + + if (isInPlace(sendBuf)) { + sendBuf = recvBuf; + } else { + ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); + } + + ctx->world.reduce(ctx->rank, + root, + (uint8_t*)sendBuf, + (uint8_t*)recvBuf, + hostDtype, + count, + hostOp); + + return MPI_SUCCESS; +} + +static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, + int32_t* sendBuf, + int32_t* recvBuf, + int32_t count, + int32_t* datatype, + int32_t* op, + int32_t* comm) +{ + ctx->checkMpiComm(comm); + faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); + faabric_op_t* hostOp = ctx->getFaasmOp(op); + + ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); + + if (isInPlace(sendBuf)) { + sendBuf = recvBuf; + } else { + ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); + } + + ctx->world.scan( + ctx->rank, (uint8_t*)sendBuf, (uint8_t*)recvBuf, hostDtype, count, hostOp); + + return MPI_SUCCESS; +} + static int32_t MPI_Send_wrapper(wasm_exec_env_t execEnv, int32_t* buffer, int32_t count, From bdc8be8f75a22b731d4e4ebcb05f6d8ced5d25c8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 11:42:27 +0000 Subject: [PATCH 03/49] wamr: fix MPI_IN_PLACE detection to get MPI_Reduce to work properly --- src/wamr/mpi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index a5d25c1c0..53975ccf4 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -932,7 +932,7 @@ static int32_t MPI_Reduce_wrapper(wasm_exec_env_t execEnv, ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); - if (isInPlace(sendBuf)) { + if (ctx->isInPlace(sendBuf)) { sendBuf = recvBuf; } else { ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); @@ -963,7 +963,7 @@ static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); - if (isInPlace(sendBuf)) { + if (ctx->isInPlace(sendBuf)) { sendBuf = recvBuf; } else { ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); From 6a0c68128510c2d28cf4d4c5860360a460133d0e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 13:31:57 +0000 Subject: [PATCH 04/49] wamr: implement more mpi calls, mpi_cart_create not working --- src/wamr/mpi.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 53975ccf4..83a56deb7 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -949,6 +949,34 @@ static int32_t MPI_Reduce_wrapper(wasm_exec_env_t execEnv, return MPI_SUCCESS; } +static int32_t MPI_Reduce_scatter_wrapper(wasm_exec_env_t execEnv, + int32_t* sendBuf, + int32_t* recvBuf, + int32_t recvCount, + int32_t* datatype, + int32_t* op, + int32_t* comm) +{ + throw std::runtime_error("MPI_Reduce_scatter not implemented!"); +} + +static int32_t MPI_Request_free_wrapper(wasm_exec_env_t execEnv, + int32_t* requestPtr) +{ + throw std::runtime_error("MPI_Request_free not implemented!"); +} + +static int32_t MPI_Rsend_wrapper(wasm_exec_env_t execEnv, + int32_t* buffer, + int32_t count, + int32_t* datatype, + int32_t destRank, + int32_t tag, + int32_t* comm) +{ + throw std::runtime_error("MPI_Rsend not implemented!"); +} + static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, int32_t* sendBuf, int32_t* recvBuf, From 2f78ccad729bce3b6ab950941121bb9fdeda3766 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 14:34:43 +0000 Subject: [PATCH 05/49] wamr: all mpi stubs for lammps in place --- src/wamr/mpi.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 83a56deb7..3b36cfc90 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -1003,6 +1003,35 @@ static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, return MPI_SUCCESS; } +static int32_t MPI_Scatter_wrapper(wasm_exec_env_t execEnv, + int32_t* sendBuf, + int32_t sendCount, + int32_t* sendType, + int32_t* recvBuf, + int32_t recvCount, + int32_t* recvType, + int32_t root, + int32_t* comm) +{ + ctx->checkMpiComm(comm); + faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); + faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); + + ctx->module->validateNativePointer(sendBuf, sendCount * hostSendDtype->size); + ctx->module->validateNativePointer(recvBuf, recvCount * hostRecvDtype->size); + + ctx->world.scatter(root, + ctx->rank, + (uint8_t*)sendBuf, + hostSendDtype, + sendCount, + (uint8_t*)recvBuf, + hostRecvDtype, + recvCount); + + return MPI_SUCCESS; +} + static int32_t MPI_Send_wrapper(wasm_exec_env_t execEnv, int32_t* buffer, int32_t count, From 621b69c6a97672edfda468a6ec0ea5c0823e01f4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 15:42:35 +0000 Subject: [PATCH 06/49] wamr: mpi tests passing locally --- src/wamr/mpi.cpp | 23 +++++++++++++++-------- src/wasm/WasmModule.cpp | 4 ++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 3b36cfc90..c4c7c834c 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -244,8 +244,10 @@ static int32_t MPI_Alltoall_wrapper(wasm_exec_env_t execEnv, faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); - ctx->module->validateNativePointer(sendBuf, sendCount * hostSendDtype->size); - ctx->module->validateNativePointer(recvBuf, recvCount * hostRecvDtype->size); + ctx->module->validateNativePointer(sendBuf, + sendCount * hostSendDtype->size); + ctx->module->validateNativePointer(recvBuf, + recvCount * hostRecvDtype->size); ctx->world.allToAll(ctx->rank, (uint8_t*)sendBuf, @@ -272,8 +274,7 @@ static int32_t MPI_Alltoallv_wrapper(wasm_exec_env_t execEnv, throw std::runtime_error("MPI_Alltoallv not implemented!"); } -static int32_t MPI_Barrier_wrapper(wasm_exec_env_t execEnv, - int32_t* comm) +static int32_t MPI_Barrier_wrapper(wasm_exec_env_t execEnv, int32_t* comm) { MPI_FUNC_ARGS("S - MPI_Alltoall {} {} {} {} {} {} {}", (uintptr_t)sendBuf, @@ -997,8 +998,12 @@ static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); } - ctx->world.scan( - ctx->rank, (uint8_t*)sendBuf, (uint8_t*)recvBuf, hostDtype, count, hostOp); + ctx->world.scan(ctx->rank, + (uint8_t*)sendBuf, + (uint8_t*)recvBuf, + hostDtype, + count, + hostOp); return MPI_SUCCESS; } @@ -1017,8 +1022,10 @@ static int32_t MPI_Scatter_wrapper(wasm_exec_env_t execEnv, faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); - ctx->module->validateNativePointer(sendBuf, sendCount * hostSendDtype->size); - ctx->module->validateNativePointer(recvBuf, recvCount * hostRecvDtype->size); + ctx->module->validateNativePointer(sendBuf, + sendCount * hostSendDtype->size); + ctx->module->validateNativePointer(recvBuf, + recvCount * hostRecvDtype->size); ctx->world.scatter(root, ctx->rank, diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index 29b01fcda..bf8ba84ac 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -830,8 +830,8 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) // TODO: why are we hitting this warning so much now? Is this something // we need to worry about? Move to debug temporarily SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", - pageAligned, - offset); + pageAligned, + offset); } } From fc5dbd9b2260a649d5d7789d296f63d3355ce8b3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 15:45:30 +0000 Subject: [PATCH 07/49] nits: self-review cleanup --- src/wasm/WasmModule.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index bf8ba84ac..71daafb00 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -827,11 +827,9 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) SPDLOG_TRACE("MEM - munmapping top of memory by {}", pageAligned); shrinkMemory(pageAligned); } else { - // TODO: why are we hitting this warning so much now? Is this something - // we need to worry about? Move to debug temporarily - SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", - pageAligned, - offset); + SPDLOG_WARN("MEM - unable to reclaim unmapped memory {} at {}", + pageAligned, + offset); } } From 52e6359a1deda7cadeec5b731d780b2c36517aab Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 19:22:05 +0000 Subject: [PATCH 08/49] mpi: add logging --- src/wamr/mpi.cpp | 130 ++++------------------------------------------- 1 file changed, 9 insertions(+), 121 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index c4c7c834c..610999e31 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -240,6 +240,15 @@ static int32_t MPI_Alltoall_wrapper(wasm_exec_env_t execEnv, int32_t* recvType, int32_t* comm) { + MPI_FUNC_ARGS("S - MPI_Alltoall {} {} {} {} {} {} {}", + (uintptr_t)sendBuf, + sendCount, + (uintptr_t)sendType, + (uintptr_t)recvBuf, + recvCount, + (uintptr_t)recvType, + (uintptr_t)comm); + ctx->checkMpiComm(comm); faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); @@ -918,127 +927,6 @@ static int32_t MPI_Scatter_wrapper(wasm_exec_env_t execEnv, return MPI_SUCCESS; } -static int32_t MPI_Reduce_wrapper(wasm_exec_env_t execEnv, - int32_t* sendBuf, - int32_t* recvBuf, - int32_t count, - int32_t* datatype, - int32_t* op, - int32_t root, - int32_t* comm) -{ - ctx->checkMpiComm(comm); - faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); - faabric_op_t* hostOp = ctx->getFaasmOp(op); - - ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); - - if (ctx->isInPlace(sendBuf)) { - sendBuf = recvBuf; - } else { - ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); - } - - ctx->world.reduce(ctx->rank, - root, - (uint8_t*)sendBuf, - (uint8_t*)recvBuf, - hostDtype, - count, - hostOp); - - return MPI_SUCCESS; -} - -static int32_t MPI_Reduce_scatter_wrapper(wasm_exec_env_t execEnv, - int32_t* sendBuf, - int32_t* recvBuf, - int32_t recvCount, - int32_t* datatype, - int32_t* op, - int32_t* comm) -{ - throw std::runtime_error("MPI_Reduce_scatter not implemented!"); -} - -static int32_t MPI_Request_free_wrapper(wasm_exec_env_t execEnv, - int32_t* requestPtr) -{ - throw std::runtime_error("MPI_Request_free not implemented!"); -} - -static int32_t MPI_Rsend_wrapper(wasm_exec_env_t execEnv, - int32_t* buffer, - int32_t count, - int32_t* datatype, - int32_t destRank, - int32_t tag, - int32_t* comm) -{ - throw std::runtime_error("MPI_Rsend not implemented!"); -} - -static int32_t MPI_Scan_wrapper(wasm_exec_env_t execEnv, - int32_t* sendBuf, - int32_t* recvBuf, - int32_t count, - int32_t* datatype, - int32_t* op, - int32_t* comm) -{ - ctx->checkMpiComm(comm); - faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); - faabric_op_t* hostOp = ctx->getFaasmOp(op); - - ctx->module->validateNativePointer(recvBuf, count * hostDtype->size); - - if (ctx->isInPlace(sendBuf)) { - sendBuf = recvBuf; - } else { - ctx->module->validateNativePointer(sendBuf, count * hostDtype->size); - } - - ctx->world.scan(ctx->rank, - (uint8_t*)sendBuf, - (uint8_t*)recvBuf, - hostDtype, - count, - hostOp); - - return MPI_SUCCESS; -} - -static int32_t MPI_Scatter_wrapper(wasm_exec_env_t execEnv, - int32_t* sendBuf, - int32_t sendCount, - int32_t* sendType, - int32_t* recvBuf, - int32_t recvCount, - int32_t* recvType, - int32_t root, - int32_t* comm) -{ - ctx->checkMpiComm(comm); - faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); - faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); - - ctx->module->validateNativePointer(sendBuf, - sendCount * hostSendDtype->size); - ctx->module->validateNativePointer(recvBuf, - recvCount * hostRecvDtype->size); - - ctx->world.scatter(root, - ctx->rank, - (uint8_t*)sendBuf, - hostSendDtype, - sendCount, - (uint8_t*)recvBuf, - hostRecvDtype, - recvCount); - - return MPI_SUCCESS; -} - static int32_t MPI_Send_wrapper(wasm_exec_env_t execEnv, int32_t* buffer, int32_t count, From 39e3aadc7d1d9accd8b4b18ce223cdca858422b2 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 22 Mar 2023 19:29:28 +0000 Subject: [PATCH 09/49] wamr: unaligned write to make ubsan happy --- src/wamr/mpi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 610999e31..a9bb5fb88 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -416,7 +416,7 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, faabric_communicator_t* hostOldCommPtr = reinterpret_cast( ctx->module->wasmOffsetToNativePointer((uintptr_t)*oldCommPtr)); - *hostNewCommPtr = *hostOldCommPtr; + faabric::util::unalignedWrite(*hostOldCommPtr, (uint8_t*)hostNewCommPtr); return MPI_SUCCESS; } From aa635c76609f5ba51a79ff5017b66bccddc1dc68 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 23 Mar 2023 10:40:43 +0000 Subject: [PATCH 10/49] wamr: make ubsan happy with mpi cart creation --- src/wamr/mpi.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index a9bb5fb88..eb7ff54b1 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -11,10 +11,10 @@ using namespace faabric::scheduler; #define MPI_FUNC(str) \ - SPDLOG_TRACE("MPI-{} {}", executingContext.getRank(), str); + SPDLOG_DEBUG("MPI-{} {}", executingContext.getRank(), str); #define MPI_FUNC_ARGS(formatStr, ...) \ - SPDLOG_TRACE("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); + SPDLOG_DEBUG("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); namespace wasm { static thread_local faabric::scheduler::MpiContext executingContext; @@ -390,6 +390,12 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, // need to convert the pointed to offset into a native pointer ctx->module->validateNativePointer(newCommPtrPtr, sizeof(MPI_Comm)); MPI_Comm* newCommPtr = reinterpret_cast(newCommPtrPtr); + /* + MPI_Comm* newCommPtr = nullptr; + faabric::util::unalignedWrite( + reinterpret_cast(*newCommPtrPtr), + reinterpret_cast(newCommPtr)); + */ // Allocate memory for the pointed-to faabric_communicator_t size_t pageAlignedMemSize = @@ -416,7 +422,7 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, faabric_communicator_t* hostOldCommPtr = reinterpret_cast( ctx->module->wasmOffsetToNativePointer((uintptr_t)*oldCommPtr)); - faabric::util::unalignedWrite(*hostOldCommPtr, (uint8_t*)hostNewCommPtr); + *hostNewCommPtr = *hostOldCommPtr; return MPI_SUCCESS; } From 8e8ef527455c0a3be7d2fde15a098432453efe84 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 23 Mar 2023 13:54:30 +0000 Subject: [PATCH 11/49] wamr: more ubsan fixes --- src/wamr/mpi.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index eb7ff54b1..610999e31 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -11,10 +11,10 @@ using namespace faabric::scheduler; #define MPI_FUNC(str) \ - SPDLOG_DEBUG("MPI-{} {}", executingContext.getRank(), str); + SPDLOG_TRACE("MPI-{} {}", executingContext.getRank(), str); #define MPI_FUNC_ARGS(formatStr, ...) \ - SPDLOG_DEBUG("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); + SPDLOG_TRACE("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); namespace wasm { static thread_local faabric::scheduler::MpiContext executingContext; @@ -390,12 +390,6 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, // need to convert the pointed to offset into a native pointer ctx->module->validateNativePointer(newCommPtrPtr, sizeof(MPI_Comm)); MPI_Comm* newCommPtr = reinterpret_cast(newCommPtrPtr); - /* - MPI_Comm* newCommPtr = nullptr; - faabric::util::unalignedWrite( - reinterpret_cast(*newCommPtrPtr), - reinterpret_cast(newCommPtr)); - */ // Allocate memory for the pointed-to faabric_communicator_t size_t pageAlignedMemSize = From 43afe7440d8783a3169ade86c69f685ff43605e8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 27 Mar 2023 16:28:06 +0000 Subject: [PATCH 12/49] wamr fixes --- include/wamr/WAMRModuleMixin.h | 23 +++++- src/runner/local_pool_runner.cpp | 9 +- src/wamr/CMakeLists.txt | 3 + src/wamr/WAMRWasmModule.cpp | 84 +++++++++++++++---- src/wamr/codegen.cpp | 2 + src/wamr/env.cpp | 12 +-- src/wamr/memory.cpp | 8 +- src/wamr/mpi.cpp | 136 +++++++++++++++++++++++++++---- src/wamr/stubs.cpp | 15 ++++ src/wasm/WasmModule.cpp | 4 +- tests/test/faaslet/test_mpi.cpp | 2 + tests/test/wamr/test_wamr.cpp | 40 +++++++++ 12 files changed, 294 insertions(+), 44 deletions(-) diff --git a/include/wamr/WAMRModuleMixin.h b/include/wamr/WAMRModuleMixin.h index 0067e9f6e..c19d9575f 100644 --- a/include/wamr/WAMRModuleMixin.h +++ b/include/wamr/WAMRModuleMixin.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -28,12 +29,18 @@ struct WAMRModuleMixin // Validate that a memory range defined by a pointer and a size is a valid // offset in the module's WASM linear memory. - // bool validateNativePointer(void* nativePtr, size_t size); - bool validateNativePointer(void* nativePtr, int size) + // If we can not throw an exception in SGX, lets implement it in the + // respective headers then + void validateNativePointer(void* nativePtr, int size) { auto moduleInstance = this->underlying().getModuleInstance(); - return wasm_runtime_validate_native_addr( + // TODO: can we throw exceptions in SGX? + bool success = wasm_runtime_validate_native_addr( moduleInstance, nativePtr, size); + + if (!success) { + throw std::runtime_error("Failed validating native pointer!"); + } } void* wasmOffsetToNativePointer(uint32_t wasmOffset) @@ -50,6 +57,16 @@ struct WAMRModuleMixin return wasm_runtime_addr_native_to_app(moduleInstance, nativePtr); } + // Allocate memory in the WASM's module heap (inside the linear memory). + // Returns the WASM offset of the newly allocated memory if succesful, 0 + // otherwise. If succesful, populate the nativePtr variable with the + // native pointer to access the returned offset + uint32_t wasmModuleMalloc(size_t size, void** nativePtr) + { + auto moduleInstance = this->underlying().getModuleInstance(); + return wasm_runtime_module_malloc(moduleInstance, size, nativePtr); + } + // Helper function to write a string array to a buffer in the WASM linear // memory, and record the offsets where each new string begins (note that // in WASM this strings are now interpreted as char pointers). diff --git a/src/runner/local_pool_runner.cpp b/src/runner/local_pool_runner.cpp index edfec53ad..9ef6de9ef 100644 --- a/src/runner/local_pool_runner.cpp +++ b/src/runner/local_pool_runner.cpp @@ -32,7 +32,7 @@ int doRunner(int argc, char* argv[]) usleep(1000 * 500); for (const auto& m : req->messages()) { - faabric::Message result = sch.getFunctionResult(m.id(), 20000); + faabric::Message result = sch.getFunctionResult(m.id(), 20000 * 1000); if (result.returnvalue() != 0) { SPDLOG_ERROR("Message ({}) returned error code: {}", m.id(), @@ -53,6 +53,13 @@ int main(int argc, char* argv[]) sch.shutdown(); sch.addHostToGlobalSet(); + // Set timeout to ensure longer functions can finish + faabric::util::SystemConfig& conf = faabric::util::getSystemConfig(); + conf::FaasmConfig& faasmConf = conf::getFaasmConfig(); + conf.boundTimeout = 120000 * 100; + conf.globalMessageTimeout = 120000 * 100; + faasmConf.chainedCallTimeout = 120000 * 100; + // WARNING: All 0MQ-related operations must take place in a self-contined // scope to ensure all sockets are destructed before closing the context. { diff --git a/src/wamr/CMakeLists.txt b/src/wamr/CMakeLists.txt index 0efff3d17..e689a7a49 100644 --- a/src/wamr/CMakeLists.txt +++ b/src/wamr/CMakeLists.txt @@ -11,6 +11,7 @@ set(WAMR_BUILD_SPEC_TEST 0) add_definitions(-DWAMR_FAASM=1) # Set AOT mode and JIT for code generation +set(WAMR_BUILD_INTERPRETER 1) set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_JIT 1) set(WAMR_BUILD_LAZY_JIT 0) @@ -21,6 +22,8 @@ set(WAMR_BUILD_LIBC_WASI 1) set(WAMR_BUILD_LIB_PTHREAD 0) # WAMR features +# Do we actually need this? +# set(WAMR_BUILD_SHARED_MEMORY 1) set(WAMR_BUILD_SIMD 1) set(WAMR_BUILD_MULTI_MODULE 1) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index e86ffa0fb..77fa31901 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -1,15 +1,16 @@ +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include - #include #include +#include #include #include @@ -37,7 +38,16 @@ void WAMRWasmModule::initialiseWAMRGlobally() } // Initialise WAMR runtime - bool success = wasm_runtime_init(); + RuntimeInitArgs initArgs; + memset(&initArgs, 0, sizeof(RuntimeInitArgs)); + + // Memory configuration + initArgs.mem_alloc_type = Alloc_With_Allocator; + initArgs.mem_alloc_option.allocator.malloc_func = (void*) ::malloc; + initArgs.mem_alloc_option.allocator.realloc_func = (void*) ::realloc; + initArgs.mem_alloc_option.allocator.free_func = (void*) ::free; + + bool success = wasm_runtime_full_init(&initArgs); if (!success) { throw std::runtime_error("Failed to initialise WAMR"); } @@ -47,6 +57,9 @@ void WAMRWasmModule::initialiseWAMRGlobally() // Initialise Faasm's own native symbols initialiseWAMRNatives(); + // Set log level: BH_LOG_LEVEL_{FATAL,ERROR,WARNING,DEBUG,VERBOSE} + bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); + wamrInitialised = true; } @@ -103,6 +116,7 @@ void WAMRWasmModule::doBindToFunction(faabric::Message& msg, bool cache) // Load the wasm file storage::FileLoader& functionLoader = storage::getFileLoader(); wasmBytes = functionLoader.loadFunctionWamrAotFile(msg); + // wasmBytes = faabric::util::readFileToBytes("/usr/local/faasm/wasm/demo/exit/function.wasm"); { faabric::util::UniqueLock lock(wamrGlobalsMutex); @@ -125,9 +139,11 @@ void WAMRWasmModule::bindInternal(faabric::Message& msg) // Prepare the filesystem filesystem.prepareFilesystem(); - // Instantiate module + // Instantiate module. Set the app-managed heap size to 0 to use + // wasi-libc's managed heap. See: + // https://bytecodealliance.github.io/wamr.dev/blog/understand-the-wamr-heap/ moduleInstance = wasm_runtime_instantiate( - wasmModule, STACK_SIZE_KB, HEAP_SIZE_KB, errorBuffer, ERROR_BUFFER_SIZE); + wasmModule, STACK_SIZE_KB, 0, errorBuffer, ERROR_BUFFER_SIZE); // Sense-check the module auto* aotModule = reinterpret_cast(moduleInstance); @@ -146,8 +162,10 @@ void WAMRWasmModule::bindInternal(faabric::Message& msg) throw std::runtime_error("Failed to instantiate WAMR module"); } currentBrk.store(getMemorySizeBytes(), std::memory_order_release); + // Set up thread stacks - createThreadStacks(); + // createThreadStacks(); + threadStacks.push_back(-1); } int32_t WAMRWasmModule::executeFunction(faabric::Message& msg) @@ -159,7 +177,7 @@ int32_t WAMRWasmModule::executeFunction(faabric::Message& msg) int returnValue = 0; // Run wasm initialisers - executeWasmFunction(WASM_CTORS_FUNC_NAME); + // executeWasmFunction(WASM_CTORS_FUNC_NAME); if (msg.funcptr() > 0) { // Run the function from the pointer @@ -220,8 +238,44 @@ int WAMRWasmModule::executeWasmFunctionFromPointer(int wasmFuncPtr) int WAMRWasmModule::executeWasmFunction(const std::string& funcName) { - SPDLOG_TRACE("WAMR executing function from string {}", funcName); + SPDLOG_DEBUG("WAMR executing function from string {}", funcName); + + /* + std::unique_ptr + execEnv(wasm_runtime_create_exec_env(moduleInstance, STACK_SIZE_KB), + &wasm_runtime_destroy_exec_env); + */ + WASMExecEnv* execEnv = wasm_runtime_get_exec_env_singleton(moduleInstance); + if (execEnv == nullptr) { + SPDLOG_ERROR("Failed to create exec env for func {}", funcName); + throw std::runtime_error("Failed to create WAMR exec env"); + } + + WASMFunctionInstanceCommon* func = + wasm_runtime_lookup_function(moduleInstance, funcName.c_str(), nullptr); + if (func == nullptr) { + SPDLOG_ERROR("Did not find function {} for module {}/{}", + funcName, + boundUser, + boundFunction); + throw std::runtime_error("Did not find named wasm function"); + } + // Note, for some reason WAMR sets the return value in the argv array you + // pass it, therefore we should provide a single integer argv even though + // it's not actually used + std::vector argv = { 0 }; + bool success = wasm_runtime_call_wasm(execEnv, func, 0, argv.data()); + uint32_t returnValue = argv[0]; + // std::vector argv = { "0" }; + /* + char* argv = { 0 }; + bool success = wasm_application_execute_main(moduleInstance, 0, &argv[0]); + uint32_t returnValue = *(int*)argv; + */ + + // TODO: error checking? + /* WASMFunctionInstanceCommon* func = wasm_runtime_lookup_function(moduleInstance, funcName.c_str(), nullptr); if (func == nullptr) { @@ -239,10 +293,10 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) // Invoke the function bool success = aot_create_exec_env_and_call_function( - reinterpret_cast(moduleInstance), - reinterpret_cast(func), - 0, - argv.data()); + reinterpret_cast(moduleInstance), + reinterpret_cast(func), + 0, + argv.data()); uint32_t returnValue = argv[0]; @@ -257,6 +311,7 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) // Special case where we've set the exit code from within the host // interface + // TODO: update if (faabric::util::startsWith(errorMessage, WAMR_EXIT_PREFIX)) { std::string returnValueString = faabric::util::removeSubstr(errorMessage, WAMR_EXIT_PREFIX); @@ -278,6 +333,7 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) return returnValue; } + */ SPDLOG_DEBUG("WAMR finished executing {}", funcName); return returnValue; } diff --git a/src/wamr/codegen.cpp b/src/wamr/codegen.cpp index 07b07b9ea..a9131240f 100644 --- a/src/wamr/codegen.cpp +++ b/src/wamr/codegen.cpp @@ -61,6 +61,8 @@ std::vector wamrCodegen(std::vector& wasmBytes, bool isSgx) option.opt_level = 3; option.size_level = 3; option.output_format = AOT_FORMAT_FILE; + // Switching this flag between 0 and 1 can make some WAMR generated code + // seg-fault unexpectedly, so modify with care option.bounds_checks = 0; option.enable_bulk_memory = false; option.enable_ref_types = true; diff --git a/src/wamr/env.cpp b/src/wamr/env.cpp index c7f4e342c..75b717589 100644 --- a/src/wamr/env.cpp +++ b/src/wamr/env.cpp @@ -87,14 +87,14 @@ uint32_t wasi_environ_sizes_get(wasm_exec_env_t exec_env, return __WASI_ESUCCESS; } -void wasi_proc_exit(wasm_exec_env_t exec_env, int32_t retCode) +void wasi_proc_exit(wasm_exec_env_t execEnv, int32_t retCode) { SPDLOG_DEBUG("S - proc_exit {}", retCode); - - WAMRWasmModule* module = getExecutingWAMRModule(); - std::string resStr = WAMR_EXIT_PREFIX; - resStr += std::to_string(retCode); - wasm_runtime_set_exception(module->getModuleInstance(), resStr.c_str()); + // WAMRWasmModule* module = getExecutingWAMRModule(); + WASMModuleInstanceCommon* module_inst = wasm_runtime_get_module_inst(execEnv); + WASIContext* wasiCtx = wasm_runtime_get_wasi_ctx(module_inst); + wasm_runtime_set_exception(module_inst, "wasi proc exit"); + wasiCtx->exit_code = retCode; } static uint32_t wasi_random_get(wasm_exec_env_t exec_env, diff --git a/src/wamr/memory.cpp b/src/wamr/memory.cpp index 70194efc8..bdf502dc6 100644 --- a/src/wamr/memory.cpp +++ b/src/wamr/memory.cpp @@ -17,12 +17,14 @@ static int32_t __sbrk_wrapper(wasm_exec_env_t exec_env, int32_t increment) if (increment == 0) { return oldBrk; - } else if (increment < 0) { + } + + if (increment < 0) { module->shrinkMemory(-1 * increment); return oldBrk; - } else { - return module->growMemory(increment); } + + return module->growMemory(increment); } static int32_t mmap_wrapper(wasm_exec_env_t exec_env, diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 610999e31..9fddc0f1a 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -11,10 +11,10 @@ using namespace faabric::scheduler; #define MPI_FUNC(str) \ - SPDLOG_TRACE("MPI-{} {}", executingContext.getRank(), str); + SPDLOG_DEBUG("MPI-{} {}", executingContext.getRank(), str); #define MPI_FUNC_ARGS(formatStr, ...) \ - SPDLOG_TRACE("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); + SPDLOG_DEBUG("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); namespace wasm { static thread_local faabric::scheduler::MpiContext executingContext; @@ -26,6 +26,8 @@ static faabric::scheduler::MpiWorld& getExecutingWorld() return reg.getWorld(executingContext.getWorldId()); } +#define WASM_OFFSET_ISEND -1 + /** * Convenience wrapper around the MPI context for use in the syscalls in this * file. @@ -61,29 +63,67 @@ class WamrMpiContextWrapper } // MPI passes an MPI_Request* as part of the asynchronous API calls. - // MPI_Request is in itself a faabric_request_t* so, to write a value to - // it, we'd have to allocate memory for the faabric_reques_t. To aovid - // doing that, we write the actual request id to a faabric_reques_t*. - void writeFaasmRequestId(int32_t* requestPtrPtr, int32_t requestId) + // MPI_Request is in itself a faabric_request_t* so requestPtrPtr is a + // faabric_request_t**, which is a double wasm offset. WAMR converts the + // first offset to a native pointer. The second pointer is still a WASM + // offset. That being said, we need to populate the contents of the second + // pointer which involves: + // i) Giving the first pointer the value of a wasm offset + // ii) Accessing the memory pointed to the newly provisioned offset and + // populating the contents + void writeFaasmRequestId(int32_t* requestPtrPtr, int32_t requestId) const { module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); MPI_Request* requestPtr = reinterpret_cast(requestPtrPtr); + + // Allocate memory for the pointed-to faabric_request_t + /* + size_t pageAlignedMemSize = + roundUpToWasmPageAligned(sizeof(faabric_request_t)); + uint32_t wasmPtr = module->growMemory(pageAlignedMemSize); + */ + // Alternative strategy using wasmModuleMalloc (not working) + faabric_request_t* hostRequestPtr = nullptr; + uint32_t wasmPtr = module->wasmModuleMalloc(sizeof(faabric_request_t), + (void**)&hostRequestPtr); + if (wasmPtr == 0) { + SPDLOG_ERROR("Error allocating memory in the WASM's heap"); + throw std::runtime_error("Error allocating memory in the WASM heap"); + } + assert(hostRequestPtr != nullptr); + + // Assign the new offset (i.e. wasm pointer) to the MPI_Request var. + // Note that we are assigning a WASM offset to a native pointer, hence + // why we need to force the casting to let the compiler know we know + // what we are doing faabric::util::unalignedWrite( - reinterpret_cast(requestId), + reinterpret_cast(wasmPtr), reinterpret_cast(requestPtr)); + + // Be careful as requestPtr is a WASM offset, not a native pointer. We + // need a naitive pointer to de-reference it and access the id field + /* + faabric_request_t* hostRequestPtr = + reinterpret_cast(module->wasmOffsetToNativePointer(wasmPtr)); + module->validateNativePointer(hostRequestPtr, sizeof(faabric_communicator_t)); + */ + hostRequestPtr->id = requestId; } - // We use the same trick described before here. We take the value of - // MPI_Request (which is a faabric_request_t*) and interpret it as an int, - // the request id - int32_t getFaasmRequestId(int32_t* requestPtrPtr) + // requestPtrPtr is of type faabric_request_t** and we need to access the + // `id` field of faabric_request_t. The first pointer is a native pointer, + // the second one is a WASM offset + int32_t getFaasmRequestId(int32_t* requestPtrPtr) const { + // First level of indirection module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); - MPI_Request* requestPtr = reinterpret_cast(requestPtrPtr); - int32_t requestId = faabric::util::unalignedRead( - reinterpret_cast(requestPtr)); - return requestId; + // Second level of indirection + faabric_request_t* hostRequestPtr = + reinterpret_cast(module->wasmOffsetToNativePointer(*requestPtrPtr)); + module->validateNativePointer(hostRequestPtr, sizeof(faabric_communicator_t)); + + return hostRequestPtr->id; } // In place execution of reduce-like calls is indicated by setting the send @@ -115,6 +155,14 @@ class WamrMpiContextWrapper wasm::WAMRWasmModule* module; faabric::scheduler::MpiWorld& world; int rank; + + // Native pointers to WAMR's heap may be invalidated after calls to + // memory.growth. This is dangerous when running something like MPI_Irecv + // where we handle faabric a pointer to a buffer that it keeps around + // until the actual asynchronous message is received. WASM offsets are, + // however, stable. We then give faabric a buffer that we control, and we + // keep track of what WASM offset it belongs to. + std::map>> offsetToBufferMap; }; static thread_local std::unique_ptr ctx = nullptr; @@ -669,6 +717,7 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* requestPtrPtr) { + /* MPI_FUNC_ARGS("S - MPI_Irecv {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -677,16 +726,33 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)requestPtrPtr); + */ ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); + // TODO: we can not keep the WASM buffer around, as it may be invalidated, + // we can only keep the offset ctx->module->validateNativePointer(buffer, count * hostDtype->size); + /* int requestId = ctx->world.irecv( sourceRank, ctx->rank, (uint8_t*)buffer, hostDtype, count); + */ + std::vector ourBuf(count * hostDtype->size); + std::copy_n(reinterpret_cast(buffer), count * hostDtype->size, ourBuf.data()); + int requestId = ctx->world.irecv( + sourceRank, ctx->rank, ourBuf.data(), hostDtype, count); + // Make sure we are not copying + SPDLOG_INFO("Irecv inserting key: {}", requestId); + ctx->offsetToBufferMap[requestId] = + std::make_pair>( + ctx->module->nativePointerToWasmOffset(buffer), + std::move(ourBuf)); ctx->writeFaasmRequestId(requestPtrPtr, requestId); + MPI_FUNC_ARGS("S - MPI_Irecv {}", requestId); + return MPI_SUCCESS; } @@ -699,6 +765,7 @@ static int32_t MPI_Isend_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* requestPtrPtr) { + /* MPI_FUNC_ARGS("S - MPI_Isend {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -707,16 +774,29 @@ static int32_t MPI_Isend_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)requestPtrPtr); + */ ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); + // Giving the buffer to faabric here directly is allright, as data will be + // sent directly, and the buffer won't be re-used in a later native call. + // We still add the requestId to the map, not to make MPI_Wait be able + // to differentiate Isend from Irecv requestIds (even though it may be a + // good idea to do so for performance reasons). ctx->module->validateNativePointer(buffer, count * hostDtype->size); int requestId = ctx->world.isend(ctx->rank, destRank, (uint8_t*)buffer, hostDtype, count); + SPDLOG_INFO("Isend inserting key: {}", requestId); + ctx->offsetToBufferMap[requestId] = + std::make_pair>( + WASM_OFFSET_ISEND, + std::vector()); ctx->writeFaasmRequestId(requestPtrPtr, requestId); + MPI_FUNC_ARGS("S - MPI_Isend {}", requestId); + return MPI_SUCCESS; } @@ -1050,7 +1130,33 @@ static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, MPI_FUNC_ARGS("S - MPI_Wait {} {}", (uintptr_t)requestPtrPtr, requestId); + // WARNING: we could be running into a problem here, as I don't think it + // is safe to re-use native pointers to the heap in WAMR, as the memory + // layout may change. Thus, we may want to keep an additional map of + // requests id, to wasm offsets, and give faabric a pointer _we_ control. + // This will increase the memory usage on the host, but this is something + // we can leave with? + // wasm_export.h#L1020 - "Note that a native address to a module instance + // can be invalidated on a memory growth" ctx->world.awaitAsyncRequest(requestId); + // TODO: too many map accesses, re-factor using pointers + SPDLOG_INFO("Requesting key: {}", requestId); + uint32_t wasmOffset = ctx->offsetToBufferMap.at(requestId).first; + + // If waiting for an Isend request, remove and return + if (wasmOffset == WASM_OFFSET_ISEND) { + ctx->offsetToBufferMap.erase(requestId); + return MPI_SUCCESS; + } + + // If wainting for an Irecv request, copy the received contents into the + // right wasm offset we stored when the request was first received + uint8_t* nativePtr = (uint8_t*) ctx->module->wasmOffsetToNativePointer(wasmOffset); + ctx->module->validateNativePointer(nativePtr, ctx->offsetToBufferMap.at(requestId).second.size()); + std::copy_n(ctx->offsetToBufferMap.at(requestId).second.data(), + ctx->offsetToBufferMap.at(requestId).second.size(), + nativePtr); + ctx->offsetToBufferMap.erase(requestId); return MPI_SUCCESS; } diff --git a/src/wamr/stubs.cpp b/src/wamr/stubs.cpp index 6bd3aa8e6..191e0d6e6 100644 --- a/src/wamr/stubs.cpp +++ b/src/wamr/stubs.cpp @@ -1,5 +1,8 @@ #include +#include + #include + #include #include @@ -47,11 +50,23 @@ static int32_t shm_open_wrapper(wasm_exec_env_t exec_env, throw std::runtime_error("Native shm_open not implemented"); } +static void __faasm_interrupt_wrapper(wasm_exec_env_t execEnv) +{ + SPDLOG_INFO("External malloc!"); + + wasm::WAMRWasmModule* module = getExecutingWAMRModule(); + + void *nativePtr; + size_t mallocSize = 3 * sizeof(int); + module->wasmModuleMalloc(mallocSize, &nativePtr); +} + static NativeSymbol ns[] = { REG_NATIVE_FUNC(__cxa_allocate_exception, "(i)i"), REG_NATIVE_FUNC(__cxa_throw, "(iii)"), REG_NATIVE_FUNC(shm_open, "($ii)i"), REG_NATIVE_FUNC(syscall, "(ii)i"), + REG_NATIVE_FUNC(__faasm_interrupt, "()"), }; uint32_t getFaasmStubs(NativeSymbol** nativeSymbols) diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index 71daafb00..c298f9976 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -709,7 +709,7 @@ uint32_t WasmModule::growMemory(size_t nBytes) // If we can reclaim old memory, just bump the break if (newBrk <= oldBytes) { - SPDLOG_TRACE( + SPDLOG_DEBUG( "MEM - Growing memory using already provisioned {} + {} <= {}", oldBrk, nBytes, @@ -737,7 +737,7 @@ uint32_t WasmModule::growMemory(size_t nBytes) throw std::runtime_error("Failed to grow memory"); } - SPDLOG_TRACE("Growing memory from {} to {} pages (max {})", + SPDLOG_DEBUG("Growing memory from {} to {} pages (max {})", oldPages, newPages, maxPages); diff --git a/tests/test/faaslet/test_mpi.cpp b/tests/test/faaslet/test_mpi.cpp index 122cc09ed..1c51199b6 100644 --- a/tests/test/faaslet/test_mpi.cpp +++ b/tests/test/faaslet/test_mpi.cpp @@ -146,6 +146,7 @@ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI message ordering", "[mpi]") } // 31/12/21 - Probe support is broken after faasm/faabric#205 +/* TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI probe", "[.]") { SECTION("WAVM") { faasmConf.wasmVm = "wavm"; } @@ -154,6 +155,7 @@ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI probe", "[.]") checkMpiFunc("mpi_probe"); } +*/ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI reduce", "[mpi]") { diff --git a/tests/test/wamr/test_wamr.cpp b/tests/test/wamr/test_wamr.cpp index 714268a85..a3620fa74 100644 --- a/tests/test/wamr/test_wamr.cpp +++ b/tests/test/wamr/test_wamr.cpp @@ -105,6 +105,46 @@ TEST_CASE_METHOD(FunctionExecTestFixture, "Test WAMR sbrk", "[wamr]") REQUIRE(module.getCurrentBrk() == sizeB); } +TEST_CASE_METHOD(FunctionExecTestFixture, + "Test allocating memory in the WASM module from the runtime", + "[wamr]") +{ + std::string user; + std::string function; + + SECTION("Simple function") + { + user = "demo"; + function = "echo"; + } + + SECTION("Complex function") + { + user = "mpi"; + function = "mpi_isendrecv"; + } + + auto req = setUpContext("demo", "echo"); + faabric::Message& call = req->mutable_messages()->at(0); + std::string inputData = "hello there"; + call.set_inputdata(inputData); + + wasm::WAMRWasmModule module; + module.bindToFunction(call); + + std::vector nums = { 1, 2, 3 }; + void* nativePtr = nullptr; + uint32_t wasmOffset = module.wasmModuleMalloc(3 * sizeof(int), &nativePtr); + REQUIRE(wasmOffset != 0); + + + SPDLOG_INFO("WASM offset: {}", wasmOffset); + if (wasmOffset == 0) { + SPDLOG_ERROR("WASM module malloc failed!"); + } +} + +// TODO - move to WASM chaining tests TEST_CASE_METHOD(FunctionExecTestFixture, "Test executing chain function with WAMR", "[wamr]") From 984a1288619bb2575d79bb29e97f8e196890ec26 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 27 Mar 2023 16:30:31 +0000 Subject: [PATCH 13/49] cmake: latest wamr --- cmake/ExternalProjects.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index a3b6a3bc7..6e7e792f9 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -111,7 +111,7 @@ FetchContent_Declare(wavm_ext FetchContent_Declare(wamr_ext GIT_REPOSITORY "https://github.com/faasm/wasm-micro-runtime" - GIT_TAG "a31e5a4fa299c4f8384f40e157b0a928ad0bda1b" + GIT_TAG "8781f5b8a27babf6563edbcf185e66c5bb49c9eb" ) # WAMR and WAVM both link to LLVM From b59dde157384cf2ef0757f1b9991b4795ebf30c3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 28 Mar 2023 10:36:08 +0000 Subject: [PATCH 14/49] wamr-mpi: fixes after rebase --- src/wamr/mpi.cpp | 47 +---------------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 9fddc0f1a..cc1835b72 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -131,7 +131,7 @@ class WamrMpiContextWrapper // WAMR automatially converts the wasm offset to a native pointer as part // of the native symbol call, so we convert it back to a wasm offset and // check its value - bool isInPlace(int32_t* wasmPtr) + bool isInPlace(int32_t* wasmPtr) const { int wasmOffset = module->nativePointerToWasmOffset(wasmPtr); return wasmOffset == FAABRIC_IN_PLACE; @@ -331,51 +331,6 @@ static int32_t MPI_Alltoallv_wrapper(wasm_exec_env_t execEnv, throw std::runtime_error("MPI_Alltoallv not implemented!"); } -static int32_t MPI_Barrier_wrapper(wasm_exec_env_t execEnv, int32_t* comm) -{ - MPI_FUNC_ARGS("S - MPI_Alltoall {} {} {} {} {} {} {}", - (uintptr_t)sendBuf, - sendCount, - (uintptr_t)sendType, - (uintptr_t)recvBuf, - recvCount, - (uintptr_t)recvType, - (uintptr_t)comm); - - ctx->checkMpiComm(comm); - faabric_datatype_t* hostSendDtype = ctx->getFaasmDataType(sendType); - faabric_datatype_t* hostRecvDtype = ctx->getFaasmDataType(recvType); - - ctx->module->validateNativePointer(sendBuf, - sendCount * hostSendDtype->size); - ctx->module->validateNativePointer(recvBuf, - recvCount * hostRecvDtype->size); - - ctx->world.allToAll(ctx->rank, - (uint8_t*)sendBuf, - hostSendDtype, - sendCount, - (uint8_t*)recvBuf, - hostRecvDtype, - recvCount); - - return MPI_SUCCESS; -} - -static int32_t MPI_Alltoallv_wrapper(wasm_exec_env_t execEnv, - int32_t* sendBuf, - int32_t sendCount, - int32_t sdispls, - int32_t* sendType, - int32_t* recvBuf, - int32_t recvCount, - int32_t rdispls, - int32_t* recvType, - int32_t* comm) -{ - throw std::runtime_error("MPI_Alltoallv not implemented!"); -} - static int32_t MPI_Barrier_wrapper(wasm_exec_env_t execEnv, int32_t* comm) { MPI_FUNC_ARGS("S - MPI_Barrier {}", (uintptr_t)comm); From f2e5214f6bf1b11525e26d3447ebca208795d205 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 28 Mar 2023 11:09:04 +0000 Subject: [PATCH 15/49] nits: format --- cmake/ExternalProjects.cmake | 2 +- include/wamr/WAMRModuleMixin.h | 4 +-- src/wamr/WAMRWasmModule.cpp | 9 +++--- src/wamr/env.cpp | 3 +- src/wamr/mpi.cpp | 56 ++++++++++++++++------------------ src/wamr/stubs.cpp | 2 +- tests/test/wamr/test_wamr.cpp | 1 - 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 6e7e792f9..12cf7ff2d 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -111,7 +111,7 @@ FetchContent_Declare(wavm_ext FetchContent_Declare(wamr_ext GIT_REPOSITORY "https://github.com/faasm/wasm-micro-runtime" - GIT_TAG "8781f5b8a27babf6563edbcf185e66c5bb49c9eb" + GIT_TAG "ff15f5ecfcbcbf69317b324f991c750a1cd7a831" ) # WAMR and WAVM both link to LLVM diff --git a/include/wamr/WAMRModuleMixin.h b/include/wamr/WAMRModuleMixin.h index c19d9575f..d295bef5e 100644 --- a/include/wamr/WAMRModuleMixin.h +++ b/include/wamr/WAMRModuleMixin.h @@ -35,8 +35,8 @@ struct WAMRModuleMixin { auto moduleInstance = this->underlying().getModuleInstance(); // TODO: can we throw exceptions in SGX? - bool success = wasm_runtime_validate_native_addr( - moduleInstance, nativePtr, size); + bool success = + wasm_runtime_validate_native_addr(moduleInstance, nativePtr, size); if (!success) { throw std::runtime_error("Failed validating native pointer!"); diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index 77fa31901..e7163866d 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -43,9 +43,9 @@ void WAMRWasmModule::initialiseWAMRGlobally() // Memory configuration initArgs.mem_alloc_type = Alloc_With_Allocator; - initArgs.mem_alloc_option.allocator.malloc_func = (void*) ::malloc; - initArgs.mem_alloc_option.allocator.realloc_func = (void*) ::realloc; - initArgs.mem_alloc_option.allocator.free_func = (void*) ::free; + initArgs.mem_alloc_option.allocator.malloc_func = (void*)::malloc; + initArgs.mem_alloc_option.allocator.realloc_func = (void*)::realloc; + initArgs.mem_alloc_option.allocator.free_func = (void*)::free; bool success = wasm_runtime_full_init(&initArgs); if (!success) { @@ -116,7 +116,8 @@ void WAMRWasmModule::doBindToFunction(faabric::Message& msg, bool cache) // Load the wasm file storage::FileLoader& functionLoader = storage::getFileLoader(); wasmBytes = functionLoader.loadFunctionWamrAotFile(msg); - // wasmBytes = faabric::util::readFileToBytes("/usr/local/faasm/wasm/demo/exit/function.wasm"); + // wasmBytes = + // faabric::util::readFileToBytes("/usr/local/faasm/wasm/demo/exit/function.wasm"); { faabric::util::UniqueLock lock(wamrGlobalsMutex); diff --git a/src/wamr/env.cpp b/src/wamr/env.cpp index 75b717589..75473d315 100644 --- a/src/wamr/env.cpp +++ b/src/wamr/env.cpp @@ -91,7 +91,8 @@ void wasi_proc_exit(wasm_exec_env_t execEnv, int32_t retCode) { SPDLOG_DEBUG("S - proc_exit {}", retCode); // WAMRWasmModule* module = getExecutingWAMRModule(); - WASMModuleInstanceCommon* module_inst = wasm_runtime_get_module_inst(execEnv); + WASMModuleInstanceCommon* module_inst = + wasm_runtime_get_module_inst(execEnv); WASIContext* wasiCtx = wasm_runtime_get_wasi_ctx(module_inst); wasm_runtime_set_exception(module_inst, "wasi proc exit"); wasiCtx->exit_code = retCode; diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index cc1835b72..bf9c17521 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -77,18 +77,13 @@ class WamrMpiContextWrapper MPI_Request* requestPtr = reinterpret_cast(requestPtrPtr); // Allocate memory for the pointed-to faabric_request_t - /* - size_t pageAlignedMemSize = - roundUpToWasmPageAligned(sizeof(faabric_request_t)); - uint32_t wasmPtr = module->growMemory(pageAlignedMemSize); - */ - // Alternative strategy using wasmModuleMalloc (not working) faabric_request_t* hostRequestPtr = nullptr; uint32_t wasmPtr = module->wasmModuleMalloc(sizeof(faabric_request_t), (void**)&hostRequestPtr); if (wasmPtr == 0) { SPDLOG_ERROR("Error allocating memory in the WASM's heap"); - throw std::runtime_error("Error allocating memory in the WASM heap"); + throw std::runtime_error( + "Error allocating memory in the WASM heap"); } assert(hostRequestPtr != nullptr); @@ -102,11 +97,6 @@ class WamrMpiContextWrapper // Be careful as requestPtr is a WASM offset, not a native pointer. We // need a naitive pointer to de-reference it and access the id field - /* - faabric_request_t* hostRequestPtr = - reinterpret_cast(module->wasmOffsetToNativePointer(wasmPtr)); - module->validateNativePointer(hostRequestPtr, sizeof(faabric_communicator_t)); - */ hostRequestPtr->id = requestId; } @@ -120,8 +110,10 @@ class WamrMpiContextWrapper // Second level of indirection faabric_request_t* hostRequestPtr = - reinterpret_cast(module->wasmOffsetToNativePointer(*requestPtrPtr)); - module->validateNativePointer(hostRequestPtr, sizeof(faabric_communicator_t)); + reinterpret_cast( + module->wasmOffsetToNativePointer(*requestPtrPtr)); + module->validateNativePointer(hostRequestPtr, + sizeof(faabric_communicator_t)); return hostRequestPtr->id; } @@ -395,9 +387,14 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, MPI_Comm* newCommPtr = reinterpret_cast(newCommPtrPtr); // Allocate memory for the pointed-to faabric_communicator_t - size_t pageAlignedMemSize = - roundUpToWasmPageAligned(sizeof(faabric_communicator_t)); - uint32_t wasmPtr = ctx->module->growMemory(pageAlignedMemSize); + faabric_communicator_t* hostNewCommPtr = nullptr; + uint32_t wasmPtr = ctx->module->wasmModuleMalloc( + sizeof(faabric_communicator_t), (void**)&hostNewCommPtr); + if (wasmPtr == 0) { + SPDLOG_ERROR("Error allocating memory in the WASM's heap"); + throw std::runtime_error("Error allocating memory in the WASM heap"); + } + assert(hostNewCommPtr != nullptr); // Assign the new offset (i.e. wasm pointer) to the MPI_Comm value. Note // that we are assigning a WASM offset to a native pointer, hence why we @@ -413,9 +410,6 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, // Be careful, as *newCommPtr is a WASM offset, not a native pointer. We // need the native pointer to copy the values from the old communicator - faabric_communicator_t* hostNewCommPtr = - reinterpret_cast( - ctx->module->wasmOffsetToNativePointer(wasmPtr)); faabric_communicator_t* hostOldCommPtr = reinterpret_cast( ctx->module->wasmOffsetToNativePointer((uintptr_t)*oldCommPtr)); @@ -694,15 +688,16 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, sourceRank, ctx->rank, (uint8_t*)buffer, hostDtype, count); */ std::vector ourBuf(count * hostDtype->size); - std::copy_n(reinterpret_cast(buffer), count * hostDtype->size, ourBuf.data()); - int requestId = ctx->world.irecv( - sourceRank, ctx->rank, ourBuf.data(), hostDtype, count); + std::copy_n(reinterpret_cast(buffer), + count * hostDtype->size, + ourBuf.data()); + int requestId = + ctx->world.irecv(sourceRank, ctx->rank, ourBuf.data(), hostDtype, count); // Make sure we are not copying SPDLOG_INFO("Irecv inserting key: {}", requestId); ctx->offsetToBufferMap[requestId] = std::make_pair>( - ctx->module->nativePointerToWasmOffset(buffer), - std::move(ourBuf)); + ctx->module->nativePointerToWasmOffset(buffer), std::move(ourBuf)); ctx->writeFaasmRequestId(requestPtrPtr, requestId); @@ -744,9 +739,8 @@ static int32_t MPI_Isend_wrapper(wasm_exec_env_t execEnv, ctx->world.isend(ctx->rank, destRank, (uint8_t*)buffer, hostDtype, count); SPDLOG_INFO("Isend inserting key: {}", requestId); ctx->offsetToBufferMap[requestId] = - std::make_pair>( - WASM_OFFSET_ISEND, - std::vector()); + std::make_pair>(WASM_OFFSET_ISEND, + std::vector()); ctx->writeFaasmRequestId(requestPtrPtr, requestId); @@ -1106,8 +1100,10 @@ static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, // If wainting for an Irecv request, copy the received contents into the // right wasm offset we stored when the request was first received - uint8_t* nativePtr = (uint8_t*) ctx->module->wasmOffsetToNativePointer(wasmOffset); - ctx->module->validateNativePointer(nativePtr, ctx->offsetToBufferMap.at(requestId).second.size()); + uint8_t* nativePtr = + (uint8_t*)ctx->module->wasmOffsetToNativePointer(wasmOffset); + ctx->module->validateNativePointer( + nativePtr, ctx->offsetToBufferMap.at(requestId).second.size()); std::copy_n(ctx->offsetToBufferMap.at(requestId).second.data(), ctx->offsetToBufferMap.at(requestId).second.size(), nativePtr); diff --git a/src/wamr/stubs.cpp b/src/wamr/stubs.cpp index 191e0d6e6..5ffd074c4 100644 --- a/src/wamr/stubs.cpp +++ b/src/wamr/stubs.cpp @@ -56,7 +56,7 @@ static void __faasm_interrupt_wrapper(wasm_exec_env_t execEnv) wasm::WAMRWasmModule* module = getExecutingWAMRModule(); - void *nativePtr; + void* nativePtr; size_t mallocSize = 3 * sizeof(int); module->wasmModuleMalloc(mallocSize, &nativePtr); } diff --git a/tests/test/wamr/test_wamr.cpp b/tests/test/wamr/test_wamr.cpp index a3620fa74..8b93fa36d 100644 --- a/tests/test/wamr/test_wamr.cpp +++ b/tests/test/wamr/test_wamr.cpp @@ -137,7 +137,6 @@ TEST_CASE_METHOD(FunctionExecTestFixture, uint32_t wasmOffset = module.wasmModuleMalloc(3 * sizeof(int), &nativePtr); REQUIRE(wasmOffset != 0); - SPDLOG_INFO("WASM offset: {}", wasmOffset); if (wasmOffset == 0) { SPDLOG_ERROR("WASM module malloc failed!"); From 547982da39fab8ddd79531c170f75846b01cf112 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 29 Mar 2023 14:22:05 +0000 Subject: [PATCH 16/49] wamr: check the return value of wasm_runtime_module_malloc --- include/wamr/WAMRModuleMixin.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/wamr/WAMRModuleMixin.h b/include/wamr/WAMRModuleMixin.h index d295bef5e..8cfc09817 100644 --- a/include/wamr/WAMRModuleMixin.h +++ b/include/wamr/WAMRModuleMixin.h @@ -64,7 +64,13 @@ struct WAMRModuleMixin uint32_t wasmModuleMalloc(size_t size, void** nativePtr) { auto moduleInstance = this->underlying().getModuleInstance(); - return wasm_runtime_module_malloc(moduleInstance, size, nativePtr); + uint32_t wasmOffset = wasm_runtime_module_malloc(moduleInstance, size, nativePtr); + + if (wasmOffset == 0 || nativePtr == nullptr) { + throw std::runtime_error("Failed malloc-ing memory in WASM module!"); + } + + return wasmOffset; } // Helper function to write a string array to a buffer in the WASM linear From e16c25b8c78eecb8e8f3910d3b40c21ff45f700a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 29 Mar 2023 14:46:43 +0000 Subject: [PATCH 17/49] wamr: lammps workinggit add src/wamr/ --- src/wamr/CMakeLists.txt | 2 - src/wamr/mpi.cpp | 107 +++++++++++++--------------------------- 2 files changed, 33 insertions(+), 76 deletions(-) diff --git a/src/wamr/CMakeLists.txt b/src/wamr/CMakeLists.txt index e689a7a49..a5f80ffd5 100644 --- a/src/wamr/CMakeLists.txt +++ b/src/wamr/CMakeLists.txt @@ -22,8 +22,6 @@ set(WAMR_BUILD_LIBC_WASI 1) set(WAMR_BUILD_LIB_PTHREAD 0) # WAMR features -# Do we actually need this? -# set(WAMR_BUILD_SHARED_MEMORY 1) set(WAMR_BUILD_SIMD 1) set(WAMR_BUILD_MULTI_MODULE 1) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index bf9c17521..12b89a441 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -11,10 +11,10 @@ using namespace faabric::scheduler; #define MPI_FUNC(str) \ - SPDLOG_DEBUG("MPI-{} {}", executingContext.getRank(), str); + SPDLOG_TRACE("MPI-{} {}", executingContext.getRank(), str); #define MPI_FUNC_ARGS(formatStr, ...) \ - SPDLOG_DEBUG("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); + SPDLOG_TRACE("MPI-{} " formatStr, executingContext.getRank(), __VA_ARGS__); namespace wasm { static thread_local faabric::scheduler::MpiContext executingContext; @@ -26,8 +26,6 @@ static faabric::scheduler::MpiWorld& getExecutingWorld() return reg.getWorld(executingContext.getWorldId()); } -#define WASM_OFFSET_ISEND -1 - /** * Convenience wrapper around the MPI context for use in the syscalls in this * file. @@ -77,27 +75,30 @@ class WamrMpiContextWrapper MPI_Request* requestPtr = reinterpret_cast(requestPtrPtr); // Allocate memory for the pointed-to faabric_request_t + /* faabric_request_t* hostRequestPtr = nullptr; uint32_t wasmPtr = module->wasmModuleMalloc(sizeof(faabric_request_t), (void**)&hostRequestPtr); - if (wasmPtr == 0) { - SPDLOG_ERROR("Error allocating memory in the WASM's heap"); - throw std::runtime_error( - "Error allocating memory in the WASM heap"); - } - assert(hostRequestPtr != nullptr); + */ // Assign the new offset (i.e. wasm pointer) to the MPI_Request var. // Note that we are assigning a WASM offset to a native pointer, hence // why we need to force the casting to let the compiler know we know // what we are doing + /* unaligned write is overwritting other stack variables! faabric::util::unalignedWrite( reinterpret_cast(wasmPtr), reinterpret_cast(requestPtr)); + */ + // Without an unaligned write, it also seems we are overwritting some + // stack variables + // *requestPtr = (faabric_request_t*)((int32_t)wasmPtr); + // Go back to the old trick of setting the value in the pointer + ::memcpy(requestPtr, &requestId, sizeof(int32_t)); // Be careful as requestPtr is a WASM offset, not a native pointer. We // need a naitive pointer to de-reference it and access the id field - hostRequestPtr->id = requestId; + // hostRequestPtr->id = requestId; } // requestPtrPtr is of type faabric_request_t** and we need to access the @@ -108,14 +109,16 @@ class WamrMpiContextWrapper // First level of indirection module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); + return (int32_t) *requestPtrPtr; // Second level of indirection + /* faabric_request_t* hostRequestPtr = reinterpret_cast( module->wasmOffsetToNativePointer(*requestPtrPtr)); module->validateNativePointer(hostRequestPtr, sizeof(faabric_communicator_t)); - return hostRequestPtr->id; + */ } // In place execution of reduce-like calls is indicated by setting the send @@ -147,14 +150,6 @@ class WamrMpiContextWrapper wasm::WAMRWasmModule* module; faabric::scheduler::MpiWorld& world; int rank; - - // Native pointers to WAMR's heap may be invalidated after calls to - // memory.growth. This is dangerous when running something like MPI_Irecv - // where we handle faabric a pointer to a buffer that it keeps around - // until the actual asynchronous message is received. WASM offsets are, - // however, stable. We then give faabric a buffer that we control, and we - // keep track of what WASM offset it belongs to. - std::map>> offsetToBufferMap; }; static thread_local std::unique_ptr ctx = nullptr; @@ -399,7 +394,8 @@ static int32_t MPI_Cart_create_wrapper(wasm_exec_env_t execEnv, // Assign the new offset (i.e. wasm pointer) to the MPI_Comm value. Note // that we are assigning a WASM offset to a native pointer, hence why we // need to force the casting to let the compiler know we know what we are - // doing + // doing. Be _very_ careful with this unaligned writes, as it has happened + // before that they smash other values in the stack if not used properly faabric::util::unalignedWrite( reinterpret_cast(wasmPtr), reinterpret_cast(newCommPtr)); @@ -666,7 +662,6 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* requestPtrPtr) { - /* MPI_FUNC_ARGS("S - MPI_Irecv {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -675,7 +670,6 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)requestPtrPtr); - */ ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); @@ -683,26 +677,19 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, // TODO: we can not keep the WASM buffer around, as it may be invalidated, // we can only keep the offset ctx->module->validateNativePointer(buffer, count * hostDtype->size); - /* + // std::vector ourBuf(count * hostDtype->size); + int requestId = ctx->world.irecv( sourceRank, ctx->rank, (uint8_t*)buffer, hostDtype, count); - */ - std::vector ourBuf(count * hostDtype->size); - std::copy_n(reinterpret_cast(buffer), - count * hostDtype->size, - ourBuf.data()); - int requestId = - ctx->world.irecv(sourceRank, ctx->rank, ourBuf.data(), hostDtype, count); // Make sure we are not copying - SPDLOG_INFO("Irecv inserting key: {}", requestId); + /* ctx->offsetToBufferMap[requestId] = std::make_pair>( ctx->module->nativePointerToWasmOffset(buffer), std::move(ourBuf)); + */ ctx->writeFaasmRequestId(requestPtrPtr, requestId); - MPI_FUNC_ARGS("S - MPI_Irecv {}", requestId); - return MPI_SUCCESS; } @@ -715,7 +702,6 @@ static int32_t MPI_Isend_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* requestPtrPtr) { - /* MPI_FUNC_ARGS("S - MPI_Isend {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -724,28 +710,16 @@ static int32_t MPI_Isend_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)requestPtrPtr); - */ ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); - // Giving the buffer to faabric here directly is allright, as data will be - // sent directly, and the buffer won't be re-used in a later native call. - // We still add the requestId to the map, not to make MPI_Wait be able - // to differentiate Isend from Irecv requestIds (even though it may be a - // good idea to do so for performance reasons). ctx->module->validateNativePointer(buffer, count * hostDtype->size); int requestId = ctx->world.isend(ctx->rank, destRank, (uint8_t*)buffer, hostDtype, count); - SPDLOG_INFO("Isend inserting key: {}", requestId); - ctx->offsetToBufferMap[requestId] = - std::make_pair>(WASM_OFFSET_ISEND, - std::vector()); ctx->writeFaasmRequestId(requestPtrPtr, requestId); - MPI_FUNC_ARGS("S - MPI_Isend {}", requestId); - return MPI_SUCCESS; } @@ -786,6 +760,7 @@ static int32_t MPI_Recv_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* statusPtr) { + /* MPI_FUNC_ARGS("S - MPI_Recv {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -794,6 +769,8 @@ static int32_t MPI_Recv_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)statusPtr); + */ + MPI_FUNC_ARGS("S - MPI_Recv {} <- {}", ctx->rank, sourceRank); ctx->checkMpiComm(comm); ctx->module->validateNativePointer(statusPtr, sizeof(MPI_Status)); @@ -964,13 +941,7 @@ static int32_t MPI_Send_wrapper(wasm_exec_env_t execEnv, int32_t tag, int32_t* comm) { - MPI_FUNC_ARGS("S - MPI_Send {} {} {} {} {} {}", - (uintptr_t)buffer, - count, - (uintptr_t)datatype, - destRank, - tag, - (uintptr_t)comm); + MPI_FUNC_ARGS("S - MPI_Send {} -> {}", ctx->rank, destRank); ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); @@ -1071,6 +1042,14 @@ static int32_t MPI_Type_size_wrapper(wasm_exec_env_t execEnv, return MPI_SUCCESS; } +// As part of our implementation of MPI's asynchronous messaging in faabric, +// we keep native pointers to WASM memory. This pointers are kept between one +// asynchronous call is made, and it is MPI_Wait-ed upon. WAMR may invalidate +// native pointers after calls to memory.grow, thus this could eventually +// cause failures. If this happens, building wamr with +// `WAMR_BUILD_SHARED_MEMORY` set to `1`, should fix the issue, as then +// addresses are not invalidated. For the time being, as this has not caused +// any errors, we don't set it. static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, int32_t* requestPtrPtr, int32_t status) @@ -1088,26 +1067,6 @@ static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, // wasm_export.h#L1020 - "Note that a native address to a module instance // can be invalidated on a memory growth" ctx->world.awaitAsyncRequest(requestId); - // TODO: too many map accesses, re-factor using pointers - SPDLOG_INFO("Requesting key: {}", requestId); - uint32_t wasmOffset = ctx->offsetToBufferMap.at(requestId).first; - - // If waiting for an Isend request, remove and return - if (wasmOffset == WASM_OFFSET_ISEND) { - ctx->offsetToBufferMap.erase(requestId); - return MPI_SUCCESS; - } - - // If wainting for an Irecv request, copy the received contents into the - // right wasm offset we stored when the request was first received - uint8_t* nativePtr = - (uint8_t*)ctx->module->wasmOffsetToNativePointer(wasmOffset); - ctx->module->validateNativePointer( - nativePtr, ctx->offsetToBufferMap.at(requestId).second.size()); - std::copy_n(ctx->offsetToBufferMap.at(requestId).second.data(), - ctx->offsetToBufferMap.at(requestId).second.size(), - nativePtr); - ctx->offsetToBufferMap.erase(requestId); return MPI_SUCCESS; } From 7ebdf88368e75a99bb048b0692ef8f4e8c6d09cd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 29 Mar 2023 16:32:02 +0000 Subject: [PATCH 18/49] nits --- clients/cpp | 2 +- src/wamr/WAMRWasmModule.cpp | 2 +- src/wamr/timing.cpp | 2 +- src/wasm/WasmModule.cpp | 12 +++++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/clients/cpp b/clients/cpp index b556c29b4..5920779e7 160000 --- a/clients/cpp +++ b/clients/cpp @@ -1 +1 @@ -Subproject commit b556c29b46869d0f07d55c53b1b6819e6c4c72f1 +Subproject commit 5920779e7e70331e04463e07a4bb3a5591004174 diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index e7163866d..142513810 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -58,7 +58,7 @@ void WAMRWasmModule::initialiseWAMRGlobally() initialiseWAMRNatives(); // Set log level: BH_LOG_LEVEL_{FATAL,ERROR,WARNING,DEBUG,VERBOSE} - bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); + bh_log_set_verbose_level(BH_LOG_LEVEL_WARNING); wamrInitialised = true; } diff --git a/src/wamr/timing.cpp b/src/wamr/timing.cpp index 2379fc4e4..80a1e43dd 100644 --- a/src/wamr/timing.cpp +++ b/src/wamr/timing.cpp @@ -14,7 +14,7 @@ uint32_t wasi_clock_time_get(wasm_exec_env_t exec_env, int64_t precision, int32_t* result) { - SPDLOG_DEBUG("S - clock_time_get"); + SPDLOG_TRACE("S - clock_time_get"); timespec ts{}; diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index c298f9976..d3773f124 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -709,7 +709,7 @@ uint32_t WasmModule::growMemory(size_t nBytes) // If we can reclaim old memory, just bump the break if (newBrk <= oldBytes) { - SPDLOG_DEBUG( + SPDLOG_TRACE( "MEM - Growing memory using already provisioned {} + {} <= {}", oldBrk, nBytes, @@ -737,7 +737,7 @@ uint32_t WasmModule::growMemory(size_t nBytes) throw std::runtime_error("Failed to grow memory"); } - SPDLOG_DEBUG("Growing memory from {} to {} pages (max {})", + SPDLOG_TRACE("Growing memory from {} to {} pages (max {})", oldPages, newPages, maxPages); @@ -827,9 +827,11 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) SPDLOG_TRACE("MEM - munmapping top of memory by {}", pageAligned); shrinkMemory(pageAligned); } else { - SPDLOG_WARN("MEM - unable to reclaim unmapped memory {} at {}", - pageAligned, - offset); + // TODO - this log statement should be a warning, but for some reason + // we are running into it a lot, so I temporarily disable it + SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", + pageAligned, + offset); } } From 7c831f10389fb08281e717576eba652983fe8224 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 29 Mar 2023 16:32:36 +0000 Subject: [PATCH 19/49] nit: run clang-format --- include/wamr/WAMRModuleMixin.h | 6 ++++-- src/wamr/mpi.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/wamr/WAMRModuleMixin.h b/include/wamr/WAMRModuleMixin.h index 8cfc09817..2e0462d5a 100644 --- a/include/wamr/WAMRModuleMixin.h +++ b/include/wamr/WAMRModuleMixin.h @@ -64,10 +64,12 @@ struct WAMRModuleMixin uint32_t wasmModuleMalloc(size_t size, void** nativePtr) { auto moduleInstance = this->underlying().getModuleInstance(); - uint32_t wasmOffset = wasm_runtime_module_malloc(moduleInstance, size, nativePtr); + uint32_t wasmOffset = + wasm_runtime_module_malloc(moduleInstance, size, nativePtr); if (wasmOffset == 0 || nativePtr == nullptr) { - throw std::runtime_error("Failed malloc-ing memory in WASM module!"); + throw std::runtime_error( + "Failed malloc-ing memory in WASM module!"); } return wasmOffset; diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 12b89a441..007b6d9ac 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -109,7 +109,7 @@ class WamrMpiContextWrapper // First level of indirection module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); - return (int32_t) *requestPtrPtr; + return (int32_t)*requestPtrPtr; // Second level of indirection /* faabric_request_t* hostRequestPtr = From 19d59301a15262be8f88193869254f75370efdad Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 29 Mar 2023 16:33:21 +0000 Subject: [PATCH 20/49] gh: bump code version --- .env | 6 +++--- .github/workflows/sgx_hw.yml | 2 +- .github/workflows/tests.yml | 18 +++++++++--------- VERSION | 2 +- deploy/k8s-common/minio.yml | 2 +- deploy/k8s-common/redis.yml | 4 ++-- deploy/k8s-sgx/upload.yml | 2 +- deploy/k8s-sgx/worker.yml | 2 +- deploy/k8s/upload.yml | 2 +- deploy/k8s/worker.yml | 2 +- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.env b/.env index 55bb20663..52e161bb8 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ -FAASM_VERSION=0.9.5 -FAASM_CLI_IMAGE=faasm.azurecr.io/cli:0.9.5 -FAASM_WORKER_IMAGE=faasm.azurecr.io/worker:0.9.5 +FAASM_VERSION=0.9.6 +FAASM_CLI_IMAGE=faasm.azurecr.io/cli:0.9.6 +FAASM_WORKER_IMAGE=faasm.azurecr.io/worker:0.9.6 CPP_VERSION=0.2.4 CPP_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.2.4 diff --git a/.github/workflows/sgx_hw.yml b/.github/workflows/sgx_hw.yml index 0b74440de..8ec0fb8b2 100644 --- a/.github/workflows/sgx_hw.yml +++ b/.github/workflows/sgx_hw.yml @@ -15,7 +15,7 @@ jobs: runs-on: self-hosted env: VM_BASE_NAME: gha-sgx-hw-vm - FAASM_VERSION: 0.9.5 + FAASM_VERSION: 0.9.6 steps: - name: "Check out the experiment-base code" uses: actions/checkout@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb863d422..289498d75 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cli:0.9.5 + image: faasm.azurecr.io/cli:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} @@ -52,7 +52,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cli:0.9.5 + image: faasm.azurecr.io/cli:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} @@ -131,7 +131,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cli:0.9.5 + image: faasm.azurecr.io/cli:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} @@ -168,18 +168,18 @@ jobs: TSAN_OPTIONS: "history_size=0 halt_on_error=1 suppressions=./thread-sanitizer-ignorelist.txt flush_memory_ms=5000" UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=./ub-sanitizer-ignorelist.txt" container: - image: faasm.azurecr.io/cli:0.9.5 + image: faasm.azurecr.io/cli:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} services: redis: - image: faasm.azurecr.io/redis:0.9.5 + image: faasm.azurecr.io/redis:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} minio: - image: faasm.azurecr.io/minio:0.9.5 + image: faasm.azurecr.io/minio:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} @@ -257,18 +257,18 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm.azurecr.io/cli-sgx-sim:0.9.5 + image: faasm.azurecr.io/cli-sgx-sim:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} services: redis: - image: faasm.azurecr.io/redis:0.9.5 + image: faasm.azurecr.io/redis:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} minio: - image: faasm.azurecr.io/minio:0.9.5 + image: faasm.azurecr.io/minio:0.9.6 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} diff --git a/VERSION b/VERSION index b0bb87854..85b7c695b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.5 +0.9.6 diff --git a/deploy/k8s-common/minio.yml b/deploy/k8s-common/minio.yml index 4b9ba0d1d..1c5101256 100644 --- a/deploy/k8s-common/minio.yml +++ b/deploy/k8s-common/minio.yml @@ -11,7 +11,7 @@ metadata: spec: containers: - name: minio-main - image: faasm.azurecr.io/minio:0.9.5 + image: faasm.azurecr.io/minio:0.9.6 env: - name: MINIO_ROOT_USER value: "minio" diff --git a/deploy/k8s-common/redis.yml b/deploy/k8s-common/redis.yml index 3b0d448e5..7e0fc2590 100644 --- a/deploy/k8s-common/redis.yml +++ b/deploy/k8s-common/redis.yml @@ -11,7 +11,7 @@ metadata: spec: containers: - name: master - image: faasm.azurecr.io/redis:0.9.5 + image: faasm.azurecr.io/redis:0.9.6 ports: - containerPort: 6379 @@ -28,7 +28,7 @@ metadata: spec: containers: - name: master - image: faasm.azurecr.io/redis:0.9.5 + image: faasm.azurecr.io/redis:0.9.6 ports: - containerPort: 6379 diff --git a/deploy/k8s-sgx/upload.yml b/deploy/k8s-sgx/upload.yml index ad73d81d4..bde350d5c 100644 --- a/deploy/k8s-sgx/upload.yml +++ b/deploy/k8s-sgx/upload.yml @@ -11,7 +11,7 @@ metadata: spec: containers: - name: upload - image: faasm.azurecr.io/upload:0.9.5 + image: faasm.azurecr.io/upload:0.9.6 ports: - containerPort: 8002 - containerPort: 5000 diff --git a/deploy/k8s-sgx/worker.yml b/deploy/k8s-sgx/worker.yml index da00ca37a..6e559f67c 100644 --- a/deploy/k8s-sgx/worker.yml +++ b/deploy/k8s-sgx/worker.yml @@ -31,7 +31,7 @@ spec: weight: 100 containers: - - image: faasm.azurecr.io/worker-sgx:0.9.5 + - image: faasm.azurecr.io/worker-sgx:0.9.6 name: faasm-worker ports: - containerPort: 8080 diff --git a/deploy/k8s/upload.yml b/deploy/k8s/upload.yml index 7545007b3..621171f4c 100644 --- a/deploy/k8s/upload.yml +++ b/deploy/k8s/upload.yml @@ -11,7 +11,7 @@ metadata: spec: containers: - name: upload - image: faasm.azurecr.io/upload:0.9.5 + image: faasm.azurecr.io/upload:0.9.6 ports: - containerPort: 8002 - containerPort: 5000 diff --git a/deploy/k8s/worker.yml b/deploy/k8s/worker.yml index 3d5a203b2..328a0faf1 100644 --- a/deploy/k8s/worker.yml +++ b/deploy/k8s/worker.yml @@ -31,7 +31,7 @@ spec: weight: 100 containers: - - image: faasm.azurecr.io/worker:0.9.5 + - image: faasm.azurecr.io/worker:0.9.6 name: faasm-worker ports: - containerPort: 8080 From 5125114741017435903ff778736eda43844155e7 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 14:27:07 +0000 Subject: [PATCH 21/49] runner: remove unnecessary logging from local pool runner --- src/runner/local_pool_runner.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/runner/local_pool_runner.cpp b/src/runner/local_pool_runner.cpp index 9ef6de9ef..5ace84973 100644 --- a/src/runner/local_pool_runner.cpp +++ b/src/runner/local_pool_runner.cpp @@ -17,13 +17,9 @@ int doRunner(int argc, char* argv[]) if (vm.count("input-data")) { msg.set_inputdata(vm["input-data"].as()); - SPDLOG_INFO("Adding input data: {}", - vm["input-data"].as()); } if (vm.count("cmdline")) { msg.set_cmdline(vm["cmdline"].as()); - SPDLOG_INFO("Adding command line arguments: {}", - vm["cmdline"].as()); } faabric::scheduler::Scheduler& sch = faabric::scheduler::getScheduler(); From 5c36f741aef3dec2efc62af7d2a138dbb2f39813 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 14:57:03 +0000 Subject: [PATCH 22/49] wamr: trace logging for munmap --- src/wamr/memory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wamr/memory.cpp b/src/wamr/memory.cpp index bdf502dc6..2d4f02ca0 100644 --- a/src/wamr/memory.cpp +++ b/src/wamr/memory.cpp @@ -64,7 +64,7 @@ static int32_t munmap_wrapper(wasm_exec_env_t exec_env, int32_t addr, int32_t length) { - SPDLOG_DEBUG("S - munmap - {} {}", addr, length); + SPDLOG_TRACE("S - munmap - {} {}", addr, length); WAMRWasmModule* executingModule = getExecutingWAMRModule(); executingModule->unmapMemory(addr, length); From 3bf0fc6ce78bfe70f1ba0402fc4a5a70e2e77723 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 14:57:39 +0000 Subject: [PATCH 23/49] wamr: module cleanup and re-order prepare filesystem --- src/wamr/WAMRWasmModule.cpp | 81 ++++--------------------------------- 1 file changed, 7 insertions(+), 74 deletions(-) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index 142513810..32abbe42f 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -19,6 +19,8 @@ #include #include +#include + namespace wasm { // The high level API for WAMR can be found here: // https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/core/iwasm/include/wasm_export.h @@ -100,7 +102,7 @@ void WAMRWasmModule::reset(faabric::Message& msg, } std::string funcStr = faabric::util::funcToString(msg, true); - SPDLOG_DEBUG("WAMR resetting after {} (snap key {})", funcStr, snapshotKey); + SPDLOG_INFO("WAMR resetting after {} (snap key {})", funcStr, snapshotKey); wasm_runtime_deinstantiate(moduleInstance); bindInternal(msg); @@ -116,8 +118,6 @@ void WAMRWasmModule::doBindToFunction(faabric::Message& msg, bool cache) // Load the wasm file storage::FileLoader& functionLoader = storage::getFileLoader(); wasmBytes = functionLoader.loadFunctionWamrAotFile(msg); - // wasmBytes = - // faabric::util::readFileToBytes("/usr/local/faasm/wasm/demo/exit/function.wasm"); { faabric::util::UniqueLock lock(wamrGlobalsMutex); @@ -178,7 +178,7 @@ int32_t WAMRWasmModule::executeFunction(faabric::Message& msg) int returnValue = 0; // Run wasm initialisers - // executeWasmFunction(WASM_CTORS_FUNC_NAME); + executeWasmFunction(WASM_CTORS_FUNC_NAME); if (msg.funcptr() > 0) { // Run the function from the pointer @@ -241,11 +241,6 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) { SPDLOG_DEBUG("WAMR executing function from string {}", funcName); - /* - std::unique_ptr - execEnv(wasm_runtime_create_exec_env(moduleInstance, STACK_SIZE_KB), - &wasm_runtime_destroy_exec_env); - */ WASMExecEnv* execEnv = wasm_runtime_get_exec_env_singleton(moduleInstance); if (execEnv == nullptr) { SPDLOG_ERROR("Failed to create exec env for func {}", funcName); @@ -267,74 +262,12 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) std::vector argv = { 0 }; bool success = wasm_runtime_call_wasm(execEnv, func, 0, argv.data()); uint32_t returnValue = argv[0]; - // std::vector argv = { "0" }; - /* - char* argv = { 0 }; - bool success = wasm_application_execute_main(moduleInstance, 0, &argv[0]); - uint32_t returnValue = *(int*)argv; - */ - - // TODO: error checking? - - /* - WASMFunctionInstanceCommon* func = - wasm_runtime_lookup_function(moduleInstance, funcName.c_str(), nullptr); - if (func == nullptr) { - SPDLOG_ERROR("Did not find function {} for module {}/{}", - funcName, - boundUser, - boundFunction); - throw std::runtime_error("Did not find named wasm function"); - } - - // Note, for some reason WAMR sets the return value in the argv array you - // pass it, therefore we should provide a single integer argv even though - // it's not actually used - std::vector argv = { 0 }; - - // Invoke the function - bool success = aot_create_exec_env_and_call_function( - reinterpret_cast(moduleInstance), - reinterpret_cast(func), - 0, - argv.data()); - - uint32_t returnValue = argv[0]; - - // Check function result - if (!success || returnValue != 0) { - std::string errorMessage( - ((AOTModuleInstance*)moduleInstance)->cur_exception); - - // Strip the prefix that WAMR puts on internally - errorMessage = faabric::util::removeSubstr( - errorMessage, WAMR_INTERNAL_EXCEPTION_PREFIX); - - // Special case where we've set the exit code from within the host - // interface - // TODO: update - if (faabric::util::startsWith(errorMessage, WAMR_EXIT_PREFIX)) { - std::string returnValueString = - faabric::util::removeSubstr(errorMessage, WAMR_EXIT_PREFIX); - int parsedReturnValue = std::stoi(returnValueString); - - SPDLOG_ERROR("Caught WAMR exit code {} (from {})", - parsedReturnValue, - errorMessage); - return parsedReturnValue; - } - - SPDLOG_ERROR("Caught wasm runtime exception: {}", errorMessage); - // Ensure return value is not zero if not successful - if (returnValue == 0) { - returnValue = 1; - } - - return returnValue; + if (!success) { + SPDLOG_ERROR("Error executing {}: {}", funcName, wasm_runtime_get_exception(moduleInstance)); + throw std::runtime_error("Error executing WASM function with WAMR"); } - */ SPDLOG_DEBUG("WAMR finished executing {}", funcName); return returnValue; } From dbdddb3099f32fd1a80f953470e0a499f6baa33d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:00:23 +0000 Subject: [PATCH 24/49] tests: add regression tests for lack of fd clearing --- tests/test/faaslet/test_shared_files.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test/faaslet/test_shared_files.cpp b/tests/test/faaslet/test_shared_files.cpp index 923ca2632..d39629129 100644 --- a/tests/test/faaslet/test_shared_files.cpp +++ b/tests/test/faaslet/test_shared_files.cpp @@ -47,7 +47,11 @@ TEST_CASE_METHOD(SharedFilesExecTestFixture, auto req = setUpContext("demo", "shared_file"); SECTION("WAVM") { execFunction(req); } - SECTION("WAMR") { execWamrFunction(req->mutable_messages()->at(0)); } + SECTION("WAMR") + { + execWamrFunction(req->mutable_messages()->at(0)); + execWamrFunction(req->mutable_messages()->at(0)); + } // Check file has been synced locally REQUIRE(boost::filesystem::exists(fullPath)); From eabc039e28ec0eda4c78d1bb8f33bcdf80748787 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:00:38 +0000 Subject: [PATCH 25/49] tests: add fixture for fd tests --- tests/test/storage/test_file_descriptor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test/storage/test_file_descriptor.cpp b/tests/test/storage/test_file_descriptor.cpp index 688916a8b..c7dbb6851 100644 --- a/tests/test/storage/test_file_descriptor.cpp +++ b/tests/test/storage/test_file_descriptor.cpp @@ -10,9 +10,9 @@ #include #include -#include #include #include +#include using namespace storage; From 8d34fca33feae1a46456a093dc641ca7f6881e76 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:03:15 +0000 Subject: [PATCH 26/49] nits: run clang-format --- tests/test/storage/test_file_descriptor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test/storage/test_file_descriptor.cpp b/tests/test/storage/test_file_descriptor.cpp index c7dbb6851..688916a8b 100644 --- a/tests/test/storage/test_file_descriptor.cpp +++ b/tests/test/storage/test_file_descriptor.cpp @@ -10,9 +10,9 @@ #include #include +#include #include #include -#include using namespace storage; From 73217d650b0346e1b92bbeab756f39600740832b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:08:10 +0000 Subject: [PATCH 27/49] nits: self-review --- tests/test/faaslet/test_shared_files.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test/faaslet/test_shared_files.cpp b/tests/test/faaslet/test_shared_files.cpp index d39629129..923ca2632 100644 --- a/tests/test/faaslet/test_shared_files.cpp +++ b/tests/test/faaslet/test_shared_files.cpp @@ -47,11 +47,7 @@ TEST_CASE_METHOD(SharedFilesExecTestFixture, auto req = setUpContext("demo", "shared_file"); SECTION("WAVM") { execFunction(req); } - SECTION("WAMR") - { - execWamrFunction(req->mutable_messages()->at(0)); - execWamrFunction(req->mutable_messages()->at(0)); - } + SECTION("WAMR") { execWamrFunction(req->mutable_messages()->at(0)); } // Check file has been synced locally REQUIRE(boost::filesystem::exists(fullPath)); From df956edc4123cf48d5028d7dcc61529d43fd5e56 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:29:01 +0000 Subject: [PATCH 28/49] local pool runner: longer timeouts for long executions --- include/wamr/WAMRModuleMixin.h | 3 --- src/runner/local_pool_runner.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/include/wamr/WAMRModuleMixin.h b/include/wamr/WAMRModuleMixin.h index 2e0462d5a..710ad1a52 100644 --- a/include/wamr/WAMRModuleMixin.h +++ b/include/wamr/WAMRModuleMixin.h @@ -29,12 +29,9 @@ struct WAMRModuleMixin // Validate that a memory range defined by a pointer and a size is a valid // offset in the module's WASM linear memory. - // If we can not throw an exception in SGX, lets implement it in the - // respective headers then void validateNativePointer(void* nativePtr, int size) { auto moduleInstance = this->underlying().getModuleInstance(); - // TODO: can we throw exceptions in SGX? bool success = wasm_runtime_validate_native_addr(moduleInstance, nativePtr, size); diff --git a/src/runner/local_pool_runner.cpp b/src/runner/local_pool_runner.cpp index 5ace84973..f28525ef2 100644 --- a/src/runner/local_pool_runner.cpp +++ b/src/runner/local_pool_runner.cpp @@ -28,7 +28,7 @@ int doRunner(int argc, char* argv[]) usleep(1000 * 500); for (const auto& m : req->messages()) { - faabric::Message result = sch.getFunctionResult(m.id(), 20000 * 1000); + faabric::Message result = sch.getFunctionResult(m.id(), 20000 * 100); if (result.returnvalue() != 0) { SPDLOG_ERROR("Message ({}) returned error code: {}", m.id(), From 5a170c9a5a0672b7e71b81ac0db26fb1123c0b08 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:42:45 +0000 Subject: [PATCH 29/49] nits: more self-review --- src/wamr/WAMRWasmModule.cpp | 5 ++-- src/wamr/mpi.cpp | 51 ++++++------------------------------- 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index 32abbe42f..778725fd9 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -102,7 +102,7 @@ void WAMRWasmModule::reset(faabric::Message& msg, } std::string funcStr = faabric::util::funcToString(msg, true); - SPDLOG_INFO("WAMR resetting after {} (snap key {})", funcStr, snapshotKey); + SPDLOG_DEBUG("WAMR resetting after {} (snap key {})", funcStr, snapshotKey); wasm_runtime_deinstantiate(moduleInstance); bindInternal(msg); @@ -165,8 +165,7 @@ void WAMRWasmModule::bindInternal(faabric::Message& msg) currentBrk.store(getMemorySizeBytes(), std::memory_order_release); // Set up thread stacks - // createThreadStacks(); - threadStacks.push_back(-1); + createThreadStacks(); } int32_t WAMRWasmModule::executeFunction(faabric::Message& msg) diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 007b6d9ac..38473346d 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -62,63 +62,28 @@ class WamrMpiContextWrapper // MPI passes an MPI_Request* as part of the asynchronous API calls. // MPI_Request is in itself a faabric_request_t* so requestPtrPtr is a - // faabric_request_t**, which is a double wasm offset. WAMR converts the - // first offset to a native pointer. The second pointer is still a WASM - // offset. That being said, we need to populate the contents of the second - // pointer which involves: - // i) Giving the first pointer the value of a wasm offset - // ii) Accessing the memory pointed to the newly provisioned offset and - // populating the contents + // faabric_request_t**, which is a double wasm offset. Allocating memory + // for the second pointer from outside WASM is tricky and error-prone, + // so we use overwrite the pointer value with the actual pointed-to value + // (the pointer just points to one int, so we can do that). void writeFaasmRequestId(int32_t* requestPtrPtr, int32_t requestId) const { module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); MPI_Request* requestPtr = reinterpret_cast(requestPtrPtr); - // Allocate memory for the pointed-to faabric_request_t - /* - faabric_request_t* hostRequestPtr = nullptr; - uint32_t wasmPtr = module->wasmModuleMalloc(sizeof(faabric_request_t), - (void**)&hostRequestPtr); - */ - - // Assign the new offset (i.e. wasm pointer) to the MPI_Request var. - // Note that we are assigning a WASM offset to a native pointer, hence - // why we need to force the casting to let the compiler know we know - // what we are doing - /* unaligned write is overwritting other stack variables! - faabric::util::unalignedWrite( - reinterpret_cast(wasmPtr), - reinterpret_cast(requestPtr)); - */ - // Without an unaligned write, it also seems we are overwritting some - // stack variables - // *requestPtr = (faabric_request_t*)((int32_t)wasmPtr); - // Go back to the old trick of setting the value in the pointer + // Be very careful with this copy, as we may overwrite other variables + // in the stack ::memcpy(requestPtr, &requestId, sizeof(int32_t)); - - // Be careful as requestPtr is a WASM offset, not a native pointer. We - // need a naitive pointer to de-reference it and access the id field - // hostRequestPtr->id = requestId; } - // requestPtrPtr is of type faabric_request_t** and we need to access the - // `id` field of faabric_request_t. The first pointer is a native pointer, - // the second one is a WASM offset + // We use the same trick explained in the previous function, whereby we + // read the integer from a pointer (without dereferencing it) int32_t getFaasmRequestId(int32_t* requestPtrPtr) const { // First level of indirection module->validateNativePointer(requestPtrPtr, sizeof(MPI_Request)); return (int32_t)*requestPtrPtr; - // Second level of indirection - /* - faabric_request_t* hostRequestPtr = - reinterpret_cast( - module->wasmOffsetToNativePointer(*requestPtrPtr)); - module->validateNativePointer(hostRequestPtr, - sizeof(faabric_communicator_t)); - return hostRequestPtr->id; - */ } // In place execution of reduce-like calls is indicated by setting the send From 55283cf989fc14dc71034ce36e35b6a025535935 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 15:42:59 +0000 Subject: [PATCH 30/49] clang-format --- src/wamr/WAMRWasmModule.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index 778725fd9..8d31062d6 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -263,7 +263,9 @@ int WAMRWasmModule::executeWasmFunction(const std::string& funcName) uint32_t returnValue = argv[0]; if (!success) { - SPDLOG_ERROR("Error executing {}: {}", funcName, wasm_runtime_get_exception(moduleInstance)); + SPDLOG_ERROR("Error executing {}: {}", + funcName, + wasm_runtime_get_exception(moduleInstance)); throw std::runtime_error("Error executing WASM function with WAMR"); } From 681af88b6de25b2ad07e2f55b15cf3f1395b26e0 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 16:38:25 +0000 Subject: [PATCH 31/49] nits: more self-review --- src/wamr/WAMRWasmModule.cpp | 2 -- src/wamr/env.cpp | 8 +++----- src/wamr/mpi.cpp | 22 ++-------------------- src/wamr/stubs.cpp | 15 --------------- tests/test/faaslet/test_mpi.cpp | 2 -- tests/test/wamr/test_wamr.cpp | 12 +++--------- 6 files changed, 8 insertions(+), 53 deletions(-) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index 8d31062d6..a653511ce 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -19,8 +19,6 @@ #include #include -#include - namespace wasm { // The high level API for WAMR can be found here: // https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/core/iwasm/include/wasm_export.h diff --git a/src/wamr/env.cpp b/src/wamr/env.cpp index 75473d315..5573d14b7 100644 --- a/src/wamr/env.cpp +++ b/src/wamr/env.cpp @@ -90,11 +90,9 @@ uint32_t wasi_environ_sizes_get(wasm_exec_env_t exec_env, void wasi_proc_exit(wasm_exec_env_t execEnv, int32_t retCode) { SPDLOG_DEBUG("S - proc_exit {}", retCode); - // WAMRWasmModule* module = getExecutingWAMRModule(); - WASMModuleInstanceCommon* module_inst = - wasm_runtime_get_module_inst(execEnv); - WASIContext* wasiCtx = wasm_runtime_get_wasi_ctx(module_inst); - wasm_runtime_set_exception(module_inst, "wasi proc exit"); + auto* moduleInstance = getExecutingWAMRModule()->getModuleInstance(); + WASIContext* wasiCtx = wasm_runtime_get_wasi_ctx(moduleInstance); + wasm_runtime_set_exception(moduleInstance, "wasi proc exit"); wasiCtx->exit_code = retCode; } diff --git a/src/wamr/mpi.cpp b/src/wamr/mpi.cpp index 38473346d..b5f63dcdd 100644 --- a/src/wamr/mpi.cpp +++ b/src/wamr/mpi.cpp @@ -639,19 +639,10 @@ static int32_t MPI_Irecv_wrapper(wasm_exec_env_t execEnv, ctx->checkMpiComm(comm); faabric_datatype_t* hostDtype = ctx->getFaasmDataType(datatype); - // TODO: we can not keep the WASM buffer around, as it may be invalidated, - // we can only keep the offset ctx->module->validateNativePointer(buffer, count * hostDtype->size); - // std::vector ourBuf(count * hostDtype->size); int requestId = ctx->world.irecv( sourceRank, ctx->rank, (uint8_t*)buffer, hostDtype, count); - // Make sure we are not copying - /* - ctx->offsetToBufferMap[requestId] = - std::make_pair>( - ctx->module->nativePointerToWasmOffset(buffer), std::move(ourBuf)); - */ ctx->writeFaasmRequestId(requestPtrPtr, requestId); @@ -725,7 +716,6 @@ static int32_t MPI_Recv_wrapper(wasm_exec_env_t execEnv, int32_t* comm, int32_t* statusPtr) { - /* MPI_FUNC_ARGS("S - MPI_Recv {} {} {} {} {} {} {}", (uintptr_t)buffer, count, @@ -734,8 +724,6 @@ static int32_t MPI_Recv_wrapper(wasm_exec_env_t execEnv, tag, (uintptr_t)comm, (uintptr_t)statusPtr); - */ - MPI_FUNC_ARGS("S - MPI_Recv {} <- {}", ctx->rank, sourceRank); ctx->checkMpiComm(comm); ctx->module->validateNativePointer(statusPtr, sizeof(MPI_Status)); @@ -1015,6 +1003,8 @@ static int32_t MPI_Type_size_wrapper(wasm_exec_env_t execEnv, // `WAMR_BUILD_SHARED_MEMORY` set to `1`, should fix the issue, as then // addresses are not invalidated. For the time being, as this has not caused // any errors, we don't set it. +// wasm_export.h#L1020 - "Note that a native address to a module instance +// can be invalidated on a memory growth" static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, int32_t* requestPtrPtr, int32_t status) @@ -1023,14 +1013,6 @@ static int32_t MPI_Wait_wrapper(wasm_exec_env_t execEnv, MPI_FUNC_ARGS("S - MPI_Wait {} {}", (uintptr_t)requestPtrPtr, requestId); - // WARNING: we could be running into a problem here, as I don't think it - // is safe to re-use native pointers to the heap in WAMR, as the memory - // layout may change. Thus, we may want to keep an additional map of - // requests id, to wasm offsets, and give faabric a pointer _we_ control. - // This will increase the memory usage on the host, but this is something - // we can leave with? - // wasm_export.h#L1020 - "Note that a native address to a module instance - // can be invalidated on a memory growth" ctx->world.awaitAsyncRequest(requestId); return MPI_SUCCESS; diff --git a/src/wamr/stubs.cpp b/src/wamr/stubs.cpp index 5ffd074c4..6bd3aa8e6 100644 --- a/src/wamr/stubs.cpp +++ b/src/wamr/stubs.cpp @@ -1,8 +1,5 @@ #include -#include - #include - #include #include @@ -50,23 +47,11 @@ static int32_t shm_open_wrapper(wasm_exec_env_t exec_env, throw std::runtime_error("Native shm_open not implemented"); } -static void __faasm_interrupt_wrapper(wasm_exec_env_t execEnv) -{ - SPDLOG_INFO("External malloc!"); - - wasm::WAMRWasmModule* module = getExecutingWAMRModule(); - - void* nativePtr; - size_t mallocSize = 3 * sizeof(int); - module->wasmModuleMalloc(mallocSize, &nativePtr); -} - static NativeSymbol ns[] = { REG_NATIVE_FUNC(__cxa_allocate_exception, "(i)i"), REG_NATIVE_FUNC(__cxa_throw, "(iii)"), REG_NATIVE_FUNC(shm_open, "($ii)i"), REG_NATIVE_FUNC(syscall, "(ii)i"), - REG_NATIVE_FUNC(__faasm_interrupt, "()"), }; uint32_t getFaasmStubs(NativeSymbol** nativeSymbols) diff --git a/tests/test/faaslet/test_mpi.cpp b/tests/test/faaslet/test_mpi.cpp index 1c51199b6..122cc09ed 100644 --- a/tests/test/faaslet/test_mpi.cpp +++ b/tests/test/faaslet/test_mpi.cpp @@ -146,7 +146,6 @@ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI message ordering", "[mpi]") } // 31/12/21 - Probe support is broken after faasm/faabric#205 -/* TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI probe", "[.]") { SECTION("WAVM") { faasmConf.wasmVm = "wavm"; } @@ -155,7 +154,6 @@ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI probe", "[.]") checkMpiFunc("mpi_probe"); } -*/ TEST_CASE_METHOD(MPIFuncTestFixture, "Test MPI reduce", "[mpi]") { diff --git a/tests/test/wamr/test_wamr.cpp b/tests/test/wamr/test_wamr.cpp index 8b93fa36d..49e78535b 100644 --- a/tests/test/wamr/test_wamr.cpp +++ b/tests/test/wamr/test_wamr.cpp @@ -118,10 +118,12 @@ TEST_CASE_METHOD(FunctionExecTestFixture, function = "echo"; } + // Note that mpi_cart_create calls MPI_Cart_create that in turn calls + // wasmModuleMalloc again SECTION("Complex function") { user = "mpi"; - function = "mpi_isendrecv"; + function = "mpi_cart_create"; } auto req = setUpContext("demo", "echo"); @@ -142,12 +144,4 @@ TEST_CASE_METHOD(FunctionExecTestFixture, SPDLOG_ERROR("WASM module malloc failed!"); } } - -// TODO - move to WASM chaining tests -TEST_CASE_METHOD(FunctionExecTestFixture, - "Test executing chain function with WAMR", - "[wamr]") -{ - executeWithWamrPool("demo", "chain", 10000); -} } From 5a2b5257aaf009208fd62375d434411639265e55 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 16:44:47 +0000 Subject: [PATCH 32/49] enclave: stop using deprecated API call --- src/enclave/inside/EnclaveWasmModule.cpp | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/enclave/inside/EnclaveWasmModule.cpp b/src/enclave/inside/EnclaveWasmModule.cpp index 5aa92769c..86c3d55a2 100644 --- a/src/enclave/inside/EnclaveWasmModule.cpp +++ b/src/enclave/inside/EnclaveWasmModule.cpp @@ -60,26 +60,31 @@ bool EnclaveWasmModule::loadWasm(void* wasmOpCodePtr, uint32_t wasmOpCodeSize) bool EnclaveWasmModule::callFunction(uint32_t argcIn, char** argvIn) { + prepareArgcArgv(argcIn, argvIn); + + WASMExecEnv* execEnv = wasm_runtime_get_exec_env_singleton(moduleInstance); + if (execEnv == nullptr) { + ocallLogError("Failed to create WAMR exec env"); + throw std::runtime_error("Failed to create WAMR exec env"); + } + WASMFunctionInstanceCommon* func = wasm_runtime_lookup_function(moduleInstance, WASM_ENTRY_FUNC, nullptr); - - prepareArgcArgv(argcIn, argvIn); + if (func == nullptr) { + ocallLogError("Did not find named WASM function"); + throw std::runtime_error("Did not find named wasm function"); + } // Set dummy argv to capture return value std::vector argv = { 0 }; + bool success = wasm_runtime_call_wasm(execEnv, func, 0, argv.data()); + uint32_t returnValue = argv[0]; - bool success = - aot_create_exec_env_and_call_function((AOTModuleInstance*)moduleInstance, - (AOTFunctionInstance*)func, - 0x0, - argv.data()); if (success) { ocallLogDebug("Success calling WASM function"); } else { - std::string errorMessage( - ((AOTModuleInstance*)moduleInstance)->cur_exception); - // TODO - better logging + std::string errorMessage(wasm_runtime_get_exception(moduleInstance)); std::string errorText = "Caught WASM runtime exception: " + errorMessage; ocallLogError(errorText.c_str()); From 858450aa974457943bc1843170c031972c40a137 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 16:54:47 +0000 Subject: [PATCH 33/49] wamr: update cmake reference --- cmake/ExternalProjects.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 12cf7ff2d..fa666adf5 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -111,7 +111,7 @@ FetchContent_Declare(wavm_ext FetchContent_Declare(wamr_ext GIT_REPOSITORY "https://github.com/faasm/wasm-micro-runtime" - GIT_TAG "ff15f5ecfcbcbf69317b324f991c750a1cd7a831" + GIT_TAG "2725028a8673dc0a2d2fa1ba46e5b52ba60b3d74" ) # WAMR and WAVM both link to LLVM From af147ebb8761aa37801dda88e30567c0412d22cf Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 30 Mar 2023 18:03:23 +0000 Subject: [PATCH 34/49] cpp: bump tag version --- .env | 4 ++-- .github/workflows/tests.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 52e161bb8..737063d66 100644 --- a/.env +++ b/.env @@ -2,8 +2,8 @@ FAASM_VERSION=0.9.6 FAASM_CLI_IMAGE=faasm.azurecr.io/cli:0.9.6 FAASM_WORKER_IMAGE=faasm.azurecr.io/worker:0.9.6 -CPP_VERSION=0.2.4 -CPP_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.2.4 +CPP_VERSION=0.2.5 +CPP_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.2.5 PYTHON_VERSION=0.2.5 PYTHON_CLI_IMAGE=faasm.azurecr.io/cpython:0.2.5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 289498d75..219979a50 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,7 +68,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cpp-sysroot:0.2.4 + image: faasm.azurecr.io/cpp-sysroot:0.2.5 credentials: username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} From 94661aa5eb2989a576e51d7fbc00c924f1dd8cae Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 31 Mar 2023 08:08:25 +0000 Subject: [PATCH 35/49] tests: fix disassemble test, which had changed after including the memory layout protection call --- tests/test/wasm/test_wasm.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test/wasm/test_wasm.cpp b/tests/test/wasm/test_wasm.cpp index 5dc4e6abc..27b686966 100644 --- a/tests/test/wasm/test_wasm.cpp +++ b/tests/test/wasm/test_wasm.cpp @@ -152,9 +152,9 @@ TEST_CASE_METHOD(SimpleWasmTestFixture, "Test disassemble module", "[wasm]") // Check a few known definitions REQUIRE(disasMap["functionDef0"] == "__wasm_call_ctors"); REQUIRE(disasMap["functionDef1"] == "_start"); - REQUIRE(disasMap["functionDef2"] == "main"); - REQUIRE(disasMap["functionDef3"] == "faasmGetInputSize"); - REQUIRE(disasMap["functionDef4"] == "faasmGetInput"); + REQUIRE(disasMap["functionDef2"] == "__faasm_memory_layout_protection()"); + REQUIRE(disasMap["functionDef3"] == "main"); + REQUIRE(disasMap["functionDef4"] == "faasmGetInputSize"); // Check a couple of imports REQUIRE(disasMap["functionImport0"] == "__faasm_read_input"); From dc75e2b8b4b76c8eb07e75a7c966cda1b791339e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 31 Mar 2023 08:51:10 +0000 Subject: [PATCH 36/49] cmake: clean-up wamr's diff --- cmake/ExternalProjects.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index fa666adf5..8fcba2afd 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -111,7 +111,7 @@ FetchContent_Declare(wavm_ext FetchContent_Declare(wamr_ext GIT_REPOSITORY "https://github.com/faasm/wasm-micro-runtime" - GIT_TAG "2725028a8673dc0a2d2fa1ba46e5b52ba60b3d74" + GIT_TAG "a4bc82f2e6af657c5a1b937aaf66da508bd4582a" ) # WAMR and WAVM both link to LLVM From c20349627ae544d983a49df00e0929831e37fbcc Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 31 Mar 2023 08:52:44 +0000 Subject: [PATCH 37/49] k8s: manually bump the k8s-wamr versions after rebase --- deploy/k8s-wamr/upload.yml | 2 +- deploy/k8s-wamr/worker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/k8s-wamr/upload.yml b/deploy/k8s-wamr/upload.yml index 8b280dd9f..30f8a72b0 100644 --- a/deploy/k8s-wamr/upload.yml +++ b/deploy/k8s-wamr/upload.yml @@ -11,7 +11,7 @@ metadata: spec: containers: - name: upload - image: faasm.azurecr.io/upload:0.9.5 + image: faasm.azurecr.io/upload:0.9.6 ports: - containerPort: 8002 - containerPort: 5000 diff --git a/deploy/k8s-wamr/worker.yml b/deploy/k8s-wamr/worker.yml index 5df05f6e5..b51742b2e 100644 --- a/deploy/k8s-wamr/worker.yml +++ b/deploy/k8s-wamr/worker.yml @@ -31,7 +31,7 @@ spec: weight: 100 containers: - - image: faasm.azurecr.io/worker:0.9.5 + - image: faasm.azurecr.io/worker:0.9.6 name: faasm-worker ports: - containerPort: 8080 From 83aa7c49acaf00e6176dfefdd9ccafef3747003c Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 09:49:18 +0000 Subject: [PATCH 38/49] wamr: move chaining into (old) funcs and refactor funcs to faasm --- include/wamr/native.h | 2 -- src/wamr/CMakeLists.txt | 3 +-- src/wamr/chaining.cpp | 26 -------------------------- src/wamr/{funcs.cpp => faasm.cpp} | 29 +++++++++++++++++++++++++++-- src/wamr/native.cpp | 1 - 5 files changed, 28 insertions(+), 33 deletions(-) delete mode 100644 src/wamr/chaining.cpp rename src/wamr/{funcs.cpp => faasm.cpp} (80%) diff --git a/include/wamr/native.h b/include/wamr/native.h index 202f7b5b3..87f63b732 100644 --- a/include/wamr/native.h +++ b/include/wamr/native.h @@ -45,8 +45,6 @@ namespace wasm { void initialiseWAMRNatives(); -uint32_t getFaasmChainingApi(NativeSymbol** nativeSymbols); - uint32_t getFaasmDynlinkApi(NativeSymbol** nativeSymbols); uint32_t getFaasmEnvApi(NativeSymbol** nativeSymbols); diff --git a/src/wamr/CMakeLists.txt b/src/wamr/CMakeLists.txt index a5f80ffd5..c62ceb43e 100644 --- a/src/wamr/CMakeLists.txt +++ b/src/wamr/CMakeLists.txt @@ -75,12 +75,11 @@ llvm_map_components_to_libnames( # Link everything together faasm_private_lib(wamrmodule WAMRWasmModule.cpp - chaining.cpp codegen.cpp dynlink.cpp env.cpp + faasm.cpp filesystem.cpp - funcs.cpp memory.cpp mpi.cpp native.cpp diff --git a/src/wamr/chaining.cpp b/src/wamr/chaining.cpp deleted file mode 100644 index 6c83b11c4..000000000 --- a/src/wamr/chaining.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include -#include - -#include -#include - -namespace wasm { -static int32_t __faasm_chain_name_wrapper(wasm_exec_env_t execEnv, - const char* name, - const uint8_t* input, - uint32_t inputSize) -{ - std::vector _input(input, input + inputSize); - SPDLOG_DEBUG("S - chain_name - {}", std::string(name)); - return wasm::makeChainedCall(std::string(name), 0, nullptr, _input); -} - -static NativeSymbol ns[] = { REG_NATIVE_FUNC(__faasm_chain_name, "($$i)i") }; - -uint32_t getFaasmChainingApi(NativeSymbol** nativeSymbols) -{ - *nativeSymbols = ns; - return sizeof(ns) / sizeof(NativeSymbol); -} -} diff --git a/src/wamr/funcs.cpp b/src/wamr/faasm.cpp similarity index 80% rename from src/wamr/funcs.cpp rename to src/wamr/faasm.cpp index 3a2e2aec5..12a6406c4 100644 --- a/src/wamr/funcs.cpp +++ b/src/wamr/faasm.cpp @@ -3,11 +3,12 @@ #include #include #include -#include +#include +#include #include +#include #include #include -#include #include @@ -43,6 +44,19 @@ static int32_t __faasm_await_call_wrapper(wasm_exec_env_t exec_env, return result; } +/** + * Chain a function by name + */ +static int32_t __faasm_chain_name_wrapper(wasm_exec_env_t execEnv, + const char* name, + const uint8_t* input, + uint32_t inputSize) +{ + std::vector _input(input, input + inputSize); + SPDLOG_DEBUG("S - chain_name - {}", std::string(name)); + return wasm::makeChainedCall(std::string(name), 0, nullptr, _input); +} + /** * Chain a function by function pointer */ @@ -58,6 +72,15 @@ static int32_t __faasm_chain_ptr_wrapper(wasm_exec_env_t exec_env, return makeChainedCall(call.function(), wasmFuncPtr, nullptr, inputData); } +static void __faasm_migrate_point_wrapper(wasm_exec_env_t execEnv, + int32_t wasmFuncPtr, + std::string funcArg) +{ + SPDLOG_DEBUG("S - faasm_migrate_point {} {}", wasmFuncPtr, funcArg); + + wasm::doMigrationPoint(wasmFuncPtr, funcArg); +} + static void __faasm_pull_state_wrapper(wasm_exec_env_t execEnv, int32_t* keyPtr, int32_t stateLen) @@ -114,7 +137,9 @@ static void __faasm_write_output_wrapper(wasm_exec_env_t exec_env, static NativeSymbol ns[] = { REG_NATIVE_FUNC(__faasm_await_call, "(i)i"), + REG_NATIVE_FUNC(__faasm_chain_name, "($$i)i"), REG_NATIVE_FUNC(__faasm_chain_ptr, "(i$i)i"), + REG_NATIVE_FUNC(__faasm_migrate_point, "(i$)"), REG_NATIVE_FUNC(__faasm_pull_state, "(*i)"), REG_NATIVE_FUNC(__faasm_push_state, "(*)"), REG_NATIVE_FUNC(__faasm_read_input, "($i)i"), diff --git a/src/wamr/native.cpp b/src/wamr/native.cpp index 5bcb8fb78..5790b4d64 100644 --- a/src/wamr/native.cpp +++ b/src/wamr/native.cpp @@ -20,7 +20,6 @@ void doWasiSymbolRegistration(uint32_t (*f)(NativeSymbol** ns)) void initialiseWAMRNatives() { // Register native symbols - doSymbolRegistration(getFaasmChainingApi); doSymbolRegistration(getFaasmDynlinkApi); doSymbolRegistration(getFaasmEnvApi); doSymbolRegistration(getFaasmFilesystemApi); From 5a11817fa31d92438757d4ed095e955d7628db2c Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 09:49:53 +0000 Subject: [PATCH 39/49] migration: move to wasm --- include/wasm/migration.h | 6 +++ src/wasm/CMakeLists.txt | 1 + src/wasm/WasmModule.cpp | 2 +- src/wasm/migration.cpp | 102 +++++++++++++++++++++++++++++++++++++++ src/wavm/faasm.cpp | 21 ++++---- 5 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 include/wasm/migration.h create mode 100644 src/wasm/migration.cpp diff --git a/include/wasm/migration.h b/include/wasm/migration.h new file mode 100644 index 000000000..3d65ead20 --- /dev/null +++ b/include/wasm/migration.h @@ -0,0 +1,6 @@ +#pragma once + +namespace wasm { +void doMigrationPoint(int32_t entrypointFuncWasmOffset, + const std::string& entrypointFuncArg); +} diff --git a/src/wasm/CMakeLists.txt b/src/wasm/CMakeLists.txt index 4c4e47fdf..f6815f3c9 100644 --- a/src/wasm/CMakeLists.txt +++ b/src/wasm/CMakeLists.txt @@ -3,6 +3,7 @@ faasm_private_lib(wasm WasmExecutionContext.cpp WasmModule.cpp chaining_util.cpp + migration.cpp ) # Shared variables with the cross-compilation toolchain diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index d3773f124..04036f77d 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -829,7 +829,7 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) } else { // TODO - this log statement should be a warning, but for some reason // we are running into it a lot, so I temporarily disable it - SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", + SPDLOG_TRACE("MEM - unable to reclaim unmapped memory {} at {}", pageAligned, offset); } diff --git a/src/wasm/migration.cpp b/src/wasm/migration.cpp new file mode 100644 index 000000000..469265774 --- /dev/null +++ b/src/wasm/migration.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include + +namespace wasm { +void doMigrationPoint(int32_t entrypointFuncWasmOffset, + const std::string& entrypointFuncArg) { + auto* call = &faabric::scheduler::ExecutorContext::get()->getMsg(); + auto& sch = faabric::scheduler::getScheduler(); + + // Detect if there is a pending migration for the current app + auto pendingMigrations = sch.getPendingAppMigrations(call->appid()); + bool appMustMigrate = pendingMigrations != nullptr; + + // Detect if this particular function needs to be migrated or not + bool funcMustMigrate = false; + std::string hostToMigrateTo = "otherHost"; + if (appMustMigrate) { + for (int i = 0; i < pendingMigrations->migrations_size(); i++) { + auto m = pendingMigrations->mutable_migrations()->at(i); + if (m.msg().id() == call->id()) { + funcMustMigrate = true; + hostToMigrateTo = m.dsthost(); + break; + } + } + } + + // Regardless if we have to individually migrate or not, we need to prepare + // for the app migration + if (appMustMigrate && call->ismpi()) { + auto& mpiWorld = faabric::scheduler::getMpiWorldRegistry().getWorld( + call->mpiworldid()); + mpiWorld.prepareMigration(call->mpirank(), pendingMigrations); + } + + // Do actual migration + if (funcMustMigrate) { + std::vector inputData(entrypointFuncArg.begin(), entrypointFuncArg.end()); + + std::string user = call->user(); + + std::shared_ptr req = + faabric::util::batchExecFactory(call->user(), call->function(), 1); + req->set_type(faabric::BatchExecuteRequest::MIGRATION); + + faabric::Message& msg = req->mutable_messages()->at(0); + msg.set_inputdata(inputData.data(), inputData.size()); + msg.set_funcptr(entrypointFuncWasmOffset); + + // Take snapshot of function and send it to the host we are migrating + // to. Note that the scheduler only pushes snapshots as part of function + // chaining from the master host of the app, and + // we are most likely migrating from a non-master host. Thus, we must + // take and push the snapshot manually. + auto* exec = faabric::scheduler::ExecutorContext::get()->getExecutor(); + auto snap = + std::make_shared(exec->getMemoryView()); + std::string snapKey = "migration_" + std::to_string(msg.id()); + auto& reg = faabric::snapshot::getSnapshotRegistry(); + reg.registerSnapshot(snapKey, snap); + sch.getSnapshotClient(hostToMigrateTo)->pushSnapshot(snapKey, snap); + msg.set_snapshotkey(snapKey); + + // Propagate the app ID and set the _same_ message ID + msg.set_appid(call->appid()); + msg.set_groupid(call->groupid()); + msg.set_groupidx(call->groupidx()); + + // If message is MPI, propagate the necessary MPI bits + if (call->ismpi()) { + msg.set_ismpi(true); + msg.set_mpiworldid(call->mpiworldid()); + msg.set_mpiworldsize(call->mpiworldsize()); + msg.set_mpirank(call->mpirank()); + } + + if (call->recordexecgraph()) { + msg.set_recordexecgraph(true); + } + + SPDLOG_INFO("Migrating {}/{} {} to {}", + msg.user(), + msg.function(), + call->id(), + hostToMigrateTo); + + // Build decision and send + faabric::util::SchedulingDecision decision(msg.appid(), msg.groupid()); + decision.addMessage(hostToMigrateTo, msg); + sch.callFunctions(req, decision); + + if (call->recordexecgraph()) { + sch.logChainedFunction(call->id(), msg.id()); + } + + // Throw an exception to be caught by the executor and terminate + throw faabric::util::FunctionMigratedException("Migrating MPI rank"); + } +} +} diff --git a/src/wavm/faasm.cpp b/src/wavm/faasm.cpp index 79c483157..785fb9364 100644 --- a/src/wavm/faasm.cpp +++ b/src/wavm/faasm.cpp @@ -1,13 +1,6 @@ #include "syscalls.h" -#include -#include -#include - -#include -#include -#include - +#include #include #include #include @@ -21,8 +14,15 @@ #include #include #include +#include +#include +#include +#include + +#include +#include +#include -#include using namespace WAVM; using namespace faabric::transport; @@ -716,6 +716,8 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, SPDLOG_DEBUG( "S - faasm_migrate_point {} {}", entrypointFuncPtr, entrypointFuncArg); + wasm::doMigrationPoint(entrypointFuncPtr, std::to_string(entrypointFuncArg)); +/* auto* call = &ExecutorContext::get()->getMsg(); auto& sch = faabric::scheduler::getScheduler(); @@ -809,6 +811,7 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, // Throw an exception to be caught by the executor and terminate throw faabric::util::FunctionMigratedException("Migrating MPI rank"); } +*/ } // ------------------------------------ From 974c53b859a49603e8d7ce179917efb0c9c2db46 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 14:12:27 +0000 Subject: [PATCH 40/49] wasm: move statement back to warn logging after memory leak has been fixed --- src/wasm/WasmModule.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index 04036f77d..3c018be57 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -827,9 +827,7 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) SPDLOG_TRACE("MEM - munmapping top of memory by {}", pageAligned); shrinkMemory(pageAligned); } else { - // TODO - this log statement should be a warning, but for some reason - // we are running into it a lot, so I temporarily disable it - SPDLOG_TRACE("MEM - unable to reclaim unmapped memory {} at {}", + SPDLOG_WARN("MEM - unable to reclaim unmapped memory {} at {}", pageAligned, offset); } From 92ad6d8462d560481cf75144852eeeb61123ecde Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 15:54:24 +0000 Subject: [PATCH 41/49] cpp: bump after merge --- clients/cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/cpp b/clients/cpp index 5920779e7..042475a77 160000 --- a/clients/cpp +++ b/clients/cpp @@ -1 +1 @@ -Subproject commit 5920779e7e70331e04463e07a4bb3a5591004174 +Subproject commit 042475a7721d03370e69fe451bb7cc996f32033a From fa74c2ac9fc69813c085c077a042710475bd45e9 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 15:55:31 +0000 Subject: [PATCH 42/49] wasm: remove todo after memory leak issue has been fixed --- src/enclave/inside/EnclaveWasmModule.cpp | 1 - src/wamr/WAMRWasmModule.cpp | 2 +- src/wasm/WasmModule.cpp | 8 +++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/enclave/inside/EnclaveWasmModule.cpp b/src/enclave/inside/EnclaveWasmModule.cpp index 86c3d55a2..46f4286b6 100644 --- a/src/enclave/inside/EnclaveWasmModule.cpp +++ b/src/enclave/inside/EnclaveWasmModule.cpp @@ -80,7 +80,6 @@ bool EnclaveWasmModule::callFunction(uint32_t argcIn, char** argvIn) bool success = wasm_runtime_call_wasm(execEnv, func, 0, argv.data()); uint32_t returnValue = argv[0]; - if (success) { ocallLogDebug("Success calling WASM function"); } else { diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index a653511ce..e949376cd 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -329,7 +329,7 @@ uint8_t* WAMRWasmModule::getMemoryBase() size_t WAMRWasmModule::getMaxMemoryPages() { - auto aotModule = reinterpret_cast(moduleInstance); + auto* aotModule = reinterpret_cast(moduleInstance); AOTMemoryInstance* aotMem = ((AOTMemoryInstance**)aotModule->memories)[0]; return aotMem->max_page_count; } diff --git a/src/wasm/WasmModule.cpp b/src/wasm/WasmModule.cpp index d3773f124..71daafb00 100644 --- a/src/wasm/WasmModule.cpp +++ b/src/wasm/WasmModule.cpp @@ -827,11 +827,9 @@ void WasmModule::unmapMemory(uint32_t offset, size_t nBytes) SPDLOG_TRACE("MEM - munmapping top of memory by {}", pageAligned); shrinkMemory(pageAligned); } else { - // TODO - this log statement should be a warning, but for some reason - // we are running into it a lot, so I temporarily disable it - SPDLOG_DEBUG("MEM - unable to reclaim unmapped memory {} at {}", - pageAligned, - offset); + SPDLOG_WARN("MEM - unable to reclaim unmapped memory {} at {}", + pageAligned, + offset); } } From b2a3c3f4cf57d4ad15a7cd46d8b5adc81b5a2fc7 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 15:57:46 +0000 Subject: [PATCH 43/49] cpp: bump client after merge --- clients/cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/cpp b/clients/cpp index 5920779e7..042475a77 160000 --- a/clients/cpp +++ b/clients/cpp @@ -1 +1 @@ -Subproject commit 5920779e7e70331e04463e07a4bb3a5591004174 +Subproject commit 042475a7721d03370e69fe451bb7cc996f32033a From e9660e44d8e9e4ac9c926c496ec61dbb57b8aa74 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 16:00:02 +0000 Subject: [PATCH 44/49] wamr: fix warning --- src/wamr/WAMRWasmModule.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wamr/WAMRWasmModule.cpp b/src/wamr/WAMRWasmModule.cpp index a653511ce..e949376cd 100644 --- a/src/wamr/WAMRWasmModule.cpp +++ b/src/wamr/WAMRWasmModule.cpp @@ -329,7 +329,7 @@ uint8_t* WAMRWasmModule::getMemoryBase() size_t WAMRWasmModule::getMaxMemoryPages() { - auto aotModule = reinterpret_cast(moduleInstance); + auto* aotModule = reinterpret_cast(moduleInstance); AOTMemoryInstance* aotMem = ((AOTMemoryInstance**)aotModule->memories)[0]; return aotMem->max_page_count; } From 9e3725b2cb50f9f81ef8e206339d39cccde57d85 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 16:45:48 +0000 Subject: [PATCH 45/49] nits: run clang-format after merge --- src/wamr/faasm.cpp | 10 +-- src/wasm/migration.cpp | 6 +- src/wavm/faasm.cpp | 181 +++++++++++++++++++++-------------------- 3 files changed, 102 insertions(+), 95 deletions(-) diff --git a/src/wamr/faasm.cpp b/src/wamr/faasm.cpp index 12a6406c4..0ac895624 100644 --- a/src/wamr/faasm.cpp +++ b/src/wamr/faasm.cpp @@ -3,12 +3,12 @@ #include #include #include -#include -#include -#include #include +#include #include #include +#include +#include #include @@ -73,8 +73,8 @@ static int32_t __faasm_chain_ptr_wrapper(wasm_exec_env_t exec_env, } static void __faasm_migrate_point_wrapper(wasm_exec_env_t execEnv, - int32_t wasmFuncPtr, - std::string funcArg) + int32_t wasmFuncPtr, + std::string funcArg) { SPDLOG_DEBUG("S - faasm_migrate_point {} {}", wasmFuncPtr, funcArg); diff --git a/src/wasm/migration.cpp b/src/wasm/migration.cpp index 469265774..3fcb20c10 100644 --- a/src/wasm/migration.cpp +++ b/src/wasm/migration.cpp @@ -5,7 +5,8 @@ namespace wasm { void doMigrationPoint(int32_t entrypointFuncWasmOffset, - const std::string& entrypointFuncArg) { + const std::string& entrypointFuncArg) +{ auto* call = &faabric::scheduler::ExecutorContext::get()->getMsg(); auto& sch = faabric::scheduler::getScheduler(); @@ -37,7 +38,8 @@ void doMigrationPoint(int32_t entrypointFuncWasmOffset, // Do actual migration if (funcMustMigrate) { - std::vector inputData(entrypointFuncArg.begin(), entrypointFuncArg.end()); + std::vector inputData(entrypointFuncArg.begin(), + entrypointFuncArg.end()); std::string user = call->user(); diff --git a/src/wavm/faasm.cpp b/src/wavm/faasm.cpp index 785fb9364..5d8bc7324 100644 --- a/src/wavm/faasm.cpp +++ b/src/wavm/faasm.cpp @@ -23,7 +23,6 @@ #include #include - using namespace WAVM; using namespace faabric::transport; using namespace faabric::scheduler; @@ -716,102 +715,108 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, SPDLOG_DEBUG( "S - faasm_migrate_point {} {}", entrypointFuncPtr, entrypointFuncArg); - wasm::doMigrationPoint(entrypointFuncPtr, std::to_string(entrypointFuncArg)); -/* - auto* call = &ExecutorContext::get()->getMsg(); - auto& sch = faabric::scheduler::getScheduler(); - - // Detect if there is a pending migration for the current app - auto pendingMigrations = sch.getPendingAppMigrations(call->appid()); - bool appMustMigrate = pendingMigrations != nullptr; - - // Detect if this particular function needs to be migrated or not - bool funcMustMigrate = false; - std::string hostToMigrateTo = "otherHost"; - if (appMustMigrate) { - for (int i = 0; i < pendingMigrations->migrations_size(); i++) { - auto m = pendingMigrations->mutable_migrations()->at(i); - if (m.msg().id() == call->id()) { - funcMustMigrate = true; - hostToMigrateTo = m.dsthost(); - break; + wasm::doMigrationPoint(entrypointFuncPtr, + std::to_string(entrypointFuncArg)); + /* + auto* call = &ExecutorContext::get()->getMsg(); + auto& sch = faabric::scheduler::getScheduler(); + + // Detect if there is a pending migration for the current app + auto pendingMigrations = sch.getPendingAppMigrations(call->appid()); + bool appMustMigrate = pendingMigrations != nullptr; + + // Detect if this particular function needs to be migrated or not + bool funcMustMigrate = false; + std::string hostToMigrateTo = "otherHost"; + if (appMustMigrate) { + for (int i = 0; i < pendingMigrations->migrations_size(); i++) { + auto m = pendingMigrations->mutable_migrations()->at(i); + if (m.msg().id() == call->id()) { + funcMustMigrate = true; + hostToMigrateTo = m.dsthost(); + break; + } } } - } - - // Regardless if we have to individually migrate or not, we need to prepare - // for the app migration - if (appMustMigrate && call->ismpi()) { - auto& mpiWorld = faabric::scheduler::getMpiWorldRegistry().getWorld( - call->mpiworldid()); - mpiWorld.prepareMigration(call->mpirank(), pendingMigrations); - } - // Do actual migration - if (funcMustMigrate) { - std::string argStr = std::to_string(entrypointFuncArg); - std::vector inputData(argStr.begin(), argStr.end()); - - std::string user = call->user(); - - std::shared_ptr req = - faabric::util::batchExecFactory(call->user(), call->function(), 1); - req->set_type(faabric::BatchExecuteRequest::MIGRATION); - - faabric::Message& msg = req->mutable_messages()->at(0); - msg.set_inputdata(inputData.data(), inputData.size()); - msg.set_funcptr(entrypointFuncPtr); - - // Take snapshot of function and send it to the host we are migrating - // to. Note that the scheduler only pushes snapshots as part of function - // chaining from the master host of the app, and - // we are most likely migrating from a non-master host. Thus, we must - // take and push the snapshot manually. - auto* exec = faabric::scheduler::ExecutorContext::get()->getExecutor(); - auto snap = - std::make_shared(exec->getMemoryView()); - std::string snapKey = "migration_" + std::to_string(msg.id()); - auto& reg = faabric::snapshot::getSnapshotRegistry(); - reg.registerSnapshot(snapKey, snap); - sch.getSnapshotClient(hostToMigrateTo)->pushSnapshot(snapKey, snap); - msg.set_snapshotkey(snapKey); - - // Propagate the app ID and set the _same_ message ID - msg.set_appid(call->appid()); - msg.set_groupid(call->groupid()); - msg.set_groupidx(call->groupidx()); - - // If message is MPI, propagate the necessary MPI bits - if (call->ismpi()) { - msg.set_ismpi(true); - msg.set_mpiworldid(call->mpiworldid()); - msg.set_mpiworldsize(call->mpiworldsize()); - msg.set_mpirank(call->mpirank()); + // Regardless if we have to individually migrate or not, we need to + prepare + // for the app migration + if (appMustMigrate && call->ismpi()) { + auto& mpiWorld = faabric::scheduler::getMpiWorldRegistry().getWorld( + call->mpiworldid()); + mpiWorld.prepareMigration(call->mpirank(), pendingMigrations); } - if (call->recordexecgraph()) { - msg.set_recordexecgraph(true); - } + // Do actual migration + if (funcMustMigrate) { + std::string argStr = std::to_string(entrypointFuncArg); + std::vector inputData(argStr.begin(), argStr.end()); + + std::string user = call->user(); + + std::shared_ptr req = + faabric::util::batchExecFactory(call->user(), call->function(), + 1); req->set_type(faabric::BatchExecuteRequest::MIGRATION); + + faabric::Message& msg = req->mutable_messages()->at(0); + msg.set_inputdata(inputData.data(), inputData.size()); + msg.set_funcptr(entrypointFuncPtr); + + // Take snapshot of function and send it to the host we are + migrating + // to. Note that the scheduler only pushes snapshots as part of + function + // chaining from the master host of the app, and + // we are most likely migrating from a non-master host. Thus, we + must + // take and push the snapshot manually. + auto* exec = + faabric::scheduler::ExecutorContext::get()->getExecutor(); auto snap = + std::make_shared(exec->getMemoryView()); + std::string snapKey = "migration_" + std::to_string(msg.id()); + auto& reg = faabric::snapshot::getSnapshotRegistry(); + reg.registerSnapshot(snapKey, snap); + sch.getSnapshotClient(hostToMigrateTo)->pushSnapshot(snapKey, snap); + msg.set_snapshotkey(snapKey); + + // Propagate the app ID and set the _same_ message ID + msg.set_appid(call->appid()); + msg.set_groupid(call->groupid()); + msg.set_groupidx(call->groupidx()); + + // If message is MPI, propagate the necessary MPI bits + if (call->ismpi()) { + msg.set_ismpi(true); + msg.set_mpiworldid(call->mpiworldid()); + msg.set_mpiworldsize(call->mpiworldsize()); + msg.set_mpirank(call->mpirank()); + } - SPDLOG_INFO("Migrating {}/{} {} to {}", - msg.user(), - msg.function(), - call->id(), - hostToMigrateTo); + if (call->recordexecgraph()) { + msg.set_recordexecgraph(true); + } - // Build decision and send - faabric::util::SchedulingDecision decision(msg.appid(), msg.groupid()); - decision.addMessage(hostToMigrateTo, msg); - sch.callFunctions(req, decision); + SPDLOG_INFO("Migrating {}/{} {} to {}", + msg.user(), + msg.function(), + call->id(), + hostToMigrateTo); - if (call->recordexecgraph()) { - sch.logChainedFunction(call->id(), msg.id()); - } + // Build decision and send + faabric::util::SchedulingDecision decision(msg.appid(), + msg.groupid()); decision.addMessage(hostToMigrateTo, msg); + sch.callFunctions(req, decision); - // Throw an exception to be caught by the executor and terminate - throw faabric::util::FunctionMigratedException("Migrating MPI rank"); - } -*/ + if (call->recordexecgraph()) { + sch.logChainedFunction(call->id(), msg.id()); + } + + // Throw an exception to be caught by the executor and terminate + throw faabric::util::FunctionMigratedException("Migrating MPI + rank"); + } + */ } // ------------------------------------ From be3d0819bf99761d6dc474520576b2c31dee17d8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 18:06:30 +0000 Subject: [PATCH 46/49] tests: fix disas test after mmap/munmap are not imported --- tests/test/wasm/test_wasm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test/wasm/test_wasm.cpp b/tests/test/wasm/test_wasm.cpp index 27b686966..4d7e37ef6 100644 --- a/tests/test/wasm/test_wasm.cpp +++ b/tests/test/wasm/test_wasm.cpp @@ -159,7 +159,7 @@ TEST_CASE_METHOD(SimpleWasmTestFixture, "Test disassemble module", "[wasm]") // Check a couple of imports REQUIRE(disasMap["functionImport0"] == "__faasm_read_input"); REQUIRE(disasMap["functionImport1"] == "__faasm_write_output"); - REQUIRE(disasMap["functionImport2"] == "mmap"); - REQUIRE(disasMap["functionImport3"] == "munmap"); + REQUIRE(disasMap["functionImport2"] == "__imported_wasi_snapshot_preview1_args_get"); + REQUIRE(disasMap["functionImport3"] == "__imported_wasi_snapshot_preview1_args_sizes_get"); } } From 692ba7ed4955e09a6bffaf3938f6b79ec26a101a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 18:06:49 +0000 Subject: [PATCH 47/49] wavm: remove commented out code --- src/wavm/faasm.cpp | 100 --------------------------------------------- 1 file changed, 100 deletions(-) diff --git a/src/wavm/faasm.cpp b/src/wavm/faasm.cpp index 5d8bc7324..acd39c09e 100644 --- a/src/wavm/faasm.cpp +++ b/src/wavm/faasm.cpp @@ -717,106 +717,6 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, wasm::doMigrationPoint(entrypointFuncPtr, std::to_string(entrypointFuncArg)); - /* - auto* call = &ExecutorContext::get()->getMsg(); - auto& sch = faabric::scheduler::getScheduler(); - - // Detect if there is a pending migration for the current app - auto pendingMigrations = sch.getPendingAppMigrations(call->appid()); - bool appMustMigrate = pendingMigrations != nullptr; - - // Detect if this particular function needs to be migrated or not - bool funcMustMigrate = false; - std::string hostToMigrateTo = "otherHost"; - if (appMustMigrate) { - for (int i = 0; i < pendingMigrations->migrations_size(); i++) { - auto m = pendingMigrations->mutable_migrations()->at(i); - if (m.msg().id() == call->id()) { - funcMustMigrate = true; - hostToMigrateTo = m.dsthost(); - break; - } - } - } - - // Regardless if we have to individually migrate or not, we need to - prepare - // for the app migration - if (appMustMigrate && call->ismpi()) { - auto& mpiWorld = faabric::scheduler::getMpiWorldRegistry().getWorld( - call->mpiworldid()); - mpiWorld.prepareMigration(call->mpirank(), pendingMigrations); - } - - // Do actual migration - if (funcMustMigrate) { - std::string argStr = std::to_string(entrypointFuncArg); - std::vector inputData(argStr.begin(), argStr.end()); - - std::string user = call->user(); - - std::shared_ptr req = - faabric::util::batchExecFactory(call->user(), call->function(), - 1); req->set_type(faabric::BatchExecuteRequest::MIGRATION); - - faabric::Message& msg = req->mutable_messages()->at(0); - msg.set_inputdata(inputData.data(), inputData.size()); - msg.set_funcptr(entrypointFuncPtr); - - // Take snapshot of function and send it to the host we are - migrating - // to. Note that the scheduler only pushes snapshots as part of - function - // chaining from the master host of the app, and - // we are most likely migrating from a non-master host. Thus, we - must - // take and push the snapshot manually. - auto* exec = - faabric::scheduler::ExecutorContext::get()->getExecutor(); auto snap = - std::make_shared(exec->getMemoryView()); - std::string snapKey = "migration_" + std::to_string(msg.id()); - auto& reg = faabric::snapshot::getSnapshotRegistry(); - reg.registerSnapshot(snapKey, snap); - sch.getSnapshotClient(hostToMigrateTo)->pushSnapshot(snapKey, snap); - msg.set_snapshotkey(snapKey); - - // Propagate the app ID and set the _same_ message ID - msg.set_appid(call->appid()); - msg.set_groupid(call->groupid()); - msg.set_groupidx(call->groupidx()); - - // If message is MPI, propagate the necessary MPI bits - if (call->ismpi()) { - msg.set_ismpi(true); - msg.set_mpiworldid(call->mpiworldid()); - msg.set_mpiworldsize(call->mpiworldsize()); - msg.set_mpirank(call->mpirank()); - } - - if (call->recordexecgraph()) { - msg.set_recordexecgraph(true); - } - - SPDLOG_INFO("Migrating {}/{} {} to {}", - msg.user(), - msg.function(), - call->id(), - hostToMigrateTo); - - // Build decision and send - faabric::util::SchedulingDecision decision(msg.appid(), - msg.groupid()); decision.addMessage(hostToMigrateTo, msg); - sch.callFunctions(req, decision); - - if (call->recordexecgraph()) { - sch.logChainedFunction(call->id(), msg.id()); - } - - // Throw an exception to be caught by the executor and terminate - throw faabric::util::FunctionMigratedException("Migrating MPI - rank"); - } - */ } // ------------------------------------ From 0bb6b1ac9788c135099add351a3eedb051ac8a34 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 3 Apr 2023 18:27:23 +0000 Subject: [PATCH 48/49] nit: run clang-format --- tests/test/wasm/test_wasm.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test/wasm/test_wasm.cpp b/tests/test/wasm/test_wasm.cpp index 4d7e37ef6..f2e0f5325 100644 --- a/tests/test/wasm/test_wasm.cpp +++ b/tests/test/wasm/test_wasm.cpp @@ -159,7 +159,9 @@ TEST_CASE_METHOD(SimpleWasmTestFixture, "Test disassemble module", "[wasm]") // Check a couple of imports REQUIRE(disasMap["functionImport0"] == "__faasm_read_input"); REQUIRE(disasMap["functionImport1"] == "__faasm_write_output"); - REQUIRE(disasMap["functionImport2"] == "__imported_wasi_snapshot_preview1_args_get"); - REQUIRE(disasMap["functionImport3"] == "__imported_wasi_snapshot_preview1_args_sizes_get"); + REQUIRE(disasMap["functionImport2"] == + "__imported_wasi_snapshot_preview1_args_get"); + REQUIRE(disasMap["functionImport3"] == + "__imported_wasi_snapshot_preview1_args_sizes_get"); } } From 669107551ab21119d723d0cec040726ab21454da Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 4 Apr 2023 10:38:32 +0000 Subject: [PATCH 49/49] wamr: squash to one-commit-diff with upstream main --- cmake/ExternalProjects.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 8fcba2afd..6a1414281 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -111,7 +111,7 @@ FetchContent_Declare(wavm_ext FetchContent_Declare(wamr_ext GIT_REPOSITORY "https://github.com/faasm/wasm-micro-runtime" - GIT_TAG "a4bc82f2e6af657c5a1b937aaf66da508bd4582a" + GIT_TAG "5e9dc3c7eb33167389d99b7e5851dc55b5911d33" ) # WAMR and WAVM both link to LLVM