feat: support IC low Wasm memory hook

Changelog.md

@@ -4,7 +4,9 @@
 
 * motoko (`moc`)
 
-  * Breaking change (minor):
+  * Support low Wasm memory hook: `system func lowmemory() : async* () { ... }` (#4849).
+
+  * Breaking change (minor) (#4854):
 
     * For enhanced orthogonal persistence: The Wasm persistence modes used internally for canister upgrades have been changed to lower case names,
       `keep` and `replace` and instead of `Keep` and `Replace`:
@@ -16,7 +18,7 @@
 
 * motoko (`moc`)
 
-  * Breaking change (minor):
+  * Breaking change (minor) (#4786):
 
     * Add new keyword `transient` with exactly the same meaning as the old keyword `flexible` (but a more familiar reading).
 
@@ -26,10 +28,8 @@
       `let` or `var` declaration is `stable` (not `flexible` or `transient`).
 
       For example, a stateful counter can now be declared as:
-
       ``` motoko
       persistent actor {
-
         // counts increments since last upgrade
         transient var invocations = 0;
 
@@ -40,10 +40,8 @@
           value += 1;
           invocations += 1;
         }
-
       }
       ```
-
       On upgrade, the transient variable `invocations` will be reset to `0` and `value`, now implicitly `stable`, will retain its current value.
 
       Legacy actors and classes declared without the `persistent` keyword have the same semantics as before.

doc/md/canister-maintenance/memory.md

@@ -0,0 +1,26 @@
+---
+sidebar_position: 5
+---
+
+# Memory diagnostics
+
+## Low memory hook
+
+The IC allows to implement a low memory hook, which is a warning trigger when main memory is becoming scarce.
+
+For this purpose, a Motoko actor or actor class instance can implement the system function `lowmemory()`. This system function is scheduled when canister's free main memory space has fallen below the defined threshold `wasm_memory_threshold`, that is is part of the canister settings. In Motoko, `lowmemory()` implements the `canister_on_low_wasm_memory` hook defined in the IC specification.
+
+Example of using the low memory hook:
+```
+actor {
+    system func lowmemory() : async* () {
+        Debug.print("Low memory!");
+    }
+}
+```
+
+The following properties apply to the low memory hook:
+* The execution of `lowmemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed.
+* Once executed, `lowmemory` is only triggered again when the main memory free space first exceeds and then falls below the threshold.
+* Traps or unhandled errors in `lowmemory` are ignored. Traps only revert the changes done in `lowmemory`.
+* Due to its `async*` return type, the `lowmemory` function may send further messages and `await` results.

nix/sources.json

@@ -21,15 +21,15 @@
         "version": "3.2.25"
     },
     "ic": {
-        "branch": "luc/lower-case-persistence-modes",
+        "branch": "luc/latest-drun",
         "description": "Internet Computer blockchain source: the client/replica software run by nodes",
         "homepage": "",
         "owner": "luc-blaeser",
         "repo": "ic",
-        "rev": "48ba595fb1a6bc828e7e9dae976db42b5526a5b6",
-        "sha256": "0136hxidak4qsk0nhjdy2nmciwb0lprm07vs117xdw3cp4m6py2z",
+        "rev": "afc94db6269ec269ddb1eb5aded38e95fb019958",
+        "sha256": "1clc0183gsk824dg1qp8cqjfp44ywp9yll0a9a1frxgsbwllyk10",
         "type": "tarball",
-        "url": "https://github.com/luc-blaeser/ic/archive/48ba595fb1a6bc828e7e9dae976db42b5526a5b6.tar.gz",
+        "url": "https://github.com/luc-blaeser/ic/archive/afc94db6269ec269ddb1eb5aded38e95fb019958.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "ic-hs": {

src/codegen/compile_classical.ml

@@ -5277,6 +5277,18 @@ module IC = struct
       edesc = nr (FuncExport (nr fi))
     })
 
+  let export_low_memory env =
+    assert (E.mode env = Flags.ICMode || E.mode env = Flags.RefMode);
+    let fi = E.add_fun env "canister_on_low_wasm_memory"
+      (Func.of_body env [] [] (fun env ->
+        G.i (Call (nr (E.built_in env "low_memory_exp"))) ^^
+        GC.collect_garbage env))
+    in
+    E.add_export env (nr {
+      name = Lib.Utf8.decode "canister_on_low_wasm_memory";
+      edesc = nr (FuncExport (nr fi))
+    })
+
   let export_wasi_start env =
     assert (E.mode env = Flags.WASIMode);
     let fi = E.add_fun env "_start" (Func.of_body env [] [] (fun env1 ->
@@ -13014,6 +13026,15 @@ and main_actor as_opt mod_env ds fs up =
        IC.export_inspect env;
     end;
 
+    (* Export low memory hook (but only when required) *)
+    begin match up.low_memory.it with
+     | Ir.PrimE (Ir.TupPrim, []) -> ()
+     | _ ->
+       Func.define_built_in env "low_memory_exp" [] [] (fun env ->
+         compile_exp_as env ae2 SR.unit up.low_memory);
+       IC.export_low_memory env;
+    end;
+
     (* Helper function to build the stable actor wrapper *)
     Func.define_built_in mod_env IC.get_actor_to_persist_function_name [] [I32Type] (fun env ->
       compile_exp_as env ae2 SR.Vanilla build_stable_actor

src/codegen/compile_enhanced.ml

@@ -4972,6 +4972,18 @@ module IC = struct
       edesc = nr (FuncExport (nr fi))
     })
 
+  let export_low_memory env =
+    assert (E.mode env = Flags.ICMode || E.mode env = Flags.RefMode);
+    let fi = E.add_fun env "canister_on_low_wasm_memory"
+      (Func.of_body env [] [] (fun env ->
+        G.i (Call (nr (E.built_in env "low_memory_exp"))) ^^
+        GC.collect_garbage env))
+    in
+    E.add_export env (nr {
+      name = Lib.Utf8.decode "canister_on_low_wasm_memory";
+      edesc = nr (FuncExport (nr fi))
+    })
+
   let initialize_main_actor_function_name = "@initialize_main_actor"
 
   let initialize_main_actor env =
@@ -13106,6 +13118,15 @@ and main_actor as_opt mod_env ds fs up =
        IC.export_inspect env;
     end;
 
+    (* Export low memory hook (but only when required) *)
+    begin match up.low_memory.it with
+    | Ir.PrimE (Ir.TupPrim, []) -> ()
+    | _ ->
+      Func.define_built_in env "low_memory_exp" [] [] (fun env ->
+        compile_exp_as env ae2 SR.unit up.low_memory);
+      IC.export_low_memory env;
+    end;
+
     (* Helper function to build the stable actor wrapper *)
     Func.define_built_in mod_env IC.get_actor_to_persist_function_name [] [I64Type] (fun env ->
       compile_exp_as env ae2 SR.Vanilla build_stable_actor

src/ir_def/arrange_ir.ml

@@ -35,13 +35,14 @@ let rec exp e = match e.it with
   | TryE (e, cs, None) -> "TryE" $$ [exp e] @ List.map case cs
   | TryE (e, cs, Some (i, _)) -> "TryE" $$ [exp e] @ List.map case cs @ Atom ";" :: [id i]
 
-and system { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type} = (* TODO: show meta? *)
+and system { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type} = (* TODO: show meta? *)
   "System" $$ [
       "Pre" $$ [exp preupgrade];
       "Post" $$ [exp postupgrade];
       "Heartbeat" $$ [exp heartbeat];
       "Timer" $$ [exp timer];
       "Inspect" $$ [exp inspect];
+      "LowMemory" $$ [exp low_memory];
       "StableRecord" $$ [exp stable_record];
       "StableType" $$ [typ stable_type]
     ]

src/ir_def/check_ir.ml

@@ -817,7 +817,7 @@ let rec check_exp env (exp:Ir.exp) : unit =
     typ exp_r <: T.(Construct.err_contT unit);
     typ exp_c <: Construct.clean_contT;
   | ActorE (ds, fs,
-      { preupgrade; postupgrade; meta; heartbeat; timer; inspect; stable_record; stable_type }, t0) ->
+      { preupgrade; postupgrade; meta; heartbeat; timer; inspect; low_memory; stable_record; stable_type }, t0) ->
     (* TODO: check meta *)
     let env' = { env with async = None } in
     let scope1 = gather_block_decs env' ds in
@@ -828,12 +828,15 @@ let rec check_exp env (exp:Ir.exp) : unit =
     check_exp env'' heartbeat;
     check_exp env'' timer;
     check_exp env'' inspect;
+    let async_cap = Some Async_cap.top_cap in
+    check_exp { env'' with async = async_cap } low_memory;
     check_exp env'' stable_record;
     typ preupgrade <: T.unit;
     typ postupgrade <: T.unit;
     typ heartbeat <: T.unit;
     typ timer <: T.unit;
     typ inspect <: T.unit;
+    typ low_memory <: T.unit;
     typ stable_record <: stable_type;
     check (T.is_obj t0) "bad annotation (object type expected)";
     let (s0, tfs0) = T.as_obj t0 in
@@ -1160,7 +1163,7 @@ let check_comp_unit env = function
     let env' = adjoin env scope in
     check_decs env' ds
   | ActorU (as_opt, ds, fs,
-      { preupgrade; postupgrade; meta; heartbeat; timer; inspect; stable_type; stable_record }, t0) ->
+      { preupgrade; postupgrade; meta; heartbeat; timer; inspect; low_memory; stable_type; stable_record }, t0) ->
     let check p = check env no_region p in
     let (<:) t1 t2 = check_sub env no_region t1 t2 in
     let env' = match as_opt with
@@ -1179,11 +1182,14 @@ let check_comp_unit env = function
     check_exp env'' timer;
     check_exp env'' inspect;
     check_exp env'' stable_record;
+    let async_cap = Some Async_cap.top_cap in
+    check_exp { env'' with async = async_cap } low_memory;
     typ preupgrade <: T.unit;
     typ postupgrade <: T.unit;
     typ heartbeat <: T.unit;
     typ timer <: T.unit;
     typ inspect <: T.unit;
+    typ low_memory <: T.unit;
     typ stable_record <: stable_type;
     check (T.is_obj t0) "bad annotation (object type expected)";
     let (s0, tfs0) = T.as_obj t0 in

src/ir_def/freevars.ml

@@ -123,12 +123,13 @@ let rec exp e : f = match e.it with
 
 and actor ds fs u = close (decs ds +++ fields fs +++ system u)
 
-and system {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; _} =
+and system {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; _} =
   under_lambda (exp preupgrade) ++
   under_lambda (exp postupgrade) ++
   under_lambda (exp heartbeat) ++
   under_lambda (exp timer) ++
   under_lambda (exp inspect) ++
+  under_lambda (exp low_memory) ++
   under_lambda (exp stable_record)
 
 and exps es : f = unions exp es

src/ir_def/ir.ml

@@ -85,6 +85,7 @@ and system = {
   heartbeat : exp;
   timer : exp; (* TODO: use an option type: (Default of exp | UserDefined of exp) option *)
   inspect : exp;
+  low_memory : exp;
   stable_record: exp;
   stable_type: Type.typ;
 }

src/ir_def/rename.ml

@@ -33,7 +33,7 @@ and exp' rho = function
   | VarE (m, i)         -> VarE (m, id rho i)
   | LitE _ as e         -> e
   | PrimE (p, es)       -> PrimE (prim rho p, List.map (exp rho) es)
-  | ActorE (ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) ->
+  | ActorE (ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) ->
     let ds', rho' = decs rho ds in
     ActorE
       (ds',
@@ -44,6 +44,7 @@ and exp' rho = function
         heartbeat = exp rho' heartbeat;
         timer = exp rho' timer;
         inspect = exp rho' inspect;
+        low_memory = exp rho' low_memory;
         stable_type = stable_type;
         stable_record = exp rho' stable_record;
        },
@@ -200,7 +201,7 @@ let comp_unit rho cu = match cu with
   | LibU (ds, e) ->
     let ds', rho' = decs rho ds
     in LibU (ds', exp rho' e)
-  | ActorU (as_opt, ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type }, t) ->
+  | ActorU (as_opt, ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type }, t) ->
     let as_opt', rho' = match as_opt with
       | None -> None, rho
       | Some as_ ->
@@ -215,6 +216,7 @@ let comp_unit rho cu = match cu with
         heartbeat = exp rho'' heartbeat;
         timer = exp rho'' timer;
         inspect = exp rho'' inspect;
+        low_memory = exp rho'' low_memory;
         stable_record = exp rho'' stable_record;
         stable_type = stable_type;
       }, t)

src/ir_passes/async.ml

@@ -442,14 +442,15 @@ let transform prog =
             | (Returns | Replies), _ -> assert false
           end
       end
-    | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) ->
+    | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) ->
       ActorE (t_decs ds, t_fields fs,
         {meta;
          preupgrade = t_exp preupgrade;
          postupgrade = t_exp postupgrade;
          heartbeat = t_exp heartbeat;
          timer = t_exp timer;
          inspect = t_exp inspect;
+         low_memory = t_exp low_memory;
          stable_record = t_exp stable_record;
          stable_type = t_typ stable_type;
         },
@@ -523,14 +524,15 @@ let transform prog =
   and t_comp_unit = function
     | LibU _ -> raise (Invalid_argument "cannot compile library")
     | ProgU ds -> ProgU (t_decs ds)
-    | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) ->
+    | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) ->
       ActorU (Option.map t_args args_opt, t_decs ds, t_fields fs,
         { meta;
           preupgrade = t_exp preupgrade;
           postupgrade = t_exp postupgrade;
           heartbeat = t_exp heartbeat;
           timer = t_exp timer;
           inspect = t_exp inspect;
+          low_memory = t_exp low_memory;
           stable_record = t_exp stable_record;
           stable_type = t_typ stable_type;
         },

src/ir_passes/await.ml

@@ -181,14 +181,15 @@ and t_exp' context exp =
     assert (not (T.is_shared_func (typ exp)));
     let context' = LabelEnv.singleton Return Label in
     FuncE (x, s, c, typbinds, pat, typs, t_exp context' exp1)
-  | ActorE (ds, ids, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) ->
+  | ActorE (ds, ids, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) ->
     ActorE (t_decs context ds, ids,
       { meta;
         preupgrade = t_exp LabelEnv.empty preupgrade;
         postupgrade = t_exp LabelEnv.empty postupgrade;
         heartbeat = t_ignore_throw LabelEnv.empty heartbeat;
         timer = t_ignore_throw LabelEnv.empty timer;
         inspect = t_exp LabelEnv.empty inspect;
+        low_memory = t_ignore_throw LabelEnv.empty low_memory;
         stable_record = t_exp LabelEnv.empty stable_record;
         stable_type;
       },
@@ -647,14 +648,15 @@ and t_comp_unit context = function
           expD (c_block context' ds (tupE []) (meta (T.unit) (fun v1 -> tupE [])))
         ]
     end
-  | ActorU (as_opt, ds, ids, { meta = m; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) ->
+  | ActorU (as_opt, ds, ids, { meta = m; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) ->
     ActorU (as_opt, t_decs context ds, ids,
       { meta = m;
         preupgrade = t_exp LabelEnv.empty preupgrade;
         postupgrade = t_exp LabelEnv.empty postupgrade;
         heartbeat = t_ignore_throw LabelEnv.empty heartbeat;
         timer = t_timer_throw LabelEnv.empty timer;
         inspect = t_exp LabelEnv.empty inspect;
+        low_memory = t_ignore_throw LabelEnv.empty low_memory;
         stable_record = t_exp LabelEnv.empty stable_record;
         stable_type;
       },

src/ir_passes/const.ml

@@ -164,14 +164,15 @@ let rec exp lvl (env : env) e : Lbool.t =
       surely_false
     | NewObjE _ -> (* mutable objects *)
       surely_false
-    | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, _typ) ->
+    | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, _typ) ->
       (* this may well be “the” top-level actor, so don’t update lvl here *)
       let (env', _) = decs lvl env ds in
       exp_ lvl env' preupgrade;
       exp_ lvl env' postupgrade;
       exp_ lvl env' heartbeat;
       exp_ lvl env' timer;
       exp_ lvl env' inspect;
+      exp_ lvl env' low_memory;
       exp_ lvl env' stable_record;
       surely_false
   in
@@ -228,7 +229,7 @@ and block lvl env (ds, body) =
 and comp_unit = function
   | LibU _ -> raise (Invalid_argument "cannot compile library")
   | ProgU ds -> decs_ TopLvl M.empty ds
-  | ActorU (as_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) ->
+  | ActorU (as_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) ->
     let env = match as_opt with
       | None -> M.empty
       | Some as_ -> args TopLvl M.empty as_
@@ -239,6 +240,7 @@ and comp_unit = function
     exp_ TopLvl env' heartbeat;
     exp_ TopLvl env' timer;
     exp_ TopLvl env' inspect;
+    exp_ TopLvl env' low_memory;
     exp_ TopLvl env' stable_record
 
 let analyze ((cu, _flavor) : prog) =