Add native modules support #4925

waruqi · 2024-04-02T14:57:01Z

We can use native modules to optimise time-consuming lua implementations, or we can import and use third-party native lua modules.

Each time the binary module interface is executed, a child process is executed, so shared modules are lighter and faster. However, the binary module is simpler to implement and does not need to depend on lua packages.

Binary module

Add a binary module in modules/binary

ruki-2:native_module ruki$ tree modules/binary/
modules/binary/
└── bar
    ├── src
    │   ├── add.cpp
    │   └── sub.cpp
    └── xmake.lua

we define two interfaces in it's xmake.lua. bar.add, bar.sub

add_rules("mode.debug", "mode.release")

target("add")
    add_rules("module.binary")
    add_files("src/add.cpp")

target("sub")
    add_rules("module.binary")
    add_files("src/sub.cpp")

bar.cpp

#include <stdio.h>
#include <stdlib.h>

int main(int argc, char** argv) {
    int a = atoi(argv[1]);
    int b = atoi(argv[2]);
    printf("%d", a + b);
    return 0;
}

Then we can use import("binary.bar") to import it, call it's interfaces. bar.add(1, 1)

add_rules("mode.debug", "mode.release")

add_moduledirs("modules")

target("test")
    set_kind("binary")
    add_files("src/*.cpp")
    on_config(function (target)
        import("binary.bar")
        print("binary: 1 + 1 = %s", bar.add(1, 1))
        print("binary: 1 - 1 = %s", bar.sub(1, 1))
    end)

Import("binary.bar") will build this module first, and run build/bar program and return it's value.

ruki-2:native_module ruki$ xmake
[ 50%]: cache compiling.release src/add.cpp
[ 50%]: cache compiling.release src/sub.cpp
[ 75%]: linking.release module_add
[ 75%]: linking.release module_sub
[100%]: build ok, spent 1.256s
binary: 1 + 1 = 2
binary: 1 - 1 = 0
[100%]: build ok, spent 0.456s

module only be built once, then we run xmake again:

ruki-2:native_module ruki$ xmake
binary: 1 + 1 = 2
binary: 1 - 1 = 0
[100%]: build ok, spent 0.456s

Shared module

We can also import native shared modules. (lua + so)

ruki-2:native_module ruki$ tree modules/shared/
modules/shared/
└── foo
    ├── src
    │   └── foo.c
    └── xmake.lua

We need write a native shared module with lua5.4.

add_rules("mode.debug", "mode.release")

target("foo")
    add_rules("module.shared")
    add_files("src/foo.c")

foo.c

#include <xmi.h>

static int add(lua_State* lua) {
    int a = lua_tointeger(lua, 1);
    int b = lua_tointeger(lua, 2);
    lua_pushinteger(lua, a + b);
    return 1;
}

static int sub(lua_State* lua) {
    int a = lua_tointeger(lua, 1);
    int b = lua_tointeger(lua, 2);
    lua_pushinteger(lua, a - b);
    return 1;
}

int luaopen(foo, lua_State* lua) {
    static const luaL_Reg funcs[] = {
        {"add", add},
        {"sub", sub},
        {NULL, NULL}
    };
    lua_newtable(lua);
    luaL_setfuncs(lua, funcs, 0);
    return 1;
}

Then we can use import("shared.foo") to import it and call it's interfaces. foo.add and foo.sub

add_rules("mode.debug", "mode.release")

add_moduledirs("modules")

target("test")
    set_kind("binary")
    add_files("src/*.cpp")
    on_config(function (target)
        import("shared.foo")
        print("shared: 1 + 1 = %d", foo.add(1, 1))
        print("shared: 1 - 1 = %d", foo.sub(1, 1))
    end)

ruki-2:native_module ruki$ xmake
[ 50%]: cache compiling.release src/foo.c
[ 75%]: linking.release libmodule_foo.dylib
[100%]: build ok, spent 1.296s
shared: 1 + 1 = 2
shared: 1 - 1 = 0
[100%]: build ok, spent 0.447s

run xmake again

ruki-2:native_module ruki$ xmake
shared: 1 + 1 = 2
shared: 1 - 1 = 0
[100%]: build ok, spent 0.447s

The text was updated successfully, but these errors were encountered:

waruqi · 2024-04-03T03:04:17Z

And we can configure import("binary.bar", {always_build = true}), it will always build module.

Examples: https://github.com/xmake-io/xmake/tree/master/tests/projects/other/native_module

waruqi · 2024-04-03T03:35:06Z

We can use template to create it. module.binary and module.shared templates.

ruki:tmp ruki$ xmake create -t module.binary test
create test ...
  [+]: xmake.lua
  [+]: modules/binary/bar/xmake.lua
  [+]: modules/binary/bar/.gitignore
  [+]: modules/binary/bar/src/add.cpp
  [+]: modules/binary/bar/src/sub.cpp
  [+]: src/main.cpp
  [+]: .gitignore
create ok!
ruki:tmp ruki$ cd test
ruki:test ruki$ xmake
building native binary module(bar) in modules/binary/bar ...
[ 50%]: cache compiling.release src/add.cpp
[ 50%]: cache compiling.release src/sub.cpp
[ 75%]: linking.release module_add
[ 75%]: linking.release module_sub
[100%]: build ok, spent 1.231s
binary: 1 + 1 = 2
binary: 1 - 1 = 0
[ 50%]: cache compiling.release src/main.cpp
[ 75%]: linking.release test
[100%]: build ok, spent 1.28s

ruki:tmp ruki$ xmake create -t module.shared -l c test
create test ...
  [+]: xmake.lua
  [+]: modules/shared/foo/xmake.lua
  [+]: modules/shared/foo/src/foo.c
  [+]: src/main.c
  [+]: .gitignore
create ok!
ruki:tmp ruki$ cd test
ruki:test ruki$ xmake
building native shared module(foo) in modules/shared/foo ...
[ 50%]: cache compiling.release src/foo.c
[ 75%]: linking.release libmodule_foo.dylib
[100%]: build ok, spent 0.688s
shared: 1 + 1 = 2
shared: 1 - 1 = 0
[ 50%]: cache compiling.release src/main.c
[ 75%]: linking.release test
[100%]: build ok, spent 0.656s

Arthapz · 2024-04-03T03:37:43Z

maybe u should add a .clang-format for future native xmake internal C module

waruqi · 2024-04-03T03:39:58Z

maybe u should add a .clang-format for future native xmake internal C module

I'm not familiar with .clang-format, you can open a pr to improve these templates.

waruqi · 2024-04-03T03:46:13Z

And we can also use module.binary modules to implement codegen.

https://github.com/xmake-io/xmake/tree/master/tests/projects/other/autogen_module

Arthapz · 2024-04-03T03:58:54Z

maybe u should add a .clang-format for future native xmake internal C module

I'm not familiar with .clang-format, you can open a pr to improve these templates.

Ok i'll try to do something that mimic the formating of tbox C code

waruqi · 2024-04-03T04:23:02Z

maybe u should add a .clang-format for future native xmake internal C module

I'm not familiar with .clang-format, you can open a pr to improve these templates.

Ok i'll try to do something that mimic the formating of tbox C code

Thanks, but tbox's code style is an earlier style, and some of the styles have changed. for example {}

void func() {
}

But I don't have time to reformat all the c code for tbox and xmake.

waruqi · 2024-04-03T15:19:33Z

Now we can remove lua package dep in module.shared and it supports xmake with lua/luajit runtime at same time.

#4932

foo.c

#include <xmi.h>

static int add(lua_State* lua) {
    int a = lua_tointeger(lua, 1);
    int b = lua_tointeger(lua, 2);
    lua_pushinteger(lua, a + b);
    return 1;
}

static int sub(lua_State* lua) {
    int a = lua_tointeger(lua, 1);
    int b = lua_tointeger(lua, 2);
    lua_pushinteger(lua, a - b);
    return 1;
}

int luaopen(foo, lua_State* lua) {
    static const luaL_Reg funcs[] = {
        {"add", add},
        {"sub", sub},
        {NULL, NULL}
    };
    lua_newtable(lua);
    luaL_setfuncs(lua, funcs, 0);
    return 1;
}

modules/shared/foo/xmake.lua

add_rules("mode.debug", "mode.release")

target("foo")
    add_rules("module.shared")
    add_files("src/foo.c")

xmi.h will include all lua c apis, we can also include lua.h to use them.

microdee · 2024-04-03T19:32:46Z

wow this is exactly the feature I needed for my project, well done!

waruqi · 2024-04-05T07:05:43Z

A whole lua-cjson shared module example: https://github.com/xmake-io/xmake/tree/dev/tests/projects/other/native_module_cjson

modules/lua/cjson.lua

add_rules("mode.debug", "mode.release")

target("cjson")
    add_rules("module.shared")
    set_warnings("all")
    if is_plat("windows") then
        set_languages("c89")
    end
    add_files("src/*.c")
    add_files("src/lua-cjson/*.c|fpconv.c")
    -- Use internal strtod() / g_fmt() code for performance and disable multi-thread
    add_defines("NDEBUG", "USE_INTERNAL_FPCONV")
    add_defines("XM_CONFIG_API_HAVE_LUA_CJSON")
    if is_plat("windows") then
        -- Windows sprintf()/strtod() handle NaN/inf differently. Not supported.
        add_defines("DISABLE_INVALID_NUMBERS")
        add_defines("inline=__inline")
    end

src/cjson.c

#include <xmi.h>

int luaopen_cjson(lua_State* lua);

int luaopen(cjson, lua_State* lua) {
    return luaopen_cjson(lua);
}

Import and use lua.cjson module

add_rules("mode.debug", "mode.release")

add_moduledirs("modules")

target("test")
    set_kind("binary")
    add_files("src/*.cpp")
    on_config(function (target)
        import("lua.cjson", {always_build = true})
        print(cjson.decode('{"foo": 1, "bar": [1, 2, 3]}'))
    end)

ruki-2:native_module_cjson ruki$ xmake
building native shared module(cjson) in modules/lua/cjson ...
[ 75%]: cache compiling.release src/cjson.c
[ 75%]: cache compiling.release src/lua-cjson/lua-cjson/g_fmt.c
[ 75%]: cache compiling.release src/lua-cjson/lua-cjson/lua_cjson.c
[ 75%]: cache compiling.release src/lua-cjson/lua-cjson/strbuf.c
[ 75%]: cache compiling.release src/lua-cjson/lua-cjson/dtoa.c
[ 87%]: linking.release libmodule_cjson.dylib
[100%]: build ok, spent 2.156s
{
  foo = 1,
  bar = {
    1,
    2,
    3
  }
}

[100%]: build ok, spent 0.491s

waruqi added the feature request label Apr 2, 2024

waruqi added this to the v2.9.1 milestone Apr 2, 2024

waruqi mentioned this issue Apr 2, 2024

Add native modules support #4918

Merged

waruqi mentioned this issue Apr 3, 2024

Improve native shared module to remove lua package dependence #4932

Merged

waruqi closed this as completed Apr 5, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add native modules support #4925

Add native modules support #4925

waruqi commented Apr 2, 2024 •

edited

Loading

waruqi commented Apr 3, 2024

waruqi commented Apr 3, 2024 •

edited

Loading

Arthapz commented Apr 3, 2024

waruqi commented Apr 3, 2024

waruqi commented Apr 3, 2024

Arthapz commented Apr 3, 2024

waruqi commented Apr 3, 2024 •

edited

Loading

waruqi commented Apr 3, 2024 •

edited

Loading

microdee commented Apr 3, 2024

waruqi commented Apr 5, 2024 •

edited

Loading

Add native modules support #4925

Add native modules support #4925

Comments

waruqi commented Apr 2, 2024 • edited Loading

Binary module

Shared module

waruqi commented Apr 3, 2024

waruqi commented Apr 3, 2024 • edited Loading

Arthapz commented Apr 3, 2024

waruqi commented Apr 3, 2024

waruqi commented Apr 3, 2024

Arthapz commented Apr 3, 2024

waruqi commented Apr 3, 2024 • edited Loading

waruqi commented Apr 3, 2024 • edited Loading

microdee commented Apr 3, 2024

waruqi commented Apr 5, 2024 • edited Loading

waruqi commented Apr 2, 2024 •

edited

Loading

waruqi commented Apr 3, 2024 •

edited

Loading

waruqi commented Apr 3, 2024 •

edited

Loading

waruqi commented Apr 3, 2024 •

edited

Loading

waruqi commented Apr 5, 2024 •

edited

Loading