diff --git a/CHANGELOG.md b/CHANGELOG.md index e552011..f1a2fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ and this project adheres to ## [Unreleased] +## Changes + +- The release files are now copied recursively instead of calling the OS command + `cp -r`. This is because we are now calculating a hash of all the deployed + files to generate a software release unique identifier. +- The deploy command now generate a MANIFEST files that contains information + about the deployed software in a term file that can be read with + file:consult/1. + ## [2.7.1] - 2024-10-11 ### Changed diff --git a/src/grisp_tools_build.erl b/src/grisp_tools_build.erl index 924745b..a908f7f 100644 --- a/src/grisp_tools_build.erl +++ b/src/grisp_tools_build.erl @@ -279,7 +279,7 @@ dockerize_command(Cmd, S0) -> apply_patch({Name, Patch}, State0) -> Dir = mapz:deep_get([paths, build], State0), Context = mapz:deep_get([build, context], State0), - grisp_tools_util:write_file(Dir, Patch, Context), + grisp_tools_util:copy_file(Dir, Patch, Context), State4 = case shell(State0, ["git apply ", Name, " --ignore-whitespace --reverse --check"], [{cd, Dir}, return_on_error]) of @@ -308,7 +308,7 @@ sort_files(Apps, Files) -> copy_file(Root, File, State0) -> State1 = event(State0, [File]), Context = mapz:deep_get([build, context], State0), - grisp_tools_util:write_file(Root, File, Context), + grisp_tools_util:copy_file(Root, File, Context), State1. run_hooks(#{paths := #{toolchain := {ToolchainType, _}}} = S0, Type, Opts) -> diff --git a/src/grisp_tools_deploy.erl b/src/grisp_tools_deploy.erl index 6d81003..a9d866b 100644 --- a/src/grisp_tools_deploy.erl +++ b/src/grisp_tools_deploy.erl @@ -31,9 +31,9 @@ run(State) -> %--- Tasks --------------------------------------------------------------------- -package(#{custom_build := true, build := #{hash := #{value := Hash}}} = State0) -> +package(State0 = #{custom_build := true, build := #{hash := #{value := Hash}}}) -> event(State0, [{type, {custom_build, Hash}}]); -package(#{build := #{hash := #{value := Hash}}} = State0) -> +package(State0 = #{build := #{hash := #{value := Hash}}}) -> State1 = event(State0, [{type, {package, Hash}}]), grisp_tools_util:weave(State1, [ fun meta/1, @@ -42,7 +42,7 @@ package(#{build := #{hash := #{value := Hash}}} = State0) -> fun extract/1 ]). -release(#{paths := #{install := InstallPath}} = State0) -> +release(State0 = #{paths := #{install := InstallPath}}) -> Release = maps:get(release, State0, #{}), release(State0, maps:merge(Release, #{erts => InstallPath})). @@ -53,6 +53,7 @@ distribute(State0 = #{distribute := Dists}) -> fun dist_prepare/1, fun dist_release/1, fun dist_files/1, + fun dist_write_manifest/1, fun dist_finish/1, fun(S) -> run_script([dist, scripts], post_script, S) end ], [fun dist_abort/1]) @@ -75,9 +76,9 @@ init(State0) -> rm(Tmp), mapz:deep_merge([State1, #{package => #{file => File, tmp => Tmp}}]). -download(#{package_source := cache} = State0) -> +download(State0 = #{package_source := cache}) -> event(State0, ['_skip']); -download(#{package := #{meta := Meta}} = State0) -> +download(State0 = #{package := #{meta := Meta}}) -> Client = http_init(), URI = grisp_tools_util:cdn_path(otp, State0), Headers = [{"If-None-Match", ETag} || #{etag := ETag} <- [Meta]], @@ -94,7 +95,7 @@ download(#{package := #{meta := Meta}} = State0) -> end, State2. -extract(#{package := #{state := State, file := File}} = State0) +extract(State0 = #{package := #{state := State, file := File}}) when (State == downloaded) or (State == not_modified) -> #{paths := #{install := InstallPath}} = State0, case filelib:is_dir(InstallPath) of @@ -111,11 +112,27 @@ extract_really(File, InstallPath, State0) -> {error, Reason} -> event(State1, [{error, Reason}]) end. +dist_hash_prepare(State0) -> + State0#{dist_hash => crypto:hash_init(sha)}. + +dist_hash_update(State0 = #{dist_hash := Ctx0}, FileId, FileContent) + when Ctx0 =/= undefined -> + Ctx1 = crypto:hash_update(Ctx0, ["<", FileId, ":", FileContent, ">"]), + State0#{dist_hash => Ctx1}; +dist_hash_update(State0 = #{dist_hash := undefined}, _FileId, _FileContent) -> + State0. + +dist_hash_finalize(State0 = #{dist_hash := Ctx0}) + when Ctx0 =/= undefined -> + {State0#{dist_hash => undefined}, crypto:hash_final(Ctx0)}; +dist_hash_finalize(State0) -> + {State0, undefined}. + dist_prepare(State0 = #{dist := #{type := copy, destination := Dest}}) -> case file:read_file_info(Dest) of {ok, #file_info{type = directory, access = Access}} when Access =:= write; Access =:= read_write -> - State0; + dist_hash_prepare(State0); {ok, #file_info{type = directory}} -> event(State0, [{error, dir_not_writable, Dest}]); {ok, #file_info{}} -> @@ -144,11 +161,11 @@ dist_prepare(State0 = #{dist := #{type := archive, destination := Dest, end, grisp_tools_util:ensure_dir(Dest), case erl_tar:open(Dest, TarOpts) of - {ok, TarDesc} -> State0#{tar_desc => TarDesc}; + {ok, TarDesc} -> dist_hash_prepare(State0#{tar_desc => TarDesc}); {error, Reason} -> event(State0, [archive, {error, Reason}]) end. -dist_files(#{dist := #{destination := Dest}, release := Release} = State0) -> +dist_files(State0 = #{dist := #{destination := Dest}, release := Release}) -> #{paths := #{install := InstallPath}} = State0, State1 = event(State0, [files, {init, Dest}]), ERTSPath = filelib:wildcard(binary_to_list(filename:join(InstallPath, "erts-*"))), @@ -161,28 +178,57 @@ dist_files(#{dist := #{destination := Dest}, release := Release} = State0) -> }, maps:fold( fun(_Name, File, S) -> - write_file(S, File, Context) + copy_file(S, File, Context) end, State1, mapz:deep_get([deploy, overlay, files], State0) ). -write_file(#{dist := #{type := archive}, tar_desc := TarDesc} = State0, - #{target := Target} = File, Context) -> - Content = iolist_to_binary(grisp_tools_util:read_file(File, Context)), - State1 = event(State0, [files, {copy, File}]), - case erl_tar:add(TarDesc, Content, binary_to_list(Target), []) of +as_binary(Data) when is_atom(Data) -> atom_to_binary(Data); +as_binary(Data) -> iolist_to_binary(Data). + +as_list(Data) when is_list(Data) -> Data; +as_list(Data) when is_binary(Data) -> binary_to_list(Data). + +copy_file(State0 = #{dist := #{destination := Dest}}, File, Context) -> + copy_file(State0, Dest, File, undefined, Context). + +copy_file(State0, Dest, File, FileInfo, Context) -> + Content = as_binary(grisp_tools_util:read_file(File, Context)), + write_file(State0, Dest, File, FileInfo, Content). + +write_file(State0 = #{dist := #{destination := Dest}}, File, Content) -> + write_file(State0, Dest, File, undefined, Content). + +write_file(State0 = #{dist := #{type := archive}, tar_desc := TarDesc}, _Dest, + #{target := Target}, FileInfo, Content) -> + State1 = event(State0, [files, {copy, Target}]), + TarOpts = case FileInfo of + undefined -> []; + Rec when is_record(Rec, file_info) -> + % TAR files lose the files mode + [{K, V} || {K, V} <- [ + {atime, Rec#file_info.atime}, + {mtime, Rec#file_info.mtime}, + {ctime, Rec#file_info.ctime}, + {uid, Rec#file_info.uid}, + {gid, Rec#file_info.gid} + ], V =/= undefined] + end, + ContentBin = iolist_to_binary(Content), + case erl_tar:add(TarDesc, ContentBin, as_list(Target), TarOpts) of {error, Reason} -> error(Reason); - ok -> State1 + ok -> dist_hash_update(State1, Target, Content) end; -write_file(#{dist := #{type := copy, destination := Dest, force := Force}} = State0, - #{target := Target} = File, Context) -> +write_file(State0 = #{dist := #{type := copy, force := Force}}, Dest, + File = #{target := Target}, FileInfo, Content) -> Path = filename:join(Dest, Target), - State1 = event(State0, [files, {copy, File}]), + State1 = event(State0, [files, {copy, Target}]), + State2 = dist_hash_update(State1, Target, Content), force_execute(Path, Force, fun(F) -> grisp_tools_util:ensure_dir(F), - grisp_tools_util:write_file(Dest, File, Context) - end, State1). + grisp_tools_util:write_file(Dest, File, Content, FileInfo) + end, State2). force_execute(File, Force, Fun, State0) -> State1 = case {filelib:is_file(File), Force} of @@ -194,29 +240,114 @@ force_execute(File, Force, Fun, State0) -> Fun(File), State1. -dist_release(#{tar_desc := TarDesc, release := Release, - dist := #{type := archive}} = State0) -> - #{name := RelName, dir := Source} = Release, +dist_create_dir_callback(State0 = #{dist := #{type := archive}}, + _DestRoot, _RelPath, _FileInfo) -> + % TAR files do not contains empty directories... + State0; +dist_create_dir_callback(State0 = #{dist := #{type := copy}}, + DestRoot, RelPath, FileInfo) -> + DestPath = filename:join(DestRoot, RelPath), + grisp_tools_util:filelib_ensure_path(filename:join(DestPath, "dummy.file")), + case file:write_file_info(DestPath, FileInfo, [{time, posix}]) of + ok -> State0; + {error, Reason} -> + erlang:error({write_file_info_error, Reason, DestPath}) + end. + +dist_copy_callback(State0, SrcPath, DestRoot, RelPath, FileInfo) -> + FileSpec = #{source => SrcPath, target => RelPath, name => RelPath}, + copy_file(State0, DestRoot, FileSpec, FileInfo, #{}). + +dist_release(State0 = #{release := #{name := RelName, dir := Source}, + dist := #{type := archive}}) -> Target = atom_to_list(RelName), State1 = event(State0, [release, {archive, Source, Target}]), - case erl_tar:add(TarDesc, Source, Target, [dereference]) of - {error, Reason} -> error(Reason); - ok -> State1 - end; -dist_release(#{release := Release, dist := #{type := copy} = Dist} = State0) -> - #{name := RelName, dir := Source} = Release, - #{destination := Dest, force := Force} = Dist, - Target = filename:join(Dest, RelName), + Opts = #{copy_fun => fun dist_copy_callback/5, + create_dir_fun => fun dist_create_dir_callback/4}, + grisp_tools_util:copy_directory(State1, Source, "", Target, Opts); +dist_release(State0 = #{release := #{name := RelName, dir := Source}, + dist := #{type := copy, destination := Dest}}) -> + Target = atom_to_list(RelName), State1 = event(State0, [release, {copy, Source, Target}]), - CopyExe = case Force of - true -> "cp -Rf"; - false -> "cp -R" - end, - Command = string:join([CopyExe, qoute(Source ++ "/"), qoute(Target)], " "), - {Output, State2} = shell(State1, Command), - event(State2, [release, {copy, {result, Output}}]). + Opts = #{copy_fun => fun dist_copy_callback/5, + create_dir_fun => fun dist_create_dir_callback/4}, + grisp_tools_util:copy_directory(State1, Source, Dest, Target, Opts). + +parse_grisp_package_file(Line) -> + case binary:split(Line, <<" ">>, [global]) of + [Path, Hash] -> {Path, Hash}; + Parts -> + % The path contains a space + Hash = lists:last(Parts), + Path = binary:part(Line, {0, byte_size(Line) - byte_size(Hash) - 1}), + {Path, Hash} + end. + +read_grisp_package_files(ErtsPath) -> + case file:read_file(filename:join(ErtsPath, "GRISP_PACKAGE_FILES")) of + {error, enoent} -> undefined; + {ok, Data} -> + Lines = binary:split(Data, <<"\n">>, [global, trim]), + [parse_grisp_package_file(Line) || Line <- Lines] + end. + +read_grisp_toolchain_revision(ErtsPath) -> + case file:read_file(filename:join(ErtsPath, "GRISP_TOOLCHAIN_REVISION")) of + {error, enoent} -> undefined; + {ok, Data} -> + re:replace(Data, <<"(^\\s+|\\s+$)">>, <<>>, + [global, {return, binary}]) + end. -qoute(String) -> "\"" ++ String ++ "\"". +validate_manifest(Data, Expected) -> + TempFilePath = string:chomp(os:cmd("mktemp")), + try + file:write_file(TempFilePath, Data), + case file:consult(TempFilePath) of + {ok, Expected} -> ok; + {ok, _Other} -> + {internal_manifest_error, inconsistent, Data}; + {error, Reason} -> + {internal_manifest_error, Reason, Data} + end + after + os:cmd("rm -f " ++ TempFilePath) + end. + +dist_write_manifest(State0 = #{platform := Platform, + otp_version := {_, _, _, OtpVer}, + release := Release}) -> + #{name := RelName, version := RelVsn, erts := ErtsPath} = Release, + Profiles = maps:get(profiles, Release, []), + {State1, HashBin} = dist_hash_finalize(State0), + Hash = iolist_to_binary([io_lib:format("~2.16.0b", [Byte]) + || <> <= HashBin]), + ToolchainRev = read_grisp_toolchain_revision(ErtsPath), + PackageFiles = read_grisp_package_files(ErtsPath), + ManifestTerms = [ + {platform, as_binary(Platform)}, + {id, Hash}, + {relname, as_binary(RelName)}, + {relvsn, as_binary(RelVsn)}, + {profiles, Profiles}, + {package, [ + {toolchain, [ + {revision, as_binary(ToolchainRev)} + ]}, + {rtems, [ + % FIXME: whe finally support rtems 6, we need a way + % to figure the version out. + {version, <<"5">>} + ]}, + {otp, [ + {version, as_binary(OtpVer)} + ]}, + {custom, PackageFiles} + ]} + ], + Manifest = grisp_tools_util:format_term(ManifestTerms), + validate_manifest(Manifest, ManifestTerms), + write_file(State1, #{target => <<"MANIFEST">>}, Manifest). dist_finish(State0 = #{tar_desc := TarDesc, dist := #{type := archive, destination := Dest}}) -> @@ -236,7 +367,7 @@ dist_abort(State0) -> % Helpers -download_loop(ReqID, #{package := #{tmp := Tmp}} = State0) -> +download_loop(ReqID, State0 = #{package := #{tmp := Tmp}}) -> with_file(Tmp, [raw, append, binary], fun(Handle) -> download_loop({ReqID, undefined}, Handle, State0, 0) end). diff --git a/src/grisp_tools_util.erl b/src/grisp_tools_util.erl index 1a98ee4..bd8d572 100644 --- a/src/grisp_tools_util.erl +++ b/src/grisp_tools_util.erl @@ -1,5 +1,7 @@ -module(grisp_tools_util). +-include_lib("kernel/include/file.hrl"). + % API -export([weave/2]). -export([weave/3]). @@ -26,7 +28,9 @@ -export([build_hash_format/1]). -export([merge_build_config/2]). -export([source_hash/2]). --export([write_file/3]). +-export([copy_directory/4, copy_directory/5]). +-export([copy_file/3, copy_file/4]). +-export([write_file/3, write_file/4]). -export([read_file/2]). -export([with_file/3]). -export([pipe/2]). @@ -38,6 +42,9 @@ -export([make_relative/2]). -export([maybe_relative/2]). -export([maybe_relative/3]). +-export([filelib_ensure_path/1]). +-export([format_term/1]). + %--- Macros -------------------------------------------------------------------- @@ -236,11 +243,36 @@ source_hash(Apps, Board) -> Targets2 = maps:merge(Targets, NIFFiles), hash_files(Targets2). -write_file(Root, #{target := Target} = File, Context) -> +copy_directory(State0, Src, DestRoot, DestRel) -> + recursive_copy(State0, Src, DestRoot, DestRel, [], #{}). + +copy_directory(State0, Src, DestRoot, DestRel, Opts) -> + recursive_copy(State0, Src, DestRoot, DestRel, [], Opts). + +copy_file(Root, File, Context) -> Content = read_file(File, Context), + write_file(Root, File, Content). + +copy_file(Root, File, Context, FileInfo) -> + Content = read_file(File, Context), + write_file(Root, File, Content, FileInfo). + +write_file(Root, File, Content) -> + write_file(Root, File, Content, undefined). + +write_file(Root, #{target := Target}, Content, FileInfo) -> Destination = filename:join(Root, Target), ensure_dir(Destination), - ok = file:write_file(Destination, Content). + ok = file:write_file(Destination, Content), + case FileInfo of + undefined -> ok; + Rec when is_record(Rec, file_info) -> + case file:write_file_info(Destination, FileInfo, [{time, posix}]) of + ok -> ok; + {error, Reason} -> + erlang:error({write_file_info_error, Reason, Destination}) + end + end. read_file(#{source := {template, Source}}, Context) -> grisp_tools_template:render(Source, Context); @@ -310,6 +342,28 @@ maybe_relative(BasePath, Path, MaxDoubleDots) -> filename:join(RelParts) end. +% For OTP < 25 +filelib_ensure_path(Path) -> + DirPath = filename:dirname(Path), + case filelib:ensure_dir(filename:join(DirPath, "dummy.file")) of + ok -> ok; + {error, enoent} -> + filelib_ensure_path(DirPath), + file:make_dir(DirPath); + {error, Reason} -> + erlang:error(Reason) + end. + +%% @doc Formats a term into a UTF8 encoded binary that can be later read by +%% file:consult/1. If the binaries are strings, they are supposed to be encoded +%% in UTF8. +-spec format_term(term()) -> binary(). +format_term(Term) -> + IoData = format_term(<<" ">>, 0, [], Term), + Bin = unicode:characters_to_binary(IoData, utf8), + <<"%% coding: utf-8\n", Bin/binary>>. + + %--- Internal ------------------------------------------------------------------ common_prefix([H | T1], [H | T2]) -> @@ -428,8 +482,7 @@ hash_file_read(Handle, Context) -> eof -> {ok, crypto:hash_final(Context)} end. -format_hash(sha256, <>) -> format_hash(Int); -format_hash(md5, <>) -> format_hash(Int). +format_hash(sha256, <>) -> format_hash(Int). format_hash(Int) when is_integer(Int) -> list_to_binary(io_lib:format("~.16b", [Int])). @@ -525,3 +578,159 @@ file_info(Name, App, Source, Target) -> path_to_binary({template, Path}) when is_list(Path) -> {template, iolist_to_binary(Path)}; path_to_binary(Path) when is_list(Path) -> iolist_to_binary(Path); path_to_binary(Path) -> Path. + +resolve_symlink(FilePath) -> + case file:read_link_info(FilePath, [{time, posix}]) of + {ok, #file_info{type = symlink}} -> + case file:read_link(FilePath) of + {ok, Target} -> resolve_symlink(Target); + {error, Reason} -> + erlang:error({read_link_error, Reason, FilePath}) + end; + {error, Reason} -> + erlang:error({read_link_info_error, Reason, FilePath}); + _ -> + FilePath + end. + +recursive_copy_file(State0, SrcPath, DestRoot, RelPath, FileInfo, + #{copy_fun := CopyFun}) + when is_function(CopyFun, 5) -> + CopyFun(State0, SrcPath, DestRoot, RelPath, FileInfo); +recursive_copy_file(State0, SrcPath, DestRoot, RelPath, FileInfo, _Opts) -> + FileSpec = #{source => SrcPath, target => RelPath, name => RelPath}, + copy_file(DestRoot, FileSpec, #{}, FileInfo), + State0. + +recursive_create_dir(State0, DestRoot, RelPath, FileInfo, + #{create_dir_fun := CreateDirFun}) + when is_function(CreateDirFun, 4) -> + CreateDirFun(State0, DestRoot, RelPath, FileInfo); +recursive_create_dir(State0, DestRoot, RelPath, FileInfo, _Opts) -> + DestPath = filename:join(DestRoot, RelPath), + filelib_ensure_path(filename:join(DestPath, "dummy.file")), + case FileInfo of + Rec when is_record(Rec, file_info) -> + case file:write_file_info(DestPath, FileInfo, [{time, posix}]) of + ok -> State0; + {error, Reason} -> + erlang:error({write_file_info_error, Reason, DestPath}) + end + end. + +recursive_copy(State0, Src, DestRoot, DestRel, RevRelNames, Opts) -> + {SrcPath, RelPath} = case RevRelNames of + [] -> {Src, DestRel}; + _ -> + R = filename:join(lists:reverse(RevRelNames)), + {filename:join(Src, R), filename:join(DestRel, R)} + end, + case file:read_file_info(SrcPath, [{time, posix}]) of + {ok, #file_info{type = directory} = FileInfo} -> + State1 = recursive_create_dir(State0, DestRoot, RelPath, FileInfo, Opts), + case file:list_dir(SrcPath) of + {error, Reason} -> + erlang:error({list_dir_error, Reason, SrcPath}); + {ok, Names} -> + lists:foldl(fun(Name, State) -> + RevRelNames2 = [Name | RevRelNames], + recursive_copy(State, Src, DestRoot, DestRel, + RevRelNames2, Opts) + end, State1, Names) + end; + {ok, #file_info{type = regular} = FileInfo} -> + case file:read_link_info(SrcPath, [{time, posix}]) of + {ok, #file_info{type = regular}} -> + recursive_copy_file(State0, SrcPath, DestRoot, RelPath, + FileInfo, Opts); + {ok, #file_info{type = symlink}} -> + SrcPath2 = resolve_symlink(SrcPath), + recursive_copy_file(State0, SrcPath2, DestRoot, RelPath, + FileInfo, Opts); + {ok, #file_info{type = Other}} -> + erlang:error({invalid_file_type, Other, SrcPath}); + {error, Reason} -> + erlang:error({read_link_info_error, Reason, SrcPath}) + end; + {ok, #file_info{type = Other}} -> + erlang:error({invalid_file_type, Other, SrcPath}); + {error, Reason} -> + erlang:error({read_file_info_error, Reason, SrcPath}) + end. + +format_term_value(Val) + when is_integer(Val); is_float(Val) -> + io_lib:format("~w", [Val]); +format_term_value(Val) + when is_atom(Val) -> + io_lib:write_atom(Val); +format_term_value([]) -> + "[]"; +format_term_value(Val) + when is_list(Val) -> + io_lib:write_string(Val); +format_term_value(Val) + when is_binary(Val) -> + try unicode:characters_to_list(Val, utf8) of + Unicode -> + try io_lib:write_latin1_string(Unicode) of + Str -> io_lib:format("<<~ts>>", [Str]) + catch + _:_ -> + Str = io_lib:write_string(Unicode), + io_lib:format("<<~ts/utf8>>", [Str]) + end + catch + _:_ -> + io_lib:format("~w", [Val]) + end. + +format_term(_Prefix, _Level, Acc, []) -> + lists:reverse(Acc); +format_term(Prefix, Level, Acc, [{Key, Val} | Rest]) + when is_atom(Key) orelse is_binary(Key), + is_atom(Val) orelse is_binary(Val) + orelse is_integer(Val) orelse is_float(Val) -> + Indent = binary:copy(Prefix, Level), + Ending = case {Level, Rest} of + {0, _} -> <<".">>; + {_, []} -> <<>>; + {_, _} -> <<",">> + end, + KeyStr = format_term_value(Key), + ValStr = format_term_value(Val), + ResStr = io_lib:format("~s{~ts, ~ts}~s~n", + [Indent, KeyStr, ValStr, Ending]), + format_term(Prefix, Level, [ResStr | Acc], Rest); +format_term(Prefix, Level, Acc, [Val | Rest]) + when is_atom(Val) orelse is_binary(Val) + orelse is_integer(Val) orelse is_float(Val) -> + Indent = binary:copy(Prefix, Level), + Ending = case {Level, Rest} of + {0, _} -> <<".">>; + {_, []} -> <<>>; + {_, _} -> <<",">> + end, + ValStr = format_term_value(Val), + ResStr = io_lib:format("~s~ts~s~n", [Indent, ValStr, Ending]), + format_term(Prefix, Level, [ResStr | Acc], Rest); +format_term(Prefix, Level, Acc, [{Key, Val} | Rest]) + when is_atom(Key) orelse is_binary(Key), is_list(Val) -> + Indent = binary:copy(Prefix, Level), + Ending = case {Level, Rest} of + {0, _} -> <<".">>; + {_, []} -> <<>>; + {_, _} -> <<",">> + end, + KeyStr = format_term_value(Key), + ResStr = case io_lib:printable_unicode_list(Val) of + true -> + ValStr = format_term_value(Val), + io_lib:format("~s{~ts, ~ts}~s~n", + [Indent, KeyStr, ValStr, Ending]); + false -> + SubStr = format_term(Prefix, Level + 1, [], Val), + io_lib:format("~s{~ts, [~n~ts~s]}~s~n", + [Indent, KeyStr, SubStr, Indent, Ending]) + end, + format_term(Prefix, Level, [ResStr | Acc], Rest).