Add takestring! and make String(::Memory) not truncate

NEWS.md

@@ -103,6 +103,7 @@ New library features
 * `RegexMatch` objects can now be used to construct `NamedTuple`s and `Dict`s ([#50988])
 * `Lockable` is now exported ([#54595])
 * New `ltruncate`, `rtruncate` and `ctruncate` functions for truncating strings to text width, accounting for char widths ([#55351])
+* `takestring!(v)` creates a `String` from `v` (a `Vector{UInt8}` or `IOBuffer`), truncating `v` and reusing its memory if possible )([#54372]).
 
 Standard library changes
 ------------------------

base/compiler/ssair/show.jl

@@ -312,7 +312,7 @@ function compute_ir_line_annotations(code::IRCode)
             loc_method = string(" "^printing_depth, loc_method)
             last_stack = stack
         end
-        push!(loc_annotations, String(take!(buf)))
+        push!(loc_annotations, takestring!(buf))
         push!(loc_lineno, (lineno != 0 && lineno != last_lineno) ? string(lineno) : "")
         push!(loc_methods, loc_method)
         (lineno != 0) && (last_lineno = lineno)

base/errorshow.jl

@@ -305,7 +305,7 @@ function showerror(io::IO, ex::MethodError)
         iob = IOContext(buf, io)     # for type abbreviation as in #49795; some, like `convert(T, x)`, should not abbreviate
         show_signature_function(iob, Core.Typeof(f))
         show_tuple_as_call(iob, :function, arg_types; hasfirst=false, kwargs = isempty(kwargs) ? nothing : kwargs)
-        str = String(take!(buf))
+        str = takestring!(buf)
         str = type_limited_string_from_context(io, str)
         print(io, str)
     end
@@ -596,7 +596,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[])
             m = parentmodule_before_main(method)
             modulecolor = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, m)
             print_module_path_file(iob, m, string(file), line; modulecolor, digit_align_width = 3)
-            push!(lines, String(take!(buf)))
+            push!(lines, takestring!(buf))
             push!(line_score, -(right_matches * 2 + (length(arg_types_param) < 2 ? 1 : 0)))
         end
     end

base/exports.jl

@@ -638,6 +638,7 @@ export
     split,
     string,
     strip,
+    takestring!,
     textwidth,
     thisind,
     titlecase,

base/filesystem.jl

@@ -141,7 +141,7 @@ import .Base:
     bytesavailable, position, read, read!, readavailable, seek, seekend, show,
     skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error,
     setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize,
-    isexecutable, isreadable, iswritable, MutableDenseArrayType
+    isexecutable, isreadable, iswritable, MutableDenseArrayType, unsafe_takestring!
 
 import .Base.RefValue
 

base/gmp.jl

@@ -11,7 +11,7 @@ import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor,
              bin, oct, dec, hex, isequal, invmod, _prevpow2, _nextpow2, ndigits0zpb,
              widen, signed, unsafe_trunc, trunc, iszero, isone, big, flipsign, signbit,
              sign, hastypemax, isodd, iseven, digits!, hash, hash_integer, top_set_bit,
-             clamp
+             clamp, unsafe_takestring
 
 if Clong == Int32
     const ClongMax = Union{Int8, Int16, Int32}
@@ -761,7 +761,7 @@ function string(n::BigInt; base::Integer = 10, pad::Integer = 1)
         sv[i] = '0' % UInt8
     end
     isneg(n) && (sv[1] = '-' % UInt8)
-    String(sv)
+    unsafe_takestring(sv)
 end
 
 function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<:Integer}

base/indices.jl

@@ -132,7 +132,7 @@ function throw_promote_shape_mismatch(a::Tuple, b::Union{Nothing,Tuple}, i = not
     if i ≢ nothing
         print(msg, ", mismatch at dim ", i)
     end
-    throw(DimensionMismatch(String(take!(msg))))
+    throw(DimensionMismatch(takestring!(msg)))
 end
 
 function promote_shape(a::Tuple{Int,}, b::Tuple{Int,})

base/int.jl

@@ -718,7 +718,7 @@ macro big_str(s)
             is_prev_dot = (c == '.')
         end
         print(bf, s[end])
-        s = String(take!(bf))
+        s = unsafe_takestring!(bf)
     end
     n = tryparse(BigInt, s)
     n === nothing || return n

base/intfuncs.jl

@@ -766,7 +766,7 @@ function bin(x::Unsigned, pad::Int, neg::Bool)
         i -= 1
     end
     neg && (@inbounds a[1] = 0x2d) # UInt8('-')
-    String(a)
+    unsafe_takestring(a)
 end
 
 function oct(x::Unsigned, pad::Int, neg::Bool)
@@ -780,7 +780,7 @@ function oct(x::Unsigned, pad::Int, neg::Bool)
         i -= 1
     end
     neg && (@inbounds a[1] = 0x2d) # UInt8('-')
-    String(a)
+    unsafe_takestring(a)
 end
 
 # 2-digit decimal characters ("00":"99")
@@ -850,7 +850,7 @@ function dec(x::Unsigned, pad::Int, neg::Bool)
     a = StringMemory(n)
     append_c_digits_fast(n, x, a, 1)
     neg && (@inbounds a[1] = 0x2d) # UInt8('-')
-    String(a)
+    unsafe_takestring(a)
 end
 
 function hex(x::Unsigned, pad::Int, neg::Bool)
@@ -871,7 +871,7 @@ function hex(x::Unsigned, pad::Int, neg::Bool)
         @inbounds a[i] = d + ifelse(d > 0x9, 0x57, 0x30)
     end
     neg && (@inbounds a[1] = 0x2d) # UInt8('-')
-    String(a)
+    unsafe_takestring(a)
 end
 
 const base36digits = UInt8['0':'9';'a':'z']
@@ -896,7 +896,7 @@ function _base(base::Integer, x::Integer, pad::Int, neg::Bool)
         i -= 1
     end
     neg && (@inbounds a[1] = 0x2d) # UInt8('-')
-    String(a)
+    unsafe_takestring(a)
 end
 
 split_sign(n::Integer) = unsigned(abs(n)), n < 0
@@ -972,7 +972,7 @@ function bitstring(x::T) where {T}
         x = lshr_int(x, 4)
         i -= 4
     end
-    return String(str)
+    return unsafe_takestring(str)
 end
 
 """

base/io.jl

@@ -277,13 +277,13 @@ julia> io = IOBuffer();
 julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.")
 56
 
-julia> String(take!(io))
+julia> takestring!(io)
 "JuliaLang is a GitHub organization. It has many members."
 
 julia> write(io, "Sometimes those members") + write(io, " write documentation.")
 44
 
-julia> String(take!(io))
+julia> takestring!(io)
 "Sometimes those members write documentation."
 ```
 User-defined plain-data types without `write` methods can be written when wrapped in a `Ref`:
@@ -543,8 +543,8 @@ julia> rm("my_file.txt")
 ```
 """
 readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String)
-readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...))
-readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = String(_unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)))
+readuntil(stream::IO, delim::UInt8; kw...) = takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...))
+readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...))
 readuntil(stream::IO, delim::T; keep::Bool=false) where T = _copyuntil(Vector{T}(), stream, delim, keep)
 
 
@@ -566,10 +566,10 @@ Similar to [`readuntil`](@ref), which returns a `String`; in contrast,
 ```jldoctest
 julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n");
 
-julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", 'L')))
+julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", 'L'))
 "Julia"
 
-julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true)))
+julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true))
 "JuliaLang is a GitHub organization."
 
 julia> rm("my_file.txt")
@@ -616,8 +616,7 @@ Logan
 """
 readline(filename::AbstractString; keep::Bool=false) =
     open(io -> readline(io; keep), filename)
-readline(s::IO=stdin; keep::Bool=false) =
-    String(_unsafe_take!(copyline(IOBuffer(sizehint=16), s; keep)))
+readline(s::IO=stdin; keep::Bool=false) = takestring!(copyline(IOBuffer(sizehint=16), s; keep))
 
 """
     copyline(out::IO, io::IO=stdin; keep::Bool=false)
@@ -642,10 +641,10 @@ See also [`copyuntil`](@ref) for reading until more general delimiters.
 ```jldoctest
 julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n");
 
-julia> String(take!(copyline(IOBuffer(), "my_file.txt")))
+julia> takestring!(copyline(IOBuffer(), "my_file.txt"))
 "JuliaLang is a GitHub organization."
 
-julia> String(take!(copyline(IOBuffer(), "my_file.txt", keep=true)))
+julia> takestring!(copyline(IOBuffer(), "my_file.txt", keep=true))
 "JuliaLang is a GitHub organization.\\n"
 
 julia> rm("my_file.txt")
@@ -1291,7 +1290,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state)
         buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
         empty!(chunks) # will cause next iteration to terminate
         seekend(r.itr.stream) # reposition to end of stream for isdone
-        s = String(_unsafe_take!(buf))
+        s = unsafe_takestring!(buf)
     else
         # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline]
         if ichunk == jchunk # common case: current and previous newline in same chunk
@@ -1308,7 +1307,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state)
             end
             write(buf, view(chunks[jchunk], 1:jnewline))
             buf.size = _stripnewline(r.itr.keep, buf.size, buf.data)
-            s = String(_unsafe_take!(buf))
+            s = unsafe_takestring!(buf)
 
             # overwrite obsolete chunks (ichunk+1:jchunk)
             i = jchunk

base/iobuffer.jl

@@ -66,7 +66,7 @@ julia> io = IOBuffer();
 julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.")
 56
 
-julia> String(take!(io))
+julia> takestring!(io)
 "JuliaLang is a GitHub organization. It has many members."
 
 julia> io = IOBuffer(b"JuliaLang is a GitHub organization.")
@@ -84,7 +84,7 @@ IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=fa
 julia> write(io, "JuliaLang is a GitHub organization.")
 34
 
-julia> String(take!(io))
+julia> takestring!(io)
 "JuliaLang is a GitHub organization"
 
 julia> length(read(IOBuffer(b"data", read=true, truncate=false)))
@@ -489,6 +489,76 @@ function take!(io::IOBuffer)
     return data
 end
 
+"Internal method. This method can be faster than takestring!, because it does not
+reset the buffer to a usable state, and it does not check for io.reinit.
+Using the buffer after calling unsafe_takestring! may cause undefined behaviour.
+This function is meant to be used when the buffer is only used as a temporary
+string builder, which is discarded after the string is built."
+function unsafe_takestring!(io::IOBuffer)
+    start = io.seekable ? io.offset + 1 : io.ptr
+    nbytes = io.size - start + 1
+    iszero(nbytes) && return ""
+    # The C function can only copy from the start of the memory.
+    # Fortunately, in most cases, the offset will be zero.
+    return if isone(start)
+        ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), io.data, nbytes)
+    else
+        mem = StringMemory(nbytes)
+        unsafe_copyto!(mem, 1, io.data, start, nbytes)
+        unsafe_takestring(mem)
+    end
+end
+
+"""
+    takestring!(io::IOBuffer) -> String
+
+Return the content of `io` as a `String`, resetting the buffer to its initial
+state.
+This is preferred over calling `String(take!(io))` to create a string from
+an `IOBuffer`.
+
+# Examples
+```jldoctest
+julia> io = IOBuffer();
+
+julia> write(io, [0x61, 0x62, 0x63]);
+
+julia> s = takestring!(io)
+"abc"
+
+julia> isempty(take!(io)) # io is now empty
+true
+```
+
+!!! compat "Julia 1.12"
+    This function requires at least Julia 1.12.
+"""
+function takestring!(io::IOBuffer)
+    # If the buffer has been used up and needs to be replaced, there are no bytes, and
+    # we can return an empty string without interacting with the buffer at all.
+    io.reinit && return ""
+
+    s = unsafe_takestring!(io)
+
+    # Restore the buffer to a usable state, making it no longer undefined behaviour to
+    # use the buffer after the `unsafe_takestring!` call.
+    # Note that if the buffer is not writable, there is no need to reinitialize the buffer,
+    # since it doesn't matter that the returned string looks into the same memory -
+    # this is because the buffer is not mutable through either the string nor the buffer.
+    if io.writable
+        io.reinit = true
+        io.mark = -1
+        io.ptr = 1
+        io.size = 0
+        io.offset = 0
+    end
+    s
+end
+
+# Fallback methods
+unsafe_takestring!(io::GenericIOBuffer) = takestring!(io)
+takestring!(io::GenericIOBuffer) = String(take!(io))
+
 """
     _unsafe_take!(io::IOBuffer)
 

base/iostream.jl

@@ -98,7 +98,7 @@ julia> write(io, "JuliaLang is a GitHub organization.")
 julia> truncate(io, 15)
 IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=15, maxsize=Inf, ptr=16, mark=-1)
 
-julia> String(take!(io))
+julia> takestring!(io)
 "JuliaLang is a "
 
 julia> io = IOBuffer();
@@ -107,7 +107,7 @@ julia> write(io, "JuliaLang is a GitHub organization.");
 
 julia> truncate(io, 40);
 
-julia> String(take!(io))
+julia> takestring!(io)
 "JuliaLang is a GitHub organization.\\0\\0\\0\\0\\0"
 ```
 """
@@ -456,7 +456,7 @@ function readuntil_string(s::IOStream, delim::UInt8, keep::Bool)
 end
 readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) =
     isascii(delim) ? readuntil_string(s, delim % UInt8, keep) :
-    String(_unsafe_take!(copyuntil(IOBuffer(sizehint=70), s, delim; keep)))
+    takestring!(copyuntil(IOBuffer(sizehint=70), s, delim; keep))
 
 function readline(s::IOStream; keep::Bool=false)
     @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2)

base/logging/ConsoleLogger.jl

@@ -141,7 +141,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module
         for (key, val) in kwargs
             key === :maxlog && continue
             showvalue(valio, val)
-            vallines = split(String(take!(valbuf)), '\n')
+            vallines = split(takestring!(valbuf), '\n')
             if length(vallines) == 1
                 push!(msglines, (indent=2, msg=SubString("$key = $(vallines[1])")))
             else

base/pkgid.jl

@@ -32,7 +32,7 @@ function binpack(pkg::PkgId)
     uuid = pkg.uuid
     write(io, uuid === nothing ? UInt128(0) : UInt128(uuid))
     write(io, pkg.name)
-    return String(take!(io))
+    return unsafe_takestring!(io)
 end
 
 function binunpack(s::String)