From 441b9fc05fed5091a415718611fceb69d98a9a5b Mon Sep 17 00:00:00 2001 From: Peter Hyatt Date: Wed, 6 Nov 2019 15:36:56 -0500 Subject: [PATCH 1/3] Fixed the conclusion example to work with Jula 1.x Replaced using Requests with using HTTP Properly handle unfinished @async log calls Replaced function log with function emit Replaced myid with getpid --- docs/src/man/conclusion.md | 71 +++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/docs/src/man/conclusion.md b/docs/src/man/conclusion.md index 3b3c030..0acb444 100644 --- a/docs/src/man/conclusion.md +++ b/docs/src/man/conclusion.md @@ -15,7 +15,7 @@ using Memento function run(f::Function, args...; kwargs...) ret = nothing - logger = getlogger(current_module()) + logger = getlogger(@__MODULE__) info(logger, "Got logger $logger") notice(logger, "Running function...") @@ -37,7 +37,7 @@ Now we want to start writing our application code that uses this package, but ou Requirements: 1. This will be run on Amazon EC2 instances and we want our log message to contain information about the machine the code is being run on. -2. We want our logs to be written to an HTTP REST service (kinda like Loggly), where the endpoint is of the form `https:////?AccessKey=`. +2. We want our logs to be written to an HTTP REST service (kinda like Loggly), where the endpoint is of the form `https:////?AccessKey=`, and an Authorization header and a Content-Type header. 3. We want our logs to be written in a CSV format... for some reason. Okay, so how do we address all of those requirements using Memento's API? @@ -54,7 +54,8 @@ NOTE: The code below is not intended to be a working example because it assumes # myapp.jl using Wrapper using Memento -using Requests # For send logs to our fake logging REST service +using Memento.TimeZones +using HTTP # For send logs to our fake logging REST service # Start by setting up our basic console logging for the root logger. logger = Memento.config!("info"; fmt="[{level} | {name}]: {msg}") @@ -78,17 +79,17 @@ mutable struct EC2Record <: Record trace = Attribute{StackTrace}(get_trace) EC2Record( - Attribute{DateTime}(() -> round(time, Dates.Second)), - Attribute(args[:level]), - Attribute(args[:levelnum]), - Attribute{AbstractString}(get_msg(args[:msg])), - Attribute(args[:name]), - Attribute(myid()), - Attribute{StackFrame}(get_lookup(trace)), + Attribute{ZonedDateTime}(() -> Dates.now(tz"UTC")), + Attribute(level), + Attribute(levelnum), + Attribute{AbstractString}(msg), + Attribute(name), + Attribute(getpid()), + Attribute{Union{StackFrame, Nothing}}(get_lookup(trace)), trace, - Attribute(ENV["INSTANCE_ID"]), - Attribute(ENV["PUBLIC_IP"]), - Attribute(ENV["IAM_USER"]), + Attribute(get(ENV, "INSTANCE_ID", "no INSTANCE_ID")), + Attribute(get(ENV, "PUBLIC_IP", "no PUBLIC_IP")), + Attribute(get(ENV, "IAM_USER", "no IAM_USER")), ) end end @@ -121,18 +122,45 @@ end # Our print method builds the correct uri using the log level # and sends the put request. -function println(io::REST, level::AbstractString, msg::AbstractString) - uri = "https://$(io.account_uri)/$(io.app_name)/$level?AccessKey=$(io.access_key)" - @async put(uri; data=msg) +global REST_LOG_TASKS = [] +global queue_cleanup = false + +function Base.println(io::REST, level::AbstractString, msg::AbstractString) + uri = "https://$(io.account_uri)/$(io.app_name)/$(level)?AccessKey=$(io.access_key)" + headers = [ "Authorization" => io.access_key, + "Content-Type" => "text/csv" ] + data = msg + t = @async HTTP.post(uri, headers, data) + + # Bookkeeping for handling @async tasks at exit + global REST_LOG_TASKS, queue_cleanup + push!(REST_LOG_TASKS, t) + filter!(t -> !istaskdone(t), REST_LOG_TASKS) + if !queue_cleanup + queue_cleanup = true + Base.atexit(finish_rest_log_tasks) + end +end + +# A flush for @async tasks, to guarantee that the logs will have some time to finish writing, hooked in with atexit lazily +function finish_rest_log_tasks(timeout=5.0) + timer = Timer(timeout) + while any(t -> !istaskdone(t), REST_LOG_TASKS) && isopen(timer) + sleep(0.05) + yield() + filter!(t -> !istaskdone(t), REST_LOG_TASKS) + end + if !isopen(timer) && any(t -> !istaskdone(t), REST_LOG_TASKS) + error("Some REST_LOG_TASKS did not complete! Gave up after $timeout seconds.") + end end # Not relevant, but good to have. -flush(io::REST) = io +Base.flush(io::REST) = io -# We still need to special case the `DefaultHandler` `log` method to call `println(io::REST, level, msg)` -function log(handler::DefaultHandler{F, O}, rec::Record) where {F<:Formatter, O<:REST} - msg = format(handler.fmt, rec) - println(handler.io, rec.level, msg) +# We still need to special case the `DefaultHandler` `emit` method to call `println(io::REST, level, msg)` otherwise we will get an error "REST does not support byte I/O" +function emit(handler::DefaultHandler{F, O}, rec::Record) where {F<:Formatter, O<:REST} + println(handler.io, getlevel(rec), format(handler.fmt, rec)) flush(handler.io) end @@ -152,6 +180,7 @@ push!( ) ) + # Don't forget to update the root logger `Record` type. setrecord!(logger, EC2Record) From 7193cc04cbbd4b4267d855f883a67572d99867a5 Mon Sep 17 00:00:00 2001 From: Peter Hyatt Date: Wed, 6 Nov 2019 15:58:39 -0500 Subject: [PATCH 2/3] Replaced `add_handler` with `push` and dropped the name argument. Replaced `STDOUT` with `stdout`. Replaced `current_module()` with `@__MODULE__`. Replaced `myid()` with `getpid()`. --- docs/quickstart.jl | 4 ++-- docs/src/faq/change-colors.md | 14 ++++++-------- docs/src/faq/json-formatting.md | 5 ++--- docs/src/faq/logging-to-syslog.md | 10 ++++------ docs/src/index.md | 10 +++++----- docs/src/man/loggers.md | 4 ++-- docs/src/man/records.md | 2 +- src/Memento.jl | 2 +- src/records.jl | 2 +- 9 files changed, 24 insertions(+), 29 deletions(-) diff --git a/docs/quickstart.jl b/docs/quickstart.jl index b542915..b5853be 100644 --- a/docs/quickstart.jl +++ b/docs/quickstart.jl @@ -11,6 +11,6 @@ critical(logger, "The world is exploding") child_logger = getlogger("Foo.bar") setlevel!(child_logger, "warn") push!(child_logger, DefaultHandler(tempname(), DefaultFormatter("[{date} | {level} | {name}]: {msg}"))); -debug(child_logger, "Something that should only be printed to STDOUT on the root_logger.") -warn(child_logger, "Warning to STDOUT and the log file.") +debug(child_logger, "Something that should only be printed to stdout on the root_logger.") +warn(child_logger, "Warning to stdout and the log file.") exit() diff --git a/docs/src/faq/change-colors.md b/docs/src/faq/change-colors.md index ef5ed01..09ea6cf 100644 --- a/docs/src/faq/change-colors.md +++ b/docs/src/faq/change-colors.md @@ -3,10 +3,9 @@ Colors can be enabled/disabled and set using via the `is_colorized` and `colors` options to the `DefaultHandler`. ```julia -julia> add_handler(logger, DefaultHandler( - STDOUT, DefaultFormatter(), - Dict{Symbol, Any}(:is_colorized => true)), - "console" +julia> push!(logger, DefaultHandler( + stdout, DefaultFormatter(), + Dict{Symbol, Any}(:is_colorized => true)) ) ``` will create a `DefaultHandler` with colorization. @@ -29,8 +28,8 @@ Dict{AbstractString, Symbol}( However, you can specify custom colors/log levels like so: ```julia -add_handler(logger, DefaultHandler( - STDOUT, DefaultFormatter(), +push!(logger, DefaultHandler( + stdout, DefaultFormatter(), Dict{Symbol, Any}( :colors => Dict{AbstractString, Symbol}( "debug" => :black, @@ -39,8 +38,7 @@ add_handler(logger, DefaultHandler( "error" => :red, "crazy" => :green ) - ), - "console" + ) ) ``` diff --git a/docs/src/faq/json-formatting.md b/docs/src/faq/json-formatting.md index 4b84e1a..896a055 100644 --- a/docs/src/faq/json-formatting.md +++ b/docs/src/faq/json-formatting.md @@ -8,12 +8,11 @@ However, the original behaviour can still be easily achieved by passing in `JSON ```julia using JSON -add_handler( +push!( logger, DefaultHandler( "json-output.log", DictFormatter(JSON.json) - ), - "JSON" + ) ) ``` diff --git a/docs/src/faq/logging-to-syslog.md b/docs/src/faq/logging-to-syslog.md index 1ad6f56..5c6708a 100644 --- a/docs/src/faq/logging-to-syslog.md +++ b/docs/src/faq/logging-to-syslog.md @@ -21,25 +21,23 @@ end Now we can start logging to syslog locally: ```julia -add_handler( +push!( logger, DefaultHandler( Syslog(), DefaultFormatter("{level}: {msg}") - ), - "Syslog" + ) ) ``` We can also log to remote syslog servers via UDP or TCP: ```julia -add_handler( +push!( logger, DefaultHandler( Syslog(ip"123.34.56.78"), DefaultFormatter("{level}: {msg}") - ), - "Syslog" + ) ) ``` diff --git a/docs/src/index.md b/docs/src/index.md index d4d62e2..d87a473 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -65,14 +65,14 @@ julia> push!(child_logger, DefaultHandler(tempname(), DefaultFormatter("[{date} Memento.DefaultHandler{Memento.DefaultFormatter,IOStream}(Memento.DefaultFormatter("[{date} | {level} | {name}]: {msg}"),IOStream(),Dict{Symbol,Any}(Pair{Symbol,Any}(:is_colorized,false))) -julia> debug(child_logger, "Something that should only be printed to STDOUT on the root_logger.") -[debug | Foo.bar]: Something that should only be printed to STDOUT on the root_logger. +julia> debug(child_logger, "Something that should only be printed to stdout on the root_logger.") +[debug | Foo.bar]: Something that should only be printed to stdout on the root_logger. -julia> warn(child_logger, "Warning to STDOUT and the log file.") -[warn | Foo.bar]: Warning to STDOUT and the log file. +julia> warn(child_logger, "Warning to stdout and the log file.") +[warn | Foo.bar]: Warning to stdout and the log file. ``` -NOTE: We used `getlogger("Foo.bar")`, but you can also do `getlogger(current_module())` which allows us to avoid hard coding in logger names. +NOTE: We used `getlogger("Foo.bar")`, but you can also do `getlogger(@__MODULE__)` which allows us to avoid hard coding in logger names. ## Piggybacking onto another package's logger diff --git a/docs/src/man/loggers.md b/docs/src/man/loggers.md index ccfc1e6..65bb048 100644 --- a/docs/src/man/loggers.md +++ b/docs/src/man/loggers.md @@ -1,10 +1,10 @@ # [Loggers](@id man_loggers) A [`Logger`](@ref) is the primary component you use to send formatted log messages to various IO outputs. This type holds information needed to manage the process of creating and storing logs. There is a default "root" logger stored in const `_loggers` inside the `Memento` module. Since Memento implements hierarchical logging you should define child loggers that can be configured independently and better describe the individual components within your code. -To create a new logger for you code it is recommended to do `getlogger(current_module())`. +To create a new logger for you code it is recommended to do `getlogger(@__MODULE__)`. ```julia -julia> logger = getlogger(current_module()) +julia> logger = getlogger(@__MODULE__) ``` Log messages are brought to different output streams by [`Handler`s](@ref man_handlers). From here you can add and remove handlers. To add a handler that writes to rotating log files, simply: diff --git a/docs/src/man/records.md b/docs/src/man/records.md index 0d18e73..866a85d 100644 --- a/docs/src/man/records.md +++ b/docs/src/man/records.md @@ -44,7 +44,7 @@ mutable struct EC2Record <: AttributeRecord Attribute(levelnum), Attribute{AbstractString}(msg), Attribute(name), - Attribute(myid()), + Attribute(getpid()), Attribute{StackFrame}(get_lookup(trace)), trace, Attribute(ENV["INSTANCE_ID"]), diff --git a/src/Memento.jl b/src/Memento.jl index 8e1535e..53d38fa 100644 --- a/src/Memento.jl +++ b/src/Memento.jl @@ -51,7 +51,7 @@ include("memento_test.jl") include("deprecated.jl") # Initializing at compile-time will work as long as the loggers which are added do not -# contain references to STDOUT. +# contain references to stdout. const _loggers = Dict{AbstractString, Logger}( "root" => Logger("root"), ) diff --git a/src/records.jl b/src/records.jl index 89584a8..253f664 100644 --- a/src/records.jl +++ b/src/records.jl @@ -160,7 +160,7 @@ function DefaultRecord(name::AbstractString, level::AbstractString, levelnum::In Attribute(levelnum), Attribute{AbstractString}(msg), Attribute(name), - Attribute(myid()), + Attribute(getpid()), Attribute{Union{StackFrame, Nothing}}(get_lookup(trace)), trace, ) From 2d28e2b7116493a9bdeb3ff875c9bba5526f0814 Mon Sep 17 00:00:00 2001 From: Peter Hyatt Date: Wed, 6 Nov 2019 16:36:58 -0500 Subject: [PATCH 3/3] Fixed contributing.md, added `using Pkg` and `compiled-modules=no` and `Pkg.develop` --- docs/src/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/contributing.md b/docs/src/contributing.md index e6f79f7..ac816fb 100644 --- a/docs/src/contributing.md +++ b/docs/src/contributing.md @@ -6,9 +6,9 @@ Detailed docs on contributing to Julia packages can be found [here](http://docs. To start hacking code or writing docs, simply: -1. `julia> Pkg.add("Memento"); Pkg.checkout("Memento")` +1. `julia> using Pkg; Pkg.develop("Memento")` 2. Make your changes. -3. Test your changes with `julia --compilecache=no -e 'Pkg.test("Memento"; coverage=true)'` +3. Test your changes with `julia --compiled-modules=no -e 'using Pkg; Pkg.test("Memento"; coverage=true)'` 4. Check that your changes haven't reduced the test coverage. From the root Memento package folder run `julia -e 'using Coverage; Coverage.get_summary(process_folder())'`. 5. Make a pull request to [Memento](https://github.com/invenia/Memento.jl) and share your changes with the rest of the community.