diff --git a/src/Pluto.jl b/src/Pluto.jl index 9763436f2..3437e1fea 100644 --- a/src/Pluto.jl +++ b/src/Pluto.jl @@ -36,7 +36,7 @@ import Pkg import Scratch include_dependency("../Project.toml") -const PLUTO_VERSION = VersionNumber(Pkg.TOML.parsefile(joinpath(ROOT_DIR, "Project.toml"))["version"]) +const PLUTO_VERSION = pkgversion(@__MODULE__) const PLUTO_VERSION_STR = "v$(string(PLUTO_VERSION))" const JULIA_VERSION_STR = "v$(string(VERSION))" diff --git a/src/analysis/DependencyCache.jl b/src/analysis/DependencyCache.jl index 7ad9f1db4..8b15f676a 100644 --- a/src/analysis/DependencyCache.jl +++ b/src/analysis/DependencyCache.jl @@ -1,3 +1,5 @@ +import UUIDs: UUID + """ Gets a dictionary of all symbols and the respective cells which are dependent on the given cell. diff --git a/src/analysis/MoreAnalysis.jl b/src/analysis/MoreAnalysis.jl index f402c9190..df673ad2b 100644 --- a/src/analysis/MoreAnalysis.jl +++ b/src/analysis/MoreAnalysis.jl @@ -46,13 +46,22 @@ function _find_bound_variables!(found::Set{Symbol}, expr::Any) end "Return the given cells, and all cells that depend on them (recursively)." -function downstream_recursive(notebook::Notebook, topology::NotebookTopology, from::Union{Vector{Cell},Set{Cell}})::Set{Cell} +function downstream_recursive( + notebook::Notebook, + topology::NotebookTopology, + from::Union{Vector{Cell},Set{Cell}}, +)::Set{Cell} found = Set{Cell}(copy(from)) _downstream_recursive!(found, notebook, topology, from) found end -function _downstream_recursive!(found::Set{Cell}, notebook::Notebook, topology::NotebookTopology, from::Vector{Cell})::Nothing +function _downstream_recursive!( + found::Set{Cell}, + notebook::Notebook, + topology::NotebookTopology, + from::Vector{Cell}, +)::Nothing for cell in from one_down = PlutoDependencyExplorer.where_referenced(topology, cell) for next in one_down @@ -68,13 +77,22 @@ end "Return all cells that are depended upon by any of the given cells." -function upstream_recursive(notebook::Notebook, topology::NotebookTopology, from::Union{Vector{Cell},Set{Cell}})::Set{Cell} +function upstream_recursive( + notebook::Notebook, + topology::NotebookTopology, + from::Union{Vector{Cell},Set{Cell}}, +)::Set{Cell} found = Set{Cell}(copy(from)) _upstream_recursive!(found, notebook, topology, from) found end -function _upstream_recursive!(found::Set{Cell}, notebook::Notebook, topology::NotebookTopology, from::Vector{Cell})::Nothing +function _upstream_recursive!( + found::Set{Cell}, + notebook::Notebook, + topology::NotebookTopology, + from::Vector{Cell}, +)::Nothing for cell in from references = topology.nodes[cell].references for upstream in PlutoDependencyExplorer.where_assigned(topology, references) diff --git a/src/analysis/Parse.jl b/src/analysis/Parse.jl index 5385b0cec..9e43cb3c8 100644 --- a/src/analysis/Parse.jl +++ b/src/analysis/Parse.jl @@ -1,3 +1,5 @@ +# This is how we go from a String of cell code to a Julia `Expr` that can be executed. + import ExpressionExplorer import Markdown diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index ae1d22357..45d48c9dc 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -2,6 +2,7 @@ import REPL: ends_with_semicolon import .Configuration import .Throttled import ExpressionExplorer: is_joined_funcname +import UUIDs: UUID """ Run given cells and all the cells that depend on them, based on the topology information before and after the changes. @@ -507,9 +508,6 @@ function cells_to_disable_to_resolve_multiple_defs(old::NotebookTopology, new::N if length(fellow_assigners_new) > length(fellow_assigners_old) other_definers = setdiff(fellow_assigners_new, (cell,)) - - @debug "Solving multiple defs" cell.cell_id cell_id.(other_definers) disjoint(cells, other_definers) - # we want cell to be the only element of cells that defines this varialbe, i.e. all other definers must have been created previously if disjoint(cells, other_definers) # all fellow cells (including the current cell) should meet some criteria: diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 5af2d4a27..057dc7bfe 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -1,5 +1,8 @@ import UUIDs: UUID, uuid1 +# Hello! 👋 Check out the `Cell` struct. + + const METADATA_DISABLED_KEY = "disabled" const METADATA_SHOW_LOGS_KEY = "show_logs" const METADATA_SKIP_AS_SCRIPT_KEY = "skip_as_script" diff --git a/src/notebook/Export.jl b/src/notebook/Export.jl index fa9cd5d02..0ccf003a3 100644 --- a/src/notebook/Export.jl +++ b/src/notebook/Export.jl @@ -94,6 +94,8 @@ function prefetch_statefile_html(statefile_js::AbstractString) end """ +This function takes the `editor.html` file from Pluto's source code, and uses string replacements to insert custom data. By inserting a statefile (and more), you can create an HTML file that will display a notebook when opened: this is how the Static HTML export works. + See [PlutoSliderServer.jl](https://github.com/JuliaPluto/PlutoSliderServer.jl) if you are interested in exporting notebooks programatically. """ function generate_html(; diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index c88145182..6373cf3ce 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -1,8 +1,9 @@ +# The `Notebook` struct! + import UUIDs: UUID, uuid1 import .Configuration import .PkgCompat: PkgCompat, PkgContext import Pkg -import TOML import .Status const DEFAULT_NOTEBOOK_METADATA = Dict{String, Any}() @@ -22,7 +23,11 @@ const ProcessStatus = ( waiting_for_permission="waiting_for_permission", ) -"Like a [`Diary`](@ref) but more serious. 📓" +""" +A Pluto notebook, yay! 📓 + +This mutable struct is a notebook session. It contains the information loaded from the `.jl` file, the cell outputs, package information, execution metadata and more. +""" Base.@kwdef mutable struct Notebook "Cells are ordered in a `Notebook`, and this order can be changed by the user. Cells will always have a constant UUID." cells_dict::Dict{UUID,Cell} @@ -111,8 +116,10 @@ end Notebook(cells::Vector{Cell}, path::AbstractString=numbered_until_new(joinpath(new_notebooks_directory(), cutename()))) = Notebook(cells, path, uuid1()) function Base.getproperty(notebook::Notebook, property::Symbol) + # This is so that you can do notebook.cells to get all cells as a vector. if property == :cells _collect_cells(notebook.cells_dict, notebook.cell_order) + # This is for Firebasey I think elseif property == :cell_inputs notebook.cells_dict else @@ -120,6 +127,7 @@ function Base.getproperty(notebook::Notebook, property::Symbol) end end +# New method for this function with a `Notebook` as input. function PlutoDependencyExplorer.topological_order(notebook::Notebook) cached = notebook._cached_topological_order if cached === nothing || cached.input_topology !== notebook.topology diff --git a/src/notebook/path helpers.jl b/src/notebook/path helpers.jl index 05093a807..c0d6649de 100644 --- a/src/notebook/path helpers.jl +++ b/src/notebook/path helpers.jl @@ -190,7 +190,7 @@ end const tamepath = abspath ∘ tryexpanduser "Block until reading the file two times in a row gave the same result." -function wait_until_file_unchanged(filename::String, timeout::Real, last_contents::String="")::Nothing +function wait_until_file_unchanged(filename::String, timeout::Real, last_contents::String="-=-=-=-")::Nothing new_contents = try read(filename, String) catch diff --git a/src/notebook/saving and loading.jl b/src/notebook/saving and loading.jl index cdac82902..45a7966a9 100644 --- a/src/notebook/saving and loading.jl +++ b/src/notebook/saving and loading.jl @@ -1,4 +1,5 @@ - +import TOML +import UUIDs: UUID const _notebook_header = "### A Pluto.jl notebook ###" const _notebook_metadata_prefix = "#> " @@ -43,7 +44,8 @@ function save_notebook(io::IO, notebook::Notebook) end end - # Anything between the version string and the first UUID delimiter will be ignored by the notebook loader. + # (Anything between the version string and the first UUID delimiter will be ignored by the notebook loader.) + # We insert these two imports because they are also imported by default in the Pluto session. You might use these packages in your code, so we add the imports to the file, so the file can run as a script. println(io, "") println(io, "using Markdown") println(io, "using InteractiveUtils") @@ -78,15 +80,18 @@ function save_notebook(io::IO, notebook::Notebook) end end end + + # Do one little string replacement to make it impossible to use the Pluto cell delimiter inside of actual cell code. If this would happen, then the notebook file cannot load correctly. So we just remove it from your code (sorry!) + current_code = replace(c.code, _cell_id_delimiter => "# ") if must_be_commented_in_file(c) print(io, _disabled_prefix) - print(io, replace(c.code, _cell_id_delimiter => "# ")) + print(io, current_code) print(io, _disabled_suffix) print(io, _cell_suffix) else # write the cell code and prevent collisions with the cell delimiter - print(io, replace(c.code, _cell_id_delimiter => "# ")) + print(io, current_code) print(io, _cell_suffix) end end @@ -155,7 +160,7 @@ save_notebook(notebook::Notebook) = save_notebook(notebook, notebook.path) # LOADING ### -function _notebook_metadata!(@nospecialize(io::IO)) +function _read_notebook_metadata!(@nospecialize(io::IO)) firstline = String(readline(io))::String if firstline != _notebook_header @@ -192,7 +197,7 @@ function _notebook_metadata!(@nospecialize(io::IO)) return notebook_metadata end -function _notebook_collected_cells!(@nospecialize(io::IO)) +function _read_notebook_collected_cells!(@nospecialize(io::IO)) collected_cells = Dict{UUID,Cell}() while !eof(io) cell_id_str = String(readline(io)) @@ -239,7 +244,7 @@ function _notebook_collected_cells!(@nospecialize(io::IO)) return collected_cells end -function _notebook_cell_order!(@nospecialize(io::IO), collected_cells) +function _read_notebook_cell_order!(@nospecialize(io::IO), collected_cells) cell_order = UUID[] while !eof(io) cell_id_str = String(readline(io)) @@ -259,7 +264,7 @@ function _notebook_cell_order!(@nospecialize(io::IO), collected_cells) return cell_order end -function _notebook_nbpkg_ctx(cell_order::Vector{UUID}, collected_cells::Dict{Base.UUID, Cell}) +function _read_notebook_nbpkg_ctx(cell_order::Vector{UUID}, collected_cells::Dict{Base.UUID, Cell}) read_package = _ptoml_cell_id ∈ cell_order && _mtoml_cell_id ∈ cell_order && @@ -295,7 +300,7 @@ function _notebook_nbpkg_ctx(cell_order::Vector{UUID}, collected_cells::Dict{Bas return nbpkg_ctx end -function _notebook_appeared_order!(cell_order::Vector{UUID}, collected_cells::Dict{Base.UUID, Cell}) +function _read_notebook_appeared_order!(cell_order::Vector{UUID}, collected_cells::Dict{Base.UUID, Cell}) setdiff!( union!( # don't include cells that only appear in the order, but no code was given @@ -310,12 +315,12 @@ end "Load a notebook without saving it or creating a backup; returns a `Notebook`. REMEMBER TO CHANGE THE NOTEBOOK PATH after loading it to prevent it from autosaving and overwriting the original file." function load_notebook_nobackup(@nospecialize(io::IO), @nospecialize(path::AbstractString))::Notebook - notebook_metadata = _notebook_metadata!(io) + notebook_metadata = _read_notebook_metadata!(io) + collected_cells = _read_notebook_collected_cells!(io) + cell_order = _read_notebook_cell_order!(io, collected_cells) + nbpkg_ctx = _read_notebook_nbpkg_ctx(cell_order, collected_cells) + appeared_order = _read_notebook_appeared_order!(cell_order, collected_cells) - collected_cells = _notebook_collected_cells!(io) - cell_order = _notebook_cell_order!(io, collected_cells) - nbpkg_ctx = _notebook_nbpkg_ctx(cell_order, collected_cells) - appeared_order = _notebook_appeared_order!(cell_order, collected_cells) appeared_cells_dict = filter(collected_cells) do (k, v) k ∈ appeared_order end @@ -336,16 +341,14 @@ end # UTILS function load_notebook_nobackup(path::String)::Notebook - local loaded open(path, "r") do io - loaded = load_notebook_nobackup(io, path) + load_notebook_nobackup(io, path) end - loaded end # BACKUPS -"Create a backup of the given file, load the file as a .jl Pluto notebook, save the loaded notebook, compare the two files, and delete the backup of the newly saved file is equal to the backup." +"Create a backup of the given file, load the file as a .jl Pluto notebook, save the loaded notebook, compare the two files, and delete the backup of the newly saved file is mostly equal to the backup." function load_notebook(path::String; disable_writing_notebook_files::Bool=false)::Notebook backup_path = backup_filename(path) # local backup_num = 1 diff --git a/src/packages/Packages.jl b/src/packages/Packages.jl index 76ceeadba..da79c0fee 100644 --- a/src/packages/Packages.jl +++ b/src/packages/Packages.jl @@ -485,9 +485,16 @@ function with_auto_fixes(f::Function, notebook::Notebook) @info "Operation failed. Updating registries and trying again..." exception=e PkgCompat.update_registries(; force=true) + + # TODO: check for resolver errors around stdlibs and fix them by doing `up Statistics` + + + + try f() catch e + # this is identical to Pkg.update, right? @warn "Operation failed. Removing Manifest and trying again..." exception=e reset_nbpkg!(notebook; keep_project=true, save=false, backup=false) diff --git a/src/runner/Loader.jl b/src/runner/Loader.jl index d1f1094ed..05b62464c 100644 --- a/src/runner/Loader.jl +++ b/src/runner/Loader.jl @@ -1,4 +1,4 @@ -# The goal of this file is to import PlutoRunner into Main. +# The goal of this file is to import PlutoRunner into Main, on the process of the notebook (created by Malt.jl). # # This is difficult because PlutoRunner uses standard libraries and packages that are not necessarily available in the standard environment. # diff --git a/src/runner/PlutoRunner/src/bonds.jl b/src/runner/PlutoRunner/src/bonds.jl index 3bf792a6c..c74e0081e 100644 --- a/src/runner/PlutoRunner/src/bonds.jl +++ b/src/runner/PlutoRunner/src/bonds.jl @@ -1,5 +1,5 @@ import Base64 - +import UUIDs: UUID const registered_bond_elements = Dict{Symbol, Any}() diff --git a/src/webserver/Dynamic.jl b/src/webserver/Dynamic.jl index 5960c0ff7..b347e1629 100644 --- a/src/webserver/Dynamic.jl +++ b/src/webserver/Dynamic.jl @@ -427,7 +427,6 @@ function _set_cells_to_queued_in_local_state(client, notebook, cells) if haskey(results, cell.cell_id) old = results[cell.cell_id]["queued"] results[cell.cell_id]["queued"] = true - @debug "Setting val!" cell.cell_id old end end end diff --git a/src/webserver/Static.jl b/src/webserver/Static.jl index c74d04561..a70ce0770 100644 --- a/src/webserver/Static.jl +++ b/src/webserver/Static.jl @@ -1,6 +1,5 @@ import HTTP import Markdown: htmlesc -import UUIDs: UUID import Pkg import MIMEs @@ -24,7 +23,7 @@ const day = let day = 24hour end -function default_404(req = nothing) +function default_404_response(req = nothing) HTTP.Response(404, "Not found!") end @@ -41,7 +40,7 @@ function asset_response(path; cacheable::Bool=false) cacheable && HTTP.setheader(response, "Cache-Control" => "public, max-age=$(30day), immutable") response else - default_404() + default_404_response() end end @@ -73,19 +72,29 @@ function notebook_response(notebook; home_url="./", as_redirect=true) end end -const found_is_pluto_dev = Ref{Union{Nothing,Bool}}(nothing) +const found_is_pluto_dev = Ref{Bool}() +""" +Is the Pluto package `dev`ed? Returns `false` for normal Pluto installation from the registry. +""" function is_pluto_dev() - if found_is_pluto_dev[] !== nothing + if isassigned(found_is_pluto_dev) return found_is_pluto_dev[] end + found_is_pluto_dev[] = try - deps = Pkg.dependencies() + # is the package located in .julia/packages ? + if startswith(pkgdir(@__MODULE__), joinpath(get(DEPOT_PATH, 1, "zzz"), "packages")) + false + else + deps = Pkg.dependencies() - p_index = findfirst(p -> p.name == "Pluto", deps) - p = deps[p_index] + p_index = findfirst(p -> p.name == "Pluto", deps) + p = deps[p_index] - p.is_tracking_path - catch + p.is_tracking_path + end + catch e + @debug "is_pluto_dev failed" e false end end diff --git a/src/webserver/Status.jl b/src/webserver/Status.jl index 72bfc023c..bb99a6d54 100644 --- a/src/webserver/Status.jl +++ b/src/webserver/Status.jl @@ -1,3 +1,11 @@ +""" +This module contains the "Status" system from Pluto, which you can see in the bottom right in the Pluto editor. It's used to track what is currently happening, for how long. (E.g. "Notebook startup > Julia process starting".) + +The Status system is hierachical: a status item can have multiple subtasks. E.g. the "Package manager" status can have subtask "instantiate" and "precompile". In the UI, these are sections that you can fold out. + +!!! warning + This module is not public API of Pluto. +""" module Status _default_update_listener() = nothing diff --git a/src/webserver/WebServer.jl b/src/webserver/WebServer.jl index 6c96f621c..84688e0c6 100644 --- a/src/webserver/WebServer.jl +++ b/src/webserver/WebServer.jl @@ -184,7 +184,7 @@ function run!(session::ServerSession) end server = HTTP.listen!(hostIP, port; stream=true, server=serversocket, on_shutdown, verbose=-1) do http::HTTP.Stream - # messy messy code so that we can use the websocket on the same port as the HTTP server + # the if statement below asks if the current request is a "websocket upgrade" request: the start of a websocket connection. if HTTP.WebSockets.isupgrade(http.message) secret_required = let s = session.options.security @@ -202,12 +202,14 @@ function run!(session::ServerSession) end if !secret_required || is_authenticated(session, http.message) try + # "upgrade" means accept and start the websocket connection that the client requested HTTP.WebSockets.upgrade(http) do clientstream if HTTP.WebSockets.isclosed(clientstream) return end found_client_id_ref = Ref(Symbol(:none)) try + # the loop below will keep running for this websocket connection, it iterates over all incoming websocket messages. for message in clientstream # This stream contains data received over the WebSocket. # It is formatted and MsgPack-encoded by send(...) in PlutoConnection.js @@ -216,6 +218,7 @@ function run!(session::ServerSession) try parentbody = unpack(message) + # for debug only let lag = session.options.server.simulated_lag (lag > 0) && sleep(lag * (0.5 + rand())) # sleep(0) would yield to the process manager which we dont want @@ -281,12 +284,16 @@ function run!(session::ServerSession) # HTTP.closeread(http) # If a "token" url parameter is passed in from binder, then we store it to add to every URL (so that you can share the URL to collaborate). - params = HTTP.queryparams(HTTP.URI(request.target)) - if haskey(params, "token") && params["token"] ∉ ("null", "undefined", "") && session.binder_token === nothing - session.binder_token = params["token"] + let + params = HTTP.queryparams(HTTP.URI(request.target)) + if haskey(params, "token") && params["token"] ∉ ("null", "undefined", "") && session.binder_token === nothing + session.binder_token = params["token"] + end end + ### response_body = app(request) + ### request.response::HTTP.Response = response_body request.response.request = request @@ -310,7 +317,7 @@ function run!(session::ServerSession) server_running() = try - HTTP.get("http://$(hostIP):$(port)$(session.options.server.base_url)ping"; status_exception = false, retry = false, connect_timeout = 10, readtimeout = 10).status == 200 + HTTP.get("http://$(hostIP):$(port)$(session.options.server.base_url)ping"; status_exception=false, retry=false, connect_timeout=10, readtimeout=10).status == 200 catch false end @@ -328,7 +335,7 @@ function run!(session::ServerSession) # Trigger ServerStartEvent with server details try_event_call(session, ServerStartEvent(address, port)) - if PLUTO_VERSION >= v"0.17.6" && frontend_directory() == "frontend" + if frontend_directory() == "frontend" @info("It looks like you are developing the Pluto package, using the unbundled frontend...") end @@ -385,7 +392,11 @@ function pretty_address(session::ServerSession, hostIP, port) string(HTTP.URI(HTTP.URI(new_root); query = url_params)) end -"All messages sent over the WebSocket get decoded+deserialized and end up here." +""" +All messages sent over the WebSocket from the client get decoded+deserialized and end up here. + +It calls one of the functions from the `responses` Dict, see the file Dynamic.jl. +""" function process_ws_message(session::ServerSession, parentbody::Dict, clientstream) client_id = Symbol(parentbody["client_id"]) client = get!(session.connected_clients, client_id) do