More embedded Pkg API (#2868)

src/Pluto.jl

@@ -88,9 +88,11 @@ include("./webserver/WebServer.jl")
 const reset_notebook_environment = PkgUtils.reset_notebook_environment
 const update_notebook_environment = PkgUtils.update_notebook_environment
 const activate_notebook_environment = PkgUtils.activate_notebook_environment
+const will_use_pluto_pkg = PkgUtils.will_use_pluto_pkg
 export reset_notebook_environment
 export update_notebook_environment
 export activate_notebook_environment
+export will_use_pluto_pkg
 
 include("./precompile.jl")
 

src/packages/PkgUtils.jl

@@ -10,15 +10,13 @@ using Markdown
 
 export activate_notebook
 
-ensure_has_nbpkg(notebook::Notebook) = if notebook.nbpkg_ctx === nothing
+ensure_has_nbpkg(notebook::Notebook) = if !will_use_pluto_pkg(notebook)
 
     # TODO: update_save the notebook to init packages and stuff?
     error("""
-    This notebook is not using Pluto's package manager. This means that either:
-    1. The notebook contains Pkg.activate or Pkg.add calls, or
-    2. The notebook was created before Pluto 0.15.
+    This notebook is not using Pluto's package manager. This means that the notebook contains Pkg.activate or Pkg.add call.
 
-    Open the notebook using Pluto to get started.
+    Open the notebook using Pluto to see what's up.
     """)
 else
     for f in [notebook |> project_file, notebook |> manifest_file]
@@ -81,7 +79,7 @@ nb_and_dir_environments_equal(notebook_path::String, dir::String) = nb_and_dir_e
 reset_notebook_environment(notebook_path::String; keep_project::Bool=false, backup::Bool=true)
 ```
 
-Remove the embedded `Project.toml` and `Manifest.toml` from a notebook file, modifying the file. If `keep_project` is true, only `Manifest.toml` will be deleted. A backup of the notebook file is created by default.
+Remove the embedded `Project.toml` and `Manifest.toml` from a notebook file, modifying the notebook file. If `keep_project` is true, only `Manifest.toml` will be deleted. A backup of the notebook file is created by default.
 """
 function reset_notebook_environment(path::String; kwargs...)
     Pluto.reset_nbpkg!(
@@ -92,10 +90,10 @@ end
 
 """
 ```julia
-reset_notebook_environment(notebook_path::String; backup::Bool=true, level::Pkg.UpgradeLevel=Pkg.UPLEVEL_MAJOR)
+update_notebook_environment(notebook_path::String; backup::Bool=true, level::Pkg.UpgradeLevel=Pkg.UPLEVEL_MAJOR)
 ```
 
-Update the embedded `Project.toml` and `Manifest.toml` in a notebook file, modifying the file. A [`Pkg.UpgradeLevel`](@ref) can be passed to the `level` keyword argument. A backup file is created by default. 
+Call `Pkg.update` in the package environment embedded in a notebook file, modifying the notebook file. A [`Pkg.UpgradeLevel`](@ref) can be passed to the `level` keyword argument. A backup file is created by default. 
 """
 function update_notebook_environment(path::String; kwargs...)
     Pluto.update_nbpkg(
@@ -105,9 +103,45 @@ function update_notebook_environment(path::String; kwargs...)
     )
 end
 
+"""
+```julia
+will_use_pluto_pkg(notebook_path::String)::Bool
+```
+
+Will this notebook use the Pluto package manager? `false` means that the notebook contains `Pkg.activate` or another deactivator.
+"""
+will_use_pluto_pkg(path::String) = will_use_pluto_pkg(load_notebook_nobackup(path))
+function will_use_pluto_pkg(notebook::Notebook)
+    ctx = notebook.nbpkg_ctx
+    # if one of the two files is not empty:
+    if ctx !== nothing && !isempty(PkgCompat.read_project_file(ctx)) || !isempty(PkgCompat.read_manifest_file(ctx))
+        return true
+    end
+
+    # otherwise, check for Pkg.activate:
+    # when nbpkg_ctx is defined but the files are empty: check if the notebook would use one (i.e. that Pkg.activate is not used).
+    topology = Pluto.updated_topology(notebook.topology, notebook, notebook.cells)
+    return Pluto.use_plutopkg(topology)
+end
+
+"""
+```julia
+activate_notebook_environment(notebook_path::String; show_help::Bool=true)::Nothing
+```
+
+Activate the package environment embedded in a notebook file, for interactive use. This will allow you to use the Pkg REPL and Pkg commands to modify the environment, and any changes you make will be automatically saved in the notebook file.
+
+More help will be displayed if `show_help` is `true`.
+
+Limitations:
+- Shut down the notebook before using this functionality.
+- Non-interactive use is limited, use the functional form instead, or insert `sleep` calls after modifying the environment.
 
-function activate_notebook_environment(path::String)
-    notebook_ref = Ref(load_notebook(path))
+!!! info
+    This functionality works using file watching. A dummy repository contains a copy of the embedded tomls and gets activated, and the notebook file is updated when the dummy repository changes.
+"""
+function activate_notebook_environment(path::String; show_help::Bool=true)
+    notebook_ref = Ref(load_notebook_nobackup(path))
 
     ensure_has_nbpkg(notebook_ref[])
 
@@ -124,7 +158,7 @@ function activate_notebook_environment(path::String)
                 if !nb_and_dir_environments_equal(notebook_ref[], ourpath)
                     write_dir_to_nb(ourpath, notebook_ref[])
                     println()
-                    @info "Saved notebook package environment ✓"
+                    @info "Notebook file updated ✓"
                     println()
                 end
             end
@@ -137,7 +171,7 @@ function activate_notebook_environment(path::String)
                 if !nb_and_dir_environments_equal(notebook_ref[], ourpath)
                     write_nb_to_dir(notebook_ref[], ourpath)
                     println()
-                    @info "New notebook package environment written to directory ✓"
+                    @info "REPL environment updated from notebook ✓"
                     println()
                 end
             end
@@ -199,39 +233,82 @@ function activate_notebook_environment(path::String)
     #     end
     # end
 
-    println()
-    """
+    if show_help
+        println()
+        """
+        
+        > Notebook environment activated!
+
+        ## Step 1.
+        _Press `]` to open the Pkg REPL._
+        
+        The notebook environment is currently active.
+        
+        ## Step 2. 
+        The notebook file and your REPL environment are now synced. This means that:
+        1. Any changes you make in the REPL environment will be written to the notebook file. For example, you can `pkg> update` or `pkg> add SomePackage`, and the notebook file will update.
+        2. Whenever the notebook file changes, the REPL environment will be updated from the notebook file.
+
+        ## Step 3.
+        When you are done, you can exit the notebook environment by deactivating it:
+
+        ```
+        pkg> activate
+        ```
+        """ |> Markdown.parse |> display
+        println()
+    end
 
-    > Notebook environment activated!
+    nothing
+end
 
-    ## Step 1.
-    _Press `]` to open the Pkg REPL._
-    
-    The notebook environment is currently active.
-    
-    ## Step 2. 
-    The notebook file and your REPL environment are now synced. This means that:
-    1. Any changes you make in the REPL environment will be written to the notebook file. For example, you can `pkg> update` or `pkg> add SomePackage`, and the notebook file will update.
-    2. Whenever the notebook file changes, the REPL environment will be updated from the notebook file.
 
-    ## Step 3.
-    When you are done, you can exit the notebook environment by deactivating it:
 
-    ```
-    pkg> activate
-    ```
-    """ |> Markdown.parse |> display
-    println()
+"""
+```julia
+activate_notebook_environment(f::Function, notebook_path::String)
+```
+
+Temporarily activate the package environment embedded in a notebook file, for use inside scripts. Inside your function `f`, you can use Pkg commands to modify the environment, and any changes you make will be automatically saved in the notebook file after your function finishes. Not thread-safe.
+
+This method is best for scripts that update notebook files. For interactive use, the method `activate_notebook_environment(notebook_path::String)` is recommended.
 
+# Example
+
+```julia
+Pluto.activate_notebook_environment("notebook.jl") do
+    Pkg.add("Example")
+end
+
+# Now the file "notebook.jl" was updated!
+```
 
+!!! warning
+    This function uses the private method `Pkg.activate(f::Function, path::String)`. This API might not be available in future Julia versions. 🤷
+"""
+function activate_notebook_environment(f::Function, path::String)
+    notebook = load_notebook_nobackup(path)
+    ensure_has_nbpkg(notebook)
+
+    ourpath = joinpath(mktempdir(), basename(path))
+    mkpath(ourpath)
+    write_nb_to_dir(notebook, ourpath)
+
+    result = Pkg.activate(f, ourpath)
+
+    if !nb_and_dir_environments_equal(notebook, ourpath)
+        write_dir_to_nb(ourpath, notebook)
+    end
+
+    result
 end
 
 const activate_notebook = activate_notebook_environment
 
-function testnb()
+function testnb(name="simple_stdlib_import.jl")
     t = tempname()
 
-    readwrite(Pluto.project_relative_path("test","packages","nb.jl"), t)
+    readwrite(Pluto.project_relative_path("test", "packages", name), t)
     t
 end
 

test/packages/PkgUtils.jl

@@ -0,0 +1,158 @@
+
+
+
+
+using Pluto
+
+
+without_pluto_version(s) = replace(s, r"# v.*" => "")
+
+
+@testset "PkgUtils" begin
+
+    @testset "activate_notebook_environment" begin
+        file = Pluto.PkgUtils.testnb()
+        before = without_pluto_version(read(file, String))
+        @assert :activate_notebook_environment in names(Pluto)
+
+        @test !occursin("Artifacts", Pluto.PkgCompat.read_project_file(Pluto.load_notebook_nobackup(file)))
+
+        ap_before = Base.ACTIVE_PROJECT[]
+
+
+        ###
+
+        Pluto.activate_notebook_environment(file; show_help=false)
+        @test Base.ACTIVE_PROJECT[] != ap_before
+
+        @test sort(collect(keys(Pkg.project().dependencies))) == ["Dates"]
+
+        Pkg.add("Artifacts")
+        @test sort(collect(keys(Pkg.project().dependencies))) == ["Artifacts", "Dates"]
+
+
+        ### EXIT, activate another env and wait for our previous changes to get picked up
+        Base.ACTIVE_PROJECT[] = ap_before
+        sleep(5)
+
+
+        ###
+        # get embedded project.toml from notebook:
+        @test occursin("Artifacts", Pluto.PkgCompat.read_project_file(Pluto.load_notebook_nobackup(file)))
+
+        after = without_pluto_version(read(file, String))
+        @test before != after
+        @test occursin("Artifacts", after)
+    end
+
+
+
+
+    @testset "activate_notebook_environment, functional 1" begin
+        file = Pluto.PkgUtils.testnb()
+        before = read(file, String)
+        @assert :activate_notebook_environment in names(Pluto)
+
+        ###
+        projs = Pluto.activate_notebook_environment(file) do
+            Pkg.project()
+        end
+
+        after = read(file, String)
+
+        @test projs !== nothing
+        @test sort(collect(keys(projs.dependencies))) == ["Dates"]
+
+        @test before == after
+    end
+
+
+
+
+    @testset "activate_notebook_environment, functional 2" begin
+        file = Pluto.PkgUtils.testnb()
+        before = without_pluto_version(read(file, String))
+        @assert :activate_notebook_environment in names(Pluto)
+
+        @test !occursin("Artifacts", Pluto.PkgCompat.read_project_file(Pluto.load_notebook_nobackup(file)))
+        @test !occursin("Artifacts", before)
+
+
+        ###
+        projs = Pluto.activate_notebook_environment(file) do
+            Pkg.add("Artifacts")
+        end
+
+        after = without_pluto_version(read(file, String))
+
+        @test occursin("Artifacts", Pluto.PkgCompat.read_project_file(Pluto.load_notebook_nobackup(file)))
+
+        @test before != after
+        @test occursin("Artifacts", after)
+    end
+
+
+
+
+    @testset "reset_notebook_environment" begin
+        file = Pluto.PkgUtils.testnb()
+        before = without_pluto_version(read(file, String))
+        @assert :reset_notebook_environment in names(Pluto)
+
+        # project.toml fake cell id
+        @test occursin("00001", before)
+        @test occursin("[deps]", before)
+
+
+        ###
+        Pluto.reset_notebook_environment(file; backup=false)
+
+
+        after = without_pluto_version(read(file, String))
+
+        @test before != after
+        # project.toml fake cell id
+        @test !occursin("00001", after)
+        @test !occursin("[deps]", after)
+
+    end
+
+
+
+
+    # TODO: too lazy to get a notebook with updatable package so just running the function and checking for errors
+    @testset "update_notebook_environment" begin
+        file = Pluto.PkgUtils.testnb()
+        before = without_pluto_version(read(file, String))
+        @assert :update_notebook_environment in names(Pluto)
+
+        ###
+        Pluto.update_notebook_environment(file)
+
+
+        # whatever
+        after = without_pluto_version(read(file, String))
+        @test occursin("[deps]", after)
+    end
+
+
+
+    @testset "will_use_pluto_pkg" begin
+        file = Pluto.PkgUtils.testnb()
+        before = read(file, String)
+        @assert :will_use_pluto_pkg in names(Pluto)
+
+        ###
+        @test Pluto.will_use_pluto_pkg(file)
+
+
+        after = read(file, String)
+        @test before == after
+
+
+        file2 = Pluto.PkgUtils.testnb("pkg_cell.jl")
+        @test !Pluto.will_use_pluto_pkg(file2)
+    end
+end
+
+

test/runtests.jl

@@ -38,6 +38,7 @@ verify_no_running_processes()
 # tests that don't start new processes:
 @timeit_include("ReloadFromFile.jl")
 @timeit_include("packages/PkgCompat.jl")
+@timeit_include("packages/PkgUtils.jl")
 @timeit_include("MethodSignatures.jl")
 @timeit_include("MoreAnalysis.jl")
 @timeit_include("is_just_text.jl")