diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8dc601..d8cbd48 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: matrix: version: - '1' - - '1.6' + - '1.9' os: - ubuntu-latest - macOS-latest @@ -40,15 +40,23 @@ jobs: os: ubuntu-latest arch: x64 backend: 'Null' + - version: '1' + os: ubuntu-latest + arch: x64 + backend: SystemPixi steps: - uses: actions/checkout@v3 + - uses: prefix-dev/setup-pixi@v0 + if: ${{ matrix.backend == 'SystemPixi' }} + with: + run-install: false - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/cache@v1 - uses: julia-actions/julia-downgrade-compat@v1 - if: ${{ matrix.version == '1.6' }} + if: ${{ matrix.version == '1.9' }} with: skip: Markdown,Pkg,TOML,Aqua,Test,TestItemRunner,OpenSSL_jll - uses: julia-actions/julia-buildpkg@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index bf90e6a..849e1b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## Unreleased +* Add `SystemPixi` backend to allow using [Pixi](https://pixi.sh/latest/) to install packages. + ## 0.2.24 (2024-11-08) * Add `pip_backend` preference to choose between `pip` and `uv`. * Add `libstdcxx_ng_version` preference to override automatic version bounds. diff --git a/Project.toml b/Project.toml index 5dcf576..e80c70c 100644 --- a/Project.toml +++ b/Project.toml @@ -15,16 +15,16 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [compat] Aqua = "0 - 999" JSON3 = "1.9" -Markdown = "1.6" +Markdown = "1" MicroMamba = "0.1.4" OpenSSL_jll = "0 - 999" Pidfile = "1.3" -Pkg = "1.6" +Pkg = "1.9" Preferences = "1.3" Test = "1" TestItemRunner = "0 - 999" TOML = "1" -julia = "1.6" +julia = "1.9" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/README.md b/README.md index cb34443..b23ac5c 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ in more detail. | Preference | Environment variable | Description | | ---------- | -------------------- | ----------- | -| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `System`, `Current` or `Null` | +| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `System`, `Current`, `SystemPixi` or `Null` | | `exe` | `JULIA_CONDAPKG_EXE` | Path to the Conda executable. | | `offline` | `JULIA_CONDAPKG_OFFLINE` | When `true`, work in offline mode. | | `env` | `JULIA_CONDAPKG_ENV` | Path to the Conda environment to use. | @@ -184,6 +184,8 @@ by setting the `backend` preference to one of the following values: [MicroMamba.jl](https://github.com/JuliaPy/MicroMamba.jl). - `System`: Use a pre-installed Conda. If the `exe` preference is set, that is used. Otherwise we look for `conda`, `mamba` or `micromamba` in your `PATH`. +- `SystemPixi`: Use a pre-installed [Pixi](https://pixi.sh). If the `exe` preference + is set, that is used. Otherwise we look for `pixi` in your `PATH`. - `Current`: Use the currently activated Conda environment instead of creating a new one. This backend will only ever install packages, never uninstall. The Conda executable used is the same as for the System backend. Similar to the default behaviour of diff --git a/src/CondaPkg.jl b/src/CondaPkg.jl index dc5f60c..da0d1f4 100644 --- a/src/CondaPkg.jl +++ b/src/CondaPkg.jl @@ -29,6 +29,7 @@ end # backend backend::Symbol = :NotSet condaexe::String = "" + pixiexe::String = "" # resolve resolved::Bool = false load_path::Vector{String} = String[] diff --git a/src/PkgREPL.jl b/src/PkgREPL.jl index 2c4e621..3c88b07 100644 --- a/src/PkgREPL.jl +++ b/src/PkgREPL.jl @@ -338,8 +338,11 @@ const gc_spec = Pkg.REPLMode.CommandSpec( function run(args) try CondaPkg.withenv() do - if args[1] == "conda" + b = CondaPkg.backend() + if b in CondaPkg.CONDA_BACKENDS && args[1] == "conda" Base.run(CondaPkg.conda_cmd(args[2:end])) + elseif b in CondaPkg.PIXI_BACKENDS && args[1] == "pixi" + Base.run(CondaPkg.pixi_cmd(args[2:end])) else Base.run(Cmd(args)) end diff --git a/src/backend.jl b/src/backend.jl index 1723aff..82ac72e 100644 --- a/src/backend.jl +++ b/src/backend.jl @@ -1,3 +1,12 @@ +"""All valid backends.""" +const ALL_BACKENDS = (:MicroMamba, :Null, :System, :Current, :SystemPixi) + +"""All backends that use a Conda/Mamba installer.""" +const CONDA_BACKENDS = (:MicroMamba, :System, :Current) + +"""All backends that use a Pixi installer.""" +const PIXI_BACKENDS = (:SystemPixi,) + function backend() if STATE.backend == :NotSet backend = getpref(String, "backend", "JULIA_CONDAPKG_BACKEND", "") @@ -27,10 +36,23 @@ function backend() error("not an executable: $exe") end end + elseif backend == "SystemPixi" + ok = false + exe2 = Sys.which(exe == "" ? "pixi" : exe) + if exe2 === nothing + if exe == "" + error("could not find a pixi executable") + else + error("not an executable: $exe") + end + end + STATE.backend = :SystemPixi + STATE.pixiexe = exe2 else error("invalid backend: $backend") end end + @assert STATE.backend in ALL_BACKENDS STATE.backend end @@ -38,13 +60,20 @@ function conda_cmd(args = ``; io::IO = stderr) b = backend() if b == :MicroMamba MicroMamba.cmd(args, io = io) - elseif b in (:System, :Current) + elseif b in CONDA_BACKENDS + STATE.condaexe == "" && error("this is a bug") `$(STATE.condaexe) $args` - elseif b == :Null - error( - "Can not run conda command when backend is Null. Manage conda actions outside of julia.", - ) else - @assert false + error("Cannot run conda when backend is $b.") + end +end + +function pixi_cmd(args = ``; io::IO = stderr) + b = backend() + if b in PIXI_BACKENDS + STATE.pixiexe == "" && error("this is a bug") + `$(STATE.pixiexe) $args` + else + error("Cannot run pixi when backend is $b.") end end diff --git a/src/deps.jl b/src/deps.jl index 9ad4d38..8e84e4a 100644 --- a/src/deps.jl +++ b/src/deps.jl @@ -114,15 +114,31 @@ function read_parsed_deps(file) end function current_packages() - cmd = conda_cmd(`list -p $(envdir()) --json`) - pkglist = JSON3.read(cmd) + b = backend() + if b in CONDA_BACKENDS + cmd = conda_cmd(`list -p $(envdir()) --json`) + pkglist = JSON3.read(cmd) + elseif b in PIXI_BACKENDS + cmd = + pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`) + pkglist = JSON3.read(cmd) + pkglist = [pkg for pkg in pkglist if pkg.kind == "conda"] + end Dict(normalise_pkg(pkg.name) => pkg for pkg in pkglist) end function current_pip_packages() - pkglist = withenv() do - cmd = `$(which("pip")) list --format=json` - JSON3.read(cmd) + b = backend() + if b in CONDA_BACKENDS + pkglist = withenv() do + cmd = `$(which("pip")) list --format=json` + JSON3.read(cmd) + end + elseif b in PIXI_BACKENDS + cmd = + pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`) + pkglist = JSON3.read(cmd) + pkglist = [pkg for pkg in pkglist if pkg.kind == "pypi"] end Dict(normalise_pip_pkg(pkg.name) => pkg for pkg in pkglist) end diff --git a/src/env.jl b/src/env.jl index 19dd705..c77f2e6 100644 --- a/src/env.jl +++ b/src/env.jl @@ -119,9 +119,13 @@ end Remove unused packages and caches. """ function gc(; io::IO = stderr) - backend() == :Null && return - resolve() - cmd = conda_cmd(`clean -y --all`, io = io) - _run(io, cmd, "Removing unused caches") + b = backend() + if b in CONDA_BACKENDS + resolve() + cmd = conda_cmd(`clean -y --all`, io = io) + _run(io, cmd, "Removing unused caches") + else + _log(io, "GC does nothing with the $b backend.") + end return end diff --git a/src/meta.jl b/src/meta.jl index becd895..5902169 100644 --- a/src/meta.jl +++ b/src/meta.jl @@ -4,7 +4,7 @@ information about the most recent resolve. """ # increment whenever the metadata format changes -const META_VERSION = 13 +const META_VERSION = 14 @kwdef mutable struct Meta timestamp::Float64 @@ -12,6 +12,7 @@ const META_VERSION = 13 load_path::Vector{String} extra_path::Vector{String} version::VersionNumber + backend::Symbol packages::Vector{PkgSpec} channels::Vector{ChannelSpec} pip_packages::Vector{PipPkgSpec} @@ -26,6 +27,7 @@ function read_meta(io::IO) load_path = read_meta(io, Vector{String}), extra_path = read_meta(io, Vector{String}), version = read_meta(io, VersionNumber), + backend = read_meta(io, Symbol), packages = read_meta(io, Vector{PkgSpec}), channels = read_meta(io, Vector{ChannelSpec}), pip_packages = read_meta(io, Vector{PipPkgSpec}), @@ -43,6 +45,9 @@ function read_meta(io::IO, ::Type{String}) end String(bytes) end +function read_meta(io::IO, ::Type{Symbol}) + Symbol(read_meta(io, String)) +end function read_meta(io::IO, ::Type{Vector{T}}) where {T} len = read(io, Int) ans = Vector{T}() @@ -81,6 +86,7 @@ function write_meta(io::IO, meta::Meta) write_meta(io, meta.load_path) write_meta(io, meta.extra_path) write_meta(io, meta.version) + write_meta(io, meta.backend) write_meta(io, meta.packages) write_meta(io, meta.channels) write_meta(io, meta.pip_packages) @@ -93,6 +99,9 @@ function write_meta(io::IO, x::String) write(io, convert(Int, sizeof(x))) write(io, x) end +function write_meta(io::IO, x::Symbol) + write_meta(io, String(x)) +end function write_meta(io::IO, x::Vector) write(io, convert(Int, length(x))) for item in x diff --git a/src/resolve.jl b/src/resolve.jl index 2edc6d5..884dc3e 100644 --- a/src/resolve.jl +++ b/src/resolve.jl @@ -52,6 +52,10 @@ function _resolve_can_skip_1(conda_env, load_path, meta_file) @debug "conda env has changed" meta.conda_env conda_env return false end + if meta.backend != backend() + @debug "backend has changed" meta.backend backend() + return false + end timestamp = max(meta.timestamp, stat(meta_file).mtime) for env in [meta.load_path; meta.extra_path] dir = isfile(env) ? dirname(env) : isdir(env) ? env : continue @@ -495,17 +499,22 @@ function _cmdlines(cmd, flags) lines end +function _logblock(io::IO, lines; kw...) + lines = collect(String, lines) + for (i, line) in enumerate(lines) + pre = i == length(lines) ? "└ " : "│ " + _log(io, label = "") do io + print(io, pre) + printstyled(io, line; kw...) + end + end +end + function _run(io::IO, cmd::Cmd, args...; flags = String[]) _log(io, args...) if _verbosity() ≥ 0 lines = _cmdlines(cmd, flags) - for (i, line) in enumerate(lines) - pre = i == length(lines) ? "└ " : "│ " - _log(io, label = "") do io - print(io, pre) - printstyled(io, line, color = :light_black) - end - end + _logblock(io, lines, color = :light_black) end run(cmd) end @@ -562,8 +571,16 @@ function resolve(; ) shared = true elseif conda_env == "" - conda_env = joinpath(meta_dir, "env") + if back in CONDA_BACKENDS + conda_env = joinpath(meta_dir, "env") + elseif back in PIXI_BACKENDS + conda_env = joinpath(meta_dir, ".pixi", "envs", "default") + else + error("this is a bug") + end shared = false + elseif !(back in CONDA_BACKENDS) + error("cannot set env preference with $back backend") elseif startswith(conda_env, "@") conda_env_name = conda_env[2:end] conda_env_name == "" && error("shared env name cannot be empty") @@ -671,55 +688,119 @@ function resolve(; _log(io, char, " ", pkg, label = "", color = color) end end - # install/uninstall packages - if !force && meta !== nothing && _resolve_env_is_clean(conda_env, meta) - # the state is sufficiently clean that we can modify the existing conda environment - changed = false - if !isempty(removed_pip_pkgs) && !shared - dry_run && return - changed = true - _resolve_pip_remove(io, removed_pip_pkgs, load_path, pip_backend) - end - if !isempty(removed_pkgs) && !shared - dry_run && return - changed = true - _resolve_conda_remove(io, conda_env, removed_pkgs) - end - if !isempty(specs) && ( - !isempty(added_pkgs) || - !isempty(changed_pkgs) || - (meta.channels != channels) || - changed - ) - dry_run && return - changed = true - _resolve_conda_install(io, conda_env, specs, channels) - end - if !isempty(pip_specs) && - (!isempty(added_pip_pkgs) || !isempty(changed_pip_pkgs) || changed) + if back in CONDA_BACKENDS + # install/uninstall packages + if !force && meta !== nothing && _resolve_env_is_clean(conda_env, meta) + # the state is sufficiently clean that we can modify the existing conda environment + changed = false + if !isempty(removed_pip_pkgs) && !shared + dry_run && return + changed = true + _resolve_pip_remove(io, removed_pip_pkgs, load_path, pip_backend) + end + if !isempty(removed_pkgs) && !shared + dry_run && return + changed = true + _resolve_conda_remove(io, conda_env, removed_pkgs) + end + if !isempty(specs) && ( + !isempty(added_pkgs) || + !isempty(changed_pkgs) || + (meta.channels != channels) || + changed + ) + dry_run && return + changed = true + _resolve_conda_install(io, conda_env, specs, channels) + end + if !isempty(pip_specs) && + (!isempty(added_pip_pkgs) || !isempty(changed_pip_pkgs) || changed) + dry_run && return + changed = true + _resolve_pip_install(io, pip_specs, load_path, pip_backend) + end + changed || _log(io, "Dependencies already up to date") + else + # the state is too dirty, recreate the conda environment from scratch dry_run && return - changed = true - _resolve_pip_install(io, pip_specs, load_path, pip_backend) + # remove environment + mkpath(meta_dir) + create = true + if isdir(conda_env) + if shared + create = false + else + _resolve_conda_remove_all(io, conda_env) + end + end + # create conda environment + _resolve_conda_install(io, conda_env, specs, channels; create = create) + # install pip packages + isempty(pip_specs) || + _resolve_pip_install(io, pip_specs, load_path, pip_backend) end - changed || _log(io, "Dependencies already up to date") - else - # the state is too dirty, recreate the conda environment from scratch + elseif back in PIXI_BACKENDS dry_run && return - # remove environment - mkpath(meta_dir) - create = true - if isdir(conda_env) - if shared - create = false - else - _resolve_conda_remove_all(io, conda_env) + cd(meta_dir) do + # remove existing files that might confuse pixi + Base.rm(joinpath(meta_dir, "pixi.toml"), force = true) + Base.rm(joinpath(meta_dir, "pyproject.toml"), force = true) + force && Base.rm(joinpath(meta_dir, "pixi.lock"), force = true) + # write .pixi/config.toml + configtomlpath = joinpath(meta_dir, ".pixi", "config.toml") + configtoml = Dict{String,Any}("detached-environments" => false) + configtomlstr = sprint(TOML.print, configtoml) + mkpath(dirname(configtomlpath)) + write(configtomlpath, configtomlstr) + # initialise pixi + _run( + io, + pixi_cmd(`init --format pixi $meta_dir`), + "Initialising pixi", + flags = ["--quiet"], + ) + # load pixi.toml + pixitomlpath = joinpath(meta_dir, "pixi.toml") + pixitoml = open(TOML.parse, pixitomlpath) + # new pixi.toml + pixitoml = Dict{String,Any}( + "project" => Dict{String,Any}( + "name" => ".CondaPkg", + "description" => "automatically generated by CondaPkg.jl", + "platforms" => pixitoml["project"]["platforms"], + "channels" => String[specstr(channel) for channel in channels], + "channel-priority" => "disabled", + ), + # TODO: deduplicate dependencies + "dependencies" => + Dict{String,Any}(spec.name => pixispec(spec) for spec in specs), + ) + if !isempty(pip_specs) + pixitoml["pypi-dependencies"] = + Dict{String,Any}(spec.name => pixispec(spec) for spec in pip_specs) + end + for spec in pip_specs + if spec.binary != "" + _log( + io, + "Warning: $b backend ignoring binary=$(spec.binary) for $(spec.name)", + ) + end end + pixitomlstr = sprint(TOML.print, pixitoml) + write(pixitomlpath, pixitomlstr) + _log(io, "Wrote $pixitomlpath") + _logblock(io, eachline(pixitomlpath), color = :light_black) + _run( + io, + pixi_cmd( + `$(force ? "update" : "install") --manifest-path $pixitomlpath`, + ), + "Installing packages", + ) end - # create conda environment - _resolve_conda_install(io, conda_env, specs, channels; create = create) - # install pip packages - isempty(pip_specs) || - _resolve_pip_install(io, pip_specs, load_path, pip_backend) + else + error("this is a bug") end # save metadata meta = Meta( @@ -728,6 +809,7 @@ function resolve(; load_path = load_path, extra_path = extra_path, version = VERSION, + backend = back, packages = specs, channels = channels, pip_packages = pip_specs, diff --git a/src/spec.jl b/src/spec.jl index da070b0..216793d 100644 --- a/src/spec.jl +++ b/src/spec.jl @@ -87,6 +87,22 @@ function specstr(x::PkgSpec) string(x.name, suffix) end +function pixispec(x::PkgSpec) + spec = Dict{String,Any}() + spec["version"] = x.version == "" ? "*" : x.version + if x.build != "" + spec["build"] = x.build + end + if x.channel != "" + spec["channel"] = x.channel + end + if length(spec) == 1 + spec["version"] + else + spec + end +end + struct ChannelSpec name::String function ChannelSpec(name) @@ -190,3 +206,37 @@ function specstr(x::PipPkgSpec) end return join(parts) end + +function pixispec(x::PipPkgSpec) + spec = Dict{String,Any}() + if startswith(x.version, "@") + url = strip(x.version[2:end]) + if startswith(url, "git+") + url = url[5:end] + if (m = match(r"^(.*?)@([^/@]*)$", url); m !== nothing) + spec["git"] = m.captures[1] + rev = m.captures[2] + if (m = match(r"^([^#]*)#(.*)$", rev); m !== nothing) + # spec["tag"] = m.captures[1] + spec["rev"] = m.captures[2] + else + spec["rev"] = rev + end + else + spec["git"] = url + end + else + spec["url"] = url + end + else + spec["version"] = x.version == "" ? "*" : x.version + end + if !isempty(x.extras) + spec["extras"] = x.extras + end + if length(spec) == 1 && haskey(spec, "version") + spec["version"] + else + spec + end +end diff --git a/test/main.jl b/test/main.jl index 6c7d996..aa95ad4 100644 --- a/test/main.jl +++ b/test/main.jl @@ -198,7 +198,7 @@ end @testitem "external conda env" begin include("setup.jl") dn = string(tempname(), backend, Sys.KERNEL, VERSION) - isnull || withenv("JULIA_CONDAPKG_ENV" => dn) do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => dn) do # create empty env CondaPkg.resolve() @test !occursin("ca-certificates", status()) @@ -219,13 +219,13 @@ end @testitem "shared env" begin include("setup.jl") - isnull || withenv("JULIA_CONDAPKG_ENV" => "@my_env") do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => "@my_env") do CondaPkg.add("python"; force = true) @test CondaPkg.envdir() == joinpath(Base.DEPOT_PATH[1], "conda_environments", "my_env") @test isfile(CondaPkg.envdir(Sys.iswindows() ? "python.exe" : "bin/python")) end - isnull || withenv("JULIA_CONDAPKG_ENV" => "@/some/absolute/path") do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => "@/some/absolute/path") do @test_throws ErrorException CondaPkg.add("python"; force = true) end end diff --git a/test/setup.jl b/test/setup.jl index 31d7094..5919bf1 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -9,11 +9,13 @@ status() = sprint(io -> CondaPkg.status(io = io)) const backend = get(ENV, "JULIA_CONDAPKG_BACKEND", "MicroMamba") const isnull = backend == "Null" +const ispixi = backend == "SystemPixi" # reset the package state (so tests are independent of the order they are run) rm(CondaPkg.cur_deps_file(), force = true) CondaPkg.STATE.backend = :NotSet CondaPkg.STATE.condaexe = "" +CondaPkg.STATE.pixiexe = "" CondaPkg.STATE.resolved = false CondaPkg.STATE.load_path = String[] CondaPkg.STATE.meta_dir = ""