diff --git a/init.rb b/init.rb index 2452f396..b73b08e8 100644 --- a/init.rb +++ b/init.rb @@ -174,7 +174,38 @@ def bundle_package(*args, &block) Autobuild::Orogen.transports.delete("mqueue") end -Rock.setup_python_configuration_options +if Autoproj.config.get "USE_PYTHON" + Rock.setup_python_configuration_options + + if Autoproj.config.get "PYTHON_UPGRADE_PIP" + # Monkeypatch Pipmanager. This should be changed in Autoproj directly: + class Autoproj::PackageManagers::PipManager + def install(pips, filter_uptodate_packages: false, install_only: false) + guess_pip_program + pips = [pips] if pips.is_a?(String) + + upgrade_cmdline = [Autobuild.tool("pip"), "install", "--user", "--upgrade", "pip"] + base_cmdline = [Autobuild.tool("pip"), "install", "--user"] + + cmdlines = [upgrade_cmdline, base_cmdline + pips] + + if pips_interaction(cmdlines) + Autoproj.message " installing/updating Python dependencies:" \ + " #{pips.sort.join(', ')}" + + cmdlines.each do |c| + Autobuild::Subprocess.run "autoproj", "osdeps", *c, + env: ws.env.resolved_env + end + + pips.each do |p| + @installed_pips << p + end + end + end + end + end +end Autoproj.config.declare 'syskit_use_bundles', 'boolean', default: true, diff --git a/overrides.rb b/overrides.rb index 9f12e50f..ba39a176 100644 --- a/overrides.rb +++ b/overrides.rb @@ -1,5 +1,10 @@ Rock.flavors.finalize +require File.join(__dir__, 'rock/python') +if Autoproj.config.get "USE_PYTHON" + Rock.check_init_venv +end + Autoproj.env_add_path 'ROCK_BUNDLE_PATH', File.join(Autobuild.prefix, 'share', 'rock') Autoproj.env_add_path 'ROCK_BUNDLE_PATH', File.join(Autoproj.root_dir, 'bundles') diff --git a/rock.osdeps-python3 b/rock.osdeps-python3 index 065684bf..a0b00a6e 100644 --- a/rock.osdeps-python3 +++ b/rock.osdeps-python3 @@ -32,6 +32,9 @@ python-setuptools: opensuse: python3-setuptools gentoo: dev-python/setuptools +python-venv: + debian,ubuntu: python3-venv + cython: debian, ubuntu: cython3 opensuse: python3-Cython diff --git a/rock/python.rb b/rock/python.rb index 4330e6c8..06a39536 100644 --- a/rock/python.rb +++ b/rock/python.rb @@ -21,280 +21,391 @@ # (however, no caching is done for that part) # 4. version constraints are checked upon activation module Rock - # Get the python version for a given python executable - # @return [String] The python version as . - def self.get_python_version(python_bin) - if !File.exist?(python_bin) - raise ArgumentError, "Rock.get_python_version executable "\ - "'#{python_bin}' does not exist" + # Get the python version for a given python executable + # @return [String] The python version as . + def self.get_python_version(python_bin) + unless File.exist?(python_bin) + raise ArgumentError, "Autoproj::Python.get_python_version executable "\ + "'#{python_bin}' does not exist" + end + + cmd = "#{python_bin} -c \"import sys;"\ + "version=sys.version_info[:3]; "\ + "print('{0}.{1}'.format(*version))\"".strip + + msg, status = Open3.capture2e(cmd) + if status.success? + msg.strip + + else + raise "Autoproj::Python.get_python_version identification"\ + " of python version for '#{python_bin}' failed: #{msg}" + end end - cmd = "#{python_bin} -c \"import sys;"\ - "version=sys.version_info[:3]; "\ - "print('{0}.{1}'.format(*version))\"".strip() - - msg, status = Open3.capture2e(cmd) - if status.success? - python_version = msg.strip() - return python_version - else - raise RuntimeError, "Rock.get_python_version identification"\ - " of python version for '#{python_bin}' failed: #{msg}" + def self.get_pip_version(pip_bin) + unless File.exist?(pip_bin) + raise ArgumentError, "Autoproj::Python.get_pip_version executable "\ + "'#{pip_bin}' does not exist" + end + + cmd = "#{pip_bin} --version" + + msg, status = Open3.capture2e(cmd) + if status.success? + msg.split(" ")[1] + + else + raise "Autoproj::Python.get_pip_version identification"\ + " of pip version for '#{pip_bin}' failed: #{msg}" + end end - end - def self.get_pip_version(pip_bin) - if !File.exist?(pip_bin) - raise ArgumentError, "Rock.get_pip_version executable "\ - "'#{pip_bin}' does not exist" + def self.validate_version(version, version_constraint) + if version_constraint + dependency = Gem::Dependency.new("python", version_constraint) + dependency.match?("python", version) + else + true + end end - cmd = "#{pip_bin} --version" + # Validate that a given python executable's version fulfills + # a given version constraint + # @param [String] python_bin the python executable + # @param [String] version_constraint version constraint, e.g., <3.8, >= 3.7, 3.6 + # @return [String,Bool] Version and validation result, i.e., + # True if binary fulfills the version constraint, false otherwise + def self.validate_python_version(python_bin, version_constraint) + version = get_python_version(python_bin) + [version, validate_version(version, version_constraint)] + end - msg, status = Open3.capture2e(cmd) - if status.success? - pip_version = msg.split(" ")[1] - return pip_version - else - raise RuntimeError, "Rock.get_pip_version identification"\ - " of pip version for '#{pip_bin}' failed: #{msg}" + # Find python given a version constraint + # @return [String,String] path to python executable and python version + def self.find_python(ws: Autoproj.workspace, + version: ws.config.get("python_version", nil)) + finders = [ + -> { Autobuild.programs["python"] }, + -> { Autobuild.find_in_path("python3") }, + -> { Autobuild.find_in_path("python") } + ] + + finders.each do |finder| + if (python_bin = finder.call) + python_version, valid = validate_python_version(python_bin, version) + return python_bin, python_version if valid + end + end + raise "Autoproj::Python.find_python_bin: failed to find python" \ + " for version '#{version}'" end - end - - def self.validate_version(version, version_constraint) - if !version_constraint - return true - else - dependency = Gem::Dependency.new("python", version_constraint) - return dependency.match?("python", version) + + # Get information about the python executable from autoproj config, + # but ensure the version constraint matches + # + # @return [String, String] Return path and version if the constraints + # are fulfilled nil otherwise + + def self.get_python_from_config(ws: Autoproj.workspace, version: nil) + config_bin = ws.config.get("python_executable", nil) + return unless config_bin + + config_version = ws.config.get("python_version", nil) + config_version ||= get_python_version(config_bin) + + # If a version constraint is given, ensure fulfillment + if validate_version(config_version, version) + [config_bin, config_version] + else + raise "python_executable in autoproj config with " \ + "version '#{config_version}' does not match "\ + "version constraints '#{version}'" + end end - end - - # Validate that a given python executable's version fulfills - # a given version constraint - # @param [String] python_bin the python executable - # @param [String] version_constraint version constraint, e.g., <3.8, >= 3.7, 3.6 - # @return [String,Bool] Version and validation result, i.e., True if binary fulfills the version constraint, false - # otherwise - def self.validate_python_version(python_bin, version_constraint) - version = get_python_version(python_bin) - return [version, validate_version(version, version_constraint)] - end - - # Find python given a version constraint - # @return [String,String] path to python executable and python version - def self.find_python(ws: Autoproj.workspace, - version: ws.config.get('python_version',nil)) - finders = [ - lambda { Autobuild.programs['python'] }, - lambda { `which python3`.strip() }, - lambda { `which python`.strip() } - ] - - finders.each do |finder| - python_bin = finder.call - if python_bin && !python_bin.empty? - python_version, valid = validate_python_version(python_bin, version) - if valid - return python_bin, python_version + + def self.get_python_from_bootstrap(ws: Autoproj.workspace, version: nil) + if ws.config.get("USE_PYTHON_VENV") == true then + config_bin = ws.config.get("PYTHON_VENV_INIT_EXECUTABLE", nil) + return unless config_bin + + config_version = ws.config.get("python_version", nil) + config_version ||= get_python_version(config_bin) + + # If a version constraint is given, ensure fulfillment + if validate_version(config_version, version) + [config_bin, config_version] + else + raise "python_executable in autoproj config with " \ + "version '#{config_version}' does not match "\ + "version constraints '#{version}'" end + else + get_python_from_config(ws, version) end end - raise RuntimeError, "Rock.find_python_bin: failed to find python" \ - " for version '#{version}'" - end - - # Get information about the python executable from autoproj config, - # but ensure the version constraint matches - # - # @return [String, String] Return path and version if the constraints - # are fulfilled nil otherwise - - def self.get_python_from_config(ws: Autoproj.workspace, version: nil) - config_bin = ws.config.get('python_executable', nil) - return unless config_bin - - config_version = ws.config.get('python_version', nil) - config_version ||= get_python_version(config_bin) - - # If a version constraint is given, ensure fulfillment - if validate_version(config_version, version) - return config_bin, config_version - else - raise RuntimeError, "python_executable in autoproj config with version '#{config_version}'"\ - " does not match version constraints '#{version}'" - end - end - - def self.custom_resolve_python(ws: Autoproj.workspace, - bin: nil, - version: nil) - version, valid = validate_python_version(bin, version) - if valid - return [bin, version] - else - raise RuntimeError, "Rock.resolve_python: requested python"\ - "executable '#{bin}' does not satisfy version"\ - "constraints '#{version}'" + + def self.custom_resolve_python(bin: nil, + version: nil) + version, valid = validate_python_version(bin, version) + if valid + [bin, version] + else + raise "Autoproj::Python.resolve_python: requested python"\ + "executable '#{bin}' does not satisfy version"\ + "constraints '#{version}'" + end end - end - def self.auto_resolve_python(ws: Autoproj.workspace, - version: nil) - version_constraint = version - resolvers = [ - lambda { get_python_from_config(ws: ws, version: version_constraint) }, - lambda { find_python(ws: ws, version: version_constraint) } - ] - - bin = nil - resolvers.each do |resolver| - begin + def self.auto_resolve_python(ws: Autoproj.workspace, + version: nil) + version_constraint = version + resolvers = [ + -> { get_python_from_config(ws: ws, version: version_constraint) }, + -> { get_python_from_bootstrap(ws: ws, version: version_constraint) }, + -> { find_python(ws: ws, version: version_constraint) } + ] + + bin = nil + resolvers.each do |resolver| bin, version = resolver.call if bin && File.exist?(bin) && version - Autoproj.debug "Rock.resolve_python: found python '#{bin}'"\ - " version '#{version}'" + Autoproj.debug "Autoproj::Python.resolve_python: " \ + "found python '#{bin}' version '#{version}'" break end rescue RuntimeError => e - Autoproj.debug "Rock.resolve_python: resolver failed: #{e}" + Autoproj.debug "Autoproj::Python.resolve_python: " \ + "resolver failed: #{e}" end + + unless bin + msg = "Autoproj::Python.resolve_python: " \ + "failed to find a python executable" + if version_constraint + msg += " satisfying version constraint '#{version_constraint}'" + end + raise msg + end + [bin, version] end - if !bin - msg = "Rock.resolve_python: failed to find a python executable" - if version_constraint - msg += " satisfying version constraint '#{version_constraint}'" + # Resolve the python executable according to a given version constraint + # @param [Autoproj.workspace] ws Autoproj workspace + # @param [String] bin Path to the python executable that shall be used, + # first fallback is the python_executable set in Autoproj's configuration, + # second fallback is a full search + # @param [String] version version constraint + # @return [String,String] python path and python version + def self.resolve_python(ws: Autoproj.workspace, + bin: nil, + version: nil) + if bin + custom_resolve_python(bin: bin, version: version) + else + auto_resolve_python(ws: ws, version: version) end - raise RuntimeError, msg end - [bin, version] - end - - # Resolve the python executable according to a given version constraint - # @param [Autoproj.workspace] ws Autoproj workspace - # @param [String] bin Path to the python executable that shall be used, - # first fallback is the python_executable set in Autoproj's configuration, - # second fallback is a full search - # @param [String] version version constraint - # @return [String,String] python path and python version - def self.resolve_python(ws: Autoproj.workspace, - bin: nil, - version: nil) - version_constraint = version - # Custom selection of python version - if bin - return custom_resolve_python(ws: ws, bin: bin, version: version) - else - return auto_resolve_python(ws: ws, version: version) + + ############### Added feature venv + def self.disable_venv(prefix_dir) + venv_path = File.join(prefix_dir, "install", "venv") + if File.exist?(venv_path) + FileUtils.rm_r venv_path + ws.config.delete("PYTHON_VENV_FOLDER") + end end - end - def self.remove_python_shims(root_dir) - shim_path = File.join(root_dir, "install","bin","python") - if File.exist?(shim_path) - FileUtils.rm shim_path + + def self.create_venv(prefix_dir) + #python_path = File.join(shim_path, 'python') + # TODO make venv-path configurable + + python_executable = get_python_from_config.first + Autobuild::Subprocess.run "config", "create_venv", python_executable, "-m", "venv", File.join(prefix_dir, "install", "venv"), "--system-site-packages" end - end - - def self.rewrite_python_shims(python_executable, root_dir) - shim_path = File.join(root_dir, "install","bin") - if !File.exist?(shim_path) - FileUtils.mkdir_p shim_path - Autoproj.warn "Rock.rewrite_python_shims: creating "\ - "#{shim_path} - "\ - "are you operating on a valid autoproj workspace?" + + # Activate configuration for python in the autoproj configuration + # @return [String,String] python path and python version + def self.activate_python_venv(ws: Autoproj.workspace, + bin: nil, + version: nil) + bin, version = resolve_python(ws: ws, bin: bin, version: version) + ws.config.set('python_executable', bin, true) + ws.config.set('python_version', version, true) + + ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./ + + rewrite_python_shims(bin, ws.root_dir) + Autoproj.env_add "VIRTUAL_ENV_DISABLE_PROMPT", "1" + Autoproj.env.source_after File.join(ws.root_dir, "install", "venv", "bin", "activate") + + [File.join(ws.root_dir, "install", "venv", "bin", "python"), version] end - python_path = File.join(shim_path, 'python') - File.open(python_path, 'w') do |io| - io.puts "#! /bin/sh" - io.puts "exec #{python_executable} \"$@\"" + #### end of venv additions + + + def self.remove_python_shims(prefix_dir) + shim_path = File.join(prefix_dir, "bin", "python") + FileUtils.rm shim_path if File.exist?(shim_path) end - FileUtils.chmod 0755, python_path - python_path - end - - def self.rewrite_pip_shims(python_executable, root_dir) - shim_path = File.join(root_dir, "install","bin") - if !File.exist?(shim_path) - FileUtils.mkdir_p shim_path - Autoproj.warn "Rock.rewrite_pip_shims: creating "\ - "#{shim_path} - "\ - "are you operating on a valid autoproj workspace?" + + def self.remove_pip_shims(prefix_dir) + shim_path = File.join(prefix_dir, "bin", "pip") + FileUtils.rm shim_path if File.exist?(shim_path) end - pip_path = File.join(shim_path, 'pip') - File.open(pip_path, 'w') do |io| - io.puts "#! /bin/sh" - io.puts "exec #{python_executable} -m pip \"$@\"" + + def self.rewrite_python_shims(python_executable, prefix_dir) + shim_path = File.join(prefix_dir, "bin") + unless File.exist?(shim_path) + FileUtils.mkdir_p shim_path + Autoproj.warn "Autoproj::Python.rewrite_python_shims: creating "\ + "#{shim_path} - "\ + "are you operating on a valid autoproj workspace?" + end + + python_path = File.join(shim_path, "python") + File.open(python_path, "w") do |io| + io.puts "#! /bin/sh" + io.puts "exec #{python_executable} \"$@\"" + end + FileUtils.chmod 0o755, python_path + python_path end - FileUtils.chmod 0755, pip_path - pip_path - end - - # Activate configuration for python in the autoproj configuration - # @return [String,String] python path and python version - def self.activate_python(ws: Autoproj.workspace, - bin: nil, - version: nil) - bin, version = resolve_python(ws: ws, bin: bin, version: version) - ws.config.set('python_executable', bin, true) - ws.config.set('python_version', version, true) - - ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./ - - rewrite_python_shims(bin, ws.root_dir) - rewrite_pip_shims(bin, ws.root_dir) - [bin, version] - end - - def self.deactivate_python(ws: Autoproj.workspace) - remove_python_shims(ws.root_dir) - ws.config.reset('python_executable') - ws.config.reset('python_version') - end - - # Allow to update the PYTHONPATH for package if autoproj configuration - # USE_PYTHON is set to true. - # Then tries to guess the python binary from Autobuild.programs['python'] - # and system's default setting - # @param [Autobuild::Package] pkg - # @param [Autoproj.workspace] ws Autoproj workspace - # @param [String] bin Path to a custom python version - # @param [String] version version constraint for python executable - # @return tuple of [executable, version, site-packages path] if set, - # otherwise nil - def self.activate_python_path(pkg, - ws: Autoproj.workspace, - bin: nil, - version: nil) - return unless ws.config.get('USE_PYTHON',nil) - - bin, version = resolve_python(ws: ws, bin: bin, version: version) - path = File.join(pkg.prefix, "lib", - "python#{version}","site-packages") - pkg.env_add_path 'PYTHONPATH', path - - [bin, version, path] - end - - def self.setup_python_configuration_options(ws: Autoproj.workspace) - ws.config.declare 'USE_PYTHON', 'boolean', - default: 'no', - doc: [ "Do you want to activate python?" ] - - if ws.config.get("USE_PYTHON") - if !ws.config.has_value_for?('python_executable') - remove_python_shims(ws.root_dir) - python_bin,_ = auto_resolve_python(ws: ws) + + def self.rewrite_pip_shims(python_executable, prefix_dir) + shim_path = File.join(prefix_dir, "bin") + unless File.exist?(shim_path) + FileUtils.mkdir_p shim_path + Autoproj.warn "Autoproj::Python.rewrite_pip_shims: creating "\ + "#{shim_path} - "\ + "are you operating on a valid autoproj workspace?" + end + pip_path = File.join(shim_path, "pip") + File.open(pip_path, "w") do |io| + io.puts "#! /bin/sh" + io.puts "exec #{python_executable} -m pip \"$@\"" end + FileUtils.chmod 0o755, pip_path + pip_path + end + + # Activate configuration for python in the autoproj configuration + # @return [String,String] python path and python version + def self.activate_python(ws: Autoproj.workspace, + bin: nil, + version: nil) + bin, version = resolve_python(ws: ws, bin: bin, version: version) + ws.config.set("python_executable", bin, true) + ws.config.set("python_version", version, true) + + ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./ + + rewrite_python_shims(bin, ws.dot_autoproj_dir) + rewrite_pip_shims(bin, ws.dot_autoproj_dir) + [bin, version] + end - ws.config.declare 'python_executable', 'string', - default: "#{python_bin}", - doc: [ "Select the path to the python executable" ] + def self.deactivate_python(ws: Autoproj.workspace) + remove_python_shims(ws.dot_autoproj_dir) + remove_pip_shims(ws.dot_autoproj_dir) + ws.config.reset("python_executable") + ws.config.reset("python_version") + end + + # Allow to update the PYTHONPATH for package if autoproj configuration + # USE_PYTHON is set to true. + # Then tries to guess the python binary from Autobuild.programs['python'] + # and system's default setting + # @param [Autobuild::Package] pkg + # @param [Autoproj.workspace] ws Autoproj workspace + # @param [String] bin Path to a custom python version + # @param [String] version version constraint for python executable + # @return tuple of [executable, version, site-packages path] if set, + # otherwise nil + def self.activate_python_path(pkg, + ws: Autoproj.workspace, + bin: nil, + version: nil) + return unless ws.config.get("USE_PYTHON", nil) + + bin, version = resolve_python(ws: ws, bin: bin, version: version) + path = File.join(pkg.prefix, "lib", + "python#{version}", "site-packages") + pkg.env_add_path "PYTHONPATH", path + + [bin, version, path] + end + + def self.assert_python_activated(ws: Autoproj.workspace) + return true if ws.config.get("USE_PYTHON") + + raise ConfigError, + "Your current package selection requires the use of python," \ + " but this is either unspecified or has been denied,"\ + " see setting of USE_PYTHON in your workspace configuration." \ + " Either remove all packages depending on pip packages " \ + " from the workspace layout (manifest) or " \ + " call 'autoproj reconfigure' to change the setting." + end + + def self.setup_python_configuration_options(ws: Autoproj.workspace) + ws.config.declare "USE_PYTHON", "boolean", + default: "no", + doc: ["Do you want to activate python?"] - activate_python(ws: ws) - else - deactivate_python(ws: ws) + if ws.config.get("USE_PYTHON") + unless ws.config.has_value_for?("python_executable") + remove_python_shims(ws.dot_autoproj_dir) + remove_pip_shims(ws.dot_autoproj_dir) + python_bin, = auto_resolve_python(ws: ws) + end + + ws.config.declare "python_executable", "string", + default: python_bin.to_s, + doc: ["Select the path to the python executable"] + os_names, os_versions = Autoproj.workspace.operating_system + ws.config.declare "USE_PYTHON_VENV", "boolean", + default: (os_names.include?('ubuntu') && os_versions.include?('24.04') ? "yes" : "no"), + doc: ["Use Python venv? (required from Ubuntu 24)"] + + ws.config.declare "PYTHON_UPGRADE_PIP", "boolean", + default: (os_names.include?('ubuntu') && os_versions.include?('20.04') ? "yes" : "no"), + doc: ["Upgrade pip version? (required before Ubuntu 22.04)"] + + if ws.config.get("USE_PYTHON_VENV") + remove_python_shims(ws.dot_autoproj_dir) + remove_pip_shims(ws.dot_autoproj_dir) + activate_python_venv(ws: ws) + ws.env.add "PATH", File.join(ws.root_dir, "install", "venv", "bin") + # tell autoproj/autobuild where the venv is + ws.env.set "PYTHONUSERBASE", File.join(ws.root_dir, "install", "venv") + ws.env.set "AUTOPROJ_PYTHONUSERBASE", File.join(ws.root_dir, "install", "venv") + else + activate_python(ws: ws) + end + else + deactivate_python(ws: ws) + end + end + + def self.check_init_venv(ws: Autoproj.workspace) + # create the actual venv, if not created before + return unless ws.config.get("USE_PYTHON_VENV") + unless ws.config.has_value_for?("PYTHON_VENV_FOLDER") && File.exist?(File.join(ws.root_dir, "install", "venv")) then + puts "creating python venv in " + ws.root_dir + ws.install_os_packages(["python-venv"]) + create_venv(ws.root_dir) + + venv_folder = File.join(ws.root_dir, "install", "venv") + ws.config.set("PYTHON_VENV_FOLDER", venv_folder) + + # switch python_executable + python_executable_init = get_python_from_config.first + python_executable_basename = File.basename(python_executable_init) + python_executable_venv = File.join(venv_folder, "bin", python_executable_basename) + ws.config.set("python_executable", python_executable_venv) + ws.config.set("PYTHON_VENV_INIT_EXECUTABLE", python_executable_init) + end end - end end