888 lines · plain
1################################################################################2# Python modules3# MLIR's Python modules are both directly used by the core project and are4# available for use and embedding into external projects (in their own5# namespace and with their own deps). In order to facilitate this, python6# artifacts are split between declarations, which make a subset of7# things available to be built and "add", which in line with the normal LLVM8# nomenclature, adds libraries.9################################################################################10 11# Function: declare_mlir_python_sources12# Declares pure python sources as part of a named grouping that can be built13# later.14# Arguments:15# ROOT_DIR: Directory where the python namespace begins (defaults to16# CMAKE_CURRENT_SOURCE_DIR). For non-relocatable sources, this will17# typically just be the root of the python source tree (current directory).18# For relocatable sources, this will point deeper into the directory that19# can be relocated. For generated sources, can be relative to20# CMAKE_CURRENT_BINARY_DIR. Generated and non generated sources cannot be21# mixed.22# ADD_TO_PARENT: Adds this source grouping to a previously declared source23# grouping. Source groupings form a DAG.24# SOURCES: List of specific source files relative to ROOT_DIR to include.25# SOURCES_GLOB: List of glob patterns relative to ROOT_DIR to include.26function(declare_mlir_python_sources name)27 cmake_parse_arguments(ARG28 ""29 "ROOT_DIR;ADD_TO_PARENT"30 "SOURCES;SOURCES_GLOB"31 ${ARGN})32 33 if(NOT ARG_ROOT_DIR)34 set(ARG_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")35 endif()36 set(_install_destination "src/python/${name}")37 38 # Process the glob.39 set(_glob_sources)40 if(ARG_SOURCES_GLOB)41 set(_glob_spec ${ARG_SOURCES_GLOB})42 list(TRANSFORM _glob_spec PREPEND "${ARG_ROOT_DIR}/")43 file(GLOB_RECURSE _glob_sources44 RELATIVE "${ARG_ROOT_DIR}"45 ${_glob_spec}46 )47 list(APPEND ARG_SOURCES ${_glob_sources})48 endif()49 50 # We create a custom target to carry properties and dependencies for51 # generated sources.52 add_library(${name} INTERFACE)53 set_target_properties(${name} PROPERTIES54 # Yes: Leading-lowercase property names are load bearing and the recommended55 # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/1926156 EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_DEPENDS"57 mlir_python_SOURCES_TYPE pure58 mlir_python_DEPENDS ""59 )60 61 # Use the interface include directories and sources on the target to carry the62 # properties we would like to export. These support generator expressions and63 # allow us to properly specify paths in both the local build and install scenarios.64 # The one caveat here is that because we don't directly build against the interface65 # library, we need to specify the INCLUDE_DIRECTORIES and SOURCES properties as well66 # via private properties because the evaluation would happen at configuration time67 # instead of build time.68 # Eventually this could be done using a FILE_SET simplifying the logic below.69 # FILE_SET is available in cmake 3.23+, so it is not an option at the moment.70 target_include_directories(${name} INTERFACE71 "$<BUILD_INTERFACE:${ARG_ROOT_DIR}>"72 "$<INSTALL_INTERFACE:${_install_destination}>"73 )74 set_property(TARGET ${name} PROPERTY INCLUDE_DIRECTORIES ${ARG_ROOT_DIR})75 76 if(ARG_SOURCES)77 list(TRANSFORM ARG_SOURCES PREPEND "${ARG_ROOT_DIR}/" OUTPUT_VARIABLE _build_sources)78 list(TRANSFORM ARG_SOURCES PREPEND "${_install_destination}/" OUTPUT_VARIABLE _install_sources)79 target_sources(${name}80 INTERFACE81 "$<INSTALL_INTERFACE:${_install_sources}>"82 "$<BUILD_INTERFACE:${_build_sources}>"83 PRIVATE ${_build_sources}84 )85 endif()86 87 # Add to parent.88 if(ARG_ADD_TO_PARENT)89 set_property(TARGET ${ARG_ADD_TO_PARENT} APPEND PROPERTY mlir_python_DEPENDS ${name})90 endif()91 92 # Install.93 set_property(GLOBAL APPEND PROPERTY MLIR_EXPORTS ${name})94 if(NOT LLVM_INSTALL_TOOLCHAIN_ONLY)95 _mlir_python_install_sources(96 ${name} "${ARG_ROOT_DIR}" "${_install_destination}"97 ${ARG_SOURCES}98 )99 endif()100endfunction()101 102# Function: mlir_generate_type_stubs103# Turns on automatic type stub generation for extension modules.104# Specifically, performs add_custom_command to run nanobind's stubgen on an extension module.105#106# Arguments:107# MODULE_NAME: The fully-qualified name of the extension module (used for importing in python).108# DEPENDS_TARGETS: List of targets these type stubs depend on being built; usually corresponding to the109# specific extension module (e.g., something like StandalonePythonModules.extension._standaloneDialectsNanobind.dso)110# and the core bindings extension module (e.g., something like StandalonePythonModules.extension._mlir.dso).111# OUTPUT_DIR: The root output directory to emit the type stubs into.112# OUTPUTS: List of expected outputs.113# DEPENDS_TARGET_SRC_DEPS: List of cpp sources for extension library (for generating a DEPFILE).114# IMPORT_PATHS: List of paths to add to PYTHONPATH for stubgen.115# PATTERN_FILE: (Optional) Pattern file (see https://nanobind.readthedocs.io/en/latest/typing.html#pattern-files).116# VERBOSE: Emit logging/status messages during stub generation (default: OFF).117# Outputs:118# NB_STUBGEN_CUSTOM_TARGET: The target corresponding to generation which other targets can depend on.119function(mlir_generate_type_stubs)120 cmake_parse_arguments(ARG121 "VERBOSE"122 "MODULE_NAME;OUTPUT_DIR;PATTERN_FILE"123 "IMPORT_PATHS;DEPENDS_TARGETS;OUTPUTS;DEPENDS_TARGET_SRC_DEPS"124 ${ARGN})125 126 # for people doing find_package(nanobind)127 if(EXISTS ${nanobind_DIR}/../src/stubgen.py)128 set(NB_STUBGEN "${nanobind_DIR}/../src/stubgen.py")129 elseif(EXISTS ${nanobind_DIR}/../stubgen.py)130 set(NB_STUBGEN "${nanobind_DIR}/../stubgen.py")131 # for people using FetchContent_Declare and FetchContent_MakeAvailable132 elseif(EXISTS ${nanobind_SOURCE_DIR}/src/stubgen.py)133 set(NB_STUBGEN "${nanobind_SOURCE_DIR}/src/stubgen.py")134 elseif(EXISTS ${nanobind_SOURCE_DIR}/stubgen.py)135 set(NB_STUBGEN "${nanobind_SOURCE_DIR}/stubgen.py")136 else()137 message(FATAL_ERROR "mlir_generate_type_stubs(): could not locate 'stubgen.py'!")138 endif()139 140 file(REAL_PATH "${NB_STUBGEN}" NB_STUBGEN)141 set(_import_paths)142 foreach(_import_path IN LISTS ARG_IMPORT_PATHS)143 file(REAL_PATH "${_import_path}" _import_path)144 list(APPEND _import_paths "-i;${_import_path}")145 endforeach()146 set(_nb_stubgen_cmd147 "${Python_EXECUTABLE}"148 "${NB_STUBGEN}"149 --module150 "${ARG_MODULE_NAME}"151 "${_import_paths}"152 --recursive153 --include-private154 --output-dir155 "${ARG_OUTPUT_DIR}")156 if(NOT ARG_VERBOSE)157 list(APPEND _nb_stubgen_cmd "--quiet")158 endif()159 if(ARG_PATTERN_FILE)160 list(APPEND _nb_stubgen_cmd "-p;${ARG_PATTERN_FILE}")161 list(APPEND ARG_DEPENDS_TARGETS "${ARG_PATTERN_FILE}")162 endif()163 164 list(TRANSFORM ARG_OUTPUTS PREPEND "${ARG_OUTPUT_DIR}/" OUTPUT_VARIABLE _generated_type_stubs)165 set(_depfile "${ARG_OUTPUT_DIR}/${ARG_MODULE_NAME}.d")166 if ((NOT EXISTS ${_depfile}) AND ARG_DEPENDS_TARGET_SRC_DEPS)167 list(JOIN ARG_DEPENDS_TARGET_SRC_DEPS " " _depfiles)168 list(TRANSFORM _generated_type_stubs APPEND ": ${_depfiles}" OUTPUT_VARIABLE _depfiles)169 list(JOIN _depfiles "\n" _depfiles)170 file(GENERATE OUTPUT "${_depfile}" CONTENT "${_depfiles}")171 endif()172 173 if(ARG_VERBOSE)174 message(STATUS "Generating type-stubs outputs ${_generated_type_stubs}")175 endif()176 177 # If PYTHONPATH is set and points to the build location of the python package then when stubgen runs, _mlir will get178 # imported twice and bad things will happen (e.g., Assertion `!instance && “PyGlobals already constructed”’).179 # This happens because mlir is a namespace package and the importer/loader can't distinguish between180 # mlir._mlir_libs._mlir and _mlir_libs._mlir imported from CWD.181 # So try to filter out any entries in PYTHONPATH that end in "MLIR_BINDINGS_PYTHON_INSTALL_PREFIX/.."182 # (e.g., python_packages/mlir_core/).183 set(_pythonpath "$ENV{PYTHONPATH}")184 cmake_path(CONVERT "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}/.." TO_NATIVE_PATH_LIST _install_prefix NORMALIZE)185 if(WIN32)186 set(_path_sep ";")187 set(_trailing_sep "\\")188 else()189 set(_path_sep ":")190 set(_trailing_sep "/")191 # `;` is the CMake list delimiter so Windows paths are automatically lists192 # and Unix paths can be made into lists by replacing `:` with `;`193 string(REPLACE "${_path_sep}" ";" _pythonpath "${_pythonpath}")194 endif()195 string(REGEX REPLACE "${_trailing_sep}$" "" _install_prefix "${_install_prefix}")196 list(FILTER _pythonpath EXCLUDE REGEX "(${_install_prefix}|${_install_prefix}${_trailing_sep})$")197 # Note, ${_pythonpath} is a list but "${_pythonpath}" is not a list - it's a string with ";" chars in it.198 string(JOIN "${_path_sep}" _pythonpath ${_pythonpath})199 add_custom_command(200 OUTPUT ${_generated_type_stubs}201 COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${_pythonpath}" ${_nb_stubgen_cmd}202 WORKING_DIRECTORY "${CMAKE_CURRENT_FUNCTION_LIST_DIR}"203 DEPENDS "${ARG_DEPENDS_TARGETS}"204 DEPFILE "${_depfile}"205 COMMENT "Generating type stubs using: ${_nb_stubgen_cmd}"206 )207 208 list(JOIN ARG_DEPENDS_TARGETS "." _name)209 set(_name "${_name}.${ARG_MODULE_NAME}.type_stubs")210 add_custom_target("${_name}" DEPENDS ${_generated_type_stubs})211 set(NB_STUBGEN_CUSTOM_TARGET "${_name}" PARENT_SCOPE)212endfunction()213 214# Function: declare_mlir_python_extension215# Declares a buildable python extension from C++ source files. The built216# module is considered a python source file and included as everything else.217# Arguments:218# ROOT_DIR: Root directory where sources are interpreted relative to.219# Defaults to CMAKE_CURRENT_SOURCE_DIR.220# MODULE_NAME: Local import name of the module (i.e. "_mlir").221# ADD_TO_PARENT: Same as for declare_mlir_python_sources.222# SOURCES: C++ sources making up the module.223# PRIVATE_LINK_LIBS: List of libraries to link in privately to the module224# regardless of how it is included in the project (generally should be225# static libraries that can be included with hidden visibility).226# EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends227# on. These will be collected for all extensions and put into an228# aggregate dylib that is linked against.229# PYTHON_BINDINGS_LIBRARY: Either pybind11 or nanobind.230function(declare_mlir_python_extension name)231 cmake_parse_arguments(ARG232 ""233 "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT;PYTHON_BINDINGS_LIBRARY"234 "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS"235 ${ARGN})236 237 if(NOT ARG_ROOT_DIR)238 set(ARG_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")239 endif()240 set(_install_destination "src/python/${name}")241 242 if(NOT ARG_PYTHON_BINDINGS_LIBRARY)243 set(ARG_PYTHON_BINDINGS_LIBRARY "pybind11")244 endif()245 246 add_library(${name} INTERFACE)247 set_target_properties(${name} PROPERTIES248 # Yes: Leading-lowercase property names are load bearing and the recommended249 # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/19261250 EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS;mlir_python_BINDINGS_LIBRARY"251 mlir_python_SOURCES_TYPE extension252 mlir_python_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}"253 mlir_python_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}"254 mlir_python_DEPENDS ""255 mlir_python_BINDINGS_LIBRARY "${ARG_PYTHON_BINDINGS_LIBRARY}"256 )257 258 # Set the interface source and link_libs properties of the target259 # These properties support generator expressions and are automatically exported260 list(TRANSFORM ARG_SOURCES PREPEND "${ARG_ROOT_DIR}/" OUTPUT_VARIABLE _build_sources)261 list(TRANSFORM ARG_SOURCES PREPEND "${_install_destination}/" OUTPUT_VARIABLE _install_sources)262 target_sources(${name} INTERFACE263 "$<BUILD_INTERFACE:${_build_sources}>"264 "$<INSTALL_INTERFACE:${_install_sources}>"265 )266 target_link_libraries(${name} INTERFACE267 ${ARG_PRIVATE_LINK_LIBS}268 )269 270 # Add to parent.271 if(ARG_ADD_TO_PARENT)272 set_property(TARGET ${ARG_ADD_TO_PARENT} APPEND PROPERTY mlir_python_DEPENDS ${name})273 endif()274 275 # Install.276 set_property(GLOBAL APPEND PROPERTY MLIR_EXPORTS ${name})277 if(NOT LLVM_INSTALL_TOOLCHAIN_ONLY)278 _mlir_python_install_sources(279 ${name} "${ARG_ROOT_DIR}" "${_install_destination}"280 ${ARG_SOURCES}281 )282 endif()283endfunction()284 285function(_mlir_python_install_sources name source_root_dir destination)286 foreach(source_relative_path ${ARGN})287 # Transform "a/b/c.py" -> "${install_prefix}/a/b" for installation.288 get_filename_component(289 dest_relative_dir "${source_relative_path}" DIRECTORY290 BASE_DIR "${source_root_dir}"291 )292 install(293 FILES "${source_root_dir}/${source_relative_path}"294 DESTINATION "${destination}/${dest_relative_dir}"295 COMPONENT mlir-python-sources296 )297 endforeach()298 get_target_export_arg(${name} MLIR export_to_mlirtargets299 UMBRELLA mlir-python-sources)300 install(TARGETS ${name}301 COMPONENT mlir-python-sources302 ${export_to_mlirtargets}303 )304endfunction()305 306# Function: add_mlir_python_modules307# Adds python modules to a project, building them from a list of declared308# source groupings (see declare_mlir_python_sources and309# declare_mlir_python_extension). One of these must be called for each310# packaging root in use.311# Arguments:312# ROOT_PREFIX: The directory in the build tree to emit sources. This will313# typically be something like ${MY_BINARY_DIR}/python_packages/foobar314# for non-relocatable modules or a deeper directory tree for relocatable.315# INSTALL_PREFIX: Prefix into the install tree for installing the package.316# Typically mirrors the path above but without an absolute path.317# DECLARED_SOURCES: List of declared source groups to include. The entire318# DAG of source modules is included.319# COMMON_CAPI_LINK_LIBS: List of dylibs (typically one) to make every320# extension depend on (see mlir_python_add_common_capi_library).321function(add_mlir_python_modules name)322 cmake_parse_arguments(ARG323 ""324 "ROOT_PREFIX;INSTALL_PREFIX"325 "COMMON_CAPI_LINK_LIBS;DECLARED_SOURCES"326 ${ARGN})327 # Helper to process an individual target.328 function(_process_target modules_target sources_target)329 get_target_property(_source_type ${sources_target} mlir_python_SOURCES_TYPE)330 331 if(_source_type STREQUAL "pure")332 # Pure python sources to link into the tree.333 set(_pure_sources_target "${modules_target}.sources.${sources_target}")334 add_mlir_python_sources_target(${_pure_sources_target}335 INSTALL_COMPONENT ${modules_target}336 INSTALL_DIR ${ARG_INSTALL_PREFIX}337 OUTPUT_DIRECTORY ${ARG_ROOT_PREFIX}338 SOURCES_TARGETS ${sources_target}339 )340 add_dependencies(${modules_target} ${_pure_sources_target})341 elseif(_source_type STREQUAL "extension")342 # Native CPP extension.343 get_target_property(_module_name ${sources_target} mlir_python_EXTENSION_MODULE_NAME)344 get_target_property(_bindings_library ${sources_target} mlir_python_BINDINGS_LIBRARY)345 # Transform relative source to based on root dir.346 set(_extension_target "${modules_target}.extension.${_module_name}.dso")347 add_mlir_python_extension(${_extension_target} "${_module_name}"348 INSTALL_COMPONENT ${modules_target}349 INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs"350 OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs"351 PYTHON_BINDINGS_LIBRARY ${_bindings_library}352 LINK_LIBS PRIVATE353 ${sources_target}354 ${ARG_COMMON_CAPI_LINK_LIBS}355 )356 add_dependencies(${modules_target} ${_extension_target})357 mlir_python_setup_extension_rpath(${_extension_target})358 else()359 message(SEND_ERROR "Unrecognized source type '${_source_type}' for python source target ${sources_target}")360 return()361 endif()362 endfunction()363 364 # Build the modules target.365 add_custom_target(${name} ALL)366 _flatten_mlir_python_targets(_flat_targets ${ARG_DECLARED_SOURCES})367 foreach(sources_target ${_flat_targets})368 _process_target(${name} ${sources_target})369 endforeach()370 371 # Create an install target.372 if(NOT LLVM_ENABLE_IDE)373 add_llvm_install_targets(374 install-${name}375 DEPENDS ${name}376 COMPONENT ${name})377 endif()378endfunction()379 380# Function: declare_mlir_dialect_python_bindings381# Helper to generate source groups for dialects, including both static source382# files and a TD_FILE to generate wrappers.383#384# This will generate a source group named ${ADD_TO_PARENT}.${DIALECT_NAME}.385#386# Arguments:387# ROOT_DIR: Same as for declare_mlir_python_sources().388# ADD_TO_PARENT: Same as for declare_mlir_python_sources(). Unique names389# for the subordinate source groups are derived from this.390# TD_FILE: Tablegen file to generate source for (relative to ROOT_DIR).391# DIALECT_NAME: Python name of the dialect.392# SOURCES: Same as declare_mlir_python_sources().393# SOURCES_GLOB: Same as declare_mlir_python_sources().394# DEPENDS: Additional dependency targets.395# GEN_ENUM_BINDINGS: Generate enum bindings.396# GEN_ENUM_BINDINGS_TD_FILE: Optional Tablegen file to generate enums for (relative to ROOT_DIR).397# This file is where the *EnumAttrs are defined, not where the *Enums are defined.398# **WARNING**: This arg will shortly be removed when the just-below TODO is satisfied. Use at your399# risk.400#401# TODO: Right now `TD_FILE` can't be the actual dialect tablegen file, since we402# use its path to determine where to place the generated python file. If403# we made the output path an additional argument here we could remove the404# need for the separate "wrapper" .td files405function(declare_mlir_dialect_python_bindings)406 cmake_parse_arguments(ARG407 "GEN_ENUM_BINDINGS"408 "ROOT_DIR;ADD_TO_PARENT;TD_FILE;DIALECT_NAME"409 "SOURCES;SOURCES_GLOB;DEPENDS;GEN_ENUM_BINDINGS_TD_FILE"410 ${ARGN})411 # Sources.412 set(_dialect_target "${ARG_ADD_TO_PARENT}.${ARG_DIALECT_NAME}")413 declare_mlir_python_sources(${_dialect_target}414 ROOT_DIR "${ARG_ROOT_DIR}"415 ADD_TO_PARENT "${ARG_ADD_TO_PARENT}"416 SOURCES "${ARG_SOURCES}"417 SOURCES_GLOB "${ARG_SOURCES_GLOB}"418 )419 420 # Tablegen421 if(ARG_TD_FILE)422 set(tblgen_target "${_dialect_target}.tablegen")423 set(td_file "${ARG_ROOT_DIR}/${ARG_TD_FILE}")424 get_filename_component(relative_td_directory "${ARG_TD_FILE}" DIRECTORY)425 file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${relative_td_directory}")426 set(dialect_filename "${relative_td_directory}/_${ARG_DIALECT_NAME}_ops_gen.py")427 set(LLVM_TARGET_DEFINITIONS ${td_file})428 mlir_tablegen("${dialect_filename}"429 -gen-python-op-bindings -bind-dialect=${ARG_DIALECT_NAME}430 DEPENDS ${ARG_DEPENDS}431 )432 add_public_tablegen_target(${tblgen_target})433 434 set(_sources ${dialect_filename})435 if(ARG_GEN_ENUM_BINDINGS OR ARG_GEN_ENUM_BINDINGS_TD_FILE)436 if(ARG_GEN_ENUM_BINDINGS_TD_FILE)437 set(td_file "${ARG_ROOT_DIR}/${ARG_GEN_ENUM_BINDINGS_TD_FILE}")438 set(LLVM_TARGET_DEFINITIONS ${td_file})439 endif()440 set(enum_filename "${relative_td_directory}/_${ARG_DIALECT_NAME}_enum_gen.py")441 mlir_tablegen(${enum_filename} -gen-python-enum-bindings)442 list(APPEND _sources ${enum_filename})443 endif()444 445 # Generated.446 declare_mlir_python_sources("${_dialect_target}.ops_gen"447 ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}"448 ADD_TO_PARENT "${_dialect_target}"449 SOURCES ${_sources}450 )451 endif()452endfunction()453 454# Function: declare_mlir_dialect_extension_python_bindings455# Helper to generate source groups for dialect extensions, including both456# static source files and a TD_FILE to generate wrappers.457#458# This will generate a source group named ${ADD_TO_PARENT}.${EXTENSION_NAME}.459#460# Arguments:461# ROOT_DIR: Same as for declare_mlir_python_sources().462# ADD_TO_PARENT: Same as for declare_mlir_python_sources(). Unique names463# for the subordinate source groups are derived from this.464# TD_FILE: Tablegen file to generate source for (relative to ROOT_DIR).465# DIALECT_NAME: Python name of the dialect.466# EXTENSION_NAME: Python name of the dialect extension.467# SOURCES: Same as declare_mlir_python_sources().468# SOURCES_GLOB: Same as declare_mlir_python_sources().469# DEPENDS: Additional dependency targets.470# GEN_ENUM_BINDINGS: Generate enum bindings.471# GEN_ENUM_BINDINGS_TD_FILE: Optional Tablegen file to generate enums for (relative to ROOT_DIR).472# This file is where the *Attrs are defined, not where the *Enums are defined.473# **WARNING**: This arg will shortly be removed when the TODO for474# declare_mlir_dialect_python_bindings is satisfied. Use at your risk.475function(declare_mlir_dialect_extension_python_bindings)476 cmake_parse_arguments(ARG477 "GEN_ENUM_BINDINGS"478 "ROOT_DIR;ADD_TO_PARENT;TD_FILE;DIALECT_NAME;EXTENSION_NAME"479 "SOURCES;SOURCES_GLOB;DEPENDS;GEN_ENUM_BINDINGS_TD_FILE"480 ${ARGN})481 # Source files.482 set(_extension_target "${ARG_ADD_TO_PARENT}.${ARG_EXTENSION_NAME}")483 declare_mlir_python_sources(${_extension_target}484 ROOT_DIR "${ARG_ROOT_DIR}"485 ADD_TO_PARENT "${ARG_ADD_TO_PARENT}"486 SOURCES "${ARG_SOURCES}"487 SOURCES_GLOB "${ARG_SOURCES_GLOB}"488 )489 490 # Tablegen491 if(ARG_TD_FILE)492 set(tblgen_target "${ARG_ADD_TO_PARENT}.${ARG_EXTENSION_NAME}.tablegen")493 set(td_file "${ARG_ROOT_DIR}/${ARG_TD_FILE}")494 get_filename_component(relative_td_directory "${ARG_TD_FILE}" DIRECTORY)495 file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${relative_td_directory}")496 set(output_filename "${relative_td_directory}/_${ARG_EXTENSION_NAME}_ops_gen.py")497 set(LLVM_TARGET_DEFINITIONS ${td_file})498 mlir_tablegen("${output_filename}" -gen-python-op-bindings499 -bind-dialect=${ARG_DIALECT_NAME}500 -dialect-extension=${ARG_EXTENSION_NAME})501 add_public_tablegen_target(${tblgen_target})502 if(ARG_DEPENDS)503 add_dependencies(${tblgen_target} ${ARG_DEPENDS})504 endif()505 506 set(_sources ${output_filename})507 if(ARG_GEN_ENUM_BINDINGS OR ARG_GEN_ENUM_BINDINGS_TD_FILE)508 if(ARG_GEN_ENUM_BINDINGS_TD_FILE)509 set(td_file "${ARG_ROOT_DIR}/${ARG_GEN_ENUM_BINDINGS_TD_FILE}")510 set(LLVM_TARGET_DEFINITIONS ${td_file})511 endif()512 set(enum_filename "${relative_td_directory}/_${ARG_EXTENSION_NAME}_enum_gen.py")513 mlir_tablegen(${enum_filename} -gen-python-enum-bindings)514 list(APPEND _sources ${enum_filename})515 endif()516 517 declare_mlir_python_sources("${_extension_target}.ops_gen"518 ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}"519 ADD_TO_PARENT "${_extension_target}"520 SOURCES ${_sources}521 )522 endif()523endfunction()524 525# Function: mlir_python_setup_extension_rpath526# Sets RPATH properties on a target, assuming that it is being output to527# an _mlir_libs directory with all other libraries. For static linkage,528# the RPATH will just be the origin. If linking dynamically, then the LLVM529# library directory will be added.530# Arguments:531# RELATIVE_INSTALL_ROOT: If building dynamically, an RPATH entry will be532# added to the install tree lib/ directory by first traversing this533# path relative to the installation location. Typically a number of ".."534# entries, one for each level of the install path.535function(mlir_python_setup_extension_rpath target)536 cmake_parse_arguments(ARG537 ""538 "RELATIVE_INSTALL_ROOT"539 ""540 ${ARGN})541 542 # RPATH handling.543 # For the build tree, include the LLVM lib directory and the current544 # directory for RPATH searching. For install, just the current directory545 # (assumes that needed dependencies have been installed).546 if(NOT APPLE AND NOT UNIX)547 return()548 endif()549 550 set(_origin_prefix "\$ORIGIN")551 if(APPLE)552 set(_origin_prefix "@loader_path")553 endif()554 set_target_properties(${target} PROPERTIES555 BUILD_WITH_INSTALL_RPATH OFF556 BUILD_RPATH "${_origin_prefix}"557 INSTALL_RPATH "${_origin_prefix}"558 )559 560 # For static builds, that is all that is needed: all dependencies will be in561 # the one directory. For shared builds, then we also need to add the global562 # lib directory. This will be absolute for the build tree and relative for563 # install.564 # When we have access to CMake >= 3.20, there is a helper to calculate this.565 if(BUILD_SHARED_LIBS AND ARG_RELATIVE_INSTALL_ROOT)566 get_filename_component(_real_lib_dir "${LLVM_LIBRARY_OUTPUT_INTDIR}" REALPATH)567 set_property(TARGET ${target} APPEND PROPERTY568 BUILD_RPATH "${_real_lib_dir}")569 set_property(TARGET ${target} APPEND PROPERTY570 INSTALL_RPATH "${_origin_prefix}/${ARG_RELATIVE_INSTALL_ROOT}/lib${LLVM_LIBDIR_SUFFIX}")571 endif()572endfunction()573 574# Function: add_mlir_python_common_capi_library575# Adds a shared library which embeds dependent CAPI libraries needed to link576# all extensions.577# Arguments:578# INSTALL_COMPONENT: Name of the install component. Typically same as the579# target name passed to add_mlir_python_modules().580# INSTALL_DESTINATION: Prefix into the install tree in which to install the581# library.582# OUTPUT_DIRECTORY: Full path in the build tree in which to create the583# library. Typically, this will be the common _mlir_libs directory where584# all extensions are emitted.585# RELATIVE_INSTALL_ROOT: See mlir_python_setup_extension_rpath().586# DECLARED_HEADERS: Source groups from which to discover headers that belong587# to the library and should be installed with it.588# DECLARED_SOURCES: Source groups from which to discover dependent589# EMBED_CAPI_LINK_LIBS.590# EMBED_LIBS: Additional libraries to embed (must be built with OBJECTS and591# have an "obj.${name}" object library associated).592function(add_mlir_python_common_capi_library name)593 cmake_parse_arguments(ARG594 ""595 "INSTALL_COMPONENT;INSTALL_DESTINATION;OUTPUT_DIRECTORY;RELATIVE_INSTALL_ROOT"596 "DECLARED_HEADERS;DECLARED_SOURCES;EMBED_LIBS"597 ${ARGN})598 # Collect all explicit and transitive embed libs.599 set(_embed_libs ${ARG_EMBED_LIBS})600 _flatten_mlir_python_targets(_all_source_targets ${ARG_DECLARED_SOURCES})601 foreach(t ${_all_source_targets})602 get_target_property(_local_embed_libs ${t} mlir_python_EMBED_CAPI_LINK_LIBS)603 if(_local_embed_libs)604 list(APPEND _embed_libs ${_local_embed_libs})605 endif()606 endforeach()607 list(REMOVE_DUPLICATES _embed_libs)608 609 # Generate the aggregate .so that everything depends on.610 add_mlir_aggregate(${name}611 SHARED612 DISABLE_INSTALL613 EMBED_LIBS ${_embed_libs}614 )615 616 # Process any headers associated with the library617 _flatten_mlir_python_targets(_flat_header_targets ${ARG_DECLARED_HEADERS})618 set(_header_sources_target "${name}.sources")619 add_mlir_python_sources_target(${_header_sources_target}620 INSTALL_COMPONENT ${ARG_INSTALL_COMPONENT}621 INSTALL_DIR "${ARG_INSTALL_DESTINATION}/include"622 OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}/include"623 SOURCES_TARGETS ${_flat_header_targets}624 )625 add_dependencies(${name} ${_header_sources_target})626 627 if(WIN32)628 set_property(TARGET ${name} PROPERTY WINDOWS_EXPORT_ALL_SYMBOLS ON)629 endif()630 set_target_properties(${name} PROPERTIES631 LIBRARY_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}"632 BINARY_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}"633 # Needed for windows (and don't hurt others).634 RUNTIME_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}"635 ARCHIVE_OUTPUT_DIRECTORY "${ARG_OUTPUT_DIRECTORY}"636 )637 mlir_python_setup_extension_rpath(${name}638 RELATIVE_INSTALL_ROOT "${ARG_RELATIVE_INSTALL_ROOT}"639 )640 install(TARGETS ${name}641 COMPONENT ${ARG_INSTALL_COMPONENT}642 LIBRARY DESTINATION "${ARG_INSTALL_DESTINATION}"643 RUNTIME DESTINATION "${ARG_INSTALL_DESTINATION}"644 )645endfunction()646 647function(_flatten_mlir_python_targets output_var)648 set(_flattened)649 foreach(t ${ARGN})650 get_target_property(_source_type ${t} mlir_python_SOURCES_TYPE)651 get_target_property(_depends ${t} mlir_python_DEPENDS)652 if(_source_type)653 list(APPEND _flattened "${t}")654 if(_depends)655 _flatten_mlir_python_targets(_local_flattened ${_depends})656 list(APPEND _flattened ${_local_flattened})657 endif()658 endif()659 endforeach()660 list(REMOVE_DUPLICATES _flattened)661 set(${output_var} "${_flattened}" PARENT_SCOPE)662endfunction()663 664# Function: add_mlir_python_sources_target665# Adds a target corresponding to an interface target that carries source666# information. This target is responsible for "building" the sources by667# placing them in the correct locations in the build and install trees.668# Arguments:669# INSTALL_COMPONENT: Name of the install component. Typically same as the670# target name passed to add_mlir_python_modules().671# INSTALL_DESTINATION: Prefix into the install tree in which to install the672# library.673# OUTPUT_DIRECTORY: Full path in the build tree in which to create the674# library. Typically, this will be the common _mlir_libs directory where675# all extensions are emitted.676# SOURCES_TARGETS: List of interface libraries that carry source information.677function(add_mlir_python_sources_target name)678 cmake_parse_arguments(ARG679 ""680 "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY"681 "SOURCES_TARGETS"682 ${ARGN})683 684 if(ARG_UNPARSED_ARGUMENTS)685 message(FATAL_ERROR "Unhandled arguments to add_mlir_python_sources_target(${name}, ... : ${ARG_UNPARSED_ARGUMENTS}")686 endif()687 688 # On Windows create_symlink requires special permissions. Use copy_if_different instead.689 if(CMAKE_HOST_WIN32)690 set(_link_or_copy copy_if_different)691 else()692 set(_link_or_copy create_symlink)693 endif()694 695 foreach(_sources_target ${ARG_SOURCES_TARGETS})696 get_target_property(_src_paths ${_sources_target} SOURCES)697 if(NOT _src_paths)698 get_target_property(_src_paths ${_sources_target} INTERFACE_SOURCES)699 if(NOT _src_paths)700 break()701 endif()702 endif()703 704 get_target_property(_root_dir ${_sources_target} INCLUDE_DIRECTORIES)705 if(NOT _root_dir)706 get_target_property(_root_dir ${_sources_target} INTERFACE_INCLUDE_DIRECTORIES)707 endif()708 709 # Initialize an empty list of all Python source destination paths.710 set(all_dest_paths "")711 foreach(_src_path ${_src_paths})712 file(RELATIVE_PATH _source_relative_path "${_root_dir}" "${_src_path}")713 set(_dest_path "${ARG_OUTPUT_DIRECTORY}/${_source_relative_path}")714 715 get_filename_component(_dest_dir "${_dest_path}" DIRECTORY)716 file(MAKE_DIRECTORY "${_dest_dir}")717 718 add_custom_command(719 OUTPUT "${_dest_path}"720 COMMENT "Copying python source ${_src_path} -> ${_dest_path}"721 DEPENDS "${_src_path}"722 COMMAND "${CMAKE_COMMAND}" -E ${_link_or_copy}723 "${_src_path}" "${_dest_path}"724 )725 726 # Track the symlink or copy command output.727 list(APPEND all_dest_paths "${_dest_path}")728 729 if(ARG_INSTALL_DIR)730 # We have to install each file individually because we need to preserve731 # the relative directory structure in the install destination.732 # As an example, ${_source_relative_path} may be dialects/math.py733 # which would be transformed to ${ARG_INSTALL_DIR}/dialects734 # here. This could be moved outside of the loop and cleaned up735 # if we used FILE_SETS (introduced in CMake 3.23).736 get_filename_component(_install_destination "${ARG_INSTALL_DIR}/${_source_relative_path}" DIRECTORY)737 install(738 FILES ${_src_path}739 DESTINATION "${_install_destination}"740 COMPONENT ${ARG_INSTALL_COMPONENT}741 )742 endif()743 endforeach()744 endforeach()745 746 # Create a new custom target that depends on all symlinked or copied sources.747 add_custom_target("${name}" DEPENDS ${all_dest_paths})748endfunction()749 750################################################################################751# Build python extension752################################################################################753function(add_mlir_python_extension libname extname)754 cmake_parse_arguments(ARG755 ""756 "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY;PYTHON_BINDINGS_LIBRARY"757 "SOURCES;LINK_LIBS"758 ${ARGN})759 if(ARG_UNPARSED_ARGUMENTS)760 message(FATAL_ERROR "Unhandled arguments to add_mlir_python_extension(${libname}, ... : ${ARG_UNPARSED_ARGUMENTS}")761 endif()762 763 # The extension itself must be compiled with RTTI and exceptions enabled.764 # Also, some warning classes triggered by pybind11 are disabled.765 set(eh_rtti_enable)766 if (MSVC)767 set(eh_rtti_enable /EHsc /GR)768 elseif(LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL)769 set(eh_rtti_enable -frtti -fexceptions)770 endif ()771 772 # The actual extension library produces a shared-object or DLL and has773 # sources that must be compiled in accordance with pybind11 needs (RTTI and774 # exceptions).775 if(NOT DEFINED ARG_PYTHON_BINDINGS_LIBRARY OR ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "pybind11")776 pybind11_add_module(${libname}777 ${ARG_SOURCES}778 )779 elseif(ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "nanobind")780 nanobind_add_module(${libname}781 NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}782 FREE_THREADED783 ${ARG_SOURCES}784 )785 786 if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES787 AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))788 # Avoid some warnings from upstream nanobind.789 # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let790 # the super project handle compile options as it wishes.791 get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)792 target_compile_options(${NB_LIBRARY_TARGET_NAME}793 PRIVATE794 -Wno-c++98-compat-extra-semi795 -Wno-cast-qual796 -Wno-covered-switch-default797 -Wno-deprecated-literal-operator798 -Wno-nested-anon-types799 -Wno-unused-parameter800 -Wno-zero-length-array801 -Wno-missing-field-initializers802 ${eh_rtti_enable})803 804 target_compile_options(${libname}805 PRIVATE806 -Wno-c++98-compat-extra-semi807 -Wno-cast-qual808 -Wno-covered-switch-default809 -Wno-deprecated-literal-operator810 -Wno-nested-anon-types811 -Wno-unused-parameter812 -Wno-zero-length-array813 -Wno-missing-field-initializers814 ${eh_rtti_enable})815 endif()816 817 if(APPLE)818 # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind819 # doesn't declare this API as undefined in its linker flags. So we need to declare it as such820 # for downstream users that do not do something like `-undefined dynamic_lookup`.821 # Same for the rest.822 target_link_options(${libname} PUBLIC823 "LINKER:-U,_PyClassMethod_New"824 "LINKER:-U,_PyCode_Addr2Location"825 "LINKER:-U,_PyFrame_GetLasti"826 )827 endif()828 endif()829 830 target_compile_options(${libname} PRIVATE ${eh_rtti_enable})831 832 # Configure the output to match python expectations.833 set_target_properties(834 ${libname} PROPERTIES835 LIBRARY_OUTPUT_DIRECTORY ${ARG_OUTPUT_DIRECTORY}836 OUTPUT_NAME "${extname}"837 NO_SONAME ON838 )839 840 if(WIN32)841 # Need to also set the RUNTIME_OUTPUT_DIRECTORY on Windows in order to842 # control where the .dll gets written.843 set_target_properties(844 ${libname} PROPERTIES845 RUNTIME_OUTPUT_DIRECTORY ${ARG_OUTPUT_DIRECTORY}846 ARCHIVE_OUTPUT_DIRECTORY ${ARG_OUTPUT_DIRECTORY}847 )848 endif()849 850 target_link_libraries(${libname}851 PRIVATE852 ${ARG_LINK_LIBS}853 )854 855 target_link_options(${libname}856 PRIVATE857 # On Linux, disable re-export of any static linked libraries that858 # came through.859 $<$<PLATFORM_ID:Linux>:LINKER:--exclude-libs,ALL>860 )861 862 if(WIN32)863 # On Windows, pyconfig.h (and by extension python.h) hardcode the version of the864 # python library which will be used for linkage depending on the flavor of the build.865 # pybind11 has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG866 # is not passed in as a compile definition, pybind11 undefs _DEBUG when including867 # python.h, so that the release python library would be used).868 # Since mlir uses pybind11, we can leverage their workaround by never directly869 # pyconfig.h or python.h and instead relying on the pybind11 headers to include the870 # necessary python headers. This results in mlir always linking against the871 # release python library via the (undocumented) cmake property Python3_LIBRARY_RELEASE.872 target_link_libraries(${libname} PRIVATE ${Python3_LIBRARY_RELEASE})873 endif()874 875 ################################################################################876 # Install877 ################################################################################878 if(ARG_INSTALL_DIR)879 install(TARGETS ${libname}880 COMPONENT ${ARG_INSTALL_COMPONENT}881 LIBRARY DESTINATION ${ARG_INSTALL_DIR}882 ARCHIVE DESTINATION ${ARG_INSTALL_DIR}883 # NOTE: Even on DLL-platforms, extensions go in the lib directory tree.884 RUNTIME DESTINATION ${ARG_INSTALL_DIR}885 )886 endif()887endfunction()888